从ScalaPB生成的Scala case类派生Avro schema

Question

我尝试使用Avro4s从ScalaPB生成的case类(来自Protobuf定义)派生Avro Schema，但不能成功。下面的例子显示了这个问题。

我的protobuf文件test.proto

message Test {

// Unix timestamp in milliseconds
required int64 time_ms = 1;

// user_id
optional string UUID = 2;
}

我使用ScalaPB生成Test case类：

// Generated by the Scala Plugin for the Protocol Buffer Compiler.
// Do not edit!
//
// Protofile syntax: PROTO2

package test

/** @param timeMs
  *   Unix timestamp in milliseconds
  * @param uUID
  *   user_id
  */
@SerialVersionUID(0L)
final case class Test(
    timeMs: _root_.scala.Long,
    uUID: _root_.scala.Option[_root_.scala.Predef.String] = None
    ) extends scalapb.GeneratedMessage with scalapb.Message[Test] with scalapb.lenses.Updatable[Test] {
    @transient
    private[this] var __serializedSizeCachedValue: _root_.scala.Int = 0
    private[this] def __computeSerializedValue(): _root_.scala.Int = {
      var __size = 0

      {
        val __value = timeMs
        __size += _root_.com.google.protobuf.CodedOutputStream.computeInt64Size(1, __value)
      };
      if (uUID.isDefined) {
        val __value = uUID.get
        __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(2, __value)
      };
      __size
    }
    final override def serializedSize: _root_.scala.Int = {
      var read = __serializedSizeCachedValue
      if (read == 0) {
        read = __computeSerializedValue()
        __serializedSizeCachedValue = read
      }
      read
    }
    def writeTo(`_output__`: _root_.com.google.protobuf.CodedOutputStream): _root_.scala.Unit = {

      {
        val __v = timeMs
        _output__.writeInt64(1, __v)
      };
      uUID.foreach { __v =>
        val __m = __v
        _output__.writeString(2, __m)
      };
    }
    def mergeFrom(`_input__`: _root_.com.google.protobuf.CodedInputStream): test.Test = {
      var __timeMs = this.timeMs
      var __uUID = this.uUID
      var __requiredFields0: _root_.scala.Long = 0x1L
      var _done__ = false
      while (!_done__) {
        val _tag__ = _input__.readTag()
        _tag__ match {
          case 0 => _done__ = true
          case 8 =>
            __timeMs = _input__.readInt64()
            __requiredFields0 &= 0xfffffffffffffffeL
          case 18 =>
            __uUID = Option(_input__.readString())
          case tag => _input__.skipField(tag)
        }
      }
      if (__requiredFields0 != 0L) { throw new _root_.com.google.protobuf.InvalidProtocolBufferException("Message missing required fields.") } 
      test.Test(
          timeMs = __timeMs,
          uUID = __uUID
      )
    }
    def withTimeMs(__v: _root_.scala.Long): Test = copy(timeMs = __v)
    def getUUID: _root_.scala.Predef.String = uUID.getOrElse("")
    def clearUUID: Test = copy(uUID = None)
    def withUUID(__v: _root_.scala.Predef.String): Test = copy(uUID = Option(__v))
    def getFieldByNumber(__fieldNumber: _root_.scala.Int): _root_.scala.Any = {
      (__fieldNumber: @_root_.scala.unchecked) match {
        case 1 => timeMs
        case 2 => uUID.orNull
      }
    }
    def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = {
      _root_.scala.Predef.require(__field.containingMessage eq companion.scalaDescriptor)
      (__field.number: @_root_.scala.unchecked) match {
        case 1 => _root_.scalapb.descriptors.PLong(timeMs)
        case 2 => uUID.map(_root_.scalapb.descriptors.PString).getOrElse(_root_.scalapb.descriptors.PEmpty)
      }
    }
    def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this)
    def companion = test.Test
}

object Test extends scalapb.GeneratedMessageCompanion[test.Test] {
  implicit def messageCompanion: scalapb.GeneratedMessageCompanion[test.Test] = this
  def fromFieldsMap(__fieldsMap: scala.collection.immutable.Map[_root_.com.google.protobuf.Descriptors.FieldDescriptor, _root_.scala.Any]): test.Test = {
    _root_.scala.Predef.require(__fieldsMap.keys.forall(_.getContainingType() == javaDescriptor), "FieldDescriptor does not match message type.")
    val __fields = javaDescriptor.getFields
    test.Test(
      __fieldsMap(__fields.get(0)).asInstanceOf[_root_.scala.Long],
      __fieldsMap.get(__fields.get(1)).asInstanceOf[_root_.scala.Option[_root_.scala.Predef.String]]
    )
  }
  implicit def messageReads: _root_.scalapb.descriptors.Reads[test.Test] = _root_.scalapb.descriptors.Reads{
    case _root_.scalapb.descriptors.PMessage(__fieldsMap) =>
      _root_.scala.Predef.require(__fieldsMap.keys.forall(_.containingMessage == scalaDescriptor), "FieldDescriptor does not match message type.")
      test.Test(
        __fieldsMap.get(scalaDescriptor.findFieldByNumber(1).get).get.as[_root_.scala.Long],
        __fieldsMap.get(scalaDescriptor.findFieldByNumber(2).get).flatMap(_.as[_root_.scala.Option[_root_.scala.Predef.String]])
      )
    case _ => throw new RuntimeException("Expected PMessage")
  }
  def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = TestProto.javaDescriptor.getMessageTypes.get(0)
  def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = TestProto.scalaDescriptor.messages(0)
  def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number)
  lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_]] = Seq.empty
  def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber)
  lazy val defaultInstance = test.Test(
    timeMs = 0L
  )
  implicit class TestLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, test.Test]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, test.Test](_l) {
    def timeMs: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Long] = field(_.timeMs)((c_, f_) => c_.copy(timeMs = f_))
    def uUID: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.getUUID)((c_, f_) => c_.copy(uUID = Option(f_)))
    def optionalUUID: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[_root_.scala.Predef.String]] = field(_.uUID)((c_, f_) => c_.copy(uUID = f_))
  }
  final val TIME_MS_FIELD_NUMBER = 1
  final val UUID_FIELD_NUMBER = 2
}

然后尝试导出Avro Schema，如下所示：

val schema = AvroSchema[Test]

我得到以下错误：

could not find implicit value for parameter schemaFor: com.sksamuel.avro4s.SchemaFor[test.Test]

我怎样才能在这里成功呢？有没有其他实用工具可以达到同样的效果？

Answer 1

ScalaPB会向生成的case类添加一个@SerialVersionUID注释。由于https://github.com/sksamuel/avro4s/issues/300，这破坏了最新的avro4s (2.0.4)

Answer 2

正如@thesamet提到的那样，avro4s有一个bug，当一个类有一个Java注解时，它会导致错误。ScalaPB添加了这样一个注解-- @SerialVersionUID。这在最新的avro4s快照(3.0.0.X-snapshot)中已修复，并将在最终的3.0.0版本中修复。

Answer 3

ScalaPB生成的类包含大量基于PB运行时库的样板。在Avro中转换/读取协议消息最安全的方法之一是使用AvroProtobuf兼容性。你可以在org.apache.avro.protobuf包上找到更多关于它的信息。

您可以使用ProtobufData获取模式或执行SerDe操作。