下面是使用unionAll的Spark2.4代码
import org.apache.spark.sql.{DataFrame, Dataset}
object UnionRewrite {
def inSource(
df1: DataFrame,
df2: DataFrame,
df3: DataFrame,
ds1: Dataset[String],
ds2: Dataset[String]
): Unit = {
val res1 = df1.unionAll(df2)
val res2 = df1.unionAll(df2).unionAll(df3)
val res3 = Seq(df1, df2, df3).reduce(_ unionAll _)
val res4 = ds1.unionAll(ds2)
val res5 = Seq(ds1, ds2).reduce(_ unionAll _)
}
}在星火公园,3.+ unionAll是不受欢迎的。下面是使用union的等效代码
import org.apache.spark.sql.{DataFrame, Dataset}
object UnionRewrite {
def inSource(
df1: DataFrame,
df2: DataFrame,
df3: DataFrame,
ds1: Dataset[String],
ds2: Dataset[String]
): Unit = {
val res1 = df1.union(df2)
val res2 = df1.union(df2).union(df3)
val res3 = Seq(df1, df2, df3).reduce(_ union _)
val res4 = ds1.union(ds2)
val res5 = Seq(ds1, ds2).reduce(_ union _)
}
}问题是如何编写Scalafix规则(使用准引号)将unionAll替换为union
没有准引号,我实现了这个规则,它是有效的
override def fix(implicit doc: SemanticDocument): Patch = {
def matchOnTree(t: Tree): Patch = {
t.collect {
case Term.Apply(
Term.Select(_, deprecated @ Term.Name(name)),
_
) if config.deprecatedMethod.contains(name) =>
Patch.replaceTree(
deprecated,
config.deprecatedMethod(name)
)
case Term.Apply(
Term.Select(_, _ @Term.Name(name)),
List(
Term.AnonymousFunction(
Term.ApplyInfix(
_,
deprecatedAnm @ Term.Name(nameAnm),
_,
_
)
)
)
) if "reduce".contains(name) && config.deprecatedMethod.contains(nameAnm) =>
Patch.replaceTree(
deprecatedAnm,
config.deprecatedMethod(nameAnm)
)
}.asPatch
}
matchOnTree(doc.tree)
}发布于 2022-11-20 18:25:47
Ver: 1
package fix
import scalafix.v1._
import scala.meta._
class RuleQuasiquotesUnionAll extends SemanticRule("RuleQuasiquotesUnionAll") {
override val description =
"""Quasiquotes in Scalafix. Replacing unionAll with union"""
override val isRewrite = true
override def fix(implicit doc: SemanticDocument): Patch = {
def matchOnTree(t: Tree): Patch = {
t.collect { case tt: Term =>
tt match {
case q"""unionAll""" =>
Patch.replaceTree(tt, """union""")
case _ => Patch.empty
}
}.asPatch
}
matchOnTree(doc.tree)
}
}https://stackoverflow.com/questions/74506444
复制相似问题