获取broadcast5的broadcast_5_piece0失败

Question

在使用2个工作实例运行spark应用程序时，出现错误"Failed to get broadcast_5_piece0 of broadcast5“。我也设置了spark.cleaner.ttl，之后我也得到了同样的错误。有人能帮帮忙吗？

完整堆栈跟踪可在以下位置找到：

java.io.IOException: org.apache.spark.SparkException: Failed to get broadcast_5_piece0 of broadcast_5
    at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1212)
    at org.apache.spark.broadcast.TorrentBroadcast.readBroadcastBlock(TorrentBroadcast.scala:165)
    at org.apache.spark.broadcast.TorrentBroadcast._value$lzycompute(TorrentBroadcast.scala:64)
    at org.apache.spark.broadcast.TorrentBroadcast._value(TorrentBroadcast.scala:64)
    at org.apache.spark.broadcast.TorrentBroadcast.getValue(TorrentBroadcast.scala:88)
    at org.apache.spark.broadcast.Broadcast.value(Broadcast.scala:70)
    at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:62)
    at org.apache.spark.scheduler.Task.run(Task.scala:89)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:227)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.spark.SparkException: Failed to get broadcast_5_piece0 of broadcast_5
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$org$apache$spark$broadcast$TorrentBroadcast$$readBlocks$1$$anonfun$2.apply(TorrentBroadcast.scala:138)
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$org$apache$spark$broadcast$TorrentBroadcast$$readBlocks$1$$anonfun$2.apply(TorrentBroadcast.scala:138)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$org$apache$spark$broadcast$TorrentBroadcast$$readBlocks$1.apply$mcVI$sp(TorrentBroadcast.scala:137)
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$org$apache$spark$broadcast$TorrentBroadcast$$readBlocks$1.apply(TorrentBroadcast.scala:120)
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$org$apache$spark$broadcast$TorrentBroadcast$$readBlocks$1.apply(TorrentBroadcast.scala:120)
    at scala.collection.immutable.List.foreach(List.scala:318)
    at org.apache.spark.broadcast.TorrentBroadcast.org$apache$spark$broadcast$TorrentBroadcast$$readBlocks(TorrentBroadcast.scala:120)
    at org.apache.spark.broadcast.TorrentBroadcast$$anonfun$readBroadcastBlock$1.apply(TorrentBroadcast.scala:175)
    at org.apache.spark.util.Utils$.tryOrIOException(Utils.scala:1205)
    ... 11 more

添加代码............

public class Insert_into_cassandra implements Serializable {
    static List<String> signal_name_pass=new ArrayList<String>();
    static int count=4;
    static SparkConf conf=new SparkConf().setAppName("Insert_into_cassandra").setMaster("local").set("spark.cassandra.connection.host", "127.0.0.1");
    static JavaSparkContext jspc=new JavaSparkContext(conf);
    static SparkContextJavaFunctions functions=CassandraJavaUtil.javaFunctions(jspc);
     static Insert_into_cassandra iic=new Insert_into_cassandra();
     static int  value_in_db=0;
public static  void main(String gg[])
{


JavaRDD<String> rbmfile=jspc.textFile("/home/amd/Desktop/prac");


JavaPairRDD<String, Parse_Object> signal_name=rbmfile.mapToPair(new PairFunction<String, String, Parse_Object>() {


    public Tuple2<String, Parse_Object> call(String x) throws Exception {

        return new Tuple2<String, Parse_Object>(x.split("  ")[0], new Parse_Object(x.split("  ")[1],x.split("  ")[2],x.split("  ")[3]));
    }
});



JavaRDD<CassandraRow> signal_name_cassandra=functions.cassandraTable("tutorialspoint", "stuff_id_2");



JavaRDD<CassandraRow> sort=signal_name_cassandra.sortBy(new Function<CassandraRow, String >() {


    public String call(CassandraRow x) throws Exception {

        return x.getString(0);
    }
}, false, 1).coalesce(1);


value_in_db=Integer.parseInt(sort.first().getString(0));


JavaPairRDD<String, String> take_signal_name=signal_name_cassandra.mapToPair(new PairFunction<CassandraRow, String, String>() {

    public Tuple2<String, String> call(CassandraRow x) throws Exception {
        // TODO Auto-generated method stub

        return new Tuple2<String, String>(x.getString(1),x.getString(0));
    }
});



/*JavaPairRDD<String, String> getting_max_id=signal_name_cassandra.mapToPair(new PairFunction<CassandraRow, String, String>() {

    @Override
    public Tuple2<String, String> call(CassandraRow x) throws Exception {
        // TODO Auto-generated method stub

        return new Tuple2<String, String>(x.getString(0),x.getString(1));
    }
});*/


 JavaPairRDD<String, Tuple2<Optional<String>, Parse_Object>> join=take_signal_name.rightOuterJoin(signal_name);



JavaPairRDD<String, String> getting_id=join.mapToPair(new PairFunction<Tuple2<String,Tuple2<Optional<String>,Parse_Object>>, String, String>() {

    public Tuple2<String, String> call(
            Tuple2<String, Tuple2<Optional<String>, Parse_Object>> x)
            throws Exception {

        if(x._2()._1().isPresent())
        {
            System.out.println("if----"+x._1());
            return new Tuple2<String, String>(x._1().toString(), x._2()._1().toString());

        }

        else
        {
            signal_name_pass.add(x._1());

        }
    return null;

    }


}).filter(new Function<Tuple2<String,String>, Boolean>() {


    public Boolean call(Tuple2<String, String> x) throws Exception {
        // TODO Auto-generated method stub
        return x!=null;
    }
});
getting_id.saveAsTextFile("/home/amd/Desktop/smal/get13");

Answer 1

在spark的现代版本中，设置Spark Cleaner TTL几乎总是错误的。这可能会导致过早删除重要的缓存数据。让内置的Context Cleaner来做它的工作要安全得多。

如果关闭TTL不能修复问题，则需要提供更多相关细节，例如：

1. executors上的任何可疑日志？
2. 异常的完整堆栈跟踪？
3. 正在运行的代码示例？