权重和偏差错误: wandb后端进程已关闭

Question

运行下面链接的colab，我得到以下错误：

“wandb后端进程已关闭”

在colab使用wandb的方式中，我没有发现任何可疑之处，我也找不到有同样问题的人。任何帮助都是非常感谢的。我正在使用colab中最新版本的wandb。

这就是我设立wandb的地方：

if WANDB:
    wandb.login()

这就是我得到错误的部分：

#setup wandb if we're using it

if WANDB:
    experiment_name = os.environ.get("EXPERIMENT_NAME")
    group = experiment_name if experiment_name != "none" else wandb.util.generate_id()

cv_scores = []
oof_data_frame = pd.DataFrame()
for fold in range(1, config.folds + 1):
    print(f"Fold {fold}/{config.folds}", end="\n"*2)
    fold_directory = os.path.join(config.output_directory, f"fold_{fold}")    
    make_directory(fold_directory)
    model_path = os.path.join(fold_directory, "model.pth")
    model_config_path = os.path.join(fold_directory, "model_config.json")
    checkpoints_directory = os.path.join(fold_directory, "checkpoints/")
    make_directory(checkpoints_directory)
    
    #Data collators are objects that will form a batch by using a list of dataset elements as input.
    collator = Collator(tokenizer=tokenizer, max_length=config.max_length)
    
    train_fold = train[~train["fold"].isin([fold])]
    train_dataset = Dataset(texts=train_fold["anchor"].values, 
                            pair_texts=train_fold["target"].values,
                            contexts=train_fold["title"].values,
                            targets=train_fold["score"].values, 
                            max_length=config.max_length,
                            sep=tokenizer.sep_token,
                            tokenizer=tokenizer)
    
    train_loader = DataLoader(dataset=train_dataset, 
                              batch_size=config.batch_size, 
                              num_workers=config.num_workers,
                              pin_memory=config.pin_memory,
                              collate_fn=collator,
                              shuffle=True, 
                              drop_last=False)
    
    print(f"Train samples: {len(train_dataset)}")
    
    validation_fold = train[train["fold"].isin([fold])]
    validation_dataset = Dataset(texts=validation_fold["anchor"].values, 
                                 pair_texts=validation_fold["target"].values,
                                 contexts=validation_fold["title"].values,
                                 targets=validation_fold["score"].values,
                                 max_length=config.max_length,
                                 sep=tokenizer.sep_token,
                                 tokenizer=tokenizer)
    
    validation_loader = DataLoader(dataset=validation_dataset, 
                                   batch_size=config.batch_size*2, 
                                   num_workers=config.num_workers,
                                   pin_memory=config.pin_memory,
                                   collate_fn=collator,
                                   shuffle=True, 
                                   drop_last=False)
    
    print(f"Validation samples: {len(validation_dataset)}")


    model = Model(**config.model)
    
    if not os.path.exists(model_config_path): 
        model.config.to_json_file(model_config_path)
    
    model_parameters = model.parameters()
    optimizer = get_optimizer(**config.optimizer, model_parameters=model_parameters)
    
    training_steps = len(train_loader) * config.epochs
    
    if "scheduler" in config:
        config.scheduler.parameters.num_training_steps = training_steps
        config.scheduler.parameters.num_warmup_steps = training_steps * config.get("warmup", 0)
        scheduler = get_scheduler(**config.scheduler, optimizer=optimizer, from_transformers=True)
    else:
        scheduler = None
        
    model_checkpoint = ModelCheckpoint(mode="min", 
                                       delta=config.delta, 
                                       directory=checkpoints_directory, 
                                       overwriting=True, 
                                       filename_format="checkpoint.pth", 
                                       num_candidates=1)


    if WANDB:
        wandb.init()
        #wandb.init(group=group, name=f"fold_{fold}", config=config)
    
    (train_loss, train_metrics), (validation_loss, validation_metrics, validation_outputs) = training_loop(model=model, 
                                                                                                           optimizer=optimizer, 
                                                                                                           scheduler=scheduler,
                                                                                                           scheduling_after=config.scheduling_after,
                                                                                                           train_loader=train_loader,
                                                                                                           validation_loader=validation_loader,
                                                                                                           epochs=config.epochs, 
                                                                                                           gradient_accumulation_steps=config.gradient_accumulation_steps, 
                                                                                                           gradient_scaling=config.gradient_scaling, 
                                                                                                           gradient_norm=config.gradient_norm, 
                                                                                                           validation_steps=config.validation_steps, 
                                                                                                           amp=config.amp,
                                                                                                           debug=config.debug, 
                                                                                                           verbose=config.verbose, 
                                                                                                           device=config.device, 
                                                                                                           recalculate_metrics_at_end=True, 
                                                                                                           return_validation_outputs=True, 
                                                                                                           logger="tqdm")
    
    if WANDB:
        wandb.finish()
    
    if config.save_model:
        model_state = model.state_dict()
        torch.save(model_state, model_path)
        print(f"Model's path: {model_path}")
    
    validation_fold["prediction"] = validation_outputs.to("cpu").numpy()
    oof_data_frame = pd.concat([oof_data_frame, validation_fold])
    
    cv_monitor_value = validation_loss if config.cv_monitor_value == "loss" else validation_metrics[config.cv_monitor_value]
    cv_scores.append(cv_monitor_value)
    
    del model, optimizer, validation_outputs, train_fold, validation_fold
    torch.cuda.empty_cache()
    gc.collect()
    
    print(end="\n"*6)

Answer 1

对我有效的解决方案是运行!wandb login --relogin。

Answer 2

TDLR;检查正在使用的wandb项目空间中生成的id是否为unique。

解释

您可以在wandb文件夹和特定的run id下的日志文件中检查发生这种情况的确切原因。Error communicating with wandb process和The wandb backend process has shutdown也有同样的问题。

我的问题是，我将run id分配给一个已经存在的特定实例，并重新运行整个搜索空间，但是run id必须是唯一的。在init中使用name通常是一个更安全的选择，如果您不打算继续之前的运行(如果您在init方法中表明了这一点，这是可能的)。

您可以尝试在脱机模式下运行Wandb，以查看这是否有帮助，并在以后执行wandb sync。