Shell脚本从不同的URL下载多个文件，使用for循环在每次调用中下载和验证文件

Question

脚本的预期行为是，如果文件不在当前目录中，则从URL下载文件，并将文件的md5校验和与服务器上的md5校验和进行比较。如果文件存在，则再次执行验证，如果验证失败，则下载该文件。由于我有一个糟糕的互联网连接，我尝试第二次下载和验证，如果初始验证失败。如果在第二次尝试中验证失败，则脚本退出并记录一个致命错误。

调用脚本时有两个输入。第一个输入是包含要下载文件名的完整URL的文件。每行一个文件。第二个输入是包含预期的md5checksum和文件名的文件。期望值格式的校验和文件列表“校验和”“文件名”没有引号。

目前，脚本按预期工作。因为我正在自学bash/sh脚本。有没有更干净的方法来实现预期的行为。我知道wget的继续选项。我选择不使用它，因为它感觉“太容易”，而不是学习sh脚本的精神。

这两个源文件可以在原始repos https://github.com/CJ-Systems/log4sh和https://github.com/CJ-Systems/shlib/blob/master/functions/shlib_安西的分叉处找到。

#!/bin/sh
#

# load log4sh
if [ -r ./lib/log4sh ]; then
    . ./lib/log4sh
else
    echo "ERROR: could not load (log4sh)" >&2
    exit 1
fi

# load shlib_ansi
if [ -r ./lib/shlib_ansi ]; then
    . ./lib/shlib_ansi
else
    echo "ERROR: could not load (shlib_ansi)" >&2
    exit 1
fi

# Download single file
downloadfile() {
    local fn=$1
    log DEBUG "${shlib_ansi_red}Downloading $bn ${shlib_ansi_none}"
    wget --quiet --show-progress --no-use-server-timestamps $fn -O $bn
}

# Compare md5 checksums
verifyfile() {
    local check_sum=$1
    exp_md5=$(cat $check_sum | grep -i $bn | cut -d' ' -f1)
    md5_local=$(md5sum $bn | cut -d' ' -f1)
    if [ $md5_local = $exp_md5 ]
    then
        log DEBUG "Checksum valid for $bn"
        return 0
    else
        log DEBUG "Checksum invalid for $bn"
        return 1
    fi
}

# Download all files in file_list
download() {
    local file_list=$1
    local check_sums=$2
    for f in $(cat $file_list)
    do
        # Get name file of file to download
        bn=$(basename $f)
        # If file not present attempt to download
        if [ ! -f $bn ]
        then
            log INFO "Downloading $bn"
            downloadfile $f
            if verifyfile $check_sums
            then
                log DEBUG "Verification of $bn successful"
            else
                log WARN "Verification of $bn failed. Retrying download"
                downloadfile $f
                if ! verifyfile $check_sums
                then
                    log FATAL "Failed to verify $bn download unsuccessful"
                    exit
                    else
                        log DEBUG "Verification successful"
                fi
            fi
        # Local copy of file exists verify checksum redownload if necessary
        else
            log INFO "File $bn already downloaded. Verifying checksum"
            if ! verifyfile $check_sums
            then
                log WARN "Verification of $bn failed. Redownloading"
                downloadfile $f
                if verifyfile $check_sums
                then
                    log INFO "Redownload and verification of $bn successful"
                else
                    log WARN "Verification of $bn failed. Retrying download"
                    downloadfile $f
                    if ! verifyfile $check_sums
                    then 
                        log FATAL "Failed to verify $bn redownload unsuccessful"
                        exit
                    fi
                fi
                else log INFO "Verification of $bn successful"
            fi
        fi
    done;
}
shlib_ansi_init auto
download $1 $2

Answer 1

在获取文件之前，我认为测试文件的可读性没有任何价值。只需尝试，并测试我们是否成功：

die()
{
    echo "$@" >&2
    exit 1
}

. ./lib/log4sh || die "ERROR: could not load (log4sh)"
. ./lib/shlib_ansi || die "ERROR: could not load (shlib_ansi)"

很多变量展开都缺少引号--请始终使用引号：

    wget --quiet --show-progress --no-use-server-timestamps "$fn" -O "$bn"

            downloadfile "$f"

我们使用变量($bn)将值传递给函数--更愿意为此使用函数参数：

# Download single URL ($1) and save it to local file ($2)
downloadfile() {
    log DEBUG "${shlib_ansi_red}Downloading $b2 ${shlib_ansi_none}"
    wget --quiet --show-progress --no-use-server-timestamps $1 -O $2
}

在verifyfile中，我们应该能够使用md5sum -c直接检查而不是使用[：

# true if checksum record line in $1 matches MD5 of file $2
verifyfile()
{
    if awk -F ' [* ]' -vfile="$2" '$2==file' "$1" | md5sum --status --check 2>/dev/null
    then
        log DEBUG "Checksum valid for $bn"
        return 0
    else
        log DEBUG "Checksum invalid for $bn"
        return 1
    fi
}

在download中，由于$(cat $file_list)，我们必须存储列表的全部内容，这可能是很大的。从这个文件中提取read可能更好，这将大大减少我们对内存的需求。

在download中有很多重复。如果我们将一个不存在的文件与校验和失败文件一样对待，我们可以简化很多：

1. 如果文件不存在，请下载。
2. 测试校验和。如果成功，我们就完蛋了。
3. 重新下载。
4. 测试校验和。

这使得功能简单得多：

download()
{
    file_list=$1
    check_sums=$2
    result=true
    while read -r f
    do
        # Get name file of file to download
        bn=$(basename "$f")
        # If file not present attempt to download
        if ! [ -f "$bn" ]
        then
            log INFO "Downloading $bn"
            downloadfile "$f" "$bn"
        fi
        if verifyfile "$check_sums" "$bn"
        then
            log DEBUG "Verification of $bn successful"
            continue;
        fi
        
        log WARN "Verification of $bn failed. Retrying download"
        if downloadfile "$f" "$bn" && verifyfile "$check_sums" "$bn"
        then
            log DEBUG "Verification of $bn successful"
        fi

        log FATAL "Failed to verify $bn download unsuccessful"
        result=false
    done <"$file_list"
    "$result"    
}

供参考，以下是Shellcheck报告的内容：

285297.sh:22:5: warning: In POSIX sh, 'local' is undefined. [SC3043]
285297.sh:22:14: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:24:61: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:24:68: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:29:5: warning: In POSIX sh, 'local' is undefined. [SC3043]
285297.sh:29:21: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:30:19: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:30:40: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:31:24: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:32:10: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:32:23: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:44:5: warning: In POSIX sh, 'local' is undefined. [SC3043]
285297.sh:44:21: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:45:5: warning: In POSIX sh, 'local' is undefined. [SC3043]
285297.sh:45:22: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:46:14: note: To read lines rather than words, pipe/redirect to a 'while read' loop. [SC2013]
285297.sh:46:20: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:49:23: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:51:19: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:54:26: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:55:27: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:60:30: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:61:33: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:72:29: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:75:30: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:76:31: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:81:34: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:82:37: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:94:10: note: Double quote to prevent globbing and word splitting. [SC2086]
285297.sh:94:13: note: Double quote to prevent globbing and word splitting. [SC2086]