公共爬行请求返回403 WARC

Question

我正在尝试从普通的爬行档案中抓取一些WARC文件，但我似乎没有将成功的请求传递给服务器。下面提供了一个最小的python示例来复制错误。我尝试在请求头中添加UserAgent，但它确实有帮助。对如何进行有什么想法吗？

import io
import time
import justext # >= 2.2.0
import argparse
import requests # >= 2.23.0
import pandas as pd # pandas >= 1.0.3
from tqdm import tqdm
from warcio.archiveiterator import ArchiveIterator warcio >= 1.7.3


def debug():

    common_crawl_data = {"filename":"crawl-data/CC-MAIN-2016-07/segments/1454702018134.95/warc/CC-MAIN-20160205195338-00121-ip-10-236-182-209.ec2.internal.warc.gz",
                     "offset":244189209,
                     "length":989
                     }

    offset, length = int(common_crawl_data['offset']), int(common_crawl_data['length'])
    offset_end = offset + length - 1

    prefix = 'https://commoncrawl.s3.amazonaws.com/'

    resp = requests.get(prefix + common_crawl_data['filename'], headers={'Range': 'bytes={}-{}'.format(offset, offset_end)})
    raw_data = io.BytesIO(resp.content)

    uri = None
    page = None
    
    for record in ArchiveIterator(raw_data, arc2warc=True):
        uri = record.rec_headers.get_header('WARC-Target-URI')
        R = record.content_stream().read()
        try:
            page = R.strip().decode('utf-8')
        except:
            page = R.strip().decode('latin1')
        print(uri, page)
    return uri, page

debug()

Answer 1

有关为一些未经身份验证的请求生成403 s的最新更改，请参见这篇无聊的博客文章。