用于Cube导师的python web爬虫上的500状态错误(魔法收集站点)

Question

这是我的代码起作用了：

import requests
from bs4 import BeautifulSoup as bs4
cookies = {
    'JSESSIONID': '15EA1C17E103E8206BAFFF73FA157231',
}

headers = {
    'Pragma': 'no-cache',
    'Origin': 'http://www.cubetutor.com',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'en-US,en;q=0.9',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'Accept': '*/*',
    'Cache-Control': 'no-cache',
    'X-Requested-With': 'XMLHttpRequest',
    'Connection': 'keep-alive',
    'Referer': 'http://www.cubetutor.com/topcardsbyset/1;jsessionid=15EA1C17E103E8206BAFFF73FA157231',
}

data = {
  't:ac': '1',
  't:formdata': 'byjWim5rtLJcD8P4BWZe94Mn6II=:H4sIAAAAAAAAAJWOPQ4BQRSAH4lCdBIRPe1oaKiQqEQkywHezj5rZOxM5j1/l3ECcQmFzh0cQKtS2DiARPsV3/edn1DaN6A+d36EIeHhMSLpMQmTJS0coOtCqtCjXpES9MQSjl2lXSBrYhUjkxrEOUQtY0M2aeaCrW8trpVH7fYuQmECFe0yCc5OcUMC1ckad9i2mKXtSILJ0v7BC5TzaPSN/v4Z/PszC04Tc7SNN4bZuOx6STrL1+leBDj4DxyMK0MBAQAA',
  'setSelect': '10E',
  't:zoneid': 'topCardsZone'
}

response = bs4(requests.post('http://www.cubetutor.com/topcardsbyset.topcardsform', headers=headers, cookies=cookies, data=data).json()['content'], 'lxml')
text = response.text.split("\n")
file = text.pop(0)
link = response.find_all("a")
arr = {file : {}}
for i in range(len(link)):
    arr[file][text[i]] = link[i]

print(file)
for i in arr[file]:
    print(i," : ", arr[file][i])

text = response.text.split("\n") on中的所有内容都不重要(没有完成，也没有问题所在)。上面的代码都很好，但它要求我去网站，查看网络请求/头，每次都获取所有这些信息。我试图使这个过程有点自动化，但是第二次我尝试创建我自己的jsession id，这样我就不必继续访问这个站点了，它给了我一个500状态错误。一开始，我想这可能是因为jsession id和头不匹配，所以我将其更改为等于session.headers，正如您在下面看到的，但是没有修复任何问题。如果您想查看另一个提交，代码位于https://github.com/icarus612/spiderPY-magicTG/。任何帮助都会很好。

import requests
from bs4 import BeautifulSoup as soup

data = {
  't:ac': '1',
  't:formdata': 'byjWim5rtLJcD8P4BWZe94Mn6II=:H4sIAAAAAAAAAJWOPQ4BQRSAH4lCdBIRPe1oaKiQqEQkywHezj5rZOxM5j1/l3ECcQmFzh0cQKtS2DiARPsV3/edn1DaN6A+d36EIeHhMSLpMQmTJS0coOtCqtCjXpES9MQSjl2lXSBrYhUjkxrEOUQtY0M2aeaCrW8trpVH7fYuQmECFe0yCc5OcUMC1ckad9i2mKXtSILJ0v7BC5TzaPSN/v4Z/PszC04Tc7SNN4bZuOx6STrL1+leBDj4DxyMK0MBAQAA',
  'setSelect': '10E',
  't:zoneid': 'topCardsZone'
}

session = requests.Session()
r = session.get('http://www.cubetutor.com')
cookies = session.cookies.get_dict()
headers = session.headers
headers['Referer'] =  f"http://www.cubetutor.com/topcardsbyset/1;jsessionid={cookies['JSESSIONID']}"
headers['Pragma'] = 'no-cache'
headers['Origin'] = 'http://www.cubetutor.com'
headers['X-Requested-With'] = 'XMLHttpRequest'

print(r)
response = session.get('http://www.cubetutor.com/topcardsbyset.topcardsform', headers=headers, data=data, cookies=cookies)
print(response)
#text = response.find_all(class_='compareCubeColumn').text
#link = response.find_all("a")

Answer 1

这似乎有效(不包括bs4解析)：

import requests

def get_session_cookie():
    resp = requests.get("https://www.cubetutor.com")
    if resp.ok and resp.cookies:
        return resp.cookies.get_dict()


def post_query():
    session_cookie = get_session_cookie()
    headers = {
        'Pragma': 'no-cache',
        'Origin': 'https://www.cubetutor.com',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'en-US,en;q=0.9',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Accept': '*/*',
        'Cache-Control': 'no-cache',
        'X-Requested-With': 'XMLHttpRequest',
        'Connection': 'keep-alive',
        'Referer': 'https://www.cubetutor.com/topcardsbyset/1;jsessionid=15EA1C17E103E8206BAFFF73FA157231',
    }
    data = {
      't:ac': '1',
      't:formdata': 'byjWim5rtLJcD8P4BWZe94Mn6II=:H4sIAAAAAAAAAJWOPQ4BQRSAH4lCdBIRPe1oaKiQqEQkywHezj5rZOxM5j1/l3ECcQmFzh0cQKtS2DiARPsV3/edn1DaN6A+d36EIeHhMSLpMQmTJS0coOtCqtCjXpES9MQSjl2lXSBrYhUjkxrEOUQtY0M2aeaCrW8trpVH7fYuQmECFe0yCc5OcUMC1ckad9i2mKXtSILJ0v7BC5TzaPSN/v4Z/PszC04Tc7SNN4bZuOx6STrL1+leBDj4DxyMK0MBAQAA',
      'setSelect': '10E',
      't:zoneid': 'topCardsZone'
    }
    return requests.post('https://www.cubetutor.com/topcardsbyset.topcardsform', headers=headers, cookies=session_cookie, data=data).json()

# resp = post_query()
# print(resp)
# {u'content': u"<div class='centeredContainer'><div class='compa ...