Python 爬虫如何分析并模拟 JS 动态请求

function generateSign(timestamp, page, limit) {
    const secret = "abc123xyz"; // 固定密钥
    const str = timestamp + page + limit + secret;
    return md5(str); // MD5加密
}

import requests
import time
import hashlib
import json
from typing import Dict, List

class JSDynamicRequestCrawler:
    def __init__(self):
        # 目标API接口
        self.api_url = "https://example.com/api/news/list"
        # 固定密钥（逆向得到）
        self.secret = "abc123xyz"
        # 请求头（从浏览器Headers中复制）
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
            "Referer": "https://example.com/news",
            "Content-Type": "application/json;charset=UTF-8"
        }

    def generate_sign(self, timestamp: str, page: int, limit: int) -> str:
        """根据逆向规则生成签名"""
        # 拼接字符串
        str_to_sign = f"{timestamp}{page}{limit}{self.secret}"
        # MD5加密
        md5_obj = hashlib.md5()
        md5_obj.update(str_to_sign.encode("utf-8"))
        sign = md5_obj.hexdigest()
        return sign

    def get_news_list(self, page: int, limit: int = 10) -> List[Dict]:
        """获取单页新闻列表数据"""
        # 生成时间戳（秒级，与JS中的一致）
        timestamp = str(int(time.time()))
        # 生成签名
        sign = self.generate_sign(timestamp, page, limit)
        # 构造请求参数
        payload = {
            "page": page,
            "limit": limit,
            "timestamp": timestamp,
            "sign": sign
        }

        try:
            # 发送POST请求
            response = requests.post(
                url=self.api_url,
                headers=self.headers,
                json=payload  # 若接口为form-data，使用data=payload
            )
            # 验证响应状态
            response.raise_for_status()
            # 解析JSON数据
            data = response.json()
            if data.get("code") == 200:  # 假设接口返回code=200表示成功
                news_list = data.get("data", {}).get("list", [])
                print(f"成功获取第{page}页数据，共{len(news_list)}条新闻")
                return news_list
            else:
                print(f"接口返回错误：{data.get('msg')}")
                return []
        except requests.exceptions.RequestException as e:
            print(f"请求失败：{str(e)}")
            return []

    def save_data(self, all_news: List[Dict], file_path: str = "news_list.json"):
        """保存数据到本地JSON文件"""
        with open(file_path, "w", encoding="utf-8") as f:
            json.dump(all_news, f, ensure_ascii=False, indent=4)
        print(f"所有数据已保存到{file_path}，共{len(all_news)}条新闻")

    def start_crawl(self, max_page: int = 5):
        """启动爬虫，爬取多页数据"""
        all_news = []
        for page in range(1, max_page + 1):
            news_list = self.get_news_list(page)
            if not news_list:
                # 若某页数据获取失败，可选择停止或继续
                print(f"第{page}页数据获取失败，停止爬取")
                break
            all_news.extend(news_list)
            # 避免请求过快，添加短暂延迟
            time.sleep(1)
        self.save_data(all_news)

if __name__ == "__main__":
    # 初始化爬虫并启动
    crawler = JSDynamicRequestCrawler()
    crawler.start_crawl(max_page=5)