Python 爬虫实战：抓取汽车之家二手车成交价格与里程数据

GET https://www.che168.com/front/api/usedcar/dealList
    ?page=1&pageSize=20&cityId=110100&seriesId=0

import time
import requests

BASE_URL = "https://www.che168.com/front/api/usedcar/dealList"

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/124.0 Safari/537.36"
    ),
    "Referer": "https://www.che168.com/",
    "Accept": "application/json, text/plain, */*",
}

def fetch_deals(city_id: int, max_pages: int = 5) -> list[dict]:
    """抓取指定城市的二手车成交记录。

    Args:
        city_id: 城市编码，如北京 110100。
        max_pages: 最多翻页数，防止无限爬取。

    Returns:
        成交记录字典列表，每条含车型/价格/里程等字段。
    """
    results: list[dict] = []
    for page in range(1, max_pages + 1):
        params = {"page": page, "pageSize": 20, "cityId": city_id}
        resp = requests.get(BASE_URL, params=params, headers=HEADERS, timeout=10)
        resp.raise_for_status()
        payload = resp.json()
        items = payload.get("data", {}).get("list", [])
        if not items:
            break
        results.extend(items)
        print(f"page={page} got {len(items)} items")
        time.sleep(1.5)  # 礼貌性限速，别把人家打挂
    return results

if __name__ == "__main__":
    deals = fetch_deals(city_id=110100)
    print(f"total: {len(deals)}")

# 亿牛云动态转发代理（16yun.cn）
# 统一转发网关，后台按请求轮换出口 IP，无需自建 IP 池
PROXY_HOST = "t.16yun.cn"
PROXY_PORT = "31111"
PROXY_USER = "YOUR_ORDER_ID"     # 亿牛云订单号
PROXY_PASS = "YOUR_PASSWORD"     # 订单密码

PROXIES = {
    "http": f"http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}",
    "https": f"http://{PROXY_USER}:{PROXY_PASS}@{PROXY_HOST}:{PROXY_PORT}",
}

resp = requests.get(
    BASE_URL, params=params, headers=HEADERS,
    proxies=PROXIES, timeout=10,
)

import pandas as pd
import re

def clean_deals(raw: list[dict]) -> pd.DataFrame:
    """把原始成交记录清洗成规整的 DataFrame。

    Args:
        raw: 接口返回的原始记录列表。

    Returns:
        规整后的 DataFrame，价格/里程均为 float。
    """
    rows = []
    for it in raw:
        price = it.get("dealPrice") or it.get("price")
        mileage = it.get("mileage") or it.get("licheng")
        rows.append({
            "model": it.get("seriesName") or it.get("carName"),
            "city": it.get("cityName"),
            "register_date": it.get("registerDate"),
            "deal_price": float(re.sub(r"[^\d.]", "", str(price))) if price else None,
            "mileage": float(re.sub(r"[^\d.]", "", str(mileage))) if mileage else None,
        })
    df = pd.DataFrame(rows).drop_duplicates()
    return df

if __name__ == "__main__":
    deals = fetch_deals(city_id=110100)
    df = clean_deals(deals)
    df.to_csv("used_car_deals.csv", index=False, encoding="utf-8-sig")
    print(df.describe())  # 快速看价格/里程分布

import matplotlib.pyplot as plt
import numpy as np

def plot_price_vs_mileage(df: pd.DataFrame, model: str) -> None:
    """绘制指定车型的价格—里程散点图与趋势线。

    Args:
        df: 清洗后的成交 DataFrame。
        model: 车型名，用于筛选与标题。
    """
    sub = df[df["model"] == model].dropna(subset=["deal_price", "mileage"])
    if len(sub) < 5:
        print(f"{model} 样本不足，跳过绘图")
        return
    plt.figure(figsize=(8, 5))
    plt.scatter(sub["mileage"], sub["deal_price"], alpha=0.6)
    # 一元线性回归趋势线
    z = np.polyfit(sub["mileage"], sub["deal_price"], 1)
    x = np.linspace(sub["mileage"].min(), sub["mileage"].max(), 100)
    plt.plot(x, np.polyval(z, x), color="red", label=f"趋势线 斜率={z[0]:.2f}万/万公里")
    plt.xlabel("里程（万公里）")
    plt.ylabel("成交价（万元）")
    plt.title(f"{model} 价格—里程关系")
    plt.legend()
    plt.savefig(f"{model}_price_mileage.png", dpi=120)
    plt.close()

# 调用示例
# plot_price_vs_mileage(df, "凯美瑞")

from concurrent.futures import ThreadPoolExecutor

CITY_IDS = [110100, 310100, 440100, 510100]  # 北京/上海/广州/成都

def crawl_city(city_id: int) -> pd.DataFrame:
    """采集单个城市并清洗，返回 DataFrame。"""
    return clean_deals(fetch_deals(city_id, max_pages=10))

with ThreadPoolExecutor(max_workers=4) as pool:
    frames = list(pool.map(crawl_city, CITY_IDS))

df_all = pd.concat(frames, ignore_index=True)
df_all.to_csv("used_car_deals_multi.csv", index=False, encoding="utf-8-sig")