如何用 Python 采集图片素材并下载保存到数据库

bash编辑pip install requests beautifulsoup4 pillow

python编辑import sqlite3

def init_db():
    conn = sqlite3.connect('images.db')
    cursor = conn.cursor()
    cursor.execute('''
        CREATE TABLE IF NOT EXISTS images (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            url TEXT NOT NULL,
            filename TEXT,
            data BLOB NOT NULL
        )
    ''')
    conn.commit()
    conn.close()

python编辑import requests
import os
from urllib.parse import urljoin, urlparse

def download_image_to_db(session, img_url, base_url=None):
    try:
        # 处理相对链接
        if base_url and not img_url.startswith(('http://', 'https://')):
            img_url = urljoin(base_url, img_url)

        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        response = session.get(img_url, headers=headers, timeout=10)
        response.raise_for_status()

        # 简单判断是否为图片（通过 Content-Type 或扩展名）
        content_type = response.headers.get('content-type', '')
        if 'image' not in content_type:
            print(f"跳过非图片资源: {img_url}")
            return False

        # 获取文件名
        parsed = urlparse(img_url)
        filename = os.path.basename(parsed.path) or "unknown.jpg"

        # 保存到数据库
        conn = sqlite3.connect('images.db')
        cursor = conn.cursor()
        cursor.execute(
            "INSERT INTO images (url, filename, data) VALUES (?, ?, ?)",
            (img_url, filename, response.content)
        )
        conn.commit()
        conn.close()
        print(f"✅ 已保存: {filename}")
        return True

    except Exception as e:
        print(f"❌ 下载失败 {img_url}: {e}")
        return False

python编辑from bs4 import BeautifulSoup

def scrape_images_from_page(url):
    session = requests.Session()
    try:
        headers = {'User-Agent': 'Mozilla/5.0'}
        res = session.get(url, headers=headers)
        res.raise_for_status()
        soup = BeautifulSoup(res.text, 'html.parser')

        img_tags = soup.find_all('img')
        img_urls = [img.get('src') for img in img_tags if img.get('src')]

        print(f"共找到 {len(img_urls)} 个图片链接")
        for img_url in img_urls:
            download_image_to_db(session, img_url, base_url=url)

    except Exception as e:
        print(f"页面解析失败: {e}")

python编辑if __name__ == "__main__":
    init_db()
    target_url = "https://www.jsonla.com/tupian/"  # 替换为目标网页
    scrape_images_from_page(target_url)

sql编辑CREATE UNIQUE INDEX IF NOT EXISTS idx_url ON images(url);

python编辑if int(response.headers.get('content-length', 0)) > 5 * 1024 * 1024:  # 5MB
    print("图片过大，跳过")
    return False