Python爬虫开发：Cookie池与定期清除的代码实现

import sqlite3
    ''')
 conn.commit()
 conn.close()
# 保存Cookie到数据库
def save_cookie(db_file, cookie):
 conn = sqlite3.connect(db_file)
 cursor = conn.cursor()
 cursor.execute('INSERT INTO cookies (cookie) VALUES (?)', (json.dumps(cookie),))
 conn.commit()
 conn.close()
# 获取一个有效的Cookie
def get_valid_cookie(db_file):
 conn = sqlite3.connect(db_file)
 cursor = conn.cursor()
 cursor.execute('SELECT id, cookie FROM cookies WHERE status=1 ORDER BY last_used ASC LIMIT 1')
 result = cursor.fetchone()
 conn.close()
 if result:
 return result[0], json.loads(result[1])
 return None, None
# 更新Cookie的使用时间
def update_cookie_usage(db_file, cookie_id):
 conn = sqlite3.connect(db_file)
 cursor = conn.cursor()
 cursor.execute('UPDATE cookies SET last_used=CURRENT_TIMESTAMP WHERE id=?', (cookie_id,))
 conn.commit()
 conn.close()
# 标记Cookie为失效
def mark_cookie_invalid(db_file, cookie_id):
 conn = sqlite3.connect(db_file)
 cursor = conn.cursor()
 cursor.execute('UPDATE cookies SET status=0 WHERE id=?', (cookie_id,))
 conn.commit()
 conn.close()
# 清理超过一定时间未使用的Cookie
def clean_expired_cookies(db_file, expire_time=3600):
 conn = sqlite3.connect(db_file)
 cursor = conn.cursor()
 cursor.execute('DELETE FROM cookies WHERE last_used < ?', (time.time() - expire_time,))
 conn.commit()
 conn.close()
# 登录并保存Cookie
def login_and_save_cookie():
 response = requests.post('https://example.com/login', data={'username': 'user', 'password': 'pass'})
 if response.status_code == 200:
 save_cookie(db_file, response.cookies.get_dict())
# 使用Cookie池发起请求
def fetch_data():
 cookie_id, cookie = get_valid_cookie(db_file)
 if cookie:
 try:
 response = requests.get('https://example.com/data', cookies=cookie)
 if response.status_code == 200:
 update_cookie_usage(db_file, cookie_id)
 return response.text
 else:
 mark_cookie_invalid(db_file, cookie_id)
 except Exception as e:
 mark_cookie_invalid(db_file, cookie_id)
 return None
# 初始化数据库
init_db(db_file)
# 示例：登录并保存Cookie
login_and_save_cookie()
# 示例：使用Cookie池发起请求
data = fetch_data()
if data:
 print(data)
else:
 print("Failed to fetch data.")
# 定时清理失效的Cookie
def scheduled_clean():
 clean_expired_cookies(db_file, expire_time=3600)
schedule.every(1).hour.do(scheduled_clean)
while True:
 schedule.run_pending()
 time.sleep(1)