大数据时代的利剑：Bright Data网页抓取与自动化工具共建高效数据采集新生态

from selenium.webdriver import Remote, ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.chromium.remote_connection import ChromiumRemoteConnection
import pandas as pd

# 替换为你自己的信息即可
AUTH = 'brd-customer-您的客户 ID-zone-您的区域:您的密码'
SBR_WEBDRIVER = f'https://{AUTH}@brd.superproxy.io:9515'


def main():
    print('连接到 Scraping Browser...')
    sbr_connection = ChromiumRemoteConnection(SBR_WEBDRIVER, 'goog', 'chrome')
    with Remote(sbr_connection, options=ChromeOptions()) as driver:
        print('连接成功！正在导航到亚马逊电脑产品列表...')
        driver.get('https://www.amazon.com/s?k=laptop')

        print('页面加载完成！正在提取商品链接...')
        driver.implicitly_wait(10)

        # 获取所有商品的容器
        products = driver.find_elements(By.XPATH, '//div[@data-component-type="s-search-result"]')
        product_links = []

        for link in product_links:
            print(f'正在访问商品页面: {link}')
            driver.get(link)
            try:
                # 找到评价页面链接
                review_link = WebDriverWait(driver, 10).until(
                    EC.presence_of_element_located((By.XPATH, '//a[@data-hook="see-all-reviews-link-foot"]'))
                ).get_attribute('href')
                driver.get(review_link)

                # 获取该款式的所有评价数
                try:
                    total_reviews_count = WebDriverWait(driver, 10).until(
                        EC.presence_of_element_located((By.XPATH, '//div[@data-hook="total-review-count"]'))
                    ).text
                except:
                    total_reviews_count = None

                # 获取评价容器
                reviews = WebDriverWait(driver, 10).until(
                    EC.presence_of_all_elements_located((By.XPATH, '//div[@data-hook="review"]'))
                )

                for review in reviews:
                    try:
                        # 提取评价标题
                        title = review.find_element(By.XPATH, './/a[@data-hook="review-title"]').text
                    except:
                        title = None

                    try:
                        # 提取评价内容
                        content = review.find_element(By.XPATH, './/span[@data-hook="review-body"]').text
                    except:
                        content = None

                    try:
                        # 提取评价星级
                        rating = review.find_element(By.XPATH, './/i[@data-hook="review-star-rating"]').text
                    except:
                        rating = None

                    try:
                        # 提取评价者
                        reviewer = review.find_element(By.XPATH, './/span[@class="a-profile-name"]').text
                    except:
                        reviewer = None

                    try:
                        # 提取评价时间
                        review_date = review.find_element(By.XPATH, './/span[@data-hook="review-date"]').text
                    except:
                        review_date = None

                    try:
                        # 提取评价的评价数
                        review_helpful_count = review.find_element(By.XPATH, './/span[@data-hook="helpful-vote-statement"]').text
                    except:
                        review_helpful_count = None

                    all_reviews.append({
                        '评价网址': review_link,
                        '评价标题': title,
                        '评价内容': content,
                        '评价星级': rating,
                        '评价者': reviewer,
                        '评价时间': review_date,
                        '该款式的所有评价数': total_reviews_count,
                        '评价的评价数': review_helpful_count
                    })


            except:
                print('未找到该商品的评价页面')

        # 将数据保存到 DataFrame
        df = pd.DataFrame(all_reviews)

        # 将数据保存到 CSV 文件
        df.to_csv('amazon_laptop_reviews.csv', index=False, encoding='utf-8-sig')

        print('数据提取完成，已保存到 amazon_laptop_reviews.csv')


if __name__ == '__main__':
    main()

定时触发 --> HTTP请求（Bright Data API）--> 函数解析 --> 条件判断 --> 钉钉/企业微信通知
                         ↓
                    数据库存储节点