用Scrapy无限滚动Ajax提取数据

Question

我是python和scrapy的初学者。我需要帮助，我需要提取一个产品清单，但是他的站点在最后有一个“查看更多”的产品，用ajax执行文本/ html请求，然后用新产品加载另一个html。

import scrapy
from scrapy.http import Request


class ProdSpider(scrapy.Spider):
name = "easy"
allowed_domains = ["easy.com.ar"]
start_urls = ["https://www.easy.com.ar/webapp/wcs/stores/servlet/es/easyar/search/AjaxCatalogSearchResultContentView?searchTermScope=&searchType=1002&filterTerm=&orderBy=&maxPrice=&showResultsPage=true&langId=-5&sType=SimpleSearch&metaData=&pageSize=12&manufacturer=&resultCatEntryType=&catalogId=10051&pageView=image&searchTerm=&minPrice=&categoryId=39652&storeId=10151&beginIndex=12"]
beginIndex_index = 12

def parse(self, response):
    SECTION_SELECTOR = '.thumb-product'
    for soar in response.css(SECTION_SELECTOR):

             Link = 'div.dojoDndItem a ::attr(href)'
             # Marca = 'p.brand a ::text'
             Nombre = 'div.thumb-name a ::text'
             # Link = 'p.brand a ::attr(href)'
             # SKU = './/p[@class="sku"]/text()' #p.sku ::text'
             Price = './/span[@id="tarj-mas-edit"]/text()' #["0"].parentElement.innerText    .//span[@class="thumb-price-e"]/text()

             yield {
               'Link': soar.css(Link).extract_first(),
            # 'Marca': soar.css(Marca).extract_first(),
               'Nombre': soar.css(Nombre).re_first(r'\n\s*(.*)'), # Limpia espacios y caracteres especiales
            # 'Link': soar.css(Link).extract_first(),   
            # 'SKU': soar.xpath(SKU).re_first(r'SKU:\s*(.*)'),
               'Price': soar.xpath(Price).re_first(r'\n\s*(.*)'),
              }
    # here if no products are available , simply return, means exiting from
    # parse and ultimately stops the spider


    self.beginIndex_index += 12
    if beginIndex_index:
        yield Request(url="https://www.easy.com.ar/webapp/wcs/stores/servlet/es/easyar/search/AjaxCatalogSearchResultContentView?searchTermScope=&searchType=1002&filterTerm=&orderBy=&maxPrice=&showResultsPage=true&langId=-5&sType=SimpleSearch&metaData=&pageSize=12&manufacturer=&resultCatEntryType=&catalogId=10051&pageView=image&searchTerm=&minPrice=&categoryId=39652&storeId=10151&beginIndex=%s" % (self.beginIndex_index + 12),
                      callback=self.parse)

我试过上面的代码，但只有12种产品被捕获。在url中唯一更改的参数是"beginIndex=12"，Y希望将+12和到url，直到产品列表结束为止。我被这个问题缠住了！

谢谢!

Answer 1

我建议您使用selenium，这样您就可以“单击”视图更多的按钮，并在蜘蛛中加载更多的数据。下面是一个蜘蛛示例(我没有测试它，但它是一般的想法)：

import scrapy
from selenium import webdriver
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary


class ProdSpider(scrapy.Spider):
    name = "easy"
    allowed_domains = ["easy.com.ar"]
    start_urls = ["https://www.easy.com.ar/webapp/wcs/stores/servlet/es/easyar/search/aditivos-y-lubricantes"]

    def __init__(self):
        super(ProdSpider, self).__init__()
        binary = FirefoxBinary('C:/Program Files (x86)/Mozilla Firefox/firefox.exe')
        self.wb = webdriver.Firefox(firefox_binary=binary)

    def parse(self, response):
        self.wb.get(response.url)
        while True:
            view_more_button = self.wb.find_element_by_xpath('//*[@id="Search_Result_div"]/div[2]/div[9]/input')
            if not view_more_button:
                break
            view_more_button.click()
            #extract your data here...

Answer 2

你明白了！

我在您的URL中看到，您有一个名为pageSize的其他参数。我对它进行了测试，网站允许你将它设置为最多50。

要知道何时停止，只需在生成另一个请求之前测试response.css(SECTION_SELECTOR)中是否有项：

import scrapy
from scrapy.http import Request
from scrapy import Selector


class ProdSpider(scrapy.Spider):
  name = "easy"
  allowed_domains = ["easy.com.ar"]

  url = "https://www.easy.com.ar/webapp/wcs/stores/servlet/es/easyar/search/AjaxCatalogSearchResultContentView?searchTermScope=&searchType=1002&filterTerm=&orderBy=&maxPrice=&showResultsPage=true&langId=-5&sType=SimpleSearch&metaData=&pageSize=50&manufacturer=&resultCatEntryType=&catalogId=10051&pageView=image&searchTerm=&minPrice=&categoryId=39652&storeId=10151&beginIndex={pagenum}"

  product_fields_xpath = {
    'Link': '//a[contains(@id, "CatalogEntry")]/@href',
    'Nombre': '//a[contains(@id, "CatalogEntry")]/text()',
    'Price': './/span[@class="thumb-price-e"]/text()'
  }

  section_selector = '//div[@class="thumb-product"]'

  begin_index = 0

  def start_request(self):
    yield Request(url=url.format(pagenum=self.begin_index), method='GET', callback=self.parse)

  def parse(self, response):
    products = response.xpath(self.section_selector).extract()
    n_items = 0
    for product in products:
      n_items += 1
      sel = Selector(text=product)
      item = dict()
      for k, v in self.product_fields_xpath.iteritems():
        item[k] = sel.xpath(v).extract_first()
      yield item

    self.begin_index += 50

    if n_items > 0:
      yield Request(url=url.format(pagenum=self.begin_index), method='GET', callback=self.parse)

我没有测试这段代码，但我希望你能理解我的意思。