python中的web抓取问题，网页没有及时加载。

Question

我正在做一个程序，以废除亚马逊网站，移动电话，但我的程序是给我超时例外，即使在页面加载准时。

这是我的密码

    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.webdriver.support import expected_conditions as EC
    from selenium.webdriver.common.by import By
    from selenium.common.exceptions import TimeoutException
    from selenium.common.exceptions import NoSuchElementException

    from bs4 import BeautifulSoup
    import urllib.request

    class Amazon_all_mobile_scraper:
        def __init__(self):
            self.driver = webdriver.Firefox()
            self.delay = 60
            self.url = "https://www.amazon.in/mobile-phones/b/ref=sd_allcat_sbc_mobcomp_all_mobiles?ie=UTF8&node=1389401031"

        def load_amazon(self):
            self.driver.get(self.url)
            try:
                wait = WebDriverWait(self.driver,self.delay)
                        wait.until(EC.presence_of_element_located((By.CLASS_NAME,"acs-ln-link")))
                print("Page is ready.")
            except TimeoutException:
                print("Took too much time to load!")
            except:
                print("Something went wrong in loading part!!")

        def extract_list_of_mobiles(self):
            try:
                mobile_list = self.driver.find_element_by_xpath('//div[@class = "acs-ln-link"]')
                print(mobile_list)
            except NoSuchElementException:
                print("Sorry, Unable to get the requested element")


    scraper = Amazon_all_mobile_scraper()
    scraper.load_amazon()
    scraper.extract_list_of_mobiles()

请帮我找出这段代码有什么问题。

Answer 1

只有从acs-ln-link到acs-ln-links的转换才不会起作用。您的xpath应该更像'//div[contains(@class,"acs-ln-nav-expanded")]//*[@class="acs-ln-links"]//a'。但是，您可以处理这一问题以获得所需的输出：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

class Amazon_all_mobile_scraper:

    url = "https://www.amazon.in/mobile-phones/b/ref=sd_allcat_sbc_mobcomp_all_mobiles?ie=UTF8&node=1389401031"

    def __init__(self):
        self.driver = webdriver.Chrome()
        self.wait = WebDriverWait(self.driver, 15)

    def load_n_get_from_amazon(self):
        self.driver.get(self.url)
        mobile_list = self.wait.until(EC.presence_of_all_elements_located((By.XPATH,'//div[contains(@class,"acs-ln-nav-expanded")]//*[@class="acs-ln-links"]//a')))
        return mobile_list

    def __del__(self):
        self.driver.close()

if __name__ == '__main__':
    scraper = Amazon_all_mobile_scraper()
    for item in scraper.load_n_get_from_amazon():
        print(f'{item.text}\n{item.get_attribute("href")}\n')

Answer 2

这个类不匹配"acs-ln-link“应该是”acs-ln-link“。