为什么我从链接列表中抓取网页的链接点击速度很慢？C#

Question

我想点击文本为"300“的所有链接。我的网页抓取代码对每个链接的点击速度非常慢。我将链接存储在一个列表中，并逐个单击它们。

我计算用于索引的链接，然后使用for(int pos = 0；pos < numberOfElementsFound；pos++)。我已经尝试过这段代码在[https://www.w3schools.com/html/default.asp]上计数和点击(By.PartialLinkText("3600") )，响应非常快，但在另一个网站上非常慢。

class Program
{
    private static IWebDriver driver = null;

    static void Main(string[] args)
    {
        driver = new InternetExplorerDriver();
        driver.Manage().Window.Maximize();
        driver.Navigate().GoToUrl("https://arbitrary.com/");

        clickAllLinks("300");
    }

    //clicking links AND get data
    public static void clickAllLinks(string tagName)
    {
        IWebElement element = 
        driver.FindElement(By.XPath("//div[@class='data']"));

        int elements = 
        element.FindElements(By.PartialLinkText(tagName)).Count();

        for (int pos = 0; pos < elements; pos++)
        {
            getElementWithIndex(By.PartialLinkText(tagName), pos).Click();
            //fetchdata();
        }
    }

    public static IWebElement getElementWithIndex(By by, int pos)
    {
        IWebElement element = 
        driver.FindElement(By.XPath("//div[@class='data']"));
        IList<IWebElement> elements = 
        element.FindElements(By.PartialLinkText("300"));
        return elements.ElementAt(pos);
    }



    //scrape data
    public static async void fetchdata()
    {
        string currentURL = driver.Url; //url to string
        Console.WriteLine("URL: " + currentURL);

        var httpclient = new HttpClient();
        var html = await httpclient.GetStringAsync(currentURL);

        var htmldoc = new HtmlDocument();
        htmldoc.LoadHtml(html); //html to htmldoc

      List<List<string>> Receipt = 
      htmldoc.DocumentNode.SelectSingleNode("//table[@class='classname']")
            //htmldoc into list TABLE->TR->TD->InnerText
            .Descendants("tr")
            .Where(tr => tr.Elements("td").Count() > 0)
            .Select(tr => tr.Elements("td")
            .ToList())
            .ToList();

Answer 1

这是您的clickAllLinks方法的简化版本。这将减少当前方法中的开销(获取元素并不必要地存储，这可能会影响执行速度)。

//clicking links AND get data
public static void clickAllLinks(string tagName)
{
    int elements = 
    driver.FindElements(By.xpath("//div[@class='data']//a[contains(.," + tagName + ")]").Count();

    for (int pos = 1; pos < elements; pos++)
    {
        driver.FindElements(By.xpath("(//div[@class='data']//a[contains(.," + tagName + ")])[" +  pos + "]").Click();
        //fetchdata();
    }
}