Maven/Java Crawler - StaleElementReferenceException

Question

我得到了一个StaleElementReferenceException，但我不知道为什么？crawler可以正常工作并循环通过排队的urls，但会失败，并显示StaleElementReferenceException错误

任何帮助都将不胜感激。

下面是代码和错误。

代码

    package com.crawlerr.pageObjects;
    import java.util.ArrayList;
    import java.util.HashSet;
    import java.util.LinkedList;
    import java.util.List;
    import java.util.Queue;

    import org.openqa.selenium.By;
    import org.openqa.selenium.WebDriver;
    import org.openqa.selenium.WebElement;

    import com.crawler.common.ExcelUtils;
    import com.crawler.common.System_Constants;

    public class Crawler4 extends BaseClass
    {
        public static HashSet<String> alreadyListed = new HashSet<String>();
        public static Queue<String> queue = new LinkedList<String>();

        public Crawler4(WebDriver driver) {
            super(driver);
        }

        public static void Execute(int iTestCaseRow) throws Exception {
            crawl(ExcelUtils.getCellData(iTestCaseRow,System_Constants.Col_BaseURL),iTestCaseRow);
        //  System.out.println("Stored"+Arrays.toString(alreadyListed.toArray()));
        }


        public static void crawl(String startingUrl, int iTestCaseRow)
        {
            queue.add(startingUrl);
            String newAddress;
            while ((newAddress = queue.poll())!=null)
            {
                processPage(newAddress, startingUrl, iTestCaseRow);
                //here you may add a code to do anything you wish with the page
            }
        }

        protected static void processPage(String url, String startingUrl, int iTestCaseRow)
        {
            try {
                System.out.println("Current URL-"+url );
                driver.get(url);
                    List<WebElement> crawlList = new ArrayList<WebElement>();
                    crawlList = findAllLinks();
                    String href;
                    System.out.println("Total number of elements found " + crawlList.size());
                      for (int i = 0; i < crawlList.size()-1; i++)
                      {
   /**this line throws the error**/ href =crawlList.get(i).getAttribute("href");
                        if (href!=null && isAddressValid(href,startingUrl))
                        {
                            if (!alreadyListed.contains(href))
                            {
                                System.out.println("Added to list - "+href);
                                alreadyListed.add(href);
                                queue.add(href);
                            }else{System.out.println("Already added to list - "+href);}
                        }else{System.out.println("Not valid - "+href);}
                    }
                } catch (NullPointerException e) {
                // TODO: handle exception
            }
        }

            protected static boolean isAddressValid(String href, String startingUrl)
            {
                //href.contains(s)
               return href.contains(startingUrl);
            }

            private static List<WebElement> findAllLinks(){
                List<WebElement> elementList = new ArrayList<WebElement>();
                elementList = driver.findElements(By.tagName("a"));
                List<WebElement> finalList = new ArrayList<WebElement>();
                for (WebElement element : elementList){
                    finalList.add(element);
                }
                return finalList;
            }
    }

输出：

FAILED: main
org.openqa.selenium.StaleElementReferenceException: Element not found in the cache - perhaps the page has changed since it was looked up
Command duration or timeout: 20.06 seconds
For documentation on this error, please visit: http://seleniumhq.org/exceptions/stale_element_reference.html
Build info: version: '2.48.2', revision: '41bccdd', time: '2015-10-09 19:59:12'
System info: host: 'DESKTOP-PCD960U', ip: '192.168.1.3', os.name: 'Windows 10', os.arch: 'amd64', os.version: '10.0', java.version: '1.8.0_65'
Driver info: org.openqa.selenium.firefox.FirefoxDriver
Capabilities [{applicationCacheEnabled=true, rotatable=false, handlesAlerts=true, databaseEnabled=true, version=42.0, platform=WINDOWS, nativeEvents=false, acceptSslCerts=true, webStorageEnabled=true, locationContextEnabled=true, browserName=firefox, takesScreenshot=true, javascriptEnabled=true, cssSelectorsEnabled=true}]
Session ID: 164b1efc-9098-43b1-ad44-45da58b71f3c
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source)
    at java.lang.reflect.Constructor.newInstance(Unknown Source)
    at org.openqa.selenium.remote.ErrorHandler.createThrowable(ErrorHandler.java:206)
    at org.openqa.selenium.remote.ErrorHandler.throwIfResponseFailed(ErrorHandler.java:158)
    at org.openqa.selenium.remote.RemoteWebDriver.execute(RemoteWebDriver.java:647)
    at org.openqa.selenium.remote.RemoteWebElement.execute(RemoteWebElement.java:326)
    at org.openqa.selenium.remote.RemoteWebElement.getAttribute(RemoteWebElement.java:149)
    at com.crawlerr.pageObjects.Crawler4.processPage(Crawler4.java:52)
    at com.crawlerr.pageObjects.Crawler4.crawl(Crawler4.java:36)
    at com.crawlerr.pageObjects.Crawler4.Execute(Crawler4.java:25)
    at com.crawler.Tests.Crawler_Test.main(Crawler_Test.java:52)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
    at java.lang.reflect.Method.invoke(Unknown Source)
    at org.testng.internal.MethodInvocationHelper.invokeMethod(MethodInvocationHelper.java:85)
    at org.testng.internal.Invoker.invokeMethod(Invoker.java:639)
    at org.testng.internal.Invoker.invokeTestMethod(Invoker.java:821)
    at org.testng.internal.Invoker.invokeTestMethods(Invoker.java:1131)
    at org.testng.internal.TestMethodWorker.invokeTestMethods(TestMethodWorker.java:124)
    at org.testng.internal.TestMethodWorker.run(TestMethodWorker.java:108)
    at org.testng.TestRunner.privateRun(TestRunner.java:773)
    at org.testng.TestRunner.run(TestRunner.java:623)
    at org.testng.SuiteRunner.runTest(SuiteRunner.java:357)
    at org.testng.SuiteRunner.runSequentially(SuiteRunner.java:352)
    at org.testng.SuiteRunner.privateRun(SuiteRunner.java:310)
    at org.testng.SuiteRunner.run(SuiteRunner.java:259)
    at org.testng.SuiteRunnerWorker.runSuite(SuiteRunnerWorker.java:52)
    at org.testng.SuiteRunnerWorker.run(SuiteRunnerWorker.java:86)
    at org.testng.TestNG.runSuitesSequentially(TestNG.java:1185)
    at org.testng.TestNG.runSuitesLocally(TestNG.java:1110)
    at org.testng.TestNG.run(TestNG.java:1018)
    at org.testng.remote.RemoteTestNG.run(RemoteTestNG.java:111)
    at org.testng.remote.RemoteTestNG.initAndRun(RemoteTestNG.java:204)
    at org.testng.remote.RemoteTestNG.main(RemoteTestNG.java:175)
Caused by: org.openqa.selenium.StaleElementReferenceException: Element not found in the cache - perhaps the page has changed since it was looked up
For documentation on this error, please visit: http://seleniumhq.org/exceptions/stale_element_reference.html
Build info: version: '2.48.2', revision: '41bccdd', time: '2015-10-09 19:59:12'
System info: host: 'DESKTOP-PCD960U', ip: '192.168.1.3', os.name: 'Windows 10', os.arch: 'amd64', os.version: '10.0', java.version: '1.8.0_65'
Driver info: driver.version: unknown
    at <anonymous class>.fxdriver.cache.getElementAt(resource://fxdriver/modules/web-element-cache.js:9351)
    at <anonymous class>.Utils.getElementAt(file:///C:/Users/Craig/AppData/Local/Temp/anonymous8982667753843323416webdriver-profile/extensions/fxdriver@googlecode.com/components/command-processor.js:8978)
    at <anonymous class>.WebElement.getElementAttribute(file:///C:/Users/Craig/AppData/Local/Temp/anonymous8982667753843323416webdriver-profile/extensions/fxdriver@googlecode.com/components/command-processor.js:12019)
    at <anonymous class>.DelayedCommand.prototype.executeInternal_/h(file:///C:/Users/Craig/AppData/Local/Temp/anonymous8982667753843323416webdriver-profile/extensions/fxdriver@googlecode.com/components/command-processor.js:12534)
    at <anonymous class>.fxdriver.Timer.prototype.setTimeout/<.notify(file:///C:/Users/Craig/AppData/Local/Temp/anonymous8982667753843323416webdriver-profile/extensions/fxdriver@googlecode.com/components/command-processor.js:621)


===============================================
    Default test
    Tests run: 1, Failures: 1, Skips: 0
===============================================


===============================================
Default suite
Total tests run: 1, Failures: 1, Skips: 0
===============================================

Answer 1

这是使用selenium抓取网页时的常见错误。StaleElementReferenceException指示您试图访问的元素已更改。这可能是由于(但不限于)以下原因：

1. 页Refresh
2. Element可能已更改，因为AJAX load
3. 元素由于等待问题

而在找到该元素后可能已更改

在您的代码中，您将调用findAllLinks()方法来获取crawlList。在此之后，您将遍历crawlList。在查找操作和迭代之间，可能已经发生了使元素过时的任何事件。

唯一的解决方案是在代码周围放置一个try catch。

try {
    href = crawlList.get(i).getAttribute("href");

    if (href != null && isAddressValid(href, startingUrl)) {
        if (!alreadyListed.contains(href)) {
            System.out.println("Added to list - " + href);
            alreadyListed.add(href);
            queue.add(href);
        } else {
            System.out.println("Already added to list - " + href);
        }
    } else {
        System.out.println("Not valid - " + href);
    }

} catch (StaleElementReferenceException e) {
    // do nothing
}

您还可以探索在findAllLinks方法中设置显式等待以等待所有链接加载的可能性。

您还可以检查以下问题，以获得web爬虫的递归解决方案。How to browse a whole website using selenium?