NodeJS爬行器登录站点

Question

我想抓取geocaching.com，但有些数据，如同弦，只供用户登录。我正在使用来自npm的“爬虫”，现在我知道如何使用爬虫登录，但我已经获得了登录表单的名称：

• ctl00$ContentBody$tbUsername: user
• $ContentBody$tbPassword: passwaord
• ctl00$ContentBody$btnSignIn："Sign+In“

到目前为止，我的代码如下：

var Crawler = require("crawler");
var url = require('url');
var mongoose = require("mongoose");
var Cache = require("./models/cache.js");

mongoose.connect("localhost:27017/Cache");

var removeTags = function(text){
    return String(text).replace(/(<([^>]+)>)/ig,'');
};
var c = new Crawler({
    maxConnections: 10,
    skipDuplicates: true,

    callback: function (error, result, $) {

        if (result.request.uri.href.startsWith("http://www.geocaching.com/geocache/")) {
            var cache = new Cache();
            var id = removeTags($(".CoordInfoCode"));
            Cache.count({
                "_id": id
            }, function (err, count) {
                if (err)
                    return;
                else if (count < 1) {
                    //Saving the data
                }

            });


        }
        if (result.headers['content-type'] == "text/html; charset=utf-8") {
            if ($('a').length != 0) {
                $('a').each(function (index, a) {
                    var toQueueUrl = $(a).attr('href');
                    process.nextTick(function () {
                        process.nextTick(function () {
                            c.queue(toQueueUrl);
                        })
                    });

                });
            }
        }

    }
});

c.queue('http://www.geocaching.com/seek/nearest.aspx?ul=Die_3sten_3');

Answer 1

我在github上做了一个javascript爬虫示例。

它是事件驱动的，并使用内存中的队列来存储所有资源(即。网址)。

如何在节点环境中使用

var Crawler = require('../lib/crawler')
var crawler = new Crawler('http://www.someUrl.com');

// crawler.maxDepth = 4;
// crawler.crawlInterval = 10;
// crawler.maxListenerCurrency = 10;
// crawler.redisQueue = true;
crawler.start();

这里我只是向您展示javascript爬虫的2种核心方法。

Crawler.prototype.run = function() {
  var crawler = this;
  process.nextTick(() => {
    //the run loop
    crawler.crawlerIntervalId = setInterval(() => {

      crawler.crawl();

    }, crawler.crawlInterval);
    //kick off first one
    crawler.crawl();
  });

  crawler.running = true;
  crawler.emit('start');
}


Crawler.prototype.crawl = function() {
  var crawler = this;

  if (crawler._openRequests >= crawler.maxListenerCurrency) return;


  //go get the item
  crawler.queue.oldestUnfetchedItem((err, queueItem, index) => {
    if (queueItem) {
      //got the item start the fetch
      crawler.fetchQueueItem(queueItem, index);
    } else if (crawler._openRequests === 0) {
      crawler.queue.complete((err, completeCount) => {
        if (err)
          throw err;
        crawler.queue.getLength((err, length) => {
          if (err)
            throw err;
          if (length === completeCount) {
            //no open Request, no unfetcheditem stop the crawler
            crawler.emit("complete", completeCount);
            clearInterval(crawler.crawlerIntervalId);
            crawler.running = false;
          }
        });
      });
    }

  });
};

这是github链接https://github.com/bfwg/node-tinycrawler。它是一个javascript爬虫，用1000行代码编写。这应该能让你走上正轨。