Cheerio WebScraping节点JS

Question

我想要抓取一个网页，并得到所有的链接(如果有)一个缩略图或图像内的'a‘标签。

我能够获得链接，但不确定如何在当前的标签im迭代中获取img > src值。

const cheerio = require('cheerio')
const request = require('request')
const throttledRequest = require('throttled-request')(request)
throttledRequest.configure({ requests: 18, milliseconds: 1000 })

let o = {
  linksOut: []
}

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body)

    $('a').map(function () {
      let link = $(this).attr('href')
      // I want to get the img url within the a tag for the current iteration
      let thumbnail = $(this).//img > src

      o.linksOut.push( {
        link: link,
        thumbnail: thumbnail
      })
    })
  } else {
   // something else
  }
}

const scrape = (() => {
  return new Promise((resolve, reject) => {
    throttledRequest({
      url: 'https://www.ibm.com/us-en',
      followAllRedirects: true,
      timeout: 30000
    }, (err, res, body) => {
      scrapeLinksOut(o, body)
      return resolve(o)
    })
  }) 
})

scrape()
  .then((res) => {
    res.linksOut.forEach((obj) => {
     console.log(obj);
   })
  })
  .catch((err) => console.log(err))

Answer 1

您可以使用find()获取img元素，然后使用attr()获取其src。

$(this).find('img').attr('src')

Answer 2

我写了一个快速的解决方案，希望能解决你的问题。我在我的机器上测试了它，它对我来说是有效的。

$('a').map(function () {

...
    let thumbnail = $(this).html();
    thumbnail = cheerio.parseHTML(thumbnail);

    if (thumbnail!== null && thumbnail.length > 0 && //Ensure link has an image
    thumbnail[0].attribs !== undefined && 
    thumbnail[0].attribs.src !== undefined) {
        thumbnail = thumbnail[0].attribs.src; //The image URL
    }

...

});

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body)

    $('a').map(function () {
      let link = $(this).attr('href')
     
      let thumbnail = $(this).html();
      thumbnail = cheerio.parseHTML(thumbnail)


      if (thumbnail !== null && thumbnail.length > 0 && thumbnail[0].attribs !== undefined && thumbnail[0].attribs.src !== undefined) {
        thumbnail = thumbnail[0].attribs.src
      } else {
        thumbnail = null
      }

      o.linksOut.push( {
        link: link,
        thumbnail: thumbnail
      })
    })
  } else {
    o.linksOut.push(links)
  }
}

这记录了：

null
null
null
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
[object Object]
null
null
null
null
null
null
null

Answer 3

这似乎是可行的。链接和图像匹配。

const scrapeLinksOut = (o, body) => {
  if (body) {
    let $ = cheerio.load(body);

    $('a').map(function () {
      let thumbnail;
      let link = $(this).attr('href');

      $(this)
        .find('img')
        .map(function (i, img) {
          thumbnail = $(img).attr('src');
        });

      if (thumbnail !== undefined) {
        o.linksOut.push({
          link: link,
          thumbnail: thumbnail,
        });
      }
    });
  } else {
    // something else
  }
};