Nodejs无法使用request-promise请求url

Question

我无法使用以下代码从OpenTable网站获取https。然而，当我尝试使用相同的代码从其他网站获取数据时，例如Instagram，它可以工作。因为我刚开始学习使用node js的web scraping，你能告诉我可能是什么问题吗？

const request = require('request-promise');



(async () => {

    const BASE_URL = 'https://www.opentable.co.uk/s?dateTime=2021-05-30T19%3A00%3A00&covers=2&latitude=51.525225&longitude=-0.079615'


    let response = await request(BASE_URL);

    let $ = cheerio.load(response);
    
    console.log(response);

  
  })();

Answer 1

您的代码很可能无法工作，因为JavaScript用于在站点上构建DOM。或者该站点正在使用防刮擦保护。不管怎样，我建议你使用像Puppeteer这样的

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

async function scrapeOpenTable() {
  const BASE_URL =
    "https://www.opentable.co.uk/s?dateTime=2021-05-30T19%3A00%3A00&covers=2&latitude=51.525225&longitude=-0.079615";

  const browser = await puppeteer.launch({
    headless: false,
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });
  const page = await browser.newPage();
  await page.goto(BASE_URL);
  // More commands here...
  /* You may view the docs at:
       https://pptr.dev/
     And more magic at:
       https://www.npmjs.com/package/puppeteer
     Github:
       https://github.com/puppeteer/puppeteer
  */

  // await browser.close();
}

scrapeOpenTable();