跨多个页面抓取(rvest) R

Question

我正在尝试使用rvest执行网络抓取，以便从网上商店抓取特定产品类别的数据。产品结果跨多个网页显示。当我使用我的代码时，我只得到前24个结果，这等于第一页上的产品。如何调整代码以遍历所有结果页？

提前感谢！

url_bol <- 'https://www.bol.com/nl/l/lichtbronnen/N/14483'
webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
head(webpage_bol)

product_title_data_html <- html_nodes(webpage_bol, '.product-title')
product_title_data <- html_text(product_title_data_html)
head(product_title_data)
product_title_data<-gsub("\n","",product_title_data)
product_title_data<-gsub(" ","",product_title_data)
head(product_title_data)
length(product_title_data)

product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
product_brand_data <-html_text(product_brand_data_html)
head(product_brand_data)
product_brand_data<-gsub("\n","",product_brand_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_brand_data)
length(product_brand_data)

product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
product_price_data <- html_text(product_price_data_html)
head(product_price_data)
product_price_data<-gsub("\n","",product_price_data)
product_price_data<-gsub(" ","",product_price_data)
head(product_price_data)
product_price_data
length(product_price_data)

bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)

View(bol.df)

Answer 1

url<-"https://www.bol.com/nl/l/lichtbronnen/N/14483/?page=3&view=list"
library(rvest)
page<-html_session(url)
total<-html_nodes(page,xpath='//*[@id="js_list_view"]/div[2]/p') %>% html_text()
total<-as.numeric(gsub("[^\\d]+", "", total, perl=TRUE))
all_df<-0
library(data.table)

for(i in 1:ceiling(total/24)){
  url_bol <- paste0('https://www.bol.com/nl/l/lichtbronnen/N/14483/?view=list&page=',i)
  webpage_bol <- read_html(url_bol,na.strings=" ",header=TRUE)
  head(webpage_bol)

  product_title_data_html <- html_nodes(webpage_bol, '.product-title')
  product_title_data <- html_text(product_title_data_html)
  head(product_title_data)
  product_title_data<-gsub("\n","",product_title_data)
  product_title_data<-gsub(" ","",product_title_data)
  head(product_title_data)
  length(product_title_data)

  product_brand_data_html <- html_nodes(webpage_bol, '.product-creator')
  product_brand_data <-html_text(product_brand_data_html)
  head(product_brand_data)
  product_brand_data<-gsub("\n","",product_brand_data)
  product_price_data<-gsub(" ","",product_price_data)
  head(product_brand_data)
  length(product_brand_data)

  product_price_data_html <- html_nodes(webpage_bol, '.promo-price')
  product_price_data <- html_text(product_price_data_html)
  head(product_price_data)
  product_price_data<-gsub("\n","",product_price_data)
  product_price_data<-gsub(" ","",product_price_data)
  head(product_price_data)
  product_price_data
  length(product_price_data)
  bol.df <- data.frame(Procuct_title = product_title_data, Brand = product_brand_data, Price = product_price_data)
  all_df[i]<-list(bol.df)
}

final<-rbindlist(all_df,fill = TRUE)




View(final)