Python刮板IEEE

Question

我试图检索特定IEEE文档的关键字。我偶然发现了这个代码here

        ieee_content = requests.get(link, timeout=180)
        soup = BeautifulSoup(ieee_content.text, 'lxml')
        tag = soup.find_all('script')
        #metadata = "".join(re.findall('global.document.metadata=(.*)', tag[9].text)).replace(";", '').replace('global.document.metadata=', '')
        for i in tag[9]:
            metadata_format = re.compile(r'global.document.metadata=.*', re.MULTILINE)
            metadata = re.findall(metadata_format, i)
            if len(metadata) != 0:
               # convert the list 
               convert_to_json = json.dumps(metadata)
               x = json.loads(convert_to_json)
               s = x[0].replace("'", '"').replace(";", '')

问题是我的元数据变量总是空的。我尝试遍历所有标记，而不是使用tag9，但是元数据在所有情况下仍然是空的。我也尝试使用'xml‘而不是'lmxl’，但是结果是一样的。我很感谢你帮我个忙。

Answer 1

import json
import re
from pprint import pprint

import requests
from bs4 import BeautifulSoup

ieee_content = requests.get("https://ieeexplore.ieee.org/document/7845555", timeout=180)
soup = BeautifulSoup(ieee_content.content, "html.parser")
scripts = soup.find_all("script")

pattern = re.compile(r"(?<=\"keywords\":)\[{.*?}\]")
keywords_dict = {}
for i, script in enumerate(scripts):
    keywords = re.findall(pattern, str(script.string))
    if len(keywords) == 1:
        raw_keywords_list = json.loads(keywords[0])
        for keyword_type in raw_keywords_list:
            keywords_dict[keyword_type["type"].strip()] = [kwd.strip() for kwd in keyword_type["kwd"]]

pprint(keywords_dict)