1. 网页文本提取
import re
def clear_text(text: str) -> str:
# 去除HTML标签
text = re.sub('<.*?>', '', text)
# 去除多余空格
text = re.sub('\s+', '', text)
return text.strip()
# 函数调用
html_text = '''<p><span style="font-family: Arial, sans-serif; font-size: 20px; color: #0066cc;">
<div>这是一个 测试文本</div>
</span></p>'''
clear_text = clear_text(html_text)
print(clear_text)
2. 文本词频统计
import jieba
from collections import Counter
text = "本周进行自然语言处理系列文档示例,示例围绕自然语言处理程序。"
words = jieba.cut(text)
words_list = list(words)
words_count = Counter(words_list)
words_filter = [word for word, count in words_count.items() if count > 1]
print("分词结果:", words_list)
print("词频:", words_count)
print("高频词汇:", words_filter)