探索自然语言处理(NLP)的旅程：从分词到文本生成

pip install nltk

import nltk
# 首次使用需下载punkt分词数据包
nltk.download('punkt')

# 英文分词示例
sentence = "This is an example sentence for tokenization."
# 核心分词函数：word_tokenize
tokens = nltk.word_tokenize(sentence)
print("英文分词结果：", tokens)
# 输出：['This', 'is', 'an', 'example', 'sentence', 'for', 'tokenization', '.']

pip install jieba

import jieba

# 中文分词示例
chinese_sentence = "自然语言处理是人工智能领域的重要分支"
# 精确模式分词（最常用）
chinese_tokens = jieba.lcut(chinese_sentence)
print("中文分词结果：", chinese_tokens)
# 输出：['自然语言处理', '是', '人工智能', '领域', '的', '重要', '分支']

# 英文停用词处理示例（需下载stopwords数据包）
nltk.download('stopwords')
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer

# 1. 定义停用词集合（英文）
stop_words = set(stopwords.words('english'))
tokens = nltk.word_tokenize("This is an example sentence for tokenization.")

# 2. 去除停用词
filtered_tokens = [token for token in tokens if token.lower() not in stop_words]
print("去除停用词后：", filtered_tokens)
# 输出：['example', 'sentence', 'tokenization', '.']

# 3. 词干提取（还原单词的词根，如running→run）
stemmer = PorterStemmer()
stemmed_tokens = [stemmer.stem(token) for token in filtered_tokens]
print("词干提取后：", stemmed_tokens)
# 输出：['exampl', 'sentenc', 'token', '.']

from sklearn.feature_extraction.text import CountVectorizer

# 示例文本集合
corpus = [
    "This is the first document",
    "This is the second second document",
    "And the third one",
    "Is this the first document"
]

# 初始化词袋模型
vectorizer = CountVectorizer(stop_words='english')  # 内置去除英文停用词
# 训练并转化为词频矩阵
bow_matrix = vectorizer.fit_transform(corpus)

print("词汇表：", vectorizer.get_feature_names_out())
print("词袋矩阵（形状：文档数×词汇数）：\n", bow_matrix.toarray())
"""
输出：
词汇表：['document' 'first' 'one' 'second' 'third']
词袋矩阵：
 [[1 1 0 0 0]
  [1 0 0 2 0]
  [0 0 1 0 1]
  [1 1 0 0 0]]
"""

from sklearn.feature_extraction.text import TfidfVectorizer

# 初始化TF-IDF模型
tfidf_vectorizer = TfidfVectorizer(stop_words='english')
# 训练并转化为TF-IDF矩阵
tfidf_matrix = tfidf_vectorizer.fit_transform(corpus)

print("TF-IDF矩阵：\n", tfidf_matrix.toarray())
"""
输出（数值为权重，越高表示词对该文档越重要）：
 [[0.46979139 0.8824969  0.         0.         0.        ]
  [0.27602602 0.         0.         0.96152395 0.        ]
  [0.         0.         0.62922752 0.         0.77722113]
  [0.46979139 0.8824969  0.         0.         0.        ]]
"""

pip install tensorflow numpy

import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense
from tensorflow.keras.optimizers import Adam

# ====================== 1. 数据准备 ======================
# 示例语料（可替换为任意文本，如小说、诗歌）
text = """
Natural language processing (NLP) is a subfield of linguistics, computer science, 
and artificial intelligence concerned with the interactions between computers and human language.
It focuses on how to program computers to process and analyze large amounts of natural language data.
"""
# 去重并排序，得到所有唯一字符
chars = sorted(list(set(text)))
char_to_idx = {c: i for i, c in enumerate(chars)}  # 字符→索引
idx_to_char = {i: c for i, c in enumerate(chars)}  # 索引→字符
vocab_size = len(chars)  # 词汇表大小（字符数）

# 构建训练数据：输入为seq_length个字符，输出为下一个字符
seq_length = 40  # 输入序列长度
step = 3  # 步长，减少数据冗余
sentences = []  # 输入序列
next_chars = []  # 目标字符

for i in range(0, len(text) - seq_length, step):
    sentences.append(text[i: i + seq_length])
    next_chars.append(text[i + seq_length])

# 将文本转化为数值矩阵（One-Hot编码）
X = np.zeros((len(sentences), seq_length, vocab_size), dtype=np.bool_)
y = np.zeros((len(sentences), vocab_size), dtype=np.bool_)

for i, sentence in enumerate(sentences):
    for t, char in enumerate(sentence):
        X[i, t, char_to_idx[char]] = 1
    y[i, char_to_idx[next_chars[i]]] = 1

# ====================== 2. 构建模型 ======================
model = Sequential()
# LSTM层：128个神经元，输入形状为(序列长度, 词汇表大小)
model.add(LSTM(128, input_shape=(seq_length, vocab_size)))
# 输出层：全连接层，softmax激活（多分类）
model.add(Dense(vocab_size, activation='softmax'))
# 编译模型：交叉熵损失（分类任务），Adam优化器
model.compile(loss='categorical_crossentropy', optimizer=Adam(learning_rate=0.01))

# ====================== 3. 训练模型 ======================
print("开始训练模型...")
model.fit(X, y, batch_size=64, epochs=50)  # 可根据需求调整epochs（训练轮数）

# ====================== 4. 文本生成函数 ======================
def generate_text(model, start_text, generate_length=200, temperature=0.7):
    """
    生成文本函数
    :param model: 训练好的模型
    :param start_text: 起始文本（种子）
    :param generate_length: 生成文本长度
    :param temperature: 温度系数（越高越随机，越低越保守）
    :return: 生成的完整文本
    """
    generated = start_text
    # 补齐起始文本长度（若不足seq_length）
    start_text = start_text[-seq_length:] if len(start_text) > seq_length else start_text
    
    for _ in range(generate_length):
        # 将起始文本转化为One-Hot矩阵
        x_pred = np.zeros((1, seq_length, vocab_size))
        for t, char in enumerate(start_text):
            x_pred[0, t, char_to_idx[char]] = 1.
        
        # 预测下一个字符的概率
        preds = model.predict(x_pred, verbose=0)[0]
        # 根据温度调整概率分布
        preds = np.log(preds) / temperature
        exp_preds = np.exp(preds)
        preds = exp_preds / np.sum(exp_preds)
        # 随机采样（按概率）
        next_idx = np.random.choice(len(preds), p=preds)
        next_char = idx_to_char[next_idx]
        
        # 更新生成文本和起始文本
        generated += next_char
        start_text = start_text[1:] + next_char
    
    return generated

# ====================== 5. 生成文本 ======================
# 设定起始文本（种子）
start_text = "Natural language processing is"
# 生成文本
generated_text = generate_text(model, start_text)
print("\n生成的文本：")
print(generated_text)