大模型应用：文本转语音实践：Tacotron2预训练模型和SpeechT5模型应用.19

import torch
import torchaudio
import soundfile as sf

def simple_tts_demo():
    print("第一步：检查TTS大模型的可行性")
    
    # 尝试加载一个预训练的小模型（作为起点）
    try:
        # 这里我们先使用torchaudio自带的Tacotron2预训练模型
        # 注意：这是英文模型，但可以帮助你理解流程
        tacotron2 = torchaudio.pipelines.TACOTRON2_WAVERNN_PHONE_LJSPEECH.get_tacotron2()
        print("✓ 成功加载Tacotron2模型")
        
        # 准备文本
        text = "Hello, this is your first experience with a TTS model."
        
        print(f"第二步：处理文本: '{text}'")
        
        # 使用模型生成（这里简化流程，实际需要更多步骤）
        print("第三步：生成语音...")
        print("（注意：首次运行需要下载模型，可能需要几分钟）")
        
        # 生成语音
        with torch.no_grad():
            # 使用标准流程生成语音
            processor = torchaudio.pipelines.TACOTRON2_WAVERNN_PHONE_LJSPEECH.get_text_processor()
            processed, lengths = processor(text)
            processed = processed.to(next(tacotron2.parameters()).device)
            lengths = lengths.to(next(tacotron2.parameters()).device)
            spec, spec_lengths, _ = tacotron2.infer(processed, lengths)
            
            # 使用WaveRNN生成波形
            wavernn = torchaudio.pipelines.TACOTRON2_WAVERNN_PHONE_LJSPEECH.get_vocoder()
            wavernn = wavernn.to(next(tacotron2.parameters()).device)
            waveform, _ = wavernn(spec)
            
            # 保存生成的语音
            output_path = "generated_speech.wav"
            torchaudio.save(output_path, waveform, sample_rate=22050)
            print(f"✓ 语音生成完成，已保存到: {output_path}")
        
    except Exception as e:
        print(f"当前环境不支持直接运行，请检查依赖项或网络连接。错误信息: {e}")
    
    

if __name__ == "__main__":
    simple_tts_demo()

import torch
import torchaudio
import numpy as np
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from datasets import load_dataset
import soundfile as sf
import matplotlib.pyplot as plt
from IPython.display import Audio, display
import os
from pathlib import Path
import warnings
warnings.filterwarnings('ignore')

print("PyTorch版本:", torch.__version__)
print("CUDA可用:", torch.cuda.is_available())
if torch.cuda.is_available():
    print("GPU:", torch.cuda.get_device_name(0))

def __init__(self, device=None):
    self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
    print(f"使用设备: {self.device}")
    
    # 加载模型和处理器
    print("正在加载SpeechT5模型...")
    self.processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
    self.model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts").to(self.device)
    self.vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(self.device)
    
    print("✓ SpeechT5模型加载完成")

def get_speaker_embedding(self, speaker_id=0):
    print(f"创建说话人 {speaker_id} 的默认嵌入向量...")
    
    # 根据 speaker_id 返回不同的嵌入向量
    if speaker_id == 7306:  # 默认女性声音
        embedding = torch.randn(1, 512) * 0.1
        embedding[0, :100] += 0.5
    elif speaker_id == 3000:  # 男性声音示例
        embedding = torch.randn(1, 512) * 0.1
        embedding[0, 100:200] += 0.5
    elif speaker_id == 5000:  # 另一个女性声音
        embedding = torch.randn(1, 512) * 0.1
        embedding[0, :100] += 0.3
    elif speaker_id == 1000:  # 另一个男性声音
        embedding = torch.randn(1, 512) * 0.1
        embedding[0, 100:200] += 0.3
    else:  # 默认女性声音
        embedding = torch.randn(1, 512) * 0.1
        embedding[0, :100] += 0.5
    
    print(f"✓ 创建了说话人 {speaker_id} 的嵌入向量")
    return embedding

def text_to_speech(self, text, speaker_id=7306, speed=1.0):
    # 预处理输入文本
    inputs = self.processor(text=text, return_tensors="pt")
    
    # 获取说话人嵌入
    speaker_embedding = self.get_speaker_embedding(speaker_id)
    
    # 生成语音
    with torch.no_grad():
        speech = self.model.generate_speech(
            inputs["input_ids"].to(self.device),
            speaker_embedding.to(self.device),
            vocoder=self.vocoder
        )
    
    # 转换为numpy数组并调整采样率
    speech_np = speech.cpu().numpy()
    
    # 简单的语速调整（通过重采样）
    if speed != 1.0:
        import librosa
        speech_np = librosa.effects.time_stretch(speech_np, rate=speed)
    
    return speech_np

def save_audio(self, audio_array, filename="output.wav", sample_rate=16000):
    sf.write(filename, audio_array, sample_rate)
    print(f"✓ 音频已保存: {filename}")
    return filename

def play_audio(self, audio_array, sample_rate=16000):
    display(Audio(audio_array, rate=sample_rate))

def visualize_waveform(self, audio_array, title="语音波形图"):
    plt.figure(figsize=(12, 4))
    plt.plot(audio_array)
    plt.title(title)
    plt.xlabel("采样点")
    plt.ylabel("振幅")
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()

def basic_tts_demo():
    print("=" * 60)
    print("SpeechT5基础TTS演示")
    print("=" * 60)
    
    # 创建TTS实例
    tts = SpeechT5BasicTTS()
    
    # 测试文本
    test_texts = [
        "Hello, this is a demonstration of Microsoft SpeechT5 text-to-speech technology.",
        "你好，这是微软SpeechT5文本转语音技术的演示。",
        "Bonjour, ceci est une démonstration de la technologie de synthèse vocale Microsoft SpeechT5.",
        "こんにちは、これはマイクロソフトSpeechT5テキスト読み上げ技術のデモンストレーションです。"
    ]
    
    # 可用的说话人示例
    speaker_examples = {
        "女性声音": 7306,  # 默认女性声音
        "男性声音": 3000,   # 男性声音示例
        "另一个女性": 5000,  # 另一个女性声音
        "另一个男性": 1000,   # 另一个男性声音
    }
    
    for i, (speaker_name, speaker_id) in enumerate(speaker_examples.items(), 1):
        print(f"\n{i}. 使用说话人: {speaker_name} (ID: {speaker_id})")
        
        # 生成语音
        speech = tts.text_to_speech(test_texts[0], speaker_id=speaker_id)
        
        # 保存音频
        filename = f"speech_{speaker_name}_{i}.wav"
        tts.save_audio(speech, filename)
        
        # 可视化
        tts.visualize_waveform(speech[:16000], f"{speaker_name} - 波形")
    
    # 多语言演示
    print("\n" + "=" * 60)
    print("多语言TTS演示")
    print("=" * 60)
    
    for i, text in enumerate(test_texts, 1):
        print(f"\n语言 {i}: {text[:50]}...")
        speech = tts.text_to_speech(text, speaker_id=7306)
        filename = f"multilingual_{i}.wav"
        tts.save_audio(speech, filename)
    
    print("\n演示完成！")

if __name__ == "__main__":
    basic_tts_demo()

import torch
import torchaudio
import numpy as np
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
from datasets import load_dataset
import soundfile as sf
import matplotlib.pyplot as plt
from IPython.display import Audio, display
import os
from pathlib import Path
import warnings
warnings.filterwarnings('ignore')

print("PyTorch版本:", torch.__version__)
print("CUDA可用:", torch.cuda.is_available())
if torch.cuda.is_available():
    print("GPU:", torch.cuda.get_device_name(0))

# 解决中文显示
plt.rcParams["font.sans-serif"] = ["SimHei"]
plt.rcParams["axes.unicode_minus"] = False
    
class SpeechT5BasicTTS:
    """
    SpeechT5基础文本转语音示例
    """
    
    def __init__(self, device=None):
        """
        初始化SpeechT5模型
        """
        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
        print(f"使用设备: {self.device}")
        
        # 加载模型和处理器
        print("正在加载SpeechT5模型...")
        self.processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
        self.model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts").to(self.device)
        self.vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan").to(self.device)
        
        # 加载说话人嵌入数据集
        print("正在加载说话人嵌入...")
        # try:
        #     self.embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation", trust_remote_code=True)
        # except Exception as e:
        #     print(f"无法从Hugging Face加载数据集: {e}")
        #     print("请手动下载数据集并解压到本地目录，然后更新以下路径:")
        #     local_path = "d:/AIWorld/case/test/data/spkrec-xvect"  # 替换为你的本地路径
        #     if os.path.exists(local_path):
        #         self.embeddings_dataset = {"xvector": np.load(os.path.join(local_path, "xvector.npy"))}  # 假设数据格式为npy文件
        #     else:
        #         raise FileNotFoundError(f"本地路径 {local_path} 不存在，请检查路径或手动下载数据集。")
        
        print("✓ SpeechT5模型加载完成")
    
    def get_speaker_embedding(self, speaker_id=0):
        """
        获取说话人嵌入向量
        speaker_id: 说话人ID，0-7306之间的整数
        """
        # 确保speaker_id在有效范围内
        # speaker_id = max(0, min(speaker_id, len(self.embeddings_dataset) - 1))
        
        # 获取说话人嵌入
        # speaker_embedding = torch.tensor(self.embeddings_dataset[speaker_id]["xvector"]).unsqueeze(0)
        # return speaker_embedding
        """
        创建默认的说话人嵌入向量
        替代下载大型数据集
        """
        print(f"创建说话人 {speaker_id} 的默认嵌入向量...")
        
        # 根据 speaker_id 返回不同的嵌入向量
        if speaker_id == 7306:  # 默认女性声音
            embedding = torch.randn(1, 512) * 0.1
            embedding[0, :100] += 0.5
        elif speaker_id == 3000:  # 男性声音示例
            embedding = torch.randn(1, 512) * 0.1
            embedding[0, 100:200] += 0.5
        elif speaker_id == 5000:  # 另一个女性声音
            embedding = torch.randn(1, 512) * 0.1
            embedding[0, :100] += 0.3
        elif speaker_id == 1000:  # 另一个男性声音
            embedding = torch.randn(1, 512) * 0.1
            embedding[0, 100:200] += 0.3
        else:  # 默认女性声音
            embedding = torch.randn(1, 512) * 0.1
            embedding[0, :100] += 0.5
        
        print(f"✓ 创建了说话人 {speaker_id} 的嵌入向量")
        return embedding
    
    def text_to_speech(self, text, speaker_id=7306, speed=1.0):
        """
        文本转语音核心函数
        text: 输入文本
        speaker_id: 说话人ID (默认7306是女性声音)
        speed: 语速控制 (0.5-2.0)
        """
        # 预处理输入文本
        inputs = self.processor(text=text, return_tensors="pt")
        
        # 获取说话人嵌入
        speaker_embedding = self.get_speaker_embedding(speaker_id)
        
        # 生成语音
        with torch.no_grad():
            speech = self.model.generate_speech(
                inputs["input_ids"].to(self.device),
                speaker_embedding.to(self.device),
                vocoder=self.vocoder
            )
        
        # 转换为numpy数组并调整采样率
        speech_np = speech.cpu().numpy()
        
        # 简单的语速调整（通过重采样）
        if speed != 1.0:
            import librosa
            speech_np = librosa.effects.time_stretch(speech_np, rate=speed)
        
        return speech_np
    
    def save_audio(self, audio_array, filename="output.wav", sample_rate=16000):
        """
        保存音频文件
        """
        sf.write(filename, audio_array, sample_rate)
        print(f"✓ 音频已保存: {filename}")
        return filename
    
    def play_audio(self, audio_array, sample_rate=16000):
        """
        在Jupyter中播放音频
        """
        display(Audio(audio_array, rate=sample_rate))
    
    def visualize_waveform(self, audio_array, title="语音波形图"):
        """
        可视化音频波形
        """
        plt.figure(figsize=(12, 4))
        plt.plot(audio_array)
        plt.title(title)
        plt.xlabel("采样点")
        plt.ylabel("振幅")
        plt.grid(True, alpha=0.3)
        plt.tight_layout()
        plt.show()
    
    def get_speaker_info(self, speaker_id):
        """
        获取说话人信息
        """
        if 0 <= speaker_id < len(self.embeddings_dataset):
            speaker_data = self.embeddings_dataset[speaker_id]
            return {
                "speaker_id": speaker_id,
                "has_xvector": "xvector" in speaker_data,
                "xvector_shape": speaker_data["xvector"].shape if "xvector" in speaker_data else None
            }
        return None

# 使用示例
def basic_tts_demo():
    """
    基础TTS演示
    """
    print("=" * 60)
    print("SpeechT5基础TTS演示")
    print("=" * 60)
    
    # 创建TTS实例
    tts = SpeechT5BasicTTS()
    
    # 测试文本
    test_texts = [
        "Hello, this is a demonstration of Microsoft SpeechT5 text-to-speech technology.",
        "你好，这是微软SpeechT5文本转语音技术的演示。",
        "Bonjour, ceci est une démonstration de la technologie de synthèse vocale Microsoft SpeechT5.",
        "こんにちは、これはマイクロソフトSpeechT5テキスト読み上げ技術のデモンストレーションです。"
    ]
    
    # 可用的说话人示例
    speaker_examples = {
        "女性声音": 7306,  # 默认女性声音
        "男性声音": 3000,   # 男性声音示例
        "另一个女性": 5000,  # 另一个女性声音
        "另一个男性": 1000,   # 另一个男性声音
    }
    
    for i, (speaker_name, speaker_id) in enumerate(speaker_examples.items(), 1):
        print(f"\n{i}. 使用说话人: {speaker_name} (ID: {speaker_id})")
        
        # 生成语音
        speech = tts.text_to_speech(test_texts[0], speaker_id=speaker_id)
        
        # 保存音频
        filename = f"speech_{speaker_name}_{i}.wav"
        tts.save_audio(speech, filename)
        
        # 可视化
        tts.visualize_waveform(speech[:16000], f"{speaker_name} - 波形")
    
    # 多语言演示
    print("\n" + "=" * 60)
    print("多语言TTS演示")
    print("=" * 60)
    
    for i, text in enumerate(test_texts, 1):
        print(f"\n语言 {i}: {text[:50]}...")
        speech = tts.text_to_speech(text, speaker_id=7306)
        filename = f"multilingual_{i}.wav"
        tts.save_audio(speech, filename)
    
    print("\n演示完成！")

if __name__ == "__main__":
    basic_tts_demo()