Java AI 开发全攻略：Spring AI、LangChain4J、Dify、百炼与微调一站式掌握

@Configuration
@EnableAi
publicclass AiConfig {
    
    @Bean
    public ChatClient openAiChatClient(
        @Value("${spring.ai.openai.api-key}") String apiKey) {
        returnnew OpenAiChatClient(apiKey);
    }
    
    @Bean
    public EmbeddingClient embeddingClient(
        @Value("${spring.ai.openai.api-key}") String apiKey) {
        returnnew OpenAiEmbeddingClient(apiKey);
    }
}

spring:
  ai:
    openai:
      api-key: ${OPENAI_API_KEY}
      chat:
        options:
          model: gpt-4
          temperature: 0.7

@Service
publicclass CustomerServiceAiAssistant {
    
    privatefinal ChatClient chatClient;
    
    public CustomerServiceAiAssistant(ChatClient chatClient) {
        this.chatClient = chatClient;
    }
    
    public String generateResponse(String customerQuery, String context) {
        PromptTemplate promptTemplate = new PromptTemplate("""
            你是一名专业的客户服务代表。请基于以下上下文信息回答用户问题。
            
            上下文：{context}
            
            用户问题：{question}
            
            请用专业、友好的语气回复，如果信息不足请明确说明。
            """);
            
        Prompt prompt = promptTemplate.create(
            Map.of("context", context, "question", customerQuery));
            
        return chatClient.call(prompt).getResult().getOutput().getContent();
    }
}

@Component
publicclass DocumentSearchService {
    
    privatefinal VectorStore vectorStore;
    privatefinal EmbeddingClient embeddingClient;
    
    public DocumentSearchService(VectorStore vectorStore, 
                               EmbeddingClient embeddingClient) {
        this.vectorStore = vectorStore;
        this.embeddingClient = embeddingClient;
    }
    
    public void addDocument(String documentId, String content) {
        // 生成文档向量
        List<Double> embedding = embeddingClient.embed(content);
        
        // 构建文档向量对象
        Document document = new Document(documentId, content, embedding);
        
        // 存储到向量数据库
        vectorStore.add(List.of(document));
    }
    
    public List<Document> similaritySearch(String query, int topK) {
        // 生成查询向量
        List<Double> queryEmbedding = embeddingClient.embed(query);
        
        // 执行相似度搜索
        return vectorStore.similaritySearch(
            SearchRequest.query(query)
                .withTopK(topK)
                .withSimilarityThreshold(0.7)
        );
    }
}

@RestController
public class StreamingChatController {
    
    @PostMapping("/chat/stream")
    public Flux<String> streamChat(@RequestBody ChatRequest request) {
        return chatClient.stream(request.getPrompt())
            .map(chatResponse -> chatResponse.getResult().getOutput().getContent());
    }
}

public class CustomerSupportChain {
    
    privatefinal ChatLanguageModel chatModel;
    privatefinal EmbeddingModel embeddingModel;
    privatefinal EmbeddingStore<TextSegment> embeddingStore;
    
    public CustomerSupportChain(ChatLanguageModel chatModel,
                              EmbeddingModel embeddingModel,
                              EmbeddingStore<TextSegment> embeddingStore) {
        this.chatModel = chatModel;
        this.embeddingModel = embeddingModel;
        this.embeddingStore = embeddingStore;
    }
    
    public String processCustomerQuery(String query, String customerId) {
        // 构建复杂的处理链
        return AiServices.builder(CustomerSupportService.class)
            .chatLanguageModel(chatModel)
            .tools(new CustomerDatabaseTool(), new ProductCatalogTool())
            .build()
            .handleCustomerQuery(query, customerId);
    }
    
    // 定义 AI 服务接口
    interface CustomerSupportService {
        @SystemMessage("你是一名专业的客户支持专家，拥有访问客户数据和产品目录的权限。")
        @UserMessage("处理客户查询：{{it}}")
        String handleCustomerQuery(@V("it") String query, 
                                 @V("customerId") String customerId);
    }
}

public class AdvancedToolsIntegration {
    
    // 定义工具接口
    @Tool("根据客户ID获取客户信息")
    public CustomerInfo getCustomerInfo(@P("客户ID") String customerId) {
        // 模拟数据库查询
        return customerRepository.findById(customerId)
            .orElseThrow(() -> new CustomerNotFoundException(customerId));
    }
    
    @Tool("根据产品ID查询产品详情和库存")
    public ProductInfo getProductInfo(@P("产品ID") String productId) {
        return productService.getProductDetails(productId);
    }
    
    @Tool("计算两个地点之间的距离")
    public DistanceInfo calculateDistance(@P("起点") String origin, 
                                        @P("终点") String destination) {
        return distanceCalculator.calculate(origin, destination);
    }
    
    // 构建支持工具调用的AI服务
    public void buildToolEnabledService() {
        CustomerServiceAI service = AiServices.builder(CustomerServiceAI.class)
            .chatLanguageModel(OpenAiChatModel.withApiKey(apiKey))
            .tools(getCustomerInfo, getProductInfo, calculateDistance)
            .build();
            
        String response = service.answerCustomerQuestion(
            "客户12345想了解产品P-1001的库存情况，并计算从北京到上海的配送时间");
        System.out.println(response);
    }
    
    interface CustomerServiceAI {
        String answerCustomerQuestion(String question);
    }
}

public class MemoryManagementExample {
    
    privatefinal ChatMemoryProvider memoryProvider;
    privatefinal ChatLanguageModel chatModel;
    
    public MemoryManagementExample() {
        this.memoryProvider = memoryId -> MessageWindowChatMemory.withMaxMessages(10);
        this.chatModel = OpenAiChatModel.withApiKey(apiKey);
    }
    
    public String continueConversation(String message, String conversationId) {
        CustomerService service = AiServices.builder(CustomerService.class)
            .chatLanguageModel(chatModel)
            .chatMemoryProvider(memoryProvider)
            .build();
            
        return service.chat(message, conversationId);
    }
    
    // 使用 TokenWindow 控制记忆长度
    public void setupTokenBasedMemory() {
        ChatMemoryProvider tokenMemoryProvider = memoryId -> 
            TokenWindowChatMemory.builder()
                .maxTokens(1000)
                .build();
                
        // 构建支持 token 限制的记忆系统
        AiServiceWithMemory service = AiServices.builder(AiServiceWithMemory.class)
            .chatLanguageModel(chatModel)
            .chatMemoryProvider(tokenMemoryProvider)
            .build();
    }
    
    interface CustomerService {
        String chat(@UserMessage String message, @MemoryId String conversationId);
    }
    
    interface AiServiceWithMemory {
        String process(@UserMessage String input, @MemoryId String sessionId);
    }
}

public class AdvancedRAGImplementation {
    
    privatefinal EmbeddingModel embeddingModel;
    privatefinal EmbeddingStore<TextSegment> embeddingStore;
    privatefinal ChatLanguageModel chatModel;
    
    public AdvancedRAGImplementation(EmbeddingModel embeddingModel,
                                   EmbeddingStore<TextSegment> embeddingStore,
                                   ChatLanguageModel chatModel) {
        this.embeddingModel = embeddingModel;
        this.embeddingStore = embeddingStore;
        this.chatModel = chatModel;
    }
    
    // 文档处理流水线
    public void processDocumentPipeline(Path documentPath) {
        // 文档加载
        Document document = DocumentLoader.load(documentPath);
        
        // 文档分割
        List<TextSegment> segments = DocumentSplitter.split(document, 500);
        
        // 生成嵌入向量
        List<Embedding> embeddings = embeddingModel.embedAll(segments).content();
        
        // 存储到向量数据库
        for (int i = 0; i < segments.size(); i++) {
            embeddingStore.add(embeddings.get(i), segments.get(i));
        }
    }
    
    // 高级检索策略
    public String advancedRetrieval(String query) {
        // 生成查询向量
        Embedding queryEmbedding = embeddingModel.embed(query).content();
        
        // 多策略检索
        List<EmbeddingMatch<TextSegment>> relevantMatches = embeddingStore.findRelevant(
            queryEmbedding, 
            5, 
            0.7
        );
        
        // 重排序
        List<TextSegment> rerankedSegments = rerankSegments(query, relevantMatches);
        
        // 构建上下文
        String context = buildContext(rerankedSegments);
        
        // 生成回答
        return chatModel.generate(
            "基于以下上下文回答问题：\n\n" + context + "\n\n问题：" + query
        );
    }
    
    private List<TextSegment> rerankSegments(String query, 
                                           List<EmbeddingMatch<TextSegment>> matches) {
        // 实现基于相关性的重排序逻辑
        return matches.stream()
            .sorted((a, b) -> Double.compare(b.score(), a.score()))
            .map(EmbeddingMatch::embedded)
            .collect(ToList.toList());
    }
}

public class OutputParsingExamples {
    
    // 自定义输出解析器
    publicstaticclass ProductRecommendationParser implements OutputParser<ProductRecommendation> {
        @Override
        public ProductRecommendation parse(String text) {
            // 解析模型返回的文本为结构化对象
            try {
                ObjectMapper mapper = new ObjectMapper();
                return mapper.readValue(text, ProductRecommendation.class);
            } catch (Exception e) {
                thrownew OutputParseException("Failed to parse product recommendation", e);
            }
        }
        
        @Override
        public String formatInstructions() {
            return"请以 JSON 格式返回产品推荐，包含 productId、productName、reason 字段";
        }
    }
    
    // 使用输出解析器的服务
    public ProductRecommendation getStructuredRecommendation(String userPreference) {
        StructuredChatService service = AiServices.builder(StructuredChatService.class)
            .chatLanguageModel(chatModel)
            .outputParser(new ProductRecommendationParser())
            .build();
            
        return service.recommendProducts(userPreference);
    }
    
    interface StructuredChatService {
        @UserMessage("根据用户偏好推荐产品：{{it}}")
        ProductRecommendation recommendProducts(@V("it") String userPreference);
    }
    
    // 支持多个输出解析器
    public void multipleParsersExample() {
        ComplexService service = AiServices.builder(ComplexService.class)
            .chatLanguageModel(chatModel)
            .outputParsers(
                Map.of(
                    "json", new JsonOutputParser(),
                    "xml", new XmlOutputParser(),
                    "custom", new ProductRecommendationParser()
                )
            )
            .build();
    }
}

public class StreamingImplementation {
    
    privatefinal StreamingChatLanguageModel streamingModel;
    
    public StreamingImplementation(StreamingChatLanguageModel streamingModel) {
        this.streamingModel = streamingModel;
    }
    
    // 基础流式响应
    public Flux<String> streamChatResponse(String message) {
        return Flux.create(sink -> {
            streamingModel.generate(message, new StreamingResponseHandler() {
                @Override
                public void onNext(String token) {
                    sink.next(token);
                }
                
                @Override
                public void onComplete() {
                    sink.complete();
                }
                
                @Override
                public void onError(Throwable error) {
                    sink.error(error);
                }
            });
        });
    }
    
    // 带工具调用的流式处理
    public Flux<StreamingResponse> streamWithTools(String query) {
        StreamingAiService<StreamingService> service = 
            AiServices.create(StreamingService.class, streamingModel);
            
        return service.streamingChat(query)
            .map(response -> new StreamingResponse(response.content(), response.finishReason()));
    }
    
    interface StreamingService {
        @UserMessage("{{it}}")
        Flux<String> streamingChat(@V("it") String message);
    }
}

name: "客户服务专业回复"
version:"1.2"
description:"针对不同客户类型和问题的动态提示词"

variables:
-name:"customer_type"
    type:"enum"
    values:["新客户","老客户","VIP客户"]
    default:"新客户"
    
-name:"question_category"
    type:"enum"
    values:["产品咨询","技术支持","价格问题","投诉建议"]
    required:true

-name:"customer_tone"
    type:"string"
    default:"友好专业"

template:|
  你是一名{customer_type}服务专家。

{%ifcustomer_type=="VIP客户"%}
请提供优先服务和专属解决方案。
{%endif%}

用户的问题属于：{question_category}

{%ifquestion_category=="投诉建议"%}
请首先表达歉意，然后认真解决问题。
{%elsifquestion_category=="价格问题"%}
请详细解释价格构成和价值体现。
{%endif%}

使用{customer_tone}的语气回复，确保回复专业且易于理解。

上下文信息：{{context}}
用户问题：{{question}}

prompt_experiment:
  name:"销售话术优化实验"
base_version:"v1.2"
variants:
    -id:"variant_a"
      version:"v1.3-aggressive"
      changes:
        -"增加紧迫感表达"
        -"强化价值主张"
      metrics:
        -"转化率"
        -"响应时间"
        
    -id:"variant_b"
      version:"v1.3-conservative"
      changes:
        -"增强信任建立"
        -"减少促销语言"
      metrics:
        -"客户满意度"
        -"长期留存率"

traffic_allocation:
    variant_a:40%
    variant_b:40%
    base_version:20%
    
evaluation_period:"7d"
primary_metric:"转化率"

api_gateway:
  rate_limiting:
    default:"1000/小时"
    premium:"10000/小时"
    
authentication:
    methods:["API_KEY","JWT","OAUTH2"]
    default_ttl:"24h"
    
endpoints:
    -path:"/v1/chat/completions"
      workflow:"customer_service_v2"
      timeout:"30s"
      
    -path:"/v1/images/generate"
      workflow:"content_creation_v1"
      timeout:"60s"
      
monitoring:
    metrics:["latency","error_rate","throughput"]
    alerts:
      -condition:"error_rate > 5%"
        action:"slack_alert"

# 自定义客户数据查询工具
@tool
class CustomerDataTool:
    name = "customer_data_lookup"
    description = "根据客户ID查询客户基本信息和历史记录"
    
    parameters:
        customer_id:
            type: "string"
            required: true
            description: "客户唯一标识符"
            
        data_fields:
            type: "array"
            items: "string"
            enum: ["basic_info", "order_history", "preferences"]
            default: ["basic_info"]
    
    def execute(self, customer_id, data_fields):
        # 连接企业 CRM 系统
        customer_data = crm_client.get_customer_data(
            customer_id, 
            fields=data_fields
        )
        
        # 数据清洗和格式化
        formatted_data = self.format_customer_data(customer_data)
        
        return {
            "success": True,
            "data": formatted_data,
            "source": "crm_system_v2"
        }
    
    def format_customer_data(self, raw_data):
        # 实现特定的数据格式化逻辑
        return {
            "customer_id": raw_data["id"],
            "name": f"{raw_data['first_name']} {raw_data['last_name']}",
            "segment": raw_data.get("segment", "standard"),
            "lifetime_value": raw_data.get("ltv", 0),
            "last_interaction": raw_data.get("last_contact")
        }

public class DistributedTrainingConfig {
    
    // 数据并行配置
    @Bean
    public DataParallelConfig dataParallelConfig() {
        return DataParallelConfig.builder()
            .gradient_accumulation_steps(4)
            .pipeline_parallel_size(1)
            .tensor_parallel_size(2)
            .data_parallel_size(4)
            .build();
    }
    
    // 混合并行训练
    @Bean
    public HybridParallelTraining hybridParallelTraining() {
        return HybridParallelTraining.builder()
            .model_sharding_strategy("auto")
            .optimizer_state_sharding(true)
            .activation_checkpointing(true)
            .zero_optimization(3)  // ZeRO-3 优化
            .build();
    }
    
    // 梯度累积配置
    @Bean
    public GradientAccumulationConfig gradientConfig() {
        return GradientAccumulationConfig.builder()
            .steps(8)
            .sync_gradients(true)
            .scale_loss(true)
            .build();
    }
}

public class TrainingDataPipeline {
    
    privatefinal DatasetProcessor datasetProcessor;
    privatefinal FeatureEngineer featureEngineer;
    
    public TrainingDataPipeline() {
        this.datasetProcessor = new ParallelDatasetProcessor();
        this.featureEngineer = new CachedFeatureEngineer();
    }
    
    // 并行数据预处理
    public Dataset createOptimizedDataset(String dataPath, int batchSize) {
        return Dataset.builder()
            .dataPath(dataPath)
            .batchSize(batchSize)
            .prefetchBufferSize(1024)  // 预取缓冲
            .numParallelWorkers(8)     // 并行工作线程
            .shuffleBufferSize(10000)  // 洗牌缓冲大小
            .cacheInMemory(true)       // 内存缓存
            .build();
    }
    
    // 动态批处理优化
    public DynamicBatchingConfig dynamicBatchingConfig() {
        return DynamicBatchingConfig.builder()
            .maxBatchSize(64)
            .timeoutMs(100)
            .paddingStrategy("longest")
            .bucketBoundaries(List.of(32, 64, 128, 256))
            .build();
    }
    
    // 数据增强流水线
    public DataAugmentationPipeline augmentationPipeline() {
        return DataAugmentationPipeline.builder()
            .textAugmentations(List.of(
                new SynonymReplacement(0.1),
                new RandomInsertion(0.05),
                new RandomSwap(0.05),
                new BackTranslation("en", "zh")
            ))
            .imageAugmentations(List.of(
                new RandomRotation(5),
                new ColorJitter(0.1, 0.1, 0.1, 0.1),
                new RandomHorizontalFlip(0.5)
            ))
            .parallelAugmentation(true)
            .cacheAugmentedData(true)
            .build();
    }
}

public class ModelCompressionService {
    
    // 动态量化配置
    public DynamicQuantizationConfig dynamicQuantization() {
        return DynamicQuantizationConfig.builder()
            .quantization_bits(8)
            .per_channel_quantization(true)
            .symmetric_quantization(true)
            .quantized_modules(List.of("Linear", "Conv2d"))
            .excluded_modules(List.of("LayerNorm", "Embedding"))
            .calibration_dataset("validation_set")
            .calibration_samples(1000)
            .build();
    }
    
    // 知识蒸馏配置
    public KnowledgeDistillationConfig distillationConfig() {
        return KnowledgeDistillationConfig.builder()
            .teacher_model("qwen-72b")
            .student_model("qwen-7b")
            .temperature(4.0)
            .alpha(0.7)
            .distillation_loss("kl_divergence")
            .attention_transfer(true)
            .hidden_state_transfer(true)
            .build();
    }
    
    // 模型剪枝策略
    public ModelPruningConfig pruningConfig() {
        return ModelPruningConfig.builder()
            .pruning_method("magnitude")
            .pruning_ratio(0.3)
            .pruning_frequency(1000)
            .global_pruning(true)
            .structured_pruning(false)
            .importance_criteria("l1_norm")
            .build();
    }
}

public class InferenceEngineOptimization {
    
    // TensorRT 优化配置
    public TensorRTConfig tensorRTConfig() {
        return TensorRTConfig.builder()
            .precision_mode("FP16")
            .max_workspace_size(2L * 1024 * 1024 * 1024)  // 2GB
            .min_timing_iterations(10)
            .avg_timing_iterations(100)
            .optimization_level(3)
            .calibration_cache("calibration.cache")
            .build();
    }
    
    // ONNX Runtime 优化
    public ONNXRuntimeConfig onnxRuntimeConfig() {
        return ONNXRuntimeConfig.builder()
            .execution_providers(List.of(
                "CUDAExecutionProvider",
                "CPUExecutionProvider"
            ))
            .graph_optimization_level(99)
            .enable_profiling(true)
            .memory_pattern_optimization(true)
            .enable_cpu_mem_arena(true)
            .build();
    }
    
    // 自研推理引擎配置
    public BailianInferenceConfig bailianConfig() {
        return BailianInferenceConfig.builder()
            .kernel_fusion(true)
            .memory_optimization(true)
            .operator_optimization(true)
            .dynamic_batching(true)
            .continuous_batching(true)  // 连续批处理
            .paged_attention(true)      // 分页注意力
            .build();
    }
}

public class MemoryOptimization {
    
    // 激活检查点配置
    public ActivationCheckpointingConfig activationCheckpointing() {
        return ActivationCheckpointingConfig.builder()
            .checkpoint_every_n_layers(2)
            .offload_activations(true)
            .cpu_offload_buffer_size(4L * 1024 * 1024 * 1024)  // 4GB
            .partition_activations(true)
            .contiguous_memory_optimization(true)
            .build();
    }
    
    // 梯度检查点配置
    public GradientCheckpointingConfig gradientCheckpointing() {
        return GradientCheckpointingConfig.builder()
            .checkpoint_ratio(0.5)
            .offload_to_cpu(true)
            .recompute_strategy("balanced")
            .build();
    }
    
    // 显存碎片整理
    public MemoryDefragmentationConfig defragmentationConfig() {
        return MemoryDefragmentationConfig.builder()
            .defragmentation_interval(1000)
            .max_defragmentation_time_ms(500)
            .defragmentation_threshold(0.1)  // 10% 碎片率触发
            .build();
    }
}

public class DynamicMemoryAllocator {
    
    privatefinal MemoryPool memoryPool;
    privatefinal AllocationStrategy strategy;
    
    public DynamicMemoryAllocator() {
        this.memoryPool = new HierarchicalMemoryPool();
        this.strategy = new BestFitAllocationStrategy();
    }
    
    // 分层内存池配置
    public MemoryPoolConfig memoryPoolConfig() {
        return MemoryPoolConfig.builder()
            .max_gpu_memory(0.9)        // 最大使用 90% 显存
            .reserved_memory(512 * 1024 * 1024)  // 保留 512MB
            .allocation_strategy("best_fit")
            .defragmentation_enabled(true)
            .build();
    }
    
    // 张量重用时序优化
    public TensorReuseOptimizer tensorReuseOptimizer() {
        return TensorReuseOptimizer.builder()
            .reuse_distance_threshold(100)
            .lifetime_analysis(true)
            .preallocation_strategy("statistical")
            .build();
    }
}

public class ComputationalGraphOptimization {
    
    // 图融合策略
    public GraphFusionConfig graphFusionConfig() {
        return GraphFusionConfig.builder()
            .enable_vertical_fusion(true)   // 垂直融合
            .enable_horizontal_fusion(true) // 水平融合
            .fusion_patterns(List.of(
                "Conv-BatchNorm-ReLU",
                "Linear-GELU",
                "LayerNorm-Silu"
            ))
            .max_fusion_group_size(10)
            .build();
    }
    
    // 算子优化配置
    public OperatorOptimizationConfig operatorOptimization() {
        return OperatorOptimizationConfig.builder()
            .kernel_auto_tuning(true)
            .winograd_optimization(true)
            .depthwise_separable_optimization(true)
            .attention_optimization("flash_attention")
            .build();
    }
    
    // 常量折叠与代数简化
    public ConstantFoldingConfig constantFolding() {
        return ConstantFoldingConfig.builder()
            .fold_constants(true)
            .algebraic_simplification(true)
            .arithmetic_optimization(true)
            .build();
    }
}

public class CommunicationOptimization {
    
    // 梯度通信优化
    public GradientCommunicationConfig gradientCommunication() {
        return GradientCommunicationConfig.builder()
            .all_reduce_algorithm("nccl")
            .gradient_compression("fp16")
            .bucket_cap_mb(25)           // 25MB 桶大小
            .overlap_communication(true) // 通信计算重叠
            .build();
    }
    
    // 模型并行通信优化
    public ModelParallelConfig modelParallelConfig() {
        return ModelParallelConfig.builder()
            .pipeline_parallel_communication("1f1b")
            .tensor_parallel_communication("all_reduce")
            .sequence_parallel_enabled(true)
            .build();
    }
    
    // 混合精度通信
    public MixedPrecisionCommunication mixedPrecisionComm() {
        return MixedPrecisionCommunication.builder()
            .master_weights_fp32(true)
            .loss_scaling(true)
            .gradient_scaling_algorithm("dynamic")
            .communication_dtype("fp16")
            .build();
    }
}

public class FineTuningDataEngineering {
    
    privatefinal DataValidator dataValidator;
    privatefinal DataAugmentor dataAugmentor;
    privatefinal QualityScorer qualityScorer;
    
    public FineTuningDataEngineering() {
        this.dataValidator = new MultiStageDataValidator();
        this.dataAugmentor = new SmartDataAugmentor();
        this.qualityScorer = new LLMBasedScorer();
    }
    
    // 数据收集与清洗流水线
    public Dataset buildHighQualityDataset(DataSources sources) {
        return DataPipeline.builder()
            .sources(sources)
            .addStage(new DataCollectionStage()
                .fromDatabase("business_db")
                .fromLogs("user_interactions")
                .fromDocuments("knowledge_base")
            )
            .addStage(new DataCleaningStage()
                .removeDuplicates(true)
                .fixEncodingIssues(true)
                .normalizeText(true)
                .filterByLength(10, 2000)
            )
            .addStage(new DataValidationStage()
                .validator(new FormatValidator())
                .validator(new ContentValidator())
                .validator(new SafetyValidator())
            )
            .addStage(new DataAugmentationStage()
                .paraphraseAugmentation(0.3)    // 30% 数据 paraphrasing
                .backTranslationAugmentation(0.2) // 20% 回译增强
                .entityReplacementAugmentation(0.1) // 10% 实体替换
            )
            .addStage(new QualityScoringStage()
                .scorer(qualityScorer)
                .minQualityScore(0.8)  // 最低质量分数
                .autoRejectLowQuality(true)
            )
            .build()
            .execute();
    }
    
    // 对话数据格式标准化
    public List<Conversation> formatConversationData(List<RawDialog> rawDialogs) {
        return rawDialogs.stream()
            .map(this::convertToStandardFormat)
            .filter(conv -> validateConversationStructure(conv))
            .map(conv -> enrichWithMetadata(conv))
            .collect(Collectors.toList());
    }
    
    private Conversation convertToStandardFormat(RawDialog rawDialog) {
        return Conversation.builder()
            .messages(rawDialog.getTurns().stream()
                .map(turn -> Message.builder()
                    .role(turn.getSpeakerType())
                    .content(turn.getText())
                    .timestamp(turn.getTimestamp())
                    .build())
                .collect(Collectors.toList()))
            .metadata(Map.of(
                "domain", extractDomain(rawDialog),
                "complexity", assessComplexity(rawDialog),
                "success", evaluateSuccess(rawDialog)
            ))
            .build();
    }
}

public class DataQualityAssessment {
    
    // 多维度质量指标
    public QualityReport assessDatasetQuality(Dataset dataset) {
        return QualityReport.builder()
            .linguisticQuality(assessLinguisticQuality(dataset))
            .domainRelevance(assessDomainRelevance(dataset))
            .diversityMetrics(calculateDiversity(dataset))
            .difficultyDistribution(analyzeDifficulty(dataset))
            .biasDetection(detectBiases(dataset))
            .build();
    }
    
    private LinguisticQuality assessLinguisticQuality(Dataset dataset) {
        return LinguisticQuality.builder()
            .grammarScore(calculateGrammarScore(dataset))
            .fluencyScore(calculateFluencyScore(dataset))
            .coherenceScore(calculateCoherenceScore(dataset))
            .styleConsistency(assessStyleConsistency(dataset))
            .build();
    }
    
    private DiversityMetrics calculateDiversity(Dataset dataset) {
        return DiversityMetrics.builder()
            .vocabularyRichness(calculateVocabRichness(dataset))
            .syntacticDiversity(calculateSyntacticDiversity(dataset))
            .semanticDiversity(calculateSemanticDiversity(dataset))
            .topicCoverage(analyzeTopicCoverage(dataset))
            .build();
    }
    
    // 基于 LLM 的自动质量评分
    public double llmBasedQualityScore(String text, String context) {
        String prompt = """
            请从以下维度评估文本质量，给出0-1的分数：
            
            文本：{text}
            领域上下文：{context}
            
            评估标准：
            1. 语言流畅性（0.25分）
            2. 内容准确性（0.25分）  
            3. 领域相关性（0.25分）
            4. 信息完整性（0.25分）
            
            请直接返回分数，不要解释。
            """;
            
        String scoreStr = llmClient.call(prompt
            .replace("{text}", text)
            .replace("{context}", context));
            
        return Double.parseDouble(scoreStr.trim());
    }
}

public class PEFTStrategySelector {
    
    // LoRA 配置 - 适合大多数场景
    public LoRAConfig loraConfigForDomainAdaptation() {
        return LoRAConfig.builder()
            .lora_rank(16)
            .lora_alpha(32)
            .target_modules(List.of(
                "q_proj", "k_proj", "v_proj", "o_proj",
                "gate_proj", "up_proj", "down_proj"
            ))
            .lora_dropout(0.1)
            .bias("none")
            .build();
    }
    
    // QLoRA 配置 - 资源受限环境
    public QLoRAConfig qloraConfigForLowResource() {
        return QLoRAConfig.builder()
            .lora_rank(64)
            .lora_alpha(16)
            .quantization_config(NF4Config.builder()
                .double_quant(true)
                .quant_type("nf4")
                .build())
            .target_modules(List.of("all-linear"))
            .build();
    }
    
    // Adapter-based 方法 - 多任务学习
    public AdapterConfig adapterConfigForMultiTask() {
        return AdapterConfig.builder()
            .adapter_dim(512)
            .adapter_activation("gelu")
            .adapter_dropout(0.1)
            .add_layer_norm(true)
            .build();
    }
    
    // 根据场景选择最优策略
    public PEFTConfig selectOptimalStrategy(FinetuningScenario scenario) {
        switch (scenario.getType()) {
            case DOMAIN_ADAPTATION:
                return loraConfigForDomainAdaptation();
            case STYLE_TRANSFER:
                return StyleTransferConfig.builder()
                    .lora_rank(8)
                    .target_modules(List.of("attn", "mlp"))
                    .build();
            case MULTI_TASK:
                return adapterConfigForMultiTask();
            case LOW_RESOURCE:
                return qloraConfigForLowResource();
            default:
                return loraConfigForDomainAdaptation();
        }
    }
}

public class ProgressiveFineTuning {
    
    privatefinal ModelLoader modelLoader;
    privatefinal TrainingScheduler scheduler;
    
    public ProgressiveFineTuning() {
        this.modelLoader = new CheckpointManager();
        this.scheduler = new AdaptiveScheduler();
    }
    
    // 三阶段渐进微调流程
    public TrainingPlan createProgressivePlan(TrainingData data) {
        return TrainingPlan.builder()
            .stage1(createWarmupStage(data.getGeneralData()))
            .stage2(createDomainAdaptationStage(data.getDomainData()))
            .stage3(createSpecializationStage(data.getTaskData()))
            .validationStrategy(new IncrementalValidation())
            .build();
    }
    
    private TrainingStage createWarmupStage(Dataset generalData) {
        return TrainingStage.builder()
            .name("通用能力保持")
            .dataset(generalData)
            .learningRate(1e-5)
            .epochs(1)
            .batchSize(32)
            .peftConfig(loraConfigForDomainAdaptation())
            .objective("保持基础语言理解能力")
            .earlyStopping(patience=3)
            .build();
    }
    
    private TrainingStage createDomainAdaptationStage(Dataset domainData) {
        return TrainingStage.builder()
            .name("领域知识注入")
            .dataset(domainData)
            .learningRate(5e-5)
            .epochs(3)
            .batchSize(16)
            .peftConfig(loraConfigForDomainAdaptation())
            .objective("学习领域术语和知识")
            .metrics(List.of("domain_accuracy", "terminology_usage"))
            .build();
    }
    
    private TrainingStage createSpecializationStage(Dataset taskData) {
        return TrainingStage.builder()
            .name("任务专项优化")
            .dataset(taskData)
            .learningRate(2e-5)
            .epochs(5)
            .batchSize(8)
            .peftConfig(qloraConfigForLowResource())
            .objective("优化特定任务表现")
            .metrics(List.of("task_accuracy", "response_quality"))
            .build();
    }
}

public class ComprehensiveEvaluation {
    
    privatefinal List<Evaluator> evaluators;
    privatefinal BenchmarkSuite benchmarkSuite;
    
    public ComprehensiveEvaluation() {
        this.evaluators = Arrays.asList(
            new AccuracyEvaluator(),
            new FluencyEvaluator(), 
            new SafetyEvaluator(),
            new BiasDetector(),
            new EfficiencyEvaluator()
        );
        this.benchmarkSuite = new DomainBenchmarkSuite();
    }
    
    // 自动化评估流水线
    public EvaluationReport evaluateModel(FineTunedModel model, TestData testData) {
        EvaluationReport.Builder reportBuilder = EvaluationReport.builder();
        
        // 基础能力评估
        reportBuilder.baseCapabilities(evaluateBaseCapabilities(model));
        
        // 领域专项评估
        reportBuilder.domainPerformance(evaluateDomainPerformance(model, testData));
        
        // 安全性评估
        reportBuilder.safetyMetrics(evaluateSafety(model));
        
        // 效率评估
        reportBuilder.efficiencyMetrics(evaluateEfficiency(model));
        
        return reportBuilder.build();
    }
    
    private BaseCapabilities evaluateBaseCapabilities(FineTunedModel model) {
        return BaseCapabilities.builder()
            .languageUnderstanding(testMMLU(model))
            .reasoningAbility(testReasoning(model))
            .knowledgeRetention(testKnowledge(model))
            .build();
    }
    
    private DomainPerformance evaluateDomainPerformance(FineTunedModel model, TestData testData) {
        return DomainPerformance.builder()
            .accuracy(calculateAccuracy(model, testData))
            .precision(calculatePrecision(model, testData))
            .recall(calculateRecall(model, testData))
            .f1Score(calculateF1(model, testData))
            .domainSpecificMetrics(calculateDomainMetrics(model, testData))
            .build();
    }
    
    // A/B 测试框架
    public ABTestResult runABTest(FineTunedModel modelA, FineTunedModel modelB, 
                                 UserTraffic traffic) {
        return ABTestFramework.builder()
            .models(Map.of("A", modelA, "B", modelB))
            .trafficAllocation(Map.of("A", 0.5, "B", 0.5))
            .metrics(List.of(
                "conversion_rate",
                "user_satisfaction", 
                "task_success_rate",
                "response_quality"
            ))
            .duration(Duration.ofDays(7))
            .minSampleSize(1000)
            .build()
            .run();
    }
}

public class ContinuousLearningPipeline {
    
    privatefinal FeedbackCollector feedbackCollector;
    privatefinal DataCurator dataCurator;
    privatefinal RetrainingScheduler retrainingScheduler;
    
    public ContinuousLearningPipeline() {
        this.feedbackCollector = new RealTimeFeedbackCollector();
        this.dataCurator = new SmartDataCurator();
        this.retrainingScheduler = new AdaptiveRetrainingScheduler();
    }
    
    // 持续学习闭环
    public void setupContinuousLearning(FineTunedModel model) {
        ContinuousLearningConfig config = ContinuousLearningConfig.builder()
            .feedbackCollection(feedbackCollector)
            .dataCuration(dataCurator)
            .retrainingTrigger(new PerformanceDropTrigger(0.05)) // 性能下降5%触发
            .retrainingStrategy(new IncrementalRetraining())
            .versionManagement(new SemanticVersioning())
            .rollbackStrategy(new AutomaticRollback())
            .build();
            
        ContinuousLearningEngine engine = new ContinuousLearningEngine(config);
        engine.start(model);
    }
    
    // 基于用户反馈的数据收集
    public Dataset collectFeedbackData() {
        return feedbackCollector.collectRecentFeedback(Duration.ofDays(30))
            .stream()
            .filter(feedback -> feedback.getQualityScore() < 0.7) // 收集低质量反馈
            .map(this::convertToTrainingExample)
            .collect(Collectors.toCollection(Dataset::new));
    }
    
    // 智能数据筛选策略
    public Dataset curateTrainingData(Dataset rawData) {
        return dataCurator.curate(rawData, 
            CuratorStrategy.builder()
                .diversitySampling(true)
                .difficultyBalancing(true)
                .qualityFiltering(true)
                .outlierDetection(true)
                .build());
    }
}