大整理: 仓库重组 + 新词表 + 新README

- 清理仓库: 删除所有废弃的generate_*脚本和旧版数据文件 - 目录重组: - data/ → 训练数据 - scripts/ → 数据生成脚本 - templates/ → Web UI - 重命名: - train_v2.py → train.py - vocab_v3.json → vocab.json (BPE, 6.8k tokens) - 更新 train.py 指向新词表路径 - 重写 README.md 完整文档 - 更新 .gitignore
2026-06-18 11:17:30 +00:00
1 parent 6e0ced4772
commit 231af72de9
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+高温多样化生成 — 把训练集全部问题用 Astron 高温度重答一次
+温度=1.2，追求多样化回答，丰富训练数据
+"""
+import sys, os, json, time
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+from api_correction import batch_generate
+
+MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
+TEMPERATURE = 1.0       # 高温度(上限1.0)，产出多样化回答
+BATCH_SIZE = 500        # 每批 500 条，分批保存防丢
+MAX_WORKERS = 5         # Astron 5并发
+
+def extract_questions(data):
+    """从训练数据中提取所有问题"""
+    questions = []
+    for item in data:
+        if isinstance(item, dict):
+            q = item.get('question', '') or item.get('input', '')
+            if q:
+                questions.append(q)
+    return questions
+
+def main():
+    # 1. 加载训练数据
+    train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json')
+    print(f"加载训练数据: {train_path}")
+    with open(train_path, encoding='utf-8') as f:
+        train_data = json.load(f)
+    
+    questions = extract_questions(train_data)
+    total = len(questions)
+    print(f"提取到 {total} 条问题，温度={TEMPERATURE}，{MAX_WORKERS}并发")
+    
+    # 2. 创建输出目录
+    out_dir = os.path.join(MODEL_DIR, 'high_temp_outputs')
+    os.makedirs(out_dir, exist_ok=True)
+    
+    # 3. 分批处理
+    all_results = []
+    existing = set()  # 去重用
+    
+    # 检查是否有已完成的中间结果
+    interim_files = sorted([f for f in os.listdir(out_dir) if f.startswith('batch_') and f.endswith('.json')])
+    if interim_files:
+        for fname in interim_files:
+            with open(os.path.join(out_dir, fname), encoding='utf-8') as f:
+                batch_data = json.load(f)
+            for r in batch_data:
+                key = (r['question'][:100], r['answer'][:100])
+                if key not in existing:
+                    existing.add(key)
+                    all_results.append(r)
+        print(f"发现已有中间结果: {len(all_results)} 条（来自 {len(interim_files)} 个文件）")
+        # 跳过已处理的 question
+        processed_questions = set(r['question'] for r in all_results)
+        remaining = [q for q in questions if q not in processed_questions]
+        print(f"已处理 {len(processed_questions)} 条，剩余 {len(remaining)} 条")
+        questions = remaining
+    
+    total_batches = (len(questions) + BATCH_SIZE - 1) // BATCH_SIZE
+    t_start = time.time()
+    
+    for batch_idx in range(total_batches):
+        batch_qs = questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
+        print(f"\n{'='*60}")
+        print(f"批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, 温度{TEMPERATURE})")
+        print(f"{'='*60}")
+        
+        t0 = time.time()
+        results = batch_generate(batch_qs, temperature=TEMPERATURE, max_workers=MAX_WORKERS)
+        elapsed = time.time() - t0
+        
+        # 标注入参温度
+        for r in results:
+            r['temperature'] = TEMPERATURE
+        
+        # 去重后加入总集
+        new_in_batch = 0
+        for r in results:
+            key = (r['question'][:100], r['answer'][:100])
+            if key not in existing:
+                existing.add(key)
+                all_results.append(r)
+                new_in_batch += 1
+        
+        # 立即保存本批结果
+        batch_file = os.path.join(out_dir, f'batch_{batch_idx + 1:04d}.json')
+        with open(batch_file, 'w', encoding='utf-8') as f:
+            json.dump(results, f, ensure_ascii=False, indent=2)
+        
+        # 追加保存总进度
+        progress_file = os.path.join(out_dir, 'all_progress.json')
+        with open(progress_file, 'w', encoding='utf-8') as f:
+            json.dump(all_results, f, ensure_ascii=False, indent=2)
+        
+        total_elapsed = time.time() - t_start
+        rate = len(all_results) / max(total_elapsed, 1) * 60
+        eta_remaining = (total - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
+        print(f"  本批新增 {new_in_batch} 条 (去重后) | 累计 {len(all_results)} 条 | "
+              f"{rate:.0f}条/分 | 已用 {total_elapsed/60:.1f}分 | 预计剩余 {eta_remaining:.1f}分")
+    
+    # 4. 去重统计
+    print(f"\n{'='*60}")
+    print(f"全部生成完成!")
+    print(f"生成结果: {len(all_results)} 条 (去重后)")
+    print(f"用时: {(time.time() - t_start)/60:.1f} 分钟")
+    
+    # 5. 保存最终去重结果（独立存放，不追加到训练集）
+    final_file = os.path.join('/data/user/work', 'high_temp_final.json')
+    with open(final_file, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    print(f"\n最终结果已保存: {final_file}")
+    print(f"如需使用，可手动合并到训练集")
+    
+    print(f"\n{'='*60}")
+    print(f"全部完成! 新增 {len(all_results)} 条多样化回答")
+    print(f"{'='*60}")
+
+if __name__ == '__main__':
+    main()