紧急修复：移除所有硬编码API Key，改为环境变量 ASTRON_KEY

generate_diverse_answers.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+多样化回答生成 — 对每个问题用 Astron 高温生成 50 个不同回答
+5并发，若重复则重生成，增量保存防丢
+"""
+import json, requests, time, os, sys, random, hashlib
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+# ====== Astron 配置 ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚，不要讲一半就停"""
+
+
+def call_astron(q, temperature):
+    """调用 Astron，返回回答或 None"""
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": q}
+                ],
+                "temperature": temperature,
+                "max_tokens": 2048,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 2:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def answer_fingerprint(text):
+    """回答指纹 — 去空格标点后取前 30 字"""
+    clean = ''.join(text.split())[:30]
+    return hashlib.md5(clean.encode()).hexdigest()
+
+
+def is_duplicate(new, existing):
+    """检查是否与已有回答重复（精确+模糊）"""
+    fp = answer_fingerprint(new)
+    if fp in existing:
+        return True
+    # 模糊：如果新回答是某个已有回答的子串或反之
+    short_new = ''.join(new.split())[:20]
+    for old_fp, old_text in existing.items():
+        short_old = ''.join(old_text.split())[:20]
+        if short_new in short_old or short_old in short_new:
+            return True if len(short_new) > 8 else False
+    return False
+
+
+def generate_answers_for_question(q, target=50, max_attempts=100, temperature=0.95):
+    """为单个问题生成 target 个不同回答"""
+    answers = {}  # fp -> full_text
+    attempt = 0
+    t_start = time.time()
+    
+    while len(answers) < target and attempt < max_attempts:
+        # 温度在 0.85~1.0 之间随机波动，避免卡在同一模式
+        temp = temperature + random.uniform(-0.1, 0.05)
+        temp = max(0.85, min(1.0, temp))
+        
+        result = call_astron(f"问题：{q}\n\n请以小狸的身份回答：", temp)
+        if result:
+            if not is_duplicate(result, answers):
+                fp = answer_fingerprint(result)
+                answers[fp] = result
+        
+        attempt += 1
+        
+        if attempt % 20 == 0:
+            elapsed = time.time() - t_start
+            rate = len(answers) / max(elapsed, 1)
+            if len(answers) < target and rate > 0:
+                eta = (target - len(answers)) / rate
+            else:
+                eta = 0
+            print(f"    [{len(answers)}/{target}] attempt={attempt}, {rate:.1f}条/秒, 预计剩余{eta:.0f}秒")
+            sys.stdout.flush()
+    
+    return list(answers.values())
+
+
+def main():
+    model_dir = '/workspace/xiaoli-llm-chat'
+    output_dir = os.path.join(model_dir, 'diverse_answers')
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # ====== 参数（可调） ======
+    QUESTIONS_PER_BATCH = 100   # 每批处理多少问题
+    ANSWERS_PER_QUESTION = 50   # 每个问题生成多少个不同回答
+    MAX_ATTEMPTS = 150          # 最多尝试多少次（超过就放弃）
+    MAX_WORKERS = 5             # Astron 5并发
+    TEMPERATURE = 0.95          # 高温（0.85~1.0 浮动）
+    
+    # ====== 加载问题列表 ======
+    train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
+    with open(train_path, encoding='utf-8') as f:
+        data = json.load(f)
+    
+    # 取所有不重复的问题
+    all_questions = list(set(item.get('question', '') for item in data))
+    random.shuffle(all_questions)
+    print(f"训练集总条数: {len(data)}")
+    print(f"不重复问题数: {len(all_questions)}")
+    
+    # 统计问题出现频次，优先处理高频问题
+    q_freq = {}
+    for item in data:
+        q = item.get('question', '')
+        q_freq[q] = q_freq.get(q, 0) + 1
+    
+    # 按频次排序（高频靠前，低频靠后，随机打乱同频）
+    freq_buckets = {}
+    for q in all_questions:
+        f = q_freq.get(q, 1)
+        freq_buckets.setdefault(f, []).append(q)
+    
+    sorted_questions = []
+    for freq in sorted(freq_buckets.keys(), reverse=True):
+        bucket = freq_buckets[freq]
+        random.shuffle(bucket)
+        sorted_questions.extend(bucket)
+    
+    print(f"按频次排序完成 (高频优先处理)")
+    
+    # ====== 恢复已有进度 ======
+    progress_file = os.path.join(output_dir, 'diversity_progress.json')
+    completed = {}  # question -> list of answers
+    if os.path.exists(progress_file):
+        with open(progress_file, encoding='utf-8') as f:
+            completed = json.load(f)
+        # 过滤掉已处理的问题
+        processed_qs = set(completed.keys())
+        sorted_questions = [q for q in sorted_questions if q not in processed_qs]
+        print(f"已有进度: {len(completed)} 个问题已完成")
+        print(f"剩余未处理: {len(sorted_questions)} 个问题")
+    
+    # 限制要处理的问题数
+    max_q = len(sorted_questions)  # 全部问题
+    
+    total_questions = len(sorted_questions)
+    total_batches = (total_questions + QUESTIONS_PER_BATCH - 1) // QUESTIONS_PER_BATCH
+    
+    print(f"\n目标: {total_questions} 个问题 × {ANSWERS_PER_QUESTION} 个回答 = {total_questions * ANSWERS_PER_QUESTION} 条")
+    print(f"批次: {total_batches} 批 (每批 ≤{QUESTIONS_PER_BATCH} 问题)")
+    print(f"并发: {MAX_WORKERS}")
+    if total_questions > 1000:
+        minutes = total_questions * ANSWERS_PER_QUESTION / (MAX_WORKERS * 8) / 60
+        print(f"预估耗时: ~{minutes:.0f} 分钟 (~{minutes/60:.1f} 小时)")
+    print(f"{'='*70}\n")
+    
+    t_start = time.time()
+    total_new_pairs = 0
+    
+    for batch_idx in range(total_batches):
+        batch_qs = sorted_questions[batch_idx * QUESTIONS_PER_BATCH : (batch_idx + 1) * QUESTIONS_PER_BATCH]
+        batch_results = {}
+        
+        print(f"\n批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)} 个问题)")
+        
+        # 并发生成
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = {
+                executor.submit(generate_answers_for_question, q, ANSWERS_PER_QUESTION, MAX_ATTEMPTS, TEMPERATURE): q
+                for q in batch_qs
+            }
+            
+            for future in as_completed(futures):
+                q = futures[future]
+                try:
+                    answers = future.result()
+                    if answers:
+                        batch_results[q] = answers
+                        print(f"  ✅ 『{q[:30]}』 → {len(answers)} 个不同回答")
+                    else:
+                        print(f"  ❌ 『{q[:30]}』 → 生成失败")
+                except Exception as e:
+                    print(f"  ❌ 『{q[:30]}』 → 错误: {e}")
+        
+        # 追加到总进度
+        completed.update(batch_results)
+        batch_new_pairs = sum(len(v) for v in batch_results.values())
+        total_new_pairs += batch_new_pairs
+        
+        # 保存进度
+        with open(progress_file, 'w', encoding='utf-8') as f:
+            json.dump(completed, f, ensure_ascii=False, indent=2)
+        
+        elapsed = time.time() - t_start
+        rate = total_new_pairs / max(elapsed, 1) * 60
+        
+        # 统计
+        print(f"\n批次 {batch_idx + 1} 完成 | "
+              f"本批新增 {batch_new_pairs} 条 | "
+              f"累计 {total_new_pairs:,} 条 ({len(completed)} 问题) | "
+              f"{rate:.0f}条/分 | "
+              f"已用 {elapsed/60:.1f}分")
+    
+    # ====== 完成 ======
+    elapsed_total = time.time() - t_start
+    print(f"\n{'='*70}")
+    print(f"全部生成完成!")
+    print(f"处理问题: {len(completed)} 个")
+    print(f"总回答数: {total_new_pairs:,} 条")
+    print(f"用时: {elapsed_total/60:.1f} 分钟")
+    
+    # ====== 保存为训练格式 ======
+    train_format = []
+    for q, answers in completed.items():
+        for a in answers:
+            train_format.append({'question': q, 'answer': a})
+    
+    output_file = os.path.join(output_dir, 'diverse_train_data_v1.json')
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(train_format, f, ensure_ascii=False, indent=2)
+    
+    # 也保存一份到工作目录
+    work_file = '/data/user/work/diverse_train_data_v1.json'
+    with open(work_file, 'w', encoding='utf-8') as f:
+        json.dump(train_format, f, ensure_ascii=False, indent=2)
+    
+    print(f"\n结果保存:")
+    print(f"  📄 {output_file}")
+    print(f"  📄 {work_file}")
+    print(f"  📊 平均每问题 {total_new_pairs/max(len(completed),1):.1f} 个回答")
+    print(f"\n如需合并到训练集：")
+    print(f"  python3 -c \"import json; d1=json.load(open('xiaoli_train_v2.json')); d2=json.load(open('{output_file}')); d1.extend(d2); json.dump(d1, open('xiaoli_train_v2.json','w'), ensure_ascii=False, indent=2)\"")
+    print(f"{'='*70}")
+
+if __name__ == '__main__':
+    main()