紧急修复：移除所有硬编码API Key，改为环境变量 ASTRON_KEY

api_correction.py

@@ -7,7 +7,7 @@ import json, requests, time, os, threading
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
-ASTRON_KEY = "658beba1beb0c911ed00672019bbf7e5:MjNmYWRkMWUyZjAxYTM3ZGE2ODI1YzFh"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
 ASTRON_MODEL = "astron-code-latest"
 
 SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。

generate_diverse_answers.py

@@ -0,0 +1,256 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+多样化回答生成 — 对每个问题用 Astron 高温生成 50 个不同回答
+5并发，若重复则重生成，增量保存防丢
+"""
+import json, requests, time, os, sys, random, hashlib
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+# ====== Astron 配置 ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚，不要讲一半就停"""
+
+
+def call_astron(q, temperature):
+    """调用 Astron，返回回答或 None"""
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": q}
+                ],
+                "temperature": temperature,
+                "max_tokens": 2048,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 2:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def answer_fingerprint(text):
+    """回答指纹 — 去空格标点后取前 30 字"""
+    clean = ''.join(text.split())[:30]
+    return hashlib.md5(clean.encode()).hexdigest()
+
+
+def is_duplicate(new, existing):
+    """检查是否与已有回答重复（精确+模糊）"""
+    fp = answer_fingerprint(new)
+    if fp in existing:
+        return True
+    # 模糊：如果新回答是某个已有回答的子串或反之
+    short_new = ''.join(new.split())[:20]
+    for old_fp, old_text in existing.items():
+        short_old = ''.join(old_text.split())[:20]
+        if short_new in short_old or short_old in short_new:
+            return True if len(short_new) > 8 else False
+    return False
+
+
+def generate_answers_for_question(q, target=50, max_attempts=100, temperature=0.95):
+    """为单个问题生成 target 个不同回答"""
+    answers = {}  # fp -> full_text
+    attempt = 0
+    t_start = time.time()
+    
+    while len(answers) < target and attempt < max_attempts:
+        # 温度在 0.85~1.0 之间随机波动，避免卡在同一模式
+        temp = temperature + random.uniform(-0.1, 0.05)
+        temp = max(0.85, min(1.0, temp))
+        
+        result = call_astron(f"问题：{q}\n\n请以小狸的身份回答：", temp)
+        if result:
+            if not is_duplicate(result, answers):
+                fp = answer_fingerprint(result)
+                answers[fp] = result
+        
+        attempt += 1
+        
+        if attempt % 20 == 0:
+            elapsed = time.time() - t_start
+            rate = len(answers) / max(elapsed, 1)
+            if len(answers) < target and rate > 0:
+                eta = (target - len(answers)) / rate
+            else:
+                eta = 0
+            print(f"    [{len(answers)}/{target}] attempt={attempt}, {rate:.1f}条/秒, 预计剩余{eta:.0f}秒")
+            sys.stdout.flush()
+    
+    return list(answers.values())
+
+
+def main():
+    model_dir = '/workspace/xiaoli-llm-chat'
+    output_dir = os.path.join(model_dir, 'diverse_answers')
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # ====== 参数（可调） ======
+    QUESTIONS_PER_BATCH = 100   # 每批处理多少问题
+    ANSWERS_PER_QUESTION = 50   # 每个问题生成多少个不同回答
+    MAX_ATTEMPTS = 150          # 最多尝试多少次（超过就放弃）
+    MAX_WORKERS = 5             # Astron 5并发
+    TEMPERATURE = 0.95          # 高温（0.85~1.0 浮动）
+    
+    # ====== 加载问题列表 ======
+    train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
+    with open(train_path, encoding='utf-8') as f:
+        data = json.load(f)
+    
+    # 取所有不重复的问题
+    all_questions = list(set(item.get('question', '') for item in data))
+    random.shuffle(all_questions)
+    print(f"训练集总条数: {len(data)}")
+    print(f"不重复问题数: {len(all_questions)}")
+    
+    # 统计问题出现频次，优先处理高频问题
+    q_freq = {}
+    for item in data:
+        q = item.get('question', '')
+        q_freq[q] = q_freq.get(q, 0) + 1
+    
+    # 按频次排序（高频靠前，低频靠后，随机打乱同频）
+    freq_buckets = {}
+    for q in all_questions:
+        f = q_freq.get(q, 1)
+        freq_buckets.setdefault(f, []).append(q)
+    
+    sorted_questions = []
+    for freq in sorted(freq_buckets.keys(), reverse=True):
+        bucket = freq_buckets[freq]
+        random.shuffle(bucket)
+        sorted_questions.extend(bucket)
+    
+    print(f"按频次排序完成 (高频优先处理)")
+    
+    # ====== 恢复已有进度 ======
+    progress_file = os.path.join(output_dir, 'diversity_progress.json')
+    completed = {}  # question -> list of answers
+    if os.path.exists(progress_file):
+        with open(progress_file, encoding='utf-8') as f:
+            completed = json.load(f)
+        # 过滤掉已处理的问题
+        processed_qs = set(completed.keys())
+        sorted_questions = [q for q in sorted_questions if q not in processed_qs]
+        print(f"已有进度: {len(completed)} 个问题已完成")
+        print(f"剩余未处理: {len(sorted_questions)} 个问题")
+    
+    # 限制要处理的问题数
+    max_q = len(sorted_questions)  # 全部问题
+    
+    total_questions = len(sorted_questions)
+    total_batches = (total_questions + QUESTIONS_PER_BATCH - 1) // QUESTIONS_PER_BATCH
+    
+    print(f"\n目标: {total_questions} 个问题 × {ANSWERS_PER_QUESTION} 个回答 = {total_questions * ANSWERS_PER_QUESTION} 条")
+    print(f"批次: {total_batches} 批 (每批 ≤{QUESTIONS_PER_BATCH} 问题)")
+    print(f"并发: {MAX_WORKERS}")
+    if total_questions > 1000:
+        minutes = total_questions * ANSWERS_PER_QUESTION / (MAX_WORKERS * 8) / 60
+        print(f"预估耗时: ~{minutes:.0f} 分钟 (~{minutes/60:.1f} 小时)")
+    print(f"{'='*70}\n")
+    
+    t_start = time.time()
+    total_new_pairs = 0
+    
+    for batch_idx in range(total_batches):
+        batch_qs = sorted_questions[batch_idx * QUESTIONS_PER_BATCH : (batch_idx + 1) * QUESTIONS_PER_BATCH]
+        batch_results = {}
+        
+        print(f"\n批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)} 个问题)")
+        
+        # 并发生成
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = {
+                executor.submit(generate_answers_for_question, q, ANSWERS_PER_QUESTION, MAX_ATTEMPTS, TEMPERATURE): q
+                for q in batch_qs
+            }
+            
+            for future in as_completed(futures):
+                q = futures[future]
+                try:
+                    answers = future.result()
+                    if answers:
+                        batch_results[q] = answers
+                        print(f"  ✅ 『{q[:30]}』 → {len(answers)} 个不同回答")
+                    else:
+                        print(f"  ❌ 『{q[:30]}』 → 生成失败")
+                except Exception as e:
+                    print(f"  ❌ 『{q[:30]}』 → 错误: {e}")
+        
+        # 追加到总进度
+        completed.update(batch_results)
+        batch_new_pairs = sum(len(v) for v in batch_results.values())
+        total_new_pairs += batch_new_pairs
+        
+        # 保存进度
+        with open(progress_file, 'w', encoding='utf-8') as f:
+            json.dump(completed, f, ensure_ascii=False, indent=2)
+        
+        elapsed = time.time() - t_start
+        rate = total_new_pairs / max(elapsed, 1) * 60
+        
+        # 统计
+        print(f"\n批次 {batch_idx + 1} 完成 | "
+              f"本批新增 {batch_new_pairs} 条 | "
+              f"累计 {total_new_pairs:,} 条 ({len(completed)} 问题) | "
+              f"{rate:.0f}条/分 | "
+              f"已用 {elapsed/60:.1f}分")
+    
+    # ====== 完成 ======
+    elapsed_total = time.time() - t_start
+    print(f"\n{'='*70}")
+    print(f"全部生成完成!")
+    print(f"处理问题: {len(completed)} 个")
+    print(f"总回答数: {total_new_pairs:,} 条")
+    print(f"用时: {elapsed_total/60:.1f} 分钟")
+    
+    # ====== 保存为训练格式 ======
+    train_format = []
+    for q, answers in completed.items():
+        for a in answers:
+            train_format.append({'question': q, 'answer': a})
+    
+    output_file = os.path.join(output_dir, 'diverse_train_data_v1.json')
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(train_format, f, ensure_ascii=False, indent=2)
+    
+    # 也保存一份到工作目录
+    work_file = '/data/user/work/diverse_train_data_v1.json'
+    with open(work_file, 'w', encoding='utf-8') as f:
+        json.dump(train_format, f, ensure_ascii=False, indent=2)
+    
+    print(f"\n结果保存:")
+    print(f"  📄 {output_file}")
+    print(f"  📄 {work_file}")
+    print(f"  📊 平均每问题 {total_new_pairs/max(len(completed),1):.1f} 个回答")
+    print(f"\n如需合并到训练集：")
+    print(f"  python3 -c \"import json; d1=json.load(open('xiaoli_train_v2.json')); d2=json.load(open('{output_file}')); d1.extend(d2); json.dump(d1, open('xiaoli_train_v2.json','w'), ensure_ascii=False, indent=2)\"")
+    print(f"{'='*70}")
+
+if __name__ == '__main__':
+    main()

generate_diverse_single.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+高温度多样化生成 — 单线程，与常规跑批共用 Astron 5并发中的1个
+对已完成批1-6的问题，用高温度重新生成，获得多样回答
+"""
+import sys, os, json, random, time, requests
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚，不要讲一半就停"""
+
+
+def call_astron(q, temperature):
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": q}
+                ],
+                "temperature": temperature,
+                "max_tokens": 8192,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def main():
+    # 从已完成批载入问题
+    all_questions = []
+    for i in range(1, 7):
+        fp = os.path.join(MODEL_DIR, f'mega_interim_{i}.json')
+        if os.path.exists(fp):
+            with open(fp) as f:
+                items = json.load(f)
+            for item in items:
+                all_questions.append(item['question'])
+            print(f"批{i}: {len(items)}条已载入")
+    
+    # 去重
+    all_questions = list(dict.fromkeys(all_questions))
+    print(f"共 {len(all_questions)} 个去重问题")
+    
+    # 温度配置
+    temp_configs = [0.9, 1.3, 1.5]
+    all_new = []
+    
+    for round_idx, temp in enumerate(temp_configs):
+        print(f"\n{'='*50}")
+        print(f"轮{round_idx+1}/3: 温度={temp}")
+        print(f"{'='*50}")
+        
+        # 单线程顺序跑
+        results = []
+        total = len(all_questions)
+        t0 = time.time()
+        
+        for idx, q in enumerate(all_questions):
+            ans = call_astron(q, temp)
+            if ans:
+                results.append({"question": q, "answer": ans})
+            
+            if (idx + 1) % 20 == 0 or idx + 1 == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                eta = (total - idx - 1) / max(idx + 1, 1) * elapsed / 60
+                print(f"  多样: {idx+1}/{total} (已获{len(results)}条, {rate:.0f}条/分, ETA:{eta:.0f}分)", end="\r")
+        
+        elapsed = time.time() - t0
+        rate = len(results) / max(elapsed, 1) * 60
+        print(f"\n  轮{round_idx+1}完成: {len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
+        
+        for r in results:
+            r['temperature'] = temp
+            r['round'] = round_idx + 1
+        all_new.extend(results)
+        
+        # ✅ 每轮跑完立即去重并追加到训练集
+        seen = set()
+        unique = []
+        for r in all_new:
+            key = (r['question'][:100], r['answer'][:100])
+            if key not in seen:
+                seen.add(key)
+                unique.append(r)
+        
+        for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
+            tp = os.path.join(MODEL_DIR, tgt)
+            with open(tp, encoding='utf-8') as f:
+                cur = json.load(f)
+            ex = set((it.get("question","")[:100], it.get("answer","")[:100]) for it in cur)
+            to_add = []
+            for r in unique:
+                key = (r['question'][:100], r['answer'][:100])
+                if key not in ex:
+                    ex.add(key)
+                    to_add.append({"question": r['question'], "answer": r['answer']})
+            cur.extend(to_add)
+            with open(tp, 'w', encoding='utf-8') as f:
+                json.dump(cur, f, ensure_ascii=False, indent=2)
+            if 'train' in tgt:
+                print(f"✅ 轮{round_idx+1} 训练集 +{len(to_add)}条 → {len(cur)}条")
+
+
+if __name__ == '__main__':
+    main()

generate_fresh_qa.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+全新问答对生成 — Astron 凭空造（新问题+新回答），5并发
+不依赖现有问题列表，从种子话题衍生
+增量保存防丢
+"""
+import json, requests, time, os, sys, random, hashlib
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+# ====== Astron 配置 ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚"""
+
+# ====== 种子场景（生成对话的起点） ======
+SCENES = [
+    # 日常场景
+    ("问候", ["打招呼", "问好", "早上起来", "放学回家", "见到朋友"]),
+    ("情绪", ["开心", "难过", "生气", "无聊", "害怕", "惊喜", "委屈", "害羞"]),
+    ("分享", ["今天发生的事", "周末做了什么", "假期计划", "新玩具", "考试成绩"]),
+    ("喜好", ["喜欢的颜色", "喜欢的动物", "喜欢的食物", "喜欢的动画片", "喜欢的游戏"]),
+    ("好奇", ["为什么问题", "想知道的事", "不懂的事", "想象的事", "奇怪的问题"]),
+    ("日常", ["起床", "刷牙", "吃饭", "上学", "做作业", "睡觉"]),
+    ("朋友", ["好朋友", "吵架", "一起玩", "分享秘密", "帮忙"]),
+    ("家庭", ["爸爸妈妈", "爷爷奶奶", "兄弟姐妹", "家人一起", "家庭活动"]),
+    ("想象", ["童话故事", "梦想", "如果我有魔法", "长大想做什么", "幻想朋友"]),
+    ("安慰", ["受伤了", "丢东西", "被骂了", "考试没考好", "被欺负", "做噩梦"]),
+    ("节日", ["生日", "过年", "圣诞节", "儿童节", "中秋"]),
+    ("自然", ["天气", "动物", "植物", "星空", "大海", "四季"]),
+    ("学校", ["老师", "同学", "上课", "课间", "课外活动", "考试"]),
+    ("晚安", ["说晚安", "睡前", "梦", "明天见", "晚安故事"]),
+]
+
+def call_astron(prompt, temperature=0.9):
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt}
+                ],
+                "temperature": temperature,
+                "max_tokens": 4096,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 10:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def parse_qa_pairs(text):
+    """解析 Astron 返回的问答对列表"""
+    lines = text.strip().split('\n')
+    pairs = []
+    current_q = None
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        q_match = __import__('re').match(r'^[Q问][：:]\s*(.*)', line)
+        a_match = __import__('re').match(r'^[A答][：:]\s*(.*)', line)
+        if q_match:
+            current_q = q_match.group(1).strip()
+        elif a_match and current_q:
+            a_text = a_match.group(1).strip()
+            if current_q and a_text and len(a_text) > 2 and len(current_q) > 2:
+                pairs.append((current_q, a_text))
+            current_q = None
+    return pairs
+
+
+def pair_fingerprint(q, a):
+    """QA对指纹"""
+    raw = f"{q[:40]}|{a[:40]}"
+    return hashlib.md5(raw.encode()).hexdigest()
+
+
+def generate_batch_pairs(batch_size=10, existing_fps=None):
+    """生成一批全新的问答对"""
+    if existing_fps is None:
+        existing_fps = set()
+    
+    # 随机选场景
+    category, sub_scenes = random.choice(SCENES)
+    sub = random.choice(sub_scenes)
+    # 随机生成要求
+    style = random.choice(["日常聊天", "撒娇可爱", "有点害羞", "好奇宝宝", "暖心安慰"])
+    mood = random.choice(["开心", "平静", "有点小情绪", "好奇", "温暖"])
+    
+    prompt = f"""请以小狸的身份，生成{batch_size}个不同的日常对话问答对。
+
+对话背景：
+- 场景：{category}（{sub}）
+- 风格：{style}
+- 小狸心情：{mood}
+
+要求：
+1. 每个问答对包含一问一答，问是用户说的，答是小狸回答
+2. 问要自然，像真实用户会说的话（不要提"小狸"）
+3. 答要是小狸的风格，1-2句话，简短自然
+4. {batch_size}个问答要对同一个话题但角度不同
+5. 不要重复一模一样的问法
+
+请按以下格式输出：
+Q：用户提问内容
+A：小狸回答内容
+---（分隔）
+Q：用户提问内容
+A：小狸回答内容
+---"""
+    
+    result = call_astron(prompt, temperature=random.uniform(0.85, 1.0))
+    if not result:
+        return []
+    
+    pairs = parse_qa_pairs(result)
+    
+    # 去重
+    deduped = []
+    for q, a in pairs:
+        fp = pair_fingerprint(q, a)
+        if fp not in existing_fps:
+            existing_fps.add(fp)
+            deduped.append({'question': q, 'answer': a})
+    
+    return deduped
+
+
+def main():
+    output_dir = '/workspace/xiaoli-llm-chat/fresh_qa_outputs'
+    os.makedirs(output_dir, exist_ok=True)
+    
+    # ====== 恢复进度 ======
+    progress_file = os.path.join(output_dir, 'fresh_qa_progress.json')
+    all_pairs = []
+    if os.path.exists(progress_file):
+        with open(progress_file, encoding='utf-8') as f:
+            all_pairs = json.load(f)
+        print(f"已有进度: {len(all_pairs)} 条")
+    
+    # 加载已有训练集做去重
+    train_path = '/workspace/xiaoli-llm-chat/xiaoli_train_v2.json'
+    with open(train_path, encoding='utf-8') as f:
+        train_data = json.load(f)
+    existing_fps = set(pair_fingerprint(item.get('question',''), item.get('answer','')) for item in train_data)
+    for p in all_pairs:
+        existing_fps.add(pair_fingerprint(p['question'], p['answer']))
+    print(f"去重指纹池: {len(existing_fps)} 个")
+    
+    # ====== 参数 ======
+    TARGET = 100000          # 目标 10 万条全新问答
+    BATCH_SIZE = 10          # 每批次生成 10 条
+    BATCHES_PER_SAVE = 20    # 每 20 批次保存一次
+    MAX_WORKERS = 5          # Astron 5并发
+    
+    remaining = TARGET - len(all_pairs)
+    total_iterations = (remaining + BATCH_SIZE - 1) // BATCH_SIZE
+    print(f"目标: {TARGET:,} 条, 还需: {remaining:,} 条")
+    print(f"轮次: {total_iterations} 轮 (每轮{BATCH_SIZE}条, {MAX_WORKERS}并发)")
+    print(f"{'='*60}\n")
+    
+    t_start = time.time()
+    batch_counter = 0
+    
+    for iteration in range(total_iterations):
+        t0 = time.time()
+        
+        # 并发生成
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = [
+                executor.submit(generate_batch_pairs, BATCH_SIZE, existing_fps)
+                for _ in range(MAX_WORKERS)
+            ]
+            for future in as_completed(futures):
+                try:
+                    new_pairs = future.result()
+                    if new_pairs:
+                        for p in new_pairs:
+                            existing_fps.add(pair_fingerprint(p['question'], p['answer']))
+                            all_pairs.append(p)
+                except Exception:
+                    pass
+        
+        batch_counter += 1
+        elapsed = time.time() - t_start
+        rate = len(all_pairs) / max(elapsed, 1) * 60
+        
+        # 定期保存
+        if batch_counter % BATCHES_PER_SAVE == 0 or iteration == total_iterations - 1:
+            with open(progress_file, 'w', encoding='utf-8') as f:
+                json.dump(all_pairs, f, ensure_ascii=False, indent=2)
+        
+        eta = (TARGET - len(all_pairs)) / max(rate, 0.1) / 60 if rate > 0 else 0
+        
+        if iteration % 5 == 0 or iteration == total_iterations - 1:
+            print(f"轮 {iteration+1}/{total_iterations} | "
+                  f"累计 {len(all_pairs):,}/{TARGET:,} | "
+                  f"{rate:.0f}条/分 | "
+                  f"已用 {elapsed/60:.1f}分 | "
+                  f"预计剩余 {eta:.1f}分")
+    
+    # ====== 完成 ======
+    elapsed_total = time.time() - t_start
+    print(f"\n{'='*60}")
+    print(f"全新问答生成完成!")
+    print(f"总数: {len(all_pairs):,} 条")
+    print(f"用时: {elapsed_total/60:.1f} 分钟")
+    print(f"速率: {len(all_pairs)/max(elapsed_total,1)*60:.0f} 条/分")
+    
+    # 保存最终
+    final_path = '/data/user/work/fresh_qa_v1.json'
+    with open(final_path, 'w', encoding='utf-8') as f:
+        json.dump(all_pairs, f, ensure_ascii=False, indent=2)
+    print(f"保存到: {final_path}")
+
+if __name__ == '__main__':
+    main()

generate_multiturn.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+多轮对话生成 — 用 Astron 造 2~5 轮小狸闲聊对话
+5并发，无限量，增量保存防丢
+"""
+import json, requests, time, os, sys, random, re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+# ====== Astron 配置（和 api_correction 一致） ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+# ====== 小狸基础提示词 ======
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚"""
+
+# ====== 多轮对话种子（开场话题池） ======
+SEEDS = [
+    # 日常问候
+    "你好呀", "你叫什么名字", "你今天开心吗", "你在干嘛呢",
+    "你好可爱呀", "我来看你啦", "你今天心情怎么样",
+    # 闲聊话题
+    "你喜欢什么颜色", "你最喜欢什么动物", "你爱吃什么零食",
+    "你最喜欢什么玩具", "你最喜欢什么动画片",
+    "你最好的朋友是谁", "幼儿园里好玩吗",
+    "你喜欢唱歌吗", "你会跳舞吗",
+    # 情绪表达
+    "我今天好开心", "我有点不开心", "我好无聊啊",
+    "我遇到了一件好事", "我今天有点难过",
+    "我好想你呀", "你能不能陪我聊聊天",
+    # 好奇问题
+    "为什么天是蓝的", "为什么月亮跟着我走",
+    "世界上最大的动物是什么", "你会讲故事吗",
+    "你会数数吗", "1+1等于几",
+    "你喜欢上学吗", "你今年几岁啦",
+    # 分享日常
+    "我今天去公园玩了", "我养了一只小猫",
+    "我今天考试考了100分", "我学会骑自行车了",
+    "妈妈给我买了个新玩具", "我今天摔了一跤",
+    # 晚安/告别
+    "我要睡觉了", "明天见", "晚安啦",
+    "我要去吃饭了", "下次再聊哦",
+]
+
+# 话题扩展 — 组合出更多种子
+PREFIXES = ["小狸", "小狸小狸", "亲爱的小狸", "嘿嘿小狸", "小狸呀"]
+SUFFIXES = ["呀", "吗", "呢", "", "呀", "吗", "呢", ""]
+
+def expand_seeds(base, max_expand=5):
+    """种子话题 → 多样化的开场白"""
+    expanded = []
+    for s in base:
+        expanded.append(s)
+        if '吗' not in s and not s.endswith('呀') and not s.endswith('呢'):
+            for _ in range(2):
+                prefix = random.choice(PREFIXES) if random.random() > 0.4 else ''
+                suffix = random.choice(SUFFIXES) if random.random() > 0.5 else ''
+                combo = f"{prefix}{s}{suffix}".strip()
+                if combo and combo not in expanded:
+                    expanded.append(combo)
+    random.shuffle(expanded)
+    return expanded[:len(base) * max_expand]
+
+
+def call_astron_multi(prompt, temperature=0.85):
+    """调用 Astron 生成一条多轮对话（2~5轮）"""
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": prompt}
+                ],
+                "temperature": temperature,
+                "max_tokens": 8192,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 10:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def parse_multi_turn(text):
+    """
+    解析 Astron 返回的多轮对话文本为 [(q, a), (q, a), ...]
+    期望返回格式（至少2轮对话）：
+    问：xxx
+    答：xxx
+    问：xxx
+    答：xxx
+    
+    或
+    用户：xxx
+    小狸：xxx
+    用户：xxx
+    小狸：xxx
+    """
+    lines = text.strip().split('\n')
+    turns = []
+    current_q = None
+    
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        
+        # 匹配 "问：" 或 "问题：" 或 "用户：" 格式
+        q_match = re.match(r'^(问|问题|用户|我)[：:]\s*(.*)', line)
+        # 匹配 "答：" 或 "回答：" 或 "小狸：" 格式
+        a_match = re.match(r'^(答|回答|小狸|小狸说|机器人)[：:]\s*(.*)', line)
+        
+        if q_match:
+            q_text = q_match.group(2).strip()
+            if q_text and len(q_text) > 1:
+                current_q = q_text
+        elif a_match and current_q:
+            a_text = a_match.group(2).strip()
+            if a_text and len(a_text) > 1 and len(a_text) < 200:
+                turns.append((current_q, a_text))
+                current_q = None  # 配对后重置，等待下一个问
+    
+    # 最少2轮才算有效
+    if len(turns) >= 2:
+        return turns
+    return None
+
+
+def format_training_sample(turns):
+    """
+    将多轮对话 [('q1','a1'), ('q2','a2'), ...] 格式化为训练样本
+    
+    格式：<SOS>q1<SEP><START>a1<END>q2<SEP><START>a2<END><EOS>
+    """
+    parts = []
+    for i, (q, a) in enumerate(turns):
+        if i == 0:
+            parts.append(f"<SOS>{q}<SEP><START>{a}<END>")
+        else:
+            parts.append(f"{q}<SEP><START>{a}<END>")
+    parts.append("<EOS>")
+    return "".join(parts)
+
+
+def make_multi_prompt():
+    """
+    构造多轮对话生成提示词
+    Astron 每次生成一条完整的4~6轮对话
+    """
+    # 随机选开场话题（2~3个组合，让对话有展开）
+    topic_count = random.randint(1, 3)
+    topics = random.sample(all_seeds, min(topic_count, len(all_seeds)))
+    topic_context = "、".join(topics)
+    
+    turn_count = random.randint(2, 5)
+    
+    prompt = f"""请以小狸的身份，模拟一段{turn_count}轮的自然闲聊对话。
+
+开场话题：{topic_context}
+
+要求：
+1. 小狸是8岁小女孩，说话简短自然
+2. 每轮包含"问"和"答"，小狸的回答1-2句话
+3. 对话要自然展开，不要强行切话题
+4. 小狸的语气：害羞但可爱，不是机器人
+
+请按以下格式输出（至少{turn_count}轮）：
+问：（用户提问）
+答：（小狸回答）
+问：（用户续问）
+答：（小狸回答）
+..."""
+    return prompt
+
+
+def generate_one_multi():
+    """生成一条多轮对话"""
+    prompt = make_multi_prompt()
+    result = call_astron_multi(prompt, temperature=random.uniform(0.8, 1.0))
+    if result:
+        turns = parse_multi_turn(result)
+        if turns and len(turns) >= 2:
+            sample = format_training_sample(turns)
+            return {
+                'source': 'multiturn_astron',
+                'turns': len(turns),
+                'sample': sample,
+                'turns_list': turns,
+                'raw': result[:100]
+            }
+    return None
+
+
+def main():
+    global all_seeds
+    
+    model_dir = os.path.dirname(os.path.abspath(__file__))
+    
+    # 加载现有训练数据的问句作为种子
+    train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
+    with open(train_path, encoding='utf-8') as f:
+        train_data = json.load(f)
+    
+    # 从现有数据中随机抽取问句作为种子
+    existing_qs = [item.get('question','') for item in train_data if item.get('question','')]
+    random.shuffle(existing_qs)
+    seed_from_data = existing_qs[:500]
+    
+    # 合并种子话题
+    all_seeds = expand_seeds(SEEDS, max_expand=10) + seed_from_data
+    random.shuffle(all_seeds)
+    
+    print(f"种子话题: {len(all_seeds)} 个 (基础{len(SEEDS)}+数据抽取{len(seed_from_data)})")
+    
+    # 输出目录
+    out_dir = os.path.join(model_dir, 'multiturn_outputs')
+    os.makedirs(out_dir, exist_ok=True)
+    
+    # 检查已有进度
+    all_results = []
+    interim_files = sorted([f for f in os.listdir(out_dir) 
+                           if f.startswith('mt_batch_') and f.endswith('.json')])
+    if interim_files:
+        for fname in interim_files:
+            with open(os.path.join(out_dir, fname), encoding='utf-8') as f:
+                data = json.load(f)
+            all_results.extend(data)
+        print(f"已有中间结果: {len(all_results)} 条 ({len(interim_files)} 批)")
+    
+    # ====== 参数 ======
+    TARGET = 20000   # 目标生成 2 万条多轮对话
+    BATCH = 50       # 每批生成 50 条
+    MAX_WORKERS = 5  # Astron 5并发
+    TEMPERATURE_RANGE = (0.80, 1.0)
+    
+    to_generate = TARGET - len(all_results)
+    total_batches = (to_generate + BATCH - 1) // BATCH
+    
+    print(f"目标: {TARGET} 条, 已有: {len(all_results)} 条, 还需: {to_generate} 条")
+    print(f"批次: {total_batches} 批 (每批{BATCH}条, {MAX_WORKERS}并发)")
+    print(f"{'='*60}\n")
+    
+    t_start = time.time()
+    
+    for batch_idx in range(total_batches):
+        t0 = time.time()
+        
+        # 并发生成一批
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = [executor.submit(generate_one_multi) for _ in range(BATCH)]
+            batch_results = []
+            for future in as_completed(futures):
+                try:
+                    r = future.result()
+                    if r:
+                        batch_results.append(r)
+                except Exception:
+                    pass
+        
+        # 去重（按 sample 内容）
+        existing_samples = set(r['sample'] for r in all_results)
+        deduped = []
+        for r in batch_results:
+            if r['sample'] not in existing_samples:
+                existing_samples.add(r['sample'])
+                deduped.append(r)
+        
+        all_results.extend(deduped)
+        
+        # 保存本批
+        elapsed = time.time() - t0
+        batch_file = os.path.join(out_dir, f'mt_batch_{batch_idx + 1:04d}.json')
+        with open(batch_file, 'w', encoding='utf-8') as f:
+            json.dump(deduped, f, ensure_ascii=False, indent=2)
+        
+        # 追加保存总进度
+        progress_file = os.path.join(out_dir, 'all_multiturn_progress.json')
+        with open(progress_file, 'w', encoding='utf-8') as f:
+            json.dump(all_results, f, ensure_ascii=False, indent=2)
+        
+        total_elapsed = time.time() - t_start
+        rate = len(all_results) / max(total_elapsed, 1) * 60
+        eta = (TARGET - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
+        
+        # 输出统计
+        avg_turns = sum(r['turns'] for r in deduped) / max(len(deduped), 1)
+        print(f"批次 {batch_idx+1}/{total_batches} | "
+              f"+{len(deduped)} 条 (去重后) | "
+              f"累计 {len(all_results)}/{TARGET} | "
+              f"均轮 {avg_turns:.1f} | "
+              f"{rate:.0f}条/分 | "
+              f"预计剩余 {eta:.1f}分")
+    
+    # ====== 完成总结 ======
+    total_elapsed = time.time() - t_start
+    print(f"\n{'='*60}")
+    print(f"多轮对话生成完成!")
+    print(f"生成: {len(all_results)} 条 (去重后)")
+    avg_turns2 = sum(r['turns'] for r in all_results) / max(len(all_results), 1)
+    print(f"平均轮数: {avg_turns2:.1f}")
+    print(f"总对话轮次: {sum(r['turns'] for r in all_results)}")
+    print(f"用时: {total_elapsed/60:.1f} 分钟")
+    print(f"速率: {len(all_results)/max(total_elapsed,1)*60:.0f} 条/分")
+    
+    # ====== 保存最终结果（独立存放，不追加到训练集） ======
+    final_file = os.path.join('/data/user/work', 'multiturn_final.json')
+    with open(final_file, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False, indent=2)
+    print(f"\n最终结果已保存: {final_file} ({len(all_results)} 条)")
+    print(f"如需后续使用，可手动合并到训练集")
+    
+    # ====== 保存多轮样本供训练时直接使用（含标记格式） ======
+    mt_samples_file = os.path.join(model_dir, 'multiturn_samples_v1.json')
+    with open(mt_samples_file, 'w', encoding='utf-8') as f:
+        samples = [{'turns': r['turns'], 'sample': r['sample']} for r in all_results]
+        json.dump(samples, f, ensure_ascii=False, indent=2)
+    print(f"  ✅ 多轮样本保存: multiturn_samples_v1.json ({len(samples)} 条)")
+    
+    print(f"\n{'='*60}")
+    print(f"全部完成! 新增 {len(all_results)} 条多轮对话")
+    print(f"{'='*60}")
+
+if __name__ == '__main__':
+    main()