v5: 两阶段生成 - Phase1纯API种子 + Phase2语义扩展

- 用 CharTokenizer.build_vocab 从 977K 数据生成词表 (vocab_size=47725)
- train_cnb.py: batch=224 + accum=2 (等效448) + AMP + torch.compile
- build_vocab.py: 词表生成脚本
- vocab_full.json: 含特殊token完整词表
- 模型 41.85M参数, L40 GPU训练

.cnb.yml

@@ -0,0 +1,21 @@
+$:
+  vscode:
+    - docker:
+        build: .ide/Dockerfile
+      runner:
+        cpus: 8
+        tags: cnb:arch:amd64:gpu
+      services:
+        - vscode
+        - docker
+      env:
+        CUDA_VISIBLE_DEVICES: "0"
+        PYTHONUNBUFFERED: "1"
+      stages:
+        - name: 开始训练
+          script: |
+            cd /workspace
+            nvidia-smi
+            echo "=== GPU就绪，开始训练 ==="
+            python3 scripts/train_cnb.py --epochs 100 --batch_size 16
+            echo "=== 训练完成 ==="

.gitignore

@@ -1,12 +1,15 @@
+# 已跟踪
 # 缓存和日志
 __pycache__/
 *.pyc
 *.log
 /tmp/
 
-# 模型权重 — 只追踪发布版本
+# 模型权重 — 只追踪发布版本(精简推理版,不含optimizer,约60MB)
 *.pth
 !/llm_model_v2_resume_epoch_7.pth
+!/xiaoli-llm-chat/xiaoli_v2_best_inference.pth
+!/xiaoli-llm-chat/xiaoli_v2_final_inference.pth
 
 # 中间输出目录（生成脚本的输出）
 high_temp_outputs/
@@ -44,4 +47,38 @@ mega_*.log
 mimo_*.log
 knowledge_*.log
 train_*.log
-web_server*.log
+web_server*.log
+# CNB 配置 — 强制跟踪关键文件
+.ide/
+!/.ide/Dockerfile
+!/.cnb.yml
+
+# 训练输出目录
+general_outputs/
+high_temp_outputs/
+multiturn_outputs/
+
+# 本地临时
+fun_test.py
+*.log
+
+# 打包/发布产物
+xiaoli_package/
+*.zip
+
+# 上传/测试脚本
+upload_*.py
+_gofile_*
+_t.*
+_up_test.*
+
+# 旧词表备份
+vocab_old.json
+vocab_backup*.json
+
+# 数据目录（太大不进git）
+data/
+
+# 训练checkpoint（不进git，通过release发布）
+xiaoli_cnb_best.pth
+xiaoli_cnb_final.pth

.ide/Dockerfile

@@ -0,0 +1,13 @@
+FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime
+
+# 基础工具
+RUN apt-get update && apt-get install -y git curl wget nano && rm -rf /var/lib/apt/lists/*
+
+# tqdm用于进度条
+RUN pip install --no-cache-dir tqdm numpy requests
+
+# 设置工作目录
+WORKDIR /workspace
+
+# 验证CUDA
+RUN python3 -c "import torch; print(f'PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}')"

api_correction.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+API生成模块 — Astron 5并发，稳定不超限
+"""
+import json, requests, time, os, threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚，不要讲一半就停"""
+
+
+def call_astron(q, temperature=0.85, system_prompt=None, max_tokens=8192):
+    """调用Astron API，支持自定义system prompt"""
+    if system_prompt is None:
+        system_prompt = SYSTEM_PROMPT
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": q}
+                ],
+                "temperature": temperature,
+                "max_tokens": max_tokens,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=120)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def batch_generate(questions, temperature=0.85, max_workers=5):
+    """Astron 5并发 — 明确限制，稳定可靠"""
+    total = len(questions)
+    results = []
+    
+    def call(q):
+        ans = call_astron(q, temperature)
+        if ans:
+            return q, ans
+        return None
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(call, item): i for i, item in enumerate(questions)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append({"question": r[0], "answer": r[1]})
+            if done % 20 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r")
+    
+    elapsed = time.time() - t0
+    rate = len(results) / max(elapsed, 1) * 60
+    print(f"    Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
+    return results
+
+
+# 保持旧函数名兼容
+batch_generate_astron = batch_generate
+batch_generate_mimo = batch_generate
+batch_generate_direct = batch_generate
+batch_generate_dual = batch_generate
+
+
+def batch_convert(items, progress=None, max_workers=5):
+    """修复版 batch_convert — 将 (q, ans) 对用 API 优化后返回"""
+    if not items:
+        return []
+    total = len(items)
+    results = []
+    
+    def convert(item):
+        q, model_ans = item
+        prompt = f"问题：{q}\n模型回答：{model_ans}\n\n请用8岁小狸的语气，重新生成更好的回答。保持简短自然，1-2句话。"
+        ans = call_astron(prompt, temperature=0.3)
+        if ans and len(ans) >= 3:
+            return {"question": q, "answer": ans}
+        return None
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(convert, item): i for i, item in enumerate(items)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append(r)
+            if progress:
+                progress(done, total)
+            if done % 50 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    API转换: {done}/{total} (+{len(results)}条, {rate:.0f}条/分)", end="\r")
+    
+    elapsed = time.time() - t0
+    rate = len(results) / max(elapsed, 1) * 60
+    print(f"    API转换: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
+    return results
+
+
+def batch_generate_multi_temp(questions, temperatures=None, max_workers=5):
+    """修复版 — 用多种温度生成多样化回答"""
+    if temperatures is None:
+        temperatures = [0.7, 0.85, 1.0]
+    if not questions:
+        return []
+    
+    total = len(questions) * len(temperatures)
+    results = []
+    
+    def call_with_temp(q, temp):
+        ans = call_astron(q, temperature=temp)
+        if ans:
+            return {"question": q, "answer": ans, "temperature": temp}
+        return None
+    
+    tasks = [(q, t) for q in questions for t in temperatures]
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(call_with_temp, q, t): i for i, (q, t) in enumerate(tasks)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append(r)
+            if done % 50 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    多温生成: {done}/{total} (+{len(results)}条, {rate:.0f}条/分)", end="\r")
+    
+    seen = set()
+    unique = []
+    for item in results:
+        key = (item["question"], item["answer"])
+        if key not in seen:
+            seen.add(key)
+            unique.append({"question": item["question"], "answer": item["answer"]})
+    
+    elapsed = time.time() - t0
+    rate = len(unique) / max(elapsed, 1) * 60
+    print(f"    多温生成: 共{len(unique)}条 (去重后, {elapsed:.0f}s, {rate:.0f}条/分)")
+    return unique

build_vocab.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+用已有的 CharTokenizer + build_vocabulary 从训练数据生成词表
+然后直接启动训练
+"""
+import sys, os, json, time
+sys.path.insert(0, '/workspace/xiaoli-llm-chat')
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
+
+from llm_chat_improved import tokenizer_instance, build_vocabulary
+
+DATA_PATH = '/workspace/data/merged_clean_v3.json'
+VOCAB_OUT = '/workspace/vocab.json'
+VOCAB_OUT_XIAOLI = '/workspace/xiaoli-llm-chat/vocab.json'
+
+def load_pairs(path):
+    """加载训练数据为 (question, answer) 对"""
+    with open(path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    pairs = []
+    for item in data:
+        if isinstance(item, dict):
+            q = item.get("question", "") or item.get("input", "") or item.get("prompt", "")
+            a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
+            if q and a:
+                pairs.append((q, a))
+    return pairs
+
+def main():
+    print("=" * 60)
+    print("用 build_vocabulary 从训练数据生成词表")
+    print("=" * 60)
+    
+    # 1. 加载训练数据
+    t0 = time.time()
+    print("\n[1] 加载训练数据...")
+    pairs = load_pairs(DATA_PATH)
+    print(f"  数据量: {len(pairs)} 对")
+    print(f"  耗时: {time.time()-t0:.1f}s")
+    
+    # 2. 用 build_vocabulary 生成词表
+    t1 = time.time()
+    print("\n[2] 用 build_vocabulary 分析数据生成词表...")
+    vocab = build_vocabulary(pairs, min_freq=1)
+    
+    # build_vocabulary 返回的是 tokenizer_instance.char_to_id
+    # 已包含特殊token (<PAD>=0, <UNK>=1, etc.)
+    special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
+                      '<START>': 4, '<END>': 5, '<SEP>': 6}
+    
+    real_vocab_size = max(vocab.values()) + 1
+    
+    # 统计
+    single_char_tokens = sum(1 for k in vocab if len(k) == 1)
+    multi_char_tokens = sum(1 for k in vocab if len(k) > 1 and not k.startswith('<'))
+    special_token_count = sum(1 for k in vocab if k.startswith('<'))
+    
+    print(f"  词表总项: {len(vocab)}")
+    print(f"  vocab_size(模型用): {real_vocab_size}")
+    print(f"  单字符token: {single_char_tokens}")
+    print(f"  多字符token(英文词): {multi_char_tokens}")
+    print(f"  特殊token: {special_token_count}")
+    print(f"  耗时: {time.time()-t1:.1f}s")
+    
+    # 3. 编码测试
+    print("\n[3] 编码测试:")
+    test_texts = [
+        "你好呀！我是小狸～",
+        "小狸最喜欢吃草莓冰淇淋了！",
+        "<SOS>你好<SEP><START>我是小狸～<END><EOS>",
+    ]
+    for text in test_texts:
+        ids = tokenizer_instance.encode(text)
+        decoded = tokenizer_instance.decode(ids)
+        print(f"  原文: '{text}'")
+        print(f"  编码: {ids[:20]}... (长度={len(ids)})")
+        print(f"  解码: '{decoded}'")
+        print()
+    
+    # 4. 估算模型参数量
+    print("\n[4] 模型参数量估算 (d_model=320, n_layers=6, d_ff=1536):")
+    d_model = 320
+    n_layers = 6
+    d_ff = 1536
+    vs = real_vocab_size
+    emb = vs * d_model
+    out = vs * d_model
+    per_layer = 4 * d_model**2 + 3 * d_model * d_ff + 2 * d_model
+    total = emb + out + n_layers * per_layer + 2 * d_model
+    print(f"  vocab_size={vs}")
+    print(f"  Embedding: {emb:,}")
+    print(f"  fc_out: {out:,}")
+    print(f"  Transformer({n_layers}层): {n_layers*per_layer:,}")
+    print(f"  总参数量: {total:,} ({total/1e6:.2f}M)")
+    
+    # 5. 保存词表
+    print(f"\n[5] 保存词表...")
+    
+    # vocab.json 格式：不含特殊token，id从7开始（与 train.py 兼容）
+    normal_vocab = {k: v for k, v in vocab.items() if k not in special_tokens}
+    
+    # 备份旧词表
+    for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
+        if os.path.exists(path):
+            backup = path.replace('vocab.json', 'vocab_old.json')
+            import shutil
+            shutil.copy2(path, backup)
+            print(f"  旧词表已备份: {backup}")
+    
+    # 保存到两个位置
+    for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(normal_vocab, f, ensure_ascii=False, indent=1)
+        print(f"  {path}: {len(normal_vocab)}项 (不含特殊token, id从7开始)")
+    
+    # 保存完整词表（含特殊token，用于参考）
+    full_path = VOCAB_OUT_XIAOLI.replace('vocab.json', 'vocab_full.json')
+    with open(full_path, 'w', encoding='utf-8') as f:
+        json.dump(vocab, f, ensure_ascii=False, indent=1)
+    print(f"  {full_path}: {len(vocab)}项 (含特殊token)")
+    
+    total_time = time.time() - t0
+    print(f"\n总耗时: {total_time:.1f}s")
+    print("=" * 60)
+    print(f"词表生成完成！vocab_size = {real_vocab_size}")
+    print(f"模型参数量 ≈ {total/1e6:.2f}M")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

data/merged_clean_v3.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e60d5f977e5b1e521d7b38d4a6c89aae0b67f540ee6754929e0407f7b408ab9
+size 175346049

distill_daemon.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+小狸 AI - 持续蒸馏守护脚本
+============================
+每轮生成 1000 条新 QA 对，去重后追加到训练集。
+5 并发调用 Astron API，不超限。
+
+使用方式:
+  python3 distill_daemon.py              # 跑一轮
+  python3 distill_daemon.py --loop       # 持续循环，每轮间隔 60 秒
+  python3 distill_daemon.py --count 5000 # 指定本轮目标条数
+"""
+import sys, os, json, time, random, argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from api_correction import call_astron, batch_generate
+
+# ── 种子话题（15 大类，持续扩展） ──────────────────────
+SEED_TOPICS = {
+    "日常问候": ["今天开心吗", "在干嘛呢", "吃了吗", "你好呀", "早上好"],
+    "情绪感受": ["我有点难过", "今天好开心", "我好无聊", "有点害怕", "好兴奋"],
+    "学校生活": ["上学好玩吗", "今天学了什么", "作业写完了吗", "喜欢上什么课", "考试考了多少"],
+    "朋友关系": ["你有好朋友吗", "怎么交朋友", "朋友吵架了", "一起玩什么", "好朋友搬家了"],
+    "家庭生活": ["喜欢爸爸妈妈吗", "有兄弟姐妹吗", "帮妈妈做家务", "周末去哪玩", "家里养宠物了吗"],
+    "兴趣爱好": ["喜欢唱歌吗", "会画画吗", "喜欢什么动画片", "爱看书吗", "喜欢什么游戏"],
+    "奇思妙想": ["为什么天是蓝的", "星星会掉下来吗", "鱼在水里怎么呼吸", "花为什么会开", "月亮上有兔子吗"],
+    "安慰鼓励": ["失败了怎么办", "被人欺负了", "好紧张怎么办", "觉得自己不够好", "想放弃了"],
+    "节日话题": ["最喜欢什么节日", "过年开心吗", "生日怎么过", "圣诞节收到什么礼物", "儿童节去哪玩"],
+    "自然科普": ["下雨了怎么办", "为什么会有彩虹", "树叶为什么变黄", "雪是怎么来的", "太阳下山去哪了"],
+    "童话故事": ["喜欢什么童话", "听过灰姑娘吗", "美人鱼的故事", "孙悟空厉害吗", "小红帽的故事"],
+    "美食话题": ["喜欢吃什么", "讨厌什么蔬菜", "冰淇淋好吃吗", "会做吃的吗", "妈妈做的饭好吃吗"],
+    "动物朋友": ["喜欢猫还是狗", "小兔子可爱吗", "小鸟会说话吗", "去过动物园吗", "恐龙是真的吗"],
+    "睡前话题": ["晚安怎么说", "做噩梦了", "睡不着怎么办", "想要抱抱", "讲个睡前故事"],
+    "成长烦恼": ["不想长大", "怕黑怎么办", "一个人在家害怕", "摔跤了很疼", "被批评了好难过"],
+}
+
+# 所有话题平铺
+ALL_TOPICS = []
+for cat, items in SEED_TOPICS.items():
+    ALL_TOPICS.extend(items)
+
+# 对话风格模板
+STYLE_TEMPLATES = [
+    "请用8岁小女孩小狸的语气回答，简短自然，1-2句话。",
+    "小狸是个有点害羞但很可爱的小女孩，请模仿她的语气回答。",
+    "像小朋友一样简单回答就好，要用小狸的口吻哦。",
+    "用小狸的口吻回答，要可爱但不要太刻意，简短就好。",
+    "你是一个8岁小女孩，请用小孩子的方式回答这个问题。",
+]
+
+
+def generate_seed_questions(count: int = 500) -> list:
+    """从种子话题生成问题列表（随机组合+衍生）"""
+    questions = []
+    
+    # 1. 直接使用种子话题
+    for topic in random.sample(ALL_TOPICS, min(count // 3, len(ALL_TOPICS))):
+        questions.append(topic)
+    
+    # 2. 组合衍生（如 "今天开心吗 + 考试考了多少" → "今天考试考得开心吗"）
+    variant_templates = [
+        lambda a, b: f"{a}，那{b}",
+        lambda a, b: f"如果{a}，是不是{b}",
+        lambda a, b: f"{a}？还有{b}",
+        lambda a, b: f"为什么{a}，然后{b}呀",
+    ]
+    needed = count // 3
+    for i in range(needed):
+        a = random.choice(ALL_TOPICS)
+        b = random.choice(ALL_TOPICS)
+        tmpl = random.choice(variant_templates)
+        questions.append(tmpl(a, b))
+    
+    # 3. "为什么"类衍生
+    why_prefixes = ["为什么", "怎么才能", "可以教我", "告诉我", "什么是", "什么时候", "在哪里", "谁"]
+    for i in range(count - len(questions)):
+        base = random.choice(ALL_TOPICS)
+        prefix = random.choice(why_prefixes)
+        questions.append(f"{prefix}{base}?")
+    
+    random.shuffle(questions)
+    return questions[:count]
+
+
+def load_existing_pairs(train_path: str) -> set:
+    """加载已有数据，供去重用"""
+    if not os.path.exists(train_path):
+        return set()
+    try:
+        with open(train_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return set((item.get("question", ""), item.get("answer", "")) for item in data)
+    except:
+        return set()
+
+
+def append_to_train(new_pairs: list, train_path: str, merged_path: str):
+    """去重后追加到训练集和合并文件"""
+    if not new_pairs:
+        return 0
+    
+    existing = load_existing_pairs(train_path)
+    truly_new = [item for item in new_pairs if (item["question"], item["answer"]) not in existing]
+    
+    if not truly_new:
+        print(f"  ℹ️ 无新内容（全部已存在）")
+        return 0
+    
+    # 追加到训练集
+    if os.path.exists(train_path):
+        with open(train_path, 'r', encoding='utf-8') as f:
+            cur_train = json.load(f)
+    else:
+        cur_train = []
+    cur_train.extend(truly_new)
+    with open(train_path, 'w', encoding='utf-8') as f:
+        json.dump(cur_train, f, ensure_ascii=False, indent=2)
+    
+    # 同步到合并文件
+    if os.path.exists(merged_path):
+        with open(merged_path, 'r', encoding='utf-8') as f:
+            cur_merged = json.load(f)
+        cur_merged.extend(truly_new)
+        with open(merged_path, 'w', encoding='utf-8') as f:
+            json.dump(cur_merged, f, ensure_ascii=False, indent=2)
+    
+    unique_qs = len(set(item["question"] for item in truly_new))
+    print(f"  ✅ 新增 {len(truly_new)} 条（{unique_qs} 道独特问题）到训练集")
+    return len(truly_new)
+
+
+def run_distill_round(target_count: int = 1000, save_dir: str = None):
+    """执行一轮蒸馏：生成问题 → API 回答 → 去重追加"""
+    if save_dir is None:
+        save_dir = os.path.dirname(os.path.abspath(__file__))
+    
+    train_path = os.path.join(save_dir, 'xiaoli_train_v2.json')
+    merged_path = os.path.join(save_dir, 'xiaoli_merged_data_v2.json')
+    
+    print(f"\n{'='*60}")
+    print(f"  蒸馏轮次开始 | 目标: {target_count} 条")
+    print(f"{'='*60}")
+    
+    # 1. 生成种子问题
+    questions = generate_seed_questions(target_count)
+    print(f"  📝 生成 {len(questions)} 个问题")
+    
+    # 2. 用 Astron API 批量回答（5 并发）
+    t0 = time.time()
+    results = batch_generate(questions, temperature=0.85, max_workers=5)
+    elapsed = time.time() - t0
+    
+    # 3. 去重追加
+    added = append_to_train(results, train_path, merged_path)
+    
+    rate = added / max(elapsed, 1) * 60
+    print(f"  📊 本轮: +{added} 条 | 耗时 {elapsed:.0f}s | 速率 {rate:.0f} 条/分")
+    print(f"  💾 训练集: {os.path.getsize(train_path)/1024/1024:.0f}MB (若存在)")
+    
+    return added
+
+
+def main():
+    parser = argparse.ArgumentParser(description="小狸 AI 持续蒸馏守护脚本")
+    parser.add_argument("--count", type=int, default=1000, help="每轮目标条数")
+    parser.add_argument("--loop", action="store_true", help="持续循环模式")
+    parser.add_argument("--interval", type=int, default=300, help="循环间隔秒数（默认300）")
+    parser.add_argument("--rounds", type=int, default=0, help="循环轮数（0=无限）")
+    args = parser.parse_args()
+    
+    if args.loop:
+        round_num = 1
+        total_added = 0
+        while True:
+            print(f"\n{'#'*60}")
+            print(f"  # 第 {round_num} 轮蒸馏")
+            print(f"{'#'*60}")
+            added = run_distill_round(args.count)
+            total_added += added
+            print(f"  📈 累计新增: {total_added} 条")
+            
+            if args.rounds > 0 and round_num >= args.rounds:
+                print(f"\n✅ 完成 {args.rounds} 轮蒸馏，共新增 {total_added} 条")
+                break
+            
+            print(f"  ⏳ 等待 {args.interval}s 后下一轮...")
+            time.sleep(args.interval)
+            round_num += 1
+    else:
+        run_distill_round(args.count)
+
+
+if __name__ == "__main__":
+    main()

gen_pretrain.py

@@ -0,0 +1,484 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+小狸 AI - 预训练数据生成器 (v5)
+==================================
+两阶段生成：
+
+Phase 1 - 种子直出:
+  5并发 API 直接生成完整多轮对话
+  每API调用 = 1条高质量对话 (15-25轮, 2000-4000字)
+  速率: ~600条/h, 17h出1万条种子
+
+Phase 2 - 语义扩展:
+  基于种子主题，API重新"编剧"出新对话
+  每条种子 → 10个变体（换角度/深度/角色）
+  速率: ~600条/h, ~7天扩展至10万+条
+
+数据格式: JSONL, 每条: {"text":"...", "domain":"...", "concept":"..."}
+"""
+import sys, os, json, time, random, hashlib, argparse, re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from api_correction import call_astron
+
+# ═══════════════════════════════════════════════════════════
+# 1. 角色定义
+# ═══════════════════════════════════════════════════════════
+
+TEACHER_SYSTEM = "你是一位AI导师，回答专业详细，包含具体数据和原理。每轮回答150-500字。"
+
+# ═══════════════════════════════════════════════════════════
+# 2. 深度学习领域图谱
+# ═══════════════════════════════════════════════════════════
+
+DOMAINS = {
+    "量子物理": ["量子叠加态","量子纠缠","量子隧穿","海森堡测不准原理","波粒二象性","量子计算","量子场论","狄拉克方程"],
+    "相对论": ["狭义相对论","质能方程","时空弯曲","引力波","黑洞物理","虫洞理论","暗物质","暗能量"],
+    "人工智能": ["深度学习原理","Transformer","强化学习","大语言模型","扩散模型","多模态学习","注意力机制","模型量化"],
+    "计算机科学": ["计算理论","操作系统","分布式系统","数据库","编译器","密码学","网络协议","计算机视觉"],
+    "分子生物学": ["DNA复制","CRISPR","中心法则","蛋白质折叠","表观遗传学","RNA干扰","免疫系统","合成生物学"],
+    "数学": ["微积分","傅里叶分析","微分方程","抽象代数","数论","拓扑学","概率论","图论"],
+    "宇宙学": ["大爆炸","恒星演化","星系形成","宇宙微波背景","暗物质粒子","暴胀模型","引力波天文学","哈勃常数危机"],
+    "神经科学": ["神经元","突触可塑性","大脑皮层","记忆机制","睡眠与意识","视觉系统","脑机接口","情感神经科学"],
+    "化学": ["化学键","反应动力学","热力学","催化原理","光谱学","纳米化学","电化学","高分子化学"],
+    "工程科技": ["航天工程","半导体","通信工程","机器人学","电力系统","材料工程","机械设计","核工程"],
+    "经济学": ["一般均衡","博弈论","信息不对称","行为经济学","货币理论","计量经济学","拍卖理论","机制设计"],
+    "哲学": ["先验哲学","权力意志","存在主义","语言游戏","证伪主义","范式理论","正义论","功利主义"],
+    "天文学": ["赫罗图","超新星","中子星","系外行星","星系分类","脉冲星","恒星形成","宇宙射线"],
+    "地球科学": ["板块构造","地震学","火山学","古生物学","气候系统","海洋环流","大气动力学","行星地质"],
+    "生物技术": ["基因测序","PCR","冷冻电镜","核磁共振","光遗传学","类器官","纳米医学","基因敲除"],
+}
+
+ALL_DOMAINS = list(DOMAINS.keys())
+
+
+# ═══════════════════════════════════════════════════════════
+# 3. Phase 1: API直接生成种子
+# ═══════════════════════════════════════════════════════════
+
+def build_seed_prompt(domain: str, concept: str) -> str:
+    return f"""请围绕【{domain}】领域的【{concept}】概念，生成一段师生深度对话。
+
+要求：
+1. 学生：从基础概念问起，逐步深入，包含追问、质疑、要求举例
+2. 导师：每轮回答150-500字，专业准确，有数据支撑
+3. 对话演进：基础定义→核心机制→实验证据→应用场景→前沿进展
+4. 至少15轮对话
+
+每轮格式（严格）：
+学生：[问题]
+导师：[回答]
+
+不要额外说明，直接开始对话。"""
+
+
+def generate_seed(domain: str, concept: str) -> dict:
+    """一次API调用生成一条种子对话"""
+    prompt = build_seed_prompt(domain, concept)
+    text = call_astron(prompt, temperature=random.uniform(0.7, 0.9),
+                       system_prompt=TEACHER_SYSTEM, max_tokens=8192)
+    if not text:
+        return None
+
+    # 解析
+    turns = []
+    for line in text.strip().split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        if re.match(r'^学生[：:]', line):
+            content = re.sub(r'^学生[：:]\s*', '', line)
+            turns.append(('user', content))
+        elif re.match(r'^导师[：:]', line):
+            content = re.sub(r'^导师[：:]\s*', '', line)
+            turns.append(('assistant', content))
+
+    if len(turns) < 10:
+        return None
+
+    lines = []
+    for role, content in turns:
+        lines.append(f"<|im_start|>{role}\n{content}<|im_end|>")
+
+    return {
+        "text": '\n'.join(lines),
+        "domain": domain,
+        "concept": concept,
+    }
+
+
+# ═══════════════════════════════════════════════════════════
+# 4. Phase 2: 语义扩展
+# ═══════════════════════════════════════════════════════════
+
+VARIATION_PROMPTS = [
+    "用更通俗的方式重新讲一遍，多用日常类比",
+    "从更高级的视角讲，假设学生已有基础，重点讲深层机制和数学原理",
+    "换个角度，从历史发展脉络讲，突出关键实验和突破性发现",
+    "从工程应用角度讲，重点讲实际怎么用、有什么技术瓶颈",
+    "从批判性视角讲，有哪些争议、局限性、未解决的问题",
+    "假设学生是高中水平，用最通俗的语言和大量类比来解释",
+    "从跨学科视角讲，这个概念与其他领域有什么联系",
+    "以问答竞赛的方式讲，学生提尖锐问题，导师用证据回击",
+]
+
+
+def generate_variation(seed: dict, seed_text: str, prompt_template: str) -> dict:
+    """基于种子主题生成一个语义变体"""
+    domain = seed["domain"]
+    concept = seed["concept"]
+    instruction = prompt_template
+
+    prompt = f"""请围绕【{domain}】领域的【{concept}】概念，生成一段师生深度对话。
+
+{instruction}
+
+要求：
+1. 至少15轮对话
+2. 导师回答150-500字，专业详细
+3. 对话要有递进：定义→机制→证据→应用→前沿
+
+每轮格式（严格）：
+学生：[问题]
+导师：[回答]
+
+直接开始对话，不要额外说明。"""
+
+    text = call_astron(prompt, temperature=random.uniform(0.75, 0.95),
+                       system_prompt=TEACHER_SYSTEM, max_tokens=8192)
+    if not text:
+        return None
+
+    turns = []
+    for line in text.strip().split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        if re.match(r'^学生[：:]', line):
+            content = re.sub(r'^学生[：:]\s*', '', line)
+            turns.append(('user', content))
+        elif re.match(r'^导师[：:]', line):
+            content = re.sub(r'^导师[：:]\s*', '', line)
+            turns.append(('assistant', content))
+
+    if len(turns) < 10:
+        return None
+
+    lines = []
+    for role, content in turns:
+        lines.append(f"<|im_start|>{role}\n{content}<|im_end|>")
+
+    return {
+        "text": '\n'.join(lines),
+        "domain": domain,
+        "concept": concept,
+        "variation": prompt_template[:30],
+    }
+
+
+# ═══════════════════════════════════════════════════════════
+# 5. 数据管理
+# ═══════════════════════════════════════════════════════════
+
+class DataStore:
+    def __init__(self, save_dir: str):
+        self.save_dir = Path(save_dir)
+        self.save_dir.mkdir(parents=True, exist_ok=True)
+        self.output_path = self.save_dir / 'dialogues.jsonl'
+        self.seed_path = self.save_dir / 'seeds.jsonl'
+
+        self.existing_hashes = set()
+        self.count = 0
+        self.load()
+
+    def load(self):
+        for path in [self.output_path, self.seed_path]:
+            if path.exists():
+                with open(path, 'r') as f:
+                    for line in f:
+                        line = line.strip()
+                        if line:
+                            try:
+                                item = json.loads(line)
+                                h = hashlib.md5(item['text'][:100].encode()).hexdigest()
+                                self.existing_hashes.add(h)
+                                self.count += 1
+                            except:
+                                pass
+        print(f"📂 已有: {self.count:,} 条")
+
+    def add_seeds(self, items: list) -> int:
+        if not items:
+            return 0
+        new = []
+        for item in items:
+            h = hashlib.md5(item['text'][:100].encode()).hexdigest()
+            if h not in self.existing_hashes:
+                self.existing_hashes.add(h)
+                new.append(item)
+        if not new:
+            return 0
+        with open(self.seed_path, 'a') as f:
+            for item in new:
+                f.write(json.dumps(item, ensure_ascii=False) + '\n')
+                self.count += 1
+        return len(new)
+
+    def add_variations(self, items: list) -> int:
+        if not items:
+            return 0
+        new = []
+        for item in items:
+            h = hashlib.md5(item['text'][:100].encode()).hexdigest()
+            if h not in self.existing_hashes:
+                self.existing_hashes.add(h)
+                new.append(item)
+        if not new:
+            return 0
+        with open(self.output_path, 'a') as f:
+            for item in new:
+                f.write(json.dumps(item, ensure_ascii=False) + '\n')
+                self.count += 1
+        return len(new)
+
+    def get_seeds(self):
+        """读取所有种子"""
+        seeds = []
+        if self.seed_path.exists():
+            with open(self.seed_path, 'r') as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        seeds.append(json.loads(line))
+        return seeds
+
+    def total_count(self):
+        return self.count
+
+
+# ═══════════════════════════════════════════════════════════
+# 6. 质量评分
+# ═══════════════════════════════════════════════════════════
+
+def score(text: str) -> float:
+    turns = (text.count('<|im_start|>user') + text.count('<|im_start|>assistant')) // 2
+    chars = len(text)
+    s = 0.0
+    if turns >= 20: s += 0.20
+    elif turns >= 12: s += 0.15
+    elif turns >= 8: s += 0.10
+    else: s += 0.05
+    if chars >= 4000: s += 0.20
+    elif chars >= 2000: s += 0.15
+    elif chars >= 1200: s += 0.10
+    else: s += 0.05
+    avg = chars / max(turns, 1)
+    if avg >= 200: s += 0.15
+    elif avg >= 100: s += 0.10
+    uc = text.count('<|im_start|>user')
+    ac = text.count('<|im_start|>assistant')
+    ratio = min(uc, ac) / max(uc, ac, 1)
+    if ratio >= 0.8: s += 0.10
+    unique = len(set(text))
+    if unique >= 150: s += 0.10
+    return max(0, min(1, s))
+
+
+# ═══════════════════════════════════════════════════════════
+# 7. 卷统计
+# ═══════════════════════════════════════════════════════════
+
+def print_stats(store: DataStore, t_start: float, phase: str, target: int):
+    elapsed = max(time.time() - t_start, 1)
+    rate = store.count / elapsed * 3600
+    eta_h = (target - store.count) / max(rate, 0.01)
+    
+    seed_count = 0
+    if store.seed_path.exists():
+        with open(store.seed_path) as f:
+            seed_count = sum(1 for _ in f if _.strip())
+    
+    tokens_est = store.count * 6000
+    
+    print(f"  [{time.strftime('%H:%M:%S')}] "
+          f"Phase {phase} | "
+          f"累计: {store.count:,}/{target:,} "
+          f"({store.count/max(target,1)*100:.1f}%) | "
+          f"速率: {rate:.0f}条/h | "
+          f"种子: {seed_count} | "
+          f"tokens: {tokens_est/1e9:.2f}B | "
+          f"预计: {eta_h/24:.1f}d", flush=True)
+
+
+# ═══════════════════════════════════════════════════════════
+# 8. 主函数
+# ═══════════════════════════════════════════════════════════
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target", type=int, default=100000, help="目标条数")
+    parser.add_argument("--phase", type=int, default=1, choices=[1, 2],
+                        help="1=生成种子, 2=语义扩展")
+    parser.add_argument("--concurrency", type=int, default=5, help="API并发数")
+    args = parser.parse_args()
+
+    save_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'pretrain_data')
+    store = DataStore(save_dir)
+
+    target = args.target
+    if store.count >= target:
+        print(f"✅ 已达目标 {target:,} 条")
+        return
+
+    print(f"\n{'='*50}")
+    print(f"  Phase {args.phase}: {'种子生成' if args.phase == 1 else '语义扩展'}")
+    print(f"  目标: {target:,} 条")
+    print(f"  并发: {args.concurrency}")
+    print(f"  当前: {store.count:,}")
+    print(f"{'='*50}\n")
+
+    t_start = time.time()
+
+    if args.phase == 1:
+        # Phase 1: 纯API种子生成
+        seeds = store.get_seeds()
+        # 从种子中的(domain, concept)对去重
+        used = set()
+        for s in seeds:
+            used.add((s.get("domain",""), s.get("concept","")))
+        print(f"  已有 {len(used)} 个不同(领域,概念)种子\n")
+
+        while store.count < target:
+            batch_items = []
+            tasks = []
+            for _ in range(args.concurrency):
+                # 选一个未用过的(领域,概念)
+                for attempt in range(20):
+                    d = random.choice(ALL_DOMAINS)
+                    c = random.choice(DOMAINS[d])
+                    if (d, c) not in used or attempt > 15:
+                        break
+                used.add((d, c))
+                tasks.append((d, c))
+
+            with ThreadPoolExecutor(max_workers=args.concurrency) as ex:
+                futures = {ex.submit(generate_seed, d, c): (d, c) for d, c in tasks}
+                for f in as_completed(futures):
+                    result = f.result()
+                    if result and score(result["text"]) >= 0.5:
+                        batch_items.append(result)
+
+            if batch_items:
+                added = store.add_seeds(batch_items)
+                n = len(batch_items)
+                print(f"  +{n}条种子 | 领域: {batch_items[0]['domain']}, "
+                      f"概念: {batch_items[0]['concept']}...", flush=True)
+
+            print_stats(store, t_start, "1", target)
+
+            if store.count >= target:
+                break
+
+    elif args.phase == 2:
+        # Phase 2: 语义扩展
+        seeds = store.get_seeds()
+        if not seeds:
+            print("❌ 没有种子数据，先跑 Phase 1")
+            return
+        print(f"  种子库: {len(seeds)} 条\n")
+
+        used_seeds = set()
+        expansion_tasks = []
+
+        while store.count < target:
+            if not expansion_tasks:
+                # 把所有种子打乱，生成扩展任务
+                random.shuffle(seeds)
+                for s in seeds:
+                    if s["text"] in used_seeds:
+                        continue
+                    used_seeds.add(s["text"])
+                    for vp in VARIATION_PROMPTS:
+                        expansion_tasks.append((s, vp))
+                print(f"  生成 {len(expansion_tasks)} 个扩展任务\n")
+
+            # 取一批任务
+            batch = []
+            while len(batch) < args.concurrency and expansion_tasks:
+                batch.append(expansion_tasks.pop(0))
+
+            if not batch:
+                break
+
+            with ThreadPoolExecutor(max_workers=args.concurrency) as ex:
+                futures = {ex.submit(generate_variation, s, s["text"], vp): (s, vp)
+                          for s, vp in batch}
+                for f in as_completed(futures):
+                    result = f.result()
+                    if result and score(result["text"]) >= 0.5:
+                        store.add_variations([result])
+
+            # 报告
+            seed_count = 0
+            if store.seed_path.exists():
+                with open(store.seed_path) as f:
+                    seed_count = sum(1 for _ in f if _.strip())
+            elapsed = max(time.time() - t_start, 1)
+            rate = store.count / elapsed * 3600
+            eta_h = (target - store.count) / max(rate, 0.01) if rate > 0 else 0
+            tokens_est = store.count * 6000
+            done_tasks = max(1, len(used_seeds) * len(VARIATION_PROMPTS) - len(expansion_tasks))
+            print(f"  [{time.strftime('%H:%M:%S')}] "
+                  f"累计: {store.count:,}/{target:,} "
+                  f"({store.count/max(target,1)*100:.1f}%) | "
+                  f"速率: {rate:.0f}条/h | "
+                  f"种子: {seed_count} | "
+                  f"扩展任务进度: {done_tasks} | "
+                  f"tokens: {tokens_est/1e9:.2f}B | "
+                  f"预计: {eta_h/24:.1f}d", flush=True)
+
+            if store.count >= target:
+                break
+
+    # 最终报告
+    total_time = time.time() - t_start
+    final_rate = store.count / max(total_time, 1) * 3600
+    tokens = store.count * 6000
+    path = store.output_path if args.phase == 2 else store.seed_path
+    size_mb = os.path.getsize(path) / 1024 / 1024 if path.exists() else 0
+
+    print(f"\n{'='*50}")
+    print(f"  ✅ Phase {args.phase} 完成!")
+    print(f"  总条数:     {store.count:,}")
+    print(f"  文件:       {path}")
+    print(f"  大小:       {size_mb:.1f}MB")
+    print(f"  估算tokens: {tokens/1e9:.2f}B")
+    print(f"  耗时:       {total_time/3600:.1f}h")
+    print(f"  平均速率:   {final_rate:.0f}条/h")
+    print(f"{'='*50}")
+
+    # 样例
+    print("\n📊 样例:")
+    with open(path) as f:
+        for i, line in enumerate(f):
+            if i >= 2:
+                break
+            if line.strip():
+                item = json.loads(line.strip())
+                text = item["text"]
+                turns = text.count("<|im_start|>") // 2
+                print(f"\n  条目{i+1}: {turns}轮 | {len(text):,}字 | {item.get('domain','')}")
+                lines = text.split("\n")[:4]
+                for l in lines:
+                    if l.strip():
+                        role = "👤" if "user" in l else "🧑‍🏫"
+                        c = l.split("<|im_start|>")[1].split("<|im_end|>")[0].strip()
+                        print(f"    {role} {c[:100]}")
+                print(f"    ...（{turns}轮）")
+
+
+if __name__ == "__main__":
+    main()

llm_chat_improved.py

@@ -7,6 +7,7 @@ LLM对话AI训练代码 - 改进版
 
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
 import torch.optim as optim
 from torch.utils.data import Dataset, DataLoader
 import math
@@ -235,21 +236,13 @@ class Attention(nn.Module):
         xk = xk.transpose(1, 2)
         xv = xv.transpose(1, 2)
         
-        # 计算注意力分数
-        scores = torch.matmul(xq, xk.transpose(-2, -1)) / math.sqrt(self.d_head)
-        
-        # 应用掩码
-        if mask is not None:
-            scores = scores.masked_fill(mask == 0, float('-inf'))
-        
-        # Softmax
-        attn_weights = torch.softmax(scores, dim=-1)
-        attn_weights = self.dropout(attn_weights)
-        
-        # 注意力输出
-        output = torch.matmul(attn_weights, xv)
+        # 融合注意力（SDPA / FlashAttention-2）- 纯因果掩码，等价于原下三角 mask
+        dropout_p = self.dropout.p if self.training else 0.0
+        output = F.scaled_dot_product_attention(
+            xq, xk, xv, attn_mask=None, dropout_p=dropout_p, is_causal=True)
+
         output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
-        
+
         return self.wo(output)
 
 
@@ -580,11 +573,14 @@ class ConversationDataset(Dataset):
 class LLMTrainer:
     """LLM训练器"""
     
-    def __init__(self, model: nn.Module, device: str = 'cpu', use_amp: bool = None):
+    def __init__(self, model: nn.Module, device: str = 'cpu', lr: float = 0.0001, use_amp: bool = None):
         self.model = model.to(device)
         self.device = device
+        # 火力全开：TF32 加速 fp32 矩阵乘法（L40/A100/H100 原生支持）
+        torch.backends.cuda.matmul.allow_tf32 = True
+        torch.backends.cudnn.allow_tf32 = True
         self.criterion = nn.CrossEntropyLoss(ignore_index=0)
-        self.optimizer = optim.AdamW(model.parameters(), lr=0.0001, weight_decay=0.01)
+        self.optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
         self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=100)
         
         # 混合精度训练
@@ -618,7 +614,9 @@ class LLMTrainer:
             
             # 生成因果掩码
             mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
-            
+
+            # CUDA graphs step 边界标记（torch.compile reduce-overhead 必需，避免输出 buffer 被覆盖）
+            torch.compiler.cudagraph_mark_step_begin()
             # 前向传播
             if self.use_amp:
                 with torch.cuda.amp.autocast():
@@ -677,30 +675,35 @@ class LLMTrainer:
                 target_ids = batch['target_ids'].to(self.device, non_blocking=True)
                 
                 mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
+                torch.compiler.cudagraph_mark_step_begin()
                 outputs = self.model(input_ids, mask)
                 outputs = outputs.reshape(-1, outputs.size(-1))
                 tgt = target_ids.reshape(-1)
                 
                 loss = self.criterion(outputs, tgt)
                 total_loss += loss.item()
-        
+
+        if len(dataloader) == 0:
+            return float('inf')
         return total_loss / len(dataloader)
     
     def save_model(self, filepath: str, vocab: dict, epoch: int = None):
+        # 兼容 torch.compile：取原始（未编译）模块，避免 state_dict 带 _orig_mod. 前缀
+        core = getattr(self.model, '_orig_mod', self.model)
         save_dict = {
-            'model_state_dict': self.model.state_dict(),
+            'model_state_dict': core.state_dict(),
             'optimizer_state_dict': self.optimizer.state_dict(),
             'vocab': vocab,
             'model_config': {
-                'vocab_size': self.model.vocab_size,
-                'd_model': self.model.d_model,
-                'n_heads': self.model.n_heads,
-                'n_layers': self.model.n_layers,
-                'd_ff': self.model.d_ff,
-                'max_len': self.model.max_len,
-                'use_memory': self.model.use_memory,
-                'use_thinking': self.model.use_thinking,
-                'use_moe': self.model.use_moe
+                'vocab_size': core.vocab_size,
+                'd_model': core.d_model,
+                'n_heads': core.n_heads,
+                'n_layers': core.n_layers,
+                'd_ff': core.d_ff,
+                'max_len': core.max_len,
+                'use_memory': core.use_memory,
+                'use_thinking': core.use_thinking,
+                'use_moe': core.use_moe
             }
         }
         if epoch is not None:
@@ -711,7 +714,8 @@ class LLMTrainer:
     
     def load_model(self, filepath: str):
         checkpoint = torch.load(filepath, map_location=self.device)
-        self.model.load_state_dict(checkpoint['model_state_dict'])
+        core = getattr(self.model, '_orig_mod', self.model)
+        core.load_state_dict(checkpoint['model_state_dict'])
         self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
         vocab = checkpoint['vocab']
         print(f"模型已从 {filepath} 加载")

scripts/api_correction.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-API生成模块 — Astron 5并发，稳定不超限
+API生成模块 — Astron 5并发（同步根目录版本）
 """
-import json, requests, time, os, threading
+import json, requests, time, os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
@@ -49,17 +49,12 @@ def call_astron(q, temperature=0.85):
     return None
 
 
-def batch_generate(questions, temperature=0.85, max_workers=4):
-    """Astron 4并发 — 明确限制，稳定可靠"""
+def batch_generate(questions, temperature=0.85, max_workers=5):
     total = len(questions)
     results = []
-    
     def call(q):
         ans = call_astron(q, temperature)
-        if ans:
-            return q, ans
-        return None
-    
+        return (q, ans) if ans else None
     with ThreadPoolExecutor(max_workers=max_workers) as executor:
         futures = {executor.submit(call, item): i for i, item in enumerate(questions)}
         done = 0
@@ -73,27 +68,81 @@ def batch_generate(questions, temperature=0.85, max_workers=4):
                 elapsed = max(time.time() - t0, 1)
                 rate = len(results) / elapsed * 60
                 print(f"    Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r")
-    
     elapsed = time.time() - t0
     rate = len(results) / max(elapsed, 1) * 60
     print(f"    Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
     return results
 
 
-# 保持旧函数名兼容
 batch_generate_astron = batch_generate
 batch_generate_mimo = batch_generate
 batch_generate_direct = batch_generate
 batch_generate_dual = batch_generate
 
 
-def batch_convert(items, *args, **kwargs):
-    """兼容占位：训练脚本导入用，已弃用"""
-    print("  [API] batch_convert已弃用，使用batch_generate")
-    return []
+def batch_convert(items, progress=None, max_workers=5):
+    if not items:
+        return []
+    total = len(items)
+    results = []
+    def convert(item):
+        q, model_ans = item
+        prompt = f"问题：{q}\n模型回答：{model_ans}\n\n请用8岁小狸的语气，重新生成更好的回答。保持简短自然，1-2句话。"
+        ans = call_astron(prompt, temperature=0.3)
+        return {"question": q, "answer": ans} if ans and len(ans) >= 3 else None
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(convert, item): i for i, item in enumerate(items)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append(r)
+            if progress:
+                progress(done, total)
+            if done % 50 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    API转换: {done}/{total} (+{len(results)}条, {rate:.0f}条/分)", end="\r")
+    elapsed = time.time() - t0
+    rate = len(results) / max(elapsed, 1) * 60
+    print(f"    API转换: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
+    return results
 
 
-def batch_generate_multi_temp(questions, *args, **kwargs):
-    """兼容占位：训练脚本导入用，已弃用"""
-    print("  [API] batch_generate_multi_temp已弃用，使用batch_generate")
-    return []
+def batch_generate_multi_temp(questions, temperatures=None, max_workers=5):
+    if temperatures is None:
+        temperatures = [0.7, 0.85, 1.0]
+    if not questions:
+        return []
+    total = len(questions) * len(temperatures)
+    results = []
+    def call_with_temp(q, temp):
+        ans = call_astron(q, temperature=temp)
+        return {"question": q, "answer": ans, "temperature": temp} if ans else None
+    tasks = [(q, t) for q in questions for t in temperatures]
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(call_with_temp, q, t): i for i, (q, t) in enumerate(tasks)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append(r)
+            if done % 50 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    多温生成: {done}/{total} (+{len(results)}条, {rate:.0f}条/分)", end="\r")
+    seen = set()
+    unique = []
+    for item in results:
+        key = (item["question"], item["answer"])
+        if key not in seen:
+            seen.add(key)
+            unique.append({"question": item["question"], "answer": item["answer"]})
+    elapsed = time.time() - t0
+    rate = len(unique) / max(elapsed, 1) * 60
+    print(f"    多温生成: 共{len(unique)}条 (去重后, {elapsed:.0f}s, {rate:.0f}条/分)")
+    return unique

scripts/generate_massive_general.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+海量通用数据蒸馏 v3 — 不设场景标签，纯自然对话，多策略并行
+目标：500万+ 纯 (question, answer) 对
+"""
+import json, requests, time, os, sys, random, re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+OUT_DIR = os.path.join(MODEL_DIR, 'general_outputs')
+os.makedirs(OUT_DIR, exist_ok=True)
+
+# ====== Astron 配置 ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+# ====== 主题全覆盖（六大类 × 子类） ======
+TOPICS = {
+    "日常": [
+        "起床洗漱", "吃早餐", "上学路上", "上课", "课间休息",
+        "午餐时间", "放学回家", "做作业", "晚饭", "睡前时光",
+        "周末活动", "天气变化", "穿衣打扮", "零花钱", "家务劳动",
+        "看电视", "玩玩具", "去公园", "逛超市", "去亲戚家",
+    ],
+    "情绪": [
+        "开心高兴", "难过伤心", "生气愤怒", "害怕恐惧", "紧张焦虑",
+        "孤独寂寞", "委屈冤枉", "失望沮丧", "感动温暖", "兴奋激动",
+        "无聊烦闷", "尴尬害羞", "后悔内疚", "羡慕嫉妒", "感激感恩",
+    ],
+    "社交": [
+        "交新朋友", "和好朋友玩", "和同学相处", "和老师交流", "和兄弟姐妹",
+        "和爸爸妈妈", "和爷爷奶奶", "和邻居", "和陌生人", "和喜欢的人",
+        "吵架和好", "被欺负", "被误解", "被表扬", "被批评",
+        "帮助别人", "请求帮助", "分享东西", "邀请做客", "告别分开",
+    ],
+    "知识": [
+        "动物世界", "植物花草", "天文星空", "天气自然", "人体健康",
+        "食物营养", "颜色形状", "数字计算", "汉字词语", "科学现象",
+        "历史故事", "地理知识", "交通工具", "乐器音乐", "绘画手工",
+        "运动体育", "节日习俗", "各国文化", "发明创造", "宇宙太空",
+    ],
+    "想象": [
+        "超能力", "魔法世界", "未来科技", "外星人", "恐龙时代",
+        "童话故事", "变成动物", "飞行梦", "海底世界", "神秘岛屿",
+        "时间旅行", "超级英雄", "会说话的玩具", "隐形衣", "变身药水",
+        "机器人朋友", "彩虹桥", "星星糖果", "云朵房子", "梦中冒险",
+    ],
+    "成长": [
+        "学新本领", "克服困难", "不怕失败", "坚持努力", "自我肯定",
+        "梦想未来", "职业理想", "好习惯", "改掉坏毛病", "学会分享",
+        "学会等待", "学会道歉", "学会感恩", "学会勇敢", "学会理解",
+        "交心谈话", "秘密分享", "烦恼倾诉", "人生道理", "温暖鼓励",
+    ],
+}
+
+FLAT_TOPICS = []
+for cat, subs in TOPICS.items():
+    for sub in subs:
+        FLAT_TOPICS.append((cat, sub))
+
+# ====== 多样化的开场方式 ======
+OPENERS = [
+    "小狸", "嗨小狸", "小狸在吗", "嘿嘿", "那个…", "喂喂喂",
+    "小狸小狸", "我想问你", "你知道吗", "你说", "我突然想到",
+    "我跟你讲", "小狸你听我说", "问你个事", "我有个问题",
+    "好无聊啊", "好开心呀", "好烦", "怎么办呀",
+    "小狸你在干嘛", "我来了", "出来玩", "跟你说个事",
+]
+
+# ====== 策略1: 直接生成纯问答对 ======
+def make_direct_pairs_prompt(count=15):
+    """无场景标签，直接生成纯问答对"""
+    # 随机选3-5个主题混搭
+    selected = random.sample(FLAT_TOPICS, min(random.randint(3, 5), len(FLAT_TOPICS)))
+    topics_str = "、".join([f"{c}({s})" for c, s in selected])
+    
+    prompt = f"""请直接生成 {count} 条聊天问答对。
+
+格式要求（严格按此格式）：
+问：xxxxxxxxx | 答：xxxxxxxxx
+
+用户的问题要：口语化、自然、像真人（5-25字）
+小狸的回答要：8岁小女孩语气，可爱活泼（10-40字）
+话题要多样：{topics_str}
+
+示例：
+问：你怕不怕黑呀？ | 答：有一点点…但是抱着小熊玩偶就不怕啦！
+问：我今天跑步摔了一跤 | 答：疼不疼呀？我给你吹吹就不疼了！
+
+现在生成 {count} 条："""
+    return prompt
+
+# ====== 策略2: 短对话（3-5轮） ======
+def make_mini_chat_prompt(count=8):
+    """微型多轮对话，每段3-5轮"""
+    ops = random.sample(OPENERS, min(count, len(OPENERS)))
+    
+    prompt = f"""请生成 {count} 段微型聊天对话，每段3-5轮（一来一回算1轮）。
+
+要求：
+- 每段一个独立话题，话题要多样（日常、情绪、知识、想象、社交、成长等）
+- 用户说话要特别自然、口语化，像真人随口说的
+- 小狸像8岁小女孩，回答要可爱、有画面感
+- 话题要自然延伸，不要生硬转折
+
+格式例子：
+话题：日常
+用户：今天好冷啊
+小狸：是呀！我穿了厚厚的羽绒服，像只小企鹅一样～
+用户：哈哈哈真的好像
+小狸：（鼓起腮帮子）才不像呢！企鹅走路摇摇摆摆的，我走路可稳啦！
+...
+
+现在生成 {count} 段，每段用"话题：xxx"开头："""
+    return prompt
+
+# ====== 策略3: 长对话（6-10轮自由展开） ======
+def make_long_chat_prompt():
+    """一场长对话，话题自然展开"""
+    topic = random.choice(FLAT_TOPICS)
+    opener = random.choice(OPENERS)
+    
+    emotions = ["开心", "难过", "兴奋", "沮丧", "好奇", "担心", "轻松", "紧张"]
+    emotion = random.choice(emotions)
+    
+    prompt = f"""请生成一段自然的日常聊天，{topic[0]}类-{topic[1]}话题。
+
+用户开场：{opener}
+用户当下的心情：{emotion}
+
+要求：
+- 用户很自然地说出开场白，就像真人发微信一样
+- 小狸正常回复，话题慢慢展开，不要硬聊
+- 聊6-10轮（一问一答算一轮）
+- 每轮格式：用户：xxx\n小狸：xxx
+- 真实感第一，不要刻意
+
+请直接输出对话，不要加说明："""
+    return prompt
+
+# ====== 策略4: 单轮问答（高吞吐） ======
+def make_single_turn_prompt(batch_size=20):
+    """纯单轮问答，批量生成"""
+    selected = random.sample(FLAT_TOPICS, min(6, len(FLAT_TOPICS)))
+    
+    prompt = f"""生成 {batch_size} 条日常问答对。
+
+要求：
+1. 用户的话：7-25字，口语化，真实感强
+2. 小狸回答：1-2句，8岁小女孩语气，可爱活泼
+3. 覆盖多样化话题：{', '.join([s for _, s in selected[:6]])}（但不仅限于这些）
+4. 每行一条，格式：问: ... | 答: ...
+5. 不要重复相同的问题
+
+示例：
+问: 你喜不喜欢下雨天呀？ | 答: 喜欢！因为可以踩水坑，啪嗒啪嗒的超好玩！
+问: 我今天被同学起外号了 | 答: （歪着头）那…你喜欢这个外号吗？不喜欢的话要大声说出来哦！
+问: 你说世界上有外星人吗？ | 答: 我觉得有！说不定他们正在用望远镜看我们呢～
+
+生成："""
+    return prompt
+
+# ====== API调用 ======
+def call_astron(prompt, temp=0.82, max_tokens=4096):
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": "你是一个擅长写日常对话的助手。小狸是一个8岁的小女孩，说话可爱自然。"},
+                    {"role": "user", "content": prompt}
+                ],
+                "temperature": temp,
+                "max_tokens": max_tokens,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=150)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 20:
+                    return content
+            else:
+                time.sleep(0.5 + attempt)
+        except Exception:
+            time.sleep(1 + attempt)
+    return None
+
+# ====== 解析 ======
+def parse_pairs(text):
+    """从"问：xxx | 答：xxx"格式解析问答对"""
+    pairs = []
+    for line in text.split('\n'):
+        line = line.strip()
+        parts = re.split(r'\s*[|｜]\s*', line)
+        if len(parts) == 2:
+            q = re.sub(r'^问\s*[：:]\s*', '', parts[0]).strip()
+            a = re.sub(r'^答\s*[：:]\s*', '', parts[1]).strip()
+            if q and a and len(q) >= 2 and len(a) >= 2 and len(a) < 300 and len(q) < 200:
+                pairs.append((q, a))
+    return pairs
+
+def parse_mini_chats(text):
+    """解析微型多轮对话"""
+    pairs = []
+    segments = re.split(r'话题[：:].*?\n', text)
+    
+    for seg in segments:
+        seg = seg.strip()
+        lines = seg.split('\n')
+        q = None
+        for line in lines:
+            line = line.strip()
+            if not line:
+                continue
+            q_match = re.match(r'^(用户|我)[：:]\s*(.*)', line)
+            a_match = re.match(r'^(小狸|答)[：:]\s*(.*)', line)
+            
+            if q_match:
+                q_text = q_match.group(2).strip()
+                if q_text and len(q_text) > 1:
+                    q = q_text
+            elif a_match and q:
+                a_text = a_match.group(2).strip()
+                if a_text and len(a_text) > 1 and len(a_text) < 300:
+                    pairs.append((q, a_text))
+                    q = None
+    return pairs
+
+def parse_long_chat(text):
+    """解析长对话"""
+    pairs = []
+    q = None
+    for line in text.split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        q_match = re.match(r'^(用户|我)[：:]\s*(.*)', line)
+        a_match = re.match(r'^(小狸)[：:]\s*(.*)', line)
+        
+        if q_match:
+            q_text = q_match.group(2).strip()
+            if q_text and len(q_text) > 1:
+                q = q_text
+        elif a_match and q:
+            a_text = a_match.group(2).strip()
+            if a_text and len(a_text) > 1 and len(a_text) < 300:
+                pairs.append((q, a_text))
+                q = None
+    return pairs
+
+# ====== 生成器（按策略分发） ======
+GENERATORS = [
+    # (name, prompt_fn, parse_fn, batch_size_estimate, weight_for_diversity)
+    ("direct", make_direct_pairs_prompt, parse_pairs, 15, 35),
+    ("mini_chat", make_mini_chat_prompt, parse_mini_chats, 20, 25),
+    ("long_chat", make_long_chat_prompt, parse_long_chat, 20, 20),
+    ("single", make_single_turn_prompt, parse_pairs, 20, 20),
+]
+
+def run_one_generation():
+    """运行一次生成任务（随机选一种策略）"""
+    # 按权重选策略
+    weights = [w for _, _, _, _, w in GENERATORS]
+    gen = random.choices(GENERATORS, weights=weights, k=1)[0]
+    name, prompt_fn, parse_fn, batch_est, _ = gen
+    
+    prompt = prompt_fn()
+    result = call_astron(prompt, temp=random.uniform(0.75, 0.92), max_tokens=4096 if 'long' not in name else 6144)
+    if not result:
+        return None
+    
+    pairs = parse_fn(result)
+    if not pairs:
+        return None
+    
+    return pairs
+
+def main():
+    print("=" * 65)
+    print("  海量通用数据蒸馏 v3")
+    print("  不设场景标签 | 6大类120+子话题 | 4种策略")
+    print("=" * 65)
+    
+    # 加载已有进度
+    all_pairs = []
+    existing_keys = set()
+    progress_file = os.path.join(OUT_DIR, 'all_general_progress.json')
+    if os.path.exists(progress_file):
+        existing = json.load(open(progress_file, encoding='utf-8'))
+        all_pairs = existing
+        for p in existing:
+            existing_keys.add((p.get('question',''), p.get('answer','')))
+        print(f"已有进度: {len(all_pairs):,} 对")
+    
+    # ====== 参数 ======
+    TARGET = 2000000       # 目标200万对
+    BATCH = 40             # 每批40个任务
+    MAX_WORKERS = 8        # 8并发（比之前5更猛）
+    CHECKPOINT = 5000      # 每5000对保存
+    
+    to_generate = TARGET - len(all_pairs)
+    total_batches = (to_generate + BATCH - 1) // BATCH
+    
+    print(f"目标: {TARGET:,} 对")
+    print(f"已有: {len(all_pairs):,} 对")
+    print(f"还需: {to_generate:,} 对")
+    print(f"批次: {total_batches:,} 批 ({BATCH}任务/批, {MAX_WORKERS}并发)")
+    
+    # 策略统计
+    strategy_stats = {g[0]: {'runs': 0, 'pairs': 0} for g in GENERATORS}
+    
+    t_start = time.time()
+    last_checkpoint = len(all_pairs)
+    
+    for batch_idx in range(total_batches):
+        t0 = time.time()
+        
+        # 并发
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = [executor.submit(run_one_generation) for _ in range(BATCH)]
+            batch_pairs = []
+            for future in as_completed(futures):
+                try:
+                    r = future.result()
+                    if r:
+                        batch_pairs.extend(r)
+                except:
+                    pass
+        
+        # 去重
+        new_pairs = []
+        for q, a in batch_pairs:
+            key = (q, a)
+            if key not in existing_keys:
+                existing_keys.add(key)
+                new_pairs.append({'question': q, 'answer': a})
+        
+        all_pairs.extend(new_pairs)
+        
+        elapsed = time.time() - t0
+        total_elapsed = time.time() - t_start
+        rate = len(all_pairs) / max(total_elapsed, 1) * 60
+        
+        # 估算
+        pairs_this_batch = len(new_pairs)
+        total_raw = len(batch_pairs)
+        dedup_rate = pairs_this_batch / max(total_raw, 1) * 100
+        
+        print(f"[{batch_idx+1}/{total_batches}] "
+              f"+{pairs_this_batch}对(有效{total_raw}/{dedup_rate:.0f}%) "
+              f"累计{len(all_pairs):,}/{TARGET:,} "
+              f"{rate:.0f}对/分 "
+              f"剩余{(TARGET-len(all_pairs))/max(rate,0.1):.0f}分")
+        
+        # 定期保存
+        if len(all_pairs) - last_checkpoint >= CHECKPOINT:
+            with open(progress_file, 'w', encoding='utf-8') as f:
+                json.dump(all_pairs, f, ensure_ascii=False)
+            last_checkpoint = len(all_pairs)
+            
+            # 也保存到数据目录一份
+            merged_path = os.path.join(MODEL_DIR, 'data', 'general_ongoing.json')
+            with open(merged_path, 'w', encoding='utf-8') as f:
+                json.dump(all_pairs, f, ensure_ascii=False)
+            
+            # 速率统计
+            stable_rate = len(all_pairs) / max(time.time() - t_start, 1) * 60
+            print(f"  💾 保存 ({len(all_pairs):,} 对, {stable_rate:.0f}对/分)")
+    
+    # ====== 完成 ======
+    total_elapsed = time.time() - t_start
+    
+    with open(progress_file, 'w', encoding='utf-8') as f:
+        json.dump(all_pairs, f, ensure_ascii=False)
+    
+    print(f"\n{'='*65}")
+    print(f"✅ 生成完成!")
+    print(f"  总对数: {len(all_pairs):,}")
+    print(f"  用时: {total_elapsed/60:.1f} 分")
+    print(f"  平均速率: {len(all_pairs)/max(total_elapsed,1)*60:.0f} 对/分")
+    print(f"  保存: {progress_file}")
+    print(f"{'='*65}")
+
+if __name__ == '__main__':
+    main()

scripts/generate_real_multiturn.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+海量真实多轮对话生成 — 模拟真实用户，8~15轮长对话
+"""
+import json, requests, time, os, sys, random, re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+from scripts.api_correction import batch_generate
+
+MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# ====== Astron 配置（复用 api_correction 的 key） ======
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+# ====== 用户场景画像（模拟真实用户） ======
+USER_SCENARIOS = [
+    # 小小孩子（3-5岁）
+    {"age": "3-5岁", "style": "说话简单，喜欢问为什么，爱说叠词", "tone": "奶声奶气"},
+    # 小孩子（6-9岁）
+    {"age": "6-9岁", "style": "小学生，爱分享学校的事，喜欢动画片和游戏", "tone": "活泼天真"},
+    # 大孩子（10-14岁）
+    {"age": "10-14岁", "style": "有点小叛逆，有自己的想法，会吐槽", "tone": "有点酷"},
+    # 妈妈/家长
+    {"age": "成年", "style": "关心孩子，会问小狸吃饭睡觉学习的事", "tone": "温柔耐心"},
+    # 孤单的人
+    {"age": "成年", "style": "有点孤独，需要陪伴，会把小狸当倾诉对象", "tone": "低落温柔"},
+    # 高中生/大学生
+    {"age": "16-22岁", "style": "有压力，会聊学业、未来、人际关系", "tone": "成熟但需要安慰"},
+    # 好奇宝宝
+    {"age": "不限", "style": "什么都想知道，十万个为什么", "tone": "充满好奇"},
+    # 伤心的人
+    {"age": "不限", "style": "心情不好，需要安慰和陪伴", "tone": "低沉脆弱"},
+]
+
+# ====== 真实对话种子（更自然的开场） ======
+REAL_SEEDS = [
+    # 日常打招呼（多种自然说法）
+    "小狸你在吗", "嘿嘿小狸", "小狸小狸快出来", "小狸你在干嘛呀",
+    "我来找你玩啦", "小狸今天乖不乖", "你在做什么呢",
+    # 情绪分享（真实用户会说的话）
+    "我今天好不开心", "我开心死了哈哈哈", "我好无聊啊没人陪我",
+    "今天倒霉死了", "我今天遇到了好事", "我有点难过",
+    "我心情不好你能陪我吗", "我好想你啊", "我好孤单",
+    "我害怕", "我有点紧张明天要考试", "我好烦",
+    # 日常分享
+    "今天我妈做了红烧肉", "我们家养了一只猫超可爱", "我今天考试考了98分",
+    "我感冒了好难受", "我今天被老师表扬了", "我和朋友吵架了",
+    "我昨天做梦梦到你了", "我学会骑自行车了！", "我今天摔了一跤好疼",
+    "学校要考试了好紧张", "放假了好开心", "又要开学了好痛苦",
+    "今天下雨了不能出去玩", "今天天气好好我们去玩吧",
+    "我妈妈给我买了个新玩具", "我养的花开了",
+    # 求助/询问
+    "我不小心把杯子打碎了怎么办", "我作业好多写不完",
+    "我睡不着你能陪我聊天吗", "我牙好疼",
+    "妈妈骂我了", "我被老师批评了",
+    "我迷路了怎么办", "我找不到我的水杯了",
+    # 好奇/知识
+    "为什么天是蓝的呀", "鱼会睡觉吗", "为什么会有影子",
+    "星星为什么会闪", "海为什么是咸的", "为什么要上学",
+    "为什么大人总说不行", "什么是爱", "什么是长大",
+    # 表达好感的
+    "小狸我最喜欢你了", "你好可爱呀", "我们是最好的朋友对吧",
+    "我送给你一个礼物", "小狸你真聪明", "你真好",
+    # 关于小狸本身的
+    "你几岁啦", "你喜欢什么", "你有爸爸妈妈吗",
+    "你住在哪里呀", "你喜欢吃什么", "你最好的朋友是谁",
+    "你上学吗", "你的梦想是什么", "你平时喜欢做什么",
+    # 奇怪/无厘头
+    "如果我能飞就好了", "我想变成一只猫", "你觉得外星人存在吗",
+    "假如你会魔法你想做什么", "你相信世界上有奥特曼吗",
+    # 夜间/睡前
+    "我睡不着", "陪我聊到睡着好不好", "给我讲个睡前故事",
+    "晚安前陪我说说话", "我怕黑",
+    # 亲情家庭
+    "我爸妈吵架了怎么办", "我想我奶奶了", "我好久没见爸爸了",
+    "妈妈生病了", "我弟弟好烦总是抢我东西",
+    # 节假日
+    "过年啦新年快乐", "今天是我生日", "圣诞节到了",
+    "六一儿童节快乐", "明天要去春游好开心",
+]
+
+CONTEXTS = [
+    "晚上睡前躺在床上", "放学回家的路上",
+    "周末在家没事做", "刚吃完饭坐着聊天",
+    "在公园里坐着", "下雨天窝在家里",
+    "一个人在房间", "在院子里玩的时候",
+    "刚写完作业休息", "早上起床的时候",
+    "在操场上看别人玩", "坐车去玩的路上",
+    "在图书馆里", "在外面吃饭等上菜",
+    "一个人发呆的时候", "看着窗外的时候",
+]
+
+def make_realistic_multi_prompt():
+    """构造非常逼真的长多轮对话提示 — 一次生成4条"""
+    parts = []
+    for _ in range(4):
+        scenario = random.choice(USER_SCENARIOS)
+        opener = random.choice(REAL_SEEDS)
+        context = random.choice(CONTEXTS)
+        min_turns = random.randint(6, 8)
+        max_turns = random.randint(10, 14)
+        
+        parts.append(f"""=== 对话 {_+1} ===
+场景：{context}
+用户：{scenario['age']}，{scenario['style']}，{scenario['tone']}
+开场：{opener}
+对话：
+用户：{opener}
+小狸：""")
+    
+    prompt = f"""请生成 {len(parts)} 段独立的日常聊天对话，每段 {min_turns}~{max_turns} 轮。
+
+重要：每段对话必须是一个完整的聊天，话题自然展开，不要强行结束。
+
+{"".join(parts)}
+
+严格要求：
+1. 每个=== 对话 ===之后的段落实例必须写完整，不要留空
+2. 用户提问必须口语化、自然，像真人在说话
+3. 小狸回答1-2句话，像8岁小朋友
+4. 话题要延续上一轮，逐渐深入延伸
+5. 用户要有真实反应（惊讶、追问、分享等）
+6. 用"问："/"答："格式
+7. 输出不能省略任何一轮
+"""
+    return prompt
+
+def call_astron_multi(prompt, temperature=0.85):
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": "你是一个擅长写真实对话的助手。"},
+                    {"role": "user", "content": prompt}
+                ],
+                "temperature": temperature,
+                "max_tokens": 8192,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=120)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content and len(content) > 50:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+def parse_multi_turn(text):
+    """解析多段对话（支持=== 对话 N ===分隔，也支持纯轮次）"""
+    lines = text.strip().split('\n')
+    conversations = []
+    current_q = None
+    current_turns = []
+    
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        
+        # 检测新对话开始标记
+        if re.match(r'===?\s*对话\s*\d+\s*===?', line):
+            if current_turns and len(current_turns) >= 4:
+                conversations.append(current_turns)
+            current_turns = []
+            current_q = None
+            continue
+        
+        # 跳过设定行（场景/用户/开场）
+        if line.startswith(('场景', '用户', '开场', '对话', '===')):
+            if line.startswith(('场景', '用户', '开场')):
+                continue
+            continue
+        
+        # 用户提问
+        q_match = re.match(r'^(用户|我|问|问题|小孩|小朋友|孩子|朋友)\s*[：:]\s*(.*)', line)
+        a_match = re.match(r'^(小狸|答|回答|机器人)\s*[：:]\s*(.*)', line)
+        
+        if q_match:
+            q_text = q_match.group(2).strip()
+            if q_text and len(q_text) > 1:
+                current_q = q_text
+        elif a_match and current_q:
+            a_text = a_match.group(2).strip()
+            if a_text and len(a_text) > 1 and len(a_text) < 300:
+                current_turns.append((current_q, a_text))
+                current_q = None
+    
+    # 保存最后一组
+    if current_turns and len(current_turns) >= 4:
+        conversations.append(current_turns)
+    
+    # 过滤：只保留>=4轮的对话，且累积轮次>=8
+    valid = [c for c in conversations if len(c) >= 4]
+    total_turns = sum(len(c) for c in valid)
+    if total_turns >= 8 and valid:
+        return valid
+    return None
+
+def make_turns_list_text(turns):
+    """将turns转成用户可见的对话文本"""
+    lines = []
+    for i, (q, a) in enumerate(turns):
+        lines.append(f"问：{q}")
+        lines.append(f"答：{a}")
+    return "\n".join(lines)
+
+def generate_one_multi():
+    """生成多条多轮对话（一次API调用返回多条）"""
+    prompt = make_realistic_multi_prompt()
+    result = call_astron_multi(prompt, temperature=random.uniform(0.75, 0.95))
+    if result:
+        convs = parse_multi_turn(result)
+        if convs:
+            items = []
+            for turns in convs:
+                items.append({
+                    'turns': len(turns),
+                    'turns_list': turns,
+                    'text': make_turns_list_text(turns),
+                    'source': 'astron_multi_v2',
+                })
+            return items
+    return None
+
+def main():
+    print("="*60)
+    print("  海量真实多轮对话生成 v2")
+    print("  生成超长、超真实的多轮聊天")
+    print("="*60)
+    
+    out_dir = os.path.join(MODEL_DIR, 'multiturn_outputs')
+    os.makedirs(out_dir, exist_ok=True)
+    
+    # 检查已有进度
+    all_results = []
+    progress_file = os.path.join(out_dir, 'all_multiturn_progress.json')
+    if os.path.exists(progress_file):
+        all_results = json.load(open(progress_file, encoding='utf-8'))
+        print(f"已有进度: {len(all_results)} 条")
+    
+    # ====== 参数 ======
+    TARGET = 500000     # 目标50万条
+    BATCH = 30          # 每批30条
+    MAX_WORKERS = 5     # Astron 5并发（稳定不超限）
+    CHECKPOINT = 1000   # 每1000条保存一次进度
+    
+    to_generate = TARGET - len(all_results)
+    total_batches = (to_generate + BATCH - 1) // BATCH
+    
+    print(f"目标: {TARGET:,} 条")
+    print(f"已有: {len(all_results):,} 条")
+    print(f"还需: {to_generate:,} 条")
+    print(f"批次: {total_batches:,} 批 (每批{BATCH}条, {MAX_WORKERS}并发)")
+    print(f"{'='*60}\n")
+    
+    t_start = time.time()
+    last_checkpoint = len(all_results)
+    
+    for batch_idx in range(total_batches):
+        t0 = time.time()
+        
+        # 并发生成
+        with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+            futures = [executor.submit(generate_one_multi) for _ in range(BATCH)]
+            batch_results = []
+            for future in as_completed(futures):
+                try:
+                    r = future.result()
+                    if r:
+                        batch_results.extend(r)
+                except Exception as e:
+                    pass
+        
+        # 去重
+        existing_texts = set(r.get('text', '')[:200] for r in all_results)
+        deduped = []
+        for r in batch_results:
+            key = r.get('text','')[:200]
+            if key not in existing_texts:
+                existing_texts.add(key)
+                deduped.append(r)
+        
+        all_results.extend(deduped)
+        
+        elapsed = time.time() - t0
+        total_elapsed = time.time() - t_start
+        rate = len(all_results) / max(total_elapsed, 1) * 60
+        eta = (TARGET - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
+        
+        # 统计
+        valid = len(batch_results)
+        avg_turns = sum(r['turns'] for r in deduped) / max(len(deduped), 1)
+        success_rate = valid / BATCH * 100
+        
+        print(f"[{batch_idx+1}/{total_batches}] "
+              f"+{len(deduped)}条(有效{valid}/{success_rate:.0f}%) "
+              f"累计{len(all_results):,}/{TARGET:,} "
+              f"均{avg_turns:.1f}轮 "
+              f"{rate:.0f}条/分 "
+              f"剩余{eta:.0f}分")
+        
+        # 定期保存进度
+        if len(all_results) - last_checkpoint >= CHECKPOINT:
+            with open(progress_file, 'w', encoding='utf-8') as f:
+                json.dump(all_results, f, ensure_ascii=False)
+            last_checkpoint = len(all_results)
+            print(f"  💾 已保存进度 ({len(all_results):,} 条)")
+    
+    # ====== 完成 ======
+    total_elapsed = time.time() - t_start
+    with open(progress_file, 'w', encoding='utf-8') as f:
+        json.dump(all_results, f, ensure_ascii=False)
+    
+    avg_turns_all = sum(r['turns'] for r in all_results) / max(len(all_results), 1)
+    total_turns = sum(r['turns'] for r in all_results)
+    
+    print(f"\n{'='*60}")
+    print(f"✅ 生成完成!")
+    print(f"  总条数: {len(all_results):,}")
+    print(f"  平均轮数: {avg_turns_all:.1f}")
+    print(f"  总对话轮次: {total_turns:,}")
+    print(f"  用时: {total_elapsed/60:.1f} 分")
+    print(f"  速率: {len(all_results)/max(total_elapsed,1)*60:.0f} 条/分")
+    print(f"  保存: {progress_file}")
+    print(f"{'='*60}")
+
+if __name__ == '__main__':
+    main()

scripts/train_cnb.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+小狸AI - CNB GPU训练脚本
+从 merged_clean_v3.json (977K对) 训练，自动GPU检测
+"""
+import sys, os, json, time, glob, re, random, argparse
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+
+from llm_chat_improved import LLMModel, LLMChat, ConversationDataset, LLMTrainer, tokenizer_instance
+
+MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+def load_pairs(path):
+    with open(path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    pairs = []
+    for item in data:
+        if isinstance(item, dict):
+            q = item.get("question", "") or item.get("input", "")
+            a = item.get("answer", "") or item.get("output", "")
+            if q and a:
+                pairs.append((q, a))
+    return pairs
+
+def find_latest_ckpt(model_dir):
+    ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_epoch_*.pth'))
+    if not ckpts:
+        ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_epoch_*.pth'))
+    if not ckpts:
+        return None, 0
+    nums = [(p, max([int(s) for s in re.findall(r'\d+', os.path.basename(p).replace('.pth',''))])) for p in ckpts]
+    best = max(nums, key=lambda x: x[1])
+    return best[0], best[1]
+
+def main():
+    parser = argparse.ArgumentParser(description='小狸AI CNB训练')
+    parser.add_argument('--epochs', type=int, default=100, help='训练轮数')
+    parser.add_argument('--batch_size', type=int, default=8, help='batch大小')
+    parser.add_argument('--lr', type=float, default=3e-4, help='学习率')
+    parser.add_argument('--accum', type=int, default=2, help='梯度累积步数')
+    parser.add_argument('--data', type=str, default='data/merged_clean_v3.json', help='训练数据路径')
+    parser.add_argument('--resume', action='store_true', help='断点续训')
+    parser.add_argument('--save_every', type=int, default=5, help='每N轮保存checkpoint')
+    args = parser.parse_args()
+
+    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    print("=" * 60)
+    print(f"小狸AI CNB训练")
+    print(f"  GPU: {torch.cuda.get_device_name(0) if device == 'cuda' else '无'}")
+    print(f"  设备: {device}")
+    print(f"  数据: {args.data}")
+    print(f"  epochs={args.epochs}, batch={args.batch_size}, lr={args.lr}")
+    print("=" * 60)
+
+    # ====== 加载数据 ======
+    data_path = args.data if os.path.isabs(args.data) else os.path.join(MODEL_DIR, args.data)
+    if not os.path.exists(data_path):
+        print(f"❌ 数据不存在: {data_path}")
+        return
+    
+    pairs = load_pairs(data_path)
+    random.seed(42)
+    random.shuffle(pairs)
+    split = int(len(pairs) * 0.95)
+    train_set = pairs[:split]
+    val_set = pairs[split:]
+    print(f"数据: {len(pairs)}对 | 训练: {len(train_set)} | 验证: {len(val_set)}")
+
+    # ====== 加载词表 ======
+    vocab_path = os.path.join(MODEL_DIR, 'vocab.json')
+    with open(vocab_path, 'r', encoding='utf-8') as f:
+        vocab = json.load(f)
+    # vocab.json 不含特殊token(0-6)，需合并进来，否则：
+    # 1) tokenizer 编码 <SOS>/<SEP>/<EOS> 等标记时会逐字符错码
+    # 2) vocab 的 id 范围是 7..max，超过 len(vocab) 会造成 embedding 索引越界
+    special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
+                      '<START>': 4, '<END>': 5, '<SEP>': 6}
+    for tok, tid in special_tokens.items():
+        if tok not in vocab:
+            vocab[tok] = tid
+    real_vocab_size = max(vocab.values()) + 1
+    print(f"词表: {len(vocab)}项, vocab_size={real_vocab_size} (字符+BPE, 来自 vocab.json)")
+    tokenizer_instance.char_to_id = vocab.copy()
+    tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
+    tokenizer_instance.special_tokens = special_tokens.copy()
+    tokenizer_instance._sorted_vocab_cache = None
+    for attr in ['_multi_trie', '_multi_keys', '_single_map']:
+        if hasattr(tokenizer_instance, attr):
+            delattr(tokenizer_instance, attr)
+
+    # ====== 模型 ======
+    model = LLMModel(vocab_size=real_vocab_size, d_model=320, n_heads=8,
+                     n_layers=6, d_ff=1536, max_len=256, dropout=0.1,
+                     use_memory=False, use_thinking=False, use_moe=False)
+    total = sum(p.numel() for p in model.parameters())
+    print(f"模型: {total:,} 参数 ({total/1e6:.2f}M)")
+
+    start_epoch = 0
+    if args.resume:
+        ckpt_path, last_epoch = find_latest_ckpt(MODEL_DIR)
+        if ckpt_path:
+            ckpt = torch.load(ckpt_path, map_location=device)
+            old_cfg = ckpt.get('model_config', {})
+            if (old_cfg.get('vocab_size', 0) == real_vocab_size
+                and old_cfg.get('d_model', 0) == 320
+                and old_cfg.get('n_layers', 0) == 6):
+                model.load_state_dict(ckpt['model_state_dict'])
+                start_epoch = last_epoch
+                print(f"断点续训: epoch {start_epoch} ← {os.path.basename(ckpt_path)}")
+            else:
+                print(f"⚠️ 架构不匹配，从头训练")
+        else:
+            print("无断点，从头训练")
+    
+    model = model.to(device)
+
+    # ====== 火力全开：torch.compile (inductor融合, 无CUDA graphs) + 梯度累积 ======
+    # 物理batch=192(不OOM), accum=2 → 等效batch=384
+    fwd_model = model
+    if device == 'cuda':
+        print("🔥 启用 torch.compile (mode=default, inductor融合)...")
+        fwd_model = torch.compile(model, mode='default')
+        print("✓ torch.compile 已就绪（首步触发编译约30-60s）")
+
+    # ====== DataLoader ======
+    n_workers = 16  # CPU火力全开，16核全用
+    train_ds = ConversationDataset(train_set, vocab, max_length=256)
+    train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True,
+                              num_workers=n_workers, pin_memory=(device == 'cuda'),
+                              persistent_workers=True, prefetch_factor=4, drop_last=True)
+    val_ds = ConversationDataset(val_set, vocab, max_length=256)
+    val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False,
+                            num_workers=n_workers, pin_memory=(device == 'cuda'),
+                            persistent_workers=True, prefetch_factor=4, drop_last=True)
+    print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)} | num_workers={n_workers} | prefetch=4 | drop_last=True")
+
+    # ====== Trainer ======
+    config = {
+        'batch_size': args.batch_size,
+        'gradient_accumulation_steps': args.accum,
+        'epochs': args.epochs,
+        'd_model': 320, 'n_heads': 8, 'n_layers': 6,
+        'd_ff': 1536, 'max_length': 256, 'dropout': 0.1,
+        'use_amp': device == 'cuda',
+    }
+    trainer = LLMTrainer(fwd_model, device, lr=args.lr, use_amp=config['use_amp'])
+
+    # ====== 训练 ======
+    print(f"\n开始训练 epoch {start_epoch+1} → {args.epochs}")
+    print("=" * 60)
+    t_start = time.time()
+    best_val = float('inf')
+
+    for epoch in range(start_epoch + 1, args.epochs + 1):
+        t0 = time.time()
+        train_loss = trainer.train_epoch(train_loader, epoch, vocab, config)
+        val_loss = trainer.evaluate(val_loader, config)
+        dt = time.time() - t0
+
+        marker = " ★" if val_loss < best_val else ""
+        if val_loss < best_val:
+            best_val = val_loss
+            trainer.save_model(os.path.join(MODEL_DIR, 'xiaoli_cnb_best.pth'), vocab, epoch)
+
+        elapsed = time.time() - t_start
+        eta = (elapsed / (epoch - start_epoch)) * (args.epochs - epoch) if epoch > start_epoch else 0
+        
+        print(f"E{epoch}/{args.epochs} | train={train_loss:.4f} val={val_loss:.4f}{marker} | "
+              f"{dt:.0f}s/epoch | {elapsed/60:.1f}min已用 ETA{eta/60:.0f}min")
+
+        # 保存
+        if epoch % args.save_every == 0:
+            ckpt_path = os.path.join(MODEL_DIR, f'llm_model_v2_resume_epoch_{epoch}.pth')
+            trainer.save_model(ckpt_path, vocab, epoch)
+            # 清理旧resume
+            for f in glob.glob(os.path.join(MODEL_DIR, 'llm_model_v2_resume_*.pth')):
+                en = max([int(s) for s in re.findall(r'\d+', os.path.basename(f).replace('.pth',''))])
+                if en <= epoch - 15:
+                    try: os.remove(f)
+                    except: pass
+            print(f"  💾 已保存 epoch {epoch}")
+
+    # ====== 完成 ======
+    final_path = os.path.join(MODEL_DIR, 'xiaoli_cnb_final.pth')
+    trainer.save_model(final_path, vocab, args.epochs)
+    total_time = (time.time() - t_start) / 60
+    print(f"\n✅ 训练完成! 耗时 {total_time:.1f}min")
+    
+    # 测试
+    print("\n测试:")
+    chat = LLMChat(model, vocab, device, max_length=256)
+    for q in ["你好", "你是谁", "你叫什么名字", "今天天气真好", "我有点难过"]:
+        r = chat.generate_response(q, max_length=60, temperature=0.3)
+        print(f"  Q: {q}")
+        print(f"  A: {r}\n")
+
+if __name__ == "__main__":
+    main()

train_max.py

@@ -0,0 +1,662 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+小狸 AI - 最大化训练脚本 v2.0
+==============================
+两阶段训练：预训练(通用中文) → 角色对齐(QA数据)
+
+相比原始 train.py 的改进：
+  ① 两阶段训练：先用通用语料建立语言能力，再用QA数据做角色对齐
+  ② Warmup + Cosine 学习率调度（业界标准）
+  ③ 更大的有效 batch size（提升收敛稳定性）
+  ④ 启用 AMP 混合精度（GPU加速）
+  ⑤ DataLoader 多进程加载
+  ⑥ 修复在线蒸馏 P2/P3 失效 bug
+  ⑦ 支持学习率分层（不同层不同 LR）
+  ⑧ 更激进的梯度裁剪策略
+  ⑨ 支持从原始 train.py checkpoint 无缝续训
+"""
+
+import sys, os, json, time, glob, random, re, math
+from typing import List, Tuple, Optional
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+
+# 复用原始项目代码
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llm_chat_improved import (
+    CharTokenizer, LLMModel, LLMChat,
+    ConversationDataset, tokenizer_instance
+)
+from api_correction import call_astron
+
+# ── 配置 ──────────────────────────────────────────────────
+class Config:
+    # ========= 模型架构 =========
+    d_model: int = 320
+    n_heads: int = 8
+    n_layers: int = 6
+    d_ff: int = 1536
+    max_length: int = 256        # 训练序列长度
+    dropout: float = 0.1
+    use_memory: bool = False
+    use_thinking: bool = False
+    use_moe: bool = False
+    num_experts: int = 1
+    top_k: int = 1
+
+    # ========= 阶段一：预训练（通用中文语料） =========
+    pt_enable: bool = True       # 是否启用预训练阶段
+    pt_data_dir: str = ""        # 预训练数据目录（留空则自动下载）
+    pt_batch_size: int = 16      # 预训练 batch
+    pt_grad_accum: int = 4       # 梯度累积
+    pt_effective_batch: int = 64 # = pt_batch_size * pt_grad_accum
+    pt_epochs: int = 5           # 预训练 epoch
+    pt_lr: float = 5e-4          # 预训练初始 LR（比微调大）
+    pt_min_lr: float = 5e-5      # 预训练最小 LR
+    pt_warmup_steps: int = 500   # 预训练 warmup
+    pt_weight_decay: float = 0.1 # 预训练 weight decay（标准）
+    pt_max_norm: float = 1.0     # 梯度裁剪
+    pt_max_length: int = 128     # 预训练用较短序列（加速）
+    pt_data_ratio: float = 1.0   # 使用数据的比例（调试用）
+
+    # ========= 阶段二：角色对齐（QA数据） =========
+    align_batch_size: int = 8
+    align_grad_accum: int = 4
+    align_effective_batch: int = 32
+    align_epochs: int = 50
+    align_lr: float = 1e-4
+    align_min_lr: float = 1e-5
+    align_warmup_steps: int = 100
+    align_weight_decay: float = 0.01
+    align_max_norm: float = 1.0
+
+    # ========= 在线蒸馏（阶段二末尾执行） =========
+    distill_enable: bool = True
+    distill_every: int = 5       # 每 N 个 epoch 执行一次蒸馏
+
+    # ========= 通用 =========
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    num_workers: int = 4 if torch.cuda.is_available() else 0
+    use_amp: bool = torch.cuda.is_available()
+    seed: int = 42
+    save_dir: str = os.path.dirname(os.path.abspath(__file__))
+    log_every: int = 0           # 0 = 每1%打印
+
+    # ========= 检查点 =========
+    save_interval: int = 1       # 每 N 个 epoch 保存 resume
+    archive_interval: int = 10   # 每 N 个 epoch 保存归档
+    keep_resume: int = 3         # 保留最近 resume 数
+
+
+config = Config()
+
+
+# ── 预训练数据集 ──────────────────────────────────────────
+class PretrainDataset(Dataset):
+    """通用中文文本预训练数据集
+
+    支持两种输入格式：
+      1. 纯文本文件：每行一句话/段落
+      2. JSON 文件：列表，每个元素含 "text" 字段
+    """
+
+    def __init__(self, data_dir: str, vocab: dict, max_length: int = 128):
+        self.vocab = vocab
+        self.max_length = max_length
+        self.pad_id = vocab.get('<PAD>', 0)
+
+        # 扫描并加载所有文本
+        self.texts = []
+        data_dir = Path(data_dir)
+        if not data_dir.exists():
+            raise FileNotFoundError(f"预训练数据目录不存在: {data_dir}")
+
+        for fpath in sorted(data_dir.glob("*")):
+            if fpath.suffix in ('.txt', '.json', '.jsonl'):
+                self._load_file(fpath)
+
+        if not self.texts:
+            raise ValueError(f"预训练数据目录为空: {data_dir}")
+
+        print(f"预训练数据: {len(self.texts)} 条文本, "
+              f"总字符数: {sum(len(t) for t in self.texts):,}")
+
+    def _load_file(self, path: Path):
+        try:
+            if path.suffix == '.txt':
+                with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+                    for line in f:
+                        line = line.strip()
+                        if len(line) >= 20:  # 过滤太短的
+                            self.texts.append(line)
+            elif path.suffix == '.json':
+                with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+                    data = json.load(f)
+                    for item in data:
+                        if isinstance(item, dict) and 'text' in item:
+                            t = item['text'].strip()
+                            if len(t) >= 20:
+                                self.texts.append(t)
+                        elif isinstance(item, str) and len(item) >= 20:
+                            self.texts.append(item)
+            elif path.suffix == '.jsonl':
+                with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+                    for line in f:
+                        try:
+                            item = json.loads(line)
+                            if isinstance(item, dict) and 'text' in item:
+                                t = item['text'].strip()
+                                if len(t) >= 20:
+                                    self.texts.append(t)
+                        except json.JSONDecodeError:
+                            continue
+        except Exception as e:
+            print(f"  跳过 {path.name}: {e}")
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        text = self.texts[idx]
+        # 随机截取一段（提升多样性）
+        tokens = tokenizer_instance.encode(text)
+        if len(tokens) > self.max_length:
+            start = random.randint(0, len(tokens) - self.max_length)
+            tokens = tokens[start:start + self.max_length]
+        else:
+            tokens = tokens + [self.pad_id] * (self.max_length - len(tokens))
+
+        # 自回归：input = tokens[:-1], target = tokens[1:]
+        input_ids = torch.tensor(tokens[:-1], dtype=torch.long)
+        target_ids = torch.tensor(tokens[1:], dtype=torch.long)
+        return {'input_ids': input_ids, 'target_ids': target_ids}
+
+
+# ── 改进版训练器 ──────────────────────────────────────────
+class MaxTrainer:
+    """增强训练器 — 支持 warmup、分层 LR、自定义 scheduler"""
+
+    def __init__(self, model: nn.Module, config: Config, stage: str = "pretrain"):
+        self.model = model.to(config.device)
+        self.device = config.device
+        self.config = config
+        self.stage = stage
+        self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
+
+        # 根据阶段选择超参数
+        if stage == "pretrain":
+            base_lr = config.pt_lr
+            wd = config.pt_weight_decay
+        else:
+            base_lr = config.align_lr
+            wd = config.align_weight_decay
+
+        # 分层学习率：embedding + fc_out 用低 LR，中间层用高 LR
+        param_groups = []
+        for name, param in model.named_parameters():
+            if not param.requires_grad:
+                continue
+            if 'embedding' in name or 'fc_out' in name:
+                param_groups.append({'params': param, 'lr': base_lr * 0.5, 'weight_decay': wd})
+            elif 'norm' in name or 'rmsnorm' in name:
+                param_groups.append({'params': param, 'lr': base_lr, 'weight_decay': 0.0})
+            else:
+                param_groups.append({'params': param, 'lr': base_lr, 'weight_decay': wd})
+
+        self.optimizer = torch.optim.AdamW(param_groups, lr=base_lr, weight_decay=wd,
+                                           betas=(0.9, 0.95), eps=1e-8)
+
+        # AMP
+        self.use_amp = config.use_amp and torch.cuda.is_available()
+        self.scaler = torch.cuda.amp.GradScaler(enabled=self.use_amp)
+
+        if torch.cuda.is_available():
+            torch.backends.cuda.matmul.allow_tf32 = True
+            torch.backends.cudnn.allow_tf32 = True
+            torch.backends.cudnn.benchmark = True
+            torch.backends.cudnn.deterministic = False
+
+    def build_scheduler(self, total_steps: int):
+        """构建 Warmup + Cosine Decay scheduler"""
+        if self.stage == "pretrain":
+            warmup = self.config.pt_warmup_steps
+            min_lr = self.config.pt_min_lr
+        else:
+            warmup = self.config.align_warmup_steps
+            min_lr = self.config.align_min_lr
+
+        def lr_lambda(step):
+            if step < warmup:
+                return step / max(1, warmup)
+            progress = (step - warmup) / max(1, total_steps - warmup)
+            cosine_decay = 0.5 * (1 + math.cos(math.pi * progress))
+            return (1 - min_lr / self.optimizer.param_groups[0]['lr']) * cosine_decay \
+                   + min_lr / self.optimizer.param_groups[0]['lr']
+
+        self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda)
+
+    def train_epoch(self, dataloader: DataLoader, epoch: int, total_epochs: int,
+                    stage_name: str = "pretrain", log_every: int = 0) -> float:
+        self.model.train()
+        total_loss = 0
+        n_batches = len(dataloader)
+
+        if stage_name == "pretrain":
+            accum = self.config.pt_grad_accum
+            max_norm = self.config.pt_max_norm
+        else:
+            accum = self.config.align_grad_accum
+            max_norm = self.config.align_max_norm
+
+        if log_every == 0:
+            log_every = max(1, n_batches // 100)
+
+        self.optimizer.zero_grad()
+
+        for batch_idx, batch in enumerate(dataloader):
+            input_ids = batch['input_ids'].to(self.device, non_blocking=True)
+            target_ids = batch['target_ids'].to(self.device, non_blocking=True)
+
+            mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
+
+            with torch.cuda.amp.autocast(enabled=self.use_amp):
+                outputs = self.model(input_ids, mask)
+                outputs = outputs.reshape(-1, outputs.size(-1))
+                tgt = target_ids.reshape(-1)
+                loss = self.criterion(outputs, tgt) / accum
+
+            self.scaler.scale(loss).backward()
+
+            if (batch_idx + 1) % accum == 0:
+                self.scaler.unscale_(self.optimizer)
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm)
+                self.scaler.step(self.optimizer)
+                self.scaler.update()
+                self.optimizer.zero_grad()
+                if self.scheduler is not None:
+                    self.scheduler.step()
+
+            total_loss += loss.item() * accum
+
+            if (batch_idx + 1) % log_every == 0:
+                avg = total_loss / (batch_idx + 1)
+                lr = self.optimizer.param_groups[0]['lr']
+                mem = ""
+                if torch.cuda.is_available():
+                    mem = f" | Mem: {torch.cuda.memory_allocated()/1e9:.2f}GB"
+                print(f"  [{stage_name}] Epoch {epoch}/{total_epochs} | "
+                      f"Batch {batch_idx+1}/{n_batches} | Loss: {avg:.4f} | LR: {lr:.2e}{mem}")
+
+        return total_loss / n_batches
+
+    def evaluate(self, dataloader: DataLoader) -> float:
+        self.model.eval()
+        total_loss = 0
+        with torch.no_grad():
+            for batch in dataloader:
+                input_ids = batch['input_ids'].to(self.device)
+                target_ids = batch['target_ids'].to(self.device)
+                mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
+                outputs = self.model(input_ids, mask)
+                outputs = outputs.reshape(-1, outputs.size(-1))
+                loss = self.criterion(outputs, target_ids.reshape(-1))
+                total_loss += loss.item()
+        return total_loss / max(1, len(dataloader))
+
+    def save_model(self, filepath: str, vocab: dict, epoch: int):
+        core = getattr(self.model, '_orig_mod', self.model)
+        torch.save({
+            'model_state_dict': core.state_dict(),
+            'optimizer_state_dict': self.optimizer.state_dict(),
+            'scheduler_state_dict': self.scheduler.state_dict() if self.scheduler else None,
+            'vocab': vocab,
+            'model_config': {
+                'vocab_size': core.vocab_size,
+                'd_model': core.d_model,
+                'n_heads': core.n_heads,
+                'n_layers': core.n_layers,
+                'd_ff': core.d_ff,
+                'max_len': core.max_len,
+                'use_memory': core.use_memory,
+                'use_thinking': core.use_thinking,
+                'use_moe': core.use_moe,
+                'stage': self.stage,
+            },
+            'epoch': epoch,
+        }, filepath)
+        print(f"  ✅ 已保存: {filepath}")
+
+
+# ── 在线蒸馏（修复版） ──────────────────────────────────
+def run_distillation(model, vocab, device, max_length, distill_questions, save_dir):
+    if not distill_questions:
+        return [], 0
+
+    from api_correction import batch_convert
+    chat = LLMChat(model, vocab, device, max_length)
+    all_new = []
+
+    print(f"  📡 蒸馏 {len(distill_questions)} 题...")
+    answers1 = []
+    for i, q in enumerate(distill_questions):
+        ans = chat.generate_response(q, max_length=40, temperature=0.15)
+        answers1.append((q, ans))
+        if (i + 1) % 100 == 0:
+            print(f"    P1: {i+1}/{len(distill_questions)}")
+
+    results1 = batch_convert(answers1)
+    all_new.extend(results1)
+    print(f"    P1: +{len(results1)} 条")
+
+    print(f"  [P2] 模型高温生成 (temp=0.8)...")
+    for i, q in enumerate(distill_questions):
+        ans = chat.generate_response(q, max_length=40, temperature=0.8)
+        if ans and len(ans) >= 5:
+            all_new.append({"question": q, "answer": ans})
+        if (i + 1) % 100 == 0:
+            print(f"    P2: {i+1}/{len(distill_questions)}")
+
+    print(f"  [P3] API 直答 (temp=1.0)...")
+    sampled = random.sample(distill_questions, min(50, len(distill_questions)))
+    for q in sampled:
+        try:
+            ans = call_astron(q, temperature=1.0)
+            if ans and len(ans) >= 5:
+                all_new.append({"question": q, "answer": ans})
+        except Exception:
+            pass
+
+    train_path = os.path.join(save_dir, 'xiaoli_train_v2.json')
+    merged_path = os.path.join(save_dir, 'xiaoli_merged_data_v2.json')
+    added = 0
+
+    if all_new and os.path.exists(train_path):
+        with open(train_path, 'r', encoding='utf-8') as f:
+            cur_train = json.load(f)
+        existing = set((item.get("question", ""), item.get("answer", "")) for item in cur_train)
+        truly_new = [item for item in all_new
+                     if (item["question"], item["answer"]) not in existing]
+
+        if truly_new:
+            cur_train.extend(truly_new)
+            with open(train_path, 'w', encoding='utf-8') as f:
+                json.dump(cur_train, f, ensure_ascii=False, indent=2)
+            if os.path.exists(merged_path):
+                with open(merged_path, 'r', encoding='utf-8') as f:
+                    cur_merged = json.load(f)
+                cur_merged.extend(truly_new)
+                with open(merged_path, 'w', encoding='utf-8') as f:
+                    json.dump(cur_merged, f, ensure_ascii=False, indent=2)
+            added = len(truly_new)
+            print(f"  ✅ 蒸馏新增 {added} 条")
+
+    return all_new, added
+
+
+# ── 主流程 ──────────────────────────────────────────────
+def main():
+    print("=" * 65)
+    print("  小狸 AI - 最大化训练 v2.0")
+    print(f"  设备: {config.device} | AMP: {config.use_amp}")
+    print(f"  模型: {config.d_model}d, {config.n_layers}L, {config.n_heads}H, {config.d_ff}ff")
+    print("=" * 65)
+
+    # 1. 加载词表
+    vocab_path = os.path.join(config.save_dir, 'vocab.json')
+    if not os.path.exists(vocab_path):
+        print("❌ 未找到 vocab.json，请先运行 build_vocab.py")
+        return
+    with open(vocab_path, 'r', encoding='utf-8') as f:
+        vocab = json.load(f)
+    tokenizer_instance.char_to_id = vocab.copy()
+    tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
+    print(f"词表: {len(vocab)} tokens")
+
+    # 2. 创建模型
+    model = LLMModel(
+        vocab_size=len(vocab),
+        d_model=config.d_model,
+        n_heads=config.n_heads,
+        n_layers=config.n_layers,
+        d_ff=config.d_ff,
+        max_len=config.max_length,
+        dropout=config.dropout,
+        use_memory=config.use_memory,
+        use_thinking=config.use_thinking,
+        use_moe=config.use_moe,
+        num_experts=config.num_experts,
+        top_k=config.top_k,
+    )
+    params = sum(p.numel() for p in model.parameters())
+    print(f"参数量: {params:,} ({params/1e6:.2f}M)")
+
+    # 3. 预训练阶段（如果启用）
+    if config.pt_enable:
+        print("\n" + "=" * 65)
+        print("  阶段一：通用中文预训练")
+        print("=" * 65)
+
+        pt_dir = config.pt_data_dir
+        if not pt_dir or not os.path.exists(pt_dir):
+            pt_dir = os.path.join(config.save_dir, 'pretrain_data')
+            os.makedirs(pt_dir, exist_ok=True)
+            if not list(Path(pt_dir).glob("*")):
+                print("  📥 下载预训练数据 (SkyPile 小样本)...")
+                _download_pretrain_data(pt_dir)
+
+        try:
+            pt_dataset = PretrainDataset(pt_dir, vocab, config.pt_max_length)
+            pt_loader = DataLoader(
+                pt_dataset,
+                batch_size=config.pt_batch_size,
+                shuffle=True,
+                num_workers=config.num_workers,
+                pin_memory=torch.cuda.is_available(),
+            )
+        except (FileNotFoundError, ValueError) as e:
+            print(f"  ⚠️ 预训练数据不可用: {e}")
+            print(f"  ⚠️ 跳过预训练阶段。请准备数据后放入: {pt_dir}")
+            pt_dataset = None
+            pt_loader = None
+
+        if pt_loader is not None:
+            total_pt_steps = len(pt_loader) * config.pt_epochs
+            trainer_pt = MaxTrainer(model, config, stage="pretrain")
+            trainer_pt.build_scheduler(total_pt_steps)
+
+            for epoch in range(1, config.pt_epochs + 1):
+                loss = trainer_pt.train_epoch(
+                    pt_loader, epoch, config.pt_epochs,
+                    stage_name="pretrain", log_every=config.log_every
+                )
+                val_loss = trainer_pt.evaluate(pt_loader)
+                print(f"  → 预训练 Epoch {epoch}/{config.pt_epochs} | "
+                      f"Loss: {loss:.4f} | Val: {val_loss:.4f}")
+
+                trainer_pt.save_model(
+                    os.path.join(config.save_dir, f'pretrain_epoch_{epoch}.pth'),
+                    vocab, epoch
+                )
+
+            print(f"  ✅ 预训练完成")
+
+    # 4. 阶段二：角色对齐
+    print("\n" + "=" * 65)
+    print("  阶段二：角色对齐 (QA 角色化训练)")
+    print("=" * 65)
+
+    merged_path = os.path.join(config.save_dir, 'xiaoli_merged_data_v2.json')
+    if not os.path.exists(merged_path):
+        print(f"❌ 未找到合并数据: {merged_path}")
+        return
+
+    with open(merged_path, 'r', encoding='utf-8') as f:
+        raw_data = json.load(f)
+
+    pairs = []
+    for item in raw_data:
+        if isinstance(item, dict):
+            q = item.get("question", "") or item.get("input", "") or item.get("prompt", "")
+            a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
+            if q and a:
+                pairs.append((q, a))
+
+    random.seed(config.seed)
+    random.shuffle(pairs)
+
+    split = int(len(pairs) * 0.9)
+    train_pairs = pairs[:split]
+    val_pairs = pairs[split:]
+    print(f"QA 数据: {len(pairs)} 条 (训练 {len(train_pairs)} + 验证 {len(val_pairs)})")
+
+    train_ds = ConversationDataset(train_pairs, vocab, config.max_length)
+    train_loader = DataLoader(
+        train_ds, batch_size=config.align_batch_size, shuffle=True,
+        num_workers=config.num_workers, pin_memory=torch.cuda.is_available(),
+    )
+    val_ds = ConversationDataset(val_pairs, vocab, config.max_length)
+    val_loader = DataLoader(
+        val_ds, batch_size=config.align_batch_size, shuffle=False,
+        num_workers=0,
+    )
+
+    distill_questions = []
+    for qpath in ['exam_distill_questions.json', 'distill_questions_v3.json', 'distill_questions_v2.json']:
+        fp = os.path.join(config.save_dir, qpath)
+        if os.path.exists(fp):
+            with open(fp, 'r', encoding='utf-8') as f:
+                distill_questions = json.load(f)
+            print(f"蒸馏问题: {len(distill_questions)} 道 ({qpath})")
+            break
+
+    total_align_steps = len(train_loader) * config.align_epochs
+    trainer_align = MaxTrainer(model, config, stage="align")
+    trainer_align.build_scheduler(total_align_steps)
+
+    best_val_loss = float('inf')
+    total_distill_added = 0
+
+    for epoch in range(1, config.align_epochs + 1):
+        train_loss = trainer_align.train_epoch(
+            train_loader, epoch, config.align_epochs,
+            stage_name="align", log_every=config.log_every
+        )
+
+        val_loss = trainer_align.evaluate(val_loader)
+        marker = " ★" if val_loss < best_val_loss else ""
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            trainer_align.save_model(
+                os.path.join(config.save_dir, 'xiaoli_v2_best.pth'), vocab, epoch
+            )
+
+        print(f"  → QA Epoch {epoch}/{config.align_epochs} | "
+              f"Loss: {train_loss:.4f} | Val: {val_loss:.4f}{marker} | "
+              f"LR: {trainer_align.optimizer.param_groups[0]['lr']:.2e}")
+
+        if epoch % config.save_interval == 0:
+            trainer_align.save_model(
+                os.path.join(config.save_dir, f'llm_model_v2_resume_epoch_{epoch}.pth'),
+                vocab, epoch
+            )
+
+        if epoch % config.archive_interval == 0:
+            trainer_align.save_model(
+                os.path.join(config.save_dir, f'llm_model_v2_epoch_{epoch}.pth'),
+                vocab, epoch
+            )
+            resumes = sorted(glob.glob(os.path.join(config.save_dir, 'llm_model_v2_resume_*.pth')))
+            for old in resumes[:-config.keep_resume]:
+                try:
+                    os.remove(old)
+                except:
+                    pass
+
+        if config.distill_enable and distill_questions and epoch % config.distill_every == 0:
+            _, added = run_distillation(
+                model, vocab, config.device, config.max_length,
+                distill_questions, config.save_dir
+            )
+            total_distill_added += added
+
+    trainer_align.save_model(
+        os.path.join(config.save_dir, 'xiaoli_v2_final.pth'), vocab, config.align_epochs
+    )
+
+    for f in glob.glob(os.path.join(config.save_dir, 'llm_model_v2_resume_*.pth')):
+        try:
+            os.remove(f)
+        except:
+            pass
+
+    print(f"\n{'='*65}")
+    print(f"  训练完成！蒸馏新增: {total_distill_added} 条")
+    print(f"  最终模型: {os.path.join(config.save_dir, 'xiaoli_v2_final.pth')}")
+    print(f"{'='*65}")
+
+    print("\n📝 快速测试:")
+    chat = LLMChat(model, vocab, config.device, config.max_length)
+    for q in ["你好", "你是谁", "1+1等于几", "讲个笑话"]:
+        ans = chat.generate_response(q, max_length=30, temperature=0.5)
+        print(f"  Q: {q}")
+        print(f"  A: {ans}")
+        print()
+
+
+def _download_pretrain_data(target_dir: str):
+    import urllib.request
+    samples = [
+        "天空是蓝色的，白云在天空中飘来飘去。小鸟在树上唱歌，花儿在阳光下开放。",
+        "小明是一个小学生，他每天都要去学校上课。今天老师教大家认识汉字，小明学会了'天'、'地'、'人'三个字。",
+        "春天来了，大地变绿了。小草从土里钻出来，树木长出了新叶子，到处都是生机勃勃的景象。",
+        "猫是一种常见的宠物，它们有着柔软的毛发和灵活的身体。猫咪喜欢晒太阳，也喜欢玩毛线球。",
+        "数学是一门重要的学科，我们生活中处处都用得到数学。比如买东西需要算钱，做菜需要称量材料。",
+        "学习编程可以培养逻辑思维能力，通过写代码，我们可以让计算机帮助我们完成各种任务。",
+        "中国有着悠久的历史和灿烂的文化，四大发明——造纸术、印刷术、火药、指南针，对世界文明做出了重要贡献。",
+        "保护环境人人有责，我们应该节约用水，减少使用塑料袋，做好垃圾分类，让地球变得更美好。",
+        "星星在夜空中闪烁，月亮有时圆有时缺。宇宙中有无数的星系，地球只是其中一颗小小的行星。",
+        "友谊是人生中宝贵的财富，好朋友会在你开心时和你一起笑，在你难过时给你安慰和鼓励。",
+        "人工智能是当今最前沿的科技领域之一，它让机器能够像人一样思考和学习，改变着我们的生活方式。",
+        "读书是获取知识的重要途径，通过阅读，我们可以了解世界各地的风土人情，学习前人的智慧和经验。",
+        "运动对身体健康非常重要，跑步、游泳、打篮球都是很好的运动方式，每天坚持运动可以让我们更有活力。",
+        "音乐是一种美妙的艺术形式，不同的乐器能演奏出不同的声音，组合在一起就形成了动听的旋律。",
+        "做饭是一门生活技能，学会做饭不仅能照顾好自己的饮食，还能给家人朋友带来美味的享受。",
+        "植物通过光合作用将阳光转化为能量，它们吸收二氧化碳，释放出氧气，为地球上的生命提供了重要的保障。",
+        "狗是人类最忠诚的朋友，它们聪明、勇敢、重情义，能够帮助人类看家护院、导盲、搜救等。",
+        "海洋覆盖了地球表面的百分之七十，海洋中有各种各样的生物，从微小的浮游生物到巨大的鲸鱼。",
+        "互联网让世界变成了一个地球村，人们可以通过网络随时随地获取信息、交流沟通、学习工作。",
+        "时间管理是一项重要的能力，合理安排时间可以提高工作效率，让我们有更多的时间做自己喜欢的事情。",
+    ]
+    out_path = os.path.join(target_dir, "pretrain_samples.txt")
+    with open(out_path, 'w', encoding='utf-8') as f:
+        for s in samples:
+            f.write(s + "\n")
+
+    wiki_samples_path = os.path.join(target_dir, "wiki_samples.json")
+    wiki_data = [
+        {"text": "孔子（公元前551年—公元前479年），名丘，字仲尼，是中国古代伟大的思想家、教育家，儒家学派的创始人。"},
+        {"text": "太阳是太阳系的中心天体，直径为139.2万公里，表面温度约为5500摄氏度，地球上的生命离不开太阳的光和热。"},
+        {"text": "长城是中国古代的军事防御工程，始建于春秋战国时期，总长度超过2.1万公里，被誉为世界七大奇迹之一。"},
+        {"text": "DNA是脱氧核糖核酸的缩写，是生物遗传信息的载体，它决定了生物的性状和特征，呈双螺旋结构。"},
+        {"text": "Python是一种广泛使用的高级编程语言，由吉多·范罗苏姆于1991年首次发布，以其简洁易读的语法而闻名。"},
+        {"text": "水的化学式是H2O，在标准大气压下，水的沸点是100摄氏度，冰点是0摄氏度，水是生命之源。"},
+        {"text": "贝多芬是德国著名的作曲家和钢琴家，他创作了《命运交响曲》《月光奏鸣曲》等不朽的音乐作品。"},
+        {"text": "光合作用是植物、藻类和某些细菌利用光能将二氧化碳和水转化为有机物和氧气的过程。"},
+        {"text": "人类基因组计划于1990年启动，2003年完成，成功绘制了人类基因组的完整序列图，对医学研究具有重要意义。"},
+        {"text": "丝绸之路是古代连接中国与中亚、欧洲的贸易路线，不仅促进了商品交流，也推动了东西方文化的传播与融合。"},
+    ]
+    with open(wiki_samples_path, 'w', encoding='utf-8') as f:
+        json.dump(wiki_data, f, ensure_ascii=False, indent=2)
+
+    print(f"  ✅ 已生成示例预训练数据: {len(samples) + len(wiki_data)} 条")
+    print(f"  ⚠️ 这仅供流程测试！实际训练请替换为大规模中文语料")
+    print(f"  💡 建议: https://huggingface.co/datasets/SkyWork/SkyPile-150B")
+
+
+if __name__ == "__main__":
+    main()

vocab.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd80a63aa00c80a755c7e0cb4d91263bedbaccaf66fd3b12428017b4230ecbe9
-size 106005
+oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
+size 999243

vocab_v2.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330
+size 41185

xiaoli-llm-chat/vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
+size 999243

xiaoli-llm-chat/vocab_full.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c66bc101096011732fcf3ce3f033a7478fe3f9d8e98b5e64bb6f3e03f0d82b85
+size 999336

xiaoli-llm-chat/vocab_v2.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330
+size 41185