feat: v1.3-lite-flash 训练代码 + 新词表(vocab_size=47725)

- 用 CharTokenizer.build_vocab 从 977K 数据生成词表 (vocab_size=47725)
- train_cnb.py: batch=224 + accum=2 (等效448) + AMP + torch.compile
- build_vocab.py: 词表生成脚本
- vocab_full.json: 含特殊token完整词表
- 模型 41.85M参数, L40 GPU训练

.gitignore

@@ -5,9 +5,11 @@ __pycache__/
 *.log
 /tmp/
 
-# 模型权重 — 只追踪发布版本
+# 模型权重 — 只追踪发布版本(精简推理版,不含optimizer,约60MB)
 *.pth
 !/llm_model_v2_resume_epoch_7.pth
+!/xiaoli-llm-chat/xiaoli_v2_best_inference.pth
+!/xiaoli-llm-chat/xiaoli_v2_final_inference.pth
 
 # 中间输出目录（生成脚本的输出）
 high_temp_outputs/
@@ -59,3 +61,24 @@ multiturn_outputs/
 # 本地临时
 fun_test.py
 *.log
+
+# 打包/发布产物
+xiaoli_package/
+*.zip
+
+# 上传/测试脚本
+upload_*.py
+_gofile_*
+_t.*
+_up_test.*
+
+# 旧词表备份
+vocab_old.json
+vocab_backup*.json
+
+# 数据目录（太大不进git）
+data/
+
+# 训练checkpoint（不进git，通过release发布）
+xiaoli_cnb_best.pth
+xiaoli_cnb_final.pth

api_correction.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+API生成模块 — Astron 5并发，稳定不超限
+"""
+import json, requests, time, os, threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
+ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
+ASTRON_MODEL = "astron-code-latest"
+
+SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
+
+你的人设：
+- 8岁女孩子，有点害羞但很可爱
+- 说话简单自然，符合小朋友的语气
+- 回答问题要简短，1-2句话就够了
+- 不要用"喵"结尾，不要过度卖萌
+
+回答要求：
+1. 语气自然，像真实的小朋友说话
+2. 不懂就说"不知道"
+3. 回答简洁但要说清楚，不要讲一半就停"""
+
+
+def call_astron(q, temperature=0.85):
+    for attempt in range(3):
+        try:
+            headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
+            payload = {
+                "model": ASTRON_MODEL,
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": q}
+                ],
+                "temperature": temperature,
+                "max_tokens": 8192,
+            }
+            resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
+            if resp.status_code == 200:
+                content = resp.json()['choices'][0]['message']['content'].strip()
+                if content:
+                    return content
+            else:
+                time.sleep(1)
+        except Exception:
+            time.sleep(2)
+    return None
+
+
+def batch_generate(questions, temperature=0.85, max_workers=4):
+    """Astron 4并发 — 明确限制，稳定可靠"""
+    total = len(questions)
+    results = []
+    
+    def call(q):
+        ans = call_astron(q, temperature)
+        if ans:
+            return q, ans
+        return None
+    
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(call, item): i for i, item in enumerate(questions)}
+        done = 0
+        t0 = time.time()
+        for f in as_completed(futures):
+            done += 1
+            r = f.result()
+            if r:
+                results.append({"question": r[0], "answer": r[1]})
+            if done % 20 == 0 or done == total:
+                elapsed = max(time.time() - t0, 1)
+                rate = len(results) / elapsed * 60
+                print(f"    Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r")
+    
+    elapsed = time.time() - t0
+    rate = len(results) / max(elapsed, 1) * 60
+    print(f"    Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
+    return results
+
+
+# 保持旧函数名兼容
+batch_generate_astron = batch_generate
+batch_generate_mimo = batch_generate
+batch_generate_direct = batch_generate
+batch_generate_dual = batch_generate
+
+
+def batch_convert(items, *args, **kwargs):
+    """兼容占位：训练脚本导入用，已弃用"""
+    print("  [API] batch_convert已弃用，使用batch_generate")
+    return []
+
+
+def batch_generate_multi_temp(questions, *args, **kwargs):
+    """兼容占位：训练脚本导入用，已弃用"""
+    print("  [API] batch_generate_multi_temp已弃用，使用batch_generate")
+    return []

build_vocab.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+用已有的 CharTokenizer + build_vocabulary 从训练数据生成词表
+然后直接启动训练
+"""
+import sys, os, json, time
+sys.path.insert(0, '/workspace/xiaoli-llm-chat')
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
+
+from llm_chat_improved import tokenizer_instance, build_vocabulary
+
+DATA_PATH = '/workspace/data/merged_clean_v3.json'
+VOCAB_OUT = '/workspace/vocab.json'
+VOCAB_OUT_XIAOLI = '/workspace/xiaoli-llm-chat/vocab.json'
+
+def load_pairs(path):
+    """加载训练数据为 (question, answer) 对"""
+    with open(path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    
+    pairs = []
+    for item in data:
+        if isinstance(item, dict):
+            q = item.get("question", "") or item.get("input", "") or item.get("prompt", "")
+            a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
+            if q and a:
+                pairs.append((q, a))
+    return pairs
+
+def main():
+    print("=" * 60)
+    print("用 build_vocabulary 从训练数据生成词表")
+    print("=" * 60)
+    
+    # 1. 加载训练数据
+    t0 = time.time()
+    print("\n[1] 加载训练数据...")
+    pairs = load_pairs(DATA_PATH)
+    print(f"  数据量: {len(pairs)} 对")
+    print(f"  耗时: {time.time()-t0:.1f}s")
+    
+    # 2. 用 build_vocabulary 生成词表
+    t1 = time.time()
+    print("\n[2] 用 build_vocabulary 分析数据生成词表...")
+    vocab = build_vocabulary(pairs, min_freq=1)
+    
+    # build_vocabulary 返回的是 tokenizer_instance.char_to_id
+    # 已包含特殊token (<PAD>=0, <UNK>=1, etc.)
+    special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
+                      '<START>': 4, '<END>': 5, '<SEP>': 6}
+    
+    real_vocab_size = max(vocab.values()) + 1
+    
+    # 统计
+    single_char_tokens = sum(1 for k in vocab if len(k) == 1)
+    multi_char_tokens = sum(1 for k in vocab if len(k) > 1 and not k.startswith('<'))
+    special_token_count = sum(1 for k in vocab if k.startswith('<'))
+    
+    print(f"  词表总项: {len(vocab)}")
+    print(f"  vocab_size(模型用): {real_vocab_size}")
+    print(f"  单字符token: {single_char_tokens}")
+    print(f"  多字符token(英文词): {multi_char_tokens}")
+    print(f"  特殊token: {special_token_count}")
+    print(f"  耗时: {time.time()-t1:.1f}s")
+    
+    # 3. 编码测试
+    print("\n[3] 编码测试:")
+    test_texts = [
+        "你好呀！我是小狸～",
+        "小狸最喜欢吃草莓冰淇淋了！",
+        "<SOS>你好<SEP><START>我是小狸～<END><EOS>",
+    ]
+    for text in test_texts:
+        ids = tokenizer_instance.encode(text)
+        decoded = tokenizer_instance.decode(ids)
+        print(f"  原文: '{text}'")
+        print(f"  编码: {ids[:20]}... (长度={len(ids)})")
+        print(f"  解码: '{decoded}'")
+        print()
+    
+    # 4. 估算模型参数量
+    print("\n[4] 模型参数量估算 (d_model=320, n_layers=6, d_ff=1536):")
+    d_model = 320
+    n_layers = 6
+    d_ff = 1536
+    vs = real_vocab_size
+    emb = vs * d_model
+    out = vs * d_model
+    per_layer = 4 * d_model**2 + 3 * d_model * d_ff + 2 * d_model
+    total = emb + out + n_layers * per_layer + 2 * d_model
+    print(f"  vocab_size={vs}")
+    print(f"  Embedding: {emb:,}")
+    print(f"  fc_out: {out:,}")
+    print(f"  Transformer({n_layers}层): {n_layers*per_layer:,}")
+    print(f"  总参数量: {total:,} ({total/1e6:.2f}M)")
+    
+    # 5. 保存词表
+    print(f"\n[5] 保存词表...")
+    
+    # vocab.json 格式：不含特殊token，id从7开始（与 train.py 兼容）
+    normal_vocab = {k: v for k, v in vocab.items() if k not in special_tokens}
+    
+    # 备份旧词表
+    for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
+        if os.path.exists(path):
+            backup = path.replace('vocab.json', 'vocab_old.json')
+            import shutil
+            shutil.copy2(path, backup)
+            print(f"  旧词表已备份: {backup}")
+    
+    # 保存到两个位置
+    for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(normal_vocab, f, ensure_ascii=False, indent=1)
+        print(f"  {path}: {len(normal_vocab)}项 (不含特殊token, id从7开始)")
+    
+    # 保存完整词表（含特殊token，用于参考）
+    full_path = VOCAB_OUT_XIAOLI.replace('vocab.json', 'vocab_full.json')
+    with open(full_path, 'w', encoding='utf-8') as f:
+        json.dump(vocab, f, ensure_ascii=False, indent=1)
+    print(f"  {full_path}: {len(vocab)}项 (含特殊token)")
+    
+    total_time = time.time() - t0
+    print(f"\n总耗时: {total_time:.1f}s")
+    print("=" * 60)
+    print(f"词表生成完成！vocab_size = {real_vocab_size}")
+    print(f"模型参数量 ≈ {total/1e6:.2f}M")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

llm_chat_improved.py

@@ -7,6 +7,7 @@ LLM对话AI训练代码 - 改进版
 
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
 import torch.optim as optim
 from torch.utils.data import Dataset, DataLoader
 import math
@@ -235,21 +236,13 @@ class Attention(nn.Module):
         xk = xk.transpose(1, 2)
         xv = xv.transpose(1, 2)
         
-        # 计算注意力分数
+        # 融合注意力（SDPA / FlashAttention-2）- 纯因果掩码，等价于原下三角 mask
-        scores = torch.matmul(xq, xk.transpose(-2, -1)) / math.sqrt(self.d_head)
+        dropout_p = self.dropout.p if self.training else 0.0
-        
+        output = F.scaled_dot_product_attention(
-        # 应用掩码
+            xq, xk, xv, attn_mask=None, dropout_p=dropout_p, is_causal=True)
-        if mask is not None:
+
-            scores = scores.masked_fill(mask == 0, float('-inf'))
-        
-        # Softmax
-        attn_weights = torch.softmax(scores, dim=-1)
-        attn_weights = self.dropout(attn_weights)
-        
-        # 注意力输出
-        output = torch.matmul(attn_weights, xv)
         output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
-        
+
         return self.wo(output)
 
 
@@ -583,6 +576,9 @@ class LLMTrainer:
     def __init__(self, model: nn.Module, device: str = 'cpu', lr: float = 0.0001, use_amp: bool = None):
         self.model = model.to(device)
         self.device = device
+        # 火力全开：TF32 加速 fp32 矩阵乘法（L40/A100/H100 原生支持）
+        torch.backends.cuda.matmul.allow_tf32 = True
+        torch.backends.cudnn.allow_tf32 = True
         self.criterion = nn.CrossEntropyLoss(ignore_index=0)
         self.optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
         self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=100)
@@ -618,7 +614,9 @@ class LLMTrainer:
             
             # 生成因果掩码
             mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
-            
+
+            # CUDA graphs step 边界标记（torch.compile reduce-overhead 必需，避免输出 buffer 被覆盖）
+            torch.compiler.cudagraph_mark_step_begin()
             # 前向传播
             if self.use_amp:
                 with torch.cuda.amp.autocast():
@@ -677,30 +675,35 @@ class LLMTrainer:
                 target_ids = batch['target_ids'].to(self.device, non_blocking=True)
                 
                 mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
+                torch.compiler.cudagraph_mark_step_begin()
                 outputs = self.model(input_ids, mask)
                 outputs = outputs.reshape(-1, outputs.size(-1))
                 tgt = target_ids.reshape(-1)
                 
                 loss = self.criterion(outputs, tgt)
                 total_loss += loss.item()
-        
+
+        if len(dataloader) == 0:
+            return float('inf')
         return total_loss / len(dataloader)
     
     def save_model(self, filepath: str, vocab: dict, epoch: int = None):
+        # 兼容 torch.compile：取原始（未编译）模块，避免 state_dict 带 _orig_mod. 前缀
+        core = getattr(self.model, '_orig_mod', self.model)
         save_dict = {
-            'model_state_dict': self.model.state_dict(),
+            'model_state_dict': core.state_dict(),
             'optimizer_state_dict': self.optimizer.state_dict(),
             'vocab': vocab,
             'model_config': {
-                'vocab_size': self.model.vocab_size,
+                'vocab_size': core.vocab_size,
-                'd_model': self.model.d_model,
+                'd_model': core.d_model,
-                'n_heads': self.model.n_heads,
+                'n_heads': core.n_heads,
-                'n_layers': self.model.n_layers,
+                'n_layers': core.n_layers,
-                'd_ff': self.model.d_ff,
+                'd_ff': core.d_ff,
-                'max_len': self.model.max_len,
+                'max_len': core.max_len,
-                'use_memory': self.model.use_memory,
+                'use_memory': core.use_memory,
-                'use_thinking': self.model.use_thinking,
+                'use_thinking': core.use_thinking,
-                'use_moe': self.model.use_moe
+                'use_moe': core.use_moe
             }
         }
         if epoch is not None:
@@ -711,7 +714,8 @@ class LLMTrainer:
     
     def load_model(self, filepath: str):
         checkpoint = torch.load(filepath, map_location=self.device)
-        self.model.load_state_dict(checkpoint['model_state_dict'])
+        core = getattr(self.model, '_orig_mod', self.model)
+        core.load_state_dict(checkpoint['model_state_dict'])
         self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
         vocab = checkpoint['vocab']
         print(f"模型已从 {filepath} 加载")

scripts/train_cnb.py

@@ -76,15 +76,26 @@ def main():
     vocab_path = os.path.join(MODEL_DIR, 'vocab.json')
     with open(vocab_path, 'r', encoding='utf-8') as f:
         vocab = json.load(f)
-    print(f"词表: {len(vocab)} (BPE)")
+    # vocab.json 不含特殊token(0-6)，需合并进来，否则：
+    # 1) tokenizer 编码 <SOS>/<SEP>/<EOS> 等标记时会逐字符错码
+    # 2) vocab 的 id 范围是 7..max，超过 len(vocab) 会造成 embedding 索引越界
+    special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
+                      '<START>': 4, '<END>': 5, '<SEP>': 6}
+    for tok, tid in special_tokens.items():
+        if tok not in vocab:
+            vocab[tok] = tid
+    real_vocab_size = max(vocab.values()) + 1
+    print(f"词表: {len(vocab)}项, vocab_size={real_vocab_size} (字符+BPE, 来自 vocab.json)")
     tokenizer_instance.char_to_id = vocab.copy()
     tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
+    tokenizer_instance.special_tokens = special_tokens.copy()
+    tokenizer_instance._sorted_vocab_cache = None
     for attr in ['_multi_trie', '_multi_keys', '_single_map']:
         if hasattr(tokenizer_instance, attr):
             delattr(tokenizer_instance, attr)
 
     # ====== 模型 ======
-    model = LLMModel(vocab_size=max(vocab.values()) + 1, d_model=320, n_heads=8,
+    model = LLMModel(vocab_size=real_vocab_size, d_model=320, n_heads=8,
                      n_layers=6, d_ff=1536, max_len=256, dropout=0.1,
                      use_memory=False, use_thinking=False, use_moe=False)
     total = sum(p.numel() for p in model.parameters())
@@ -96,7 +107,7 @@ def main():
         if ckpt_path:
             ckpt = torch.load(ckpt_path, map_location=device)
             old_cfg = ckpt.get('model_config', {})
-            if (old_cfg.get('vocab_size', 0) == len(vocab)
+            if (old_cfg.get('vocab_size', 0) == real_vocab_size
                 and old_cfg.get('d_model', 0) == 320
                 and old_cfg.get('n_layers', 0) == 6):
                 model.load_state_dict(ckpt['model_state_dict'])
@@ -109,12 +120,25 @@ def main():
     
     model = model.to(device)
 
+    # ====== 火力全开：torch.compile (inductor融合, 无CUDA graphs) + 梯度累积 ======
+    # 物理batch=192(不OOM), accum=2 → 等效batch=384
+    fwd_model = model
+    if device == 'cuda':
+        print("🔥 启用 torch.compile (mode=default, inductor融合)...")
+        fwd_model = torch.compile(model, mode='default')
+        print("✓ torch.compile 已就绪（首步触发编译约30-60s）")
+
     # ====== DataLoader ======
+    n_workers = 16  # CPU火力全开，16核全用
     train_ds = ConversationDataset(train_set, vocab, max_length=256)
-    train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True)
+    train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True,
+                              num_workers=n_workers, pin_memory=(device == 'cuda'),
+                              persistent_workers=True, prefetch_factor=4, drop_last=True)
     val_ds = ConversationDataset(val_set, vocab, max_length=256)
-    val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False)
+    val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False,
-    print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)}")
+                            num_workers=n_workers, pin_memory=(device == 'cuda'),
+                            persistent_workers=True, prefetch_factor=4, drop_last=True)
+    print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)} | num_workers={n_workers} | prefetch=4 | drop_last=True")
 
     # ====== Trainer ======
     config = {
@@ -125,7 +149,7 @@ def main():
         'd_ff': 1536, 'max_length': 256, 'dropout': 0.1,
         'use_amp': device == 'cuda',
     }
-    trainer = LLMTrainer(model, device, lr=args.lr, use_amp=config['use_amp'])
+    trainer = LLMTrainer(fwd_model, device, lr=args.lr, use_amp=config['use_amp'])
 
     # ====== 训练 ======
     print(f"\n开始训练 epoch {start_epoch+1} → {args.epochs}")

vocab.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd80a63aa00c80a755c7e0cb4d91263bedbaccaf66fd3b12428017b4230ecbe9
+oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
-size 106005
+size 999243

xiaoli-llm-chat/vocab.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
+size 999243

xiaoli-llm-chat/vocab_full.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c66bc101096011732fcf3ce3f033a7478fe3f9d8e98b5e64bb6f3e03f0d82b85
+size 999336

xiaoli-llm-chat/vocab_v2.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330
+size 41185