diff --git a/.gitignore b/.gitignore index d00222d..59a948d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,11 @@ __pycache__/ *.log /tmp/ -# 模型权重 — 只追踪发布版本 +# 模型权重 — 只追踪发布版本(精简推理版,不含optimizer,约60MB) *.pth !/llm_model_v2_resume_epoch_7.pth +!/xiaoli-llm-chat/xiaoli_v2_best_inference.pth +!/xiaoli-llm-chat/xiaoli_v2_final_inference.pth # 中间输出目录（生成脚本的输出） high_temp_outputs/ @@ -59,3 +61,24 @@ multiturn_outputs/ # 本地临时 fun_test.py *.log + +# 打包/发布产物 +xiaoli_package/ +*.zip + +# 上传/测试脚本 +upload_*.py +_gofile_* +_t.* +_up_test.* + +# 旧词表备份 +vocab_old.json +vocab_backup*.json + +# 数据目录（太大不进git） +data/ + +# 训练checkpoint（不进git，通过release发布） +xiaoli_cnb_best.pth +xiaoli_cnb_final.pth diff --git a/api_correction.py b/api_correction.py new file mode 100644 index 0000000..dabe22b --- /dev/null +++ b/api_correction.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +API生成模块 — Astron 5并发，稳定不超限 +""" +import json, requests, time, os, threading +from concurrent.futures import ThreadPoolExecutor, as_completed + +ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") +ASTRON_MODEL = "astron-code-latest" + +SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 + +你的人设： +- 8岁女孩子，有点害羞但很可爱 +- 说话简单自然，符合小朋友的语气 +- 回答问题要简短，1-2句话就够了 +- 不要用"喵"结尾，不要过度卖萌 + +回答要求： +1. 语气自然，像真实的小朋友说话 +2. 不懂就说"不知道" +3. 回答简洁但要说清楚，不要讲一半就停""" + + +def call_astron(q, temperature=0.85): + for attempt in range(3): + try: + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} + payload = { + "model": ASTRON_MODEL, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": q} + ], + "temperature": temperature, + "max_tokens": 8192, + } + resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) + if resp.status_code == 200: + content = resp.json()['choices'][0]['message']['content'].strip() + if content: + return content + else: + time.sleep(1) + except Exception: + time.sleep(2) + return None + + +def batch_generate(questions, temperature=0.85, max_workers=4): + """Astron 4并发 — 明确限制，稳定可靠""" + total = len(questions) + results = [] + + def call(q): + ans = call_astron(q, temperature) + if ans: + return q, ans + return None + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = {executor.submit(call, item): i for i, item in enumerate(questions)} + done = 0 + t0 = time.time() + for f in as_completed(futures): + done += 1 + r = f.result() + if r: + results.append({"question": r[0], "answer": r[1]}) + if done % 20 == 0 or done == total: + elapsed = max(time.time() - t0, 1) + rate = len(results) / elapsed * 60 + print(f" Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r") + + elapsed = time.time() - t0 + rate = len(results) / max(elapsed, 1) * 60 + print(f" Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)") + return results + + +# 保持旧函数名兼容 +batch_generate_astron = batch_generate +batch_generate_mimo = batch_generate +batch_generate_direct = batch_generate +batch_generate_dual = batch_generate + + +def batch_convert(items, *args, **kwargs): + """兼容占位：训练脚本导入用，已弃用""" + print(" [API] batch_convert已弃用，使用batch_generate") + return [] + + +def batch_generate_multi_temp(questions, *args, **kwargs): + """兼容占位：训练脚本导入用，已弃用""" + print(" [API] batch_generate_multi_temp已弃用，使用batch_generate") + return [] \ No newline at end of file diff --git a/build_vocab.py b/build_vocab.py new file mode 100644 index 0000000..92dc3ad --- /dev/null +++ b/build_vocab.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +用已有的 CharTokenizer + build_vocabulary 从训练数据生成词表 +然后直接启动训练 +""" +import sys, os, json, time +sys.path.insert(0, '/workspace/xiaoli-llm-chat') +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) +sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1) + +from llm_chat_improved import tokenizer_instance, build_vocabulary + +DATA_PATH = '/workspace/data/merged_clean_v3.json' +VOCAB_OUT = '/workspace/vocab.json' +VOCAB_OUT_XIAOLI = '/workspace/xiaoli-llm-chat/vocab.json' + +def load_pairs(path): + """加载训练数据为 (question, answer) 对""" + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + + pairs = [] + for item in data: + if isinstance(item, dict): + q = item.get("question", "") or item.get("input", "") or item.get("prompt", "") + a = item.get("answer", "") or item.get("output", "") or item.get("response", "") + if q and a: + pairs.append((q, a)) + return pairs + +def main(): + print("=" * 60) + print("用 build_vocabulary 从训练数据生成词表") + print("=" * 60) + + # 1. 加载训练数据 + t0 = time.time() + print("\n[1] 加载训练数据...") + pairs = load_pairs(DATA_PATH) + print(f" 数据量: {len(pairs)} 对") + print(f" 耗时: {time.time()-t0:.1f}s") + + # 2. 用 build_vocabulary 生成词表 + t1 = time.time() + print("\n[2] 用 build_vocabulary 分析数据生成词表...") + vocab = build_vocabulary(pairs, min_freq=1) + + # build_vocabulary 返回的是 tokenizer_instance.char_to_id + # 已包含特殊token (=0, =1, etc.) + special_tokens = {'': 0, '': 1, '': 2, '': 3, + '': 4, '': 5, '': 6} + + real_vocab_size = max(vocab.values()) + 1 + + # 统计 + single_char_tokens = sum(1 for k in vocab if len(k) == 1) + multi_char_tokens = sum(1 for k in vocab if len(k) > 1 and not k.startswith('<')) + special_token_count = sum(1 for k in vocab if k.startswith('<')) + + print(f" 词表总项: {len(vocab)}") + print(f" vocab_size(模型用): {real_vocab_size}") + print(f" 单字符token: {single_char_tokens}") + print(f" 多字符token(英文词): {multi_char_tokens}") + print(f" 特殊token: {special_token_count}") + print(f" 耗时: {time.time()-t1:.1f}s") + + # 3. 编码测试 + print("\n[3] 编码测试:") + test_texts = [ + "你好呀！我是小狸～", + "小狸最喜欢吃草莓冰淇淋了！", + "你好我是小狸～", + ] + for text in test_texts: + ids = tokenizer_instance.encode(text) + decoded = tokenizer_instance.decode(ids) + print(f" 原文: '{text}'") + print(f" 编码: {ids[:20]}... (长度={len(ids)})") + print(f" 解码: '{decoded}'") + print() + + # 4. 估算模型参数量 + print("\n[4] 模型参数量估算 (d_model=320, n_layers=6, d_ff=1536):") + d_model = 320 + n_layers = 6 + d_ff = 1536 + vs = real_vocab_size + emb = vs * d_model + out = vs * d_model + per_layer = 4 * d_model**2 + 3 * d_model * d_ff + 2 * d_model + total = emb + out + n_layers * per_layer + 2 * d_model + print(f" vocab_size={vs}") + print(f" Embedding: {emb:,}") + print(f" fc_out: {out:,}") + print(f" Transformer({n_layers}层): {n_layers*per_layer:,}") + print(f" 总参数量: {total:,} ({total/1e6:.2f}M)") + + # 5. 保存词表 + print(f"\n[5] 保存词表...") + + # vocab.json 格式：不含特殊token，id从7开始（与 train.py 兼容） + normal_vocab = {k: v for k, v in vocab.items() if k not in special_tokens} + + # 备份旧词表 + for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]: + if os.path.exists(path): + backup = path.replace('vocab.json', 'vocab_old.json') + import shutil + shutil.copy2(path, backup) + print(f" 旧词表已备份: {backup}") + + # 保存到两个位置 + for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]: + with open(path, 'w', encoding='utf-8') as f: + json.dump(normal_vocab, f, ensure_ascii=False, indent=1) + print(f" {path}: {len(normal_vocab)}项 (不含特殊token, id从7开始)") + + # 保存完整词表（含特殊token，用于参考） + full_path = VOCAB_OUT_XIAOLI.replace('vocab.json', 'vocab_full.json') + with open(full_path, 'w', encoding='utf-8') as f: + json.dump(vocab, f, ensure_ascii=False, indent=1) + print(f" {full_path}: {len(vocab)}项 (含特殊token)") + + total_time = time.time() - t0 + print(f"\n总耗时: {total_time:.1f}s") + print("=" * 60) + print(f"词表生成完成！vocab_size = {real_vocab_size}") + print(f"模型参数量 ≈ {total/1e6:.2f}M") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/llm_chat_improved.py b/llm_chat_improved.py index 9e30385..62aec67 100644 --- a/llm_chat_improved.py +++ b/llm_chat_improved.py @@ -7,6 +7,7 @@ LLM对话AI训练代码 - 改进版 import torch import torch.nn as nn +import torch.nn.functional as F import torch.optim as optim from torch.utils.data import Dataset, DataLoader import math @@ -235,21 +236,13 @@ class Attention(nn.Module): xk = xk.transpose(1, 2) xv = xv.transpose(1, 2) - # 计算注意力分数 - scores = torch.matmul(xq, xk.transpose(-2, -1)) / math.sqrt(self.d_head) - - # 应用掩码 - if mask is not None: - scores = scores.masked_fill(mask == 0, float('-inf')) - - # Softmax - attn_weights = torch.softmax(scores, dim=-1) - attn_weights = self.dropout(attn_weights) - - # 注意力输出 - output = torch.matmul(attn_weights, xv) + # 融合注意力（SDPA / FlashAttention-2）- 纯因果掩码，等价于原下三角 mask + dropout_p = self.dropout.p if self.training else 0.0 + output = F.scaled_dot_product_attention( + xq, xk, xv, attn_mask=None, dropout_p=dropout_p, is_causal=True) + output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model) - + return self.wo(output) @@ -583,6 +576,9 @@ class LLMTrainer: def __init__(self, model: nn.Module, device: str = 'cpu', lr: float = 0.0001, use_amp: bool = None): self.model = model.to(device) self.device = device + # 火力全开：TF32 加速 fp32 矩阵乘法（L40/A100/H100 原生支持） + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True self.criterion = nn.CrossEntropyLoss(ignore_index=0) self.optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01) self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=100) @@ -618,7 +614,9 @@ class LLMTrainer: # 生成因果掩码 mask = self.model.generate_mask(input_ids.size(1)).to(self.device) - + + # CUDA graphs step 边界标记（torch.compile reduce-overhead 必需，避免输出 buffer 被覆盖） + torch.compiler.cudagraph_mark_step_begin() # 前向传播 if self.use_amp: with torch.cuda.amp.autocast(): @@ -677,30 +675,35 @@ class LLMTrainer: target_ids = batch['target_ids'].to(self.device, non_blocking=True) mask = self.model.generate_mask(input_ids.size(1)).to(self.device) + torch.compiler.cudagraph_mark_step_begin() outputs = self.model(input_ids, mask) outputs = outputs.reshape(-1, outputs.size(-1)) tgt = target_ids.reshape(-1) loss = self.criterion(outputs, tgt) total_loss += loss.item() - + + if len(dataloader) == 0: + return float('inf') return total_loss / len(dataloader) def save_model(self, filepath: str, vocab: dict, epoch: int = None): + # 兼容 torch.compile：取原始（未编译）模块，避免 state_dict 带 _orig_mod. 前缀 + core = getattr(self.model, '_orig_mod', self.model) save_dict = { - 'model_state_dict': self.model.state_dict(), + 'model_state_dict': core.state_dict(), 'optimizer_state_dict': self.optimizer.state_dict(), 'vocab': vocab, 'model_config': { - 'vocab_size': self.model.vocab_size, - 'd_model': self.model.d_model, - 'n_heads': self.model.n_heads, - 'n_layers': self.model.n_layers, - 'd_ff': self.model.d_ff, - 'max_len': self.model.max_len, - 'use_memory': self.model.use_memory, - 'use_thinking': self.model.use_thinking, - 'use_moe': self.model.use_moe + 'vocab_size': core.vocab_size, + 'd_model': core.d_model, + 'n_heads': core.n_heads, + 'n_layers': core.n_layers, + 'd_ff': core.d_ff, + 'max_len': core.max_len, + 'use_memory': core.use_memory, + 'use_thinking': core.use_thinking, + 'use_moe': core.use_moe } } if epoch is not None: @@ -711,7 +714,8 @@ class LLMTrainer: def load_model(self, filepath: str): checkpoint = torch.load(filepath, map_location=self.device) - self.model.load_state_dict(checkpoint['model_state_dict']) + core = getattr(self.model, '_orig_mod', self.model) + core.load_state_dict(checkpoint['model_state_dict']) self.optimizer.load_state_dict(checkpoint['optimizer_state_dict']) vocab = checkpoint['vocab'] print(f"模型已从 {filepath} 加载") diff --git a/scripts/train_cnb.py b/scripts/train_cnb.py index e4ca109..75e7c17 100644 --- a/scripts/train_cnb.py +++ b/scripts/train_cnb.py @@ -76,15 +76,26 @@ def main(): vocab_path = os.path.join(MODEL_DIR, 'vocab.json') with open(vocab_path, 'r', encoding='utf-8') as f: vocab = json.load(f) - print(f"词表: {len(vocab)} (BPE)") + # vocab.json 不含特殊token(0-6)，需合并进来，否则： + # 1) tokenizer 编码 // 等标记时会逐字符错码 + # 2) vocab 的 id 范围是 7..max，超过 len(vocab) 会造成 embedding 索引越界 + special_tokens = {'': 0, '': 1, '': 2, '': 3, + '': 4, '': 5, '': 6} + for tok, tid in special_tokens.items(): + if tok not in vocab: + vocab[tok] = tid + real_vocab_size = max(vocab.values()) + 1 + print(f"词表: {len(vocab)}项, vocab_size={real_vocab_size} (字符+BPE, 来自 vocab.json)") tokenizer_instance.char_to_id = vocab.copy() tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()} + tokenizer_instance.special_tokens = special_tokens.copy() + tokenizer_instance._sorted_vocab_cache = None for attr in ['_multi_trie', '_multi_keys', '_single_map']: if hasattr(tokenizer_instance, attr): delattr(tokenizer_instance, attr) # ====== 模型 ====== - model = LLMModel(vocab_size=max(vocab.values()) + 1, d_model=320, n_heads=8, + model = LLMModel(vocab_size=real_vocab_size, d_model=320, n_heads=8, n_layers=6, d_ff=1536, max_len=256, dropout=0.1, use_memory=False, use_thinking=False, use_moe=False) total = sum(p.numel() for p in model.parameters()) @@ -96,7 +107,7 @@ def main(): if ckpt_path: ckpt = torch.load(ckpt_path, map_location=device) old_cfg = ckpt.get('model_config', {}) - if (old_cfg.get('vocab_size', 0) == len(vocab) + if (old_cfg.get('vocab_size', 0) == real_vocab_size and old_cfg.get('d_model', 0) == 320 and old_cfg.get('n_layers', 0) == 6): model.load_state_dict(ckpt['model_state_dict']) @@ -109,12 +120,25 @@ def main(): model = model.to(device) + # ====== 火力全开：torch.compile (inductor融合, 无CUDA graphs) + 梯度累积 ====== + # 物理batch=192(不OOM), accum=2 → 等效batch=384 + fwd_model = model + if device == 'cuda': + print("🔥 启用 torch.compile (mode=default, inductor融合)...") + fwd_model = torch.compile(model, mode='default') + print("✓ torch.compile 已就绪（首步触发编译约30-60s）") + # ====== DataLoader ====== + n_workers = 16 # CPU火力全开，16核全用 train_ds = ConversationDataset(train_set, vocab, max_length=256) - train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True) + train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True, + num_workers=n_workers, pin_memory=(device == 'cuda'), + persistent_workers=True, prefetch_factor=4, drop_last=True) val_ds = ConversationDataset(val_set, vocab, max_length=256) - val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False) - print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)}") + val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False, + num_workers=n_workers, pin_memory=(device == 'cuda'), + persistent_workers=True, prefetch_factor=4, drop_last=True) + print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)} | num_workers={n_workers} | prefetch=4 | drop_last=True") # ====== Trainer ====== config = { @@ -125,7 +149,7 @@ def main(): 'd_ff': 1536, 'max_length': 256, 'dropout': 0.1, 'use_amp': device == 'cuda', } - trainer = LLMTrainer(model, device, lr=args.lr, use_amp=config['use_amp']) + trainer = LLMTrainer(fwd_model, device, lr=args.lr, use_amp=config['use_amp']) # ====== 训练 ====== print(f"\n开始训练 epoch {start_epoch+1} → {args.epochs}") diff --git a/vocab.json b/vocab.json index b932bca..bdb580c 100644 --- a/vocab.json +++ b/vocab.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cd80a63aa00c80a755c7e0cb4d91263bedbaccaf66fd3b12428017b4230ecbe9 -size 106005 +oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8 +size 999243 diff --git a/xiaoli-llm-chat/vocab.json b/xiaoli-llm-chat/vocab.json new file mode 100644 index 0000000..bdb580c --- /dev/null +++ b/xiaoli-llm-chat/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8 +size 999243 diff --git a/xiaoli-llm-chat/vocab_full.json b/xiaoli-llm-chat/vocab_full.json new file mode 100644 index 0000000..baee9cb --- /dev/null +++ b/xiaoli-llm-chat/vocab_full.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c66bc101096011732fcf3ce3f033a7478fe3f9d8e98b5e64bb6f3e03f0d82b85 +size 999336 diff --git a/xiaoli-llm-chat/vocab_v2.json b/xiaoli-llm-chat/vocab_v2.json new file mode 100644 index 0000000..3de7706 --- /dev/null +++ b/xiaoli-llm-chat/vocab_v2.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330 +size 41185