delete test file

- api_correction.py: 嵌入 Astron key, max_workers=5, 修复 batch_convert/multi_temp 返回空列表bug
- scripts/api_correction.py: 同步更新
- train_max.py: 两阶段训练(预训练+角色对齐), Warmup+Cosine, 分层LR, AMP
- distill_daemon.py: 持续蒸馏守护脚本, 15大类种子话题, 5并发

data/expanded_distilled.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea54f7fc24d33f096626babb1d24c15b492451494d4417c814aa310df43a20c4
+size 1483630

setup_cloud.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+# -*- coding: utf-8 -*-
+"""
+XiaLi 云 GPU 一键搭建脚本
+=========================
+适用于 AutoDL / 矩池云 / 恒源云 等国内云GPU平台
+
+用法:
+  bash setup_cloud.sh                    # 完整搭建
+  bash setup_cloud.sh --train-only       # 只拉代码，不下载数据
+"""
+
+set -e
+
+echo "========================================"
+echo "  XiaLi 云 GPU 训练环境搭建"
+echo "========================================"
+
+REPO_URL="https://gitee.com/shuiyu1123/xiaoli-llm-chat.git"
+WORK_DIR="/root/xiaoli-llm-chat"
+
+# 1. 克隆代码
+echo ""
+echo "[1/4] 克隆代码..."
+if [ -d "$WORK_DIR" ]; then
+    echo "  目录已存在，拉取更新..."
+    cd "$WORK_DIR" && git pull
+else
+    git clone "$REPO_URL" "$WORK_DIR"
+    cd "$WORK_DIR"
+fi
+
+# 2. 安装依赖
+echo ""
+echo "[2/4] 安装依赖..."
+pip install torch numpy -q
+# 检查 torch 是否有 CUDA
+python3 -c "import torch; print(f'CUDA可用: {torch.cuda.is_available()}, GPU数: {torch.cuda.device_count()}')" 2>/dev/null || {
+    echo "  ⚠️ 检测到无 CUDA，尝试安装 GPU 版 PyTorch..."
+    pip install torch --index-url https://download.pytorch.org/whl/cu118 -q
+}
+pip install modelscope -q
+
+# 确认词表存在
+echo ""
+echo "[3/4] 检查词表..."
+ls -la vocab.json 2>/dev/null || {
+    echo "  ⚠️ vocab.json 不存在！请确认仓库完整"
+    exit 1
+}
+python3 -c "
+import json
+with open('vocab.json') as f:
+    v = json.load(f)
+print(f'  词表: {len(v)} tokens, vocab_size={max(v.values())+1}')
+"
+
+# 4. 确认训练脚本
+echo ""
+echo "[4/4] 确认训练脚本..."
+ls -la train_cloud.py && echo "  ✅ train_cloud.py 就绪"
+
+# 显存信息
+echo ""
+echo "========================================"
+echo "  环境就绪！"
+echo "========================================"
+python3 -c "
+import torch
+if torch.cuda.is_available():
+    total = torch.cuda.get_device_properties(0).total_mem / 1024**3
+    name = torch.cuda.get_device_name(0)
+    print(f'  GPU: {name} ({total:.1f} GiB)')
+else:
+    print('  ⚠️ 未检测到 GPU！')
+"
+echo ""
+echo "启动训练命令:"
+echo "  cd $WORK_DIR"
+echo ""
+echo "  # 单卡训练（完整流程）"
+echo "  python3 train_cloud.py --epochs 5 --align-epochs 20"
+echo ""
+echo "  # 从检查点续训"
+echo "  python3 train_cloud.py --resume --epochs 10"
+echo ""
+echo "  # 24G显存用更小batch"
+echo "  python3 train_cloud.py --batch-size 32"
+echo ""
+echo "========================================"

train_cloud.py

@@ -0,0 +1,437 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+XiaLi 云端 GPU 训练脚本 v1.0
+=============================
+两阶段训练 + ModelScope 自动下载数据 + 断点续训
+
+用法（云GPU上）:
+  python3 train_cloud.py              # 默认训练
+  python3 train_cloud.py --resume      # 从最新检查点续训
+  python3 train_cloud.py --epochs 10   # 指定预训练轮数
+"""
+import sys, os, json, time, glob, re, math, random
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llm_chat_improved import (
+    CharTokenizer, LLMModel, LLMChat,
+    ConversationDataset, tokenizer_instance
+)
+
+# ── 配置 ──────────────────────────────────────────────────
+class CloudConfig:
+    # 模型架构
+    d_model: int = 320
+    n_heads: int = 8
+    n_layers: int = 6
+    d_ff: int = 1536
+    max_length: int = 256          # 对齐用
+    dropout: float = 0.1
+    use_memory: bool = False       # 关掉记忆/思考/MoE 加速
+    use_thinking: bool = False
+    use_moe: bool = False
+
+    # 预训练
+    pt_batch_size: int = 64        # 32G显存可跑64, 24G改32
+    pt_grad_accum: int = 1         # 大batch不需要累积
+    pt_epochs: int = 5
+    pt_lr: float = 5e-4
+    pt_min_lr: float = 5e-5
+    pt_warmup_steps: int = 200
+    pt_max_length: int = 256       # 长序列学更好
+    pt_weight_decay: float = 0.1
+    pt_max_norm: float = 1.0
+
+    # 对齐
+    align_batch_size: int = 32
+    align_grad_accum: int = 1
+    align_epochs: int = 20
+    align_lr: float = 1e-4
+    align_min_lr: float = 1e-5
+    align_warmup_steps: int = 50
+    align_weight_decay: float = 0.01
+    align_max_norm: float = 1.0
+
+    # 通用
+    device: str = "cuda" if torch.cuda.is_available() else "cpu"
+    num_workers: int = 4
+    use_amp: bool = True
+    seed: int = 42
+    save_dir: str = "/root/xiaoli-llm-chat"   # AutoDL 默认路径
+    log_every: int = 10
+    save_interval: int = 1
+    keep_checkpoints: int = 3
+
+
+# ── 词表加载 ──────────────────────────────────────────────
+def load_vocab(vocab_path: str) -> dict:
+    special = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
+               '<START>': 4, '<END>': 5, '<SEP>': 6}
+    with open(vocab_path, 'r') as f:
+        normal_vocab = json.load(f)
+    full_vocab = {**special, **normal_vocab}
+    # 同步到全局 tokenizer
+    tokenizer_instance.char_to_id = full_vocab.copy()
+    tokenizer_instance.id_to_char = {v: k for k, v in full_vocab.items()}
+    for attr in ['_multi_trie', '_multi_keys', '_single_map']:
+        if hasattr(tokenizer_instance, attr): delattr(tokenizer_instance, attr)
+    return full_vocab
+
+
+# ── 预训练数据集（流式加载，不爆内存）────────────────────
+class PretrainDataset(Dataset):
+    def __init__(self, data_dir: str, vocab: dict, max_length: int = 128):
+        self.vocab = vocab
+        self.max_length = max_length
+        self.pad_id = vocab.get('<PAD>', 0)
+        self.texts = []
+        data_dir = Path(data_dir)
+        for fpath in sorted(data_dir.glob("*")):
+            if fpath.suffix in ('.txt', '.json', '.jsonl'):
+                self._load_file(fpath)
+        if not self.texts:
+            raise ValueError(f"预训练数据目录为空: {data_dir}")
+        total_chars = sum(len(t) for t in self.texts)
+        print(f"  预训练数据: {len(self.texts):,} 条, {total_chars:,} 字符")
+
+    def _load_file(self, path: Path):
+        try:
+            if path.suffix == '.jsonl':
+                with open(path, 'r', encoding='utf-8', errors='ignore') as f:
+                    for line in f:
+                        try:
+                            item = json.loads(line)
+                            t = item.get('text', '').strip()
+                            if len(t) >= 10: self.texts.append(t)
+                        except: continue
+        except: pass
+
+    def __len__(self): return len(self.texts)
+
+    def __getitem__(self, idx):
+        text = self.texts[idx]
+        tokens = tokenizer_instance.encode(text)
+        if len(tokens) > self.max_length:
+            start = random.randint(0, len(tokens) - self.max_length)
+            tokens = tokens[start:start + self.max_length]
+        else:
+            tokens = tokens + [self.pad_id] * (self.max_length - len(tokens))
+        input_ids = torch.tensor(tokens[:-1], dtype=torch.long)
+        target_ids = torch.tensor(tokens[1:], dtype=torch.long)
+        loss_mask = torch.ones_like(target_ids)
+        return {'input_ids': input_ids, 'target_ids': target_ids,
+                'loss_mask': loss_mask, 'weights': 1.0}
+
+
+# ── 训练器 ──────────────────────────────────────────────
+class CloudTrainer:
+    def __init__(self, model: nn.Module, config: CloudConfig, stage: str = "pretrain"):
+        self.model = model.to(config.device)
+        self.device = config.device
+        self.config = config
+        self.stage = stage
+        self.criterion = nn.CrossEntropyLoss(ignore_index=0, label_smoothing=0.1)
+
+        if stage == "pretrain":
+            base_lr = config.pt_lr
+            wd = config.pt_weight_decay
+        else:
+            base_lr = config.align_lr
+            wd = config.align_weight_decay
+
+        # 分层学习率：embedding/fc_out 用半速
+        param_groups = []
+        for name, param in model.named_parameters():
+            if not param.requires_grad: continue
+            if 'embedding' in name or 'fc_out' in name:
+                param_groups.append({'params': param, 'lr': base_lr * 0.5, 'weight_decay': wd})
+            elif 'norm' in name or 'rmsnorm' in name:
+                param_groups.append({'params': param, 'lr': base_lr, 'weight_decay': 0.0})
+            else:
+                param_groups.append({'params': param, 'lr': base_lr, 'weight_decay': wd})
+
+        self.optimizer = torch.optim.AdamW(param_groups, lr=base_lr, betas=(0.9, 0.95), eps=1e-8)
+        self.scaler = torch.cuda.amp.GradScaler(enabled=True)
+        self.scheduler = None
+
+        if torch.cuda.is_available():
+            torch.backends.cuda.matmul.allow_tf32 = True
+            torch.backends.cudnn.allow_tf32 = True
+            torch.backends.cudnn.benchmark = True
+
+    def build_scheduler(self, total_steps: int):
+        if self.stage == "pretrain":
+            warmup = self.config.pt_warmup_steps
+            min_lr = self.config.pt_min_lr
+        else:
+            warmup = self.config.align_warmup_steps
+            min_lr = self.config.align_min_lr
+        base_lr = self.optimizer.param_groups[0]['lr']
+
+        def lr_lambda(step):
+            if step < warmup:
+                return step / max(1, warmup)
+            progress = (step - warmup) / max(1, total_steps - warmup)
+            cosine = 0.5 * (1 + math.cos(math.pi * progress))
+            return (1 - min_lr / base_lr) * cosine + min_lr / base_lr
+
+        self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda)
+
+    def train_epoch(self, dataloader, epoch, total_epochs, stage_name="pretrain"):
+        self.model.train()
+        total_loss = 0
+        n_batches = len(dataloader)
+        accum_steps = self.config.pt_grad_accum if stage_name == "pretrain" else self.config.align_grad_accum
+        max_norm = self.config.pt_max_norm if stage_name == "pretrain" else self.config.align_max_norm
+
+        self.optimizer.zero_grad()
+        for batch_idx, batch in enumerate(dataloader):
+            input_ids = batch['input_ids'].to(self.device, non_blocking=True)
+            target_ids = batch['target_ids'].to(self.device, non_blocking=True)
+            mask = self.model.generate_mask(input_ids.size(1)).to(self.device, non_blocking=True)
+
+            with torch.amp.autocast('cuda', enabled=True):
+                outputs = self.model(input_ids, mask)
+                loss = self.criterion(outputs.reshape(-1, outputs.size(-1)), target_ids.reshape(-1))
+                loss = loss / accum_steps
+
+            self.scaler.scale(loss).backward()
+
+            if (batch_idx + 1) % accum_steps == 0:
+                self.scaler.unscale_(self.optimizer)
+                torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm)
+                self.scaler.step(self.optimizer)
+                self.scaler.update()
+                self.optimizer.zero_grad()
+                if self.scheduler: self.scheduler.step()
+
+            total_loss += loss.item() * accum_steps
+
+            if (batch_idx + 1) % max(1, n_batches // 10) == 0:
+                lr = self.optimizer.param_groups[0]['lr']
+                mem = f" | Mem: {torch.cuda.max_memory_allocated()/1024**3:.1f}GiB" if torch.cuda.is_available() else ""
+                print(f"  [{stage_name}] Epoch {epoch}/{total_epochs} | "
+                      f"Batch {batch_idx+1}/{n_batches} | Loss: {total_loss/(batch_idx+1):.4f} | LR: {lr:.2e}{mem}")
+
+        return total_loss / n_batches
+
+    def save_checkpoint(self, path: str, vocab: dict, epoch: int, loss: float):
+        core = getattr(self.model, '_orig_mod', self.model)
+        torch.save({
+            'model_state_dict': core.state_dict(),
+            'optimizer_state_dict': self.optimizer.state_dict(),
+            'scheduler_state_dict': self.scheduler.state_dict() if self.scheduler else None,
+            'vocab': vocab,
+            'epoch': epoch,
+            'loss': loss,
+            'config': {k: getattr(self.config, k) for k in dir(self.config)
+                       if not k.startswith('_') and not callable(getattr(self.config, k))},
+        }, path)
+        print(f"  ✅ 已保存: {path} (epoch {epoch}, loss {loss:.4f})")
+
+    def load_checkpoint(self, path: str) -> tuple:
+        cp = torch.load(path, map_location=self.device, weights_only=False)
+        core = getattr(self.model, '_orig_mod', self.model)
+        core.load_state_dict(cp['model_state_dict'])
+        self.optimizer.load_state_dict(cp['optimizer_state_dict'])
+        if cp.get('scheduler_state_dict') and self.scheduler:
+            self.scheduler.load_state_dict(cp['scheduler_state_dict'])
+        return cp.get('epoch', 0), cp.get('loss', float('inf')), cp.get('vocab', {})
+
+
+# ── 数据下载 ──────────────────────────────────────────────
+def download_data(save_dir: str):
+    """从 ModelScope 下载预训练数据"""
+    data_dir = os.path.join(save_dir, 'pretrain_data')
+    if os.path.exists(data_dir) and list(Path(data_dir).glob("*.jsonl")):
+        print("  预训练数据已存在，跳过下载")
+        return data_dir
+
+    os.makedirs(data_dir, exist_ok=True)
+    print("  📥 从 ModelScope 下载预训练数据...")
+    try:
+        import subprocess
+        subprocess.run([
+            sys.executable, '-m', 'pip', 'install', 'modelscope', '-q'
+        ], check=True, capture_output=True)
+        subprocess.run([
+            sys.executable, '-m', 'modelscope', 'download',
+            '--dataset', 'gongjy/minimind_dataset',
+            '--include', 'pretrain_t2t_mini*',
+            '--local_dir', data_dir,
+        ], check=True)
+        print(f"  ✅ 下载完成: {data_dir}")
+    except Exception as e:
+        print(f"  ⚠️ 自动下载失败: {e}")
+        print(f"  ⚠️ 请手动下载放到 {data_dir}")
+        print(f"  ⚠️ 命令: modelscope download --dataset gongjy/minimind_dataset --include 'pretrain_t2t_mini*' --local_dir {data_dir}")
+    return data_dir
+
+
+# ── 主函数 ──────────────────────────────────────────────
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--resume", action="store_true", help="从最新检查点续训")
+    parser.add_argument("--epochs", type=int, default=5, help="预训练轮数")
+    parser.add_argument("--align-epochs", type=int, default=20, help="对齐轮数")
+    parser.add_argument("--save-dir", type=str, default="/root/xiaoli-llm-chat")
+    parser.add_argument("--batch-size", type=int, default=0, help="覆盖batch大小")
+    args = parser.parse_args()
+
+    config = CloudConfig()
+    config.pt_epochs = args.epochs
+    config.align_epochs = args.align_epochs
+    config.save_dir = args.save_dir
+
+    if args.batch_size > 0:
+        config.pt_batch_size = args.batch_size
+        config.align_batch_size = args.batch_size // 2
+
+    if not torch.cuda.is_available():
+        print("⚠️ 未检测到 GPU！云端训练建议使用 AutoDL / 矩池云 / 恒源云")
+
+    print("=" * 65)
+    print(f"  XiaLi 云端训练")
+    print(f"  GPU: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'N/A'}")
+    print(f"  显存: {torch.cuda.get_device_properties(0).total_mem / 1024**3:.1f} GiB" if torch.cuda.is_available() else "")
+    print(f"  模型: {config.d_model}d, {config.n_layers}L, {config.n_heads}H")
+    print(f"  预训练: {config.pt_epochs} epochs, batch={config.pt_batch_size}")
+    print(f"  对齐: {config.align_epochs} epochs, batch={config.align_batch_size}")
+    print("=" * 65)
+
+    os.makedirs(config.save_dir, exist_ok=True)
+
+    # 1. 加载词表
+    vocab_path = os.path.join(config.save_dir, 'vocab.json')
+    if not os.path.exists(vocab_path):
+        print(f"❌ 未找到 {vocab_path}")
+        print(f"   请从 Gitee 仓库获取: https://gitee.com/shuiyu1123/xiaoli-llm-chat")
+        return
+    vocab = load_vocab(vocab_path)
+    print(f"词表: {len(vocab)} tokens, vocab_size={max(vocab.values())+1}")
+
+    # 2. 创建模型
+    model = LLMModel(
+        vocab_size=max(vocab.values())+1,
+        d_model=config.d_model,
+        n_heads=config.n_heads,
+        n_layers=config.n_layers,
+        d_ff=config.d_ff,
+        max_len=config.max_length,
+        dropout=config.dropout,
+        use_memory=False, use_thinking=False, use_moe=False,
+    )
+    params = sum(p.numel() for p in model.parameters())
+    print(f"参数量: {params:,} ({params/1e6:.2f}M)")
+
+    # 3. 续训检测
+    start_epoch = 1
+    if args.resume:
+        checkpoints = sorted(glob.glob(os.path.join(config.save_dir, 'pretrain_epoch_*.pth')))
+        if checkpoints:
+            latest = checkpoints[-1]
+            trainer_pt = CloudTrainer(model, config, stage="pretrain")
+            loaded_epoch, loaded_loss, _ = trainer_pt.load_checkpoint(latest)
+            start_epoch = loaded_epoch + 1
+            print(f"  从 {latest} 续训 (epoch {loaded_epoch}, loss {loaded_loss:.4f})")
+
+    # 4. 下载数据
+    pt_dir = download_data(config.save_dir)
+
+    # 5. 预训练
+    print(f"\n{'='*65}")
+    print(f"  阶段一：预训练 ({config.pt_epochs} epochs)")
+    print(f"{'='*65}")
+
+    pt_dataset = PretrainDataset(pt_dir, vocab, config.pt_max_length)
+    pt_loader = DataLoader(
+        pt_dataset, batch_size=config.pt_batch_size, shuffle=True,
+        num_workers=config.num_workers,
+        pin_memory=True,
+    )
+
+    trainer = CloudTrainer(model, config, stage="pretrain")
+    total_steps = len(pt_loader) * config.pt_epochs
+    trainer.build_scheduler(total_steps)
+
+    for epoch in range(start_epoch, config.pt_epochs + 1):
+        t0 = time.time()
+        loss = trainer.train_epoch(pt_loader, epoch, config.pt_epochs, "pretrain")
+        epoch_time = time.time() - t0
+        lr = trainer.optimizer.param_groups[0]['lr']
+        print(f"  → Epoch {epoch}/{config.pt_epochs} | Loss: {loss:.4f} | LR: {lr:.2e} | 耗时: {epoch_time:.0f}s")
+
+        if epoch % config.save_interval == 0:
+            ckpt_path = os.path.join(config.save_dir, f'pretrain_epoch_{epoch}.pth')
+            trainer.save_checkpoint(ckpt_path, vocab, epoch, loss)
+            # 清理旧检查点
+            all_ckpts = sorted(glob.glob(os.path.join(config.save_dir, 'pretrain_epoch_*.pth')))
+            for old in all_ckpts[:-config.keep_checkpoints]:
+                os.remove(old)
+
+    print(f"\n  ✅ 预训练完成")
+
+    # 6. 对齐
+    print(f"\n{'='*65}")
+    print(f"  阶段二：角色对齐 ({config.align_epochs} epochs)")
+    print(f"{'='*65}")
+
+    merged_path = os.path.join(config.save_dir, 'xiaoli_merged_data_v2.json')
+    if not os.path.exists(merged_path):
+        print(f"  ⚠️ 未找到合并数据: {merged_path}")
+        print(f"  ⚠️ 跳过对齐阶段")
+    else:
+        with open(merged_path, 'r') as f:
+            raw_data = json.load(f)
+        pairs = []
+        for item in raw_data:
+            q = item.get("question", "") or item.get("input", "") or item.get("prompt", "")
+            a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
+            if q and a: pairs.append((q, a))
+
+        random.seed(config.seed)
+        random.shuffle(pairs)
+        split = int(len(pairs) * 0.95)
+        train_pairs = pairs[:split]
+        print(f"  对齐数据: {len(pairs)} 条 (训练 {len(train_pairs)} + 验证 {len(pairs)-split})")
+
+        train_ds = ConversationDataset(train_pairs, vocab, config.max_length)
+        train_loader = DataLoader(
+            train_ds, batch_size=config.align_batch_size, shuffle=True,
+            num_workers=config.num_workers, pin_memory=True,
+        )
+
+        trainer2 = CloudTrainer(model, config, stage="align")
+        total_align_steps = len(train_loader) * config.align_epochs
+        trainer2.build_scheduler(total_align_steps)
+
+        for epoch in range(1, config.align_epochs + 1):
+            t0 = time.time()
+            loss = trainer2.train_epoch(train_loader, epoch, config.align_epochs, "align")
+            epoch_time = time.time() - t0
+            print(f"  → Epoch {epoch}/{config.align_epochs} | Loss: {loss:.4f} | 耗时: {epoch_time:.0f}s")
+
+            if epoch % 5 == 0:
+                ckpt_path = os.path.join(config.save_dir, f'xiaoli_align_epoch_{epoch}.pth')
+                trainer2.save_checkpoint(ckpt_path, vocab, epoch, loss)
+
+        # 保存最终模型
+        final_path = os.path.join(config.save_dir, 'xiaoli_final.pth')
+        trainer2.save_checkpoint(final_path, vocab, config.align_epochs, loss)
+        print(f"\n  ✅ 对齐完成！最终模型: {final_path}")
+
+    print(f"\n{'='*65}")
+    print(f"  🎉 训练完成！")
+    print(f"  模型保存在: {config.save_dir}")
+    print(f"  {'='*65}")
+
+
+if __name__ == "__main__":
+    main()

vocab.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
-size 999243
+oid sha256:9746baace6ebbb410065f45ae4862d2a439e094b5477af8de6b20869adf84309
+size 314876

xiaoli_next.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+XiaLi-Next 统一架构 v1.0
+=========================
+同一套架构，从 20M 到 7B+ 通用
+核心设计：RoPE + SwiGLU + Pre-RMSNorm + GQA + Weight Tying
+
+用法:
+  from xiaoli_next import XiaLiConfig, XiaLiModel
+  
+  # 20M 超小模型
+  cfg = XiaLiConfig.preset('20m')
+  model = XiaLiModel(cfg)
+  
+  # 1B 中等模型  
+  cfg = XiaLiConfig.preset('1b')
+  model = XiaLiModel(cfg)
+  
+  # 7B 大模型
+  cfg = XiaLiConfig.preset('7b')
+  model = XiaLiModel(cfg)
+"""
+
+import math
+import json
+from typing import Optional, Tuple
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+# ════════════════════════════════════════════════════════════
+# 1. 配置系统
+# ════════════════════════════════════════════════════════════
+
+class XiaLiConfig:
+    """统一配置 - 通过 preset 快速切换规模"""
+    
+    def __init__(self):
+        # 架构
+        self.d_model: int = 320
+        self.n_layers: int = 8
+        self.n_heads: int = 8
+        self.n_kv_heads: int = 4       # GQA: KV头数（=n_heads 就是MHA，=1 就是MQA）
+        self.d_ff: int = 0             # 0=自动 = int(d_model * 8/3 * 2)  SwiGLU需要额外gate
+        self.max_seq_len: int = 2048   # 最大序列长度
+        self.dropout: float = 0.0      # 大模型dropout=0
+        self.rope_theta: float = 10000.0  # RoPE base frequency
+        self.tie_weights: bool = True  # Weight Tying
+        self.vocab_size: int = 20007
+        
+        # 训练
+        self.norm_eps: float = 1e-5
+        
+    @classmethod
+    def preset(cls, name: str) -> 'XiaLiConfig':
+        """按规模取配置"""
+        cfgs = {
+            # ── 超小模型 ──
+            '20m': dict(d_model=288, n_layers=6, n_heads=6, n_kv_heads=3, max_seq_len=1024),
+            '26m': dict(d_model=320, n_layers=8, n_heads=8, n_kv_heads=4, max_seq_len=2048),
+            # ── 百M级 ──
+            '100m': dict(d_model=512, n_layers=12, n_heads=8, n_kv_heads=4, max_seq_len=2048),
+            '300m': dict(d_model=768, n_layers=16, n_heads=12, n_kv_heads=4, max_seq_len=4096),
+            # ── 十亿级 ──
+            '500m': dict(d_model=1024, n_layers=24, n_heads=16, n_kv_heads=8, max_seq_len=8192),
+            '1b':   dict(d_model=1536, n_layers=24, n_heads=16, n_kv_heads=8, max_seq_len=8192),
+            '3b':   dict(d_model=2560, n_layers=32, n_heads=24, n_kv_heads=8, max_seq_len=8192),
+            # ── 十亿级+ ──
+            '7b':   dict(d_model=4096, n_layers=32, n_heads=32, n_kv_heads=8, max_seq_len=8192, rope_theta=500000.0),
+            '13b':  dict(d_model=5120, n_layers=40, n_heads=40, n_kv_heads=10, max_seq_len=8192, rope_theta=500000.0),
+            '70b':  dict(d_model=8192, n_layers=80, n_heads=64, n_kv_heads=8, max_seq_len=8192, rope_theta=500000.0),
+        }
+        cfg = cls()
+        if name in cfgs:
+            for k, v in cfgs[name].items():
+                setattr(cfg, k, v)
+        if cfg.d_ff == 0:
+            # SwiGLU 标准: 8/3 * d_model（参数量等价于 4x 标准 FFN）
+            cfg.d_ff = int(cfg.d_model * 8 / 3)
+        return cfg
+
+
+# ════════════════════════════════════════════════════════════
+# 2. RMSNorm
+# ════════════════════════════════════════════════════════════
+
+class XiaLiRMSNorm(nn.Module):
+    """RMS Layer Normalization"""
+    def __init__(self, dim: int, eps: float = 1e-5):
+        super().__init__()
+        self.eps = eps
+        self.weight = nn.Parameter(torch.ones(dim))
+    
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        rms = torch.sqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+        return x / rms * self.weight
+
+
+# ════════════════════════════════════════════════════════════
+# 3. RoPE（旋转位置编码）
+# ════════════════════════════════════════════════════════════
+
+def precompute_rope_freqs(dim: int, max_len: int, theta: float = 10000.0) -> Tuple[torch.Tensor, torch.Tensor]:
+    """预计算 RoPE 频率"""
+    freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+    t = torch.arange(max_len, dtype=torch.float32)
+    angles = torch.outer(t, freqs)  # [max_len, dim//2]
+    cos = angles.cos().unsqueeze(0).unsqueeze(2)  # [1, max_len, 1, dim//2]
+    sin = angles.sin().unsqueeze(0).unsqueeze(2)
+    return cos, sin
+
+def apply_rope(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+    """对 x 应用 RoPE。x: [batch, seq, n_heads, head_dim]"""
+    head_dim = x.size(-1)
+    # cos/sin 是每对维度一个值 (head_dim//2)，需要扩张到 head_dim
+    cos = torch.repeat_interleave(cos, 2, dim=-1)  # [..., d//2] -> [..., d]
+    sin = torch.repeat_interleave(sin, 2, dim=-1)
+    x1 = x[..., :head_dim // 2]          # [..., d//2]
+    x2 = x[..., head_dim // 2:]           # [..., d//2]
+    rotated = torch.cat([-x2, x1], dim=-1)  # [..., d]
+    return x * cos + rotated * sin
+
+
+# ════════════════════════════════════════════════════════════
+# 4. GQA 注意力（分组查询注意力）
+# ════════════════════════════════════════════════════════════
+
+class GroupedQueryAttention(nn.Module):
+    """分组查询注意力 - 小模型=MHA，大模型节省KV Cache"""
+    
+    def __init__(self, config: XiaLiConfig):
+        super().__init__()
+        self.n_heads = config.n_heads
+        self.n_kv_heads = config.n_kv_heads  # GQA: KV头数
+        self.head_dim = config.d_model // config.n_heads
+        self.n_rep = self.n_heads // self.n_kv_heads  # 每个KV头被几个Q头共享
+        
+        # 投影（无bias）
+        self.wq = nn.Linear(config.d_model, config.n_heads * self.head_dim, bias=False)
+        self.wk = nn.Linear(config.d_model, self.n_kv_heads * self.head_dim, bias=False)
+        self.wv = nn.Linear(config.d_model, self.n_kv_heads * self.head_dim, bias=False)
+        self.wo = nn.Linear(config.n_heads * self.head_dim, config.d_model, bias=False)
+        self.attn_dropout = nn.Dropout(config.dropout)
+        
+    def forward(
+        self, 
+        x: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        batch, seq_len, _ = x.shape
+        
+        # QKV 投影
+        q = self.wq(x).view(batch, seq_len, self.n_heads, self.head_dim)
+        k = self.wk(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)
+        v = self.wv(x).view(batch, seq_len, self.n_kv_heads, self.head_dim)
+        
+        # RoPE
+        cos_ = cos[:, :seq_len]
+        sin_ = sin[:, :seq_len]
+        q = apply_rope(q, cos_, sin_)
+        k = apply_rope(k, cos_, sin_)
+        
+        # GQA: KV 头扩展到 Q 头数
+        if self.n_rep > 1:
+            k = k.repeat_interleave(self.n_rep, dim=2)
+            v = v.repeat_interleave(self.n_rep, dim=2)
+        
+        # 转置为 [batch, n_heads, seq, head_dim]
+        q = q.transpose(1, 2)
+        k = k.transpose(1, 2)
+        v = v.transpose(1, 2)
+        
+        # 注意力计算
+        scale = self.head_dim ** -0.5
+        attn = torch.matmul(q, k.transpose(-2, -1)) * scale
+        
+        if mask is not None:
+            attn = attn + mask[:, :, :seq_len, :seq_len]
+        
+        attn = F.softmax(attn, dim=-1, dtype=torch.float32).to(q.dtype)
+        attn = self.attn_dropout(attn)
+        
+        out = torch.matmul(attn, v)  # [batch, n_heads, seq, head_dim]
+        out = out.transpose(1, 2).contiguous().view(batch, seq_len, -1)
+        
+        return self.wo(out)
+
+
+# ════════════════════════════════════════════════════════════
+# 5. SwiGLU 前馈网络
+# ════════════════════════════════════════════════════════════
+
+class SwiGLU(nn.Module):
+    """SwiGLU 前馈网络 - 比 ReLU FFN 质量更高"""
+    
+    def __init__(self, config: XiaLiConfig):
+        super().__init__()
+        # d_ff 已按 8/3*d_model 计算。SwiGLU: w1和w3同维度，w2降回
+        hidden = config.d_ff
+        self.w1 = nn.Linear(config.d_model, hidden, bias=False)
+        self.w2 = nn.Linear(hidden, config.d_model, bias=False)
+        self.w3 = nn.Linear(config.d_model, hidden, bias=False)
+        self.dropout = nn.Dropout(config.dropout)
+        
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # SwiGLU(x) = (SiLU(x @ W1) * (x @ W3)) @ W2
+        return self.w2(self.dropout(F.silu(self.w1(x)) * self.w3(x)))
+
+
+# ════════════════════════════════════════════════════════════
+# 6. Transformer 块
+# ════════════════════════════════════════════════════════════
+
+class XiaLiBlock(nn.Module):
+    """Transformer 块 - Pre-RMSNorm + GQA + SwiGLU"""
+    
+    def __init__(self, config: XiaLiConfig, layer_idx: int):
+        super().__init__()
+        self.layer_idx = layer_idx
+        
+        self.attention = GroupedQueryAttention(config)
+        self.feed_forward = SwiGLU(config)
+        
+        self.attn_norm = XiaLiRMSNorm(config.d_model, config.norm_eps)
+        self.ff_norm = XiaLiRMSNorm(config.d_model, config.norm_eps)
+        
+    def forward(
+        self,
+        x: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+        mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        h = x + self.attention(self.attn_norm(x), cos, sin, mask)
+        out = h + self.feed_forward(self.ff_norm(h))
+        return out
+
+
+# ════════════════════════════════════════════════════════════
+# 7. 主模型
+# ════════════════════════════════════════════════════════════
+
+class XiaLiModel(nn.Module):
+    """
+    XiaLi-Next 统一架构
+    
+    核心设计：
+    - RoPE 位置编码（无学习的 position embedding）
+    - Pre-RMSNorm（训练稳定）
+    - SwiGLU FFN（同等参数量更高质量）
+    - GQA 注意力（大模型省 KV Cache）
+    - Weight Tying（embedding 与输出层共享权重）
+    - 所有 Linear 无 bias（更干净）
+    - 从 20M 到 7B+ 统一架构
+    """
+    
+    def __init__(self, config: XiaLiConfig):
+        super().__init__()
+        self.config = config
+        
+        # 词嵌入
+        self.embedding = nn.Embedding(config.vocab_size, config.d_model)
+        
+        # 预计算 RoPE
+        cos, sin = precompute_rope_freqs(
+            config.d_model // config.n_heads,
+            config.max_seq_len * 2,
+            config.rope_theta,
+        )
+        self.register_buffer('rope_cos', cos, persistent=False)
+        self.register_buffer('rope_sin', sin, persistent=False)
+        
+        # Transformer 块
+        self.blocks = nn.ModuleList([
+            XiaLiBlock(config, i) for i in range(config.n_layers)
+        ])
+        
+        # 最终归一化
+        self.norm = XiaLiRMSNorm(config.d_model, config.norm_eps)
+        
+        # 输出层（与 embedding 共享权重）
+        self.fc_out = nn.Linear(config.d_model, config.vocab_size, bias=False)
+        
+        if config.tie_weights:
+            self.fc_out.weight = self.embedding.weight  # Weight Tying
+        
+        self._init_weights()
+    
+    def _init_weights(self):
+        """初始化权重"""
+        for name, param in self.named_parameters():
+            if param.dim() >= 2:
+                nn.init.normal_(param, mean=0.0, std=0.02)
+            else:
+                nn.init.zeros_(param)
+    
+    def forward(self, x: torch.Tensor, mask: Optional[torch.Tensor] = None) -> torch.Tensor:
+        batch, seq_len = x.shape
+        
+        # 词嵌入
+        h = self.embedding(x)
+        
+        # RoPE
+        cos = self.rope_cos[:, :seq_len]
+        sin = self.rope_sin[:, :seq_len]
+        
+        # Transformer 层
+        for block in self.blocks:
+            h = block(h, cos, sin, mask)
+        
+        # 最终归一化
+        h = self.norm(h)
+        
+        # 输出
+        return self.fc_out(h)
+    
+    def generate_mask(self, seq_len: int) -> torch.Tensor:
+        """生成因果掩码 [1, 1, seq, seq]"""
+        mask = torch.full((seq_len, seq_len), float('-inf'))
+        mask = torch.triu(mask, diagonal=1)
+        return mask.unsqueeze(0).unsqueeze(0)
+    
+    @torch.no_grad()
+    def generate(
+        self,
+        input_ids: torch.Tensor,
+        max_new_tokens: int = 100,
+        temperature: float = 0.7,
+        top_k: int = 40,
+        eos_id: int = 3,
+    ) -> torch.Tensor:
+        """自回归生成"""
+        self.eval()
+        for _ in range(max_new_tokens):
+            # 截断到 max_seq_len
+            seq = input_ids if input_ids.size(1) <= self.config.max_seq_len else input_ids[:, -self.config.max_seq_len:]
+            
+            mask = self.generate_mask(seq.size(1)).to(seq.device)
+            logits = self.forward(seq, mask)
+            logits = logits[:, -1, :] / temperature
+            
+            # Top-k 采样
+            if top_k > 0:
+                v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
+                logits[logits < v[:, -1:]] = float('-inf')
+            
+            probs = F.softmax(logits, dim=-1)
+            next_token = torch.multinomial(probs, 1)
+            input_ids = torch.cat([input_ids, next_token], dim=-1)
+            
+            if next_token.item() == eos_id:
+                break
+        
+        return input_ids
+    
+    def count_params(self) -> dict:
+        """统计参数量"""
+        total = sum(p.numel() for p in self.parameters())
+        trainable = sum(p.numel() for p in self.parameters() if p.requires_grad)
+        # 各模块
+        emb = self.embedding.weight.numel()
+        attn = sum(p.numel() for n, p in self.named_parameters() if 'attention' in n)
+        ffn = sum(p.numel() for n, p in self.named_parameters() if 'feed_forward' in n)
+        norm = sum(p.numel() for n, p in self.named_parameters() if 'norm' in n)
+        return {
+            'total': total,
+            'trainable': trainable,
+            'embedding': emb,
+            'attention': attn,
+            'ffn': ffn,
+            'norm': norm,
+            'total_m': total / 1e6,
+        }
+
+
+# ════════════════════════════════════════════════════════════
+# 8. 与原有系统兼容的训练适配
+# ════════════════════════════════════════════════════════════
+
+# 导出兼容名（方便从原系统切换）
+XiaLiNextModel = XiaLiModel
+XiaLiNextConfig = XiaLiConfig
+
+
+# ════════════════════════════════════════════════════════════
+# 9. 快速测试
+# ════════════════════════════════════════════════════════════
+
+if __name__ == "__main__":
+    print("=" * 65)
+    print("  XiaLi-Next 架构测试")
+    print("=" * 65)
+    
+    for name in ['20m', '26m', '100m']:
+        cfg = XiaLiConfig.preset(name)
+        try:
+            model = XiaLiModel(cfg)
+            stats = model.count_params()
+            print(f"\n  [{name}] {cfg.d_model}d × {cfg.n_layers}L × {cfg.n_heads}H (KV={cfg.n_kv_heads})")
+            print(f"    参数量: {stats['total_m']:.2f}M")
+            print(f"    其中: emb={stats['embedding']/1e6:.2f}M, attn={stats['attention']/1e6:.2f}M, "
+                  f"ffn={stats['ffn']/1e6:.2f}M")
+            
+            # 前向测试
+            x = torch.randint(0, cfg.vocab_size - 1, (1, 64))
+            try:
+                out = model(x)
+                print(f"    前向: ✅ {out.shape}")
+            except Exception as e:
+                print(f"    前向: ❌ {e}")
+        except Exception as e:
+            print(f"\n  [{name}] ❌ {e}")
+    
+    print(f"\n{'='*65}")
+    print(f"  ✅ XiaLi-Next 就绪！")
+    print(f"  从 20M 到 7B+ 统一架构")
+    print(f"{'='*65}")