180 lines
7.4 KiB
Python
180 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
小狸AI - CNB GPU训练脚本
|
|
从 merged_clean_v3.json (977K对) 训练，自动GPU检测
|
|
"""
|
|
import sys, os, json, time, glob, re, random, argparse
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
from torch.utils.data import Dataset, DataLoader
|
|
|
|
from llm_chat_improved import LLMModel, LLMChat, ConversationDataset, LLMTrainer, tokenizer_instance
|
|
|
|
MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
def load_pairs(path):
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
pairs = []
|
|
for item in data:
|
|
if isinstance(item, dict):
|
|
q = item.get("question", "") or item.get("input", "")
|
|
a = item.get("answer", "") or item.get("output", "")
|
|
if q and a:
|
|
pairs.append((q, a))
|
|
return pairs
|
|
|
|
def find_latest_ckpt(model_dir):
|
|
ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_epoch_*.pth'))
|
|
if not ckpts:
|
|
ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_epoch_*.pth'))
|
|
if not ckpts:
|
|
return None, 0
|
|
nums = [(p, max([int(s) for s in re.findall(r'\d+', os.path.basename(p).replace('.pth',''))])) for p in ckpts]
|
|
best = max(nums, key=lambda x: x[1])
|
|
return best[0], best[1]
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='小狸AI CNB训练')
|
|
parser.add_argument('--epochs', type=int, default=100, help='训练轮数')
|
|
parser.add_argument('--batch_size', type=int, default=8, help='batch大小')
|
|
parser.add_argument('--lr', type=float, default=3e-4, help='学习率')
|
|
parser.add_argument('--accum', type=int, default=2, help='梯度累积步数')
|
|
parser.add_argument('--data', type=str, default='data/merged_clean_v3.json', help='训练数据路径')
|
|
parser.add_argument('--resume', action='store_true', help='断点续训')
|
|
parser.add_argument('--save_every', type=int, default=5, help='每N轮保存checkpoint')
|
|
args = parser.parse_args()
|
|
|
|
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
|
print("=" * 60)
|
|
print(f"小狸AI CNB训练")
|
|
print(f" GPU: {torch.cuda.get_device_name(0) if device == 'cuda' else '无'}")
|
|
print(f" 设备: {device}")
|
|
print(f" 数据: {args.data}")
|
|
print(f" epochs={args.epochs}, batch={args.batch_size}, lr={args.lr}")
|
|
print("=" * 60)
|
|
|
|
# ====== 加载数据 ======
|
|
data_path = args.data if os.path.isabs(args.data) else os.path.join(MODEL_DIR, args.data)
|
|
if not os.path.exists(data_path):
|
|
print(f"❌ 数据不存在: {data_path}")
|
|
return
|
|
|
|
pairs = load_pairs(data_path)
|
|
random.seed(42)
|
|
random.shuffle(pairs)
|
|
split = int(len(pairs) * 0.95)
|
|
train_set = pairs[:split]
|
|
val_set = pairs[split:]
|
|
print(f"数据: {len(pairs)}对 | 训练: {len(train_set)} | 验证: {len(val_set)}")
|
|
|
|
# ====== 加载词表 ======
|
|
vocab_path = os.path.join(MODEL_DIR, 'vocab.json')
|
|
with open(vocab_path, 'r', encoding='utf-8') as f:
|
|
vocab = json.load(f)
|
|
print(f"词表: {len(vocab)} (BPE)")
|
|
tokenizer_instance.char_to_id = vocab.copy()
|
|
tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
|
|
for attr in ['_multi_trie', '_multi_keys', '_single_map']:
|
|
if hasattr(tokenizer_instance, attr):
|
|
delattr(tokenizer_instance, attr)
|
|
|
|
# ====== 模型 ======
|
|
model = LLMModel(vocab_size=max(vocab.values()) + 1, d_model=320, n_heads=8,
|
|
n_layers=6, d_ff=1536, max_len=256, dropout=0.1,
|
|
use_memory=False, use_thinking=False, use_moe=False)
|
|
total = sum(p.numel() for p in model.parameters())
|
|
print(f"模型: {total:,} 参数 ({total/1e6:.2f}M)")
|
|
|
|
start_epoch = 0
|
|
if args.resume:
|
|
ckpt_path, last_epoch = find_latest_ckpt(MODEL_DIR)
|
|
if ckpt_path:
|
|
ckpt = torch.load(ckpt_path, map_location=device)
|
|
old_cfg = ckpt.get('model_config', {})
|
|
if (old_cfg.get('vocab_size', 0) == len(vocab)
|
|
and old_cfg.get('d_model', 0) == 320
|
|
and old_cfg.get('n_layers', 0) == 6):
|
|
model.load_state_dict(ckpt['model_state_dict'])
|
|
start_epoch = last_epoch
|
|
print(f"断点续训: epoch {start_epoch} ← {os.path.basename(ckpt_path)}")
|
|
else:
|
|
print(f"⚠️ 架构不匹配，从头训练")
|
|
else:
|
|
print("无断点，从头训练")
|
|
|
|
model = model.to(device)
|
|
|
|
# ====== DataLoader ======
|
|
train_ds = ConversationDataset(train_set, vocab, max_length=256)
|
|
train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True)
|
|
val_ds = ConversationDataset(val_set, vocab, max_length=256)
|
|
val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False)
|
|
print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)}")
|
|
|
|
# ====== Trainer ======
|
|
config = {
|
|
'batch_size': args.batch_size,
|
|
'gradient_accumulation_steps': args.accum,
|
|
'epochs': args.epochs,
|
|
'd_model': 320, 'n_heads': 8, 'n_layers': 6,
|
|
'd_ff': 1536, 'max_length': 256, 'dropout': 0.1,
|
|
'use_amp': device == 'cuda',
|
|
}
|
|
trainer = LLMTrainer(model, device, lr=args.lr, use_amp=config['use_amp'])
|
|
|
|
# ====== 训练 ======
|
|
print(f"\n开始训练 epoch {start_epoch+1} → {args.epochs}")
|
|
print("=" * 60)
|
|
t_start = time.time()
|
|
best_val = float('inf')
|
|
|
|
for epoch in range(start_epoch + 1, args.epochs + 1):
|
|
t0 = time.time()
|
|
train_loss = trainer.train_epoch(train_loader, epoch, vocab, config)
|
|
val_loss = trainer.evaluate(val_loader, config)
|
|
dt = time.time() - t0
|
|
|
|
marker = " ★" if val_loss < best_val else ""
|
|
if val_loss < best_val:
|
|
best_val = val_loss
|
|
trainer.save_model(os.path.join(MODEL_DIR, 'xiaoli_cnb_best.pth'), vocab, epoch)
|
|
|
|
elapsed = time.time() - t_start
|
|
eta = (elapsed / (epoch - start_epoch)) * (args.epochs - epoch) if epoch > start_epoch else 0
|
|
|
|
print(f"E{epoch}/{args.epochs} | train={train_loss:.4f} val={val_loss:.4f}{marker} | "
|
|
f"{dt:.0f}s/epoch | {elapsed/60:.1f}min已用 ETA{eta/60:.0f}min")
|
|
|
|
# 保存
|
|
if epoch % args.save_every == 0:
|
|
ckpt_path = os.path.join(MODEL_DIR, f'llm_model_v2_resume_epoch_{epoch}.pth')
|
|
trainer.save_model(ckpt_path, vocab, epoch)
|
|
# 清理旧resume
|
|
for f in glob.glob(os.path.join(MODEL_DIR, 'llm_model_v2_resume_*.pth')):
|
|
en = max([int(s) for s in re.findall(r'\d+', os.path.basename(f).replace('.pth',''))])
|
|
if en <= epoch - 15:
|
|
try: os.remove(f)
|
|
except: pass
|
|
print(f" 💾 已保存 epoch {epoch}")
|
|
|
|
# ====== 完成 ======
|
|
final_path = os.path.join(MODEL_DIR, 'xiaoli_cnb_final.pth')
|
|
trainer.save_model(final_path, vocab, args.epochs)
|
|
total_time = (time.time() - t_start) / 60
|
|
print(f"\n✅ 训练完成! 耗时 {total_time:.1f}min")
|
|
|
|
# 测试
|
|
print("\n测试:")
|
|
chat = LLMChat(model, vocab, device, max_length=256)
|
|
for q in ["你好", "你是谁", "你叫什么名字", "今天天气真好", "我有点难过"]:
|
|
r = chat.generate_response(q, max_length=60, temperature=0.3)
|
|
print(f" Q: {q}")
|
|
print(f" A: {r}\n")
|
|
|
|
if __name__ == "__main__":
|
|
main() |