diff --git a/.gitignore b/.gitignore index 815e04b..cdb757d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,54 +2,26 @@ __pycache__/ *.pyc *.log -# 模型文件 — 只追踪发布版本 -*.pth -!/llm_model_v2_resume_epoch_7.pth /tmp/ -# 中间数据（旧版临时文件） -mega_interim_*.json -essay_interim_*.json -checkpoints_v2/ +# 模型权重 — 只追踪发布版本 +*.pth +!/llm_model_v2_resume_epoch_7.pth + +# 中间输出目录（生成脚本的输出） high_temp_outputs/ diverse_answers/ fresh_qa_outputs/ multiturn_outputs/ +checkpoints_v2/ old_ckpts/ backup_old_arch/ -# 旧版数据（不再使用） -xiaoli_mega_*.json -xiaoli_multiturn_memory*.json -xiaoli_offline_distill*.json -xiaoli_personality*.json -xiaoli_safety*.json -xiaoli_sassy*.json -xiaoli_short_cot*.json -xiaoli_cot*.json -xiaoli_distill*.json -xiaoli_diverse*.json -xiaoli_drowning*.json -xiaoli_english*.json -xiaoli_final*.json -xiaoli_girl*.json -xiaoli_knowledge*.json -xiaoli_math*.json -xiaoli_merged_data.json -xiaoli_body*.json -xiaoli_child*.json -xiaoli_cute*.json -xiaoli_complete*.json -xiaoli_data_optimized*.json -xiaoli_riddle*.json -xiaoli_final*.json - -# 测试输出 -test_*.json -test_*.py -/var/folders - -# 旧版配置文件 +# 旧版废弃数据 +xiaoli_*.json +mega_interim_*.json +essay_interim_*.json +distill_*.json cot_*.json memory_long*.json multiturn_batch_*.json @@ -60,7 +32,16 @@ essay_questions*.json cot_candidates*.json model_test_results*.json online_distill_report*.json + +# 测试/临时文件 +test_*.json +test_*.py +/var/folders + +# 日志 batch_runner*.log mega_*.log mimo_*.log -knowledge_*.log \ No newline at end of file +knowledge_*.log +train_*.log +web_server*.log \ No newline at end of file diff --git a/README.en.md b/README.en.md deleted file mode 100644 index 8f75441..0000000 --- a/README.en.md +++ /dev/null @@ -1,29 +0,0 @@ -# xiaoli-llm-chat - -#### Description -XiaoliAI - A casual chat & psychological therapy model designed to run on the smallest devices. - -A from-scratch domain-specific Transformer. No pre-trained base model, no fine-tuning, no alignment — train it and talk to it directly. - -#### Features -- **Ultra-lightweight**: 13M params, ~20MB footprint -- **From Scratch**: Not based on LLaMA/GPT or any backbone -- **Ready to Use**: No SFT / RLHF needed -- **Safe by Design**: Data-embedded alignment, no harmful outputs -- **Emotional Companion**: Built for conversation, not general tasks - -#### Tech Stack -- Architecture: Causal Decoder-Only Transformer (RoPE + RMSNorm + SwiGLU) -- Tokenizer: Character-level (3236 vocab) -- Training: Custom QA data + Astron API distillation - -#### Use Cases -- Child emotional companionship -- Lightweight psychological support -- IoT / Embedded offline chat -- Privacy-focused local conversation - -#### Performance -- Memory: ~20MB -- Inference: Real-time on CPU (no GPU needed) -- Runs on: Raspberry Pi, phones, smartwatches, low-end PCs \ No newline at end of file diff --git a/README.md b/README.md index c91b6b6..0484ab1 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,95 @@ -# xiaoli-llm-chat +# 小狸 AI — Xiaoli Chat -#### 介绍 -小狸AI - 旨在最小设备上运行的闲聊心理治疗模型 +> 一个从零训练的 **13M 参数闲聊心理治疗模型**。纯 CPU 实时推理，无需 GPU。 -从零训练的特化 Transformer，不依赖任何预训练模型，无需微调对齐，训练完可直接对话。 +小狸是一个 8 岁小女孩的角色化对话 AI，基于 Causal Decoder-Only Transformer 架构，支持日常闲聊、情感陪伴、角色扮演等场景。 -#### 模型特点 -- **超轻量**: 仅 13M 参数，约 20MB 占用 -- **从零训练**: 不基于 LLaMA/GPT 等任何基座，权重完全随机初始化 -- **开箱即用**: 无需 SFT / RLHF，训练完即可对话 -- **安全可控**: 数据内嵌对齐，天然不输出违规内容 -- **闲聊陪伴**: 专注情感价值，不追求通用能力 +## 快速开始 -#### 技术栈 -- 架构: Causal Decoder-Only Transformer (RoPE + RMSNorm + SwiGLU) -- 分词: 字符级词表（3236 tokens） -- 训练: 自建问答数据 + Astron API 蒸馏 +```bash +# 1. 推理（对话） +python3 -c " +import json, torch +from llm_chat_improved import LLMModel, LLMChat, tokenizer_instance -#### 使用场景 -- 儿童情感陪伴 -- 轻量级心理倾诉 -- IoT / 嵌入式设备离线对话 -- 隐私安全的本地聊天 +vocab = json.load(open('vocab.json')) +ckpt = torch.load('llm_model_v2_resume_epoch_7.pth', map_location='cpu') +cfg = ckpt['model_config'] -#### 性能 -- 内存占用: ~20MB -- 推理速度: CPU 实时（无需 GPU） -- 可运行平台: 树莓派、手机、手表、低端PC \ No newline at end of file +model = LLMModel(vocab_size=len(vocab), d_model=cfg['d_model'], n_heads=cfg['n_heads'], + n_layers=cfg['n_layers'], d_ff=cfg['d_ff'], max_len=cfg['max_len'], dropout=0.1) +model.load_state_dict(ckpt['model_state_dict']) +model.eval() + +chat = LLMChat(model, vocab, device='cpu') +tokenizer_instance.char_to_id = vocab.copy() +tokenizer_instance.id_to_char = {v:k for k,v in vocab.items()} + +print(chat.generate_response('小狸你好呀')) +" + +# 2. 训练 +python3 train.py +``` + +## 模型架构 + +| 参数 | 值 | +|------|-----| +| 架构 | Causal Decoder-Only Transformer | +| 参数量 | 13M | +| 隐藏层 (d_model) | 320 | +| 层数 (n_layers) | 6 | +| 注意力头 (n_heads) | 8 | +| FFN 宽度 (d_ff) | 1280 | +| 最大序列长度 | 512 | +| 位置编码 | RoPE (Rotary Position Embedding) | +| 激活函数 | SwiGLU | +| 归一化 | RMSNorm | +| 词表 | BPE, 6,812 tokens | + +## 项目结构 + +``` +xiaoli-llm-chat/ +├── llm_chat_improved.py # 模型架构 + 推理代码（核心） +├── train.py # 训练脚本 +├── vocab.json # BPE 词表（6.8k tokens） +├── web_server.py # Web 聊天服务 +├── chat_server.py # Flask 聊天服务 +├── templates/ +│ └── chat.html # 聊天网页 UI +├── data/ +│ ├── xiaoli_data_v1_part_1of2.json # 训练数据（33万条） +│ └── xiaoli_data_v1_part_2of2.json # 训练数据（33万条） +└── scripts/ # 数据生成脚本 + ├── generate_fresh_qa.py + ├── generate_multiturn.py + ├── regenerate_all_high_temp.py + ├── daily_data_gen.py + └── api_correction.py +``` + +## 训练数据 + +- **总计 666,379 条** 去重 QA 对 +- 来源：Astron 30B API 蒸馏生成（高温采样 + 多轮对话 + 新问答） +- 场景覆盖：日常闲聊、情感陪伴、儿童对话、角色扮演 +- 纯人工蒸馏，零网页爬取，数据安全可控 + +## 模型权重 + +当前发布版本：**Epoch 7**（v0.1） + +模型权重文件 `llm_model_v2_resume_epoch_7.pth`（154MB）因超出 Gitee 单文件限制，需单独下载。 + +## 系统要求 + +- Python 3.8+ +- PyTorch（CPU 即可，无需 GPU） +- 内存：~200MB +- 磁盘：~500MB（含训练数据和权重） + +## 许可 + +MIT License \ No newline at end of file diff --git a/all_data_cleaned.json b/all_data_cleaned.json deleted file mode 100644 index a01fa37..0000000 --- a/all_data_cleaned.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:136863dba34ed2300a56611600b9cd758aa6e01cdd7e3dcdd35d0371be32d41c -size 43711 diff --git a/xiaoli_data_v1_part_1of2.json b/data/xiaoli_data_v1_part_1of2.json similarity index 100% rename from xiaoli_data_v1_part_1of2.json rename to data/xiaoli_data_v1_part_1of2.json diff --git a/xiaoli_data_v1_part_2of2.json b/data/xiaoli_data_v1_part_2of2.json similarity index 100% rename from xiaoli_data_v1_part_2of2.json rename to data/xiaoli_data_v1_part_2of2.json diff --git a/distill_questions_v2.json b/distill_questions_v2.json deleted file mode 100644 index 8043495..0000000 --- a/distill_questions_v2.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e5154511dddc7a03fcad94b7273bf7ab23ff9ccfe16957058b6a18eed608bf3b -size 4919 diff --git a/distill_questions_v3.json b/distill_questions_v3.json deleted file mode 100644 index 9a63e93..0000000 --- a/distill_questions_v3.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c74ebfca559a906d6cbd2d77eae0752ab72add28fca9b8de47d165fadd00e419 -size 30616 diff --git a/exam_distill_questions.json b/exam_distill_questions.json deleted file mode 100644 index 4011fe9..0000000 --- a/exam_distill_questions.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:59f8dabcf5cbfc960810e99e51d8777a238e2ad4798405e8dd1c7bb7e5050a93 -size 2686585 diff --git a/llm_chat_improved.py b/llm_chat_improved.py index e9ae4c4..89521fe 100644 --- a/llm_chat_improved.py +++ b/llm_chat_improved.py @@ -786,7 +786,17 @@ class LLMChat: outputs = self.model(seq) # 获取最后一个时间步的预测 - next_token_logits = outputs[0, -1, :] / temperature + logits = outputs[0, -1, :] + + # 温度=0 → greedy解码（取概率最高的token） + if temperature == 0: + next_token = torch.argmax(logits).item() + generated.append(next_token) + new_token = torch.tensor([[next_token]], device=self.device) + input_tensor = torch.cat([input_tensor, new_token], dim=1) + continue + + next_token_logits = logits / temperature # === 重复惩罚 === # 1. 对最近生成的token进行惩罚（更强） diff --git a/multiturn_data.json b/multiturn_data.json deleted file mode 100644 index 46a5d22..0000000 --- a/multiturn_data.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7c4d05531958838c1c6750216a98fc781ea13085b278a0154eb1378680959fe9 -size 10038 diff --git a/api_correction.py b/scripts/api_correction.py similarity index 100% rename from api_correction.py rename to scripts/api_correction.py diff --git a/daily_data_gen.py b/scripts/daily_data_gen.py similarity index 100% rename from daily_data_gen.py rename to scripts/daily_data_gen.py diff --git a/generate_batch.py b/scripts/generate_batch.py similarity index 100% rename from generate_batch.py rename to scripts/generate_batch.py diff --git a/generate_diverse.py b/scripts/generate_diverse.py similarity index 100% rename from generate_diverse.py rename to scripts/generate_diverse.py diff --git a/generate_diverse_single.py b/scripts/generate_diverse_single.py similarity index 100% rename from generate_diverse_single.py rename to scripts/generate_diverse_single.py diff --git a/generate_fresh_qa.py b/scripts/generate_fresh_qa.py similarity index 100% rename from generate_fresh_qa.py rename to scripts/generate_fresh_qa.py diff --git a/generate_multiturn.py b/scripts/generate_multiturn.py similarity index 100% rename from generate_multiturn.py rename to scripts/generate_multiturn.py diff --git a/regenerate_all_high_temp.py b/scripts/regenerate_all_high_temp.py similarity index 100% rename from regenerate_all_high_temp.py rename to scripts/regenerate_all_high_temp.py diff --git a/train_v2.py b/train.py similarity index 99% rename from train_v2.py rename to train.py index 94435e8..95a57a8 100644 --- a/train_v2.py +++ b/train.py @@ -217,13 +217,13 @@ def main(): print(f" 训练集: {len(train_set)}条 | 验证集: {len(val_set)}条") # ====== 加载固定vocab ====== - vocab_path = os.path.join(model_dir, 'vocab_v2.json') + vocab_path = os.path.join(model_dir, 'vocab.json') if not os.path.exists(vocab_path): - print("❌ 未找到 vocab_v2.json，请先运行固定vocab生成脚本") + print("❌ 未找到 vocab.json，请先运行BPE词表生成脚本") return with open(vocab_path, 'r', encoding='utf-8') as f: vocab = json.load(f) - print(f"词汇表: {len(vocab)} (固定, 来自 vocab_v2.json)") + print(f"词汇表: {len(vocab)} (BPE, 来自 vocab.json)") # 同步tokenizer（新字符映射为UNK） tokenizer_instance.char_to_id = vocab.copy() tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()} diff --git a/vocab_v3.json b/vocab.json similarity index 100% rename from vocab_v3.json rename to vocab.json diff --git a/vocab_v2.json b/vocab_v2.json deleted file mode 100644 index 3de7706..0000000 --- a/vocab_v2.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330 -size 41185 diff --git a/xiaoli_data.json b/xiaoli_data.json deleted file mode 100644 index 416407d..0000000 --- a/xiaoli_data.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c0c0848f74cc7604d41699134a5f34f54d5958779a6653e27973ef495790c29c -size 5267 diff --git a/xiaoli_merged_data_v2.json b/xiaoli_merged_data_v2.json deleted file mode 100644 index f936b6c..0000000 --- a/xiaoli_merged_data_v2.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b117303df258c2296699ad987c2e5f9ef141e9adef2b693723c2a41511a85d4d -size 12616207 diff --git a/xiaoli_train_v2.json b/xiaoli_train_v2.json deleted file mode 100644 index bc32cd6..0000000 --- a/xiaoli_train_v2.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bfa8970a4a176f23451002081afec16c909b6c8889b42819d698bc2e44e06aeb -size 12290087 diff --git a/xiaoli_valid_v2.json b/xiaoli_valid_v2.json deleted file mode 100644 index 8b55843..0000000 --- a/xiaoli_valid_v2.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c445541e8aaefe41ef3223ec0f83988acf5e282a52f34d13f8560e551615a2d2 -size 327442