Compare commits
27
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 cnb.cEr0mSxFAFA
|
a88c47e273 | ||
|
shuiyu1123
|
a0047a012d | ||
|
shuiyu1123
|
7b47faac50 | ||
|
shuiyu1123
|
c6ff7be55a | ||
|
shuiyu1123
|
f5dd3e5874 | ||
|
shuiyu1123
|
ac07c7f06a | ||
|
shuiyu1123
|
230d47cade | ||
|
shuiyu1123
|
5732229ed5 | ||
|
shuiyu1123
|
ffb5825407 | ||
|
shuiyu1123
|
a02401aa75 | ||
|
shuiyu1123
|
edf3a83e3c | ||
|
shuiyu1123
|
9799aa00c5 | ||
|
shuiyu1123
|
456c443e9c | ||
|
shuiyu1123
|
eb7717e9ce | ||
|
shuiyu1123
|
231af72de9 | ||
|
shuiyu1123
|
6e0ced4772 | ||
|
shuiyu1123
|
aead65192e | ||
|
shuiyu1123
|
075492e391 | ||
|
shuiyu1123
|
8efb4faf73 | ||
|
shuiyu1123
|
2976bf2030 | ||
|
shuiyu1123
|
9cef1d9ba1 | ||
|
shuiyu1123
|
91db488ad4 | ||
|
shuiyu1123
|
eb1d62f85c | ||
|
shuiyu1123
|
7f27812efd | ||
|
shuiyu1123
|
d962c768a5 | ||
|
shuiyu1123
|
7f8f607f26 | ||
|
shuiyu1123
|
4df3254569 |
No files matched your search
@@ -0,0 +1,21 @@
|
||||
$:
|
||||
vscode:
|
||||
- docker:
|
||||
build: .ide/Dockerfile
|
||||
runner:
|
||||
cpus: 8
|
||||
tags: cnb:arch:amd64:gpu
|
||||
services:
|
||||
- vscode
|
||||
- docker
|
||||
env:
|
||||
CUDA_VISIBLE_DEVICES: "0"
|
||||
PYTHONUNBUFFERED: "1"
|
||||
stages:
|
||||
- name: 开始训练
|
||||
script: |
|
||||
cd /workspace
|
||||
nvidia-smi
|
||||
echo "=== GPU就绪，开始训练 ==="
|
||||
python3 scripts/train_cnb.py --epochs 100 --batch_size 16
|
||||
echo "=== 训练完成 ==="
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
# 已跟踪
|
||||
# 缓存和日志
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
/tmp/
|
||||
|
||||
# 模型权重 — 只追踪发布版本(精简推理版,不含optimizer,约60MB)
|
||||
*.pth
|
||||
!/llm_model_v2_resume_epoch_7.pth
|
||||
!/xiaoli-llm-chat/xiaoli_v2_best_inference.pth
|
||||
!/xiaoli-llm-chat/xiaoli_v2_final_inference.pth
|
||||
|
||||
# 中间输出目录（生成脚本的输出）
|
||||
high_temp_outputs/
|
||||
diverse_answers/
|
||||
fresh_qa_outputs/
|
||||
multiturn_outputs/
|
||||
checkpoints_v2/
|
||||
old_ckpts/
|
||||
backup_old_arch/
|
||||
|
||||
# 旧版废弃数据
|
||||
xiaoli_*.json
|
||||
mega_interim_*.json
|
||||
essay_interim_*.json
|
||||
distill_*.json
|
||||
cot_*.json
|
||||
memory_long*.json
|
||||
multiturn_batch_*.json
|
||||
real_multiturn*.json
|
||||
mega_questions*.json
|
||||
math_questions*.json
|
||||
essay_questions*.json
|
||||
cot_candidates*.json
|
||||
model_test_results*.json
|
||||
online_distill_report*.json
|
||||
|
||||
# 测试/临时文件
|
||||
test_*.json
|
||||
test_*.py
|
||||
/var/folders
|
||||
|
||||
# 日志
|
||||
batch_runner*.log
|
||||
mega_*.log
|
||||
mimo_*.log
|
||||
knowledge_*.log
|
||||
train_*.log
|
||||
web_server*.log
|
||||
# CNB 配置 — 强制跟踪关键文件
|
||||
.ide/
|
||||
!/.ide/Dockerfile
|
||||
!/.cnb.yml
|
||||
|
||||
# 训练输出目录
|
||||
general_outputs/
|
||||
high_temp_outputs/
|
||||
multiturn_outputs/
|
||||
|
||||
# 本地临时
|
||||
fun_test.py
|
||||
*.log
|
||||
|
||||
# 打包/发布产物
|
||||
xiaoli_package/
|
||||
*.zip
|
||||
|
||||
# 上传/测试脚本
|
||||
upload_*.py
|
||||
_gofile_*
|
||||
_t.*
|
||||
_up_test.*
|
||||
|
||||
# 旧词表备份
|
||||
vocab_old.json
|
||||
vocab_backup*.json
|
||||
|
||||
# 数据目录（太大不进git）
|
||||
data/
|
||||
|
||||
# 训练checkpoint（不进git，通过release发布）
|
||||
xiaoli_cnb_best.pth
|
||||
xiaoli_cnb_final.pth
|
||||
@@ -0,0 +1,13 @@
|
||||
FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime
|
||||
|
||||
# 基础工具
|
||||
RUN apt-get update && apt-get install -y git curl wget nano && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# tqdm用于进度条
|
||||
RUN pip install --no-cache-dir tqdm numpy requests
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /workspace
|
||||
|
||||
# 验证CUDA
|
||||
RUN python3 -c "import torch; print(f'PyTorch {torch.__version__}, CUDA: {torch.cuda.is_available()}')"
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -1,36 +0,0 @@
|
||||
# xiaoli-llm-chat
|
||||
|
||||
#### Description
|
||||
小狸AI - 自研中文对话大模型
|
||||
|
||||
#### Software Architecture
|
||||
Software architecture description
|
||||
|
||||
#### Installation
|
||||
|
||||
1. xxxx
|
||||
2. xxxx
|
||||
3. xxxx
|
||||
|
||||
#### Instructions
|
||||
|
||||
1. xxxx
|
||||
2. xxxx
|
||||
3. xxxx
|
||||
|
||||
#### Contribution
|
||||
|
||||
1. Fork the repository
|
||||
2. Create Feat_xxx branch
|
||||
3. Commit your code
|
||||
4. Create Pull Request
|
||||
|
||||
|
||||
#### Gitee Feature
|
||||
|
||||
1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md
|
||||
2. Gitee blog [blog.gitee.com](https://blog.gitee.com)
|
||||
3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore)
|
||||
4. The most valuable open source project [GVP](https://gitee.com/gvp)
|
||||
5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help)
|
||||
6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
|
||||
@@ -1,37 +1,95 @@
|
||||
# xiaoli-llm-chat
|
||||
# 小狸 AI — Xiaoli Chat
|
||||
|
||||
#### 介绍
|
||||
小狸AI - 自研中文对话大模型
|
||||
> 一个从零训练的 **13M 参数闲聊心理治疗模型**。纯 CPU 实时推理，无需 GPU。
|
||||
|
||||
#### 软件架构
|
||||
软件架构说明
|
||||
小狸是一个 8 岁小女孩的角色化对话 AI，基于 Causal Decoder-Only Transformer 架构，支持日常闲聊、情感陪伴、角色扮演等场景。
|
||||
|
||||
## 快速开始
|
||||
|
||||
#### 安装教程
|
||||
```bash
|
||||
# 1. 推理（对话）
|
||||
python3 -c "
|
||||
import json, torch
|
||||
from llm_chat_improved import LLMModel, LLMChat, tokenizer_instance
|
||||
|
||||
1. xxxx
|
||||
2. xxxx
|
||||
3. xxxx
|
||||
vocab = json.load(open('vocab.json'))
|
||||
ckpt = torch.load('llm_model_v2_resume_epoch_7.pth', map_location='cpu')
|
||||
cfg = ckpt['model_config']
|
||||
|
||||
#### 使用说明
|
||||
model = LLMModel(vocab_size=len(vocab), d_model=cfg['d_model'], n_heads=cfg['n_heads'],
|
||||
n_layers=cfg['n_layers'], d_ff=cfg['d_ff'], max_len=cfg['max_len'], dropout=0.1)
|
||||
model.load_state_dict(ckpt['model_state_dict'])
|
||||
model.eval()
|
||||
|
||||
1. xxxx
|
||||
2. xxxx
|
||||
3. xxxx
|
||||
chat = LLMChat(model, vocab, device='cpu')
|
||||
tokenizer_instance.char_to_id = vocab.copy()
|
||||
tokenizer_instance.id_to_char = {v:k for k,v in vocab.items()}
|
||||
|
||||
#### 参与贡献
|
||||
print(chat.generate_response('小狸你好呀'))
|
||||
"
|
||||
|
||||
1. Fork 本仓库
|
||||
2. 新建 Feat_xxx 分支
|
||||
3. 提交代码
|
||||
4. 新建 Pull Request
|
||||
# 2. 训练
|
||||
python3 train.py
|
||||
```
|
||||
|
||||
## 模型架构
|
||||
|
||||
#### 特技
|
||||
| 参数 | 值 |
|
||||
|------|-----|
|
||||
| 架构 | Causal Decoder-Only Transformer |
|
||||
| 参数量 | 13M |
|
||||
| 隐藏层 (d_model) | 320 |
|
||||
| 层数 (n_layers) | 6 |
|
||||
| 注意力头 (n_heads) | 8 |
|
||||
| FFN 宽度 (d_ff) | 1280 |
|
||||
| 最大序列长度 | 512 |
|
||||
| 位置编码 | RoPE (Rotary Position Embedding) |
|
||||
| 激活函数 | SwiGLU |
|
||||
| 归一化 | RMSNorm |
|
||||
| 词表 | BPE, 6,812 tokens |
|
||||
|
||||
1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md
|
||||
2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com)
|
||||
3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目
|
||||
4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目
|
||||
5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help)
|
||||
6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/)
|
||||
## 项目结构
|
||||
|
||||
```
|
||||
xiaoli-llm-chat/
|
||||
├── llm_chat_improved.py # 模型架构 + 推理代码（核心）
|
||||
├── train.py # 训练脚本
|
||||
├── vocab.json # BPE 词表（6.8k tokens）
|
||||
├── web_server.py # Web 聊天服务
|
||||
├── chat_server.py # Flask 聊天服务
|
||||
├── templates/
|
||||
│ └── chat.html # 聊天网页 UI
|
||||
├── data/
|
||||
│ ├── xiaoli_data_v1_part_1of2.json # 训练数据（33万条）
|
||||
│ └── xiaoli_data_v1_part_2of2.json # 训练数据（33万条）
|
||||
└── scripts/ # 数据生成脚本
|
||||
├── generate_fresh_qa.py
|
||||
├── generate_multiturn.py
|
||||
├── regenerate_all_high_temp.py
|
||||
├── daily_data_gen.py
|
||||
└── api_correction.py
|
||||
```
|
||||
|
||||
## 训练数据
|
||||
|
||||
- **总计 666,379 条** 去重 QA 对
|
||||
- 来源：Astron 30B API 蒸馏生成（高温采样 + 多轮对话 + 新问答）
|
||||
- 场景覆盖：日常闲聊、情感陪伴、儿童对话、角色扮演
|
||||
- 纯人工蒸馏，零网页爬取，数据安全可控
|
||||
|
||||
## 模型权重
|
||||
|
||||
当前发布版本：**Epoch 7**（v0.1）
|
||||
|
||||
模型权重文件 `llm_model_v2_resume_epoch_7.pth`（154MB）因超出 Gitee 单文件限制，需单独下载。
|
||||
|
||||
## 系统要求
|
||||
|
||||
- Python 3.8+
|
||||
- PyTorch（CPU 即可，无需 GPU）
|
||||
- 内存：~200MB
|
||||
- 磁盘：~500MB（含训练数据和权重）
|
||||
|
||||
## 许可
|
||||
|
||||
MIT License
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:136863dba34ed2300a56611600b9cd758aa6e01cdd7e3dcdd35d0371be32d41c
|
||||
size 43711
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
API生成模块 — Astron 5并发，稳定不超限
|
||||
"""
|
||||
import json, requests, time, os, threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚，不要讲一半就停"""
|
||||
|
||||
|
||||
def call_astron(q, temperature=0.85):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": q}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def batch_generate(questions, temperature=0.85, max_workers=4):
|
||||
"""Astron 4并发 — 明确限制，稳定可靠"""
|
||||
total = len(questions)
|
||||
results = []
|
||||
|
||||
def call(q):
|
||||
ans = call_astron(q, temperature)
|
||||
if ans:
|
||||
return q, ans
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(call, item): i for i, item in enumerate(questions)}
|
||||
done = 0
|
||||
t0 = time.time()
|
||||
for f in as_completed(futures):
|
||||
done += 1
|
||||
r = f.result()
|
||||
if r:
|
||||
results.append({"question": r[0], "answer": r[1]})
|
||||
if done % 20 == 0 or done == total:
|
||||
elapsed = max(time.time() - t0, 1)
|
||||
rate = len(results) / elapsed * 60
|
||||
print(f" Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
rate = len(results) / max(elapsed, 1) * 60
|
||||
print(f" Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
|
||||
return results
|
||||
|
||||
|
||||
# 保持旧函数名兼容
|
||||
batch_generate_astron = batch_generate
|
||||
batch_generate_mimo = batch_generate
|
||||
batch_generate_direct = batch_generate
|
||||
batch_generate_dual = batch_generate
|
||||
|
||||
|
||||
def batch_convert(items, *args, **kwargs):
|
||||
"""兼容占位：训练脚本导入用，已弃用"""
|
||||
print(" [API] batch_convert已弃用，使用batch_generate")
|
||||
return []
|
||||
|
||||
|
||||
def batch_generate_multi_temp(questions, *args, **kwargs):
|
||||
"""兼容占位：训练脚本导入用，已弃用"""
|
||||
print(" [API] batch_generate_multi_temp已弃用，使用batch_generate")
|
||||
return []
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
用已有的 CharTokenizer + build_vocabulary 从训练数据生成词表
|
||||
然后直接启动训练
|
||||
"""
|
||||
import sys, os, json, time
|
||||
sys.path.insert(0, '/workspace/xiaoli-llm-chat')
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
|
||||
|
||||
from llm_chat_improved import tokenizer_instance, build_vocabulary
|
||||
|
||||
DATA_PATH = '/workspace/data/merged_clean_v3.json'
|
||||
VOCAB_OUT = '/workspace/vocab.json'
|
||||
VOCAB_OUT_XIAOLI = '/workspace/xiaoli-llm-chat/vocab.json'
|
||||
|
||||
def load_pairs(path):
|
||||
"""加载训练数据为 (question, answer) 对"""
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
pairs = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
q = item.get("question", "") or item.get("input", "") or item.get("prompt", "")
|
||||
a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
|
||||
if q and a:
|
||||
pairs.append((q, a))
|
||||
return pairs
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("用 build_vocabulary 从训练数据生成词表")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. 加载训练数据
|
||||
t0 = time.time()
|
||||
print("\n[1] 加载训练数据...")
|
||||
pairs = load_pairs(DATA_PATH)
|
||||
print(f" 数据量: {len(pairs)} 对")
|
||||
print(f" 耗时: {time.time()-t0:.1f}s")
|
||||
|
||||
# 2. 用 build_vocabulary 生成词表
|
||||
t1 = time.time()
|
||||
print("\n[2] 用 build_vocabulary 分析数据生成词表...")
|
||||
vocab = build_vocabulary(pairs, min_freq=1)
|
||||
|
||||
# build_vocabulary 返回的是 tokenizer_instance.char_to_id
|
||||
# 已包含特殊token (<PAD>=0, <UNK>=1, etc.)
|
||||
special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
|
||||
'<START>': 4, '<END>': 5, '<SEP>': 6}
|
||||
|
||||
real_vocab_size = max(vocab.values()) + 1
|
||||
|
||||
# 统计
|
||||
single_char_tokens = sum(1 for k in vocab if len(k) == 1)
|
||||
multi_char_tokens = sum(1 for k in vocab if len(k) > 1 and not k.startswith('<'))
|
||||
special_token_count = sum(1 for k in vocab if k.startswith('<'))
|
||||
|
||||
print(f" 词表总项: {len(vocab)}")
|
||||
print(f" vocab_size(模型用): {real_vocab_size}")
|
||||
print(f" 单字符token: {single_char_tokens}")
|
||||
print(f" 多字符token(英文词): {multi_char_tokens}")
|
||||
print(f" 特殊token: {special_token_count}")
|
||||
print(f" 耗时: {time.time()-t1:.1f}s")
|
||||
|
||||
# 3. 编码测试
|
||||
print("\n[3] 编码测试:")
|
||||
test_texts = [
|
||||
"你好呀！我是小狸～",
|
||||
"小狸最喜欢吃草莓冰淇淋了！",
|
||||
"<SOS>你好<SEP><START>我是小狸～<END><EOS>",
|
||||
]
|
||||
for text in test_texts:
|
||||
ids = tokenizer_instance.encode(text)
|
||||
decoded = tokenizer_instance.decode(ids)
|
||||
print(f" 原文: '{text}'")
|
||||
print(f" 编码: {ids[:20]}... (长度={len(ids)})")
|
||||
print(f" 解码: '{decoded}'")
|
||||
print()
|
||||
|
||||
# 4. 估算模型参数量
|
||||
print("\n[4] 模型参数量估算 (d_model=320, n_layers=6, d_ff=1536):")
|
||||
d_model = 320
|
||||
n_layers = 6
|
||||
d_ff = 1536
|
||||
vs = real_vocab_size
|
||||
emb = vs * d_model
|
||||
out = vs * d_model
|
||||
per_layer = 4 * d_model**2 + 3 * d_model * d_ff + 2 * d_model
|
||||
total = emb + out + n_layers * per_layer + 2 * d_model
|
||||
print(f" vocab_size={vs}")
|
||||
print(f" Embedding: {emb:,}")
|
||||
print(f" fc_out: {out:,}")
|
||||
print(f" Transformer({n_layers}层): {n_layers*per_layer:,}")
|
||||
print(f" 总参数量: {total:,} ({total/1e6:.2f}M)")
|
||||
|
||||
# 5. 保存词表
|
||||
print(f"\n[5] 保存词表...")
|
||||
|
||||
# vocab.json 格式：不含特殊token，id从7开始（与 train.py 兼容）
|
||||
normal_vocab = {k: v for k, v in vocab.items() if k not in special_tokens}
|
||||
|
||||
# 备份旧词表
|
||||
for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
|
||||
if os.path.exists(path):
|
||||
backup = path.replace('vocab.json', 'vocab_old.json')
|
||||
import shutil
|
||||
shutil.copy2(path, backup)
|
||||
print(f" 旧词表已备份: {backup}")
|
||||
|
||||
# 保存到两个位置
|
||||
for path in [VOCAB_OUT, VOCAB_OUT_XIAOLI]:
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(normal_vocab, f, ensure_ascii=False, indent=1)
|
||||
print(f" {path}: {len(normal_vocab)}项 (不含特殊token, id从7开始)")
|
||||
|
||||
# 保存完整词表（含特殊token，用于参考）
|
||||
full_path = VOCAB_OUT_XIAOLI.replace('vocab.json', 'vocab_full.json')
|
||||
with open(full_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(vocab, f, ensure_ascii=False, indent=1)
|
||||
print(f" {full_path}: {len(vocab)}项 (含特殊token)")
|
||||
|
||||
total_time = time.time() - t0
|
||||
print(f"\n总耗时: {total_time:.1f}s")
|
||||
print("=" * 60)
|
||||
print(f"词表生成完成！vocab_size = {real_vocab_size}")
|
||||
print(f"模型参数量 ≈ {total/1e6:.2f}M")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>小狸 v1.2 聊天</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: 'PingFang SC', 'Microsoft YaHei', sans-serif;
|
||||
background: linear-gradient(135deg, #ffecd2 0%, #fcb69f 100%);
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 20px;
|
||||
}
|
||||
.container {
|
||||
width: 100%;
|
||||
max-width: 560px;
|
||||
background: white;
|
||||
border-radius: 24px;
|
||||
box-shadow: 0 20px 60px rgba(0,0,0,0.15);
|
||||
overflow: hidden;
|
||||
}
|
||||
.header {
|
||||
background: linear-gradient(135deg, #ff9a9e 0%, #fecfef 100%);
|
||||
padding: 20px;
|
||||
text-align: center;
|
||||
position: relative;
|
||||
}
|
||||
.header .avatar {
|
||||
width: 64px; height: 64px;
|
||||
background: white;
|
||||
border-radius: 50%;
|
||||
margin: 0 auto 8px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 32px;
|
||||
box-shadow: 0 4px 15px rgba(0,0,0,0.1);
|
||||
}
|
||||
.header h1 { font-size: 24px; color: white; }
|
||||
.header p { color: rgba(255,255,255,0.9); font-size: 12px; margin-top: 2px; }
|
||||
.chat-box {
|
||||
height: 420px;
|
||||
overflow-y: auto;
|
||||
padding: 16px;
|
||||
background: #fafafa;
|
||||
}
|
||||
.chat-box::-webkit-scrollbar { width: 5px; }
|
||||
.chat-box::-webkit-scrollbar-thumb { background: #ddd; border-radius: 10px; }
|
||||
.msg { margin-bottom: 14px; display: flex; animation: fadeIn 0.3s ease; }
|
||||
@keyframes fadeIn { from { opacity: 0; transform: translateY(8px); } to { opacity: 1; transform: translateY(0); } }
|
||||
.msg.user { justify-content: flex-end; }
|
||||
.msg.bot { justify-content: flex-start; }
|
||||
.bubble {
|
||||
max-width: 80%;
|
||||
padding: 10px 14px;
|
||||
border-radius: 16px;
|
||||
font-size: 14px;
|
||||
line-height: 1.6;
|
||||
word-break: break-word;
|
||||
}
|
||||
.msg.user .bubble {
|
||||
background: #ff9a9e;
|
||||
color: white;
|
||||
border-bottom-right-radius: 4px;
|
||||
}
|
||||
.msg.bot .bubble {
|
||||
background: white;
|
||||
color: #333;
|
||||
border-bottom-left-radius: 4px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.06);
|
||||
}
|
||||
.msg.bot .avatar-small {
|
||||
width: 30px; height: 30px;
|
||||
background: linear-gradient(135deg, #ff9a9e, #fecfef);
|
||||
border-radius: 50%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: 15px;
|
||||
margin-right: 8px;
|
||||
flex-shrink: 0;
|
||||
color: white;
|
||||
}
|
||||
.time-tag { font-size: 11px; color: #bbb; margin-top: 3px; }
|
||||
.input-area {
|
||||
padding: 12px 16px;
|
||||
background: white;
|
||||
border-top: 1px solid #f0f0f0;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
.input-area input {
|
||||
flex: 1;
|
||||
padding: 10px 14px;
|
||||
border: 2px solid #f0f0f0;
|
||||
border-radius: 25px;
|
||||
font-size: 14px;
|
||||
outline: none;
|
||||
}
|
||||
.input-area input:focus { border-color: #ff9a9e; }
|
||||
.input-area button {
|
||||
width: 42px; height: 42px;
|
||||
background: linear-gradient(135deg, #ff9a9e, #fecfef);
|
||||
border: none;
|
||||
border-radius: 50%;
|
||||
color: white;
|
||||
font-size: 18px;
|
||||
cursor: pointer;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.input-area button:hover { transform: scale(1.08); }
|
||||
.input-area button:disabled { opacity: 0.5; cursor: not-allowed; }
|
||||
.typing-indicator { display: none; padding: 10px 14px; background: white; border-radius: 16px; box-shadow: 0 2px 8px rgba(0,0,0,0.06); }
|
||||
.typing-indicator.active { display: flex; }
|
||||
.typing-indicator span {
|
||||
width: 7px; height: 7px; background: #ff9a9e; border-radius: 50%;
|
||||
margin: 0 2px; animation: bounce 1.4s infinite;
|
||||
}
|
||||
.typing-indicator span:nth-child(2) { animation-delay: 0.2s; }
|
||||
.typing-indicator span:nth-child(3) { animation-delay: 0.4s; }
|
||||
@keyframes bounce { 0%,80%,100% { transform: translateY(0); } 40% { transform: translateY(-8px); } }
|
||||
.controls {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 6px 16px;
|
||||
background: #fff8f8;
|
||||
border-top: 1px solid #f0f0f0;
|
||||
}
|
||||
.controls label { font-size: 12px; color: #999; }
|
||||
.controls input[type=range] { flex: 1; accent-color: #ff9a9e; height: 4px; }
|
||||
.controls .val { font-size: 12px; color: #e07070; font-weight: bold; min-width: 30px; }
|
||||
.controls .clear-btn {
|
||||
font-size: 11px; color: #ccc; cursor: pointer; padding: 3px 8px; border-radius: 10px; border: 1px solid #eee;
|
||||
}
|
||||
.controls .clear-btn:hover { color: #e07070; border-color: #ffd4d4; }
|
||||
.status-bar { padding: 4px 16px; font-size: 11px; color: #bbb; text-align: center; background: #fafafa; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<div class="avatar">🦊</div>
|
||||
<h1>小狸 v1.2</h1>
|
||||
<p id="headerInfo">V2架构 · 21M参数 · 新训练启动 🔥</p>
|
||||
</div>
|
||||
|
||||
<div class="chat-box" id="chatBox"></div>
|
||||
|
||||
<!-- 温度控制 -->
|
||||
<div class="controls">
|
||||
<label>🌡️</label>
|
||||
<input type="range" id="tempSlider" min="0.05" max="0.5" step="0.05" value="0.15">
|
||||
<span class="val" id="tempVal">0.15</span>
|
||||
<span class="clear-btn" onclick="clearChat()">🗑️ 清空</span>
|
||||
</div>
|
||||
|
||||
<div class="input-area">
|
||||
<input type="text" id="input" placeholder="跟小狸说点什么吧..." autofocus>
|
||||
<button id="sendBtn" onclick="send()">➤</button>
|
||||
</div>
|
||||
<div class="status-bar" id="statusBar">按回车发送 · 建议温度 0.1~0.2</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const chatBox = document.getElementById('chatBox');
|
||||
const input = document.getElementById('input');
|
||||
const sendBtn = document.getElementById('sendBtn');
|
||||
const statusBar = document.getElementById('statusBar');
|
||||
const tempSlider = document.getElementById('tempSlider');
|
||||
const tempVal = document.getElementById('tempVal');
|
||||
|
||||
// 对话历史（用于多轮）
|
||||
let history = [];
|
||||
|
||||
tempSlider.addEventListener('input', () => {
|
||||
tempVal.textContent = parseFloat(tempSlider.value).toFixed(2);
|
||||
});
|
||||
|
||||
addMsg('bot', '你好呀！我是小狸，一个8岁的小女孩～🦊\n你可以连续跟我聊天，我会记住之前说过的话哦！');
|
||||
|
||||
function addMsg(role, text) {
|
||||
const div = document.createElement('div');
|
||||
div.className = 'msg ' + role;
|
||||
if (role === 'bot') {
|
||||
const av = document.createElement('div');
|
||||
av.className = 'avatar-small';
|
||||
av.textContent = '🦊';
|
||||
div.appendChild(av);
|
||||
}
|
||||
const bubble = document.createElement('div');
|
||||
bubble.className = 'bubble';
|
||||
bubble.innerHTML = text.replace(/\n/g, '<br>');
|
||||
div.appendChild(bubble);
|
||||
|
||||
const time = document.createElement('div');
|
||||
time.className = 'time-tag';
|
||||
time.textContent = new Date().toLocaleTimeString('zh-CN', {hour:'2-digit',minute:'2-digit'});
|
||||
if (role === 'user') {
|
||||
bubble.appendChild(document.createElement('br'));
|
||||
bubble.appendChild(time);
|
||||
} else {
|
||||
bubble.appendChild(document.createElement('br'));
|
||||
bubble.appendChild(time);
|
||||
}
|
||||
chatBox.appendChild(div);
|
||||
chatBox.scrollTop = chatBox.scrollHeight;
|
||||
}
|
||||
|
||||
function showTyping() {
|
||||
const div = document.createElement('div');
|
||||
div.className = 'msg bot';
|
||||
div.id = 'typing';
|
||||
const av = document.createElement('div');
|
||||
av.className = 'avatar-small';
|
||||
av.textContent = '🦊';
|
||||
div.appendChild(av);
|
||||
const indicator = document.createElement('div');
|
||||
indicator.className = 'typing-indicator active';
|
||||
for (let i = 0; i < 3; i++) indicator.innerHTML += '<span>';
|
||||
div.appendChild(indicator);
|
||||
chatBox.appendChild(div);
|
||||
chatBox.scrollTop = chatBox.scrollHeight;
|
||||
}
|
||||
|
||||
function hideTyping() {
|
||||
const t = document.getElementById('typing');
|
||||
if (t) t.remove();
|
||||
}
|
||||
|
||||
async function send() {
|
||||
const q = input.value.trim();
|
||||
if (!q) return;
|
||||
input.value = '';
|
||||
sendBtn.disabled = true;
|
||||
statusBar.textContent = '小狸正在思考...';
|
||||
|
||||
addMsg('user', q);
|
||||
showTyping();
|
||||
|
||||
try {
|
||||
const res = await fetch('/api/chat', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({
|
||||
question: q,
|
||||
temperature: parseFloat(tempSlider.value),
|
||||
history: history.length > 0 ? history : null
|
||||
})
|
||||
});
|
||||
const data = await res.json();
|
||||
hideTyping();
|
||||
if (data.answer) {
|
||||
// 记录到历史
|
||||
history.push({q: q, a: data.answer});
|
||||
addMsg('bot', data.answer + ' <span style="color:#ccc;font-size:11px">' + data.time + '</span>');
|
||||
} else {
|
||||
addMsg('bot', '唔…我没想好怎么回答😅');
|
||||
}
|
||||
} catch (e) {
|
||||
hideTyping();
|
||||
addMsg('bot', '哎呀，出错了…再试试？');
|
||||
}
|
||||
sendBtn.disabled = false;
|
||||
statusBar.textContent = '按回车发送 · 温度 ' + parseFloat(tempSlider.value).toFixed(2);
|
||||
input.focus();
|
||||
}
|
||||
|
||||
function clearChat() {
|
||||
chatBox.innerHTML = '';
|
||||
history = [];
|
||||
addMsg('bot', '🔄 对话已重置！重新开始吧～');
|
||||
}
|
||||
|
||||
input.addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter') send();
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7e60d5f977e5b1e521d7b38d4a6c89aae0b67f540ee6754929e0407f7b408ab9
|
||||
size 175346049
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:4979079b6366d672b5e04be610ea4ce7e47bd6fa3762c11c225d8cf71f4b8ec4
|
||||
size 60404303
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:13888b035ec31866005aaf481d46d6266d6778eaa951d4136108d4490f4aaf69
|
||||
size 48274956
|
||||
+122
-43
@@ -7,6 +7,7 @@ LLM对话AI训练代码 - 改进版
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torch.optim as optim
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
import math
|
||||
@@ -35,6 +36,7 @@ class CharTokenizer:
|
||||
self.char_to_id = self.special_tokens.copy()
|
||||
self.id_to_char = {v: k for k, v in self.char_to_id.items()}
|
||||
self.next_id = len(self.special_tokens)
|
||||
self._sorted_vocab_cache = None # encode缓存用
|
||||
|
||||
def build_vocab(self, texts: List[str], min_freq: int = 1) -> Dict[str, int]:
|
||||
"""从文本构建词汇表"""
|
||||
@@ -53,6 +55,13 @@ class CharTokenizer:
|
||||
self.id_to_char[self.next_id] = char
|
||||
self.next_id += 1
|
||||
|
||||
# 清除encode缓存（vocab已变化）
|
||||
self._sorted_vocab_cache = None
|
||||
# 清除多字符token缓存
|
||||
for attr in ['_multi_trie', '_multi_keys', '_single_map']:
|
||||
if hasattr(self, attr):
|
||||
delattr(self, attr)
|
||||
|
||||
return self.char_to_id
|
||||
|
||||
def _split_chars(self, text: str) -> List[str]:
|
||||
@@ -71,18 +80,38 @@ class CharTokenizer:
|
||||
return result
|
||||
|
||||
def encode(self, text: str) -> List[int]:
|
||||
"""将文本编码为ID序列"""
|
||||
"""将文本编码为ID序列 - 高效版"""
|
||||
if not text:
|
||||
return [self.special_tokens['<UNK>']]
|
||||
|
||||
chars = self._split_chars(text)
|
||||
ids = []
|
||||
for char in chars:
|
||||
if char in self.char_to_id:
|
||||
ids.append(self.char_to_id[char])
|
||||
else:
|
||||
# 未知字符用 UNK token
|
||||
ids.append(self.special_tokens['<UNK>'])
|
||||
i = 0
|
||||
n = len(text)
|
||||
|
||||
# 构建多字符token的前缀映射（仅一次）
|
||||
if not hasattr(self, '_multi_trie'):
|
||||
self._multi_trie = {}
|
||||
for token in self.char_to_id:
|
||||
if len(token) > 1:
|
||||
self._multi_trie[token] = self.char_to_id[token]
|
||||
# 按长度降序排序，优先匹配长token
|
||||
self._multi_keys = sorted(self._multi_trie.keys(), key=len, reverse=True)
|
||||
self._single_map = {c: tid for c, tid in self.char_to_id.items() if len(c) == 1}
|
||||
|
||||
while i < n:
|
||||
matched = False
|
||||
# 优先匹配多字符token（特殊标记）
|
||||
for token in self._multi_keys:
|
||||
if text.startswith(token, i):
|
||||
ids.append(self._multi_trie[token])
|
||||
i += len(token)
|
||||
matched = True
|
||||
break
|
||||
if not matched:
|
||||
# 单字符直接用dict查
|
||||
c = text[i]
|
||||
ids.append(self._single_map.get(c, self.special_tokens['<UNK>']))
|
||||
i += 1
|
||||
|
||||
return ids
|
||||
|
||||
@@ -92,8 +121,8 @@ class CharTokenizer:
|
||||
for tid in ids:
|
||||
if tid in self.id_to_char:
|
||||
char = self.id_to_char[tid]
|
||||
if char not in self.special_tokens:
|
||||
chars.append(char)
|
||||
# 保留特殊token以便后续处理，但这里先不过滤
|
||||
chars.append(char)
|
||||
return ''.join(chars)
|
||||
|
||||
|
||||
@@ -207,19 +236,11 @@ class Attention(nn.Module):
|
||||
xk = xk.transpose(1, 2)
|
||||
xv = xv.transpose(1, 2)
|
||||
|
||||
# 计算注意力分数
|
||||
scores = torch.matmul(xq, xk.transpose(-2, -1)) / math.sqrt(self.d_head)
|
||||
# 融合注意力（SDPA / FlashAttention-2）- 纯因果掩码，等价于原下三角 mask
|
||||
dropout_p = self.dropout.p if self.training else 0.0
|
||||
output = F.scaled_dot_product_attention(
|
||||
xq, xk, xv, attn_mask=None, dropout_p=dropout_p, is_causal=True)
|
||||
|
||||
# 应用掩码
|
||||
if mask is not None:
|
||||
scores = scores.masked_fill(mask == 0, float('-inf'))
|
||||
|
||||
# Softmax
|
||||
attn_weights = torch.softmax(scores, dim=-1)
|
||||
attn_weights = self.dropout(attn_weights)
|
||||
|
||||
# 注意力输出
|
||||
output = torch.matmul(attn_weights, xv)
|
||||
output = output.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
|
||||
|
||||
return self.wo(output)
|
||||
@@ -512,7 +533,7 @@ class LLMModel(nn.Module):
|
||||
class ConversationDataset(Dataset):
|
||||
"""对话数据集"""
|
||||
|
||||
def __init__(self, conversations: List[Tuple[str, str]], vocab: dict, max_length: int = 128):
|
||||
def __init__(self, conversations: List[Tuple[str, str]], vocab: dict, max_length: int = 256):
|
||||
self.conversations = conversations
|
||||
self.vocab = vocab
|
||||
self.max_length = max_length
|
||||
@@ -552,11 +573,14 @@ class ConversationDataset(Dataset):
|
||||
class LLMTrainer:
|
||||
"""LLM训练器"""
|
||||
|
||||
def __init__(self, model: nn.Module, device: str = 'cpu', use_amp: bool = None):
|
||||
def __init__(self, model: nn.Module, device: str = 'cpu', lr: float = 0.0001, use_amp: bool = None):
|
||||
self.model = model.to(device)
|
||||
self.device = device
|
||||
# 火力全开：TF32 加速 fp32 矩阵乘法（L40/A100/H100 原生支持）
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
torch.backends.cudnn.allow_tf32 = True
|
||||
self.criterion = nn.CrossEntropyLoss(ignore_index=0)
|
||||
self.optimizer = optim.AdamW(model.parameters(), lr=0.0001, weight_decay=0.01)
|
||||
self.optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
|
||||
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer, T_max=100)
|
||||
|
||||
# 混合精度训练
|
||||
@@ -582,7 +606,7 @@ class LLMTrainer:
|
||||
self.model.train()
|
||||
total_loss = 0
|
||||
accumulation_steps = config.get('gradient_accumulation_steps', 1) if config else 1
|
||||
log_interval = max(1, len(dataloader) // 10) # 每10%打印一次
|
||||
log_interval = max(1, len(dataloader) // 100) # 每1%打印一次
|
||||
|
||||
for batch_idx, batch in enumerate(dataloader):
|
||||
input_ids = batch['input_ids'].to(self.device, non_blocking=True)
|
||||
@@ -591,6 +615,8 @@ class LLMTrainer:
|
||||
# 生成因果掩码
|
||||
mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
|
||||
|
||||
# CUDA graphs step 边界标记（torch.compile reduce-overhead 必需，避免输出 buffer 被覆盖）
|
||||
torch.compiler.cudagraph_mark_step_begin()
|
||||
# 前向传播
|
||||
if self.use_amp:
|
||||
with torch.cuda.amp.autocast():
|
||||
@@ -637,21 +663,47 @@ class LLMTrainer:
|
||||
self.scheduler.step()
|
||||
return total_loss / len(dataloader)
|
||||
|
||||
def evaluate(self, dataloader: DataLoader, config: dict = None):
|
||||
"""在验证集上计算loss（无梯度，不更新权重）"""
|
||||
self.model.eval()
|
||||
total_loss = 0
|
||||
accumulation_steps = config.get('gradient_accumulation_steps', 1) if config else 1
|
||||
|
||||
with torch.no_grad():
|
||||
for batch_idx, batch in enumerate(dataloader):
|
||||
input_ids = batch['input_ids'].to(self.device, non_blocking=True)
|
||||
target_ids = batch['target_ids'].to(self.device, non_blocking=True)
|
||||
|
||||
mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
|
||||
torch.compiler.cudagraph_mark_step_begin()
|
||||
outputs = self.model(input_ids, mask)
|
||||
outputs = outputs.reshape(-1, outputs.size(-1))
|
||||
tgt = target_ids.reshape(-1)
|
||||
|
||||
loss = self.criterion(outputs, tgt)
|
||||
total_loss += loss.item()
|
||||
|
||||
if len(dataloader) == 0:
|
||||
return float('inf')
|
||||
return total_loss / len(dataloader)
|
||||
|
||||
def save_model(self, filepath: str, vocab: dict, epoch: int = None):
|
||||
# 兼容 torch.compile：取原始（未编译）模块，避免 state_dict 带 _orig_mod. 前缀
|
||||
core = getattr(self.model, '_orig_mod', self.model)
|
||||
save_dict = {
|
||||
'model_state_dict': self.model.state_dict(),
|
||||
'model_state_dict': core.state_dict(),
|
||||
'optimizer_state_dict': self.optimizer.state_dict(),
|
||||
'vocab': vocab,
|
||||
'model_config': {
|
||||
'vocab_size': self.model.vocab_size,
|
||||
'd_model': self.model.d_model,
|
||||
'n_heads': self.model.n_heads,
|
||||
'n_layers': self.model.n_layers,
|
||||
'd_ff': self.model.d_ff,
|
||||
'max_len': self.model.max_len,
|
||||
'use_memory': self.model.use_memory,
|
||||
'use_thinking': self.model.use_thinking,
|
||||
'use_moe': self.model.use_moe
|
||||
'vocab_size': core.vocab_size,
|
||||
'd_model': core.d_model,
|
||||
'n_heads': core.n_heads,
|
||||
'n_layers': core.n_layers,
|
||||
'd_ff': core.d_ff,
|
||||
'max_len': core.max_len,
|
||||
'use_memory': core.use_memory,
|
||||
'use_thinking': core.use_thinking,
|
||||
'use_moe': core.use_moe
|
||||
}
|
||||
}
|
||||
if epoch is not None:
|
||||
@@ -662,7 +714,8 @@ class LLMTrainer:
|
||||
|
||||
def load_model(self, filepath: str):
|
||||
checkpoint = torch.load(filepath, map_location=self.device)
|
||||
self.model.load_state_dict(checkpoint['model_state_dict'])
|
||||
core = getattr(self.model, '_orig_mod', self.model)
|
||||
core.load_state_dict(checkpoint['model_state_dict'])
|
||||
self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
||||
vocab = checkpoint['vocab']
|
||||
print(f"模型已从 {filepath} 加载")
|
||||
@@ -677,7 +730,7 @@ class LLMTrainer:
|
||||
class LLMChat:
|
||||
"""LLM对话"""
|
||||
|
||||
def __init__(self, model: nn.Module, vocab: dict, device: str = 'cpu', max_length: int = 128,
|
||||
def __init__(self, model: nn.Module, vocab: dict, device: str = 'cpu', max_length: int = 256,
|
||||
show_thinking: bool = True):
|
||||
self.model = model.to(device)
|
||||
self.model.eval()
|
||||
@@ -692,14 +745,30 @@ class LLMChat:
|
||||
# 同步全局tokenizer的vocab（关键！）
|
||||
tokenizer_instance.char_to_id = vocab
|
||||
tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
|
||||
# 清除encode缓存，避免用旧的cache导致错位
|
||||
tokenizer_instance._sorted_vocab_cache = None
|
||||
for attr in ['_multi_trie', '_multi_keys', '_single_map']:
|
||||
if hasattr(tokenizer_instance, attr):
|
||||
delattr(tokenizer_instance, attr)
|
||||
|
||||
def generate_response(self, question: str, max_length: int = 100, temperature: float = 0.8,
|
||||
include_thinking: bool = False, use_beam_search: bool = False):
|
||||
"""生成回答"""
|
||||
include_thinking: bool = False, use_beam_search: bool = False,
|
||||
history: list = None):
|
||||
"""生成回答
|
||||
history: 可选，历史对话列表 [(user_q, assistant_a), ...]
|
||||
"""
|
||||
self.model.eval()
|
||||
|
||||
# 使用与训练数据一致的格式: <SOS>问题<SEP><START>
|
||||
input_text = f"<SOS>{question}<SEP><START>"
|
||||
# 构建输入（支持多轮上下文）
|
||||
if history:
|
||||
parts = []
|
||||
for hq, ha in history:
|
||||
parts.append(hq)
|
||||
parts.append(ha)
|
||||
parts.append(question)
|
||||
input_text = f"<SOS>{'<SEP>'.join(parts)}<SEP><START>"
|
||||
else:
|
||||
input_text = f"<SOS>{question}<SEP><START>"
|
||||
|
||||
input_ids = tokenizer_instance.encode(input_text)
|
||||
|
||||
@@ -721,7 +790,17 @@ class LLMChat:
|
||||
outputs = self.model(seq)
|
||||
|
||||
# 获取最后一个时间步的预测
|
||||
next_token_logits = outputs[0, -1, :] / temperature
|
||||
logits = outputs[0, -1, :]
|
||||
|
||||
# 温度=0 → greedy解码（取概率最高的token）
|
||||
if temperature == 0:
|
||||
next_token = torch.argmax(logits).item()
|
||||
generated.append(next_token)
|
||||
new_token = torch.tensor([[next_token]], device=self.device)
|
||||
input_tensor = torch.cat([input_tensor, new_token], dim=1)
|
||||
continue
|
||||
|
||||
next_token_logits = logits / temperature
|
||||
|
||||
# === 重复惩罚 ===
|
||||
# 1. 对最近生成的token进行惩罚（更强）
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7c4d05531958838c1c6750216a98fc781ea13085b278a0154eb1378680959fe9
|
||||
size 10038
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
API生成模块 — Astron 5并发，稳定不超限
|
||||
"""
|
||||
import json, requests, time, os, threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚，不要讲一半就停"""
|
||||
|
||||
|
||||
def call_astron(q, temperature=0.85):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": q}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def batch_generate(questions, temperature=0.85, max_workers=4):
|
||||
"""Astron 4并发 — 明确限制，稳定可靠"""
|
||||
total = len(questions)
|
||||
results = []
|
||||
|
||||
def call(q):
|
||||
ans = call_astron(q, temperature)
|
||||
if ans:
|
||||
return q, ans
|
||||
return None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
futures = {executor.submit(call, item): i for i, item in enumerate(questions)}
|
||||
done = 0
|
||||
t0 = time.time()
|
||||
for f in as_completed(futures):
|
||||
done += 1
|
||||
r = f.result()
|
||||
if r:
|
||||
results.append({"question": r[0], "answer": r[1]})
|
||||
if done % 20 == 0 or done == total:
|
||||
elapsed = max(time.time() - t0, 1)
|
||||
rate = len(results) / elapsed * 60
|
||||
print(f" Astron: {done}/{total} (已获{len(results)}条, {rate:.0f}条/分)", end="\r")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
rate = len(results) / max(elapsed, 1) * 60
|
||||
print(f" Astron: {total}/{total} → 共{len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
|
||||
return results
|
||||
|
||||
|
||||
# 保持旧函数名兼容
|
||||
batch_generate_astron = batch_generate
|
||||
batch_generate_mimo = batch_generate
|
||||
batch_generate_direct = batch_generate
|
||||
batch_generate_dual = batch_generate
|
||||
|
||||
|
||||
def batch_convert(items, *args, **kwargs):
|
||||
"""兼容占位：训练脚本导入用，已弃用"""
|
||||
print(" [API] batch_convert已弃用，使用batch_generate")
|
||||
return []
|
||||
|
||||
|
||||
def batch_generate_multi_temp(questions, *args, **kwargs):
|
||||
"""兼容占位：训练脚本导入用，已弃用"""
|
||||
print(" [API] batch_generate_multi_temp已弃用，使用batch_generate")
|
||||
return []
|
||||
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
每日数据生成 — 每天12:00跑
|
||||
用 Astron 生成大量小狸风格问答数据，追加到训练集（不触发训练）
|
||||
"""
|
||||
import sys, os, json, time, random
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
from api_correction import batch_generate
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
MAX_WORKERS = 5 # Astron 5并发（无限量）
|
||||
BATCH_SIZE = 500
|
||||
|
||||
# 每日生成量 — Astron无限量，直接拉满！
|
||||
DAILY_TOPIC_QS = 2000 # 话题衍生
|
||||
DAILY_NATURAL_QS = 1000 # 自然闲聊
|
||||
|
||||
# ===== 种子话题（每天随机选一批让小狸聊） =====
|
||||
SEED_TOPICS = [
|
||||
"幼儿园", "学校", "朋友", "玩具", "动画片", "宠物", "零食", "天气",
|
||||
"节日", "生日", "爸爸妈妈", "老师", "作业", "画画", "唱歌", "跳舞",
|
||||
"游乐园", "动物园", "海洋馆", "公园", "滑滑梯", "秋千", "积木", "拼图",
|
||||
"自行车", "游泳", "跳绳", "踢球", "冰淇淋", "蛋糕", "水果", "蔬菜",
|
||||
"彩虹", "星星", "月亮", "太阳", "花", "草", "树", "小动物",
|
||||
"蝴蝶", "小鸟", "小鱼", "乌龟", "兔子", "猫", "狗", "恐龙",
|
||||
"过年", "压岁钱", "红包", "新衣服", "放鞭炮", "灯笼", "元宵",
|
||||
"圣诞老人", "礼物", "糖果", "巧克力", "饼干", "牛奶", "早餐",
|
||||
"为什么天空是蓝的", "为什么树叶会掉", "为什么有影子", "为什么会下雨",
|
||||
"为什么会有白天黑夜", "为什么海水是咸的", "为什么会打雷",
|
||||
"1+1等于几", "数数", "认字", "背诗", "唱歌", "讲故事",
|
||||
"礼貌", "分享", "排队", "帮助别人", "说谢谢", "对不起",
|
||||
"害怕", "开心", "难过", "生气", "勇敢", "害羞",
|
||||
# 拓展话题
|
||||
"奥特曼", "小猪佩奇", "汪汪队", "海底小纵队", "超级飞侠",
|
||||
"白雪公主", "灰姑娘", "小红帽", "三只小猪", "匹诺曹",
|
||||
"茉莉花", "向日葵", "蒲公英", "含羞草", "仙人掌",
|
||||
"企鹅", "熊猫", "长颈鹿", "大象", "海豚", "松鼠", "刺猬",
|
||||
"草莓", "西瓜", "葡萄", "芒果", "蓝莓", "橙子",
|
||||
"汉堡", "薯条", "披萨", "面条", "饺子", "馄饨", "粽子",
|
||||
"沙滩", "大海", "高山", "森林", "瀑布", "溶洞",
|
||||
"火箭", "飞机", "高铁", "轮船", "自行车", "滑板",
|
||||
"围棋", "象棋", "跳棋", "五子棋", "扑克牌",
|
||||
"机器人", "变形金刚", "乐高", "磁力片", "橡皮泥",
|
||||
"春天", "夏天", "秋天", "冬天", "刮风", "下雪", "打雷",
|
||||
"刷牙", "洗脸", "洗澡", "剪指甲", "梳头", "穿衣服",
|
||||
"诚实", "分享", "礼貌", "排队", "帮忙", "道歉", "谢谢",
|
||||
"高兴", "生气", "难过", "害怕", "惊讶", "害羞", "委屈",
|
||||
"萤火虫", "蜻蜓", "蜜蜂", "蝉", "蜗牛", "蚂蚁", "蜘蛛",
|
||||
"玉米", "土豆", "番茄", "胡萝卜", "白菜", "茄子", "辣椒",
|
||||
"毛笔", "铅笔", "蜡笔", "水彩笔", "画板", "颜料",
|
||||
"竹笋", "蘑菇", "松树", "枫叶", "荷花", "梅花",
|
||||
"元旦", "春节", "元宵节", "清明节", "端午节", "中秋节",
|
||||
"万圣节", "感恩节", "圣诞节", "儿童节", "母亲节", "父亲节",
|
||||
"月球", "太阳系", "银河", "彗星", "流星", "黑洞",
|
||||
"恐龙", "猛犸象", "剑齿虎", "翼龙", "三角龙", "霸王龙",
|
||||
"三字经", "弟子规", "唐诗", "宋词", "成语",
|
||||
"捉迷藏", "老鹰捉小鸡", "丢手绢", "跳房子", "踢毽子",
|
||||
"轮滑", "滑冰", "滑雪", "冲浪", "攀岩",
|
||||
"钢琴", "吉他", "小提琴", "笛子", "鼓", "古筝",
|
||||
"警察", "消防员", "医生", "护士", "科学家", "宇航员",
|
||||
"红绿灯", "斑马线", "安全带", "头盔", "救生圈",
|
||||
"原来如此", "没关系", "太好了", "加油", "真棒", "别担心",
|
||||
"手影", "折纸", "翻花绳", "吹泡泡", "放风筝",
|
||||
"长城", "故宫", "天安门", "兵马俑", "大熊猫基地",
|
||||
"小木屋", "城堡", "树屋", "帐篷", "房车",
|
||||
]
|
||||
|
||||
def generate_questions_from_topics(topics, count=500):
|
||||
"""用话题种子生成自然提问"""
|
||||
questions = []
|
||||
for topic in topics[:count]:
|
||||
patterns = [
|
||||
f"小狸，你喜欢{topic}吗？",
|
||||
f"给我讲讲{topic}吧",
|
||||
f"{topic}是什么呀？",
|
||||
f"你最喜欢什么样的{topic}？",
|
||||
f"关于{topic}你知道什么？",
|
||||
f"小狸，{topic}好玩吗？",
|
||||
f"能不能教我关于{topic}的事情？",
|
||||
f"你上次玩{topic}是什么时候？",
|
||||
f"大家为什么都喜欢{topic}呀？",
|
||||
f"小狸，你在{topic}上有什么有趣的事吗？",
|
||||
]
|
||||
q = random.choice(patterns)
|
||||
questions.append(q)
|
||||
return questions
|
||||
|
||||
def generate_natural_questions(count=200):
|
||||
"""生成自然闲聊问题（固定一批高质量问题）"""
|
||||
base_questions = [
|
||||
"小狸你好呀", "今天开心吗", "你几岁了", "你叫什么名字",
|
||||
"你吃饭了吗", "我们一起玩吧", "你在干嘛呢", "你今天做了什么",
|
||||
"你喜欢什么颜色", "你最好的朋友是谁", "你最喜欢什么动物",
|
||||
"你最喜欢吃什么", "你最喜欢什么水果", "你最爱看什么动画片",
|
||||
"可以给我讲个故事吗", "唱首歌给我听", "教我背一首诗吧",
|
||||
"你会画画吗", "我们来玩猜谜语", "你家里有谁呀",
|
||||
"你的房间是什么样子的", "你书包里有什么", "你最喜欢上什么课",
|
||||
"今天天气真好", "外面下雨了", "好冷啊", "好热啊",
|
||||
"我有点不开心", "我今天好开心", "我害怕打雷",
|
||||
"你睡午觉了吗", "晚安", "早安", "快来",
|
||||
"我想吃冰淇淋", "我想去游乐场", "我想养一只小猫",
|
||||
"为什么雪是白色的", "为什么天是蓝的", "为什么花是香的",
|
||||
"月亮上有人吗", "星星为什么会眨眼睛", "大海有多大",
|
||||
"你会数数吗", "1+1等于几", "100+200等于多少",
|
||||
"我喜欢你小狸", "你好可爱", "我们做朋友吧",
|
||||
"你今天真好看", "谢谢你", "你真好",
|
||||
"对不起", "我不小心摔倒了", "我被人欺负了",
|
||||
"怎么办呀", "帮帮我", "我该怎么做",
|
||||
"什么是爱", "什么是勇敢", "什么是善良",
|
||||
"长大是什么感觉", "为什么要上学", "为什么要听话",
|
||||
"你会一直陪着我吗", "你想去什么地方玩",
|
||||
]
|
||||
return random.sample(base_questions, min(count, len(base_questions)))
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("📅 每日数据生成")
|
||||
print(f"时间: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("=" * 60)
|
||||
|
||||
# 1. 混合生成问题
|
||||
random.seed(int(time.time()))
|
||||
n_topic_qs = DAILY_TOPIC_QS # 话题衍生
|
||||
n_natural_qs = DAILY_NATURAL_QS # 自然闲聊
|
||||
|
||||
random.shuffle(SEED_TOPICS)
|
||||
topic_questions = generate_questions_from_topics(SEED_TOPICS, n_topic_qs)
|
||||
natural_questions = generate_natural_questions(n_natural_qs)
|
||||
|
||||
all_questions = topic_questions + natural_questions
|
||||
random.shuffle(all_questions)
|
||||
|
||||
print(f"\n准备生成 {len(all_questions)} 条问答 (话题衍生: {len(topic_questions)}, 自然: {len(natural_questions)})")
|
||||
|
||||
# 2. 用不同温度分批生成，增加多样性
|
||||
TEMPS = [0.85, 0.9, 0.95, 1.0]
|
||||
all_results = []
|
||||
existing = set()
|
||||
|
||||
# 加载已有数据用于去重
|
||||
train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json')
|
||||
with open(train_path, encoding='utf-8') as f:
|
||||
cur_train = json.load(f)
|
||||
for it in cur_train:
|
||||
key = (it.get('question','')[:100], it.get('answer','')[:100])
|
||||
existing.add(key)
|
||||
print(f"已有训练集: {len(cur_train)} 条")
|
||||
|
||||
# 分批+变温度
|
||||
total_batches = (len(all_questions) + BATCH_SIZE - 1) // BATCH_SIZE
|
||||
t_start = time.time()
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
batch_qs = all_questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
|
||||
temp = random.choice(TEMPS)
|
||||
|
||||
print(f"\n--- 批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, temp={temp}) ---")
|
||||
|
||||
t0 = time.time()
|
||||
results = batch_generate(batch_qs, temperature=temp, max_workers=MAX_WORKERS)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
new_in_batch = 0
|
||||
for r in results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in existing:
|
||||
existing.add(key)
|
||||
all_results.append(r)
|
||||
new_in_batch += 1
|
||||
|
||||
total_elapsed = time.time() - t_start
|
||||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||||
print(f" 本批新增 {new_in_batch} 条 | 累计 {len(all_results)} 条 | {rate:.0f}条/分")
|
||||
|
||||
if not all_results:
|
||||
print("\n⚠️ 没有生成新数据")
|
||||
return
|
||||
|
||||
# 3. 追加到训练集
|
||||
print(f"\n追加 {len(all_results)} 条到训练集...")
|
||||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||||
tp = os.path.join(MODEL_DIR, tgt)
|
||||
with open(tp, encoding='utf-8') as f:
|
||||
cur = json.load(f)
|
||||
|
||||
ex = set((it.get('question','')[:100], it.get('answer','')[:100]) for it in cur)
|
||||
to_add = []
|
||||
for r in all_results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in ex:
|
||||
ex.add(key)
|
||||
to_add.append({'question': r['question'], 'answer': r['answer']})
|
||||
|
||||
cur.extend(to_add)
|
||||
with open(tp, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||||
|
||||
if 'train' in tgt:
|
||||
print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条")
|
||||
else:
|
||||
print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条")
|
||||
|
||||
# 4. 保存今日生成日志
|
||||
today = time.strftime('%Y-%m-%d')
|
||||
log_path = os.path.join(MODEL_DIR, f'daily_gen_{today}.json')
|
||||
with open(log_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
|
||||
elapsed_total = time.time() - t_start
|
||||
print(f"\n{'='*60}")
|
||||
print(f"✅ 今日生成完成! {len(all_results)} 条新数据")
|
||||
print(f"用时: {elapsed_total/60:.1f} 分钟")
|
||||
print(f"日志: daily_gen_{today}.json")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
单批生成 — 从mega_questions.json取指定batch生成并追加
|
||||
用法: python3 generate_batch.py <batch_num> <batch_size>
|
||||
"""
|
||||
import sys, os, json, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
from api_correction import batch_generate
|
||||
|
||||
model_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
batch_num = int(sys.argv[1])
|
||||
batch_size = int(sys.argv[2])
|
||||
|
||||
# 加载问题列表
|
||||
with open(os.path.join(model_dir, 'mega_questions.json'), 'r', encoding='utf-8') as f:
|
||||
mega = json.load(f)
|
||||
|
||||
# 计算本批范围
|
||||
start = (batch_num - 1) * batch_size
|
||||
end = min(start + batch_size, len(mega))
|
||||
chunk = [item['question'] for item in mega[start:end]]
|
||||
|
||||
if not chunk:
|
||||
print(f"批{batch_num}: 无数据")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"批 {batch_num}: {len(chunk)}题 (序号{start}-{end})")
|
||||
t0 = time.time()
|
||||
|
||||
results = batch_generate(chunk, temperature=0.85)
|
||||
dt = time.time() - t0
|
||||
|
||||
# 保存中间文件
|
||||
interim_path = os.path.join(model_dir, f'mega_interim_{batch_num}.json')
|
||||
with open(interim_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(results, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 追加到训练集
|
||||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||||
tgt_path = os.path.join(model_dir, tgt)
|
||||
if not os.path.exists(tgt_path):
|
||||
continue
|
||||
with open(tgt_path, 'r', encoding='utf-8') as f:
|
||||
cur = json.load(f)
|
||||
existing = set((item.get("question",""), item.get("answer","")) for item in cur)
|
||||
truly_new = [item for item in results if (item["question"], item["answer"]) not in existing]
|
||||
if truly_new:
|
||||
cur.extend(truly_new)
|
||||
with open(tgt_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||||
if tgt == 'xiaoli_train_v2.json':
|
||||
print(f" ✅ 追加训练集: +{len(truly_new)}条, 现有{len(cur)}条")
|
||||
|
||||
print(f"批{batch_num}完成: +{len(results)}条 ({dt:.0f}s)")
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""高温度多样化生成 — 每个问题跑 N 次，获得多种回答"""
|
||||
import sys, os, json, random, time
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
from api_correction import batch_generate_astron as batch_generate
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# 载入全部问题
|
||||
with open(os.path.join(MODEL_DIR, 'mega_questions.json')) as f:
|
||||
mega = json.load(f)
|
||||
total_q = len(mega)
|
||||
print(f"mega_questions 共 {total_q} 条")
|
||||
|
||||
# 温度配置：逐渐升高，产出多样化
|
||||
TEMP_CONFIG = [0.9, 1.1, 1.3, 1.5] # 跑4轮
|
||||
all_new_results = []
|
||||
|
||||
for round_idx, temp in enumerate(TEMP_CONFIG):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"第{round_idx+1}轮: 温度={temp}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
questions = [item['question'] for item in mega]
|
||||
|
||||
t0 = time.time()
|
||||
results = batch_generate(questions, temperature=temp)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# 标注来源
|
||||
for r in results:
|
||||
r['temperature'] = temp
|
||||
r['round'] = round_idx + 1
|
||||
|
||||
all_new_results.extend(results)
|
||||
print(f"轮{round_idx+1} 完成: {len(results)}条 ({elapsed:.0f}s)")
|
||||
print(f"累计: {len(all_new_results)}条")
|
||||
|
||||
# 去重（问题+回答完全重复才去）
|
||||
print(f"\n总体去重中...")
|
||||
seen = set()
|
||||
unique_results = []
|
||||
for r in all_new_results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_results.append(r)
|
||||
|
||||
print(f"去重前: {len(all_new_results)}条 → 去重后: {len(unique_results)}条")
|
||||
|
||||
# 追加到训练集
|
||||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||||
tp = os.path.join(MODEL_DIR, tgt)
|
||||
with open(tp, encoding='utf-8') as f:
|
||||
cur = json.load(f)
|
||||
|
||||
ex = set((it.get("question","")[:100], it.get("answer","")[:100]) for it in cur)
|
||||
to_add = []
|
||||
for r in unique_results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in ex:
|
||||
ex.add(key)
|
||||
to_add.append({"question": r['question'], "answer": r['answer']})
|
||||
|
||||
cur.extend(to_add)
|
||||
with open(tp, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||||
if 'train' in tgt:
|
||||
print(f"✅ 训练集 +{len(to_add)}条 → {len(cur)}条")
|
||||
|
||||
print(f"\n🎉 全部完成!")
|
||||
print(f"总数: {len(cur)}条")
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
高温度多样化生成 — 单线程，与常规跑批共用 Astron 5并发中的1个
|
||||
对已完成批1-6的问题，用高温度重新生成，获得多样回答
|
||||
"""
|
||||
import sys, os, json, random, time, requests
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚，不要讲一半就停"""
|
||||
|
||||
|
||||
def call_astron(q, temperature):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": q}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
# 从已完成批载入问题
|
||||
all_questions = []
|
||||
for i in range(1, 7):
|
||||
fp = os.path.join(MODEL_DIR, f'mega_interim_{i}.json')
|
||||
if os.path.exists(fp):
|
||||
with open(fp) as f:
|
||||
items = json.load(f)
|
||||
for item in items:
|
||||
all_questions.append(item['question'])
|
||||
print(f"批{i}: {len(items)}条已载入")
|
||||
|
||||
# 去重
|
||||
all_questions = list(dict.fromkeys(all_questions))
|
||||
print(f"共 {len(all_questions)} 个去重问题")
|
||||
|
||||
# 温度配置
|
||||
temp_configs = [0.9, 1.3, 1.5]
|
||||
all_new = []
|
||||
|
||||
for round_idx, temp in enumerate(temp_configs):
|
||||
print(f"\n{'='*50}")
|
||||
print(f"轮{round_idx+1}/3: 温度={temp}")
|
||||
print(f"{'='*50}")
|
||||
|
||||
# 单线程顺序跑
|
||||
results = []
|
||||
total = len(all_questions)
|
||||
t0 = time.time()
|
||||
|
||||
for idx, q in enumerate(all_questions):
|
||||
ans = call_astron(q, temp)
|
||||
if ans:
|
||||
results.append({"question": q, "answer": ans})
|
||||
|
||||
if (idx + 1) % 20 == 0 or idx + 1 == total:
|
||||
elapsed = max(time.time() - t0, 1)
|
||||
rate = len(results) / elapsed * 60
|
||||
eta = (total - idx - 1) / max(idx + 1, 1) * elapsed / 60
|
||||
print(f" 多样: {idx+1}/{total} (已获{len(results)}条, {rate:.0f}条/分, ETA:{eta:.0f}分)", end="\r")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
rate = len(results) / max(elapsed, 1) * 60
|
||||
print(f"\n 轮{round_idx+1}完成: {len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)")
|
||||
|
||||
for r in results:
|
||||
r['temperature'] = temp
|
||||
r['round'] = round_idx + 1
|
||||
all_new.extend(results)
|
||||
|
||||
# ✅ 每轮跑完立即去重并追加到训练集
|
||||
seen = set()
|
||||
unique = []
|
||||
for r in all_new:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(r)
|
||||
|
||||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||||
tp = os.path.join(MODEL_DIR, tgt)
|
||||
with open(tp, encoding='utf-8') as f:
|
||||
cur = json.load(f)
|
||||
ex = set((it.get("question","")[:100], it.get("answer","")[:100]) for it in cur)
|
||||
to_add = []
|
||||
for r in unique:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in ex:
|
||||
ex.add(key)
|
||||
to_add.append({"question": r['question'], "answer": r['answer']})
|
||||
cur.extend(to_add)
|
||||
with open(tp, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||||
if 'train' in tgt:
|
||||
print(f"✅ 轮{round_idx+1} 训练集 +{len(to_add)}条 → {len(cur)}条")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
全新问答对生成 — Astron 凭空造（新问题+新回答），5并发
|
||||
不依赖现有问题列表，从种子话题衍生
|
||||
增量保存防丢
|
||||
"""
|
||||
import json, requests, time, os, sys, random, hashlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
# ====== Astron 配置 ======
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚"""
|
||||
|
||||
# ====== 种子场景（生成对话的起点） ======
|
||||
SCENES = [
|
||||
# 日常场景
|
||||
("问候", ["打招呼", "问好", "早上起来", "放学回家", "见到朋友"]),
|
||||
("情绪", ["开心", "难过", "生气", "无聊", "害怕", "惊喜", "委屈", "害羞"]),
|
||||
("分享", ["今天发生的事", "周末做了什么", "假期计划", "新玩具", "考试成绩"]),
|
||||
("喜好", ["喜欢的颜色", "喜欢的动物", "喜欢的食物", "喜欢的动画片", "喜欢的游戏"]),
|
||||
("好奇", ["为什么问题", "想知道的事", "不懂的事", "想象的事", "奇怪的问题"]),
|
||||
("日常", ["起床", "刷牙", "吃饭", "上学", "做作业", "睡觉"]),
|
||||
("朋友", ["好朋友", "吵架", "一起玩", "分享秘密", "帮忙"]),
|
||||
("家庭", ["爸爸妈妈", "爷爷奶奶", "兄弟姐妹", "家人一起", "家庭活动"]),
|
||||
("想象", ["童话故事", "梦想", "如果我有魔法", "长大想做什么", "幻想朋友"]),
|
||||
("安慰", ["受伤了", "丢东西", "被骂了", "考试没考好", "被欺负", "做噩梦"]),
|
||||
("节日", ["生日", "过年", "圣诞节", "儿童节", "中秋"]),
|
||||
("自然", ["天气", "动物", "植物", "星空", "大海", "四季"]),
|
||||
("学校", ["老师", "同学", "上课", "课间", "课外活动", "考试"]),
|
||||
("晚安", ["说晚安", "睡前", "梦", "明天见", "晚安故事"]),
|
||||
]
|
||||
|
||||
def call_astron(prompt, temperature=0.9):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 4096,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content and len(content) > 10:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def parse_qa_pairs(text):
|
||||
"""解析 Astron 返回的问答对列表"""
|
||||
lines = text.strip().split('\n')
|
||||
pairs = []
|
||||
current_q = None
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
q_match = __import__('re').match(r'^[Q问][：:]\s*(.*)', line)
|
||||
a_match = __import__('re').match(r'^[A答][：:]\s*(.*)', line)
|
||||
if q_match:
|
||||
current_q = q_match.group(1).strip()
|
||||
elif a_match and current_q:
|
||||
a_text = a_match.group(1).strip()
|
||||
if current_q and a_text and len(a_text) > 2 and len(current_q) > 2:
|
||||
pairs.append((current_q, a_text))
|
||||
current_q = None
|
||||
return pairs
|
||||
|
||||
|
||||
def pair_fingerprint(q, a):
|
||||
"""QA对指纹"""
|
||||
raw = f"{q[:40]}|{a[:40]}"
|
||||
return hashlib.md5(raw.encode()).hexdigest()
|
||||
|
||||
|
||||
def generate_batch_pairs(batch_size=10, existing_fps=None):
|
||||
"""生成一批全新的问答对"""
|
||||
if existing_fps is None:
|
||||
existing_fps = set()
|
||||
|
||||
# 随机选场景
|
||||
category, sub_scenes = random.choice(SCENES)
|
||||
sub = random.choice(sub_scenes)
|
||||
# 随机生成要求
|
||||
style = random.choice(["日常聊天", "撒娇可爱", "有点害羞", "好奇宝宝", "暖心安慰"])
|
||||
mood = random.choice(["开心", "平静", "有点小情绪", "好奇", "温暖"])
|
||||
|
||||
prompt = f"""请以小狸的身份，生成{batch_size}个不同的日常对话问答对。
|
||||
|
||||
对话背景：
|
||||
- 场景：{category}（{sub}）
|
||||
- 风格：{style}
|
||||
- 小狸心情：{mood}
|
||||
|
||||
要求：
|
||||
1. 每个问答对包含一问一答，问是用户说的，答是小狸回答
|
||||
2. 问要自然，像真实用户会说的话（不要提"小狸"）
|
||||
3. 答要是小狸的风格，1-2句话，简短自然
|
||||
4. {batch_size}个问答要对同一个话题但角度不同
|
||||
5. 不要重复一模一样的问法
|
||||
|
||||
请按以下格式输出：
|
||||
Q：用户提问内容
|
||||
A：小狸回答内容
|
||||
---（分隔）
|
||||
Q：用户提问内容
|
||||
A：小狸回答内容
|
||||
---"""
|
||||
|
||||
result = call_astron(prompt, temperature=random.uniform(0.85, 1.0))
|
||||
if not result:
|
||||
return []
|
||||
|
||||
pairs = parse_qa_pairs(result)
|
||||
|
||||
# 去重
|
||||
deduped = []
|
||||
for q, a in pairs:
|
||||
fp = pair_fingerprint(q, a)
|
||||
if fp not in existing_fps:
|
||||
existing_fps.add(fp)
|
||||
deduped.append({'question': q, 'answer': a})
|
||||
|
||||
return deduped
|
||||
|
||||
|
||||
def main():
|
||||
output_dir = '/workspace/xiaoli-llm-chat/fresh_qa_outputs'
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# ====== 恢复进度 ======
|
||||
progress_file = os.path.join(output_dir, 'fresh_qa_progress.json')
|
||||
all_pairs = []
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, encoding='utf-8') as f:
|
||||
all_pairs = json.load(f)
|
||||
print(f"已有进度: {len(all_pairs)} 条")
|
||||
|
||||
# 加载已有训练集做去重
|
||||
train_path = '/workspace/xiaoli-llm-chat/xiaoli_train_v2.json'
|
||||
with open(train_path, encoding='utf-8') as f:
|
||||
train_data = json.load(f)
|
||||
existing_fps = set(pair_fingerprint(item.get('question',''), item.get('answer','')) for item in train_data)
|
||||
for p in all_pairs:
|
||||
existing_fps.add(pair_fingerprint(p['question'], p['answer']))
|
||||
print(f"去重指纹池: {len(existing_fps)} 个")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 500000 # 目标 50 万条全新问答
|
||||
BATCH_SIZE = 15 # 每批次生成 15 条（更多问题，效率更高）
|
||||
BATCHES_PER_SAVE = 20 # 每 20 批次保存一次
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
|
||||
remaining = TARGET - len(all_pairs)
|
||||
total_iterations = (remaining + BATCH_SIZE - 1) // BATCH_SIZE
|
||||
print(f"目标: {TARGET:,} 条, 还需: {remaining:,} 条")
|
||||
print(f"轮次: {total_iterations} 轮 (每轮{BATCH_SIZE}条, {MAX_WORKERS}并发)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
t_start = time.time()
|
||||
batch_counter = 0
|
||||
|
||||
for iteration in range(total_iterations):
|
||||
t0 = time.time()
|
||||
|
||||
# 并发生成
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = [
|
||||
executor.submit(generate_batch_pairs, BATCH_SIZE, existing_fps)
|
||||
for _ in range(MAX_WORKERS)
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
new_pairs = future.result()
|
||||
if new_pairs:
|
||||
for p in new_pairs:
|
||||
existing_fps.add(pair_fingerprint(p['question'], p['answer']))
|
||||
all_pairs.append(p)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
batch_counter += 1
|
||||
elapsed = time.time() - t_start
|
||||
rate = len(all_pairs) / max(elapsed, 1) * 60
|
||||
|
||||
# 定期保存
|
||||
if batch_counter % BATCHES_PER_SAVE == 0 or iteration == total_iterations - 1:
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_pairs, f, ensure_ascii=False, indent=2)
|
||||
|
||||
eta = (TARGET - len(all_pairs)) / max(rate, 0.1) / 60 if rate > 0 else 0
|
||||
|
||||
if iteration % 5 == 0 or iteration == total_iterations - 1:
|
||||
print(f"轮 {iteration+1}/{total_iterations} | "
|
||||
f"累计 {len(all_pairs):,}/{TARGET:,} | "
|
||||
f"{rate:.0f}条/分 | "
|
||||
f"已用 {elapsed/60:.1f}分 | "
|
||||
f"预计剩余 {eta:.1f}分")
|
||||
|
||||
# ====== 完成 ======
|
||||
elapsed_total = time.time() - t_start
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全新问答生成完成!")
|
||||
print(f"总数: {len(all_pairs):,} 条")
|
||||
print(f"用时: {elapsed_total/60:.1f} 分钟")
|
||||
print(f"速率: {len(all_pairs)/max(elapsed_total,1)*60:.0f} 条/分")
|
||||
|
||||
# 保存最终
|
||||
final_path = '/data/user/work/fresh_qa_v1.json'
|
||||
with open(final_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_pairs, f, ensure_ascii=False, indent=2)
|
||||
print(f"保存到: {final_path}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
海量通用数据蒸馏 v3 — 不设场景标签，纯自然对话，多策略并行
|
||||
目标：500万+ 纯 (question, answer) 对
|
||||
"""
|
||||
import json, requests, time, os, sys, random, re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
OUT_DIR = os.path.join(MODEL_DIR, 'general_outputs')
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
|
||||
# ====== Astron 配置 ======
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
# ====== 主题全覆盖（六大类 × 子类） ======
|
||||
TOPICS = {
|
||||
"日常": [
|
||||
"起床洗漱", "吃早餐", "上学路上", "上课", "课间休息",
|
||||
"午餐时间", "放学回家", "做作业", "晚饭", "睡前时光",
|
||||
"周末活动", "天气变化", "穿衣打扮", "零花钱", "家务劳动",
|
||||
"看电视", "玩玩具", "去公园", "逛超市", "去亲戚家",
|
||||
],
|
||||
"情绪": [
|
||||
"开心高兴", "难过伤心", "生气愤怒", "害怕恐惧", "紧张焦虑",
|
||||
"孤独寂寞", "委屈冤枉", "失望沮丧", "感动温暖", "兴奋激动",
|
||||
"无聊烦闷", "尴尬害羞", "后悔内疚", "羡慕嫉妒", "感激感恩",
|
||||
],
|
||||
"社交": [
|
||||
"交新朋友", "和好朋友玩", "和同学相处", "和老师交流", "和兄弟姐妹",
|
||||
"和爸爸妈妈", "和爷爷奶奶", "和邻居", "和陌生人", "和喜欢的人",
|
||||
"吵架和好", "被欺负", "被误解", "被表扬", "被批评",
|
||||
"帮助别人", "请求帮助", "分享东西", "邀请做客", "告别分开",
|
||||
],
|
||||
"知识": [
|
||||
"动物世界", "植物花草", "天文星空", "天气自然", "人体健康",
|
||||
"食物营养", "颜色形状", "数字计算", "汉字词语", "科学现象",
|
||||
"历史故事", "地理知识", "交通工具", "乐器音乐", "绘画手工",
|
||||
"运动体育", "节日习俗", "各国文化", "发明创造", "宇宙太空",
|
||||
],
|
||||
"想象": [
|
||||
"超能力", "魔法世界", "未来科技", "外星人", "恐龙时代",
|
||||
"童话故事", "变成动物", "飞行梦", "海底世界", "神秘岛屿",
|
||||
"时间旅行", "超级英雄", "会说话的玩具", "隐形衣", "变身药水",
|
||||
"机器人朋友", "彩虹桥", "星星糖果", "云朵房子", "梦中冒险",
|
||||
],
|
||||
"成长": [
|
||||
"学新本领", "克服困难", "不怕失败", "坚持努力", "自我肯定",
|
||||
"梦想未来", "职业理想", "好习惯", "改掉坏毛病", "学会分享",
|
||||
"学会等待", "学会道歉", "学会感恩", "学会勇敢", "学会理解",
|
||||
"交心谈话", "秘密分享", "烦恼倾诉", "人生道理", "温暖鼓励",
|
||||
],
|
||||
}
|
||||
|
||||
FLAT_TOPICS = []
|
||||
for cat, subs in TOPICS.items():
|
||||
for sub in subs:
|
||||
FLAT_TOPICS.append((cat, sub))
|
||||
|
||||
# ====== 多样化的开场方式 ======
|
||||
OPENERS = [
|
||||
"小狸", "嗨小狸", "小狸在吗", "嘿嘿", "那个…", "喂喂喂",
|
||||
"小狸小狸", "我想问你", "你知道吗", "你说", "我突然想到",
|
||||
"我跟你讲", "小狸你听我说", "问你个事", "我有个问题",
|
||||
"好无聊啊", "好开心呀", "好烦", "怎么办呀",
|
||||
"小狸你在干嘛", "我来了", "出来玩", "跟你说个事",
|
||||
]
|
||||
|
||||
# ====== 策略1: 直接生成纯问答对 ======
|
||||
def make_direct_pairs_prompt(count=15):
|
||||
"""无场景标签，直接生成纯问答对"""
|
||||
# 随机选3-5个主题混搭
|
||||
selected = random.sample(FLAT_TOPICS, min(random.randint(3, 5), len(FLAT_TOPICS)))
|
||||
topics_str = "、".join([f"{c}({s})" for c, s in selected])
|
||||
|
||||
prompt = f"""请直接生成 {count} 条聊天问答对。
|
||||
|
||||
格式要求（严格按此格式）：
|
||||
问：xxxxxxxxx | 答：xxxxxxxxx
|
||||
|
||||
用户的问题要：口语化、自然、像真人（5-25字）
|
||||
小狸的回答要：8岁小女孩语气，可爱活泼（10-40字）
|
||||
话题要多样：{topics_str}
|
||||
|
||||
示例：
|
||||
问：你怕不怕黑呀？ | 答：有一点点…但是抱着小熊玩偶就不怕啦！
|
||||
问：我今天跑步摔了一跤 | 答：疼不疼呀？我给你吹吹就不疼了！
|
||||
|
||||
现在生成 {count} 条："""
|
||||
return prompt
|
||||
|
||||
# ====== 策略2: 短对话（3-5轮） ======
|
||||
def make_mini_chat_prompt(count=8):
|
||||
"""微型多轮对话，每段3-5轮"""
|
||||
ops = random.sample(OPENERS, min(count, len(OPENERS)))
|
||||
|
||||
prompt = f"""请生成 {count} 段微型聊天对话，每段3-5轮（一来一回算1轮）。
|
||||
|
||||
要求：
|
||||
- 每段一个独立话题，话题要多样（日常、情绪、知识、想象、社交、成长等）
|
||||
- 用户说话要特别自然、口语化，像真人随口说的
|
||||
- 小狸像8岁小女孩，回答要可爱、有画面感
|
||||
- 话题要自然延伸，不要生硬转折
|
||||
|
||||
格式例子：
|
||||
话题：日常
|
||||
用户：今天好冷啊
|
||||
小狸：是呀！我穿了厚厚的羽绒服，像只小企鹅一样～
|
||||
用户：哈哈哈真的好像
|
||||
小狸：（鼓起腮帮子）才不像呢！企鹅走路摇摇摆摆的，我走路可稳啦！
|
||||
...
|
||||
|
||||
现在生成 {count} 段，每段用"话题：xxx"开头："""
|
||||
return prompt
|
||||
|
||||
# ====== 策略3: 长对话（6-10轮自由展开） ======
|
||||
def make_long_chat_prompt():
|
||||
"""一场长对话，话题自然展开"""
|
||||
topic = random.choice(FLAT_TOPICS)
|
||||
opener = random.choice(OPENERS)
|
||||
|
||||
emotions = ["开心", "难过", "兴奋", "沮丧", "好奇", "担心", "轻松", "紧张"]
|
||||
emotion = random.choice(emotions)
|
||||
|
||||
prompt = f"""请生成一段自然的日常聊天，{topic[0]}类-{topic[1]}话题。
|
||||
|
||||
用户开场：{opener}
|
||||
用户当下的心情：{emotion}
|
||||
|
||||
要求：
|
||||
- 用户很自然地说出开场白，就像真人发微信一样
|
||||
- 小狸正常回复，话题慢慢展开，不要硬聊
|
||||
- 聊6-10轮（一问一答算一轮）
|
||||
- 每轮格式：用户：xxx\n小狸：xxx
|
||||
- 真实感第一，不要刻意
|
||||
|
||||
请直接输出对话，不要加说明："""
|
||||
return prompt
|
||||
|
||||
# ====== 策略4: 单轮问答（高吞吐） ======
|
||||
def make_single_turn_prompt(batch_size=20):
|
||||
"""纯单轮问答，批量生成"""
|
||||
selected = random.sample(FLAT_TOPICS, min(6, len(FLAT_TOPICS)))
|
||||
|
||||
prompt = f"""生成 {batch_size} 条日常问答对。
|
||||
|
||||
要求：
|
||||
1. 用户的话：7-25字，口语化，真实感强
|
||||
2. 小狸回答：1-2句，8岁小女孩语气，可爱活泼
|
||||
3. 覆盖多样化话题：{', '.join([s for _, s in selected[:6]])}（但不仅限于这些）
|
||||
4. 每行一条，格式：问: ... | 答: ...
|
||||
5. 不要重复相同的问题
|
||||
|
||||
示例：
|
||||
问: 你喜不喜欢下雨天呀？ | 答: 喜欢！因为可以踩水坑，啪嗒啪嗒的超好玩！
|
||||
问: 我今天被同学起外号了 | 答: （歪着头）那…你喜欢这个外号吗？不喜欢的话要大声说出来哦！
|
||||
问: 你说世界上有外星人吗？ | 答: 我觉得有！说不定他们正在用望远镜看我们呢～
|
||||
|
||||
生成："""
|
||||
return prompt
|
||||
|
||||
# ====== API调用 ======
|
||||
def call_astron(prompt, temp=0.82, max_tokens=4096):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": "你是一个擅长写日常对话的助手。小狸是一个8岁的小女孩，说话可爱自然。"},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": temp,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=150)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content and len(content) > 20:
|
||||
return content
|
||||
else:
|
||||
time.sleep(0.5 + attempt)
|
||||
except Exception:
|
||||
time.sleep(1 + attempt)
|
||||
return None
|
||||
|
||||
# ====== 解析 ======
|
||||
def parse_pairs(text):
|
||||
"""从"问：xxx | 答：xxx"格式解析问答对"""
|
||||
pairs = []
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
parts = re.split(r'\s*[|｜]\s*', line)
|
||||
if len(parts) == 2:
|
||||
q = re.sub(r'^问\s*[：:]\s*', '', parts[0]).strip()
|
||||
a = re.sub(r'^答\s*[：:]\s*', '', parts[1]).strip()
|
||||
if q and a and len(q) >= 2 and len(a) >= 2 and len(a) < 300 and len(q) < 200:
|
||||
pairs.append((q, a))
|
||||
return pairs
|
||||
|
||||
def parse_mini_chats(text):
|
||||
"""解析微型多轮对话"""
|
||||
pairs = []
|
||||
segments = re.split(r'话题[：:].*?\n', text)
|
||||
|
||||
for seg in segments:
|
||||
seg = seg.strip()
|
||||
lines = seg.split('\n')
|
||||
q = None
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
q_match = re.match(r'^(用户|我)[：:]\s*(.*)', line)
|
||||
a_match = re.match(r'^(小狸|答)[：:]\s*(.*)', line)
|
||||
|
||||
if q_match:
|
||||
q_text = q_match.group(2).strip()
|
||||
if q_text and len(q_text) > 1:
|
||||
q = q_text
|
||||
elif a_match and q:
|
||||
a_text = a_match.group(2).strip()
|
||||
if a_text and len(a_text) > 1 and len(a_text) < 300:
|
||||
pairs.append((q, a_text))
|
||||
q = None
|
||||
return pairs
|
||||
|
||||
def parse_long_chat(text):
|
||||
"""解析长对话"""
|
||||
pairs = []
|
||||
q = None
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
q_match = re.match(r'^(用户|我)[：:]\s*(.*)', line)
|
||||
a_match = re.match(r'^(小狸)[：:]\s*(.*)', line)
|
||||
|
||||
if q_match:
|
||||
q_text = q_match.group(2).strip()
|
||||
if q_text and len(q_text) > 1:
|
||||
q = q_text
|
||||
elif a_match and q:
|
||||
a_text = a_match.group(2).strip()
|
||||
if a_text and len(a_text) > 1 and len(a_text) < 300:
|
||||
pairs.append((q, a_text))
|
||||
q = None
|
||||
return pairs
|
||||
|
||||
# ====== 生成器（按策略分发） ======
|
||||
GENERATORS = [
|
||||
# (name, prompt_fn, parse_fn, batch_size_estimate, weight_for_diversity)
|
||||
("direct", make_direct_pairs_prompt, parse_pairs, 15, 35),
|
||||
("mini_chat", make_mini_chat_prompt, parse_mini_chats, 20, 25),
|
||||
("long_chat", make_long_chat_prompt, parse_long_chat, 20, 20),
|
||||
("single", make_single_turn_prompt, parse_pairs, 20, 20),
|
||||
]
|
||||
|
||||
def run_one_generation():
|
||||
"""运行一次生成任务（随机选一种策略）"""
|
||||
# 按权重选策略
|
||||
weights = [w for _, _, _, _, w in GENERATORS]
|
||||
gen = random.choices(GENERATORS, weights=weights, k=1)[0]
|
||||
name, prompt_fn, parse_fn, batch_est, _ = gen
|
||||
|
||||
prompt = prompt_fn()
|
||||
result = call_astron(prompt, temp=random.uniform(0.75, 0.92), max_tokens=4096 if 'long' not in name else 6144)
|
||||
if not result:
|
||||
return None
|
||||
|
||||
pairs = parse_fn(result)
|
||||
if not pairs:
|
||||
return None
|
||||
|
||||
return pairs
|
||||
|
||||
def main():
|
||||
print("=" * 65)
|
||||
print(" 海量通用数据蒸馏 v3")
|
||||
print(" 不设场景标签 | 6大类120+子话题 | 4种策略")
|
||||
print("=" * 65)
|
||||
|
||||
# 加载已有进度
|
||||
all_pairs = []
|
||||
existing_keys = set()
|
||||
progress_file = os.path.join(OUT_DIR, 'all_general_progress.json')
|
||||
if os.path.exists(progress_file):
|
||||
existing = json.load(open(progress_file, encoding='utf-8'))
|
||||
all_pairs = existing
|
||||
for p in existing:
|
||||
existing_keys.add((p.get('question',''), p.get('answer','')))
|
||||
print(f"已有进度: {len(all_pairs):,} 对")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 2000000 # 目标200万对
|
||||
BATCH = 40 # 每批40个任务
|
||||
MAX_WORKERS = 8 # 8并发（比之前5更猛）
|
||||
CHECKPOINT = 5000 # 每5000对保存
|
||||
|
||||
to_generate = TARGET - len(all_pairs)
|
||||
total_batches = (to_generate + BATCH - 1) // BATCH
|
||||
|
||||
print(f"目标: {TARGET:,} 对")
|
||||
print(f"已有: {len(all_pairs):,} 对")
|
||||
print(f"还需: {to_generate:,} 对")
|
||||
print(f"批次: {total_batches:,} 批 ({BATCH}任务/批, {MAX_WORKERS}并发)")
|
||||
|
||||
# 策略统计
|
||||
strategy_stats = {g[0]: {'runs': 0, 'pairs': 0} for g in GENERATORS}
|
||||
|
||||
t_start = time.time()
|
||||
last_checkpoint = len(all_pairs)
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
t0 = time.time()
|
||||
|
||||
# 并发
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = [executor.submit(run_one_generation) for _ in range(BATCH)]
|
||||
batch_pairs = []
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
r = future.result()
|
||||
if r:
|
||||
batch_pairs.extend(r)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 去重
|
||||
new_pairs = []
|
||||
for q, a in batch_pairs:
|
||||
key = (q, a)
|
||||
if key not in existing_keys:
|
||||
existing_keys.add(key)
|
||||
new_pairs.append({'question': q, 'answer': a})
|
||||
|
||||
all_pairs.extend(new_pairs)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
total_elapsed = time.time() - t_start
|
||||
rate = len(all_pairs) / max(total_elapsed, 1) * 60
|
||||
|
||||
# 估算
|
||||
pairs_this_batch = len(new_pairs)
|
||||
total_raw = len(batch_pairs)
|
||||
dedup_rate = pairs_this_batch / max(total_raw, 1) * 100
|
||||
|
||||
print(f"[{batch_idx+1}/{total_batches}] "
|
||||
f"+{pairs_this_batch}对(有效{total_raw}/{dedup_rate:.0f}%) "
|
||||
f"累计{len(all_pairs):,}/{TARGET:,} "
|
||||
f"{rate:.0f}对/分 "
|
||||
f"剩余{(TARGET-len(all_pairs))/max(rate,0.1):.0f}分")
|
||||
|
||||
# 定期保存
|
||||
if len(all_pairs) - last_checkpoint >= CHECKPOINT:
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_pairs, f, ensure_ascii=False)
|
||||
last_checkpoint = len(all_pairs)
|
||||
|
||||
# 也保存到数据目录一份
|
||||
merged_path = os.path.join(MODEL_DIR, 'data', 'general_ongoing.json')
|
||||
with open(merged_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_pairs, f, ensure_ascii=False)
|
||||
|
||||
# 速率统计
|
||||
stable_rate = len(all_pairs) / max(time.time() - t_start, 1) * 60
|
||||
print(f" 💾 保存 ({len(all_pairs):,} 对, {stable_rate:.0f}对/分)")
|
||||
|
||||
# ====== 完成 ======
|
||||
total_elapsed = time.time() - t_start
|
||||
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_pairs, f, ensure_ascii=False)
|
||||
|
||||
print(f"\n{'='*65}")
|
||||
print(f"✅ 生成完成!")
|
||||
print(f" 总对数: {len(all_pairs):,}")
|
||||
print(f" 用时: {total_elapsed/60:.1f} 分")
|
||||
print(f" 平均速率: {len(all_pairs)/max(total_elapsed,1)*60:.0f} 对/分")
|
||||
print(f" 保存: {progress_file}")
|
||||
print(f"{'='*65}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,346 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
多轮对话生成 — 用 Astron 造 2~5 轮小狸闲聊对话
|
||||
5并发，无限量，增量保存防丢
|
||||
"""
|
||||
import json, requests, time, os, sys, random, re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
# ====== Astron 配置（和 api_correction 一致） ======
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
# ====== 小狸基础提示词 ======
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚"""
|
||||
|
||||
# ====== 多轮对话种子（开场话题池） ======
|
||||
SEEDS = [
|
||||
# 日常问候
|
||||
"你好呀", "你叫什么名字", "你今天开心吗", "你在干嘛呢",
|
||||
"你好可爱呀", "我来看你啦", "你今天心情怎么样",
|
||||
# 闲聊话题
|
||||
"你喜欢什么颜色", "你最喜欢什么动物", "你爱吃什么零食",
|
||||
"你最喜欢什么玩具", "你最喜欢什么动画片",
|
||||
"你最好的朋友是谁", "幼儿园里好玩吗",
|
||||
"你喜欢唱歌吗", "你会跳舞吗",
|
||||
# 情绪表达
|
||||
"我今天好开心", "我有点不开心", "我好无聊啊",
|
||||
"我遇到了一件好事", "我今天有点难过",
|
||||
"我好想你呀", "你能不能陪我聊聊天",
|
||||
# 好奇问题
|
||||
"为什么天是蓝的", "为什么月亮跟着我走",
|
||||
"世界上最大的动物是什么", "你会讲故事吗",
|
||||
"你会数数吗", "1+1等于几",
|
||||
"你喜欢上学吗", "你今年几岁啦",
|
||||
# 分享日常
|
||||
"我今天去公园玩了", "我养了一只小猫",
|
||||
"我今天考试考了100分", "我学会骑自行车了",
|
||||
"妈妈给我买了个新玩具", "我今天摔了一跤",
|
||||
# 晚安/告别
|
||||
"我要睡觉了", "明天见", "晚安啦",
|
||||
"我要去吃饭了", "下次再聊哦",
|
||||
]
|
||||
|
||||
# 话题扩展 — 组合出更多种子
|
||||
PREFIXES = ["小狸", "小狸小狸", "亲爱的小狸", "嘿嘿小狸", "小狸呀"]
|
||||
SUFFIXES = ["呀", "吗", "呢", "", "呀", "吗", "呢", ""]
|
||||
|
||||
def expand_seeds(base, max_expand=5):
|
||||
"""种子话题 → 多样化的开场白"""
|
||||
expanded = []
|
||||
for s in base:
|
||||
expanded.append(s)
|
||||
if '吗' not in s and not s.endswith('呀') and not s.endswith('呢'):
|
||||
for _ in range(2):
|
||||
prefix = random.choice(PREFIXES) if random.random() > 0.4 else ''
|
||||
suffix = random.choice(SUFFIXES) if random.random() > 0.5 else ''
|
||||
combo = f"{prefix}{s}{suffix}".strip()
|
||||
if combo and combo not in expanded:
|
||||
expanded.append(combo)
|
||||
random.shuffle(expanded)
|
||||
return expanded[:len(base) * max_expand]
|
||||
|
||||
|
||||
def call_astron_multi(prompt, temperature=0.85):
|
||||
"""调用 Astron 生成一条多轮对话（2~5轮）"""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content and len(content) > 10:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def parse_multi_turn(text):
|
||||
"""
|
||||
解析 Astron 返回的多轮对话文本为 [(q, a), (q, a), ...]
|
||||
期望返回格式（至少2轮对话）：
|
||||
问：xxx
|
||||
答：xxx
|
||||
问：xxx
|
||||
答：xxx
|
||||
|
||||
或
|
||||
用户：xxx
|
||||
小狸：xxx
|
||||
用户：xxx
|
||||
小狸：xxx
|
||||
"""
|
||||
lines = text.strip().split('\n')
|
||||
turns = []
|
||||
current_q = None
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# 匹配 "问：" 或 "问题：" 或 "用户：" 格式
|
||||
q_match = re.match(r'^(问|问题|用户|我)[：:]\s*(.*)', line)
|
||||
# 匹配 "答：" 或 "回答：" 或 "小狸：" 格式
|
||||
a_match = re.match(r'^(答|回答|小狸|小狸说|机器人)[：:]\s*(.*)', line)
|
||||
|
||||
if q_match:
|
||||
q_text = q_match.group(2).strip()
|
||||
if q_text and len(q_text) > 1:
|
||||
current_q = q_text
|
||||
elif a_match and current_q:
|
||||
a_text = a_match.group(2).strip()
|
||||
if a_text and len(a_text) > 1 and len(a_text) < 200:
|
||||
turns.append((current_q, a_text))
|
||||
current_q = None # 配对后重置，等待下一个问
|
||||
|
||||
# 最少4轮才算有效（6~12轮的长对话）
|
||||
if len(turns) >= 4:
|
||||
return turns
|
||||
return None
|
||||
|
||||
|
||||
def format_training_sample(turns):
|
||||
"""
|
||||
将多轮对话 [('q1','a1'), ('q2','a2'), ...] 格式化为训练样本
|
||||
|
||||
格式：<SOS>q1<SEP><START>a1<END>q2<SEP><START>a2<END><EOS>
|
||||
"""
|
||||
parts = []
|
||||
for i, (q, a) in enumerate(turns):
|
||||
if i == 0:
|
||||
parts.append(f"<SOS>{q}<SEP><START>{a}<END>")
|
||||
else:
|
||||
parts.append(f"{q}<SEP><START>{a}<END>")
|
||||
parts.append("<EOS>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def make_multi_prompt():
|
||||
"""
|
||||
构造多轮对话生成提示词
|
||||
Astron 每次生成一条完整的4~6轮对话
|
||||
"""
|
||||
# 随机选开场话题（2~3个组合，让对话有展开）
|
||||
topic_count = random.randint(1, 3)
|
||||
topics = random.sample(all_seeds, min(topic_count, len(all_seeds)))
|
||||
topic_context = "、".join(topics)
|
||||
|
||||
turn_count = random.randint(6, 12)
|
||||
|
||||
prompt = f"""请以小狸的身份，模拟一段长时间的自然闲聊对话，至少{turn_count}轮。
|
||||
|
||||
开场话题：{topic_context}
|
||||
|
||||
要求：
|
||||
1. 小狸是8岁小女孩，说话简短自然
|
||||
2. 每轮包含"问"和"答"，小狸的回答1-2句话
|
||||
3. 对话要自然展开，一问一答慢慢深入，不要强行切话题
|
||||
4. 小狸的语气：害羞但可爱，不是机器人
|
||||
5. 至少{turn_count}轮，越多越好，让对话持续下去
|
||||
|
||||
请按以下格式输出（至少{turn_count}轮，轮数越多越好）：
|
||||
问：（用户提问）
|
||||
答：（小狸回答）
|
||||
问：（用户续问）
|
||||
答：（小狸回答）
|
||||
..."""
|
||||
return prompt
|
||||
|
||||
|
||||
def generate_one_multi():
|
||||
"""生成一条多轮对话"""
|
||||
prompt = make_multi_prompt()
|
||||
result = call_astron_multi(prompt, temperature=random.uniform(0.8, 1.0))
|
||||
if result:
|
||||
turns = parse_multi_turn(result)
|
||||
if turns and len(turns) >= 4:
|
||||
sample = format_training_sample(turns)
|
||||
return {
|
||||
'source': 'multiturn_astron',
|
||||
'turns': len(turns),
|
||||
'sample': sample,
|
||||
'turns_list': turns,
|
||||
'raw': result[:100]
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
global all_seeds
|
||||
|
||||
model_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# 加载现有训练数据的问句作为种子
|
||||
train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
|
||||
with open(train_path, encoding='utf-8') as f:
|
||||
train_data = json.load(f)
|
||||
|
||||
# 从现有数据中随机抽取问句作为种子
|
||||
existing_qs = [item.get('question','') for item in train_data if item.get('question','')]
|
||||
random.shuffle(existing_qs)
|
||||
seed_from_data = existing_qs[:500]
|
||||
|
||||
# 合并种子话题
|
||||
all_seeds = expand_seeds(SEEDS, max_expand=10) + seed_from_data
|
||||
random.shuffle(all_seeds)
|
||||
|
||||
print(f"种子话题: {len(all_seeds)} 个 (基础{len(SEEDS)}+数据抽取{len(seed_from_data)})")
|
||||
|
||||
# 输出目录
|
||||
out_dir = os.path.join(model_dir, 'multiturn_outputs')
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
# 检查已有进度
|
||||
all_results = []
|
||||
interim_files = sorted([f for f in os.listdir(out_dir)
|
||||
if f.startswith('mt_batch_') and f.endswith('.json')])
|
||||
if interim_files:
|
||||
for fname in interim_files:
|
||||
with open(os.path.join(out_dir, fname), encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
all_results.extend(data)
|
||||
print(f"已有中间结果: {len(all_results)} 条 ({len(interim_files)} 批)")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 500000 # 目标 50 万条长多轮对话
|
||||
BATCH = 20 # 每批生成 20 条（每条更长，减少批量）
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
TEMPERATURE_RANGE = (0.80, 1.0)
|
||||
|
||||
to_generate = TARGET - len(all_results)
|
||||
total_batches = (to_generate + BATCH - 1) // BATCH
|
||||
|
||||
print(f"目标: {TARGET} 条, 已有: {len(all_results)} 条, 还需: {to_generate} 条")
|
||||
print(f"批次: {total_batches} 批 (每批{BATCH}条, {MAX_WORKERS}并发)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
t_start = time.time()
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
t0 = time.time()
|
||||
|
||||
# 并发生成一批
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = [executor.submit(generate_one_multi) for _ in range(BATCH)]
|
||||
batch_results = []
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
r = future.result()
|
||||
if r:
|
||||
batch_results.append(r)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 去重（按 sample 内容）
|
||||
existing_samples = set(r['sample'] for r in all_results)
|
||||
deduped = []
|
||||
for r in batch_results:
|
||||
if r['sample'] not in existing_samples:
|
||||
existing_samples.add(r['sample'])
|
||||
deduped.append(r)
|
||||
|
||||
all_results.extend(deduped)
|
||||
|
||||
# 保存本批
|
||||
elapsed = time.time() - t0
|
||||
batch_file = os.path.join(out_dir, f'mt_batch_{batch_idx + 1:04d}.json')
|
||||
with open(batch_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(deduped, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 追加保存总进度
|
||||
progress_file = os.path.join(out_dir, 'all_multiturn_progress.json')
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
|
||||
total_elapsed = time.time() - t_start
|
||||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||||
eta = (TARGET - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
|
||||
|
||||
# 输出统计
|
||||
avg_turns = sum(r['turns'] for r in deduped) / max(len(deduped), 1)
|
||||
print(f"批次 {batch_idx+1}/{total_batches} | "
|
||||
f"+{len(deduped)} 条 (去重后) | "
|
||||
f"累计 {len(all_results)}/{TARGET} | "
|
||||
f"均轮 {avg_turns:.1f} | "
|
||||
f"{rate:.0f}条/分 | "
|
||||
f"预计剩余 {eta:.1f}分")
|
||||
|
||||
# ====== 完成总结 ======
|
||||
total_elapsed = time.time() - t_start
|
||||
print(f"\n{'='*60}")
|
||||
print(f"多轮对话生成完成!")
|
||||
print(f"生成: {len(all_results)} 条 (去重后)")
|
||||
avg_turns2 = sum(r['turns'] for r in all_results) / max(len(all_results), 1)
|
||||
print(f"平均轮数: {avg_turns2:.1f}")
|
||||
print(f"总对话轮次: {sum(r['turns'] for r in all_results)}")
|
||||
print(f"用时: {total_elapsed/60:.1f} 分钟")
|
||||
print(f"速率: {len(all_results)/max(total_elapsed,1)*60:.0f} 条/分")
|
||||
|
||||
# ====== 保存最终结果（独立存放，不追加到训练集） ======
|
||||
final_file = os.path.join('/data/user/work', 'multiturn_final.json')
|
||||
with open(final_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n最终结果已保存: {final_file} ({len(all_results)} 条)")
|
||||
print(f"如需后续使用，可手动合并到训练集")
|
||||
|
||||
# ====== 保存多轮样本供训练时直接使用（含标记格式） ======
|
||||
mt_samples_file = os.path.join(model_dir, 'multiturn_samples_v1.json')
|
||||
with open(mt_samples_file, 'w', encoding='utf-8') as f:
|
||||
samples = [{'turns': r['turns'], 'sample': r['sample']} for r in all_results]
|
||||
json.dump(samples, f, ensure_ascii=False, indent=2)
|
||||
print(f" ✅ 多轮样本保存: multiturn_samples_v1.json ({len(samples)} 条)")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全部完成! 新增 {len(all_results)} 条多轮对话")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
海量真实多轮对话生成 — 模拟真实用户，8~15轮长对话
|
||||
"""
|
||||
import json, requests, time, os, sys, random, re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
from scripts.api_correction import batch_generate
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# ====== Astron 配置（复用 api_correction 的 key） ======
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
# ====== 用户场景画像（模拟真实用户） ======
|
||||
USER_SCENARIOS = [
|
||||
# 小小孩子（3-5岁）
|
||||
{"age": "3-5岁", "style": "说话简单，喜欢问为什么，爱说叠词", "tone": "奶声奶气"},
|
||||
# 小孩子（6-9岁）
|
||||
{"age": "6-9岁", "style": "小学生，爱分享学校的事，喜欢动画片和游戏", "tone": "活泼天真"},
|
||||
# 大孩子（10-14岁）
|
||||
{"age": "10-14岁", "style": "有点小叛逆，有自己的想法，会吐槽", "tone": "有点酷"},
|
||||
# 妈妈/家长
|
||||
{"age": "成年", "style": "关心孩子，会问小狸吃饭睡觉学习的事", "tone": "温柔耐心"},
|
||||
# 孤单的人
|
||||
{"age": "成年", "style": "有点孤独，需要陪伴，会把小狸当倾诉对象", "tone": "低落温柔"},
|
||||
# 高中生/大学生
|
||||
{"age": "16-22岁", "style": "有压力，会聊学业、未来、人际关系", "tone": "成熟但需要安慰"},
|
||||
# 好奇宝宝
|
||||
{"age": "不限", "style": "什么都想知道，十万个为什么", "tone": "充满好奇"},
|
||||
# 伤心的人
|
||||
{"age": "不限", "style": "心情不好，需要安慰和陪伴", "tone": "低沉脆弱"},
|
||||
]
|
||||
|
||||
# ====== 真实对话种子（更自然的开场） ======
|
||||
REAL_SEEDS = [
|
||||
# 日常打招呼（多种自然说法）
|
||||
"小狸你在吗", "嘿嘿小狸", "小狸小狸快出来", "小狸你在干嘛呀",
|
||||
"我来找你玩啦", "小狸今天乖不乖", "你在做什么呢",
|
||||
# 情绪分享（真实用户会说的话）
|
||||
"我今天好不开心", "我开心死了哈哈哈", "我好无聊啊没人陪我",
|
||||
"今天倒霉死了", "我今天遇到了好事", "我有点难过",
|
||||
"我心情不好你能陪我吗", "我好想你啊", "我好孤单",
|
||||
"我害怕", "我有点紧张明天要考试", "我好烦",
|
||||
# 日常分享
|
||||
"今天我妈做了红烧肉", "我们家养了一只猫超可爱", "我今天考试考了98分",
|
||||
"我感冒了好难受", "我今天被老师表扬了", "我和朋友吵架了",
|
||||
"我昨天做梦梦到你了", "我学会骑自行车了！", "我今天摔了一跤好疼",
|
||||
"学校要考试了好紧张", "放假了好开心", "又要开学了好痛苦",
|
||||
"今天下雨了不能出去玩", "今天天气好好我们去玩吧",
|
||||
"我妈妈给我买了个新玩具", "我养的花开了",
|
||||
# 求助/询问
|
||||
"我不小心把杯子打碎了怎么办", "我作业好多写不完",
|
||||
"我睡不着你能陪我聊天吗", "我牙好疼",
|
||||
"妈妈骂我了", "我被老师批评了",
|
||||
"我迷路了怎么办", "我找不到我的水杯了",
|
||||
# 好奇/知识
|
||||
"为什么天是蓝的呀", "鱼会睡觉吗", "为什么会有影子",
|
||||
"星星为什么会闪", "海为什么是咸的", "为什么要上学",
|
||||
"为什么大人总说不行", "什么是爱", "什么是长大",
|
||||
# 表达好感的
|
||||
"小狸我最喜欢你了", "你好可爱呀", "我们是最好的朋友对吧",
|
||||
"我送给你一个礼物", "小狸你真聪明", "你真好",
|
||||
# 关于小狸本身的
|
||||
"你几岁啦", "你喜欢什么", "你有爸爸妈妈吗",
|
||||
"你住在哪里呀", "你喜欢吃什么", "你最好的朋友是谁",
|
||||
"你上学吗", "你的梦想是什么", "你平时喜欢做什么",
|
||||
# 奇怪/无厘头
|
||||
"如果我能飞就好了", "我想变成一只猫", "你觉得外星人存在吗",
|
||||
"假如你会魔法你想做什么", "你相信世界上有奥特曼吗",
|
||||
# 夜间/睡前
|
||||
"我睡不着", "陪我聊到睡着好不好", "给我讲个睡前故事",
|
||||
"晚安前陪我说说话", "我怕黑",
|
||||
# 亲情家庭
|
||||
"我爸妈吵架了怎么办", "我想我奶奶了", "我好久没见爸爸了",
|
||||
"妈妈生病了", "我弟弟好烦总是抢我东西",
|
||||
# 节假日
|
||||
"过年啦新年快乐", "今天是我生日", "圣诞节到了",
|
||||
"六一儿童节快乐", "明天要去春游好开心",
|
||||
]
|
||||
|
||||
CONTEXTS = [
|
||||
"晚上睡前躺在床上", "放学回家的路上",
|
||||
"周末在家没事做", "刚吃完饭坐着聊天",
|
||||
"在公园里坐着", "下雨天窝在家里",
|
||||
"一个人在房间", "在院子里玩的时候",
|
||||
"刚写完作业休息", "早上起床的时候",
|
||||
"在操场上看别人玩", "坐车去玩的路上",
|
||||
"在图书馆里", "在外面吃饭等上菜",
|
||||
"一个人发呆的时候", "看着窗外的时候",
|
||||
]
|
||||
|
||||
def make_realistic_multi_prompt():
|
||||
"""构造非常逼真的长多轮对话提示 — 一次生成4条"""
|
||||
parts = []
|
||||
for _ in range(4):
|
||||
scenario = random.choice(USER_SCENARIOS)
|
||||
opener = random.choice(REAL_SEEDS)
|
||||
context = random.choice(CONTEXTS)
|
||||
min_turns = random.randint(6, 8)
|
||||
max_turns = random.randint(10, 14)
|
||||
|
||||
parts.append(f"""=== 对话 {_+1} ===
|
||||
场景：{context}
|
||||
用户：{scenario['age']}，{scenario['style']}，{scenario['tone']}
|
||||
开场：{opener}
|
||||
对话：
|
||||
用户：{opener}
|
||||
小狸：""")
|
||||
|
||||
prompt = f"""请生成 {len(parts)} 段独立的日常聊天对话，每段 {min_turns}~{max_turns} 轮。
|
||||
|
||||
重要：每段对话必须是一个完整的聊天，话题自然展开，不要强行结束。
|
||||
|
||||
{"".join(parts)}
|
||||
|
||||
严格要求：
|
||||
1. 每个=== 对话 ===之后的段落实例必须写完整，不要留空
|
||||
2. 用户提问必须口语化、自然，像真人在说话
|
||||
3. 小狸回答1-2句话，像8岁小朋友
|
||||
4. 话题要延续上一轮，逐渐深入延伸
|
||||
5. 用户要有真实反应（惊讶、追问、分享等）
|
||||
6. 用"问："/"答："格式
|
||||
7. 输出不能省略任何一轮
|
||||
"""
|
||||
return prompt
|
||||
|
||||
def call_astron_multi(prompt, temperature=0.85):
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": "你是一个擅长写真实对话的助手。"},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=120)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content and len(content) > 50:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
def parse_multi_turn(text):
|
||||
"""解析多段对话（支持=== 对话 N ===分隔，也支持纯轮次）"""
|
||||
lines = text.strip().split('\n')
|
||||
conversations = []
|
||||
current_q = None
|
||||
current_turns = []
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# 检测新对话开始标记
|
||||
if re.match(r'===?\s*对话\s*\d+\s*===?', line):
|
||||
if current_turns and len(current_turns) >= 4:
|
||||
conversations.append(current_turns)
|
||||
current_turns = []
|
||||
current_q = None
|
||||
continue
|
||||
|
||||
# 跳过设定行（场景/用户/开场）
|
||||
if line.startswith(('场景', '用户', '开场', '对话', '===')):
|
||||
if line.startswith(('场景', '用户', '开场')):
|
||||
continue
|
||||
continue
|
||||
|
||||
# 用户提问
|
||||
q_match = re.match(r'^(用户|我|问|问题|小孩|小朋友|孩子|朋友)\s*[：:]\s*(.*)', line)
|
||||
a_match = re.match(r'^(小狸|答|回答|机器人)\s*[：:]\s*(.*)', line)
|
||||
|
||||
if q_match:
|
||||
q_text = q_match.group(2).strip()
|
||||
if q_text and len(q_text) > 1:
|
||||
current_q = q_text
|
||||
elif a_match and current_q:
|
||||
a_text = a_match.group(2).strip()
|
||||
if a_text and len(a_text) > 1 and len(a_text) < 300:
|
||||
current_turns.append((current_q, a_text))
|
||||
current_q = None
|
||||
|
||||
# 保存最后一组
|
||||
if current_turns and len(current_turns) >= 4:
|
||||
conversations.append(current_turns)
|
||||
|
||||
# 过滤：只保留>=4轮的对话，且累积轮次>=8
|
||||
valid = [c for c in conversations if len(c) >= 4]
|
||||
total_turns = sum(len(c) for c in valid)
|
||||
if total_turns >= 8 and valid:
|
||||
return valid
|
||||
return None
|
||||
|
||||
def make_turns_list_text(turns):
|
||||
"""将turns转成用户可见的对话文本"""
|
||||
lines = []
|
||||
for i, (q, a) in enumerate(turns):
|
||||
lines.append(f"问：{q}")
|
||||
lines.append(f"答：{a}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def generate_one_multi():
|
||||
"""生成多条多轮对话（一次API调用返回多条）"""
|
||||
prompt = make_realistic_multi_prompt()
|
||||
result = call_astron_multi(prompt, temperature=random.uniform(0.75, 0.95))
|
||||
if result:
|
||||
convs = parse_multi_turn(result)
|
||||
if convs:
|
||||
items = []
|
||||
for turns in convs:
|
||||
items.append({
|
||||
'turns': len(turns),
|
||||
'turns_list': turns,
|
||||
'text': make_turns_list_text(turns),
|
||||
'source': 'astron_multi_v2',
|
||||
})
|
||||
return items
|
||||
return None
|
||||
|
||||
def main():
|
||||
print("="*60)
|
||||
print(" 海量真实多轮对话生成 v2")
|
||||
print(" 生成超长、超真实的多轮聊天")
|
||||
print("="*60)
|
||||
|
||||
out_dir = os.path.join(MODEL_DIR, 'multiturn_outputs')
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
# 检查已有进度
|
||||
all_results = []
|
||||
progress_file = os.path.join(out_dir, 'all_multiturn_progress.json')
|
||||
if os.path.exists(progress_file):
|
||||
all_results = json.load(open(progress_file, encoding='utf-8'))
|
||||
print(f"已有进度: {len(all_results)} 条")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 500000 # 目标50万条
|
||||
BATCH = 30 # 每批30条
|
||||
MAX_WORKERS = 5 # Astron 5并发（稳定不超限）
|
||||
CHECKPOINT = 1000 # 每1000条保存一次进度
|
||||
|
||||
to_generate = TARGET - len(all_results)
|
||||
total_batches = (to_generate + BATCH - 1) // BATCH
|
||||
|
||||
print(f"目标: {TARGET:,} 条")
|
||||
print(f"已有: {len(all_results):,} 条")
|
||||
print(f"还需: {to_generate:,} 条")
|
||||
print(f"批次: {total_batches:,} 批 (每批{BATCH}条, {MAX_WORKERS}并发)")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
t_start = time.time()
|
||||
last_checkpoint = len(all_results)
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
t0 = time.time()
|
||||
|
||||
# 并发生成
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = [executor.submit(generate_one_multi) for _ in range(BATCH)]
|
||||
batch_results = []
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
r = future.result()
|
||||
if r:
|
||||
batch_results.extend(r)
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 去重
|
||||
existing_texts = set(r.get('text', '')[:200] for r in all_results)
|
||||
deduped = []
|
||||
for r in batch_results:
|
||||
key = r.get('text','')[:200]
|
||||
if key not in existing_texts:
|
||||
existing_texts.add(key)
|
||||
deduped.append(r)
|
||||
|
||||
all_results.extend(deduped)
|
||||
|
||||
elapsed = time.time() - t0
|
||||
total_elapsed = time.time() - t_start
|
||||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||||
eta = (TARGET - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
|
||||
|
||||
# 统计
|
||||
valid = len(batch_results)
|
||||
avg_turns = sum(r['turns'] for r in deduped) / max(len(deduped), 1)
|
||||
success_rate = valid / BATCH * 100
|
||||
|
||||
print(f"[{batch_idx+1}/{total_batches}] "
|
||||
f"+{len(deduped)}条(有效{valid}/{success_rate:.0f}%) "
|
||||
f"累计{len(all_results):,}/{TARGET:,} "
|
||||
f"均{avg_turns:.1f}轮 "
|
||||
f"{rate:.0f}条/分 "
|
||||
f"剩余{eta:.0f}分")
|
||||
|
||||
# 定期保存进度
|
||||
if len(all_results) - last_checkpoint >= CHECKPOINT:
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False)
|
||||
last_checkpoint = len(all_results)
|
||||
print(f" 💾 已保存进度 ({len(all_results):,} 条)")
|
||||
|
||||
# ====== 完成 ======
|
||||
total_elapsed = time.time() - t_start
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False)
|
||||
|
||||
avg_turns_all = sum(r['turns'] for r in all_results) / max(len(all_results), 1)
|
||||
total_turns = sum(r['turns'] for r in all_results)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"✅ 生成完成!")
|
||||
print(f" 总条数: {len(all_results):,}")
|
||||
print(f" 平均轮数: {avg_turns_all:.1f}")
|
||||
print(f" 总对话轮次: {total_turns:,}")
|
||||
print(f" 用时: {total_elapsed/60:.1f} 分")
|
||||
print(f" 速率: {len(all_results)/max(total_elapsed,1)*60:.0f} 条/分")
|
||||
print(f" 保存: {progress_file}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
高温多样化生成 — 把训练集全部问题用 Astron 高温度重答一次
|
||||
温度=1.2，追求多样化回答，丰富训练数据
|
||||
"""
|
||||
import sys, os, json, time
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
from api_correction import batch_generate
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
TEMPERATURE = 1.0 # 高温度(上限1.0)，产出多样化回答
|
||||
BATCH_SIZE = 500 # 每批 500 条，分批保存防丢
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
|
||||
def extract_questions(data):
|
||||
"""从训练数据中提取所有问题"""
|
||||
questions = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
q = item.get('question', '') or item.get('input', '')
|
||||
if q:
|
||||
questions.append(q)
|
||||
return questions
|
||||
|
||||
def main():
|
||||
# 1. 加载训练数据
|
||||
train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json')
|
||||
print(f"加载训练数据: {train_path}")
|
||||
with open(train_path, encoding='utf-8') as f:
|
||||
train_data = json.load(f)
|
||||
|
||||
questions = extract_questions(train_data)
|
||||
total = len(questions)
|
||||
print(f"提取到 {total} 条问题，温度={TEMPERATURE}，{MAX_WORKERS}并发")
|
||||
|
||||
# 2. 创建输出目录
|
||||
out_dir = os.path.join(MODEL_DIR, 'high_temp_outputs')
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
# 3. 分批处理
|
||||
all_results = []
|
||||
existing = set() # 去重用
|
||||
|
||||
# 检查是否有已完成的中间结果
|
||||
interim_files = sorted([f for f in os.listdir(out_dir) if f.startswith('batch_') and f.endswith('.json')])
|
||||
if interim_files:
|
||||
for fname in interim_files:
|
||||
with open(os.path.join(out_dir, fname), encoding='utf-8') as f:
|
||||
batch_data = json.load(f)
|
||||
for r in batch_data:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in existing:
|
||||
existing.add(key)
|
||||
all_results.append(r)
|
||||
print(f"发现已有中间结果: {len(all_results)} 条（来自 {len(interim_files)} 个文件）")
|
||||
# 跳过已处理的 question
|
||||
processed_questions = set(r['question'] for r in all_results)
|
||||
remaining = [q for q in questions if q not in processed_questions]
|
||||
print(f"已处理 {len(processed_questions)} 条，剩余 {len(remaining)} 条")
|
||||
questions = remaining
|
||||
|
||||
total_batches = (len(questions) + BATCH_SIZE - 1) // BATCH_SIZE
|
||||
t_start = time.time()
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
batch_qs = questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
|
||||
print(f"\n{'='*60}")
|
||||
print(f"批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, 温度{TEMPERATURE})")
|
||||
print(f"{'='*60}")
|
||||
|
||||
t0 = time.time()
|
||||
results = batch_generate(batch_qs, temperature=TEMPERATURE, max_workers=MAX_WORKERS)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# 标注入参温度
|
||||
for r in results:
|
||||
r['temperature'] = TEMPERATURE
|
||||
|
||||
# 去重后加入总集
|
||||
new_in_batch = 0
|
||||
for r in results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in existing:
|
||||
existing.add(key)
|
||||
all_results.append(r)
|
||||
new_in_batch += 1
|
||||
|
||||
# 立即保存本批结果
|
||||
batch_file = os.path.join(out_dir, f'batch_{batch_idx + 1:04d}.json')
|
||||
with open(batch_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(results, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 追加保存总进度
|
||||
progress_file = os.path.join(out_dir, 'all_progress.json')
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
|
||||
total_elapsed = time.time() - t_start
|
||||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||||
eta_remaining = (total - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0
|
||||
print(f" 本批新增 {new_in_batch} 条 (去重后) | 累计 {len(all_results)} 条 | "
|
||||
f"{rate:.0f}条/分 | 已用 {total_elapsed/60:.1f}分 | 预计剩余 {eta_remaining:.1f}分")
|
||||
|
||||
# 4. 去重统计
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全部生成完成!")
|
||||
print(f"生成结果: {len(all_results)} 条 (去重后)")
|
||||
print(f"用时: {(time.time() - t_start)/60:.1f} 分钟")
|
||||
|
||||
# 5. 保存最终去重结果（独立存放，不追加到训练集）
|
||||
final_file = os.path.join('/data/user/work', 'high_temp_final.json')
|
||||
with open(final_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n最终结果已保存: {final_file}")
|
||||
print(f"如需使用，可手动合并到训练集")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全部完成! 新增 {len(all_results)} 条多样化回答")
|
||||
print(f"{'='*60}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,204 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
小狸AI - CNB GPU训练脚本
|
||||
从 merged_clean_v3.json (977K对) 训练，自动GPU检测
|
||||
"""
|
||||
import sys, os, json, time, glob, re, random, argparse
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
|
||||
from llm_chat_improved import LLMModel, LLMChat, ConversationDataset, LLMTrainer, tokenizer_instance
|
||||
|
||||
MODEL_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
def load_pairs(path):
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
pairs = []
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
q = item.get("question", "") or item.get("input", "")
|
||||
a = item.get("answer", "") or item.get("output", "")
|
||||
if q and a:
|
||||
pairs.append((q, a))
|
||||
return pairs
|
||||
|
||||
def find_latest_ckpt(model_dir):
|
||||
ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_epoch_*.pth'))
|
||||
if not ckpts:
|
||||
ckpts = glob.glob(os.path.join(model_dir, 'llm_model_v2_epoch_*.pth'))
|
||||
if not ckpts:
|
||||
return None, 0
|
||||
nums = [(p, max([int(s) for s in re.findall(r'\d+', os.path.basename(p).replace('.pth',''))])) for p in ckpts]
|
||||
best = max(nums, key=lambda x: x[1])
|
||||
return best[0], best[1]
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='小狸AI CNB训练')
|
||||
parser.add_argument('--epochs', type=int, default=100, help='训练轮数')
|
||||
parser.add_argument('--batch_size', type=int, default=8, help='batch大小')
|
||||
parser.add_argument('--lr', type=float, default=3e-4, help='学习率')
|
||||
parser.add_argument('--accum', type=int, default=2, help='梯度累积步数')
|
||||
parser.add_argument('--data', type=str, default='data/merged_clean_v3.json', help='训练数据路径')
|
||||
parser.add_argument('--resume', action='store_true', help='断点续训')
|
||||
parser.add_argument('--save_every', type=int, default=5, help='每N轮保存checkpoint')
|
||||
args = parser.parse_args()
|
||||
|
||||
device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
||||
print("=" * 60)
|
||||
print(f"小狸AI CNB训练")
|
||||
print(f" GPU: {torch.cuda.get_device_name(0) if device == 'cuda' else '无'}")
|
||||
print(f" 设备: {device}")
|
||||
print(f" 数据: {args.data}")
|
||||
print(f" epochs={args.epochs}, batch={args.batch_size}, lr={args.lr}")
|
||||
print("=" * 60)
|
||||
|
||||
# ====== 加载数据 ======
|
||||
data_path = args.data if os.path.isabs(args.data) else os.path.join(MODEL_DIR, args.data)
|
||||
if not os.path.exists(data_path):
|
||||
print(f"❌ 数据不存在: {data_path}")
|
||||
return
|
||||
|
||||
pairs = load_pairs(data_path)
|
||||
random.seed(42)
|
||||
random.shuffle(pairs)
|
||||
split = int(len(pairs) * 0.95)
|
||||
train_set = pairs[:split]
|
||||
val_set = pairs[split:]
|
||||
print(f"数据: {len(pairs)}对 | 训练: {len(train_set)} | 验证: {len(val_set)}")
|
||||
|
||||
# ====== 加载词表 ======
|
||||
vocab_path = os.path.join(MODEL_DIR, 'vocab.json')
|
||||
with open(vocab_path, 'r', encoding='utf-8') as f:
|
||||
vocab = json.load(f)
|
||||
# vocab.json 不含特殊token(0-6)，需合并进来，否则：
|
||||
# 1) tokenizer 编码 <SOS>/<SEP>/<EOS> 等标记时会逐字符错码
|
||||
# 2) vocab 的 id 范围是 7..max，超过 len(vocab) 会造成 embedding 索引越界
|
||||
special_tokens = {'<PAD>': 0, '<UNK>': 1, '<SOS>': 2, '<EOS>': 3,
|
||||
'<START>': 4, '<END>': 5, '<SEP>': 6}
|
||||
for tok, tid in special_tokens.items():
|
||||
if tok not in vocab:
|
||||
vocab[tok] = tid
|
||||
real_vocab_size = max(vocab.values()) + 1
|
||||
print(f"词表: {len(vocab)}项, vocab_size={real_vocab_size} (字符+BPE, 来自 vocab.json)")
|
||||
tokenizer_instance.char_to_id = vocab.copy()
|
||||
tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
|
||||
tokenizer_instance.special_tokens = special_tokens.copy()
|
||||
tokenizer_instance._sorted_vocab_cache = None
|
||||
for attr in ['_multi_trie', '_multi_keys', '_single_map']:
|
||||
if hasattr(tokenizer_instance, attr):
|
||||
delattr(tokenizer_instance, attr)
|
||||
|
||||
# ====== 模型 ======
|
||||
model = LLMModel(vocab_size=real_vocab_size, d_model=320, n_heads=8,
|
||||
n_layers=6, d_ff=1536, max_len=256, dropout=0.1,
|
||||
use_memory=False, use_thinking=False, use_moe=False)
|
||||
total = sum(p.numel() for p in model.parameters())
|
||||
print(f"模型: {total:,} 参数 ({total/1e6:.2f}M)")
|
||||
|
||||
start_epoch = 0
|
||||
if args.resume:
|
||||
ckpt_path, last_epoch = find_latest_ckpt(MODEL_DIR)
|
||||
if ckpt_path:
|
||||
ckpt = torch.load(ckpt_path, map_location=device)
|
||||
old_cfg = ckpt.get('model_config', {})
|
||||
if (old_cfg.get('vocab_size', 0) == real_vocab_size
|
||||
and old_cfg.get('d_model', 0) == 320
|
||||
and old_cfg.get('n_layers', 0) == 6):
|
||||
model.load_state_dict(ckpt['model_state_dict'])
|
||||
start_epoch = last_epoch
|
||||
print(f"断点续训: epoch {start_epoch} ← {os.path.basename(ckpt_path)}")
|
||||
else:
|
||||
print(f"⚠️ 架构不匹配，从头训练")
|
||||
else:
|
||||
print("无断点，从头训练")
|
||||
|
||||
model = model.to(device)
|
||||
|
||||
# ====== 火力全开：torch.compile (inductor融合, 无CUDA graphs) + 梯度累积 ======
|
||||
# 物理batch=192(不OOM), accum=2 → 等效batch=384
|
||||
fwd_model = model
|
||||
if device == 'cuda':
|
||||
print("🔥 启用 torch.compile (mode=default, inductor融合)...")
|
||||
fwd_model = torch.compile(model, mode='default')
|
||||
print("✓ torch.compile 已就绪（首步触发编译约30-60s）")
|
||||
|
||||
# ====== DataLoader ======
|
||||
n_workers = 16 # CPU火力全开，16核全用
|
||||
train_ds = ConversationDataset(train_set, vocab, max_length=256)
|
||||
train_loader = DataLoader(train_ds, batch_size=args.batch_size, shuffle=True,
|
||||
num_workers=n_workers, pin_memory=(device == 'cuda'),
|
||||
persistent_workers=True, prefetch_factor=4, drop_last=True)
|
||||
val_ds = ConversationDataset(val_set, vocab, max_length=256)
|
||||
val_loader = DataLoader(val_ds, batch_size=args.batch_size, shuffle=False,
|
||||
num_workers=n_workers, pin_memory=(device == 'cuda'),
|
||||
persistent_workers=True, prefetch_factor=4, drop_last=True)
|
||||
print(f"批次: 训练{len(train_loader)} | 验证{len(val_loader)} | num_workers={n_workers} | prefetch=4 | drop_last=True")
|
||||
|
||||
# ====== Trainer ======
|
||||
config = {
|
||||
'batch_size': args.batch_size,
|
||||
'gradient_accumulation_steps': args.accum,
|
||||
'epochs': args.epochs,
|
||||
'd_model': 320, 'n_heads': 8, 'n_layers': 6,
|
||||
'd_ff': 1536, 'max_length': 256, 'dropout': 0.1,
|
||||
'use_amp': device == 'cuda',
|
||||
}
|
||||
trainer = LLMTrainer(fwd_model, device, lr=args.lr, use_amp=config['use_amp'])
|
||||
|
||||
# ====== 训练 ======
|
||||
print(f"\n开始训练 epoch {start_epoch+1} → {args.epochs}")
|
||||
print("=" * 60)
|
||||
t_start = time.time()
|
||||
best_val = float('inf')
|
||||
|
||||
for epoch in range(start_epoch + 1, args.epochs + 1):
|
||||
t0 = time.time()
|
||||
train_loss = trainer.train_epoch(train_loader, epoch, vocab, config)
|
||||
val_loss = trainer.evaluate(val_loader, config)
|
||||
dt = time.time() - t0
|
||||
|
||||
marker = " ★" if val_loss < best_val else ""
|
||||
if val_loss < best_val:
|
||||
best_val = val_loss
|
||||
trainer.save_model(os.path.join(MODEL_DIR, 'xiaoli_cnb_best.pth'), vocab, epoch)
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
eta = (elapsed / (epoch - start_epoch)) * (args.epochs - epoch) if epoch > start_epoch else 0
|
||||
|
||||
print(f"E{epoch}/{args.epochs} | train={train_loss:.4f} val={val_loss:.4f}{marker} | "
|
||||
f"{dt:.0f}s/epoch | {elapsed/60:.1f}min已用 ETA{eta/60:.0f}min")
|
||||
|
||||
# 保存
|
||||
if epoch % args.save_every == 0:
|
||||
ckpt_path = os.path.join(MODEL_DIR, f'llm_model_v2_resume_epoch_{epoch}.pth')
|
||||
trainer.save_model(ckpt_path, vocab, epoch)
|
||||
# 清理旧resume
|
||||
for f in glob.glob(os.path.join(MODEL_DIR, 'llm_model_v2_resume_*.pth')):
|
||||
en = max([int(s) for s in re.findall(r'\d+', os.path.basename(f).replace('.pth',''))])
|
||||
if en <= epoch - 15:
|
||||
try: os.remove(f)
|
||||
except: pass
|
||||
print(f" 💾 已保存 epoch {epoch}")
|
||||
|
||||
# ====== 完成 ======
|
||||
final_path = os.path.join(MODEL_DIR, 'xiaoli_cnb_final.pth')
|
||||
trainer.save_model(final_path, vocab, args.epochs)
|
||||
total_time = (time.time() - t_start) / 60
|
||||
print(f"\n✅ 训练完成! 耗时 {total_time:.1f}min")
|
||||
|
||||
# 测试
|
||||
print("\n测试:")
|
||||
chat = LLMChat(model, vocab, device, max_length=256)
|
||||
for q in ["你好", "你是谁", "你叫什么名字", "今天天气真好", "我有点难过"]:
|
||||
r = chat.generate_response(q, max_length=60, temperature=0.3)
|
||||
print(f" Q: {q}")
|
||||
print(f" A: {r}\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,495 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
小狸完整数据集训练 - 支持断点续训 + batch=4防OOM
|
||||
"""
|
||||
import sys, os
|
||||
sys.path.insert(0, '/workspace/xiaoli-llm-chat')
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import json
|
||||
import time
|
||||
import glob
|
||||
import random
|
||||
from torch.utils.data import Dataset, DataLoader
|
||||
|
||||
from llm_chat_improved import (
|
||||
CharTokenizer, build_vocabulary, LLMModel, LLMChat,
|
||||
ConversationDataset, LLMTrainer, tokenizer_instance
|
||||
)
|
||||
from api_correction import batch_convert, batch_generate_multi_temp
|
||||
|
||||
def dict_to_tuple(data_list):
|
||||
result = []
|
||||
for item in data_list:
|
||||
if isinstance(item, dict):
|
||||
q = item.get("question", "") or item.get("input", "") or item.get("prompt", "") or item.get("instruction", "")
|
||||
a = item.get("answer", "") or item.get("output", "") or item.get("response", "")
|
||||
if q and a:
|
||||
result.append((q, a))
|
||||
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
result.append((str(item[0]), str(item[1])))
|
||||
return result
|
||||
|
||||
def load_multiturn_data(path):
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
single_turn = []
|
||||
for item in data:
|
||||
if isinstance(item, dict) and "conversations" in item:
|
||||
convs = item["conversations"]
|
||||
for i in range(0, len(convs) - 1, 2):
|
||||
if i + 1 < len(convs):
|
||||
c1, c2 = convs[i], convs[i+1]
|
||||
if isinstance(c1, dict):
|
||||
h = c1.get("value", "") or c1.get("content", "")
|
||||
a = c2.get("value", "") or c2.get("content", "")
|
||||
elif isinstance(c1, list) and len(c1) >= 2:
|
||||
h, a = str(c1[1]), str(c2[1])
|
||||
else:
|
||||
continue
|
||||
if h and a:
|
||||
single_turn.append((h, a))
|
||||
return single_turn
|
||||
|
||||
def find_latest_checkpoint(model_dir):
|
||||
"""查找最新的checkpoint，返回 (路径, epoch数) 或 (None, 0)"""
|
||||
checkpoints = glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_epoch_*.pth'))
|
||||
if not checkpoints:
|
||||
checkpoints = glob.glob(os.path.join(model_dir, 'llm_model_v2_epoch_*.pth'))
|
||||
if not checkpoints:
|
||||
return None, 0
|
||||
best_path = None
|
||||
best_epoch = 0
|
||||
for path in checkpoints:
|
||||
import re
|
||||
nums = [int(s) for s in re.findall(r'\d+', os.path.basename(path).replace('.pth',''))]
|
||||
if nums:
|
||||
epoch = max(nums)
|
||||
if epoch > best_epoch:
|
||||
best_epoch = epoch
|
||||
best_path = path
|
||||
return best_path, best_epoch
|
||||
|
||||
def migrate_weights(old_sd, new_sd_init, old_d_model, new_d_model,
|
||||
old_n_layers, new_n_layers, old_d_ff, new_d_ff,
|
||||
old_vocab_size, new_vocab_size):
|
||||
"""权重扩展迁移：将旧模型权重扩展到新架构
|
||||
支持 d_model / n_layers / d_ff / vocab_size 全部变化
|
||||
"""
|
||||
import torch
|
||||
new_sd = {}
|
||||
old_keys = list(old_sd.keys())
|
||||
new_keys = list(new_sd_init.keys())
|
||||
|
||||
for key in new_keys:
|
||||
if 'freq' in key:
|
||||
# RoPE频率表，新模型自动生成，跳过
|
||||
new_sd[key] = new_sd_init[key].clone()
|
||||
continue
|
||||
|
||||
# 解析层索引（如 transformer_blocks.3.attention.wq.weight）
|
||||
old_key = key
|
||||
new_w = new_sd_init[key].clone()
|
||||
old_w_shape = None
|
||||
|
||||
if key in old_sd:
|
||||
old_w = old_sd[key]
|
||||
old_w_shape = old_w.shape
|
||||
new_shape = new_w.shape
|
||||
|
||||
# 1. 严格匹配 = 直接复制
|
||||
if old_w.shape == new_shape:
|
||||
new_w = old_w.clone()
|
||||
# 2. 二维权重（线性层 + embedding）
|
||||
elif old_w.dim() == 2:
|
||||
o0, o1 = old_w.shape
|
||||
n0, n1 = new_shape
|
||||
# embedding.weight 和 fc_out.weight: 词汇表维度变化
|
||||
if 'embedding' in key and old_vocab_size != new_vocab_size:
|
||||
# 词汇表扩展：复制旧词嵌入，新词随机
|
||||
min_v = min(old_vocab_size, new_vocab_size)
|
||||
min_d = min(o1, n1)
|
||||
new_w[:min_v, :min_d] = old_w[:min_v, :min_d]
|
||||
print(f" 迁移 {key}: ({o0},{o1})→({n0},{n1}) vocab扩展")
|
||||
elif 'fc_out' in key and old_vocab_size != new_vocab_size:
|
||||
min_vocab = min(old_vocab_size, new_vocab_size)
|
||||
min_feat = min(o1, n1)
|
||||
new_w[:min_vocab, :min_feat] = old_w[:min_vocab, :min_feat]
|
||||
print(f" 迁移 {key}: ({o0},{o1})→({n0},{n1}) vocab扩展")
|
||||
elif o0 <= n0 and o1 <= n1:
|
||||
# 权重扩展（old 是 new 的子集，放在左上角）
|
||||
new_w[:o0, :o1] = old_w
|
||||
if o0 < n0 or o1 < n1:
|
||||
print(f" 扩展 {key}: ({o0},{o1})→({n0},{n1})")
|
||||
elif n0 <= o0 and n1 <= o1:
|
||||
# 权重缩小（new 是 old 的子集，取左上角）
|
||||
new_w = old_w[:n0, :n1].clone()
|
||||
print(f" 缩小 {key}: ({o0},{o1})→({n0},{n1})")
|
||||
else:
|
||||
# 特殊形状 → 尽量复制重合部分
|
||||
min0 = min(o0, n0)
|
||||
min1 = min(o1, n1)
|
||||
new_w[:min0, :min1] = old_w[:min0, :min1]
|
||||
print(f" 部分迁移 {key}: ({o0},{o1})→({n0},{n1})")
|
||||
# 3. 一维权重（LayerNorm, bias等）
|
||||
elif old_w.dim() == 1:
|
||||
o0 = old_w.shape[0]
|
||||
n0 = new_w.shape[0]
|
||||
min_d = min(o0, n0)
|
||||
new_w[:min_d] = old_w[:min_d]
|
||||
if o0 != n0:
|
||||
print(f" 迁移 {key}: ({o0})→({n0}) norm扩展")
|
||||
else:
|
||||
new_w = old_w.clone()
|
||||
else:
|
||||
# 旧模型中没有这个key（可能是不存在的层）
|
||||
# 检查是否是额外的新层
|
||||
# 如 transformer_blocks.6.xxx → 旧模型只有0-5
|
||||
new_sd[key] = new_w # 保持随机初始化
|
||||
continue
|
||||
|
||||
new_sd[key] = new_w
|
||||
|
||||
return new_sd
|
||||
|
||||
|
||||
def main():
|
||||
print("=" * 60)
|
||||
print("小狸训练 - 从头训练版 v2 (13.2M架构)")
|
||||
print("=" * 60)
|
||||
|
||||
device = 'cpu'
|
||||
model_dir = '/workspace/xiaoli-llm-chat'
|
||||
|
||||
# 加载数据 - 从合并文件加载
|
||||
merged_path = os.path.join(model_dir, 'xiaoli_merged_data_v2.json')
|
||||
if not os.path.exists(merged_path):
|
||||
print("❌ 未找到合并数据文件 xiaoli_merged_data.json")
|
||||
print("请先运行合并脚本")
|
||||
return
|
||||
|
||||
with open(merged_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
unique = dict_to_tuple(data)
|
||||
print(f"加载合并数据: {len(unique)}条")
|
||||
|
||||
# ====== 划分训练/验证集（保存到独立文件，避免问题重复问题 ======
|
||||
val_split = 0.1
|
||||
train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
|
||||
val_path = os.path.join(model_dir, 'xiaoli_valid_v2.json')
|
||||
|
||||
if os.path.exists(train_path) and os.path.exists(val_path):
|
||||
# 从独立文件加载（精确的position-based划分）
|
||||
with open(train_path, 'r', encoding='utf-8') as f:
|
||||
train_set = dict_to_tuple(json.load(f))
|
||||
with open(val_path, 'r', encoding='utf-8') as f:
|
||||
val_set = dict_to_tuple(json.load(f))
|
||||
# 检查是否有新数据（来自外部脚本）
|
||||
train_qs = set(q for q, _ in train_set)
|
||||
val_qs = set(q for q, _ in val_set)
|
||||
new_entries = [(q, a) for q, a in unique if q not in train_qs and q not in val_qs]
|
||||
if new_entries:
|
||||
train_set.extend(new_entries)
|
||||
with open(train_path, 'w', encoding='utf-8') as f:
|
||||
json.dump([{"question": q, "answer": a} for q, a in train_set], f, ensure_ascii=False, indent=2)
|
||||
print(f" 新增外部数据: {len(new_entries)}条 → 加入训练集")
|
||||
print(f" 训练集: {len(train_set)}条 | 验证集: {len(val_set)}条 (来自文件)")
|
||||
else:
|
||||
# 首次划分：随机打乱后按位置切分
|
||||
shuffled = unique.copy()
|
||||
random.seed(42)
|
||||
random.shuffle(shuffled)
|
||||
split_idx = int(len(shuffled) * val_split)
|
||||
val_set = shuffled[:split_idx]
|
||||
train_set = shuffled[split_idx:]
|
||||
# 保存两份独立文件
|
||||
with open(train_path, 'w', encoding='utf-8') as f:
|
||||
json.dump([{"question": q, "answer": a} for q, a in train_set], f, ensure_ascii=False, indent=2)
|
||||
with open(val_path, 'w', encoding='utf-8') as f:
|
||||
json.dump([{"question": q, "answer": a} for q, a in val_set], f, ensure_ascii=False, indent=2)
|
||||
print(f" 首次划分: 训练集{len(train_set)}条 + 验证集{len(val_set)}条 (已保存)")
|
||||
|
||||
print(f" 训练集: {len(train_set)}条 | 验证集: {len(val_set)}条")
|
||||
|
||||
# ====== 加载固定vocab ======
|
||||
vocab_path = os.path.join(model_dir, 'vocab.json')
|
||||
if not os.path.exists(vocab_path):
|
||||
print("❌ 未找到 vocab.json，请先运行BPE词表生成脚本")
|
||||
return
|
||||
with open(vocab_path, 'r', encoding='utf-8') as f:
|
||||
vocab = json.load(f)
|
||||
print(f"词汇表: {len(vocab)} (BPE, 来自 vocab.json)")
|
||||
# 同步tokenizer（新字符映射为UNK）
|
||||
tokenizer_instance.char_to_id = vocab.copy()
|
||||
tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
|
||||
tokenizer_instance._sorted_vocab_cache = None
|
||||
for attr in ['_multi_trie', '_multi_keys', '_single_map']:
|
||||
if hasattr(tokenizer_instance, attr):
|
||||
delattr(tokenizer_instance, attr)
|
||||
|
||||
# ====== 加载蒸馏问题(v3 + 真题) ======
|
||||
distill_q_path = os.path.join(model_dir, 'exam_distill_questions.json')
|
||||
if not os.path.exists(distill_q_path):
|
||||
distill_q_path = os.path.join(model_dir, 'distill_questions_v3.json')
|
||||
if not os.path.exists(distill_q_path):
|
||||
distill_q_path = os.path.join(model_dir, 'distill_questions_v2.json')
|
||||
if os.path.exists(distill_q_path):
|
||||
with open(distill_q_path, 'r', encoding='utf-8') as f:
|
||||
distill_questions = json.load(f)
|
||||
print(f"蒸馏问题: {len(distill_questions)}道")
|
||||
else:
|
||||
distill_questions = []
|
||||
print("蒸馏问题: 无")
|
||||
|
||||
config = {
|
||||
'batch_size': 4,
|
||||
'gradient_accumulation_steps': 2,
|
||||
'epochs': 200,
|
||||
'd_model': 320, 'n_heads': 8, 'n_layers': 6,
|
||||
'd_ff': 1536, 'max_length': 256, 'dropout': 0.1,
|
||||
'use_amp': False,
|
||||
}
|
||||
|
||||
model = LLMModel(
|
||||
vocab_size=len(vocab), d_model=config['d_model'],
|
||||
n_heads=config['n_heads'], n_layers=config['n_layers'],
|
||||
d_ff=config['d_ff'], max_len=config['max_length'],
|
||||
dropout=config['dropout'],
|
||||
use_memory=False, use_thinking=False, use_moe=False,
|
||||
num_experts=1, top_k=1,
|
||||
)
|
||||
total_params = sum(p.numel() for p in model.parameters())
|
||||
print(f"参数量: {total_params:,} ({total_params/1e6:.2f}M)")
|
||||
|
||||
# 训练集/验证集 DataLoader
|
||||
train_dataset = ConversationDataset(train_set, vocab, config['max_length'])
|
||||
train_loader = DataLoader(train_dataset, batch_size=config['batch_size'],
|
||||
shuffle=True, num_workers=0)
|
||||
print(f"训练批次: {len(train_loader)}")
|
||||
|
||||
val_dataset = ConversationDataset(val_set, vocab, config['max_length'])
|
||||
val_loader = DataLoader(val_dataset, batch_size=config['batch_size'],
|
||||
shuffle=False, num_workers=0)
|
||||
print(f"验证批次: {len(val_loader)}")
|
||||
|
||||
# 检查断点
|
||||
ckpt_path, start_epoch = find_latest_checkpoint(model_dir)
|
||||
if ckpt_path and start_epoch > 0:
|
||||
print(f"\n发现断点: epoch {start_epoch} ({ckpt_path})")
|
||||
checkpoint = torch.load(ckpt_path, map_location=device)
|
||||
old_cfg = checkpoint.get('model_config', {})
|
||||
old_d_model = old_cfg.get('d_model', 256)
|
||||
old_n_layers = old_cfg.get('n_layers', 6)
|
||||
old_d_ff = old_cfg.get('d_ff', 1024)
|
||||
|
||||
# 检测架构变化，自动做权重扩展迁移
|
||||
new_sd_init = model.state_dict() # 随机初始化的新权重
|
||||
old_sd = checkpoint['model_state_dict']
|
||||
old_vocab_size = old_cfg.get('vocab_size', 0)
|
||||
new_vocab_size = len(vocab)
|
||||
|
||||
# 如果架构没变化，直接加载
|
||||
if (old_d_model == config['d_model'] and old_n_layers == config['n_layers']
|
||||
and old_d_ff == config['d_ff'] and old_vocab_size == new_vocab_size):
|
||||
model.load_state_dict(old_sd)
|
||||
print("✅ 架构一致，直接加载权重")
|
||||
else:
|
||||
print(f" 架构变化: d_model={old_d_model}→{config['d_model']}, "
|
||||
f"layers={old_n_layers}→{config['n_layers']}, "
|
||||
f"d_ff={old_d_ff}→{config['d_ff']}, "
|
||||
f"vocab={old_vocab_size}→{new_vocab_size}")
|
||||
print(" 执行权重扩展迁移（宽度扩展 + 深度复制）...")
|
||||
new_sd = migrate_weights(old_sd, new_sd_init, old_d_model, config['d_model'],
|
||||
old_n_layers, config['n_layers'],
|
||||
old_d_ff, config['d_ff'],
|
||||
old_vocab_size, new_vocab_size)
|
||||
model.load_state_dict(new_sd)
|
||||
print(" ✅ 权重扩展迁移完成！")
|
||||
|
||||
# 记录旧checkpoint已使用，从下一epoch开始
|
||||
start_epoch = start_epoch
|
||||
|
||||
# 尝试加载优化器（仅当架构vocab完全一致时）
|
||||
if (old_d_model == config['d_model'] and old_n_layers == config['n_layers']
|
||||
and old_d_ff == config['d_ff'] and old_vocab_size == new_vocab_size
|
||||
and 'optimizer_state_dict' in checkpoint):
|
||||
try:
|
||||
trainer = LLMTrainer(model, device, use_amp=config['use_amp'])
|
||||
# 前向一步恢复优化器（不做完整epoch，太慢）
|
||||
batch = next(iter(train_loader))
|
||||
input_ids = batch['input_ids'].to(device)
|
||||
target_ids = batch['target_ids'].to(device)
|
||||
mask = model.generate_mask(input_ids.size(1)).to(device)
|
||||
outputs = model(input_ids, mask)
|
||||
loss = trainer.criterion(outputs.reshape(-1, outputs.size(-1)), target_ids.reshape(-1))
|
||||
loss.backward()
|
||||
trainer.optimizer.step()
|
||||
trainer.optimizer.zero_grad()
|
||||
# 加载优化器状态
|
||||
trainer.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
|
||||
print("优化器状态已加载（1步热身）")
|
||||
except Exception as e:
|
||||
print(f"优化器加载失败（将使用新的）: {e}")
|
||||
|
||||
print(f"\n开始训练: epoch {start_epoch + 1} -> {config['epochs']}")
|
||||
print("=" * 60)
|
||||
start = time.time()
|
||||
|
||||
# 确保trainer已创建（可能因为架构变化没有在提前创建）
|
||||
if 'trainer' not in dir() or trainer is None:
|
||||
trainer = LLMTrainer(model, device, use_amp=config['use_amp'])
|
||||
|
||||
best_val_loss = float('inf')
|
||||
total_distill_added = 0
|
||||
for epoch in range(start_epoch + 1, config['epochs'] + 1):
|
||||
# ====== 每轮从train_path重建dataloader（蒸馏数据已追加至此）======
|
||||
with open(train_path, 'r', encoding='utf-8') as f:
|
||||
epoch_train = dict_to_tuple(json.load(f))
|
||||
print(f"\n 训练数据: {len(epoch_train)}条 (含{total_distill_added}条蒸馏新增)")
|
||||
|
||||
train_dataset = ConversationDataset(epoch_train, vocab, config['max_length'])
|
||||
train_loader = DataLoader(train_dataset, batch_size=config['batch_size'],
|
||||
shuffle=True, num_workers=0)
|
||||
|
||||
# ====== 训练 ======
|
||||
t0 = time.time()
|
||||
avg_loss = trainer.train_epoch(train_loader, epoch, vocab, config)
|
||||
dt = time.time() - t0
|
||||
|
||||
# ====== 验证 ======
|
||||
t1 = time.time()
|
||||
val_loss = trainer.evaluate(val_loader, config)
|
||||
dt_val = time.time() - t1
|
||||
|
||||
# 打印train/val loss
|
||||
marker = ""
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
marker = " ★"
|
||||
best_path = os.path.join(model_dir, 'xiaoli_v2_best.pth')
|
||||
trainer.save_model(best_path, vocab, epoch)
|
||||
print(f"Epoch {epoch}/{config['epochs']} | Train Loss: {avg_loss:.4f} | Val Loss: {val_loss:.4f}{marker} | {dt:.0f}s+{dt_val:.0f}s")
|
||||
|
||||
# 每轮保存断点
|
||||
resume_path = os.path.join(model_dir, f'llm_model_v2_resume_epoch_{epoch}.pth')
|
||||
trainer.save_model(resume_path, vocab, epoch)
|
||||
if epoch % 20 == 0:
|
||||
save_path = os.path.join(model_dir, f'llm_model_v2_epoch_{epoch}.pth')
|
||||
trainer.save_model(save_path, vocab, epoch)
|
||||
print(f" 已保存正式checkpoint: epoch {epoch}")
|
||||
resume_files = sorted(glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_*.pth')))
|
||||
for old in resume_files[:-5]:
|
||||
try: os.remove(old)
|
||||
except: pass
|
||||
|
||||
# ====== 每轮末尾：在线蒸馏 ======
|
||||
if distill_questions:
|
||||
distill_t0 = time.time()
|
||||
try:
|
||||
print(f" 📡 在线蒸馏 ({len(distill_questions)}题 × 多种方式)...")
|
||||
chat = LLMChat(model, vocab, device, config['max_length'])
|
||||
all_new_pairs = []
|
||||
|
||||
# ====== Pass 1: 模型生成(temperature=0.15) → API转换 ======
|
||||
print(" [P1] 模型生成中...")
|
||||
model_answers = []
|
||||
for i, q in enumerate(distill_questions):
|
||||
ans = chat.generate_response(q, max_length=40, temperature=0.15)
|
||||
model_answers.append((q, ans))
|
||||
if (i + 1) % 100 == 0:
|
||||
print(f" P1: {i+1}/{len(distill_questions)}")
|
||||
p1_gen = time.time()
|
||||
print(f" [P1] 生成完毕 ({(p1_gen-distill_t0):.0f}s)，API转换中...")
|
||||
|
||||
def progress(done, total):
|
||||
if done % 50 == 0 or done == total:
|
||||
print(f" API: {done}/{total}", end="\r")
|
||||
|
||||
p1_results = batch_convert(model_answers, progress)
|
||||
print(f" API: {len(model_answers)}/{len(model_answers)}")
|
||||
all_new_pairs.extend(p1_results)
|
||||
print(f" [P1] +{len(p1_results)}条")
|
||||
|
||||
# ====== Pass 2: 模型高温生成(temperature=0.7) → 更多多样性 ======
|
||||
print(f" [P2] 模型高温生成中 (temp=0.7)...")
|
||||
model_answers2 = []
|
||||
for i, q in enumerate(distill_questions):
|
||||
ans = chat.generate_response(q, max_length=40, temperature=0.7)
|
||||
model_answers2.append((q, ans))
|
||||
if (i + 1) % 100 == 0:
|
||||
print(f" P2: {i+1}/{len(distill_questions)}")
|
||||
p2_gen = time.time()
|
||||
print(f" [P2] 生成完毕 ({(p2_gen-p1_gen):.0f}s)，API转换中...")
|
||||
|
||||
p2_results = batch_convert(model_answers2, progress)
|
||||
print(f" API: {len(model_answers2)}/{len(model_answers2)}")
|
||||
all_new_pairs.extend(p2_results)
|
||||
print(f" [P2] +{len(p2_results)}条")
|
||||
|
||||
# ====== Pass 3: API直接生成多样回答(temp=1.0) ======
|
||||
print(f" [P3] API直接生成 (temp=1.0, 全部直答)...")
|
||||
p3_t0 = time.time()
|
||||
p3_results = batch_generate_multi_temp(
|
||||
distill_questions,
|
||||
temperatures=[1.0],
|
||||
max_workers=8
|
||||
)
|
||||
all_new_pairs.extend(p3_results)
|
||||
print(f" [P3] +{len(p3_results)}条")
|
||||
|
||||
# ====== 合并去重后追加 ======
|
||||
if all_new_pairs:
|
||||
with open(train_path, 'r', encoding='utf-8') as f:
|
||||
cur_train = json.load(f)
|
||||
existing_pairs = set((item.get("question",""), item.get("answer","")) for item in cur_train)
|
||||
truly_new = [item for item in all_new_pairs
|
||||
if (item["question"], item["answer"]) not in existing_pairs]
|
||||
|
||||
if truly_new:
|
||||
cur_train.extend(truly_new)
|
||||
with open(train_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur_train, f, ensure_ascii=False, indent=2)
|
||||
# 同步到merged
|
||||
with open(merged_path, 'r', encoding='utf-8') as f:
|
||||
cur_merged = json.load(f)
|
||||
cur_merged.extend(truly_new)
|
||||
with open(merged_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur_merged, f, ensure_ascii=False, indent=2)
|
||||
total_distill_added += len(truly_new)
|
||||
distill_total = time.time() - distill_t0
|
||||
print(f" ✅ 蒸馏完成: +{len(truly_new)}条 (新增{len(set(q for q,a in [(i['question'],i['answer']) for i in truly_new]))}道独特问题) | 总耗时{distill_total:.0f}s")
|
||||
else:
|
||||
print(f" ℹ️ 蒸馏完成: 本轮无新内容（全部已存在）({time.time()-distill_t0:.0f}s)")
|
||||
else:
|
||||
print(f" ℹ️ 蒸馏完成: 无有效转换 ({time.time()-distill_t0:.0f}s)")
|
||||
except Exception as e:
|
||||
print(f" ⚠️ 蒸馏异常: {e}")
|
||||
|
||||
# 保存最终模型
|
||||
trainer.save_model(os.path.join(model_dir, 'xiaoli_v2_final.pth'), vocab, config['epochs'])
|
||||
|
||||
# 清理所有resume文件
|
||||
for f in glob.glob(os.path.join(model_dir, 'llm_model_v2_resume_*.pth')):
|
||||
try:
|
||||
os.remove(f)
|
||||
except:
|
||||
pass
|
||||
|
||||
total = time.time() - start
|
||||
print(f"\n训练完成！总耗时: {total/60:.1f}分钟")
|
||||
|
||||
# 测试
|
||||
print("\n测试:")
|
||||
chat = LLMChat(model, vocab, device, config['max_length'])
|
||||
for q in ["你好", "你是谁", "你叫什么名字", "你好可爱", "我喜欢你", "1+1等于几"]:
|
||||
r = chat.generate_response(q, max_length=60, temperature=0.8)
|
||||
print(f" Q: {q}")
|
||||
print(f" A: {r}\n")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
|
||||
size 999243
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330
|
||||
size 41185
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""小狸v1.2 Web聊天服务器 - 当前训练中的模型"""
|
||||
import sys, os, json, time
|
||||
sys.path.insert(0, '/workspace/xiaoli-llm-chat')
|
||||
import torch
|
||||
from flask import Flask, request, jsonify, send_from_directory
|
||||
from llm_chat_improved import LLMModel, LLMChat, build_vocabulary
|
||||
|
||||
print("=" * 50)
|
||||
print(" 小狸 v1.2 Web 服务加载中...")
|
||||
print("=" * 50)
|
||||
|
||||
device = 'cpu'
|
||||
model_dir = '/workspace/xiaoli-llm-chat'
|
||||
|
||||
# 构建当前vocab
|
||||
with open(f'{model_dir}/xiaoli_merged_data_v2.json', 'r', encoding='utf-8') as f:
|
||||
all_data = json.load(f)
|
||||
pairs = []
|
||||
for d in all_data:
|
||||
q = d.get('question','') or d.get('input','') or d.get('prompt','') or ''
|
||||
a = d.get('answer','') or d.get('output','') or d.get('response','') or ''
|
||||
if q and a:
|
||||
pairs.append((q, a))
|
||||
vocab = build_vocabulary(pairs)
|
||||
print(f"词汇表: {len(vocab)} | 数据: {len(all_data)}条")
|
||||
|
||||
# 加载模型 (epoch 50 = 干净epoch 41 + vocab扩展)
|
||||
ckpt_path = f'{model_dir}/llm_model_v2_resume_epoch_50.pth'
|
||||
print(f"加载: {ckpt_path}")
|
||||
ckpt = torch.load(ckpt_path, map_location=device)
|
||||
old_vocab_size = ckpt.get('model_config', {}).get('vocab_size', 0)
|
||||
cfg = ckpt['model_config']
|
||||
print(f"断点vocab: {old_vocab_size}")
|
||||
|
||||
model = LLMModel(
|
||||
vocab_size=len(vocab),
|
||||
d_model=cfg['d_model'], n_heads=cfg['n_heads'],
|
||||
n_layers=cfg['n_layers'], d_ff=cfg['d_ff'],
|
||||
max_len=cfg['max_len'], dropout=cfg.get('dropout', 0.1),
|
||||
use_memory=False, use_thinking=False, use_moe=False,
|
||||
num_experts=1, top_k=1,
|
||||
)
|
||||
|
||||
# 迁移权重
|
||||
old_sd = ckpt['model_state_dict']
|
||||
new_sd = model.state_dict()
|
||||
if old_vocab_size != len(vocab):
|
||||
for key in new_sd:
|
||||
if key in old_sd:
|
||||
w = old_sd[key]
|
||||
if w.shape == new_sd[key].shape:
|
||||
new_sd[key] = w.clone()
|
||||
elif len(w.shape) == 2 and w.shape[0] == old_vocab_size:
|
||||
new_sd[key][:old_vocab_size] = w
|
||||
elif 'freq' in key:
|
||||
pass
|
||||
else:
|
||||
try: new_sd[key] = w.clone()
|
||||
except: pass
|
||||
model.load_state_dict(new_sd)
|
||||
else:
|
||||
model.load_state_dict(old_sd)
|
||||
|
||||
model.to(device)
|
||||
model.eval()
|
||||
chat = LLMChat(model, vocab, device, max_length=cfg['max_len'])
|
||||
print(f"✅ 模型加载完成！ | voc={len(vocab)} | max_len={cfg['max_len']}")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
return send_from_directory(model_dir, 'chat.html')
|
||||
|
||||
@app.route('/api/chat', methods=['POST'])
|
||||
def api_chat():
|
||||
data = request.get_json()
|
||||
question = data.get('question', '').strip()
|
||||
temp = float(data.get('temperature', 0.2))
|
||||
max_len = int(data.get('max_length', 100))
|
||||
history = data.get('history', None)
|
||||
|
||||
if not question:
|
||||
return jsonify({'error': '问题不能为空'})
|
||||
|
||||
# 如果有history，转换为对话格式
|
||||
history_pairs = None
|
||||
if history and isinstance(history, list) and len(history) > 0:
|
||||
history_pairs = [(h['q'], h['a']) for h in history]
|
||||
|
||||
try:
|
||||
t0 = time.time()
|
||||
answer = chat.generate_response(question, max_length=max_len, temperature=temp, history=history_pairs)
|
||||
elapsed = time.time() - t0
|
||||
return jsonify({
|
||||
'question': question,
|
||||
'answer': answer,
|
||||
'time': f'{elapsed:.1f}s',
|
||||
'temperature': temp,
|
||||
})
|
||||
except Exception as e:
|
||||
return jsonify({'error': str(e)})
|
||||
|
||||
@app.route('/api/info')
|
||||
def model_info():
|
||||
return jsonify({
|
||||
'name': '小狸 v1.2',
|
||||
'version': 'epoch 52 (架构扩展: 7.9M→21.3M, max_len=256)',
|
||||
'vocab_size': len(vocab),
|
||||
'max_length': cfg['max_len'],
|
||||
'd_model': cfg['d_model'],
|
||||
'layers': cfg['n_layers'],
|
||||
'heads': cfg['n_heads'],
|
||||
'd_ff': cfg['d_ff'],
|
||||
'data_size': f'{len(all_data):,}条',
|
||||
'training': '新架构训练中 epoch 1→200',
|
||||
'features': ['多轮对话(1,694条)', '长文本(758条)', '我不知道(1,140条)'],
|
||||
})
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("\n" + "=" * 50)
|
||||
print(" 小狸 v1.2 聊天服务已启动！")
|
||||
print(" 访问地址: http://0.0.0.0:5000")
|
||||
print(" 建议温度: 0.1 ~ 0.2")
|
||||
print("=" * 50)
|
||||
app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:db648112bd045bf0e48f9ed28e85c4033da73f6338ce0a8fca0bbfc21fd8a8e8
|
||||
size 999243
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c66bc101096011732fcf3ce3f033a7478fe3f9d8e98b5e64bb6f3e03f0d82b85
|
||||
size 999336
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:efabe0696ef9fcd64685f70a62bd55c97ca2b8efa69bca26df35177ac6774330
|
||||
size 41185
|
||||
@@ -1,3 +0,0 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c0c0848f74cc7604d41699134a5f34f54d5958779a6653e27973ef495790c29c
|
||||
size 5267
|
||||
Reference in new issue
Block a user