shuiyu1123
231af72de9
- 清理仓库: 删除所有废弃的generate_*脚本和旧版数据文件 - 目录重组: - data/ → 训练数据 - scripts/ → 数据生成脚本 - templates/ → Web UI - 重命名: - train_v2.py → train.py - vocab_v3.json → vocab.json (BPE, 6.8k tokens) - 更新 train.py 指向新词表路径 - 重写 README.md 完整文档 - 更新 .gitignore
221 lines
10 KiB
Python
221 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
每日数据生成 — 每天12:00跑
|
||
用 Astron 生成大量小狸风格问答数据，追加到训练集（不触发训练）
|
||
"""
|
||
import sys, os, json, time, random
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||
|
||
from api_correction import batch_generate
|
||
|
||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
MAX_WORKERS = 5 # Astron 5并发（无限量）
|
||
BATCH_SIZE = 500
|
||
|
||
# 每日生成量 — Astron无限量，直接拉满！
|
||
DAILY_TOPIC_QS = 2000 # 话题衍生
|
||
DAILY_NATURAL_QS = 1000 # 自然闲聊
|
||
|
||
# ===== 种子话题（每天随机选一批让小狸聊） =====
|
||
SEED_TOPICS = [
|
||
"幼儿园", "学校", "朋友", "玩具", "动画片", "宠物", "零食", "天气",
|
||
"节日", "生日", "爸爸妈妈", "老师", "作业", "画画", "唱歌", "跳舞",
|
||
"游乐园", "动物园", "海洋馆", "公园", "滑滑梯", "秋千", "积木", "拼图",
|
||
"自行车", "游泳", "跳绳", "踢球", "冰淇淋", "蛋糕", "水果", "蔬菜",
|
||
"彩虹", "星星", "月亮", "太阳", "花", "草", "树", "小动物",
|
||
"蝴蝶", "小鸟", "小鱼", "乌龟", "兔子", "猫", "狗", "恐龙",
|
||
"过年", "压岁钱", "红包", "新衣服", "放鞭炮", "灯笼", "元宵",
|
||
"圣诞老人", "礼物", "糖果", "巧克力", "饼干", "牛奶", "早餐",
|
||
"为什么天空是蓝的", "为什么树叶会掉", "为什么有影子", "为什么会下雨",
|
||
"为什么会有白天黑夜", "为什么海水是咸的", "为什么会打雷",
|
||
"1+1等于几", "数数", "认字", "背诗", "唱歌", "讲故事",
|
||
"礼貌", "分享", "排队", "帮助别人", "说谢谢", "对不起",
|
||
"害怕", "开心", "难过", "生气", "勇敢", "害羞",
|
||
# 拓展话题
|
||
"奥特曼", "小猪佩奇", "汪汪队", "海底小纵队", "超级飞侠",
|
||
"白雪公主", "灰姑娘", "小红帽", "三只小猪", "匹诺曹",
|
||
"茉莉花", "向日葵", "蒲公英", "含羞草", "仙人掌",
|
||
"企鹅", "熊猫", "长颈鹿", "大象", "海豚", "松鼠", "刺猬",
|
||
"草莓", "西瓜", "葡萄", "芒果", "蓝莓", "橙子",
|
||
"汉堡", "薯条", "披萨", "面条", "饺子", "馄饨", "粽子",
|
||
"沙滩", "大海", "高山", "森林", "瀑布", "溶洞",
|
||
"火箭", "飞机", "高铁", "轮船", "自行车", "滑板",
|
||
"围棋", "象棋", "跳棋", "五子棋", "扑克牌",
|
||
"机器人", "变形金刚", "乐高", "磁力片", "橡皮泥",
|
||
"春天", "夏天", "秋天", "冬天", "刮风", "下雪", "打雷",
|
||
"刷牙", "洗脸", "洗澡", "剪指甲", "梳头", "穿衣服",
|
||
"诚实", "分享", "礼貌", "排队", "帮忙", "道歉", "谢谢",
|
||
"高兴", "生气", "难过", "害怕", "惊讶", "害羞", "委屈",
|
||
"萤火虫", "蜻蜓", "蜜蜂", "蝉", "蜗牛", "蚂蚁", "蜘蛛",
|
||
"玉米", "土豆", "番茄", "胡萝卜", "白菜", "茄子", "辣椒",
|
||
"毛笔", "铅笔", "蜡笔", "水彩笔", "画板", "颜料",
|
||
"竹笋", "蘑菇", "松树", "枫叶", "荷花", "梅花",
|
||
"元旦", "春节", "元宵节", "清明节", "端午节", "中秋节",
|
||
"万圣节", "感恩节", "圣诞节", "儿童节", "母亲节", "父亲节",
|
||
"月球", "太阳系", "银河", "彗星", "流星", "黑洞",
|
||
"恐龙", "猛犸象", "剑齿虎", "翼龙", "三角龙", "霸王龙",
|
||
"三字经", "弟子规", "唐诗", "宋词", "成语",
|
||
"捉迷藏", "老鹰捉小鸡", "丢手绢", "跳房子", "踢毽子",
|
||
"轮滑", "滑冰", "滑雪", "冲浪", "攀岩",
|
||
"钢琴", "吉他", "小提琴", "笛子", "鼓", "古筝",
|
||
"警察", "消防员", "医生", "护士", "科学家", "宇航员",
|
||
"红绿灯", "斑马线", "安全带", "头盔", "救生圈",
|
||
"原来如此", "没关系", "太好了", "加油", "真棒", "别担心",
|
||
"手影", "折纸", "翻花绳", "吹泡泡", "放风筝",
|
||
"长城", "故宫", "天安门", "兵马俑", "大熊猫基地",
|
||
"小木屋", "城堡", "树屋", "帐篷", "房车",
|
||
]
|
||
|
||
def generate_questions_from_topics(topics, count=500):
|
||
"""用话题种子生成自然提问"""
|
||
questions = []
|
||
for topic in topics[:count]:
|
||
patterns = [
|
||
f"小狸，你喜欢{topic}吗？",
|
||
f"给我讲讲{topic}吧",
|
||
f"{topic}是什么呀？",
|
||
f"你最喜欢什么样的{topic}？",
|
||
f"关于{topic}你知道什么？",
|
||
f"小狸，{topic}好玩吗？",
|
||
f"能不能教我关于{topic}的事情？",
|
||
f"你上次玩{topic}是什么时候？",
|
||
f"大家为什么都喜欢{topic}呀？",
|
||
f"小狸，你在{topic}上有什么有趣的事吗？",
|
||
]
|
||
q = random.choice(patterns)
|
||
questions.append(q)
|
||
return questions
|
||
|
||
def generate_natural_questions(count=200):
|
||
"""生成自然闲聊问题（固定一批高质量问题）"""
|
||
base_questions = [
|
||
"小狸你好呀", "今天开心吗", "你几岁了", "你叫什么名字",
|
||
"你吃饭了吗", "我们一起玩吧", "你在干嘛呢", "你今天做了什么",
|
||
"你喜欢什么颜色", "你最好的朋友是谁", "你最喜欢什么动物",
|
||
"你最喜欢吃什么", "你最喜欢什么水果", "你最爱看什么动画片",
|
||
"可以给我讲个故事吗", "唱首歌给我听", "教我背一首诗吧",
|
||
"你会画画吗", "我们来玩猜谜语", "你家里有谁呀",
|
||
"你的房间是什么样子的", "你书包里有什么", "你最喜欢上什么课",
|
||
"今天天气真好", "外面下雨了", "好冷啊", "好热啊",
|
||
"我有点不开心", "我今天好开心", "我害怕打雷",
|
||
"你睡午觉了吗", "晚安", "早安", "快来",
|
||
"我想吃冰淇淋", "我想去游乐场", "我想养一只小猫",
|
||
"为什么雪是白色的", "为什么天是蓝的", "为什么花是香的",
|
||
"月亮上有人吗", "星星为什么会眨眼睛", "大海有多大",
|
||
"你会数数吗", "1+1等于几", "100+200等于多少",
|
||
"我喜欢你小狸", "你好可爱", "我们做朋友吧",
|
||
"你今天真好看", "谢谢你", "你真好",
|
||
"对不起", "我不小心摔倒了", "我被人欺负了",
|
||
"怎么办呀", "帮帮我", "我该怎么做",
|
||
"什么是爱", "什么是勇敢", "什么是善良",
|
||
"长大是什么感觉", "为什么要上学", "为什么要听话",
|
||
"你会一直陪着我吗", "你想去什么地方玩",
|
||
]
|
||
return random.sample(base_questions, min(count, len(base_questions)))
|
||
|
||
def main():
|
||
print("=" * 60)
|
||
print("📅 每日数据生成")
|
||
print(f"时间: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||
print("=" * 60)
|
||
|
||
# 1. 混合生成问题
|
||
random.seed(int(time.time()))
|
||
n_topic_qs = DAILY_TOPIC_QS # 话题衍生
|
||
n_natural_qs = DAILY_NATURAL_QS # 自然闲聊
|
||
|
||
random.shuffle(SEED_TOPICS)
|
||
topic_questions = generate_questions_from_topics(SEED_TOPICS, n_topic_qs)
|
||
natural_questions = generate_natural_questions(n_natural_qs)
|
||
|
||
all_questions = topic_questions + natural_questions
|
||
random.shuffle(all_questions)
|
||
|
||
print(f"\n准备生成 {len(all_questions)} 条问答 (话题衍生: {len(topic_questions)}, 自然: {len(natural_questions)})")
|
||
|
||
# 2. 用不同温度分批生成，增加多样性
|
||
TEMPS = [0.85, 0.9, 0.95, 1.0]
|
||
all_results = []
|
||
existing = set()
|
||
|
||
# 加载已有数据用于去重
|
||
train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json')
|
||
with open(train_path, encoding='utf-8') as f:
|
||
cur_train = json.load(f)
|
||
for it in cur_train:
|
||
key = (it.get('question','')[:100], it.get('answer','')[:100])
|
||
existing.add(key)
|
||
print(f"已有训练集: {len(cur_train)} 条")
|
||
|
||
# 分批+变温度
|
||
total_batches = (len(all_questions) + BATCH_SIZE - 1) // BATCH_SIZE
|
||
t_start = time.time()
|
||
|
||
for batch_idx in range(total_batches):
|
||
batch_qs = all_questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
|
||
temp = random.choice(TEMPS)
|
||
|
||
print(f"\n--- 批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, temp={temp}) ---")
|
||
|
||
t0 = time.time()
|
||
results = batch_generate(batch_qs, temperature=temp, max_workers=MAX_WORKERS)
|
||
elapsed = time.time() - t0
|
||
|
||
new_in_batch = 0
|
||
for r in results:
|
||
key = (r['question'][:100], r['answer'][:100])
|
||
if key not in existing:
|
||
existing.add(key)
|
||
all_results.append(r)
|
||
new_in_batch += 1
|
||
|
||
total_elapsed = time.time() - t_start
|
||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||
print(f" 本批新增 {new_in_batch} 条 | 累计 {len(all_results)} 条 | {rate:.0f}条/分")
|
||
|
||
if not all_results:
|
||
print("\n⚠️ 没有生成新数据")
|
||
return
|
||
|
||
# 3. 追加到训练集
|
||
print(f"\n追加 {len(all_results)} 条到训练集...")
|
||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||
tp = os.path.join(MODEL_DIR, tgt)
|
||
with open(tp, encoding='utf-8') as f:
|
||
cur = json.load(f)
|
||
|
||
ex = set((it.get('question','')[:100], it.get('answer','')[:100]) for it in cur)
|
||
to_add = []
|
||
for r in all_results:
|
||
key = (r['question'][:100], r['answer'][:100])
|
||
if key not in ex:
|
||
ex.add(key)
|
||
to_add.append({'question': r['question'], 'answer': r['answer']})
|
||
|
||
cur.extend(to_add)
|
||
with open(tp, 'w', encoding='utf-8') as f:
|
||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||
|
||
if 'train' in tgt:
|
||
print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条")
|
||
else:
|
||
print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条")
|
||
|
||
# 4. 保存今日生成日志
|
||
today = time.strftime('%Y-%m-%d')
|
||
log_path = os.path.join(MODEL_DIR, f'daily_gen_{today}.json')
|
||
with open(log_path, 'w', encoding='utf-8') as f:
|
||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||
|
||
elapsed_total = time.time() - t_start
|
||
print(f"\n{'='*60}")
|
||
print(f"✅ 今日生成完成! {len(all_results)} 条新数据")
|
||
print(f"用时: {elapsed_total/60:.1f} 分钟")
|
||
print(f"日志: daily_gen_{today}.json")
|
||
print(f"{'='*60}")
|
||
|
||
if __name__ == '__main__':
|
||
main() |