shuiyu1123
7f8f607f26
- 训练脚本: train_v2.py + llm_chat_improved.py - API生成: api_correction.py - 高温多样化: regenerate_all_high_temp.py - 每日数据生成: daily_data_gen.py - 数据: xiaoli_train_v2.json(40776条) + vocab_v2.json - 蒸馏问题: exam_distill_questions等 - Web服务: web_server.py + chat.html - 新增 .gitignore 排除旧版文件
184 lines
8.0 KiB
Python
184 lines
8.0 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
每日数据生成 — 每天12:00跑
|
||
用 Astron 生成大量小狸风格问答数据，追加到训练集（不触发训练）
|
||
"""
|
||
import sys, os, json, time, random
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||
|
||
from api_correction import batch_generate
|
||
|
||
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
MAX_WORKERS = 5 # 5并发
|
||
BATCH_SIZE = 500
|
||
|
||
# ===== 种子话题（每天随机选一批让小狸聊） =====
|
||
SEED_TOPICS = [
|
||
"幼儿园", "学校", "朋友", "玩具", "动画片", "宠物", "零食", "天气",
|
||
"节日", "生日", "爸爸妈妈", "老师", "作业", "画画", "唱歌", "跳舞",
|
||
"游乐园", "动物园", "海洋馆", "公园", "滑滑梯", "秋千", "积木", "拼图",
|
||
"自行车", "游泳", "跳绳", "踢球", "冰淇淋", "蛋糕", "水果", "蔬菜",
|
||
"彩虹", "星星", "月亮", "太阳", "花", "草", "树", "小动物",
|
||
"蝴蝶", "小鸟", "小鱼", "乌龟", "兔子", "猫", "狗", "恐龙",
|
||
"过年", "压岁钱", "红包", "新衣服", "放鞭炮", "灯笼", "元宵",
|
||
"圣诞老人", "礼物", "糖果", "巧克力", "饼干", "牛奶", "早餐",
|
||
"为什么天空是蓝的", "为什么树叶会掉", "为什么有影子", "为什么会下雨",
|
||
"为什么会有白天黑夜", "为什么海水是咸的", "为什么会打雷",
|
||
"1+1等于几", "数数", "认字", "背诗", "唱歌", "讲故事",
|
||
"礼貌", "分享", "排队", "帮助别人", "说谢谢", "对不起",
|
||
"害怕", "开心", "难过", "生气", "勇敢", "害羞",
|
||
]
|
||
|
||
def generate_questions_from_topics(topics, count=500):
|
||
"""用话题种子生成自然提问"""
|
||
questions = []
|
||
for topic in topics[:count]:
|
||
patterns = [
|
||
f"小狸，你喜欢{topic}吗？",
|
||
f"给我讲讲{topic}吧",
|
||
f"{topic}是什么呀？",
|
||
f"你最喜欢什么样的{topic}？",
|
||
f"关于{topic}你知道什么？",
|
||
f"小狸，{topic}好玩吗？",
|
||
f"能不能教我关于{topic}的事情？",
|
||
f"你上次玩{topic}是什么时候？",
|
||
f"大家为什么都喜欢{topic}呀？",
|
||
f"小狸，你在{topic}上有什么有趣的事吗？",
|
||
]
|
||
q = random.choice(patterns)
|
||
questions.append(q)
|
||
return questions
|
||
|
||
def generate_natural_questions(count=200):
|
||
"""生成自然闲聊问题（固定一批高质量问题）"""
|
||
base_questions = [
|
||
"小狸你好呀", "今天开心吗", "你几岁了", "你叫什么名字",
|
||
"你吃饭了吗", "我们一起玩吧", "你在干嘛呢", "你今天做了什么",
|
||
"你喜欢什么颜色", "你最好的朋友是谁", "你最喜欢什么动物",
|
||
"你最喜欢吃什么", "你最喜欢什么水果", "你最爱看什么动画片",
|
||
"可以给我讲个故事吗", "唱首歌给我听", "教我背一首诗吧",
|
||
"你会画画吗", "我们来玩猜谜语", "你家里有谁呀",
|
||
"你的房间是什么样子的", "你书包里有什么", "你最喜欢上什么课",
|
||
"今天天气真好", "外面下雨了", "好冷啊", "好热啊",
|
||
"我有点不开心", "我今天好开心", "我害怕打雷",
|
||
"你睡午觉了吗", "晚安", "早安", "快来",
|
||
"我想吃冰淇淋", "我想去游乐场", "我想养一只小猫",
|
||
"为什么雪是白色的", "为什么天是蓝的", "为什么花是香的",
|
||
"月亮上有人吗", "星星为什么会眨眼睛", "大海有多大",
|
||
"你会数数吗", "1+1等于几", "100+200等于多少",
|
||
"我喜欢你小狸", "你好可爱", "我们做朋友吧",
|
||
"你今天真好看", "谢谢你", "你真好",
|
||
"对不起", "我不小心摔倒了", "我被人欺负了",
|
||
"怎么办呀", "帮帮我", "我该怎么做",
|
||
"什么是爱", "什么是勇敢", "什么是善良",
|
||
"长大是什么感觉", "为什么要上学", "为什么要听话",
|
||
"你会一直陪着我吗", "你想去什么地方玩",
|
||
]
|
||
return random.sample(base_questions, min(count, len(base_questions)))
|
||
|
||
def main():
|
||
print("=" * 60)
|
||
print("📅 每日数据生成")
|
||
print(f"时间: {time.strftime('%Y-%m-%d %H:%M:%S')}")
|
||
print("=" * 60)
|
||
|
||
# 1. 混合生成问题
|
||
random.seed(int(time.time()))
|
||
n_topic_qs = random.randint(300, 500) # 话题衍生问题
|
||
n_natural_qs = random.randint(200, 300) # 自然闲聊问题
|
||
|
||
random.shuffle(SEED_TOPICS)
|
||
topic_questions = generate_questions_from_topics(SEED_TOPICS, n_topic_qs)
|
||
natural_questions = generate_natural_questions(n_natural_qs)
|
||
|
||
all_questions = topic_questions + natural_questions
|
||
random.shuffle(all_questions)
|
||
|
||
print(f"\n准备生成 {len(all_questions)} 条问答 (话题衍生: {len(topic_questions)}, 自然: {len(natural_questions)})")
|
||
|
||
# 2. 用不同温度分批生成，增加多样性
|
||
TEMPS = [0.85, 0.9, 0.95, 1.0]
|
||
all_results = []
|
||
existing = set()
|
||
|
||
# 加载已有数据用于去重
|
||
train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json')
|
||
with open(train_path, encoding='utf-8') as f:
|
||
cur_train = json.load(f)
|
||
for it in cur_train:
|
||
key = (it.get('question','')[:100], it.get('answer','')[:100])
|
||
existing.add(key)
|
||
print(f"已有训练集: {len(cur_train)} 条")
|
||
|
||
# 分批+变温度
|
||
total_batches = (len(all_questions) + BATCH_SIZE - 1) // BATCH_SIZE
|
||
t_start = time.time()
|
||
|
||
for batch_idx in range(total_batches):
|
||
batch_qs = all_questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE]
|
||
temp = random.choice(TEMPS)
|
||
|
||
print(f"\n--- 批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, temp={temp}) ---")
|
||
|
||
t0 = time.time()
|
||
results = batch_generate(batch_qs, temperature=temp, max_workers=MAX_WORKERS)
|
||
elapsed = time.time() - t0
|
||
|
||
new_in_batch = 0
|
||
for r in results:
|
||
key = (r['question'][:100], r['answer'][:100])
|
||
if key not in existing:
|
||
existing.add(key)
|
||
all_results.append(r)
|
||
new_in_batch += 1
|
||
|
||
total_elapsed = time.time() - t_start
|
||
rate = len(all_results) / max(total_elapsed, 1) * 60
|
||
print(f" 本批新增 {new_in_batch} 条 | 累计 {len(all_results)} 条 | {rate:.0f}条/分")
|
||
|
||
if not all_results:
|
||
print("\n⚠️ 没有生成新数据")
|
||
return
|
||
|
||
# 3. 追加到训练集
|
||
print(f"\n追加 {len(all_results)} 条到训练集...")
|
||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||
tp = os.path.join(MODEL_DIR, tgt)
|
||
with open(tp, encoding='utf-8') as f:
|
||
cur = json.load(f)
|
||
|
||
ex = set((it.get('question','')[:100], it.get('answer','')[:100]) for it in cur)
|
||
to_add = []
|
||
for r in all_results:
|
||
key = (r['question'][:100], r['answer'][:100])
|
||
if key not in ex:
|
||
ex.add(key)
|
||
to_add.append({'question': r['question'], 'answer': r['answer']})
|
||
|
||
cur.extend(to_add)
|
||
with open(tp, 'w', encoding='utf-8') as f:
|
||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||
|
||
if 'train' in tgt:
|
||
print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条")
|
||
else:
|
||
print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条")
|
||
|
||
# 4. 保存今日生成日志
|
||
today = time.strftime('%Y-%m-%d')
|
||
log_path = os.path.join(MODEL_DIR, f'daily_gen_{today}.json')
|
||
with open(log_path, 'w', encoding='utf-8') as f:
|
||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||
|
||
elapsed_total = time.time() - t_start
|
||
print(f"\n{'='*60}")
|
||
print(f"✅ 今日生成完成! {len(all_results)} 条新数据")
|
||
print(f"用时: {elapsed_total/60:.1f} 分钟")
|
||
print(f"日志: daily_gen_{today}.json")
|
||
print(f"{'='*60}")
|
||
|
||
if __name__ == '__main__':
|
||
main() |