#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 每日数据生成 — 每天12:00跑 用 Astron 生成大量小狸风格问答数据，追加到训练集（不触发训练） """ import sys, os, json, time, random sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) from api_correction import batch_generate MODEL_DIR = os.path.dirname(os.path.abspath(__file__)) MAX_WORKERS = 5 # 5并发 BATCH_SIZE = 500 # ===== 种子话题（每天随机选一批让小狸聊） ===== SEED_TOPICS = [ "幼儿园", "学校", "朋友", "玩具", "动画片", "宠物", "零食", "天气", "节日", "生日", "爸爸妈妈", "老师", "作业", "画画", "唱歌", "跳舞", "游乐园", "动物园", "海洋馆", "公园", "滑滑梯", "秋千", "积木", "拼图", "自行车", "游泳", "跳绳", "踢球", "冰淇淋", "蛋糕", "水果", "蔬菜", "彩虹", "星星", "月亮", "太阳", "花", "草", "树", "小动物", "蝴蝶", "小鸟", "小鱼", "乌龟", "兔子", "猫", "狗", "恐龙", "过年", "压岁钱", "红包", "新衣服", "放鞭炮", "灯笼", "元宵", "圣诞老人", "礼物", "糖果", "巧克力", "饼干", "牛奶", "早餐", "为什么天空是蓝的", "为什么树叶会掉", "为什么有影子", "为什么会下雨", "为什么会有白天黑夜", "为什么海水是咸的", "为什么会打雷", "1+1等于几", "数数", "认字", "背诗", "唱歌", "讲故事", "礼貌", "分享", "排队", "帮助别人", "说谢谢", "对不起", "害怕", "开心", "难过", "生气", "勇敢", "害羞", ] def generate_questions_from_topics(topics, count=500): """用话题种子生成自然提问""" questions = [] for topic in topics[:count]: patterns = [ f"小狸，你喜欢{topic}吗？", f"给我讲讲{topic}吧", f"{topic}是什么呀？", f"你最喜欢什么样的{topic}？", f"关于{topic}你知道什么？", f"小狸，{topic}好玩吗？", f"能不能教我关于{topic}的事情？", f"你上次玩{topic}是什么时候？", f"大家为什么都喜欢{topic}呀？", f"小狸，你在{topic}上有什么有趣的事吗？", ] q = random.choice(patterns) questions.append(q) return questions def generate_natural_questions(count=200): """生成自然闲聊问题（固定一批高质量问题）""" base_questions = [ "小狸你好呀", "今天开心吗", "你几岁了", "你叫什么名字", "你吃饭了吗", "我们一起玩吧", "你在干嘛呢", "你今天做了什么", "你喜欢什么颜色", "你最好的朋友是谁", "你最喜欢什么动物", "你最喜欢吃什么", "你最喜欢什么水果", "你最爱看什么动画片", "可以给我讲个故事吗", "唱首歌给我听", "教我背一首诗吧", "你会画画吗", "我们来玩猜谜语", "你家里有谁呀", "你的房间是什么样子的", "你书包里有什么", "你最喜欢上什么课", "今天天气真好", "外面下雨了", "好冷啊", "好热啊", "我有点不开心", "我今天好开心", "我害怕打雷", "你睡午觉了吗", "晚安", "早安", "快来", "我想吃冰淇淋", "我想去游乐场", "我想养一只小猫", "为什么雪是白色的", "为什么天是蓝的", "为什么花是香的", "月亮上有人吗", "星星为什么会眨眼睛", "大海有多大", "你会数数吗", "1+1等于几", "100+200等于多少", "我喜欢你小狸", "你好可爱", "我们做朋友吧", "你今天真好看", "谢谢你", "你真好", "对不起", "我不小心摔倒了", "我被人欺负了", "怎么办呀", "帮帮我", "我该怎么做", "什么是爱", "什么是勇敢", "什么是善良", "长大是什么感觉", "为什么要上学", "为什么要听话", "你会一直陪着我吗", "你想去什么地方玩", ] return random.sample(base_questions, min(count, len(base_questions))) def main(): print("=" * 60) print("📅 每日数据生成") print(f"时间: {time.strftime('%Y-%m-%d %H:%M:%S')}") print("=" * 60) # 1. 混合生成问题 random.seed(int(time.time())) n_topic_qs = random.randint(300, 500) # 话题衍生问题 n_natural_qs = random.randint(200, 300) # 自然闲聊问题 random.shuffle(SEED_TOPICS) topic_questions = generate_questions_from_topics(SEED_TOPICS, n_topic_qs) natural_questions = generate_natural_questions(n_natural_qs) all_questions = topic_questions + natural_questions random.shuffle(all_questions) print(f"\n准备生成 {len(all_questions)} 条问答 (话题衍生: {len(topic_questions)}, 自然: {len(natural_questions)})") # 2. 用不同温度分批生成，增加多样性 TEMPS = [0.85, 0.9, 0.95, 1.0] all_results = [] existing = set() # 加载已有数据用于去重 train_path = os.path.join(MODEL_DIR, 'xiaoli_train_v2.json') with open(train_path, encoding='utf-8') as f: cur_train = json.load(f) for it in cur_train: key = (it.get('question','')[:100], it.get('answer','')[:100]) existing.add(key) print(f"已有训练集: {len(cur_train)} 条") # 分批+变温度 total_batches = (len(all_questions) + BATCH_SIZE - 1) // BATCH_SIZE t_start = time.time() for batch_idx in range(total_batches): batch_qs = all_questions[batch_idx * BATCH_SIZE : (batch_idx + 1) * BATCH_SIZE] temp = random.choice(TEMPS) print(f"\n--- 批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)}条, temp={temp}) ---") t0 = time.time() results = batch_generate(batch_qs, temperature=temp, max_workers=MAX_WORKERS) elapsed = time.time() - t0 new_in_batch = 0 for r in results: key = (r['question'][:100], r['answer'][:100]) if key not in existing: existing.add(key) all_results.append(r) new_in_batch += 1 total_elapsed = time.time() - t_start rate = len(all_results) / max(total_elapsed, 1) * 60 print(f" 本批新增 {new_in_batch} 条 | 累计 {len(all_results)} 条 | {rate:.0f}条/分") if not all_results: print("\n⚠️ 没有生成新数据") return # 3. 追加到训练集 print(f"\n追加 {len(all_results)} 条到训练集...") for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']: tp = os.path.join(MODEL_DIR, tgt) with open(tp, encoding='utf-8') as f: cur = json.load(f) ex = set((it.get('question','')[:100], it.get('answer','')[:100]) for it in cur) to_add = [] for r in all_results: key = (r['question'][:100], r['answer'][:100]) if key not in ex: ex.add(key) to_add.append({'question': r['question'], 'answer': r['answer']}) cur.extend(to_add) with open(tp, 'w', encoding='utf-8') as f: json.dump(cur, f, ensure_ascii=False, indent=2) if 'train' in tgt: print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条") else: print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条") # 4. 保存今日生成日志 today = time.strftime('%Y-%m-%d') log_path = os.path.join(MODEL_DIR, f'daily_gen_{today}.json') with open(log_path, 'w', encoding='utf-8') as f: json.dump(all_results, f, ensure_ascii=False, indent=2) elapsed_total = time.time() - t_start print(f"\n{'='*60}") print(f"✅ 今日生成完成! {len(all_results)} 条新数据") print(f"用时: {elapsed_total/60:.1f} 分钟") print(f"日志: daily_gen_{today}.json") print(f"{'='*60}") if __name__ == '__main__': main()