From 91db488ad4fd1a5857176acd55510f7a93ba502b Mon Sep 17 00:00:00 2001 From: shuiyu1123 Date: Tue, 16 Jun 2026 11:48:30 +0000 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=BB=A3=E7=A0=81=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=20(2026-06-16)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除 generate_diverse_answers.py（多样化回答已废止） - 更新 generate_fresh_qa.py（目标50万条新问答） - 更新 generate_multiturn.py（6~12轮长对话，目标50万） - 更新 regenerate_all_high_temp.py（独立保存，不追加训练集） - 更新 .gitignore（忽略新输出目录） - 所有API Key改为环境变量读取 --- .gitignore | 3 + generate_diverse_answers.py | 256 ------------------------------------ generate_fresh_qa.py | 4 +- generate_multiturn.py | 19 +-- regenerate_all_high_temp.py | 30 +---- 5 files changed, 21 insertions(+), 291 deletions(-) delete mode 100644 generate_diverse_answers.py diff --git a/.gitignore b/.gitignore index c7b74b3..c88f70c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ mega_interim_*.json essay_interim_*.json checkpoints_v2/ high_temp_outputs/ +diverse_answers/ +fresh_qa_outputs/ +multiturn_outputs/ old_ckpts/ backup_old_arch/ diff --git a/generate_diverse_answers.py b/generate_diverse_answers.py deleted file mode 100644 index 9d9d1ce..0000000 --- a/generate_diverse_answers.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -多样化回答生成 — 对每个问题用 Astron 高温生成 50 个不同回答 -5并发，若重复则重生成，增量保存防丢 -""" -import json, requests, time, os, sys, random, hashlib -from concurrent.futures import ThreadPoolExecutor, as_completed -sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) - -# ====== Astron 配置 ====== -ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" -ASTRON_KEY = os.environ.get("ASTRON_KEY", "") -ASTRON_MODEL = "astron-code-latest" - -SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 - -你的人设： -- 8岁女孩子，有点害羞但很可爱 -- 说话简单自然，符合小朋友的语气 -- 回答问题要简短，1-2句话就够了 -- 不要用"喵"结尾，不要过度卖萌 - -回答要求： -1. 语气自然，像真实的小朋友说话 -2. 不懂就说"不知道" -3. 回答简洁但要说清楚，不要讲一半就停""" - - -def call_astron(q, temperature): - """调用 Astron，返回回答或 None""" - for attempt in range(3): - try: - headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} - payload = { - "model": ASTRON_MODEL, - "messages": [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": q} - ], - "temperature": temperature, - "max_tokens": 2048, - } - resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) - if resp.status_code == 200: - content = resp.json()['choices'][0]['message']['content'].strip() - if content and len(content) > 2: - return content - else: - time.sleep(1) - except Exception: - time.sleep(2) - return None - - -def answer_fingerprint(text): - """回答指纹 — 去空格标点后取前 30 字""" - clean = ''.join(text.split())[:30] - return hashlib.md5(clean.encode()).hexdigest() - - -def is_duplicate(new, existing): - """检查是否与已有回答重复（精确+模糊）""" - fp = answer_fingerprint(new) - if fp in existing: - return True - # 模糊：如果新回答是某个已有回答的子串或反之 - short_new = ''.join(new.split())[:20] - for old_fp, old_text in existing.items(): - short_old = ''.join(old_text.split())[:20] - if short_new in short_old or short_old in short_new: - return True if len(short_new) > 8 else False - return False - - -def generate_answers_for_question(q, target=50, max_attempts=100, temperature=0.95): - """为单个问题生成 target 个不同回答""" - answers = {} # fp -> full_text - attempt = 0 - t_start = time.time() - - while len(answers) < target and attempt < max_attempts: - # 温度在 0.85~1.0 之间随机波动，避免卡在同一模式 - temp = temperature + random.uniform(-0.1, 0.05) - temp = max(0.85, min(1.0, temp)) - - result = call_astron(f"问题：{q}\n\n请以小狸的身份回答：", temp) - if result: - if not is_duplicate(result, answers): - fp = answer_fingerprint(result) - answers[fp] = result - - attempt += 1 - - if attempt % 20 == 0: - elapsed = time.time() - t_start - rate = len(answers) / max(elapsed, 1) - if len(answers) < target and rate > 0: - eta = (target - len(answers)) / rate - else: - eta = 0 - print(f" [{len(answers)}/{target}] attempt={attempt}, {rate:.1f}条/秒, 预计剩余{eta:.0f}秒") - sys.stdout.flush() - - return list(answers.values()) - - -def main(): - model_dir = '/workspace/xiaoli-llm-chat' - output_dir = os.path.join(model_dir, 'diverse_answers') - os.makedirs(output_dir, exist_ok=True) - - # ====== 参数（可调） ====== - QUESTIONS_PER_BATCH = 100 # 每批处理多少问题 - ANSWERS_PER_QUESTION = 50 # 每个问题生成多少个不同回答 - MAX_ATTEMPTS = 150 # 最多尝试多少次（超过就放弃） - MAX_WORKERS = 5 # Astron 5并发 - TEMPERATURE = 0.95 # 高温（0.85~1.0 浮动） - - # ====== 加载问题列表 ====== - train_path = os.path.join(model_dir, 'xiaoli_train_v2.json') - with open(train_path, encoding='utf-8') as f: - data = json.load(f) - - # 取所有不重复的问题 - all_questions = list(set(item.get('question', '') for item in data)) - random.shuffle(all_questions) - print(f"训练集总条数: {len(data)}") - print(f"不重复问题数: {len(all_questions)}") - - # 统计问题出现频次，优先处理高频问题 - q_freq = {} - for item in data: - q = item.get('question', '') - q_freq[q] = q_freq.get(q, 0) + 1 - - # 按频次排序（高频靠前，低频靠后，随机打乱同频） - freq_buckets = {} - for q in all_questions: - f = q_freq.get(q, 1) - freq_buckets.setdefault(f, []).append(q) - - sorted_questions = [] - for freq in sorted(freq_buckets.keys(), reverse=True): - bucket = freq_buckets[freq] - random.shuffle(bucket) - sorted_questions.extend(bucket) - - print(f"按频次排序完成 (高频优先处理)") - - # ====== 恢复已有进度 ====== - progress_file = os.path.join(output_dir, 'diversity_progress.json') - completed = {} # question -> list of answers - if os.path.exists(progress_file): - with open(progress_file, encoding='utf-8') as f: - completed = json.load(f) - # 过滤掉已处理的问题 - processed_qs = set(completed.keys()) - sorted_questions = [q for q in sorted_questions if q not in processed_qs] - print(f"已有进度: {len(completed)} 个问题已完成") - print(f"剩余未处理: {len(sorted_questions)} 个问题") - - # 限制要处理的问题数 - max_q = len(sorted_questions) # 全部问题 - - total_questions = len(sorted_questions) - total_batches = (total_questions + QUESTIONS_PER_BATCH - 1) // QUESTIONS_PER_BATCH - - print(f"\n目标: {total_questions} 个问题 × {ANSWERS_PER_QUESTION} 个回答 = {total_questions * ANSWERS_PER_QUESTION} 条") - print(f"批次: {total_batches} 批 (每批 ≤{QUESTIONS_PER_BATCH} 问题)") - print(f"并发: {MAX_WORKERS}") - if total_questions > 1000: - minutes = total_questions * ANSWERS_PER_QUESTION / (MAX_WORKERS * 8) / 60 - print(f"预估耗时: ~{minutes:.0f} 分钟 (~{minutes/60:.1f} 小时)") - print(f"{'='*70}\n") - - t_start = time.time() - total_new_pairs = 0 - - for batch_idx in range(total_batches): - batch_qs = sorted_questions[batch_idx * QUESTIONS_PER_BATCH : (batch_idx + 1) * QUESTIONS_PER_BATCH] - batch_results = {} - - print(f"\n批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)} 个问题)") - - # 并发生成 - with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: - futures = { - executor.submit(generate_answers_for_question, q, ANSWERS_PER_QUESTION, MAX_ATTEMPTS, TEMPERATURE): q - for q in batch_qs - } - - for future in as_completed(futures): - q = futures[future] - try: - answers = future.result() - if answers: - batch_results[q] = answers - print(f" ✅ 『{q[:30]}』 → {len(answers)} 个不同回答") - else: - print(f" ❌ 『{q[:30]}』 → 生成失败") - except Exception as e: - print(f" ❌ 『{q[:30]}』 → 错误: {e}") - - # 追加到总进度 - completed.update(batch_results) - batch_new_pairs = sum(len(v) for v in batch_results.values()) - total_new_pairs += batch_new_pairs - - # 保存进度 - with open(progress_file, 'w', encoding='utf-8') as f: - json.dump(completed, f, ensure_ascii=False, indent=2) - - elapsed = time.time() - t_start - rate = total_new_pairs / max(elapsed, 1) * 60 - - # 统计 - print(f"\n批次 {batch_idx + 1} 完成 | " - f"本批新增 {batch_new_pairs} 条 | " - f"累计 {total_new_pairs:,} 条 ({len(completed)} 问题) | " - f"{rate:.0f}条/分 | " - f"已用 {elapsed/60:.1f}分") - - # ====== 完成 ====== - elapsed_total = time.time() - t_start - print(f"\n{'='*70}") - print(f"全部生成完成!") - print(f"处理问题: {len(completed)} 个") - print(f"总回答数: {total_new_pairs:,} 条") - print(f"用时: {elapsed_total/60:.1f} 分钟") - - # ====== 保存为训练格式 ====== - train_format = [] - for q, answers in completed.items(): - for a in answers: - train_format.append({'question': q, 'answer': a}) - - output_file = os.path.join(output_dir, 'diverse_train_data_v1.json') - with open(output_file, 'w', encoding='utf-8') as f: - json.dump(train_format, f, ensure_ascii=False, indent=2) - - # 也保存一份到工作目录 - work_file = '/data/user/work/diverse_train_data_v1.json' - with open(work_file, 'w', encoding='utf-8') as f: - json.dump(train_format, f, ensure_ascii=False, indent=2) - - print(f"\n结果保存:") - print(f" 📄 {output_file}") - print(f" 📄 {work_file}") - print(f" 📊 平均每问题 {total_new_pairs/max(len(completed),1):.1f} 个回答") - print(f"\n如需合并到训练集：") - print(f" python3 -c \"import json; d1=json.load(open('xiaoli_train_v2.json')); d2=json.load(open('{output_file}')); d1.extend(d2); json.dump(d1, open('xiaoli_train_v2.json','w'), ensure_ascii=False, indent=2)\"") - print(f"{'='*70}") - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/generate_fresh_qa.py b/generate_fresh_qa.py index 8427f20..0693a26 100644 --- a/generate_fresh_qa.py +++ b/generate_fresh_qa.py @@ -171,8 +171,8 @@ def main(): print(f"去重指纹池: {len(existing_fps)} 个") # ====== 参数 ====== - TARGET = 100000 # 目标 10 万条全新问答 - BATCH_SIZE = 10 # 每批次生成 10 条 + TARGET = 500000 # 目标 50 万条全新问答 + BATCH_SIZE = 15 # 每批次生成 15 条（更多问题，效率更高） BATCHES_PER_SAVE = 20 # 每 20 批次保存一次 MAX_WORKERS = 5 # Astron 5并发 diff --git a/generate_multiturn.py b/generate_multiturn.py index 085c915..1cdd73f 100644 --- a/generate_multiturn.py +++ b/generate_multiturn.py @@ -141,8 +141,8 @@ def parse_multi_turn(text): turns.append((current_q, a_text)) current_q = None # 配对后重置，等待下一个问 - # 最少2轮才算有效 - if len(turns) >= 2: + # 最少4轮才算有效（6~12轮的长对话） + if len(turns) >= 4: return turns return None @@ -173,19 +173,20 @@ def make_multi_prompt(): topics = random.sample(all_seeds, min(topic_count, len(all_seeds))) topic_context = "、".join(topics) - turn_count = random.randint(2, 5) + turn_count = random.randint(6, 12) - prompt = f"""请以小狸的身份，模拟一段{turn_count}轮的自然闲聊对话。 + prompt = f"""请以小狸的身份，模拟一段长时间的自然闲聊对话，至少{turn_count}轮。 开场话题：{topic_context} 要求： 1. 小狸是8岁小女孩，说话简短自然 2. 每轮包含"问"和"答"，小狸的回答1-2句话 -3. 对话要自然展开，不要强行切话题 +3. 对话要自然展开，一问一答慢慢深入，不要强行切话题 4. 小狸的语气：害羞但可爱，不是机器人 +5. 至少{turn_count}轮，越多越好，让对话持续下去 -请按以下格式输出（至少{turn_count}轮）： +请按以下格式输出（至少{turn_count}轮，轮数越多越好）： 问：（用户提问） 答：（小狸回答） 问：（用户续问） @@ -200,7 +201,7 @@ def generate_one_multi(): result = call_astron_multi(prompt, temperature=random.uniform(0.8, 1.0)) if result: turns = parse_multi_turn(result) - if turns and len(turns) >= 2: + if turns and len(turns) >= 4: sample = format_training_sample(turns) return { 'source': 'multiturn_astron', @@ -249,8 +250,8 @@ def main(): print(f"已有中间结果: {len(all_results)} 条 ({len(interim_files)} 批)") # ====== 参数 ====== - TARGET = 20000 # 目标生成 2 万条多轮对话 - BATCH = 50 # 每批生成 50 条 + TARGET = 500000 # 目标 50 万条长多轮对话 + BATCH = 20 # 每批生成 20 条（每条更长，减少批量） MAX_WORKERS = 5 # Astron 5并发 TEMPERATURE_RANGE = (0.80, 1.0) diff --git a/regenerate_all_high_temp.py b/regenerate_all_high_temp.py index cdcb91d..d97214f 100644 --- a/regenerate_all_high_temp.py +++ b/regenerate_all_high_temp.py @@ -111,30 +111,12 @@ def main(): print(f"生成结果: {len(all_results)} 条 (去重后)") print(f"用时: {(time.time() - t_start)/60:.1f} 分钟") - # 5. 追加到训练集 - for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']: - tp = os.path.join(MODEL_DIR, tgt) - print(f"\n追加到 {tgt}...") - with open(tp, encoding='utf-8') as f: - cur = json.load(f) - - # 去重 - ex = set((it.get('question', '')[:100], it.get('answer', '')[:100]) for it in cur) - to_add = [] - for r in all_results: - key = (r['question'][:100], r['answer'][:100]) - if key not in ex: - ex.add(key) - to_add.append({'question': r['question'], 'answer': r['answer']}) - - cur.extend(to_add) - with open(tp, 'w', encoding='utf-8') as f: - json.dump(cur, f, ensure_ascii=False, indent=2) - - if 'train' in tgt: - print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条") - else: - print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条") + # 5. 保存最终去重结果（独立存放，不追加到训练集） + final_file = os.path.join('/data/user/work', 'high_temp_final.json') + with open(final_file, 'w', encoding='utf-8') as f: + json.dump(all_results, f, ensure_ascii=False, indent=2) + print(f"\n最终结果已保存: {final_file}") + print(f"如需使用，可手动合并到训练集") print(f"\n{'='*60}") print(f"全部完成! 新增 {len(all_results)} 条多样化回答")