diff --git a/api_correction.py b/api_correction.py index 09b2572..dabe22b 100644 --- a/api_correction.py +++ b/api_correction.py @@ -7,7 +7,7 @@ import json, requests, time, os, threading from concurrent.futures import ThreadPoolExecutor, as_completed ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" -ASTRON_KEY = "658beba1beb0c911ed00672019bbf7e5:MjNmYWRkMWUyZjAxYTM3ZGE2ODI1YzFh" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") ASTRON_MODEL = "astron-code-latest" SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 diff --git a/generate_diverse_answers.py b/generate_diverse_answers.py new file mode 100644 index 0000000..9d9d1ce --- /dev/null +++ b/generate_diverse_answers.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +多样化回答生成 — 对每个问题用 Astron 高温生成 50 个不同回答 +5并发,若重复则重生成,增量保存防丢 +""" +import json, requests, time, os, sys, random, hashlib +from concurrent.futures import ThreadPoolExecutor, as_completed +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) + +# ====== Astron 配置 ====== +ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") +ASTRON_MODEL = "astron-code-latest" + +SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 + +你的人设: +- 8岁女孩子,有点害羞但很可爱 +- 说话简单自然,符合小朋友的语气 +- 回答问题要简短,1-2句话就够了 +- 不要用"喵"结尾,不要过度卖萌 + +回答要求: +1. 语气自然,像真实的小朋友说话 +2. 不懂就说"不知道" +3. 回答简洁但要说清楚,不要讲一半就停""" + + +def call_astron(q, temperature): + """调用 Astron,返回回答或 None""" + for attempt in range(3): + try: + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} + payload = { + "model": ASTRON_MODEL, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": q} + ], + "temperature": temperature, + "max_tokens": 2048, + } + resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) + if resp.status_code == 200: + content = resp.json()['choices'][0]['message']['content'].strip() + if content and len(content) > 2: + return content + else: + time.sleep(1) + except Exception: + time.sleep(2) + return None + + +def answer_fingerprint(text): + """回答指纹 — 去空格标点后取前 30 字""" + clean = ''.join(text.split())[:30] + return hashlib.md5(clean.encode()).hexdigest() + + +def is_duplicate(new, existing): + """检查是否与已有回答重复(精确+模糊)""" + fp = answer_fingerprint(new) + if fp in existing: + return True + # 模糊:如果新回答是某个已有回答的子串或反之 + short_new = ''.join(new.split())[:20] + for old_fp, old_text in existing.items(): + short_old = ''.join(old_text.split())[:20] + if short_new in short_old or short_old in short_new: + return True if len(short_new) > 8 else False + return False + + +def generate_answers_for_question(q, target=50, max_attempts=100, temperature=0.95): + """为单个问题生成 target 个不同回答""" + answers = {} # fp -> full_text + attempt = 0 + t_start = time.time() + + while len(answers) < target and attempt < max_attempts: + # 温度在 0.85~1.0 之间随机波动,避免卡在同一模式 + temp = temperature + random.uniform(-0.1, 0.05) + temp = max(0.85, min(1.0, temp)) + + result = call_astron(f"问题:{q}\n\n请以小狸的身份回答:", temp) + if result: + if not is_duplicate(result, answers): + fp = answer_fingerprint(result) + answers[fp] = result + + attempt += 1 + + if attempt % 20 == 0: + elapsed = time.time() - t_start + rate = len(answers) / max(elapsed, 1) + if len(answers) < target and rate > 0: + eta = (target - len(answers)) / rate + else: + eta = 0 + print(f" [{len(answers)}/{target}] attempt={attempt}, {rate:.1f}条/秒, 预计剩余{eta:.0f}秒") + sys.stdout.flush() + + return list(answers.values()) + + +def main(): + model_dir = '/workspace/xiaoli-llm-chat' + output_dir = os.path.join(model_dir, 'diverse_answers') + os.makedirs(output_dir, exist_ok=True) + + # ====== 参数(可调) ====== + QUESTIONS_PER_BATCH = 100 # 每批处理多少问题 + ANSWERS_PER_QUESTION = 50 # 每个问题生成多少个不同回答 + MAX_ATTEMPTS = 150 # 最多尝试多少次(超过就放弃) + MAX_WORKERS = 5 # Astron 5并发 + TEMPERATURE = 0.95 # 高温(0.85~1.0 浮动) + + # ====== 加载问题列表 ====== + train_path = os.path.join(model_dir, 'xiaoli_train_v2.json') + with open(train_path, encoding='utf-8') as f: + data = json.load(f) + + # 取所有不重复的问题 + all_questions = list(set(item.get('question', '') for item in data)) + random.shuffle(all_questions) + print(f"训练集总条数: {len(data)}") + print(f"不重复问题数: {len(all_questions)}") + + # 统计问题出现频次,优先处理高频问题 + q_freq = {} + for item in data: + q = item.get('question', '') + q_freq[q] = q_freq.get(q, 0) + 1 + + # 按频次排序(高频靠前,低频靠后,随机打乱同频) + freq_buckets = {} + for q in all_questions: + f = q_freq.get(q, 1) + freq_buckets.setdefault(f, []).append(q) + + sorted_questions = [] + for freq in sorted(freq_buckets.keys(), reverse=True): + bucket = freq_buckets[freq] + random.shuffle(bucket) + sorted_questions.extend(bucket) + + print(f"按频次排序完成 (高频优先处理)") + + # ====== 恢复已有进度 ====== + progress_file = os.path.join(output_dir, 'diversity_progress.json') + completed = {} # question -> list of answers + if os.path.exists(progress_file): + with open(progress_file, encoding='utf-8') as f: + completed = json.load(f) + # 过滤掉已处理的问题 + processed_qs = set(completed.keys()) + sorted_questions = [q for q in sorted_questions if q not in processed_qs] + print(f"已有进度: {len(completed)} 个问题已完成") + print(f"剩余未处理: {len(sorted_questions)} 个问题") + + # 限制要处理的问题数 + max_q = len(sorted_questions) # 全部问题 + + total_questions = len(sorted_questions) + total_batches = (total_questions + QUESTIONS_PER_BATCH - 1) // QUESTIONS_PER_BATCH + + print(f"\n目标: {total_questions} 个问题 × {ANSWERS_PER_QUESTION} 个回答 = {total_questions * ANSWERS_PER_QUESTION} 条") + print(f"批次: {total_batches} 批 (每批 ≤{QUESTIONS_PER_BATCH} 问题)") + print(f"并发: {MAX_WORKERS}") + if total_questions > 1000: + minutes = total_questions * ANSWERS_PER_QUESTION / (MAX_WORKERS * 8) / 60 + print(f"预估耗时: ~{minutes:.0f} 分钟 (~{minutes/60:.1f} 小时)") + print(f"{'='*70}\n") + + t_start = time.time() + total_new_pairs = 0 + + for batch_idx in range(total_batches): + batch_qs = sorted_questions[batch_idx * QUESTIONS_PER_BATCH : (batch_idx + 1) * QUESTIONS_PER_BATCH] + batch_results = {} + + print(f"\n批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)} 个问题)") + + # 并发生成 + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = { + executor.submit(generate_answers_for_question, q, ANSWERS_PER_QUESTION, MAX_ATTEMPTS, TEMPERATURE): q + for q in batch_qs + } + + for future in as_completed(futures): + q = futures[future] + try: + answers = future.result() + if answers: + batch_results[q] = answers + print(f" ✅ 『{q[:30]}』 → {len(answers)} 个不同回答") + else: + print(f" ❌ 『{q[:30]}』 → 生成失败") + except Exception as e: + print(f" ❌ 『{q[:30]}』 → 错误: {e}") + + # 追加到总进度 + completed.update(batch_results) + batch_new_pairs = sum(len(v) for v in batch_results.values()) + total_new_pairs += batch_new_pairs + + # 保存进度 + with open(progress_file, 'w', encoding='utf-8') as f: + json.dump(completed, f, ensure_ascii=False, indent=2) + + elapsed = time.time() - t_start + rate = total_new_pairs / max(elapsed, 1) * 60 + + # 统计 + print(f"\n批次 {batch_idx + 1} 完成 | " + f"本批新增 {batch_new_pairs} 条 | " + f"累计 {total_new_pairs:,} 条 ({len(completed)} 问题) | " + f"{rate:.0f}条/分 | " + f"已用 {elapsed/60:.1f}分") + + # ====== 完成 ====== + elapsed_total = time.time() - t_start + print(f"\n{'='*70}") + print(f"全部生成完成!") + print(f"处理问题: {len(completed)} 个") + print(f"总回答数: {total_new_pairs:,} 条") + print(f"用时: {elapsed_total/60:.1f} 分钟") + + # ====== 保存为训练格式 ====== + train_format = [] + for q, answers in completed.items(): + for a in answers: + train_format.append({'question': q, 'answer': a}) + + output_file = os.path.join(output_dir, 'diverse_train_data_v1.json') + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(train_format, f, ensure_ascii=False, indent=2) + + # 也保存一份到工作目录 + work_file = '/data/user/work/diverse_train_data_v1.json' + with open(work_file, 'w', encoding='utf-8') as f: + json.dump(train_format, f, ensure_ascii=False, indent=2) + + print(f"\n结果保存:") + print(f" 📄 {output_file}") + print(f" 📄 {work_file}") + print(f" 📊 平均每问题 {total_new_pairs/max(len(completed),1):.1f} 个回答") + print(f"\n如需合并到训练集:") + print(f" python3 -c \"import json; d1=json.load(open('xiaoli_train_v2.json')); d2=json.load(open('{output_file}')); d1.extend(d2); json.dump(d1, open('xiaoli_train_v2.json','w'), ensure_ascii=False, indent=2)\"") + print(f"{'='*70}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/generate_diverse_single.py b/generate_diverse_single.py new file mode 100644 index 0000000..2218106 --- /dev/null +++ b/generate_diverse_single.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +高温度多样化生成 — 单线程,与常规跑批共用 Astron 5并发中的1个 +对已完成批1-6的问题,用高温度重新生成,获得多样回答 +""" +import sys, os, json, random, time, requests +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) + +ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") +ASTRON_MODEL = "astron-code-latest" +MODEL_DIR = os.path.dirname(os.path.abspath(__file__)) + +SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 + +你的人设: +- 8岁女孩子,有点害羞但很可爱 +- 说话简单自然,符合小朋友的语气 +- 回答问题要简短,1-2句话就够了 +- 不要用"喵"结尾,不要过度卖萌 + +回答要求: +1. 语气自然,像真实的小朋友说话 +2. 不懂就说"不知道" +3. 回答简洁但要说清楚,不要讲一半就停""" + + +def call_astron(q, temperature): + for attempt in range(3): + try: + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} + payload = { + "model": ASTRON_MODEL, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": q} + ], + "temperature": temperature, + "max_tokens": 8192, + } + resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) + if resp.status_code == 200: + content = resp.json()['choices'][0]['message']['content'].strip() + if content: + return content + else: + time.sleep(1) + except Exception: + time.sleep(2) + return None + + +def main(): + # 从已完成批载入问题 + all_questions = [] + for i in range(1, 7): + fp = os.path.join(MODEL_DIR, f'mega_interim_{i}.json') + if os.path.exists(fp): + with open(fp) as f: + items = json.load(f) + for item in items: + all_questions.append(item['question']) + print(f"批{i}: {len(items)}条已载入") + + # 去重 + all_questions = list(dict.fromkeys(all_questions)) + print(f"共 {len(all_questions)} 个去重问题") + + # 温度配置 + temp_configs = [0.9, 1.3, 1.5] + all_new = [] + + for round_idx, temp in enumerate(temp_configs): + print(f"\n{'='*50}") + print(f"轮{round_idx+1}/3: 温度={temp}") + print(f"{'='*50}") + + # 单线程顺序跑 + results = [] + total = len(all_questions) + t0 = time.time() + + for idx, q in enumerate(all_questions): + ans = call_astron(q, temp) + if ans: + results.append({"question": q, "answer": ans}) + + if (idx + 1) % 20 == 0 or idx + 1 == total: + elapsed = max(time.time() - t0, 1) + rate = len(results) / elapsed * 60 + eta = (total - idx - 1) / max(idx + 1, 1) * elapsed / 60 + print(f" 多样: {idx+1}/{total} (已获{len(results)}条, {rate:.0f}条/分, ETA:{eta:.0f}分)", end="\r") + + elapsed = time.time() - t0 + rate = len(results) / max(elapsed, 1) * 60 + print(f"\n 轮{round_idx+1}完成: {len(results)}条 ({elapsed:.0f}s, {rate:.0f}条/分)") + + for r in results: + r['temperature'] = temp + r['round'] = round_idx + 1 + all_new.extend(results) + + # ✅ 每轮跑完立即去重并追加到训练集 + seen = set() + unique = [] + for r in all_new: + key = (r['question'][:100], r['answer'][:100]) + if key not in seen: + seen.add(key) + unique.append(r) + + for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']: + tp = os.path.join(MODEL_DIR, tgt) + with open(tp, encoding='utf-8') as f: + cur = json.load(f) + ex = set((it.get("question","")[:100], it.get("answer","")[:100]) for it in cur) + to_add = [] + for r in unique: + key = (r['question'][:100], r['answer'][:100]) + if key not in ex: + ex.add(key) + to_add.append({"question": r['question'], "answer": r['answer']}) + cur.extend(to_add) + with open(tp, 'w', encoding='utf-8') as f: + json.dump(cur, f, ensure_ascii=False, indent=2) + if 'train' in tgt: + print(f"✅ 轮{round_idx+1} 训练集 +{len(to_add)}条 → {len(cur)}条") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/generate_fresh_qa.py b/generate_fresh_qa.py new file mode 100644 index 0000000..8427f20 --- /dev/null +++ b/generate_fresh_qa.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +全新问答对生成 — Astron 凭空造(新问题+新回答),5并发 +不依赖现有问题列表,从种子话题衍生 +增量保存防丢 +""" +import json, requests, time, os, sys, random, hashlib +from concurrent.futures import ThreadPoolExecutor, as_completed +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) + +# ====== Astron 配置 ====== +ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") +ASTRON_MODEL = "astron-code-latest" + +SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 + +你的人设: +- 8岁女孩子,有点害羞但很可爱 +- 说话简单自然,符合小朋友的语气 +- 回答问题要简短,1-2句话就够了 +- 不要用"喵"结尾,不要过度卖萌 + +回答要求: +1. 语气自然,像真实的小朋友说话 +2. 不懂就说"不知道" +3. 回答简洁但要说清楚""" + +# ====== 种子场景(生成对话的起点) ====== +SCENES = [ + # 日常场景 + ("问候", ["打招呼", "问好", "早上起来", "放学回家", "见到朋友"]), + ("情绪", ["开心", "难过", "生气", "无聊", "害怕", "惊喜", "委屈", "害羞"]), + ("分享", ["今天发生的事", "周末做了什么", "假期计划", "新玩具", "考试成绩"]), + ("喜好", ["喜欢的颜色", "喜欢的动物", "喜欢的食物", "喜欢的动画片", "喜欢的游戏"]), + ("好奇", ["为什么问题", "想知道的事", "不懂的事", "想象的事", "奇怪的问题"]), + ("日常", ["起床", "刷牙", "吃饭", "上学", "做作业", "睡觉"]), + ("朋友", ["好朋友", "吵架", "一起玩", "分享秘密", "帮忙"]), + ("家庭", ["爸爸妈妈", "爷爷奶奶", "兄弟姐妹", "家人一起", "家庭活动"]), + ("想象", ["童话故事", "梦想", "如果我有魔法", "长大想做什么", "幻想朋友"]), + ("安慰", ["受伤了", "丢东西", "被骂了", "考试没考好", "被欺负", "做噩梦"]), + ("节日", ["生日", "过年", "圣诞节", "儿童节", "中秋"]), + ("自然", ["天气", "动物", "植物", "星空", "大海", "四季"]), + ("学校", ["老师", "同学", "上课", "课间", "课外活动", "考试"]), + ("晚安", ["说晚安", "睡前", "梦", "明天见", "晚安故事"]), +] + +def call_astron(prompt, temperature=0.9): + for attempt in range(3): + try: + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} + payload = { + "model": ASTRON_MODEL, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt} + ], + "temperature": temperature, + "max_tokens": 4096, + } + resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) + if resp.status_code == 200: + content = resp.json()['choices'][0]['message']['content'].strip() + if content and len(content) > 10: + return content + else: + time.sleep(1) + except Exception: + time.sleep(2) + return None + + +def parse_qa_pairs(text): + """解析 Astron 返回的问答对列表""" + lines = text.strip().split('\n') + pairs = [] + current_q = None + for line in lines: + line = line.strip() + if not line: + continue + q_match = __import__('re').match(r'^[Q问][::]\s*(.*)', line) + a_match = __import__('re').match(r'^[A答][::]\s*(.*)', line) + if q_match: + current_q = q_match.group(1).strip() + elif a_match and current_q: + a_text = a_match.group(1).strip() + if current_q and a_text and len(a_text) > 2 and len(current_q) > 2: + pairs.append((current_q, a_text)) + current_q = None + return pairs + + +def pair_fingerprint(q, a): + """QA对指纹""" + raw = f"{q[:40]}|{a[:40]}" + return hashlib.md5(raw.encode()).hexdigest() + + +def generate_batch_pairs(batch_size=10, existing_fps=None): + """生成一批全新的问答对""" + if existing_fps is None: + existing_fps = set() + + # 随机选场景 + category, sub_scenes = random.choice(SCENES) + sub = random.choice(sub_scenes) + # 随机生成要求 + style = random.choice(["日常聊天", "撒娇可爱", "有点害羞", "好奇宝宝", "暖心安慰"]) + mood = random.choice(["开心", "平静", "有点小情绪", "好奇", "温暖"]) + + prompt = f"""请以小狸的身份,生成{batch_size}个不同的日常对话问答对。 + +对话背景: +- 场景:{category}({sub}) +- 风格:{style} +- 小狸心情:{mood} + +要求: +1. 每个问答对包含一问一答,问是用户说的,答是小狸回答 +2. 问要自然,像真实用户会说的话(不要提"小狸") +3. 答要是小狸的风格,1-2句话,简短自然 +4. {batch_size}个问答要对同一个话题但角度不同 +5. 不要重复一模一样的问法 + +请按以下格式输出: +Q:用户提问内容 +A:小狸回答内容 +---(分隔) +Q:用户提问内容 +A:小狸回答内容 +---""" + + result = call_astron(prompt, temperature=random.uniform(0.85, 1.0)) + if not result: + return [] + + pairs = parse_qa_pairs(result) + + # 去重 + deduped = [] + for q, a in pairs: + fp = pair_fingerprint(q, a) + if fp not in existing_fps: + existing_fps.add(fp) + deduped.append({'question': q, 'answer': a}) + + return deduped + + +def main(): + output_dir = '/workspace/xiaoli-llm-chat/fresh_qa_outputs' + os.makedirs(output_dir, exist_ok=True) + + # ====== 恢复进度 ====== + progress_file = os.path.join(output_dir, 'fresh_qa_progress.json') + all_pairs = [] + if os.path.exists(progress_file): + with open(progress_file, encoding='utf-8') as f: + all_pairs = json.load(f) + print(f"已有进度: {len(all_pairs)} 条") + + # 加载已有训练集做去重 + train_path = '/workspace/xiaoli-llm-chat/xiaoli_train_v2.json' + with open(train_path, encoding='utf-8') as f: + train_data = json.load(f) + existing_fps = set(pair_fingerprint(item.get('question',''), item.get('answer','')) for item in train_data) + for p in all_pairs: + existing_fps.add(pair_fingerprint(p['question'], p['answer'])) + print(f"去重指纹池: {len(existing_fps)} 个") + + # ====== 参数 ====== + TARGET = 100000 # 目标 10 万条全新问答 + BATCH_SIZE = 10 # 每批次生成 10 条 + BATCHES_PER_SAVE = 20 # 每 20 批次保存一次 + MAX_WORKERS = 5 # Astron 5并发 + + remaining = TARGET - len(all_pairs) + total_iterations = (remaining + BATCH_SIZE - 1) // BATCH_SIZE + print(f"目标: {TARGET:,} 条, 还需: {remaining:,} 条") + print(f"轮次: {total_iterations} 轮 (每轮{BATCH_SIZE}条, {MAX_WORKERS}并发)") + print(f"{'='*60}\n") + + t_start = time.time() + batch_counter = 0 + + for iteration in range(total_iterations): + t0 = time.time() + + # 并发生成 + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = [ + executor.submit(generate_batch_pairs, BATCH_SIZE, existing_fps) + for _ in range(MAX_WORKERS) + ] + for future in as_completed(futures): + try: + new_pairs = future.result() + if new_pairs: + for p in new_pairs: + existing_fps.add(pair_fingerprint(p['question'], p['answer'])) + all_pairs.append(p) + except Exception: + pass + + batch_counter += 1 + elapsed = time.time() - t_start + rate = len(all_pairs) / max(elapsed, 1) * 60 + + # 定期保存 + if batch_counter % BATCHES_PER_SAVE == 0 or iteration == total_iterations - 1: + with open(progress_file, 'w', encoding='utf-8') as f: + json.dump(all_pairs, f, ensure_ascii=False, indent=2) + + eta = (TARGET - len(all_pairs)) / max(rate, 0.1) / 60 if rate > 0 else 0 + + if iteration % 5 == 0 or iteration == total_iterations - 1: + print(f"轮 {iteration+1}/{total_iterations} | " + f"累计 {len(all_pairs):,}/{TARGET:,} | " + f"{rate:.0f}条/分 | " + f"已用 {elapsed/60:.1f}分 | " + f"预计剩余 {eta:.1f}分") + + # ====== 完成 ====== + elapsed_total = time.time() - t_start + print(f"\n{'='*60}") + print(f"全新问答生成完成!") + print(f"总数: {len(all_pairs):,} 条") + print(f"用时: {elapsed_total/60:.1f} 分钟") + print(f"速率: {len(all_pairs)/max(elapsed_total,1)*60:.0f} 条/分") + + # 保存最终 + final_path = '/data/user/work/fresh_qa_v1.json' + with open(final_path, 'w', encoding='utf-8') as f: + json.dump(all_pairs, f, ensure_ascii=False, indent=2) + print(f"保存到: {final_path}") + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/generate_multiturn.py b/generate_multiturn.py new file mode 100644 index 0000000..085c915 --- /dev/null +++ b/generate_multiturn.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +多轮对话生成 — 用 Astron 造 2~5 轮小狸闲聊对话 +5并发,无限量,增量保存防丢 +""" +import json, requests, time, os, sys, random, re +from concurrent.futures import ThreadPoolExecutor, as_completed +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) + +# ====== Astron 配置(和 api_correction 一致) ====== +ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions" +ASTRON_KEY = os.environ.get("ASTRON_KEY", "") +ASTRON_MODEL = "astron-code-latest" + +# ====== 小狸基础提示词 ====== +SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。 + +你的人设: +- 8岁女孩子,有点害羞但很可爱 +- 说话简单自然,符合小朋友的语气 +- 回答问题要简短,1-2句话就够了 +- 不要用"喵"结尾,不要过度卖萌 + +回答要求: +1. 语气自然,像真实的小朋友说话 +2. 不懂就说"不知道" +3. 回答简洁但要说清楚""" + +# ====== 多轮对话种子(开场话题池) ====== +SEEDS = [ + # 日常问候 + "你好呀", "你叫什么名字", "你今天开心吗", "你在干嘛呢", + "你好可爱呀", "我来看你啦", "你今天心情怎么样", + # 闲聊话题 + "你喜欢什么颜色", "你最喜欢什么动物", "你爱吃什么零食", + "你最喜欢什么玩具", "你最喜欢什么动画片", + "你最好的朋友是谁", "幼儿园里好玩吗", + "你喜欢唱歌吗", "你会跳舞吗", + # 情绪表达 + "我今天好开心", "我有点不开心", "我好无聊啊", + "我遇到了一件好事", "我今天有点难过", + "我好想你呀", "你能不能陪我聊聊天", + # 好奇问题 + "为什么天是蓝的", "为什么月亮跟着我走", + "世界上最大的动物是什么", "你会讲故事吗", + "你会数数吗", "1+1等于几", + "你喜欢上学吗", "你今年几岁啦", + # 分享日常 + "我今天去公园玩了", "我养了一只小猫", + "我今天考试考了100分", "我学会骑自行车了", + "妈妈给我买了个新玩具", "我今天摔了一跤", + # 晚安/告别 + "我要睡觉了", "明天见", "晚安啦", + "我要去吃饭了", "下次再聊哦", +] + +# 话题扩展 — 组合出更多种子 +PREFIXES = ["小狸", "小狸小狸", "亲爱的小狸", "嘿嘿小狸", "小狸呀"] +SUFFIXES = ["呀", "吗", "呢", "", "呀", "吗", "呢", ""] + +def expand_seeds(base, max_expand=5): + """种子话题 → 多样化的开场白""" + expanded = [] + for s in base: + expanded.append(s) + if '吗' not in s and not s.endswith('呀') and not s.endswith('呢'): + for _ in range(2): + prefix = random.choice(PREFIXES) if random.random() > 0.4 else '' + suffix = random.choice(SUFFIXES) if random.random() > 0.5 else '' + combo = f"{prefix}{s}{suffix}".strip() + if combo and combo not in expanded: + expanded.append(combo) + random.shuffle(expanded) + return expanded[:len(base) * max_expand] + + +def call_astron_multi(prompt, temperature=0.85): + """调用 Astron 生成一条多轮对话(2~5轮)""" + for attempt in range(3): + try: + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"} + payload = { + "model": ASTRON_MODEL, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt} + ], + "temperature": temperature, + "max_tokens": 8192, + } + resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60) + if resp.status_code == 200: + content = resp.json()['choices'][0]['message']['content'].strip() + if content and len(content) > 10: + return content + else: + time.sleep(1) + except Exception: + time.sleep(2) + return None + + +def parse_multi_turn(text): + """ + 解析 Astron 返回的多轮对话文本为 [(q, a), (q, a), ...] + 期望返回格式(至少2轮对话): + 问:xxx + 答:xxx + 问:xxx + 答:xxx + + 或 + 用户:xxx + 小狸:xxx + 用户:xxx + 小狸:xxx + """ + lines = text.strip().split('\n') + turns = [] + current_q = None + + for line in lines: + line = line.strip() + if not line: + continue + + # 匹配 "问:" 或 "问题:" 或 "用户:" 格式 + q_match = re.match(r'^(问|问题|用户|我)[::]\s*(.*)', line) + # 匹配 "答:" 或 "回答:" 或 "小狸:" 格式 + a_match = re.match(r'^(答|回答|小狸|小狸说|机器人)[::]\s*(.*)', line) + + if q_match: + q_text = q_match.group(2).strip() + if q_text and len(q_text) > 1: + current_q = q_text + elif a_match and current_q: + a_text = a_match.group(2).strip() + if a_text and len(a_text) > 1 and len(a_text) < 200: + turns.append((current_q, a_text)) + current_q = None # 配对后重置,等待下一个问 + + # 最少2轮才算有效 + if len(turns) >= 2: + return turns + return None + + +def format_training_sample(turns): + """ + 将多轮对话 [('q1','a1'), ('q2','a2'), ...] 格式化为训练样本 + + 格式:q1a1q2a2 + """ + parts = [] + for i, (q, a) in enumerate(turns): + if i == 0: + parts.append(f"{q}{a}") + else: + parts.append(f"{q}{a}") + parts.append("") + return "".join(parts) + + +def make_multi_prompt(): + """ + 构造多轮对话生成提示词 + Astron 每次生成一条完整的4~6轮对话 + """ + # 随机选开场话题(2~3个组合,让对话有展开) + topic_count = random.randint(1, 3) + topics = random.sample(all_seeds, min(topic_count, len(all_seeds))) + topic_context = "、".join(topics) + + turn_count = random.randint(2, 5) + + prompt = f"""请以小狸的身份,模拟一段{turn_count}轮的自然闲聊对话。 + +开场话题:{topic_context} + +要求: +1. 小狸是8岁小女孩,说话简短自然 +2. 每轮包含"问"和"答",小狸的回答1-2句话 +3. 对话要自然展开,不要强行切话题 +4. 小狸的语气:害羞但可爱,不是机器人 + +请按以下格式输出(至少{turn_count}轮): +问:(用户提问) +答:(小狸回答) +问:(用户续问) +答:(小狸回答) +...""" + return prompt + + +def generate_one_multi(): + """生成一条多轮对话""" + prompt = make_multi_prompt() + result = call_astron_multi(prompt, temperature=random.uniform(0.8, 1.0)) + if result: + turns = parse_multi_turn(result) + if turns and len(turns) >= 2: + sample = format_training_sample(turns) + return { + 'source': 'multiturn_astron', + 'turns': len(turns), + 'sample': sample, + 'turns_list': turns, + 'raw': result[:100] + } + return None + + +def main(): + global all_seeds + + model_dir = os.path.dirname(os.path.abspath(__file__)) + + # 加载现有训练数据的问句作为种子 + train_path = os.path.join(model_dir, 'xiaoli_train_v2.json') + with open(train_path, encoding='utf-8') as f: + train_data = json.load(f) + + # 从现有数据中随机抽取问句作为种子 + existing_qs = [item.get('question','') for item in train_data if item.get('question','')] + random.shuffle(existing_qs) + seed_from_data = existing_qs[:500] + + # 合并种子话题 + all_seeds = expand_seeds(SEEDS, max_expand=10) + seed_from_data + random.shuffle(all_seeds) + + print(f"种子话题: {len(all_seeds)} 个 (基础{len(SEEDS)}+数据抽取{len(seed_from_data)})") + + # 输出目录 + out_dir = os.path.join(model_dir, 'multiturn_outputs') + os.makedirs(out_dir, exist_ok=True) + + # 检查已有进度 + all_results = [] + interim_files = sorted([f for f in os.listdir(out_dir) + if f.startswith('mt_batch_') and f.endswith('.json')]) + if interim_files: + for fname in interim_files: + with open(os.path.join(out_dir, fname), encoding='utf-8') as f: + data = json.load(f) + all_results.extend(data) + print(f"已有中间结果: {len(all_results)} 条 ({len(interim_files)} 批)") + + # ====== 参数 ====== + TARGET = 20000 # 目标生成 2 万条多轮对话 + BATCH = 50 # 每批生成 50 条 + MAX_WORKERS = 5 # Astron 5并发 + TEMPERATURE_RANGE = (0.80, 1.0) + + to_generate = TARGET - len(all_results) + total_batches = (to_generate + BATCH - 1) // BATCH + + print(f"目标: {TARGET} 条, 已有: {len(all_results)} 条, 还需: {to_generate} 条") + print(f"批次: {total_batches} 批 (每批{BATCH}条, {MAX_WORKERS}并发)") + print(f"{'='*60}\n") + + t_start = time.time() + + for batch_idx in range(total_batches): + t0 = time.time() + + # 并发生成一批 + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: + futures = [executor.submit(generate_one_multi) for _ in range(BATCH)] + batch_results = [] + for future in as_completed(futures): + try: + r = future.result() + if r: + batch_results.append(r) + except Exception: + pass + + # 去重(按 sample 内容) + existing_samples = set(r['sample'] for r in all_results) + deduped = [] + for r in batch_results: + if r['sample'] not in existing_samples: + existing_samples.add(r['sample']) + deduped.append(r) + + all_results.extend(deduped) + + # 保存本批 + elapsed = time.time() - t0 + batch_file = os.path.join(out_dir, f'mt_batch_{batch_idx + 1:04d}.json') + with open(batch_file, 'w', encoding='utf-8') as f: + json.dump(deduped, f, ensure_ascii=False, indent=2) + + # 追加保存总进度 + progress_file = os.path.join(out_dir, 'all_multiturn_progress.json') + with open(progress_file, 'w', encoding='utf-8') as f: + json.dump(all_results, f, ensure_ascii=False, indent=2) + + total_elapsed = time.time() - t_start + rate = len(all_results) / max(total_elapsed, 1) * 60 + eta = (TARGET - len(all_results)) / max(rate, 0.1) / 60 if rate > 0 else 0 + + # 输出统计 + avg_turns = sum(r['turns'] for r in deduped) / max(len(deduped), 1) + print(f"批次 {batch_idx+1}/{total_batches} | " + f"+{len(deduped)} 条 (去重后) | " + f"累计 {len(all_results)}/{TARGET} | " + f"均轮 {avg_turns:.1f} | " + f"{rate:.0f}条/分 | " + f"预计剩余 {eta:.1f}分") + + # ====== 完成总结 ====== + total_elapsed = time.time() - t_start + print(f"\n{'='*60}") + print(f"多轮对话生成完成!") + print(f"生成: {len(all_results)} 条 (去重后)") + avg_turns2 = sum(r['turns'] for r in all_results) / max(len(all_results), 1) + print(f"平均轮数: {avg_turns2:.1f}") + print(f"总对话轮次: {sum(r['turns'] for r in all_results)}") + print(f"用时: {total_elapsed/60:.1f} 分钟") + print(f"速率: {len(all_results)/max(total_elapsed,1)*60:.0f} 条/分") + + # ====== 保存最终结果(独立存放,不追加到训练集) ====== + final_file = os.path.join('/data/user/work', 'multiturn_final.json') + with open(final_file, 'w', encoding='utf-8') as f: + json.dump(all_results, f, ensure_ascii=False, indent=2) + print(f"\n最终结果已保存: {final_file} ({len(all_results)} 条)") + print(f"如需后续使用,可手动合并到训练集") + + # ====== 保存多轮样本供训练时直接使用(含标记格式) ====== + mt_samples_file = os.path.join(model_dir, 'multiturn_samples_v1.json') + with open(mt_samples_file, 'w', encoding='utf-8') as f: + samples = [{'turns': r['turns'], 'sample': r['sample']} for r in all_results] + json.dump(samples, f, ensure_ascii=False, indent=2) + print(f" ✅ 多轮样本保存: multiturn_samples_v1.json ({len(samples)} 条)") + + print(f"\n{'='*60}") + print(f"全部完成! 新增 {len(all_results)} 条多轮对话") + print(f"{'='*60}") + +if __name__ == '__main__': + main() \ No newline at end of file