同步代码更新 (2026-06-16)
- 移除 generate_diverse_answers.py（多样化回答已废止） - 更新 generate_fresh_qa.py（目标50万条新问答） - 更新 generate_multiturn.py（6~12轮长对话，目标50万） - 更新 regenerate_all_high_temp.py（独立保存，不追加训练集） - 更新 .gitignore（忽略新输出目录） - 所有API Key改为环境变量读取
This commit is contained in:
shuiyu1123 committed
1 parent
eb1d62f85c
commit
91db488ad4
5 files changed
+21
-291
No files matched your search
@@ -11,6 +11,9 @@ mega_interim_*.json
|
||||
essay_interim_*.json
|
||||
checkpoints_v2/
|
||||
high_temp_outputs/
|
||||
diverse_answers/
|
||||
fresh_qa_outputs/
|
||||
multiturn_outputs/
|
||||
old_ckpts/
|
||||
backup_old_arch/
|
||||
|
||||
|
||||
@@ -1,256 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
多样化回答生成 — 对每个问题用 Astron 高温生成 50 个不同回答
|
||||
5并发，若重复则重生成，增量保存防丢
|
||||
"""
|
||||
import json, requests, time, os, sys, random, hashlib
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
||||
|
||||
# ====== Astron 配置 ======
|
||||
ASTRON_URL = "https://maas-coding-api.cn-huabei-1.xf-yun.com/v2/chat/completions"
|
||||
ASTRON_KEY = os.environ.get("ASTRON_KEY", "")
|
||||
ASTRON_MODEL = "astron-code-latest"
|
||||
|
||||
SYSTEM_PROMPT = """你是一个名叫小狸的8岁中国小女孩。
|
||||
|
||||
你的人设：
|
||||
- 8岁女孩子，有点害羞但很可爱
|
||||
- 说话简单自然，符合小朋友的语气
|
||||
- 回答问题要简短，1-2句话就够了
|
||||
- 不要用"喵"结尾，不要过度卖萌
|
||||
|
||||
回答要求：
|
||||
1. 语气自然，像真实的小朋友说话
|
||||
2. 不懂就说"不知道"
|
||||
3. 回答简洁但要说清楚，不要讲一半就停"""
|
||||
|
||||
|
||||
def call_astron(q, temperature):
|
||||
"""调用 Astron，返回回答或 None"""
|
||||
for attempt in range(3):
|
||||
try:
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {ASTRON_KEY}"}
|
||||
payload = {
|
||||
"model": ASTRON_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": q}
|
||||
],
|
||||
"temperature": temperature,
|
||||
"max_tokens": 2048,
|
||||
}
|
||||
resp = requests.post(ASTRON_URL, headers=headers, json=payload, timeout=60)
|
||||
if resp.status_code == 200:
|
||||
content = resp.json()['choices'][0]['message']['content'].strip()
|
||||
if content and len(content) > 2:
|
||||
return content
|
||||
else:
|
||||
time.sleep(1)
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
return None
|
||||
|
||||
|
||||
def answer_fingerprint(text):
|
||||
"""回答指纹 — 去空格标点后取前 30 字"""
|
||||
clean = ''.join(text.split())[:30]
|
||||
return hashlib.md5(clean.encode()).hexdigest()
|
||||
|
||||
|
||||
def is_duplicate(new, existing):
|
||||
"""检查是否与已有回答重复（精确+模糊）"""
|
||||
fp = answer_fingerprint(new)
|
||||
if fp in existing:
|
||||
return True
|
||||
# 模糊：如果新回答是某个已有回答的子串或反之
|
||||
short_new = ''.join(new.split())[:20]
|
||||
for old_fp, old_text in existing.items():
|
||||
short_old = ''.join(old_text.split())[:20]
|
||||
if short_new in short_old or short_old in short_new:
|
||||
return True if len(short_new) > 8 else False
|
||||
return False
|
||||
|
||||
|
||||
def generate_answers_for_question(q, target=50, max_attempts=100, temperature=0.95):
|
||||
"""为单个问题生成 target 个不同回答"""
|
||||
answers = {} # fp -> full_text
|
||||
attempt = 0
|
||||
t_start = time.time()
|
||||
|
||||
while len(answers) < target and attempt < max_attempts:
|
||||
# 温度在 0.85~1.0 之间随机波动，避免卡在同一模式
|
||||
temp = temperature + random.uniform(-0.1, 0.05)
|
||||
temp = max(0.85, min(1.0, temp))
|
||||
|
||||
result = call_astron(f"问题：{q}\n\n请以小狸的身份回答：", temp)
|
||||
if result:
|
||||
if not is_duplicate(result, answers):
|
||||
fp = answer_fingerprint(result)
|
||||
answers[fp] = result
|
||||
|
||||
attempt += 1
|
||||
|
||||
if attempt % 20 == 0:
|
||||
elapsed = time.time() - t_start
|
||||
rate = len(answers) / max(elapsed, 1)
|
||||
if len(answers) < target and rate > 0:
|
||||
eta = (target - len(answers)) / rate
|
||||
else:
|
||||
eta = 0
|
||||
print(f" [{len(answers)}/{target}] attempt={attempt}, {rate:.1f}条/秒, 预计剩余{eta:.0f}秒")
|
||||
sys.stdout.flush()
|
||||
|
||||
return list(answers.values())
|
||||
|
||||
|
||||
def main():
|
||||
model_dir = '/workspace/xiaoli-llm-chat'
|
||||
output_dir = os.path.join(model_dir, 'diverse_answers')
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# ====== 参数（可调） ======
|
||||
QUESTIONS_PER_BATCH = 100 # 每批处理多少问题
|
||||
ANSWERS_PER_QUESTION = 50 # 每个问题生成多少个不同回答
|
||||
MAX_ATTEMPTS = 150 # 最多尝试多少次（超过就放弃）
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
TEMPERATURE = 0.95 # 高温（0.85~1.0 浮动）
|
||||
|
||||
# ====== 加载问题列表 ======
|
||||
train_path = os.path.join(model_dir, 'xiaoli_train_v2.json')
|
||||
with open(train_path, encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# 取所有不重复的问题
|
||||
all_questions = list(set(item.get('question', '') for item in data))
|
||||
random.shuffle(all_questions)
|
||||
print(f"训练集总条数: {len(data)}")
|
||||
print(f"不重复问题数: {len(all_questions)}")
|
||||
|
||||
# 统计问题出现频次，优先处理高频问题
|
||||
q_freq = {}
|
||||
for item in data:
|
||||
q = item.get('question', '')
|
||||
q_freq[q] = q_freq.get(q, 0) + 1
|
||||
|
||||
# 按频次排序（高频靠前，低频靠后，随机打乱同频）
|
||||
freq_buckets = {}
|
||||
for q in all_questions:
|
||||
f = q_freq.get(q, 1)
|
||||
freq_buckets.setdefault(f, []).append(q)
|
||||
|
||||
sorted_questions = []
|
||||
for freq in sorted(freq_buckets.keys(), reverse=True):
|
||||
bucket = freq_buckets[freq]
|
||||
random.shuffle(bucket)
|
||||
sorted_questions.extend(bucket)
|
||||
|
||||
print(f"按频次排序完成 (高频优先处理)")
|
||||
|
||||
# ====== 恢复已有进度 ======
|
||||
progress_file = os.path.join(output_dir, 'diversity_progress.json')
|
||||
completed = {} # question -> list of answers
|
||||
if os.path.exists(progress_file):
|
||||
with open(progress_file, encoding='utf-8') as f:
|
||||
completed = json.load(f)
|
||||
# 过滤掉已处理的问题
|
||||
processed_qs = set(completed.keys())
|
||||
sorted_questions = [q for q in sorted_questions if q not in processed_qs]
|
||||
print(f"已有进度: {len(completed)} 个问题已完成")
|
||||
print(f"剩余未处理: {len(sorted_questions)} 个问题")
|
||||
|
||||
# 限制要处理的问题数
|
||||
max_q = len(sorted_questions) # 全部问题
|
||||
|
||||
total_questions = len(sorted_questions)
|
||||
total_batches = (total_questions + QUESTIONS_PER_BATCH - 1) // QUESTIONS_PER_BATCH
|
||||
|
||||
print(f"\n目标: {total_questions} 个问题 × {ANSWERS_PER_QUESTION} 个回答 = {total_questions * ANSWERS_PER_QUESTION} 条")
|
||||
print(f"批次: {total_batches} 批 (每批 ≤{QUESTIONS_PER_BATCH} 问题)")
|
||||
print(f"并发: {MAX_WORKERS}")
|
||||
if total_questions > 1000:
|
||||
minutes = total_questions * ANSWERS_PER_QUESTION / (MAX_WORKERS * 8) / 60
|
||||
print(f"预估耗时: ~{minutes:.0f} 分钟 (~{minutes/60:.1f} 小时)")
|
||||
print(f"{'='*70}\n")
|
||||
|
||||
t_start = time.time()
|
||||
total_new_pairs = 0
|
||||
|
||||
for batch_idx in range(total_batches):
|
||||
batch_qs = sorted_questions[batch_idx * QUESTIONS_PER_BATCH : (batch_idx + 1) * QUESTIONS_PER_BATCH]
|
||||
batch_results = {}
|
||||
|
||||
print(f"\n批次 {batch_idx + 1}/{total_batches} ({len(batch_qs)} 个问题)")
|
||||
|
||||
# 并发生成
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||||
futures = {
|
||||
executor.submit(generate_answers_for_question, q, ANSWERS_PER_QUESTION, MAX_ATTEMPTS, TEMPERATURE): q
|
||||
for q in batch_qs
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
q = futures[future]
|
||||
try:
|
||||
answers = future.result()
|
||||
if answers:
|
||||
batch_results[q] = answers
|
||||
print(f" ✅ 『{q[:30]}』 → {len(answers)} 个不同回答")
|
||||
else:
|
||||
print(f" ❌ 『{q[:30]}』 → 生成失败")
|
||||
except Exception as e:
|
||||
print(f" ❌ 『{q[:30]}』 → 错误: {e}")
|
||||
|
||||
# 追加到总进度
|
||||
completed.update(batch_results)
|
||||
batch_new_pairs = sum(len(v) for v in batch_results.values())
|
||||
total_new_pairs += batch_new_pairs
|
||||
|
||||
# 保存进度
|
||||
with open(progress_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(completed, f, ensure_ascii=False, indent=2)
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
rate = total_new_pairs / max(elapsed, 1) * 60
|
||||
|
||||
# 统计
|
||||
print(f"\n批次 {batch_idx + 1} 完成 | "
|
||||
f"本批新增 {batch_new_pairs} 条 | "
|
||||
f"累计 {total_new_pairs:,} 条 ({len(completed)} 问题) | "
|
||||
f"{rate:.0f}条/分 | "
|
||||
f"已用 {elapsed/60:.1f}分")
|
||||
|
||||
# ====== 完成 ======
|
||||
elapsed_total = time.time() - t_start
|
||||
print(f"\n{'='*70}")
|
||||
print(f"全部生成完成!")
|
||||
print(f"处理问题: {len(completed)} 个")
|
||||
print(f"总回答数: {total_new_pairs:,} 条")
|
||||
print(f"用时: {elapsed_total/60:.1f} 分钟")
|
||||
|
||||
# ====== 保存为训练格式 ======
|
||||
train_format = []
|
||||
for q, answers in completed.items():
|
||||
for a in answers:
|
||||
train_format.append({'question': q, 'answer': a})
|
||||
|
||||
output_file = os.path.join(output_dir, 'diverse_train_data_v1.json')
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(train_format, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# 也保存一份到工作目录
|
||||
work_file = '/data/user/work/diverse_train_data_v1.json'
|
||||
with open(work_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(train_format, f, ensure_ascii=False, indent=2)
|
||||
|
||||
print(f"\n结果保存:")
|
||||
print(f" 📄 {output_file}")
|
||||
print(f" 📄 {work_file}")
|
||||
print(f" 📊 平均每问题 {total_new_pairs/max(len(completed),1):.1f} 个回答")
|
||||
print(f"\n如需合并到训练集：")
|
||||
print(f" python3 -c \"import json; d1=json.load(open('xiaoli_train_v2.json')); d2=json.load(open('{output_file}')); d1.extend(d2); json.dump(d1, open('xiaoli_train_v2.json','w'), ensure_ascii=False, indent=2)\"")
|
||||
print(f"{'='*70}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -171,8 +171,8 @@ def main():
|
||||
print(f"去重指纹池: {len(existing_fps)} 个")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 100000 # 目标 10 万条全新问答
|
||||
BATCH_SIZE = 10 # 每批次生成 10 条
|
||||
TARGET = 500000 # 目标 50 万条全新问答
|
||||
BATCH_SIZE = 15 # 每批次生成 15 条（更多问题，效率更高）
|
||||
BATCHES_PER_SAVE = 20 # 每 20 批次保存一次
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
|
||||
|
||||
+10
-9
@@ -141,8 +141,8 @@ def parse_multi_turn(text):
|
||||
turns.append((current_q, a_text))
|
||||
current_q = None # 配对后重置，等待下一个问
|
||||
|
||||
# 最少2轮才算有效
|
||||
if len(turns) >= 2:
|
||||
# 最少4轮才算有效（6~12轮的长对话）
|
||||
if len(turns) >= 4:
|
||||
return turns
|
||||
return None
|
||||
|
||||
@@ -173,19 +173,20 @@ def make_multi_prompt():
|
||||
topics = random.sample(all_seeds, min(topic_count, len(all_seeds)))
|
||||
topic_context = "、".join(topics)
|
||||
|
||||
turn_count = random.randint(2, 5)
|
||||
turn_count = random.randint(6, 12)
|
||||
|
||||
prompt = f"""请以小狸的身份，模拟一段{turn_count}轮的自然闲聊对话。
|
||||
prompt = f"""请以小狸的身份，模拟一段长时间的自然闲聊对话，至少{turn_count}轮。
|
||||
|
||||
开场话题：{topic_context}
|
||||
|
||||
要求：
|
||||
1. 小狸是8岁小女孩，说话简短自然
|
||||
2. 每轮包含"问"和"答"，小狸的回答1-2句话
|
||||
3. 对话要自然展开，不要强行切话题
|
||||
3. 对话要自然展开，一问一答慢慢深入，不要强行切话题
|
||||
4. 小狸的语气：害羞但可爱，不是机器人
|
||||
5. 至少{turn_count}轮，越多越好，让对话持续下去
|
||||
|
||||
请按以下格式输出（至少{turn_count}轮）：
|
||||
请按以下格式输出（至少{turn_count}轮，轮数越多越好）：
|
||||
问：（用户提问）
|
||||
答：（小狸回答）
|
||||
问：（用户续问）
|
||||
@@ -200,7 +201,7 @@ def generate_one_multi():
|
||||
result = call_astron_multi(prompt, temperature=random.uniform(0.8, 1.0))
|
||||
if result:
|
||||
turns = parse_multi_turn(result)
|
||||
if turns and len(turns) >= 2:
|
||||
if turns and len(turns) >= 4:
|
||||
sample = format_training_sample(turns)
|
||||
return {
|
||||
'source': 'multiturn_astron',
|
||||
@@ -249,8 +250,8 @@ def main():
|
||||
print(f"已有中间结果: {len(all_results)} 条 ({len(interim_files)} 批)")
|
||||
|
||||
# ====== 参数 ======
|
||||
TARGET = 20000 # 目标生成 2 万条多轮对话
|
||||
BATCH = 50 # 每批生成 50 条
|
||||
TARGET = 500000 # 目标 50 万条长多轮对话
|
||||
BATCH = 20 # 每批生成 20 条（每条更长，减少批量）
|
||||
MAX_WORKERS = 5 # Astron 5并发
|
||||
TEMPERATURE_RANGE = (0.80, 1.0)
|
||||
|
||||
|
||||
@@ -111,30 +111,12 @@ def main():
|
||||
print(f"生成结果: {len(all_results)} 条 (去重后)")
|
||||
print(f"用时: {(time.time() - t_start)/60:.1f} 分钟")
|
||||
|
||||
# 5. 追加到训练集
|
||||
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
||||
tp = os.path.join(MODEL_DIR, tgt)
|
||||
print(f"\n追加到 {tgt}...")
|
||||
with open(tp, encoding='utf-8') as f:
|
||||
cur = json.load(f)
|
||||
|
||||
# 去重
|
||||
ex = set((it.get('question', '')[:100], it.get('answer', '')[:100]) for it in cur)
|
||||
to_add = []
|
||||
for r in all_results:
|
||||
key = (r['question'][:100], r['answer'][:100])
|
||||
if key not in ex:
|
||||
ex.add(key)
|
||||
to_add.append({'question': r['question'], 'answer': r['answer']})
|
||||
|
||||
cur.extend(to_add)
|
||||
with open(tp, 'w', encoding='utf-8') as f:
|
||||
json.dump(cur, f, ensure_ascii=False, indent=2)
|
||||
|
||||
if 'train' in tgt:
|
||||
print(f" ✅ 训练集 +{len(to_add)} 条 → {len(cur)} 条")
|
||||
else:
|
||||
print(f" ✅ 合并集 +{len(to_add)} 条 → {len(cur)} 条")
|
||||
# 5. 保存最终去重结果（独立存放，不追加到训练集）
|
||||
final_file = os.path.join('/data/user/work', 'high_temp_final.json')
|
||||
with open(final_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(all_results, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n最终结果已保存: {final_file}")
|
||||
print(f"如需使用，可手动合并到训练集")
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"全部完成! 新增 {len(all_results)} 条多样化回答")
|
||||
|
||||
Reference in new issue
Block a user