shuiyu1123
231af72de9
- 清理仓库: 删除所有废弃的generate_*脚本和旧版数据文件 - 目录重组: - data/ → 训练数据 - scripts/ → 数据生成脚本 - templates/ → Web UI - 重命名: - train_v2.py → train.py - vocab_v3.json → vocab.json (BPE, 6.8k tokens) - 更新 train.py 指向新词表路径 - 重写 README.md 完整文档 - 更新 .gitignore
73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""高温度多样化生成 — 每个问题跑 N 次，获得多种回答"""
|
|
import sys, os, json, random, time
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
|
|
from api_correction import batch_generate_astron as batch_generate
|
|
|
|
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
# 载入全部问题
|
|
with open(os.path.join(MODEL_DIR, 'mega_questions.json')) as f:
|
|
mega = json.load(f)
|
|
total_q = len(mega)
|
|
print(f"mega_questions 共 {total_q} 条")
|
|
|
|
# 温度配置：逐渐升高，产出多样化
|
|
TEMP_CONFIG = [0.9, 1.1, 1.3, 1.5] # 跑4轮
|
|
all_new_results = []
|
|
|
|
for round_idx, temp in enumerate(TEMP_CONFIG):
|
|
print(f"\n{'='*60}")
|
|
print(f"第{round_idx+1}轮: 温度={temp}")
|
|
print(f"{'='*60}")
|
|
|
|
questions = [item['question'] for item in mega]
|
|
|
|
t0 = time.time()
|
|
results = batch_generate(questions, temperature=temp)
|
|
elapsed = time.time() - t0
|
|
|
|
# 标注来源
|
|
for r in results:
|
|
r['temperature'] = temp
|
|
r['round'] = round_idx + 1
|
|
|
|
all_new_results.extend(results)
|
|
print(f"轮{round_idx+1} 完成: {len(results)}条 ({elapsed:.0f}s)")
|
|
print(f"累计: {len(all_new_results)}条")
|
|
|
|
# 去重（问题+回答完全重复才去）
|
|
print(f"\n总体去重中...")
|
|
seen = set()
|
|
unique_results = []
|
|
for r in all_new_results:
|
|
key = (r['question'][:100], r['answer'][:100])
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique_results.append(r)
|
|
|
|
print(f"去重前: {len(all_new_results)}条 → 去重后: {len(unique_results)}条")
|
|
|
|
# 追加到训练集
|
|
for tgt in ['xiaoli_train_v2.json', 'xiaoli_merged_data_v2.json']:
|
|
tp = os.path.join(MODEL_DIR, tgt)
|
|
with open(tp, encoding='utf-8') as f:
|
|
cur = json.load(f)
|
|
|
|
ex = set((it.get("question","")[:100], it.get("answer","")[:100]) for it in cur)
|
|
to_add = []
|
|
for r in unique_results:
|
|
key = (r['question'][:100], r['answer'][:100])
|
|
if key not in ex:
|
|
ex.add(key)
|
|
to_add.append({"question": r['question'], "answer": r['answer']})
|
|
|
|
cur.extend(to_add)
|
|
with open(tp, 'w', encoding='utf-8') as f:
|
|
json.dump(cur, f, ensure_ascii=False, indent=2)
|
|
if 'train' in tgt:
|
|
print(f"✅ 训练集 +{len(to_add)}条 → {len(cur)}条")
|
|
|
|
print(f"\n🎉 全部完成!")
|
|
print(f"总数: {len(cur)}条") |