Initial commit: BitFlow MoE - 微型中文 MoE 语言模型 for micro:bit
This commit is contained in:
shuiyu67 committed
commit
586917e75c
14 files changed
+2522
No files matched your search
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
从训练输出解析数据并绘制训练过程图
|
||||
"""
|
||||
import re
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
# 设置中文字体
|
||||
plt.rcParams['font.sans-serif'] = ['Noto Sans CJK SC', 'WenQuanYi Micro Hei', 'Microsoft YaHei', 'SimHei']
|
||||
plt.rcParams['axes.unicode_minus'] = False
|
||||
|
||||
# 从训练输出手动提取的10000步数据
|
||||
# 每500步打印一次
|
||||
data_raw = """
|
||||
ep=0 loss=5.8066 pred=5.2434 router=1.1062 acc=0.34 bal=1.0189
|
||||
ep=500 loss=0.4863 pred=0.4721 router=0.0053 acc=1.00 bal=1.1489
|
||||
ep=1000 loss=0.2003 pred=0.1876 router=0.0032 acc=1.00 bal=1.1169
|
||||
ep=1500 loss=0.1824 pred=0.1681 router=0.0079 acc=1.00 bal=1.0295
|
||||
ep=2000 loss=0.1486 pred=0.1371 router=0.0026 acc=1.00 bal=1.0134
|
||||
ep=2500 loss=0.1164 pred=0.1051 router=0.0006 acc=1.00 bal=1.0957
|
||||
ep=3000 loss=0.1065 pred=0.0929 router=0.0032 acc=1.00 bal=1.1952
|
||||
ep=3500 loss=0.1086 pred=0.0985 router=0.0001 acc=1.00 bal=1.0078
|
||||
ep=4000 loss=0.1040 pred=0.0936 router=0.0000 acc=1.00 bal=1.0371
|
||||
ep=4500 loss=0.1158 pred=0.1050 router=0.0001 acc=1.00 bal=1.0723
|
||||
ep=5000 loss=0.1117 pred=0.1009 router=0.0003 acc=1.00 bal=1.0723
|
||||
ep=5500 loss=0.0988 pred=0.0866 router=0.0001 acc=1.00 bal=1.2127
|
||||
ep=6000 loss=0.0888 pred=0.0780 router=0.0004 acc=1.00 bal=1.0605
|
||||
ep=6500 loss=0.1031 pred=0.0926 router=0.0000 acc=1.00 bal=1.0547
|
||||
ep=7000 loss=0.0986 pred=0.0884 router=0.0000 acc=1.00 bal=1.0137
|
||||
ep=7500 loss=0.0861 pred=0.0750 router=0.0000 acc=1.00 bal=1.1016
|
||||
ep=8000 loss=0.0940 pred=0.0837 router=0.0000 acc=1.00 bal=1.0312
|
||||
ep=8500 loss=0.0971 pred=0.0838 router=0.0002 acc=1.00 bal=1.3179
|
||||
ep=9000 loss=0.0908 pred=0.0807 router=0.0000 acc=1.00 bal=1.0137
|
||||
ep=9500 loss=0.0869 pred=0.0755 router=0.0000 acc=1.00 bal=1.1426
|
||||
ep=9999 loss=0.0863 pred=0.0759 router=0.0000 acc=1.00 bal=1.0371
|
||||
"""
|
||||
|
||||
# 解析数据
|
||||
epochs = []
|
||||
losses = []
|
||||
pred_losses = []
|
||||
router_losses = []
|
||||
accuracies = []
|
||||
balance_losses = []
|
||||
qat_phase = []
|
||||
|
||||
pattern = r'ep=(\d+) loss=([\d.]+) pred=([\d.]+) router=([\d.]+) acc=([\d.]+) bal=([\d.]+)(.*)$'
|
||||
|
||||
for line in data_raw.strip().split('\n'):
|
||||
m = re.match(pattern, line.strip())
|
||||
if m:
|
||||
ep = int(m.group(1))
|
||||
epochs.append(ep)
|
||||
losses.append(float(m.group(2)))
|
||||
pred_losses.append(float(m.group(3)))
|
||||
router_losses.append(float(m.group(4)))
|
||||
accuracies.append(float(m.group(5)))
|
||||
balance_losses.append(float(m.group(6)))
|
||||
qat_phase.append(' [QAT]' in m.group(7))
|
||||
|
||||
epochs = np.array(epochs)
|
||||
losses = np.array(losses)
|
||||
pred_losses = np.array(pred_losses)
|
||||
router_losses = np.array(router_losses)
|
||||
accuracies = np.array(accuracies)
|
||||
balance_losses = np.array(balance_losses)
|
||||
|
||||
# ========== 绘制训练过程图 ==========
|
||||
fig = plt.figure(figsize=(16, 10))
|
||||
fig.suptitle('BitFlow MoE v4 训练过程 — 10,000 轮', fontsize=16, fontweight='bold', y=0.97)
|
||||
|
||||
# 子图1: 总Loss
|
||||
ax1 = plt.subplot(2, 3, 1)
|
||||
# 找到 QAT 起始 epoch
|
||||
qat_epochs = [e for e in epochs if e >= 8000]
|
||||
qat_epoch_val = qat_epochs[0] if qat_epochs else 8000
|
||||
ax1.plot(epochs, losses, 'b-', linewidth=1.5, label='总损失')
|
||||
ax1.axvspan(qat_epoch_val, epochs[-1], alpha=0.1, color='red', label='QAT量化阶段')
|
||||
ax1.set_xlabel('训练轮次 (Epoch)')
|
||||
ax1.set_ylabel('损失值')
|
||||
ax1.set_title('总损失 (Total Loss)', fontsize=11)
|
||||
ax1.legend(fontsize=8)
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax1.set_yscale('log')
|
||||
|
||||
# 子图2: 预测Loss
|
||||
ax2 = plt.subplot(2, 3, 2)
|
||||
ax2.plot(epochs, pred_losses, 'g-', linewidth=1.5, label='预测损失')
|
||||
ax2.axvspan(qat_epoch_val, epochs[-1], alpha=0.1, color='red')
|
||||
ax2.set_xlabel('训练轮次 (Epoch)')
|
||||
ax2.set_ylabel('损失值')
|
||||
ax2.set_title('预测损失 (交叉熵)', fontsize=11)
|
||||
ax2.legend(fontsize=8)
|
||||
ax2.grid(True, alpha=0.3)
|
||||
ax2.set_yscale('log')
|
||||
|
||||
# 子图3: 路由Loss
|
||||
ax3 = plt.subplot(2, 3, 3)
|
||||
ax3.plot(epochs, router_losses, 'r-', linewidth=1.5, label='路由损失')
|
||||
ax3.axvspan(qat_epoch_val, epochs[-1], alpha=0.1, color='red')
|
||||
ax3.set_xlabel('训练轮次 (Epoch)')
|
||||
ax3.set_ylabel('损失值')
|
||||
ax3.set_title('路由损失 (专家选择)', fontsize=11)
|
||||
ax3.legend(fontsize=8)
|
||||
ax3.grid(True, alpha=0.3)
|
||||
ax3.set_yscale('log')
|
||||
|
||||
# 子图4: 路由准确率
|
||||
ax4 = plt.subplot(2, 3, 4)
|
||||
ax4.plot(epochs, accuracies, 'purple', linewidth=1.5, marker='o', markevery=500, markersize=4)
|
||||
ax4.axvspan(qat_epoch_val, epochs[-1], alpha=0.1, color='red')
|
||||
ax4.set_xlabel('训练轮次 (Epoch)')
|
||||
ax4.set_ylabel('准确率')
|
||||
ax4.set_title('路由准确率 (专家路由)', fontsize=11)
|
||||
ax4.legend(fontsize=8)
|
||||
ax4.grid(True, alpha=0.3)
|
||||
ax4.set_ylim([0, 1.05])
|
||||
|
||||
# 子图5: 负载均衡Loss
|
||||
ax5 = plt.subplot(2, 3, 5)
|
||||
ax5.plot(epochs, balance_losses, 'orange', linewidth=1.5, label='负载均衡损失')
|
||||
ax5.axvspan(qat_epoch_val, epochs[-1], alpha=0.1, color='red')
|
||||
ax5.set_xlabel('训练轮次 (Epoch)')
|
||||
ax5.set_ylabel('损失值')
|
||||
ax5.set_title('负载均衡 (专家利用率)', fontsize=11)
|
||||
ax5.legend(fontsize=8)
|
||||
ax5.grid(True, alpha=0.3)
|
||||
ax5.set_yscale('log')
|
||||
|
||||
# 子图6: 各阶段对比 (分三段)
|
||||
ax6 = plt.subplot(2, 3, 6)
|
||||
segments = [
|
||||
('早期 (0-2K)', epochs[epochs<=2000], losses[epochs<=2000], 'steelblue'),
|
||||
('中期 (2K-6K)', epochs[(epochs>2000)&(epochs<=6000)], losses[(epochs>2000)&(epochs<=6000)], 'seagreen'),
|
||||
('QAT量化 (8K-10K)', epochs[epochs>=8000], losses[epochs>=8000], 'firebrick'),
|
||||
]
|
||||
for label, ep, lo, color in segments:
|
||||
ax6.plot(ep, lo, 'o-', color=color, linewidth=1.5, markersize=4, label=label, alpha=0.7)
|
||||
ax6.set_xlabel('训练轮次 (Epoch)')
|
||||
ax6.set_ylabel('损失值')
|
||||
ax6.set_title('分阶段对比 (早期/中期/QAT)', fontsize=11)
|
||||
ax6.legend(fontsize=8)
|
||||
ax6.grid(True, alpha=0.3)
|
||||
ax6.set_yscale('log')
|
||||
|
||||
plt.tight_layout(rect=[0, 0, 1, 0.95])
|
||||
output_path = 'c:\\Users\\Administrator\\.trae-cn\\work\\6a62468bb999696f3874e2ac\\training_analysis.png'
|
||||
plt.savefig(output_path, dpi=150, bbox_inches='tight', facecolor='white')
|
||||
plt.close()
|
||||
print(f"[save] training_analysis.png -> {output_path}")
|
||||
|
||||
# 输出关键统计
|
||||
print(f"\n关键数据点:")
|
||||
print(f" 初始 loss: {losses[0]:.4f} router_acc: {accuracies[0]:.0%}")
|
||||
print(f" 500 epoch: {losses[1]:.4f} router_acc: {accuracies[1]:.0%}")
|
||||
print(f" 2000 epoch: {losses[np.where(epochs==2000)[0][0]]:.4f} router_acc: {accuracies[np.where(epochs==2000)[0][0]]:.0%}")
|
||||
print(f" 6000 epoch: {losses[np.where(epochs==6000)[0][0]]:.4f} router_acc: {accuracies[np.where(epochs==6000)[0][0]]:.0%}")
|
||||
print(f" 9999 epoch: {losses[-1]:.4f} router_acc: {accuracies[-1]:.0%}")
|
||||
print(f" 最佳 loss: {losses.min():.4f} at epoch {epochs[np.argmin(losses)]}")
|
||||
|
||||
# QAT 阶段
|
||||
if any(qat_phase):
|
||||
qat_idx = np.where(np.array(qat_phase))[0][0]
|
||||
print(f"\nQAT 量化阶段:")
|
||||
print(f" QAT前(7500): {losses[np.where(epochs==7500)[0][0]]:.4f}")
|
||||
print(f" QAT后(8000): {losses[np.where(epochs==8000)[0][0]]:.4f}")
|
||||
print(f" QAT最终(9999): {losses[-1]:.4f}")
|
||||
print(f" QAT导致loss上升: +{losses[qat_idx] - losses[qat_idx-1]:.4f} (正常，量化引入噪声)")
|
||||
else:
|
||||
print(f"\nQAT 量化阶段: 未检测到QAT数据")
|
||||
Reference in new issue
Block a user