qing
52c0ade3c4
根因：预训练全零图像占位使 BN running_var 停在初始小值，推理时 特征放大至 ±1e6，DFC 在 token merge 点按最大幅度分配共享量化 scale，pred_token/act_tokens 全部被压成 0，HEF 输出对任何输入 恒定（详见 docs/reports/actpred_hef_insensitivity_rootcause_20260813.md） - export_deploy.py：新增 check_token_ranges 导出前 token 量级检查 （|img|>100 中止导出并提示微调；act/img 比例过大警告；归一化 模型在 z 空间采样，避免零方差列爆炸） - pretrain_mt.py：新增 --random-images（ImageNet 归一化噪声占位， 恢复 BN 统计）、--init-from（微调起点，含动作空间一致性警告）、 --normalize-actions 完整支持（训练集 stats 计算与保存、验证集复用） - gen_calibration.py：归一化模型的校准集自动 z-score 动作历史 - _onnx_validate.py / _npu_test.py：归一化模型适配（torch 参考 z-score、HailoPredictor 传 stats_path） - README：预训练章节补充新参数与 NPU 部署警告 - 单测 76/76 通过；冒烟验证：1 epoch 噪声微调后 |img| 2.46e5→8， 归一化后 |act| 694→2.9
126 lines
5.1 KiB
Python
126 lines
5.1 KiB
Python
"""actpred HEF NPU 输入敏感性复验脚本（重编译后的验收门槛）
|
||
|
||
用法: venv/bin/python _npu_test.py
|
||
|
||
4 角测试（真实/全零帧 × 真实/全零历史）+ 与 PyTorch 参考对比。
|
||
HEF 尺寸/布局从 vstream info 自动推导，checkpoint 用
|
||
checkpoints/pretrain_best_inference.pt。
|
||
"""
|
||
import sys
|
||
import time
|
||
sys.path.insert(0, "/opt/blackbird/actpred")
|
||
|
||
import numpy as np
|
||
import torch
|
||
import torch.nn as nn
|
||
from PIL import Image
|
||
from hailo_platform import HEF
|
||
|
||
from config import parse_checkpoint_config
|
||
from model import ActionPredictor
|
||
from hailo_infer import HailoPredictor
|
||
|
||
HEF_PATH = "checkpoints/actpred_pretrain.hef"
|
||
CKPT = "checkpoints/pretrain_best_inference.pt"
|
||
IMG_DIR = "data/episode_0000/images"
|
||
|
||
hef = HEF(HEF_PATH)
|
||
print("=== HEF vstream infos ===")
|
||
for i in hef.get_input_vstream_infos():
|
||
q = i.quant_info
|
||
print(f"IN {i.name:16s} {tuple(i.shape)} {i.format.type} "
|
||
f"zp={q.qp_zp} scale={q.qp_scale:.6g}")
|
||
for o in hef.get_output_vstream_infos():
|
||
q = o.quant_info
|
||
print(f"OUT {o.name:16s} {tuple(o.shape)} {o.format.type} "
|
||
f"zp={q.qp_zp} scale={q.qp_scale:.6g}")
|
||
|
||
imgs_real = [np.array(Image.open(f"{IMG_DIR}/000{i}.jpg").convert("RGB"))
|
||
for i in range(3)]
|
||
imgs_zero = [np.zeros_like(imgs_real[0]) for _ in range(3)]
|
||
|
||
hist_real = np.array([
|
||
[45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 10, 200, 0, 0, 0, 0, 0, 0],
|
||
[45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 20, 0, 0, 0, 0, 0, 0],
|
||
[45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
|
||
[45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
|
||
[45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
|
||
], dtype=np.float32)
|
||
hist_zero = np.zeros((5, 18), dtype=np.float32)
|
||
|
||
# PyTorch 参考（checkpoint 自带 config）
|
||
ck = torch.load(CKPT, map_location="cpu", weights_only=True)
|
||
cfg = parse_checkpoint_config(ck["config"])
|
||
model = ActionPredictor(cfg.model, cfg.data, pretrained=False)
|
||
model.load_state_dict(ck["model_state_dict"])
|
||
model.eval()
|
||
pred_learned = model.pred_token.data.unsqueeze(1).numpy() # (1,1,1,d)
|
||
|
||
# 归一化模型：torch 参考在喂入前 z-score；HailoPredictor 传 stats_path
|
||
# 由封装内部归一化（hist_real/hist_zero 保持原始值，避免双重归一化）
|
||
import os
|
||
stats_path = None
|
||
if cfg.data.normalize_actions:
|
||
stats_path = cfg.data.action_stats_path or os.path.join(
|
||
os.path.dirname(os.path.abspath(CKPT)), "action_stats.npy")
|
||
if not os.path.isabs(stats_path):
|
||
stats_path = os.path.join(os.path.dirname(os.path.abspath(CKPT)),
|
||
stats_path)
|
||
if not os.path.exists(stats_path):
|
||
raise FileNotFoundError(f"统计文件不存在: {stats_path}")
|
||
print(f"HailoPredictor stats: {stats_path}")
|
||
|
||
from torchvision import transforms
|
||
tf = transforms.Compose([
|
||
transforms.Resize((cfg.data.image_height, cfg.data.image_width)),
|
||
transforms.ToTensor(),
|
||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
||
])
|
||
|
||
|
||
def torch_pred(imgs, hist, use_learned_pred=True):
|
||
x = torch.stack([tf(Image.fromarray(im)) for im in imgs]).unsqueeze(0)
|
||
h = torch.tensor(hist).unsqueeze(0)
|
||
if stats_path is not None:
|
||
stats = np.load(stats_path)
|
||
h = (h - torch.tensor(stats[0])) / torch.tensor(stats[1])
|
||
if use_learned_pred:
|
||
with torch.no_grad():
|
||
return model(x, h).squeeze(0).numpy()
|
||
saved = model.pred_token.data.clone()
|
||
model.pred_token.data.zero_()
|
||
with torch.no_grad():
|
||
out = model(x, h).squeeze(0).numpy()
|
||
model.pred_token.data.copy_(saved)
|
||
return out
|
||
|
||
|
||
with HailoPredictor(HEF_PATH, stats_path=stats_path) as p:
|
||
print(f"\ndims: frame_len={p.frame_len} img={p.image_width}x{p.image_height} "
|
||
f"history={p.history_len} action_dim={p.action_dim}")
|
||
t0 = time.perf_counter()
|
||
a = p.predict(imgs_real, hist_real, pred_token=pred_learned)
|
||
b = p.predict(imgs_zero, hist_real, pred_token=pred_learned)
|
||
c = p.predict(imgs_real, hist_zero, pred_token=pred_learned)
|
||
d = p.predict(imgs_zero, hist_zero, pred_token=pred_learned)
|
||
t1 = time.perf_counter()
|
||
print(f"4x predict: {(t1 - t0) / 4 * 1000:.1f} ms/次")
|
||
for name, r in [("real_img+real_hist", a), ("zero_img+real_hist", b),
|
||
("real_img+zero_hist", c), ("zero_img+zero_hist", d)]:
|
||
print(f"{name:20s} max_abs={np.abs(r).max():.4f} out={np.round(r[:6], 3)}...")
|
||
|
||
img_sens = np.abs(a - b).max()
|
||
hist_sens = np.abs(a - c).max()
|
||
print(f"\nimg sensitivity = {img_sens:.4f}（>0.1 视为输入连通）")
|
||
print(f"hist sensitivity = {hist_sens:.4f}（>0.1 视为输入连通）")
|
||
|
||
ref = torch_pred(imgs_real, hist_real, True)
|
||
ref_zero_pred = torch_pred(imgs_real, hist_real, False)
|
||
print(f"torch learned_pred : {np.round(ref[:6], 3)}...")
|
||
print(f"torch zero_pred : {np.round(ref_zero_pred[:6], 3)}...")
|
||
print(f"HEF vs torch(learned) max_diff={np.abs(a - ref).max():.4f}")
|
||
|
||
ok = img_sens > 0.1 and hist_sens > 0.1
|
||
print("\nPASS: HEF 输入连通，可继续与 torch 对比量化误差"
|
||
if ok else "\nFAIL: HEF 输出对输入不敏感，编译仍有问题")
|