fix(actpred): 修复 Hailo HEF 编译链路的输入断连与封装问题

- hailo_infer: 帧输入按 input_layerN 排序绑定（vstream 报告顺序实测
  为 1,5,4,3,2，按报告顺序会 frame1/frame2 错位），新增 --pred-token
- gen_calibration: pred_token 校准改为学习值 ±30% 抖动，修复常量
  校准量化退化（scale=18571.7，任何输入都被量化成 0）
- export_deploy: 新增 --pred-token-out 保存学习值供推理端喂入
- dfc_compile: parser/optimize 后自动检查输入层连通性，断开即报错
- 新增 _npu_test.py（NPU 4 角敏感性复验）与 _onnx_validate.py
  （编译前 ONNX 数值门槛），test_actpred 补乱序帧序用例

actpred/.gitignore

@@ -20,6 +20,7 @@ checkpoints/
 # DFC logs
 acceleras.log
 hailo_sdk.*.log
+hailort.log
 
 # Dataset (too large for git)
 data/

actpred/_npu_test.py

@@ -0,0 +1,108 @@
+"""actpred HEF NPU 输入敏感性复验脚本（重编译后的验收门槛）
+
+用法: venv/bin/python _npu_test.py
+
+4 角测试（真实/全零帧 × 真实/全零历史）+ 与 PyTorch 参考对比。
+HEF 尺寸/布局从 vstream info 自动推导，checkpoint 用
+checkpoints/pretrain_best_inference.pt。
+"""
+import sys
+import time
+sys.path.insert(0, "/opt/blackbird/actpred")
+
+import numpy as np
+import torch
+import torch.nn as nn
+from PIL import Image
+from hailo_platform import HEF
+
+from config import parse_checkpoint_config
+from model import ActionPredictor
+from hailo_infer import HailoPredictor
+
+HEF_PATH = "checkpoints/actpred_pretrain.hef"
+CKPT = "checkpoints/pretrain_best_inference.pt"
+IMG_DIR = "data/episode_0000/images"
+
+hef = HEF(HEF_PATH)
+print("=== HEF vstream infos ===")
+for i in hef.get_input_vstream_infos():
+    q = i.quant_info
+    print(f"IN  {i.name:16s} {tuple(i.shape)} {i.format.type} "
+          f"zp={q.qp_zp} scale={q.qp_scale:.6g}")
+for o in hef.get_output_vstream_infos():
+    q = o.quant_info
+    print(f"OUT {o.name:16s} {tuple(o.shape)} {o.format.type} "
+          f"zp={q.qp_zp} scale={q.qp_scale:.6g}")
+
+imgs_real = [np.array(Image.open(f"{IMG_DIR}/000{i}.jpg").convert("RGB"))
+             for i in range(3)]
+imgs_zero = [np.zeros_like(imgs_real[0]) for _ in range(3)]
+
+hist_real = np.array([
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 10, 200, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 20, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+], dtype=np.float32)
+hist_zero = np.zeros((5, 18), dtype=np.float32)
+
+# PyTorch 参考（checkpoint 自带 config）
+ck = torch.load(CKPT, map_location="cpu", weights_only=True)
+cfg = parse_checkpoint_config(ck["config"])
+model = ActionPredictor(cfg.model, cfg.data, pretrained=False)
+model.load_state_dict(ck["model_state_dict"])
+model.eval()
+pred_learned = model.pred_token.data.unsqueeze(1).numpy()   # (1,1,1,d)
+
+from torchvision import transforms
+tf = transforms.Compose([
+    transforms.Resize((cfg.data.image_height, cfg.data.image_width)),
+    transforms.ToTensor(),
+    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+])
+
+
+def torch_pred(imgs, hist, use_learned_pred=True):
+    x = torch.stack([tf(Image.fromarray(im)) for im in imgs]).unsqueeze(0)
+    h = torch.tensor(hist).unsqueeze(0)
+    if use_learned_pred:
+        with torch.no_grad():
+            return model(x, h).squeeze(0).numpy()
+    saved = model.pred_token.data.clone()
+    model.pred_token.data.zero_()
+    with torch.no_grad():
+        out = model(x, h).squeeze(0).numpy()
+    model.pred_token.data.copy_(saved)
+    return out
+
+
+with HailoPredictor(HEF_PATH) as p:
+    print(f"\ndims: frame_len={p.frame_len} img={p.image_width}x{p.image_height} "
+          f"history={p.history_len} action_dim={p.action_dim}")
+    t0 = time.perf_counter()
+    a = p.predict(imgs_real, hist_real, pred_token=pred_learned)
+    b = p.predict(imgs_zero, hist_real, pred_token=pred_learned)
+    c = p.predict(imgs_real, hist_zero, pred_token=pred_learned)
+    d = p.predict(imgs_zero, hist_zero, pred_token=pred_learned)
+    t1 = time.perf_counter()
+    print(f"4x predict: {(t1 - t0) / 4 * 1000:.1f} ms/次")
+    for name, r in [("real_img+real_hist", a), ("zero_img+real_hist", b),
+                    ("real_img+zero_hist", c), ("zero_img+zero_hist", d)]:
+        print(f"{name:20s} max_abs={np.abs(r).max():.4f}  out={np.round(r[:6], 3)}...")
+
+    img_sens = np.abs(a - b).max()
+    hist_sens = np.abs(a - c).max()
+    print(f"\nimg sensitivity  = {img_sens:.4f}（>0.1 视为输入连通）")
+    print(f"hist sensitivity = {hist_sens:.4f}（>0.1 视为输入连通）")
+
+    ref = torch_pred(imgs_real, hist_real, True)
+    ref_zero_pred = torch_pred(imgs_real, hist_real, False)
+    print(f"torch learned_pred  : {np.round(ref[:6], 3)}...")
+    print(f"torch zero_pred     : {np.round(ref_zero_pred[:6], 3)}...")
+    print(f"HEF vs torch(learned) max_diff={np.abs(a - ref).max():.4f}")
+
+    ok = img_sens > 0.1 and hist_sens > 0.1
+    print("\nPASS: HEF 输入连通，可继续与 torch 对比量化误差"
+          if ok else "\nFAIL: HEF 输出对输入不敏感，编译仍有问题")

actpred/_onnx_validate.py

@@ -0,0 +1,92 @@
+"""部署版 ONNX 数值验证：输入敏感性 + 与 PyTorch 一致性（重编译前门槛）
+
+用法: venv/bin/python _onnx_validate.py
+"""
+import sys
+sys.path.insert(0, "/opt/blackbird/actpred")
+
+import numpy as np
+import torch
+from PIL import Image
+from torchvision import transforms
+import onnxruntime as ort
+
+from config import parse_checkpoint_config
+from model import ActionPredictor
+
+CKPT = "checkpoints/pretrain_best_inference.pt"
+ONNX = "checkpoints/actpred_deploy.onnx"
+IMG_DIR = "data/episode_0000/images"
+
+ck = torch.load(CKPT, map_location="cpu", weights_only=True)
+cfg = parse_checkpoint_config(ck["config"])
+model = ActionPredictor(cfg.model, cfg.data, pretrained=False)
+model.load_state_dict(ck["model_state_dict"])
+model.eval()
+
+sess = ort.InferenceSession(ONNX)
+in_names = [i.name for i in sess.get_inputs()]
+print("ONNX inputs:", in_names)
+
+h, w = cfg.data.image_height, cfg.data.image_width
+tf = transforms.Compose([
+    transforms.Resize((h, w)),
+    transforms.ToTensor(),
+    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+])
+
+imgs_real = [np.array(Image.open(f"{IMG_DIR}/000{i}.jpg").convert("RGB"))
+             for i in range(3)]
+imgs_zero = [np.zeros_like(imgs_real[0]) for _ in range(3)]
+
+hist_real = np.array([
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 10, 200, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 20, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+    [45, 0, 45, 0, 45, 0, 45, 0, 90, 90, 0, 0, 0, 0, 0, 0, 0, 0],
+], dtype=np.float32)
+hist_zero = np.zeros((5, 18), dtype=np.float32)
+
+pred_learned = model.pred_token.data.unsqueeze(1).numpy()   # (1,1,1,128)
+pred_none = np.zeros((1, 1, 1, 128), dtype=np.float32)
+
+
+def frames_nhwc(imgs):
+    """列表帧 → (1,H,W,3) 归一化 float32 NHWC × frame_len"""
+    chw = np.stack([tf(Image.fromarray(im)).numpy() for im in imgs])
+    return [chw[i].transpose(1, 2, 0)[None] for i in range(len(imgs))]
+
+
+def run_onnx(imgs, hist, pred):
+    feeds = {f"frame{i}": f for i, f in enumerate(frames_nhwc(imgs))}
+    feeds["action_history"] = hist[None, None]
+    feeds["pred_token"] = pred
+    return sess.run(None, feeds)[0].reshape(-1)
+
+
+def run_torch(imgs, hist):
+    x = torch.stack([tf(Image.fromarray(im)) for im in imgs]).unsqueeze(0)
+    hh = torch.tensor(hist).unsqueeze(0)
+    with torch.no_grad():
+        return model(x, hh).squeeze(0).numpy()
+
+
+a = run_onnx(imgs_real, hist_real, pred_learned)
+b = run_onnx(imgs_zero, hist_real, pred_learned)
+c = run_onnx(imgs_real, hist_zero, pred_learned)
+d = run_onnx(imgs_zero, hist_zero, pred_learned)
+p0 = run_onnx(imgs_real, hist_real, pred_none)
+
+print(f"onnx real_img+real_hist: {np.round(a[:6], 3)}...")
+print(f"onnx pred_token=0 diff : max={np.abs(a - p0).max():.4f}")
+print(f"img sensitivity  (a-b): max={np.abs(a - b).max():.4f}")
+print(f"hist sensitivity (a-c): max={np.abs(a - c).max():.4f}")
+
+ref = run_torch(imgs_real, hist_real)
+print(f"torch reference        : {np.round(ref[:6], 3)}...")
+print(f"onnx vs torch max_diff : {np.abs(a - ref).max():.6e}")
+
+ok = (np.abs(a - b).max() > 1e-4 and np.abs(a - c).max() > 1e-4
+      and np.abs(a - ref).max() < 1e-4)
+print("PASS" if ok else "FAIL")

actpred/dfc_compile.py

@@ -57,6 +57,44 @@ def har_input_layer_names(har_path: str) -> list:
     return [name for name, _ in layers]
 
 
+def check_input_connectivity(har_path: str, input_layers: list):
+    """检查 HAR 中输入层是否都有下游边
+
+    输入层被优化断开时编译照常成功，但 HEF 输出恒定值且无任何报错
+    （2026-08-13 实测踩坑），必须在编译前拦截。.edges 文件缺失或格式
+    无法识别时仅告警不阻断。
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        with tarfile.open(har_path) as tf:
+            tf.extractall(tmp)
+        edge_files = [f for f in os.listdir(tmp) if f.endswith(".edges")]
+        if not edge_files:
+            print(f"WARNING: {har_path} 无 .edges 文件，跳过输入连通性检查")
+            return
+        with open(os.path.join(tmp, edge_files[0])) as f:
+            edges = json.load(f)
+
+    if not isinstance(edges, list):
+        print(f"WARNING: .edges 不是列表（{type(edges).__name__}），"
+              f"跳过连通性检查")
+        return
+    adjacency = set()
+    for e in edges:
+        if isinstance(e, dict) and "from" in e:
+            adjacency.add(e["from"])
+        elif isinstance(e, (list, tuple)) and len(e) == 2:
+            adjacency.add(e[0])
+        else:
+            print(f"WARNING: 无法识别 .edges 条目格式: {e!r}，跳过连通性检查")
+            return
+    disconnected = [n for n in input_layers if n not in adjacency]
+    if disconnected:
+        sys.exit(
+            f"错误: {har_path} 中输入层无下游边（输入被优化断开，"
+            f"编译出的 HEF 将输出恒定值）: {disconnected}")
+    print(f"输入连通性检查通过: {len(input_layers)} 个输入层均有下游边")
+
+
 def main():
     parser = argparse.ArgumentParser(description="Compile actpred deploy ONNX to HEF")
     parser.add_argument("--onnx", default="checkpoints/actpred_deploy.onnx")
@@ -93,11 +131,13 @@ def main():
         names = har_input_layer_names(har_raw)
         if len(names) != len(START_NODES):
             sys.exit(f"错误: HAR 输入层数 {len(names)} != {len(START_NODES)}，解析可能被截断")
+        check_input_connectivity(har_raw, names)
 
         print("[2/3] optimize: 量化校准")
         hailo("optimize", har_raw, "--hw-arch", args.hw_arch,
               "--calib-set-path", os.path.abspath(args.calib_npz),
               "--output-har-path", har_opt, venv_bin=venv_bin)
+        check_input_connectivity(har_opt, har_input_layer_names(har_opt))
 
         print("[3/3] compiler: HAR → HEF")
         hailo("compiler", har_opt, "--hw-arch", args.hw_arch,

actpred/export_deploy.py

@@ -128,6 +128,9 @@ def main():
     parser = argparse.ArgumentParser(description="Export Hailo-friendly ONNX")
     parser.add_argument("--model", required=True, help="checkpoint path")
     parser.add_argument("--output", default="checkpoints/actpred_deploy.onnx")
+    parser.add_argument("--pred-token-out", default=None,
+                        help="保存学习到的 pred_token 到 .npy（HEF 推理端喂入用，"
+                             "配合抖动校准避免常量量化退化）")
     args = parser.parse_args()
 
     checkpoint = torch.load(args.model, map_location="cpu", weights_only=True)
@@ -156,6 +159,11 @@ def main():
     )
     print(f"部署版 ONNX 已导出: {args.output}（{f} 个帧输入 (1,{h},{w},3) NHWC + action_history + pred_token）")
 
+    if args.pred_token_out:
+        import numpy as np
+        np.save(args.pred_token_out, model.pred_token.data.unsqueeze(1).numpy())
+        print(f"pred_token 已保存: {args.pred_token_out} (1,1,1,{model.pred_token.shape[-1]})")
+
     stacked = torch.stack([fr.permute(0, 3, 1, 2) for fr in dummy_frames], dim=1)
     with torch.no_grad():
         orig_out = model(stacked, dummy_history.squeeze(1))

actpred/gen_calibration.py

@@ -6,7 +6,8 @@
   --deploy 模式（data/calibration_deploy.npz）：为 export_deploy.py 导出
   的部署版 ONNX 生成校准集，键名匹配 HN 层名 actpred_deploy/input_layerN，
   形状匹配 HEF 输入：frame0~2 (N,H,W,3) 已归一化 float32、
-  action_history (N,1,H_len,D)、pred_token (N,1,1,d) 常量。
+  action_history (N,1,H_len,D)、pred_token (N,1,1,d) 学习值 ± 抖动
+  （常量校准会使量化范围退化，见 --pred-jitter）。
   图像分辨率与 --model checkpoint 的配置一致。
 
 数据源：
@@ -81,9 +82,15 @@ def build_deploy_set(args):
                             cfg, pretrained=False)
     model.load_state_dict(ckpt["model_state_dict"])
     d = model.pred_token.shape[-1]
-    pred = np.broadcast_to(
-        model.pred_token.data.unsqueeze(1).numpy(),
-        (args.samples, 1, 1, d)).astype(np.float32).copy()
+    v = model.pred_token.data.unsqueeze(1).numpy()  # (1, 1, d) 学习到的常量
+    # 常量校准会使量化范围退化（上次实测 scale=18571.7，任何输入都被
+    # 量化成 0）；按学习值 ±相对抖动生成校准样本，运行时喂学习值即可
+    # 无损还原（量化误差 ~1e-4）
+    jitter = v * rng.uniform(-args.pred_jitter, args.pred_jitter,
+                             size=(args.samples, 1, 1, d)).astype(np.float32)
+    jitter = jitter + rng.normal(0.0, args.pred_jitter_abs,
+                                 size=jitter.shape).astype(np.float32)
+    pred = np.broadcast_to(v, (args.samples, 1, 1, d)).astype(np.float32) + jitter
 
     np.savez_compressed(
         args.output,
@@ -91,7 +98,8 @@ def build_deploy_set(args):
            for i, arr in enumerate([*frames, history, pred])})
     print(f"部署校准集已保存: {args.output}")
     print(f"  frame0~2: {frames[0].shape}（分辨率 {w}x{h}，与 checkpoint 一致）")
-    print(f"  action_history: {history.shape}  pred_token: {pred.shape}")
+    print(f"  action_history: {history.shape}  pred_token: {pred.shape}"
+          f"（学习值 ±{args.pred_jitter:.0%} 抖动）")
 
 
 def main():
@@ -104,6 +112,10 @@ def main():
     parser.add_argument("--output", default=None)
     parser.add_argument("--samples", type=int, default=32)
     parser.add_argument("--seed", type=int, default=42)
+    parser.add_argument("--pred-jitter", type=float, default=0.3,
+                        help="pred_token 校准相对抖动幅度（默认 ±30%）")
+    parser.add_argument("--pred-jitter-abs", type=float, default=0.005,
+                        help="pred_token 校准绝对抖动标准差（近零元素保底）")
     args = parser.parse_args()
 
     if args.deploy:

actpred/hailo_infer.py

@@ -3,30 +3,40 @@
 接口对齐 predict.ActionPredictorInference：predict(images, action_history)。
 
 HEF 由 actpred ONNX 经 Hailo DFC 编译生成（量化校准后）。本封装不硬编码
-vstream 名称/形状/量化参数，全部从 HEF 的 input/output vstream info 动态推导：
+vstream 名称/形状/量化参数，全部从 HEF 的 input/output vstream info 动态推导，
+支持两种布局：
 
-    图像输入:  (1, F, 3, H, W) → NHWC uint8 (1, F, H, W, 3)
-               或 (1, H, W, F*3) → 帧沿通道拼接
-    动作输入:  (1, H, D) float32，uint8 边缘时按 quant_info 量化
-    动作输出:  (1, D)，uint8 边缘时按 quant_info 反量化
+    新版 5 输入（export_deploy.py + dfc_compile.py 产物）：
+        frame0~2 各 (H,W,3) NHWC uint8（ImageNet 归一化校准，负零偏自动识别）
+        + action_history (1,H_len,D) + pred_token (1,1,d)
+    旧版单图像输入：
+        (1, F, 3, H, W) → NHWC uint8 (1, F, H, W, 3)
+        或 (1, H, W, F*3) → 帧沿通道拼接；action_history (1, H, D)
+
+uint8 边缘按 quant_info 量化/反量化；pred_token 输入默认喂零，
+新编译 HEF（抖动校准，见 gen_calibration.py --pred-jitter）应喂
+checkpoint 的学习值（--pred-token npy 或 predict(pred_token=...)），
+否则该 token 语义与训练时不一致。
 
 背景与踩坑记录见
 docs/reports/actpred_hailo_hat2_inference_research_20260813.md：
     - hailort 5.x 对 Hailo-10 仅支持新 API（create_infer_model），
       旧版 VDevice.configure(hef) 报 HAILO_NOT_IMPLEMENTED；
-    - 绑定缓冲区 dtype/大小必须匹配 HEF 边缘格式（量化 uint8）；
-    - 图像输入按 Hailo 惯例为 uint8 原始像素（ImageNet 归一化由
-      DFC 编译时折叠进网络）。
+    - 绑定缓冲区 dtype/大小必须匹配 HEF 边缘格式（量化 uint8）。
 """
 
 import argparse
 import contextlib
 import json
 import os
+import re
 
 import numpy as np
 from PIL import Image
 
+IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
+IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)
+
 # 与 predict.parse_action_history 相同逻辑；此处内联以保持
 # 本模块零 torch 依赖（HAT 部署环境只装 hailo_platform + numpy + PIL）
 
@@ -43,9 +53,17 @@ def parse_action_history(s: str, action_dim: int) -> np.ndarray:
     return np.array(history, dtype=np.float32)
 
 
+def _input_layer_index(name: str) -> int:
+    m = re.search(r"input_layer(\d+)$", name)
+    return int(m.group(1)) if m else 0
+
+
 class HailoPredictor:
     """HEF 动作预测推理器（Hailo NPU 后端）
 
+    支持新版 5 输入 HEF（逐帧输入 + pred_token）与旧版单图像输入 HEF，
+    布局与量化参数从 vstream info 自动推导。
+
     用法：
         with HailoPredictor("actpred.hef") as predictor:
             action = predictor.predict(images, action_history)
@@ -68,9 +86,9 @@ class HailoPredictor:
             if len(self._out_infos) != 1:
                 raise ValueError(
                     f"HEF 应只有一个输出 vstream，实际 {len(self._out_infos)}")
-            self._image_info = self._find_image_input()
-            self._history_info = self._find_history_input()
             self._out_info = self._out_infos[0]
+            self._frame_infos, self._history_info, self._pred_info = \
+                self._classify_inputs()
             self._derive_dims()
 
             self._action_mean = None
@@ -89,40 +107,77 @@ class HailoPredictor:
             self._stack.close()       # 初始化失败也释放设备，避免 fd 泄漏
             raise
 
-    # ---------------- 形状推导 ----------------
+    # ---------------- 输入分类与形状推导 ----------------
 
-    def _find_image_input(self):
-        cands = [i for i in self._in_infos if len(i.shape) in (4, 5)]
-        if len(cands) != 1:
-            raise ValueError(
-                "HEF 输入中无法唯一识别图像 vstream（期望一个 4/5 维输入），"
-                f"实际: {[(i.name, tuple(i.shape)) for i in self._in_infos]}")
-        return cands[0]
+    def _classify_inputs(self):
+        """将 HEF 输入分为 帧列表 / 动作历史 / pred_token（可能为 None）
 
-    def _find_history_input(self):
-        cands = [i for i in self._in_infos
-                 if len(i.shape) == 3 and i is not self._image_info]
-        if len(cands) != 1:
+        多帧输入的绑定顺序按 input_layerN 数字后缀排序，而非 vstream
+        报告顺序：hailort 5.3.0 实测报告顺序被打乱（1,5,4,3,2），
+        按报告顺序绑定会导致 frame1/frame2 错位。
+        """
+        ins = list(self._in_infos)
+        out_dim = tuple(self._out_info.shape)[-1]
+        # 新版：每帧独立 (H,W,3) 输入
+        frames = [i for i in ins if len(i.shape) == 3
+                  and i.shape[2] == 3 and i.shape[0] > 3]
+        if frames:
+            frames.sort(key=lambda i: _input_layer_index(i.name))
+        else:
+            # 旧版：单一 4/5 维图像输入
+            frames = [i for i in ins if len(i.shape) in (4, 5)]
+        rest = [i for i in ins if i not in frames]
+        history, pred = None, None
+        for i in rest:
+            s = tuple(i.shape)
+            if len(s) == 3 and s[-1] == out_dim:
+                history = i
+            elif len(s) == 3 and s[1] == 1:
+                pred = i
+            else:
+                raise ValueError(
+                    f"无法识别 HEF 输入 {i.name} 形状 {s}（期望动作历史 "
+                    f"(1,H_len,D) 或 pred_token (1,1,d)）")
+        if not frames:
+            raise ValueError("HEF 输入中未找到图像输入")
+        if history is None:
             raise ValueError(
-                "HEF 输入中无法唯一识别动作历史 vstream（期望一个 3 维输入），"
-                f"实际: {[(i.name, tuple(i.shape)) for i in self._in_infos]}")
-        return cands[0]
+                f"HEF 输入中未找到动作历史输入（输出维度 {out_dim}），"
+                f"实际: {[(i.name, tuple(i.shape)) for i in ins]}")
+        return frames, history, pred
 
     def _derive_dims(self):
-        shape = tuple(self._image_info.shape)
-        if len(shape) == 5:
-            _, self.frame_len, c, self.image_height, self.image_width = shape
-            if c != 3:
-                raise ValueError(f"图像输入通道维应为 3，实际形状 {shape}")
-            self._image_layout = "ncfhw"
-        else:  # (1, H, W, F*3) NHWC 帧拼通道
-            _, self.image_height, self.image_width, c = shape
-            if c % 3 != 0 or c == 3:
-                raise ValueError(
-                    f"无法从 4 维图像输入推导帧数（期望 (1,H,W,F*3)），"
-                    f"实际形状 {shape}")
-            self.frame_len = c // 3
-            self._image_layout = "nhwc_fused"
+        # 帧输入布局
+        if len(self._frame_infos) == 1 and len(self._frame_infos[0].shape) in (4, 5):
+            shape = tuple(self._frame_infos[0].shape)
+            if len(shape) == 5:
+                _, self.frame_len, c, self.image_height, self.image_width = shape
+                if c != 3:
+                    raise ValueError(f"图像输入通道维应为 3，实际形状 {shape}")
+                self._image_layout = "ncfhw"
+            else:  # (1, H, W, F*3) NHWC 帧拼通道
+                _, self.image_height, self.image_width, c = shape
+                if c % 3 != 0 or c == 3:
+                    raise ValueError(
+                        f"无法从 4 维图像输入推导帧数（期望 (1,H,W,F*3)），"
+                        f"实际形状 {shape}")
+                self.frame_len = c // 3
+                self._image_layout = "nhwc_fused"
+        else:
+            # 新版：每帧独立 (H,W,3) 输入，形状须一致
+            shapes = [tuple(i.shape) for i in self._frame_infos]
+            if any(len(s) != 3 or s[2] != 3
+                   or s[0] != shapes[0][0] or s[1] != shapes[0][1]
+                   for s in shapes):
+                raise ValueError(f"帧输入形状不一致: {shapes}")
+            self.image_height, self.image_width, _ = shapes[0]
+            self.frame_len = len(self._frame_infos)
+            self._image_layout = "nhwc_frames"
+        self._frame_quant = self._frame_infos[0].quant_info
+        # 帧量化零偏为负 ⇒ 校准数据是 ImageNet 归一化后的浮点，
+        # 推理端需先归一化再量化；零偏为 0 则为原始像素直通
+        self._normalize_frames = \
+            self._frame_quant.qp_zp * self._frame_quant.qp_scale < 0
 
         h_shape = tuple(self._history_info.shape)
         self.history_len, self.action_dim = h_shape[1], h_shape[2]
@@ -147,10 +202,12 @@ class HailoPredictor:
 
     # ---------------- 推理 ----------------
 
-    def predict(self, images, action_history: np.ndarray) -> np.ndarray:
+    def predict(self, images, action_history: np.ndarray,
+                pred_token: np.ndarray = None) -> np.ndarray:
         """images: 长度为 frame_len 的 RGB 图像列表（旧→新，末帧为当前帧）
 
-        返回 (action_dim,) float32 动作向量。
+        返回 (action_dim,) float32 动作向量。pred_token 仅新版 5 输入 HEF
+        需要；默认喂零，抖动校准的新 HEF 建议喂 checkpoint 学习值。
         """
         if not isinstance(images, (list, tuple)):
             images = [images]
@@ -170,9 +227,8 @@ class HailoPredictor:
         from hailo_platform import FormatType
         bindings = self._configured.create_bindings()
 
-        frames = np.stack([self._preprocess(img) for img in images])
-        bindings.input(self._image_info.name).set_buffer(
-            self._image_buffer(frames))
+        frame_float = np.stack([self._frame_float(img) for img in images])
+        self._bind_frames(bindings, frame_float)
 
         h_shape = tuple(self._history_info.shape)
         if self._history_info.format.type == FormatType.UINT8:
@@ -183,6 +239,18 @@ class HailoPredictor:
             buf = action_history.astype(np.float32).reshape(h_shape)
         bindings.input(self._history_info.name).set_buffer(buf)
 
+        if self._pred_info is not None:
+            p_shape = tuple(self._pred_info.shape)
+            if pred_token is None:
+                values = np.zeros(p_shape, dtype=np.float32)
+            else:
+                values = np.asarray(pred_token, dtype=np.float32).reshape(p_shape)
+            if self._pred_info.format.type == FormatType.UINT8:
+                q = self._pred_info.quant_info
+                values = np.clip(np.round(values / q.qp_scale + q.qp_zp),
+                                 0, 255).astype(np.uint8)
+            bindings.input(self._pred_info.name).set_buffer(values)
+
         out_shape = tuple(self._out_info.shape)
         if self._out_info.format.type == FormatType.UINT8:
             out_buf = np.zeros(out_shape, dtype=np.uint8)
@@ -203,19 +271,43 @@ class HailoPredictor:
             result = result * self._action_std + self._action_mean
         return result
 
-    def _preprocess(self, image) -> np.ndarray:
+    def _frame_float(self, image) -> np.ndarray:
+        """Resize → float32 HWC；归一化 HEF（负零偏）再走 ImageNet 归一化"""
         img = Image.fromarray(image) if isinstance(image, np.ndarray) else image
         img = img.convert("RGB").resize(
             (self.image_width, self.image_height), Image.BILINEAR)
-        return np.asarray(img, dtype=np.uint8)      # HWC uint8
+        x = np.asarray(img, dtype=np.float32)          # HWC 0..255
+        if self._normalize_frames:
+            x = (x / 255.0 - IMAGENET_MEAN) / IMAGENET_STD
+        return x
+
+    def _bind_frames(self, bindings, frame_float: np.ndarray):
+        """(F,H,W,3) float 帧 → HEF 帧输入缓冲区（按布局与量化）"""
+        from hailo_platform import FormatType
+
+        def quantize(buf, q):
+            return np.clip(np.round(buf / q.qp_scale + q.qp_zp),
+                           0, 255).astype(np.uint8)
 
-    def _image_buffer(self, frames: np.ndarray) -> np.ndarray:
-        """(F, H, W, 3) → HEF 图像输入缓冲区（按编译后布局）"""
         if self._image_layout == "ncfhw":
-            return frames.transpose(0, 3, 1, 2)[np.newaxis]  # (1,F,3,H,W)
-        # nhwc_fused: 帧沿通道拼接 (1, H, W, F*3)
-        return frames.reshape(1, self.image_height, self.image_width,
-                              self.frame_len * 3)
+            buf = frame_float.transpose(0, 3, 1, 2)[np.newaxis]   # (1,F,3,H,W)
+            info = self._frame_infos[0]
+            if info.format.type == FormatType.UINT8:
+                buf = quantize(buf, info.quant_info)
+            bindings.input(info.name).set_buffer(buf)
+        elif self._image_layout == "nhwc_fused":
+            buf = frame_float.reshape(
+                1, self.image_height, self.image_width, self.frame_len * 3)
+            info = self._frame_infos[0]
+            if info.format.type == FormatType.UINT8:
+                buf = quantize(buf, info.quant_info)
+            bindings.input(info.name).set_buffer(buf)
+        else:  # nhwc_frames：每帧独立输入
+            for i, info in enumerate(self._frame_infos):
+                buf = frame_float[i]
+                if info.format.type == FormatType.UINT8:
+                    buf = quantize(buf, info.quant_info)
+                bindings.input(info.name).set_buffer(buf)
 
 
 def main():
@@ -227,10 +319,14 @@ def main():
                         help="冒号分隔各步，每步 action_dim 个值")
     parser.add_argument("--stats", default=None,
                         help="action_stats.npy 路径（训练时启用归一化才需要）")
+    parser.add_argument("--pred-token", default=None,
+                        help="pred_token .npy 路径（export_deploy.py "
+                             "--pred-token-out 产物；缺省喂零）")
     parser.add_argument("--output-format", choices=["json", "raw"],
                         default="json")
     args = parser.parse_args()
 
+    pred_token = np.load(args.pred_token) if args.pred_token else None
     with HailoPredictor(args.hef, stats_path=args.stats) as predictor:
         if len(args.images) != predictor.frame_len:
             raise SystemExit(
@@ -238,7 +334,7 @@ def main():
                 f"实际为 {len(args.images)} 个")
         images = [np.array(Image.open(p).convert("RGB")) for p in args.images]
         history = parse_action_history(args.action_history, predictor.action_dim)
-        predicted = predictor.predict(images, history)
+        predicted = predictor.predict(images, history, pred_token=pred_token)
 
     if args.output_format == "json":
         print(json.dumps({"predicted_action": predicted.tolist()}, indent=2))

actpred/test_actpred.py

@@ -1093,6 +1093,88 @@ class TestHailoPredictor(unittest.TestCase):
         np.testing.assert_allclose(result, [0, 0, 0])
         self.assertEqual(seen_shapes["image"], (1, 4, 4, 6))
 
+    def test_multiframe_inputs_layout(self):
+        """新版 5 输入 HEF：frame×3 (H,W,3) + history + pred_token 常量"""
+        from hailo_infer import HailoPredictor
+        q_frame = _FakeQuantInfo(qp_zp=114.0, qp_scale=0.018723)
+        q_pred = _FakeQuantInfo(qp_zp=123.0, qp_scale=18571.0)
+        infos_in = [
+            _FakeVStreamInfo("net/input_layer1", (64, 64, 3),
+                             _FakeFormatType.UINT8, q_frame),
+            _FakeVStreamInfo("net/input_layer2", (64, 64, 3),
+                             _FakeFormatType.UINT8, q_frame),
+            _FakeVStreamInfo("net/input_layer3", (64, 64, 3),
+                             _FakeFormatType.UINT8, q_frame),
+            _FakeVStreamInfo("net/input_layer4", (1, 5, 18),
+                             _FakeFormatType.UINT8,
+                             _FakeQuantInfo(qp_zp=0.0, qp_scale=1.960784)),
+            _FakeVStreamInfo("net/input_layer5", (1, 1, 128),
+                             _FakeFormatType.UINT8, q_pred),
+        ]
+        infos_out = [_FakeVStreamInfo(
+            "net/conv49", (1, 1, 18), _FakeFormatType.UINT8,
+            _FakeQuantInfo(qp_zp=2.0, qp_scale=0.837115))]
+        seen = {}
+
+        def transform(b):
+            for name in ("net/input_layer1", "net/input_layer2", "net/input_layer3"):
+                seen[name] = np.asarray(b.input(name).get_buffer())
+            seen["history"] = np.asarray(b.input("net/input_layer4").get_buffer())
+            seen["pred"] = np.asarray(b.input("net/input_layer5").get_buffer())
+            b.output("net/conv49").get_buffer()[...] = 2   # 反量化后为 0
+
+        _install_fake_hailo(self, infos_in, infos_out, transform)
+        with HailoPredictor("fake.hef") as p:
+            self.assertEqual((p.frame_len, p.history_len, p.action_dim), (3, 5, 18))
+            self.assertEqual((p.image_width, p.image_height), (64, 64))
+            result = p.predict(self._frames(3, size=64), np.zeros((5, 18), np.float32))
+        self.assertEqual(seen["net/input_layer1"].dtype, np.uint8)
+        self.assertEqual(seen["net/input_layer1"].shape, (64, 64, 3))
+        self.assertEqual(seen["history"].shape, (1, 5, 18))
+        np.testing.assert_array_equal(seen["history"], 0)
+        np.testing.assert_array_equal(seen["pred"], 123)   # 退化量化：喂 0 → q=123
+        np.testing.assert_allclose(result, 0)
+
+    def test_multiframe_scrambled_report_order(self):
+        """HEF vstream 报告顺序可能被打乱（实测 1,5,4,3,2），
+        帧输入必须按 input_layerN 排序绑定，否则 frame1/frame2 错位"""
+        from hailo_infer import HailoPredictor
+        infos_in = [
+            _FakeVStreamInfo("net/input_layer1", (8, 8, 3),
+                             _FakeFormatType.FLOAT32),
+            _FakeVStreamInfo("net/input_layer3", (8, 8, 3),
+                             _FakeFormatType.FLOAT32),
+            _FakeVStreamInfo("net/input_layer2", (8, 8, 3),
+                             _FakeFormatType.FLOAT32),
+            _FakeVStreamInfo("net/input_layer4", (1, 2, 3),
+                             _FakeFormatType.FLOAT32),
+            _FakeVStreamInfo("net/input_layer5", (1, 1, 4),
+                             _FakeFormatType.FLOAT32),
+        ]
+        infos_out = [_FakeVStreamInfo("net/conv49", (1, 1, 3),
+                                      _FakeFormatType.FLOAT32)]
+        seen = {}
+
+        def transform(b):
+            for name in ("net/input_layer1", "net/input_layer2",
+                         "net/input_layer3"):
+                seen[name] = np.asarray(b.input(name).get_buffer()).copy()
+            _first_row_transform("net/input_layer4", "net/conv49")(b)
+
+        _install_fake_hailo(self, infos_in, infos_out, transform)
+        frames = self._frames(3)
+        frames[0][:] = 10
+        frames[1][:] = 20
+        frames[2][:] = 30
+        with HailoPredictor("fake.hef") as p:
+            p.predict(frames, np.zeros((2, 3), dtype=np.float32))
+        np.testing.assert_array_equal(
+            seen["net/input_layer1"], np.full((8, 8, 3), 10, np.float32))
+        np.testing.assert_array_equal(
+            seen["net/input_layer2"], np.full((8, 8, 3), 20, np.float32))
+        np.testing.assert_array_equal(
+            seen["net/input_layer3"], np.full((8, 8, 3), 30, np.float32))
+
     def test_frame_count_mismatch_raises(self):
         from hailo_infer import HailoPredictor
         infos_in = [