fix(pybot×actpred): 审计修复 7 项——Hailo 部署链语义对齐 + NaN 安全

P1-1: export_deploy.py forward() 从硬编码 3 帧改为 *inputs 动态解析
P1-2: hailo_infer.py 补 predict_delta 支持（残差模式 + --ckpt 自动检测）
P1-3: pybot _load_predictor 自动查找 pred_token.npy/action_stats.npy/
      companion checkpoint，恢复 Hailo 推理训练语义
P1-4: pretrain_mt.py random_images 每 batch 重新生成噪声，防 BN 退化
P2-1: predict.py/hailo_infer.py/robot.py 三层 NaN/Inf 防线
P2-3: recorder.py imu_dim 默认 8→0，避免外部直接使用时形状错误
P2-5: _npu_test.py imu_dim 移到函数定义前，消除执行顺序依赖

actpred/_npu_test.py

@@ -55,6 +55,7 @@ model = ActionPredictor(cfg.model, cfg.data, pretrained=False)
 model.load_state_dict(ck["model_state_dict"])
 model.eval()
 pred_learned = model.pred_token.data.unsqueeze(1).numpy()   # (1,1,1,d)
+imu_dim = cfg.data.imu_dim
 
 # 归一化模型：torch 参考在喂入前 z-score；HailoPredictor 传 stats_path
 # 由封装内部归一化（hist_real/hist_zero 保持原始值，避免双重归一化）
@@ -101,7 +102,6 @@ def torch_pred(imgs, hist, use_learned_pred=True, imu=None):
 
 
 # R2 方案一：imu_dim>0 时每步传原始姿态（HailoPredictor 内部归一化）
-imu_dim = cfg.data.imu_dim
 imu_real = np.array([5.0, -3.0] + [0.0] * 6, dtype=np.float32)
 
 with HailoPredictor(HEF_PATH, stats_path=stats_path) as p:

actpred/export_deploy.py

@@ -132,11 +132,13 @@ class DeployWrapper(nn.Module):
         x = self.proj_convs[i](x)                     # 1x1 conv ≡ 池化后的 img_proj
         return self.pool(x)                           # (1, d, 1, 1) NCHW
 
-    def forward(self, frame0: torch.Tensor, frame1: torch.Tensor, frame2: torch.Tensor,
+    def forward(self, *inputs: torch.Tensor) -> torch.Tensor:
-                action_history: torch.Tensor, pred_token: torch.Tensor,
+        # inputs = frame0..frame{F-1}, action_history, pred_token, [imu]
-                imu: torch.Tensor = None) -> torch.Tensor:
         # 全程 (1, 1, S, d) 布局：seq 占 W 维，避免任何 transpose
-        frames = [frame0, frame1, frame2]
+        frames = list(inputs[:self._frame_len])
+        action_history = inputs[self._frame_len]
+        pred_token = inputs[self._frame_len + 1]
+        imu = inputs[self._frame_len + 2] if self._imu_dim > 0 else None
         feats = [self._frame_feat(frames[i], i).permute(0, 2, 3, 1)
                  for i in range(self._frame_len)]     # (1, 1, 1, d) NHWC
         img_tokens = torch.cat(feats, dim=2) + self.proj_bias  # (1, 1, F, d)

actpred/hailo_infer.py

@@ -72,9 +72,26 @@ class HailoPredictor:
             action = predictor.predict(images, action_history)
     """
 
-    def __init__(self, hef_path: str, stats_path: str = None, vdevice=None):
+    def __init__(self, hef_path: str, stats_path: str = None, vdevice=None,
+                 predict_delta: bool = False, pred_token=None):
+        """HEF 推理器。
+
+        predict_delta: 模型用 --predict-delta 训练时须置 True，推理输出
+            （归一化空间）会加回 history 末步再反归一化，与 predict.py 对齐。
+        pred_token: np.ndarray 或 .npy 路径；export_deploy.py --pred-token-out
+            产物。缺省时喂零（仅适合 pred_token 恰好全零的模型）。
+        """
         from hailo_platform import HEF, VDevice
 
+        self._predict_delta = predict_delta
+        self._default_pred_token = None
+        if pred_token is not None:
+            if isinstance(pred_token, str):
+                self._default_pred_token = np.load(pred_token)
+            else:
+                self._default_pred_token = np.asarray(pred_token,
+                                                       dtype=np.float32)
+
         self._stack = contextlib.ExitStack()
         if vdevice is None:
             vdevice = self._stack.enter_context(VDevice())
@@ -258,6 +275,8 @@ class HailoPredictor:
 
         if self._pred_info is not None:
             p_shape = tuple(self._pred_info.shape)
+            if pred_token is None:
+                pred_token = self._default_pred_token
             if pred_token is None:
                 values = np.zeros(p_shape, dtype=np.float32)
             else:
@@ -294,8 +313,14 @@ class HailoPredictor:
             q = self._out_info.quant_info
             result = (result - q.qp_zp) * q.qp_scale
         result = result.reshape(-1)
+        if self._predict_delta:
+            # 残差模式：Δ̂ + a_{t-1}（归一化空间），与 predict.py 对齐
+            result = result + action_history[-1]
         if self._action_mean is not None:
             result = result * self._action_std + self._action_mean
+        if not np.all(np.isfinite(result)):
+            print("WARNING: HEF output contains NaN/Inf, replacing with zeros")
+            result = np.nan_to_num(result, nan=0.0, posinf=0.0, neginf=0.0)
         return result
 
     def _frame_float(self, image) -> np.ndarray:
@@ -349,6 +374,9 @@ def main():
     parser.add_argument("--pred-token", default=None,
                         help="pred_token .npy 路径（export_deploy.py "
                              "--pred-token-out 产物；缺省喂零）")
+    parser.add_argument("--ckpt", default=None,
+                        help="checkpoint .pt 路径（自动检测 predict_delta；"
+                             "HEF 部署时需要以恢复训练配置）")
     parser.add_argument("--imu", type=str, default=None,
                         help="可选姿态向量（逗号分隔，原始量纲：roll,pitch 度 / "
                              "ax..gz int16 原始值）；HEF 含 imu 输入且缺省时按零向量")
@@ -361,7 +389,18 @@ def main():
     if args.imu is not None:
         imu = np.array([float(v) for v in args.imu.split(",")],
                        dtype=np.float32)
-    with HailoPredictor(args.hef, stats_path=args.stats) as predictor:
+
+    predict_delta = False
+    if args.ckpt:
+        import torch
+        ck = torch.load(args.ckpt, map_location="cpu", weights_only=True)
+        cfg = ck.get("config", {})
+        predict_delta = cfg.get("train", {}).get("predict_delta", False)
+        if predict_delta:
+            print("predict_delta=True: HEF 输出将加回历史末步（残差模式）")
+
+    with HailoPredictor(args.hef, stats_path=args.stats,
+                        predict_delta=predict_delta) as predictor:
         if len(args.images) != predictor.frame_len:
             raise SystemExit(
                 f"--images 需传入 {predictor.frame_len} 个路径（旧→新），"

actpred/predict.py

@@ -108,7 +108,11 @@ class ActionPredictorInference:
             pred = history[0, -1] + pred
         if self._action_mean is not None:
             pred = pred * self._action_std + self._action_mean
-        return pred.cpu().numpy()
+        result = pred.cpu().numpy()
+        if not np.all(np.isfinite(result)):
+            print("WARNING: predict output contains NaN/Inf, replacing with zeros")
+            result = np.nan_to_num(result, nan=0.0, posinf=0.0, neginf=0.0)
+        return result
 
 
 def parse_action_history(s: str, action_dim: int) -> np.ndarray:

actpred/pretrain_mt.py

@@ -121,7 +121,7 @@ def pretrain_one_epoch(model, loader, optimizer, device, image_width,
         if noise_std > 0:
             history = history + torch.randn_like(history) * noise_std
             target = target + torch.randn_like(target) * noise_std
-        if images is None or images.size(0) != history.size(0):
+        if images is None or images.size(0) != history.size(0) or random_images:
             images = _placeholder_images(
                 history.size(0), frame_len, image_height, image_width,
                 device, random_images)
@@ -147,7 +147,7 @@ def validate(model, loader, device, image_width, image_height, frame_len,
     for batch in tqdm(loader, desc="  val  ", leave=False):
         history = batch["action_history"].to(device)
         target = batch["target"].to(device)
-        if images is None or images.size(0) != history.size(0):
+        if images is None or images.size(0) != history.size(0) or random_images:
             images = _placeholder_images(
                 history.size(0), frame_len, image_height, image_width,
                 device, random_images)

pybot/pybot/main.py

@@ -134,10 +134,57 @@ def _load_predictor(checkpoint):
     pybot 不硬依赖 actpred：仅 --loop 时才尝试 import，actpred 不在
     sys.path 时给出可操作提示。checkpoint 目录名决定后端（.hef → Hailo
     NPU，否则 → PyTorch），推理类由 actpred 提供。
+
+    HEF 后端会自动查找同目录下的 pred_token.npy 和 companion checkpoint
+    （pretrain_best_inference.pt / best_inference.pt），以恢复 pred_token
+    默认值和 predict_delta 训练配置——否则 Hailo 推理输出与训练语义不一致。
     """
+    import os
     try:
         if checkpoint.endswith(".hef"):
             from actpred.hailo_infer import HailoPredictor as _P
+            hef_dir = os.path.dirname(os.path.abspath(checkpoint))
+
+            # pred_token：export_deploy.py --pred-token-out 的产物
+            pred_token = None
+            pt_path = os.path.join(hef_dir, "pred_token.npy")
+            if os.path.exists(pt_path):
+                pred_token = pt_path
+                logging.info("found pred_token: %s", pt_path)
+
+            # stats_path：归一化模型的 action_stats.npy
+            stats_path = None
+            sp = os.path.join(hef_dir, "action_stats.npy")
+            if os.path.exists(sp):
+                stats_path = sp
+
+            # predict_delta：从同目录 checkpoint 读取训练配置
+            predict_delta = False
+            for name in ("pretrain_best_inference.pt",
+                         "best_inference.pt"):
+                ckpt_path = os.path.join(hef_dir, name)
+                if os.path.exists(ckpt_path):
+                    try:
+                        import torch
+                        ck = torch.load(ckpt_path, map_location="cpu",
+                                        weights_only=True)
+                        cfg = ck.get("config", {})
+                        predict_delta = cfg.get("train", {}).get(
+                            "predict_delta", False)
+                        if predict_delta:
+                            logging.info(
+                                "predict_delta=True from %s", ckpt_path)
+                    except Exception:
+                        pass
+                    break
+
+            try:
+                return _P(checkpoint, stats_path=stats_path,
+                          predict_delta=predict_delta,
+                          pred_token=pred_token)
+            except Exception:
+                logging.exception("加载 HEF 推理器失败: %s", checkpoint)
+                return None
         else:
             from actpred.predict import ActionPredictorInference as _P
     except ImportError:

pybot/pybot/recorder.py

@@ -34,7 +34,7 @@ class EpisodeRecorder:
     全程 None（无 IMU 硬件）则不产出 imu.npy，保持旧格式兼容。
     """
 
-    def __init__(self, output_dir, action_dim=18, imu_dim=8):
+    def __init__(self, output_dir, action_dim=18, imu_dim=0):
         self.action_dim = action_dim
         self.imu_dim = imu_dim
         self._frame_idx = 0

pybot/pybot/robot.py

@@ -368,6 +368,10 @@ class RobotStruct:
         SpeakerVar.play 强制要求 delay 键；闭环已剔除 delay 列（避免发声
         阻塞卡死节拍），此处为其补 delay=0 —— 发声但不阻塞，节拍由 hz 掌控。
         """
+        import numpy as np
+        if not np.all(np.isfinite(act)):
+            log.warning("action vector contains NaN/Inf, skipping step")
+            return
         row_sets = {}
         for j, dev, rp in exec_cols:
             row_sets.setdefault(dev, {})[rp] = float(act[j])

pybot/tests/test_recorder.py

@@ -314,7 +314,7 @@ cam0.frame;
 
 def test_imu_npy_written_with_dict(tmp_path):
     """record 传姿态 dict → flush 产出 imu.npy（T×8），键序 roll..gz"""
-    rec = EpisodeRecorder(str(tmp_path), action_dim=18)
+    rec = EpisodeRecorder(str(tmp_path), action_dim=18, imu_dim=8)
     imu = {"roll": 1.0, "pitch": 2.0, "ax": 3.0, "ay": 4.0,
            "az": 5.0, "gx": 6.0, "gy": 7.0, "gz": 8.0}
     rec.record(None, [0.0], imu_vec=imu)
@@ -328,7 +328,7 @@ def test_imu_npy_written_with_dict(tmp_path):
 
 def test_imu_mixed_none_marks_zero(tmp_path):
     """混合数据集：部分帧无姿态 → 该帧记零向量，仍产出 imu.npy"""
-    rec = EpisodeRecorder(str(tmp_path), action_dim=18)
+    rec = EpisodeRecorder(str(tmp_path), action_dim=18, imu_dim=8)
     rec.record(None, [0.0], imu_vec=None)
     rec.record(None, [0.0], imu_vec={"roll": 9.0})
     rec.flush()