From aa271e395a52345f08c753f481c229b49ca57205 Mon Sep 17 00:00:00 2001 From: qing <1776177662@qq.com> Date: Fri, 14 Aug 2026 16:23:05 +0800 Subject: [PATCH] =?UTF-8?q?fix(pybot=C3=97actpred):=20=E5=AE=A1=E8=AE=A1?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=207=20=E9=A1=B9=E2=80=94=E2=80=94Hailo=20?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E9=93=BE=E8=AF=AD=E4=B9=89=E5=AF=B9=E9=BD=90?= =?UTF-8?q?=20+=20NaN=20=E5=AE=89=E5=85=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1-1: export_deploy.py forward() 从硬编码 3 帧改为 *inputs 动态解析 P1-2: hailo_infer.py 补 predict_delta 支持（残差模式 + --ckpt 自动检测） P1-3: pybot _load_predictor 自动查找 pred_token.npy/action_stats.npy/ companion checkpoint，恢复 Hailo 推理训练语义 P1-4: pretrain_mt.py random_images 每 batch 重新生成噪声，防 BN 退化 P2-1: predict.py/hailo_infer.py/robot.py 三层 NaN/Inf 防线 P2-3: recorder.py imu_dim 默认 8→0，避免外部直接使用时形状错误 P2-5: _npu_test.py imu_dim 移到函数定义前，消除执行顺序依赖 --- actpred/_npu_test.py | 2 +- actpred/export_deploy.py | 10 +++++--- actpred/hailo_infer.py | 43 +++++++++++++++++++++++++++++++-- actpred/predict.py | 6 ++++- actpred/pretrain_mt.py | 4 +-- pybot/pybot/main.py | 47 ++++++++++++++++++++++++++++++++++++ pybot/pybot/recorder.py | 2 +- pybot/pybot/robot.py | 4 +++ pybot/tests/test_recorder.py | 4 +-- 9 files changed, 109 insertions(+), 13 deletions(-) diff --git a/actpred/_npu_test.py b/actpred/_npu_test.py index 63da6c7..85d9ece 100644 --- a/actpred/_npu_test.py +++ b/actpred/_npu_test.py @@ -55,6 +55,7 @@ model = ActionPredictor(cfg.model, cfg.data, pretrained=False) model.load_state_dict(ck["model_state_dict"]) model.eval() pred_learned = model.pred_token.data.unsqueeze(1).numpy() # (1,1,1,d) +imu_dim = cfg.data.imu_dim # 归一化模型：torch 参考在喂入前 z-score；HailoPredictor 传 stats_path # 由封装内部归一化（hist_real/hist_zero 保持原始值，避免双重归一化） @@ -101,7 +102,6 @@ def torch_pred(imgs, hist, use_learned_pred=True, imu=None): # R2 方案一：imu_dim>0 时每步传原始姿态（HailoPredictor 内部归一化） -imu_dim = cfg.data.imu_dim imu_real = np.array([5.0, -3.0] + [0.0] * 6, dtype=np.float32) with HailoPredictor(HEF_PATH, stats_path=stats_path) as p: diff --git a/actpred/export_deploy.py b/actpred/export_deploy.py index 7d35d97..b2b29dc 100644 --- a/actpred/export_deploy.py +++ b/actpred/export_deploy.py @@ -132,11 +132,13 @@ class DeployWrapper(nn.Module): x = self.proj_convs[i](x) # 1x1 conv ≡ 池化后的 img_proj return self.pool(x) # (1, d, 1, 1) NCHW - def forward(self, frame0: torch.Tensor, frame1: torch.Tensor, frame2: torch.Tensor, - action_history: torch.Tensor, pred_token: torch.Tensor, - imu: torch.Tensor = None) -> torch.Tensor: + def forward(self, *inputs: torch.Tensor) -> torch.Tensor: + # inputs = frame0..frame{F-1}, action_history, pred_token, [imu] # 全程 (1, 1, S, d) 布局：seq 占 W 维，避免任何 transpose - frames = [frame0, frame1, frame2] + frames = list(inputs[:self._frame_len]) + action_history = inputs[self._frame_len] + pred_token = inputs[self._frame_len + 1] + imu = inputs[self._frame_len + 2] if self._imu_dim > 0 else None feats = [self._frame_feat(frames[i], i).permute(0, 2, 3, 1) for i in range(self._frame_len)] # (1, 1, 1, d) NHWC img_tokens = torch.cat(feats, dim=2) + self.proj_bias # (1, 1, F, d) diff --git a/actpred/hailo_infer.py b/actpred/hailo_infer.py index 0a0dc67..7636407 100644 --- a/actpred/hailo_infer.py +++ b/actpred/hailo_infer.py @@ -72,9 +72,26 @@ class HailoPredictor: action = predictor.predict(images, action_history) """ - def __init__(self, hef_path: str, stats_path: str = None, vdevice=None): + def __init__(self, hef_path: str, stats_path: str = None, vdevice=None, + predict_delta: bool = False, pred_token=None): + """HEF 推理器。 + + predict_delta: 模型用 --predict-delta 训练时须置 True，推理输出 + （归一化空间）会加回 history 末步再反归一化，与 predict.py 对齐。 + pred_token: np.ndarray 或 .npy 路径；export_deploy.py --pred-token-out + 产物。缺省时喂零（仅适合 pred_token 恰好全零的模型）。 + """ from hailo_platform import HEF, VDevice + self._predict_delta = predict_delta + self._default_pred_token = None + if pred_token is not None: + if isinstance(pred_token, str): + self._default_pred_token = np.load(pred_token) + else: + self._default_pred_token = np.asarray(pred_token, + dtype=np.float32) + self._stack = contextlib.ExitStack() if vdevice is None: vdevice = self._stack.enter_context(VDevice()) @@ -258,6 +275,8 @@ class HailoPredictor: if self._pred_info is not None: p_shape = tuple(self._pred_info.shape) + if pred_token is None: + pred_token = self._default_pred_token if pred_token is None: values = np.zeros(p_shape, dtype=np.float32) else: @@ -294,8 +313,14 @@ class HailoPredictor: q = self._out_info.quant_info result = (result - q.qp_zp) * q.qp_scale result = result.reshape(-1) + if self._predict_delta: + # 残差模式：Δ̂ + a_{t-1}（归一化空间），与 predict.py 对齐 + result = result + action_history[-1] if self._action_mean is not None: result = result * self._action_std + self._action_mean + if not np.all(np.isfinite(result)): + print("WARNING: HEF output contains NaN/Inf, replacing with zeros") + result = np.nan_to_num(result, nan=0.0, posinf=0.0, neginf=0.0) return result def _frame_float(self, image) -> np.ndarray: @@ -349,6 +374,9 @@ def main(): parser.add_argument("--pred-token", default=None, help="pred_token .npy 路径（export_deploy.py " "--pred-token-out 产物；缺省喂零）") + parser.add_argument("--ckpt", default=None, + help="checkpoint .pt 路径（自动检测 predict_delta；" + "HEF 部署时需要以恢复训练配置）") parser.add_argument("--imu", type=str, default=None, help="可选姿态向量（逗号分隔，原始量纲：roll,pitch 度 / " "ax..gz int16 原始值）；HEF 含 imu 输入且缺省时按零向量") @@ -361,7 +389,18 @@ def main(): if args.imu is not None: imu = np.array([float(v) for v in args.imu.split(",")], dtype=np.float32) - with HailoPredictor(args.hef, stats_path=args.stats) as predictor: + + predict_delta = False + if args.ckpt: + import torch + ck = torch.load(args.ckpt, map_location="cpu", weights_only=True) + cfg = ck.get("config", {}) + predict_delta = cfg.get("train", {}).get("predict_delta", False) + if predict_delta: + print("predict_delta=True: HEF 输出将加回历史末步（残差模式）") + + with HailoPredictor(args.hef, stats_path=args.stats, + predict_delta=predict_delta) as predictor: if len(args.images) != predictor.frame_len: raise SystemExit( f"--images 需传入 {predictor.frame_len} 个路径（旧→新），" diff --git a/actpred/predict.py b/actpred/predict.py index e671e43..195a3b9 100644 --- a/actpred/predict.py +++ b/actpred/predict.py @@ -108,7 +108,11 @@ class ActionPredictorInference: pred = history[0, -1] + pred if self._action_mean is not None: pred = pred * self._action_std + self._action_mean - return pred.cpu().numpy() + result = pred.cpu().numpy() + if not np.all(np.isfinite(result)): + print("WARNING: predict output contains NaN/Inf, replacing with zeros") + result = np.nan_to_num(result, nan=0.0, posinf=0.0, neginf=0.0) + return result def parse_action_history(s: str, action_dim: int) -> np.ndarray: diff --git a/actpred/pretrain_mt.py b/actpred/pretrain_mt.py index 95ef97e..c55138b 100644 --- a/actpred/pretrain_mt.py +++ b/actpred/pretrain_mt.py @@ -121,7 +121,7 @@ def pretrain_one_epoch(model, loader, optimizer, device, image_width, if noise_std > 0: history = history + torch.randn_like(history) * noise_std target = target + torch.randn_like(target) * noise_std - if images is None or images.size(0) != history.size(0): + if images is None or images.size(0) != history.size(0) or random_images: images = _placeholder_images( history.size(0), frame_len, image_height, image_width, device, random_images) @@ -147,7 +147,7 @@ def validate(model, loader, device, image_width, image_height, frame_len, for batch in tqdm(loader, desc=" val ", leave=False): history = batch["action_history"].to(device) target = batch["target"].to(device) - if images is None or images.size(0) != history.size(0): + if images is None or images.size(0) != history.size(0) or random_images: images = _placeholder_images( history.size(0), frame_len, image_height, image_width, device, random_images) diff --git a/pybot/pybot/main.py b/pybot/pybot/main.py index 04ffed2..8cde321 100644 --- a/pybot/pybot/main.py +++ b/pybot/pybot/main.py @@ -134,10 +134,57 @@ def _load_predictor(checkpoint): pybot 不硬依赖 actpred：仅 --loop 时才尝试 import，actpred 不在 sys.path 时给出可操作提示。checkpoint 目录名决定后端（.hef → Hailo NPU，否则 → PyTorch），推理类由 actpred 提供。 + + HEF 后端会自动查找同目录下的 pred_token.npy 和 companion checkpoint + （pretrain_best_inference.pt / best_inference.pt），以恢复 pred_token + 默认值和 predict_delta 训练配置——否则 Hailo 推理输出与训练语义不一致。 """ + import os try: if checkpoint.endswith(".hef"): from actpred.hailo_infer import HailoPredictor as _P + hef_dir = os.path.dirname(os.path.abspath(checkpoint)) + + # pred_token：export_deploy.py --pred-token-out 的产物 + pred_token = None + pt_path = os.path.join(hef_dir, "pred_token.npy") + if os.path.exists(pt_path): + pred_token = pt_path + logging.info("found pred_token: %s", pt_path) + + # stats_path：归一化模型的 action_stats.npy + stats_path = None + sp = os.path.join(hef_dir, "action_stats.npy") + if os.path.exists(sp): + stats_path = sp + + # predict_delta：从同目录 checkpoint 读取训练配置 + predict_delta = False + for name in ("pretrain_best_inference.pt", + "best_inference.pt"): + ckpt_path = os.path.join(hef_dir, name) + if os.path.exists(ckpt_path): + try: + import torch + ck = torch.load(ckpt_path, map_location="cpu", + weights_only=True) + cfg = ck.get("config", {}) + predict_delta = cfg.get("train", {}).get( + "predict_delta", False) + if predict_delta: + logging.info( + "predict_delta=True from %s", ckpt_path) + except Exception: + pass + break + + try: + return _P(checkpoint, stats_path=stats_path, + predict_delta=predict_delta, + pred_token=pred_token) + except Exception: + logging.exception("加载 HEF 推理器失败: %s", checkpoint) + return None else: from actpred.predict import ActionPredictorInference as _P except ImportError: diff --git a/pybot/pybot/recorder.py b/pybot/pybot/recorder.py index 698b920..36cb2e0 100644 --- a/pybot/pybot/recorder.py +++ b/pybot/pybot/recorder.py @@ -34,7 +34,7 @@ class EpisodeRecorder: 全程 None（无 IMU 硬件）则不产出 imu.npy，保持旧格式兼容。 """ - def __init__(self, output_dir, action_dim=18, imu_dim=8): + def __init__(self, output_dir, action_dim=18, imu_dim=0): self.action_dim = action_dim self.imu_dim = imu_dim self._frame_idx = 0 diff --git a/pybot/pybot/robot.py b/pybot/pybot/robot.py index 8dbcd21..cd7845d 100644 --- a/pybot/pybot/robot.py +++ b/pybot/pybot/robot.py @@ -368,6 +368,10 @@ class RobotStruct: SpeakerVar.play 强制要求 delay 键；闭环已剔除 delay 列（避免发声 阻塞卡死节拍），此处为其补 delay=0 —— 发声但不阻塞，节拍由 hz 掌控。 """ + import numpy as np + if not np.all(np.isfinite(act)): + log.warning("action vector contains NaN/Inf, skipping step") + return row_sets = {} for j, dev, rp in exec_cols: row_sets.setdefault(dev, {})[rp] = float(act[j]) diff --git a/pybot/tests/test_recorder.py b/pybot/tests/test_recorder.py index dd6be13..4d58743 100644 --- a/pybot/tests/test_recorder.py +++ b/pybot/tests/test_recorder.py @@ -314,7 +314,7 @@ cam0.frame; def test_imu_npy_written_with_dict(tmp_path): """record 传姿态 dict → flush 产出 imu.npy（T×8），键序 roll..gz""" - rec = EpisodeRecorder(str(tmp_path), action_dim=18) + rec = EpisodeRecorder(str(tmp_path), action_dim=18, imu_dim=8) imu = {"roll": 1.0, "pitch": 2.0, "ax": 3.0, "ay": 4.0, "az": 5.0, "gx": 6.0, "gy": 7.0, "gz": 8.0} rec.record(None, [0.0], imu_vec=imu) @@ -328,7 +328,7 @@ def test_imu_npy_written_with_dict(tmp_path): def test_imu_mixed_none_marks_zero(tmp_path): """混合数据集：部分帧无姿态 → 该帧记零向量，仍产出 imu.npy""" - rec = EpisodeRecorder(str(tmp_path), action_dim=18) + rec = EpisodeRecorder(str(tmp_path), action_dim=18, imu_dim=8) rec.record(None, [0.0], imu_vec=None) rec.record(None, [0.0], imu_vec={"roll": 9.0}) rec.flush()