feat(pybot×actpred): R2 姿态反馈方案一——MPU6050 独立 IMU token 全链路
- pybot:recorder 逐帧落盘 imu.npy(T×8,roll,pitch,ax,ay,az,gx,gy,gz,与帧/动作同调用对齐);run_loop 对声明 imu_dim 的 predictor 以 predict(..., imu=) 调用(鸭子类型,旧两参调用向后兼容) - actpred:新增 --imu-dim;model 序列 [img×F, act×H, (IMU), PRED](imu_dim>0 时 F+H+2,IMU token 位于 PRED 前);dataset/train 读 imu.npy(缺失按零向量兜底);predict/ONNX/Hailo 三后端与 export/gen_calibration/dfc_compile/_npu_test 部署链适配 imu 输入 - 姿态按物理量程固定缩放归一化(imu_norm.py,训练/推理共用,零状态) - 测试:actpred 128 passed / pybot loop+recorder 28 passed
This commit is contained in:
1 parent
3e773debc9
commit
a2ba797410
22 files changed
+1236
-133
No files matched your search
+31
-10
@@ -69,6 +69,17 @@ if action_stats is not None:
|
||||
pred_learned = model.pred_token.data.unsqueeze(1).numpy() # (1,1,1,128)
|
||||
pred_none = np.zeros((1, 1, 1, 128), dtype=np.float32)
|
||||
|
||||
# R2 方案一:imu_dim>0 时模型/ONNX 多了归一化姿态输入
|
||||
imu_dim = cfg.data.imu_dim
|
||||
imu_real = None
|
||||
imu_zero = None
|
||||
if imu_dim > 0:
|
||||
from imu_norm import normalize_imu
|
||||
# 原始姿态(roll=5°, pitch=-3°, 其余 0)→ 部署版输入布局 (1,1,1,imu_dim)
|
||||
imu_real = normalize_imu([5.0, -3.0] + [0.0] * 6, imu_dim).reshape(
|
||||
1, 1, 1, imu_dim)
|
||||
imu_zero = np.zeros((1, 1, 1, imu_dim), dtype=np.float32)
|
||||
|
||||
|
||||
def frames_nhwc(imgs):
|
||||
"""列表帧 → (1,H,W,3) 归一化 float32 NHWC × frame_len"""
|
||||
@@ -76,35 +87,45 @@ def frames_nhwc(imgs):
|
||||
return [chw[i].transpose(1, 2, 0)[None] for i in range(len(imgs))]
|
||||
|
||||
|
||||
def run_onnx(imgs, hist, pred):
|
||||
def run_onnx(imgs, hist, pred, imu=None):
|
||||
feeds = {f"frame{i}": f for i, f in enumerate(frames_nhwc(imgs))}
|
||||
feeds["action_history"] = hist[None, None]
|
||||
feeds["pred_token"] = pred
|
||||
if imu_dim > 0:
|
||||
feeds["imu"] = imu if imu is not None else imu_zero
|
||||
return sess.run(None, feeds)[0].reshape(-1)
|
||||
|
||||
|
||||
def run_torch(imgs, hist):
|
||||
def run_torch(imgs, hist, imu=None):
|
||||
x = torch.stack([tf(Image.fromarray(im)) for im in imgs]).unsqueeze(0)
|
||||
hh = torch.tensor(hist).unsqueeze(0)
|
||||
ii = None
|
||||
if imu_dim > 0:
|
||||
ii = torch.tensor(
|
||||
imu if imu is not None else imu_zero).reshape(1, imu_dim).float()
|
||||
with torch.no_grad():
|
||||
return model(x, hh).squeeze(0).numpy()
|
||||
return model(x, hh, ii).squeeze(0).numpy()
|
||||
|
||||
|
||||
a = run_onnx(imgs_real, hist_real, pred_learned)
|
||||
b = run_onnx(imgs_zero, hist_real, pred_learned)
|
||||
c = run_onnx(imgs_real, hist_zero, pred_learned)
|
||||
a = run_onnx(imgs_real, hist_real, pred_learned, imu_real)
|
||||
b = run_onnx(imgs_zero, hist_real, pred_learned, imu_real)
|
||||
c = run_onnx(imgs_real, hist_zero, pred_learned, imu_real)
|
||||
d = run_onnx(imgs_zero, hist_zero, pred_learned)
|
||||
p0 = run_onnx(imgs_real, hist_real, pred_none)
|
||||
p0 = run_onnx(imgs_real, hist_real, pred_none, imu_real)
|
||||
|
||||
print(f"onnx real_img+real_hist: {np.round(a[:6], 3)}...")
|
||||
print(f"onnx pred_token=0 diff : max={np.abs(a - p0).max():.4f}")
|
||||
print(f"img sensitivity (a-b): max={np.abs(a - b).max():.4f}")
|
||||
print(f"hist sensitivity (a-c): max={np.abs(a - c).max():.4f}")
|
||||
|
||||
ref = run_torch(imgs_real, hist_real)
|
||||
ok = np.abs(a - b).max() > 1e-4 and np.abs(a - c).max() > 1e-4
|
||||
if imu_dim > 0:
|
||||
e = run_onnx(imgs_real, hist_real, pred_learned, imu_zero)
|
||||
print(f"imu sensitivity (a-e): max={np.abs(a - e).max():.4f}")
|
||||
|
||||
ref = run_torch(imgs_real, hist_real, imu_real)
|
||||
print(f"torch reference : {np.round(ref[:6], 3)}...")
|
||||
print(f"onnx vs torch max_diff : {np.abs(a - ref).max():.6e}")
|
||||
|
||||
ok = (np.abs(a - b).max() > 1e-4 and np.abs(a - c).max() > 1e-4
|
||||
and np.abs(a - ref).max() < 1e-4)
|
||||
ok = ok and np.abs(a - ref).max() < 1e-4
|
||||
print("PASS" if ok else "FAIL")
|
||||
Reference in new issue
Block a user