feat(pybot×actpred): R2 姿态反馈方案一——MPU6050 独立 IMU token 全链路
- pybot:recorder 逐帧落盘 imu.npy(T×8,roll,pitch,ax,ay,az,gx,gy,gz,与帧/动作同调用对齐);run_loop 对声明 imu_dim 的 predictor 以 predict(..., imu=) 调用(鸭子类型,旧两参调用向后兼容) - actpred:新增 --imu-dim;model 序列 [img×F, act×H, (IMU), PRED](imu_dim>0 时 F+H+2,IMU token 位于 PRED 前);dataset/train 读 imu.npy(缺失按零向量兜底);predict/ONNX/Hailo 三后端与 export/gen_calibration/dfc_compile/_npu_test 部署链适配 imu 输入 - 姿态按物理量程固定缩放归一化(imu_norm.py,训练/推理共用,零状态) - 测试:actpred 128 passed / pybot loop+recorder 28 passed
This commit is contained in:
1 parent
3e773debc9
commit
a2ba797410
22 files changed
+1236
-133
No files matched your search
+12
-6
@@ -57,7 +57,8 @@ def train_one_epoch(model, loader, optimizer, device, max_grad_norm: float = Non
|
||||
images = batch["images"].to(device)
|
||||
history = batch["action_history"].to(device)
|
||||
target = batch["target"].to(device)
|
||||
pred = model(images, history)
|
||||
imu = batch["imu"].to(device) if "imu" in batch else None
|
||||
pred = model(images, history, imu)
|
||||
loss = torch.nn.functional.mse_loss(pred, target)
|
||||
bs = images.size(0)
|
||||
else:
|
||||
@@ -71,10 +72,11 @@ def train_one_epoch(model, loader, optimizer, device, max_grad_norm: float = Non
|
||||
true_history = step["action_history"].to(device)
|
||||
target = step["target"].to(device)
|
||||
prev_action = step["prev_action"].to(device)
|
||||
imu = step["imu"].to(device) if "imu" in step else None
|
||||
if k == 0 or rng.random_sample() < ss_prob:
|
||||
history = true_history
|
||||
# 否则沿用上一轮回传的 history(自预测构造)
|
||||
pred = model(images, history)
|
||||
pred = model(images, history, imu)
|
||||
loss = loss + torch.nn.functional.mse_loss(pred, target)
|
||||
bs = images.size(0)
|
||||
# 回传:残差模式先把 Δ̂ 加回 a_{t-1} 得到绝对动作,再移入历史
|
||||
@@ -101,7 +103,8 @@ def validate(model, loader, device, rollout_len: int = 1, predict_delta: bool =
|
||||
images = batch["images"].to(device)
|
||||
history = batch["action_history"].to(device)
|
||||
target = batch["target"].to(device)
|
||||
pred = model(images, history)
|
||||
imu = batch["imu"].to(device) if "imu" in batch else None
|
||||
pred = model(images, history, imu)
|
||||
loss = torch.nn.functional.mse_loss(pred, target)
|
||||
bs = images.size(0)
|
||||
else:
|
||||
@@ -113,7 +116,8 @@ def validate(model, loader, device, rollout_len: int = 1, predict_delta: bool =
|
||||
images = step["images"].to(device)
|
||||
history = step["action_history"].to(device)
|
||||
target = step["target"].to(device)
|
||||
pred = model(images, history)
|
||||
imu = step["imu"].to(device) if "imu" in step else None
|
||||
pred = model(images, history, imu)
|
||||
loss = loss + torch.nn.functional.mse_loss(pred, target)
|
||||
bs = images.size(0)
|
||||
loss = loss / len(batch["steps"])
|
||||
@@ -151,7 +155,8 @@ def main():
|
||||
cfg.data.image_width, cfg.data.image_height, cfg.data.normalize_actions,
|
||||
cfg.data.action_stats_path, train_indices, cfg.data.frame_len,
|
||||
augment=cfg.train.augment,
|
||||
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len)
|
||||
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len,
|
||||
imu_dim=cfg.data.imu_dim)
|
||||
if cfg.data.normalize_actions and not cfg.data.action_stats_path:
|
||||
# 归一化统计量必须只用训练集计算,再供验证集复用,否则 val loss 不可信
|
||||
stats_path = os.path.join(cfg.train.checkpoint_dir, "action_stats.npy")
|
||||
@@ -163,7 +168,8 @@ def main():
|
||||
cfg.data.data_dir, cfg.data.history_len, cfg.data.action_dim,
|
||||
cfg.data.image_width, cfg.data.image_height, cfg.data.normalize_actions,
|
||||
cfg.data.action_stats_path, val_indices, cfg.data.frame_len,
|
||||
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len)
|
||||
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len,
|
||||
imu_dim=cfg.data.imu_dim)
|
||||
|
||||
print(f"Samples: {len(train_ds)} train, {len(val_ds)} val")
|
||||
if len(train_ds) == 0:
|
||||
|
||||
Reference in new issue
Block a user