qing/blackbird
3
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(actpred): R1 残差预测+多步展开训练 与 R4 离线评估基准

Browse Source 
R1（可选开关，默认关，向后兼容）：
- --predict-delta：残差预测，target=Δ=a[t]-a[t-1]（归一化空间），
  推理端 predict 自动加回 a_{t-1}（惯性先验平滑动作）
- --rollout-len K + --ss-prob：多步展开 + scheduled sampling，
  首步真历史、后续步按概率用真历史或自预测回传（detach 截断），
  治 exposure bias；dataset 多步样本返回 {steps:[...]}，自定义 collate
- 旧 checkpoint getattr 默认 False 兼容；pybot 闭环/部署链零改动

R4（新增 evaluate.py）：
- 留出 episode 开环滚动预测，复刻闭环口径（图像真值滚动、动作自预测回写）
- 5 个闭环代理指标：overall_mse/error_growth(发散)/error_curve/
  smoothness_ratio(平滑)/oob_rate(越界，数据经验界 min/max+margin)
- 支持 --compare A/B 对比与 --output JSON 报告
- 端到端验证：正确捕捉开环发散(单步 MSE 盲区)与残差平滑收益

新增 17 用例（R1×10 + R4×7），全量 100 passed 无回归
This commit is contained in:
qing committed 2026-08-14 07:52:22 +08:00
1 parent fe12dd7b33
commit cc2f7bca79
7 files changed
+739 -42
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

actpred/train.py
+95 -22
View File
@@ -21,39 +21,104 @@ def split_episodes(data_dir: str, val_ratio: float, seed: int = 42):
return indices[val_count:], indices[:val_count]
def train_one_epoch(model, loader, optimizer, device, max_grad_norm: float = None):
def _collate(batch):
"""多步样本（{"steps":[...]}）逐 k 聚合成 batch 列表；单步样本走默认 collate。
rollout_len==1 时 __getitem__ 返回标准 dict，直接默认 collate；
rollout_len>1 时返回 {"steps":[step_0..step_{K-1}]}，把每个 step_k 跨样本堆叠，
返回 {"steps":[batch_0..batch_{K-1}]}，供 train_one_epoch 按步 scheduled sampling。
"""
from torch.utils.data._utils.collate import default_collate
if batch and isinstance(batch[0], dict) and "steps" in batch[0]:
k = len(batch[0]["steps"])
return {"steps": [default_collate([b["steps"][i] for b in batch])
for i in range(k)]}
return default_collate(batch)
def _shift_history(history: torch.Tensor, new_action: torch.Tensor) -> torch.Tensor:
"""动作历史左移一步、追加 new_action 到末尾（scheduled sampling 回传自预测用）。
history: (B, H, D)，new_action: (B, D) → (B, H, D)
"""
return torch.cat([history[:, 1:], new_action.unsqueeze(1)], dim=1)
def train_one_epoch(model, loader, optimizer, device, max_grad_norm: float = None,
rollout_len: int = 1, ss_prob: float = 1.0,
predict_delta: bool = False):
model.train()
total_loss = 0.0
count = 0
rng = np.random.RandomState()
for batch in tqdm(loader, desc=" train", leave=False):
images = batch["images"].to(device)
history = batch["action_history"].to(device)
target = batch["target"].to(device)
optimizer.zero_grad()
pred = model(images, history)
loss = torch.nn.functional.mse_loss(pred, target)
if rollout_len == 1:
images = batch["images"].to(device)
history = batch["action_history"].to(device)
target = batch["target"].to(device)
pred = model(images, history)
loss = torch.nn.functional.mse_loss(pred, target)
bs = images.size(0)
else:
# 多步 scheduled sampling：首步必用真历史；后续步以 ss_prob 用真历史、
# 否则用上一步模型预测（detach 截断，避免梯度跨步爆炸）回传构造历史。
loss = 0.0
bs = 0
history = None
for k, step in enumerate(batch["steps"]):
images = step["images"].to(device)
true_history = step["action_history"].to(device)
target = step["target"].to(device)
prev_action = step["prev_action"].to(device)
if k == 0 or rng.random_sample() < ss_prob:
history = true_history
# 否则沿用上一轮回传的 history（自预测构造）
pred = model(images, history)
loss = loss + torch.nn.functional.mse_loss(pred, target)
bs = images.size(0)
# 回传：残差模式先把 Δ̂ 加回 a_{t-1} 得到绝对动作，再移入历史
with torch.no_grad():
abs_action = prev_action + pred if predict_delta else pred
history = _shift_history(true_history, abs_action.detach())
loss = loss / len(batch["steps"])
loss.backward()
if max_grad_norm:
torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
optimizer.step()
total_loss += loss.item() * images.size(0)
count += images.size(0)
total_loss += loss.item() * bs
count += bs
return total_loss / max(count, 1)
@torch.no_grad()
def validate(model, loader, device):
def validate(model, loader, device, rollout_len: int = 1, predict_delta: bool = False):
model.eval()
total_loss = 0.0
count = 0
for batch in tqdm(loader, desc=" val ", leave=False):
images = batch["images"].to(device)
history = batch["action_history"].to(device)
target = batch["target"].to(device)
pred = model(images, history)
loss = torch.nn.functional.mse_loss(pred, target)
total_loss += loss.item() * images.size(0)
count += images.size(0)
if rollout_len == 1:
images = batch["images"].to(device)
history = batch["action_history"].to(device)
target = batch["target"].to(device)
pred = model(images, history)
loss = torch.nn.functional.mse_loss(pred, target)
bs = images.size(0)
else:
# 验证固定全真历史（确定性，不带 scheduled sampling 随机性），
# 对 K 步累加，保证 best 模型选择口径稳定可比。
loss = 0.0
bs = 0
for step in batch["steps"]:
images = step["images"].to(device)
history = step["action_history"].to(device)
target = step["target"].to(device)
pred = model(images, history)
loss = loss + torch.nn.functional.mse_loss(pred, target)
bs = images.size(0)
loss = loss / len(batch["steps"])
total_loss += loss.item() * bs
count += bs
if count == 0:
return float("inf") # 空验证集不参与比较，避免假 best
return total_loss / count
@@ -85,7 +150,8 @@ def main():
cfg.data.data_dir, cfg.data.history_len, cfg.data.action_dim,
cfg.data.image_width, cfg.data.image_height, cfg.data.normalize_actions,
cfg.data.action_stats_path, train_indices, cfg.data.frame_len,
augment=cfg.train.augment)
augment=cfg.train.augment,
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len)
if cfg.data.normalize_actions and not cfg.data.action_stats_path:
# 归一化统计量必须只用训练集计算，再供验证集复用，否则 val loss 不可信
stats_path = os.path.join(cfg.train.checkpoint_dir, "action_stats.npy")
@@ -96,7 +162,8 @@ def main():
val_ds = ActionDataset(
cfg.data.data_dir, cfg.data.history_len, cfg.data.action_dim,
cfg.data.image_width, cfg.data.image_height, cfg.data.normalize_actions,
cfg.data.action_stats_path, val_indices, cfg.data.frame_len)
cfg.data.action_stats_path, val_indices, cfg.data.frame_len,
predict_delta=cfg.train.predict_delta, rollout_len=cfg.train.rollout_len)
print(f"Samples: {len(train_ds)} train, {len(val_ds)} val")
if len(train_ds) == 0:
@@ -111,9 +178,10 @@ def main():
loader_gen.manual_seed(cfg.train.seed)
train_loader = DataLoader(train_ds, batch_size=cfg.train.batch_size,
shuffle=True, num_workers=cfg.train.num_workers,
generator=loader_gen)
generator=loader_gen, collate_fn=_collate)
val_loader = DataLoader(val_ds, batch_size=cfg.train.batch_size,
shuffle=False, num_workers=cfg.train.num_workers)
shuffle=False, num_workers=cfg.train.num_workers,
collate_fn=_collate)
model = ActionPredictor(cfg.model, cfg.data).to(device)
n_params = sum(p.numel() for p in model.parameters())
@@ -154,8 +222,13 @@ def main():
for epoch in range(start_epoch, cfg.train.epochs + 1):
print(f"\nEpoch {epoch}/{cfg.train.epochs} lr={optimizer.param_groups[0]['lr']:.6f}")
train_loss = train_one_epoch(model, train_loader, optimizer, device,
cfg.train.max_grad_norm)
val_loss = validate(model, val_loader, device)
cfg.train.max_grad_norm,
rollout_len=cfg.train.rollout_len,
ss_prob=cfg.train.ss_prob,
predict_delta=cfg.train.predict_delta)
val_loss = validate(model, val_loader, device,
rollout_len=cfg.train.rollout_len,
predict_delta=cfg.train.predict_delta)
scheduler.step()
print(f" train_loss={train_loss:.6f} val_loss={val_loss:.6f}")