Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6a1cadf25 | ||
|
|
78bf74b067 | ||
|
|
575c2a27a4 |
No files matched your search
@@ -0,0 +1,24 @@
|
||||
# Virtual environment
|
||||
venv/
|
||||
.env/
|
||||
|
||||
# Python bytecode
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# Checkpoints and trained models
|
||||
checkpoints/
|
||||
*.pt
|
||||
*.pth
|
||||
*.onnx
|
||||
|
||||
# Dataset (too large for git)
|
||||
data/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
@@ -0,0 +1,83 @@
|
||||
# actpred — 动作预测模块
|
||||
|
||||
输入摄像头图像 + 历史动作序列,预测下一个连续动作(回归)。
|
||||
|
||||
纯神经网络实现(PyTorch),支持轻量 CNN 或 MobileNetV2 图像编码,MLP/GRU 动作历史编码。总参数量 ~140K(默认)至 ~2.3M。
|
||||
|
||||
## 架构
|
||||
|
||||
```
|
||||
图像 (3×64×64) ──► CNN 编码器 ──► img_feat (128)
|
||||
│
|
||||
├─ concat ──► 融合头 ──► 预测动作 (action_dim)
|
||||
│
|
||||
动作历史 (H×D) ──► MLP/GRU ──► act_feat (64)
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
```bash
|
||||
# 安装
|
||||
cd actpred && bash _set.sh
|
||||
|
||||
# 训练
|
||||
bash _run.sh train --data-dir ./data --epochs 100 --action-dim 8
|
||||
|
||||
# 推理
|
||||
bash _run.sh --model checkpoints/best_inference.pt --image frame.jpg \
|
||||
--action-history "45,0,45,0,45,0,45,0:45,10,45,10,45,10,45,10"
|
||||
|
||||
# 导出 ONNX
|
||||
bash _run.sh export --model checkpoints/best_inference.pt --output model.onnx
|
||||
```
|
||||
|
||||
## 数据格式
|
||||
|
||||
```
|
||||
data/
|
||||
episode_001/
|
||||
images/
|
||||
0000.jpg
|
||||
0001.jpg
|
||||
...
|
||||
actions.npy # shape: (T, action_dim) float32
|
||||
episode_002/
|
||||
...
|
||||
```
|
||||
|
||||
每个 timestep 的样本:当前图像 + 前 `history_len` 步动作 → 当前步动作。
|
||||
|
||||
### 数据采集
|
||||
|
||||
```python
|
||||
from dataset import DataRecorder
|
||||
|
||||
recorder = DataRecorder("./data", action_dim=8)
|
||||
for image, action in robot_loop():
|
||||
recorder.record(image, action)
|
||||
recorder.flush()
|
||||
```
|
||||
|
||||
## 配置
|
||||
|
||||
所有超参数通过 CLI 参数覆盖,默认值见 `config.py`:
|
||||
|
||||
| 参数 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `--image-size` | 64 | 图像缩放尺寸 |
|
||||
| `--action-dim` | 8 | 动作向量维度 |
|
||||
| `--history-len` | 5 | 历史动作窗口长度 |
|
||||
| `--backbone` | simple | CNN 类型 (simple/mobilenet) |
|
||||
| `--history-encoder` | mlp | 历史编码器 (mlp/gru) |
|
||||
| `--epochs` | 100 | 训练轮数 |
|
||||
| `--lr` | 0.001 | 学习率 |
|
||||
| `--batch-size` | 32 | 批次大小 |
|
||||
|
||||
## 编程 API
|
||||
|
||||
```python
|
||||
from predict import ActionPredictorInference
|
||||
|
||||
predictor = ActionPredictorInference("checkpoints/best_inference.pt")
|
||||
action = predictor.predict(image_rgb, action_history) # np.ndarray (action_dim,)
|
||||
```
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")" || exit
|
||||
source ./venv/bin/activate
|
||||
|
||||
if [ "${1:-}" = "train" ]; then
|
||||
shift
|
||||
exec python3 train.py "$@"
|
||||
elif [ "${1:-}" = "export" ]; then
|
||||
shift
|
||||
exec python3 export_onnx.py "$@"
|
||||
else
|
||||
exec python3 predict.py "$@"
|
||||
fi
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")" || exit
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
# CPU-only PyTorch (no CUDA dependency)
|
||||
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
|
||||
|
||||
# Remaining deps from Tsinghua mirror
|
||||
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
|
||||
mkdir -p checkpoints data
|
||||
@@ -0,0 +1,110 @@
|
||||
import argparse
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataConfig:
|
||||
image_size: int = 64
|
||||
action_dim: int = 8
|
||||
history_len: int = 5
|
||||
data_dir: str = "./data"
|
||||
normalize_actions: bool = False
|
||||
action_stats_path: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelConfig:
|
||||
cnn_backbone: str = "simple"
|
||||
cnn_feature_dim: int = 128
|
||||
action_hidden_dim: int = 64
|
||||
fused_hidden_dim: int = 128
|
||||
dropout: float = 0.1
|
||||
history_encoder: str = "mlp"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainConfig:
|
||||
batch_size: int = 32
|
||||
epochs: int = 100
|
||||
lr: float = 1e-3
|
||||
weight_decay: float = 1e-4
|
||||
val_split: float = 0.15
|
||||
checkpoint_dir: str = "./checkpoints"
|
||||
patience: int = 15
|
||||
num_workers: int = 2
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
data: DataConfig = field(default_factory=DataConfig)
|
||||
model: ModelConfig = field(default_factory=ModelConfig)
|
||||
train: TrainConfig = field(default_factory=TrainConfig)
|
||||
|
||||
def to_dict(self):
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d):
|
||||
return cls(
|
||||
data=DataConfig(**d.get("data", {})),
|
||||
model=ModelConfig(**d.get("model", {})),
|
||||
train=TrainConfig(**d.get("train", {})),
|
||||
)
|
||||
|
||||
|
||||
def build_argparser(description: str = "") -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=description)
|
||||
parser.add_argument("--image-size", type=int, default=None)
|
||||
parser.add_argument("--action-dim", type=int, default=None)
|
||||
parser.add_argument("--history-len", type=int, default=None)
|
||||
parser.add_argument("--data-dir", type=str, default=None)
|
||||
parser.add_argument("--normalize-actions", action="store_true", default=None)
|
||||
parser.add_argument("--action-stats-path", type=str, default=None)
|
||||
parser.add_argument("--backbone", type=str, default=None, choices=["simple", "mobilenet"])
|
||||
parser.add_argument("--cnn-feature-dim", type=int, default=None)
|
||||
parser.add_argument("--action-hidden-dim", type=int, default=None)
|
||||
parser.add_argument("--fused-hidden-dim", type=int, default=None)
|
||||
parser.add_argument("--dropout", type=float, default=None)
|
||||
parser.add_argument("--history-encoder", type=str, default=None, choices=["mlp", "gru"])
|
||||
parser.add_argument("--batch-size", type=int, default=None)
|
||||
parser.add_argument("--epochs", type=int, default=None)
|
||||
parser.add_argument("--lr", type=float, default=None)
|
||||
parser.add_argument("--weight-decay", type=float, default=None)
|
||||
parser.add_argument("--val-split", type=float, default=None)
|
||||
parser.add_argument("--checkpoint-dir", type=str, default=None)
|
||||
parser.add_argument("--patience", type=int, default=None)
|
||||
parser.add_argument("--num-workers", type=int, default=None)
|
||||
parser.add_argument("--save-config", type=str, default=None, help="保存配置到 JSON 文件")
|
||||
return parser
|
||||
|
||||
|
||||
def apply_overrides(cfg: Config, args: argparse.Namespace) -> Config:
|
||||
mapping = {
|
||||
"image_size": ("data", "image_size"),
|
||||
"action_dim": ("data", "action_dim"),
|
||||
"history_len": ("data", "history_len"),
|
||||
"data_dir": ("data", "data_dir"),
|
||||
"normalize_actions": ("data", "normalize_actions"),
|
||||
"action_stats_path": ("data", "action_stats_path"),
|
||||
"backbone": ("model", "cnn_backbone"),
|
||||
"cnn_feature_dim": ("model", "cnn_feature_dim"),
|
||||
"action_hidden_dim": ("model", "action_hidden_dim"),
|
||||
"fused_hidden_dim": ("model", "fused_hidden_dim"),
|
||||
"dropout": ("model", "dropout"),
|
||||
"history_encoder": ("model", "history_encoder"),
|
||||
"batch_size": ("train", "batch_size"),
|
||||
"epochs": ("train", "epochs"),
|
||||
"lr": ("train", "lr"),
|
||||
"weight_decay": ("train", "weight_decay"),
|
||||
"val_split": ("train", "val_split"),
|
||||
"checkpoint_dir": ("train", "checkpoint_dir"),
|
||||
"patience": ("train", "patience"),
|
||||
"num_workers": ("train", "num_workers"),
|
||||
}
|
||||
args_dict = vars(args)
|
||||
for arg_name, (section, field) in mapping.items():
|
||||
val = args_dict.get(arg_name)
|
||||
if val is not None:
|
||||
setattr(getattr(cfg, section), field, val)
|
||||
return cfg
|
||||
@@ -0,0 +1,104 @@
|
||||
import os
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from PIL import Image
|
||||
|
||||
import torch
|
||||
from torch.utils.data import Dataset
|
||||
from torchvision import transforms
|
||||
|
||||
|
||||
class ActionDataset(Dataset):
|
||||
def __init__(self, data_dir: str, history_len: int, action_dim: int,
|
||||
image_size: int = 64, normalize_actions: bool = False,
|
||||
action_stats_path: str = None, episode_indices: list = None):
|
||||
self.history_len = history_len
|
||||
self.action_dim = action_dim
|
||||
self.image_size = image_size
|
||||
|
||||
self.transform = transforms.Compose([
|
||||
transforms.Resize((image_size, image_size)),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
||||
])
|
||||
|
||||
data_path = Path(data_dir)
|
||||
episodes = sorted([d for d in data_path.iterdir() if d.is_dir()])
|
||||
if episode_indices is not None:
|
||||
episodes = [episodes[i] for i in episode_indices]
|
||||
|
||||
self.samples = []
|
||||
for ep_dir in episodes:
|
||||
actions_path = ep_dir / "actions.npy"
|
||||
images_dir = ep_dir / "images"
|
||||
if not actions_path.exists() or not images_dir.is_dir():
|
||||
continue
|
||||
actions = np.load(actions_path).astype(np.float32)
|
||||
n_frames = len(actions)
|
||||
image_files = sorted(images_dir.glob("*.jpg")) + sorted(images_dir.glob("*.png"))
|
||||
n_images = len(image_files)
|
||||
n = min(n_frames, n_images)
|
||||
for t in range(history_len, n):
|
||||
self.samples.append((image_files[t], actions[t - history_len:t], actions[t]))
|
||||
|
||||
self._action_mean = None
|
||||
self._action_std = None
|
||||
if normalize_actions:
|
||||
if action_stats_path and os.path.exists(action_stats_path):
|
||||
stats = np.load(action_stats_path)
|
||||
self._action_mean = stats[0]
|
||||
self._action_std = stats[1]
|
||||
else:
|
||||
all_actions = np.stack([s[2] for s in self.samples])
|
||||
self._action_mean = all_actions.mean(axis=0)
|
||||
self._action_std = all_actions.std(axis=0) + 1e-8
|
||||
|
||||
def __len__(self):
|
||||
return len(self.samples)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
image_path, action_history, target = self.samples[idx]
|
||||
image = Image.open(image_path).convert("RGB")
|
||||
image = self.transform(image)
|
||||
action_history = torch.tensor(action_history, dtype=torch.float32)
|
||||
target = torch.tensor(target, dtype=torch.float32)
|
||||
if self._action_mean is not None:
|
||||
action_history = (action_history - self._action_mean) / self._action_std
|
||||
target = (target - self._action_mean) / self._action_std
|
||||
return {"image": image, "action_history": action_history, "target": target}
|
||||
|
||||
def get_action_stats(self):
|
||||
if self._action_mean is not None:
|
||||
return np.stack([self._action_mean, self._action_std])
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_episode_count(data_dir: str) -> int:
|
||||
data_path = Path(data_dir)
|
||||
return len([d for d in data_path.iterdir() if d.is_dir()])
|
||||
|
||||
|
||||
class DataRecorder:
|
||||
def __init__(self, output_dir: str, action_dim: int):
|
||||
self.output_dir = Path(output_dir)
|
||||
self.action_dim = action_dim
|
||||
self._frame_idx = 0
|
||||
self._actions = []
|
||||
ep_idx = 0
|
||||
while (self.output_dir / f"episode_{ep_idx:04d}").exists():
|
||||
ep_idx += 1
|
||||
self.episode_dir = self.output_dir / f"episode_{ep_idx:04d}"
|
||||
self.images_dir = self.episode_dir / "images"
|
||||
self.images_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def record(self, image: np.ndarray, action: np.ndarray):
|
||||
img = Image.fromarray(image)
|
||||
img.save(self.images_dir / f"{self._frame_idx:04d}.jpg", quality=90)
|
||||
self._actions.append(action.astype(np.float32))
|
||||
self._frame_idx += 1
|
||||
|
||||
def flush(self):
|
||||
if self._actions:
|
||||
actions = np.array(self._actions, dtype=np.float32)
|
||||
np.save(self.episode_dir / "actions.npy", actions)
|
||||
return str(self.episode_dir)
|
||||
@@ -0,0 +1,44 @@
|
||||
import argparse
|
||||
import torch
|
||||
from config import DataConfig, ModelConfig
|
||||
from model import ActionPredictor
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Export actpred model to ONNX")
|
||||
parser.add_argument("--model", required=True, help="checkpoint path (best_inference.pt)")
|
||||
parser.add_argument("--output", default="model.onnx", help="output ONNX file path")
|
||||
args = parser.parse_args()
|
||||
|
||||
checkpoint = torch.load(args.model, map_location="cpu", weights_only=True)
|
||||
cfg_dict = checkpoint["config"]
|
||||
data_cfg = DataConfig(**cfg_dict["data"])
|
||||
model_cfg = ModelConfig(**cfg_dict["model"])
|
||||
|
||||
model = ActionPredictor(model_cfg, data_cfg)
|
||||
model.load_state_dict(checkpoint["model_state_dict"])
|
||||
model.eval()
|
||||
|
||||
dummy_image = torch.randn(1, 3, data_cfg.image_size, data_cfg.image_size)
|
||||
if model_cfg.history_encoder == "gru":
|
||||
dummy_history = torch.randn(1, data_cfg.history_len, data_cfg.action_dim)
|
||||
else:
|
||||
dummy_history = torch.randn(1, data_cfg.history_len, data_cfg.action_dim)
|
||||
|
||||
torch.onnx.export(
|
||||
model, (dummy_image, dummy_history),
|
||||
args.output,
|
||||
input_names=["image", "action_history"],
|
||||
output_names=["predicted_action"],
|
||||
dynamic_axes={
|
||||
"image": {0: "batch"},
|
||||
"action_history": {0: "batch"},
|
||||
"predicted_action": {0: "batch"},
|
||||
},
|
||||
opset_version=17,
|
||||
)
|
||||
print(f"ONNX model exported to: {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,117 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from config import DataConfig, ModelConfig
|
||||
|
||||
|
||||
class SimpleCNN(nn.Module):
|
||||
def __init__(self, feature_dim: int = 128):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Conv2d(3, 16, 3, stride=2, padding=1),
|
||||
nn.BatchNorm2d(16),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Conv2d(16, 32, 3, stride=2, padding=1),
|
||||
nn.BatchNorm2d(32),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Conv2d(32, 64, 3, stride=2, padding=1),
|
||||
nn.BatchNorm2d(64),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Conv2d(64, feature_dim, 3, stride=2, padding=1),
|
||||
nn.BatchNorm2d(feature_dim),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.AdaptiveAvgPool2d(1),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.net(x).flatten(1)
|
||||
|
||||
|
||||
class MobileNetEncoder(nn.Module):
|
||||
def __init__(self, feature_dim: int = 128):
|
||||
super().__init__()
|
||||
from torchvision.models import mobilenet_v2, MobileNet_V2_Weights
|
||||
backbone = mobilenet_v2(weights=MobileNet_V2_Weights.DEFAULT)
|
||||
self.features = backbone.features
|
||||
self.pool = nn.AdaptiveAvgPool2d(1)
|
||||
self.proj = nn.Linear(1280, feature_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = self.features(x)
|
||||
x = self.pool(x).flatten(1)
|
||||
return self.proj(x)
|
||||
|
||||
|
||||
class MLPHistoryEncoder(nn.Module):
|
||||
def __init__(self, input_dim: int, hidden_dim: int = 64, dropout: float = 0.1):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(input_dim, hidden_dim),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Dropout(dropout),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(inplace=True),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class GRUHistoryEncoder(nn.Module):
|
||||
def __init__(self, input_dim: int, hidden_dim: int = 64, dropout: float = 0.1):
|
||||
super().__init__()
|
||||
self.gru = nn.GRU(input_size=input_dim, hidden_size=hidden_dim,
|
||||
num_layers=1, batch_first=True)
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
_, h = self.gru(x)
|
||||
return self.dropout(h.squeeze(0))
|
||||
|
||||
|
||||
class FusionHead(nn.Module):
|
||||
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, dropout: float = 0.1):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(input_dim, hidden_dim),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Dropout(dropout),
|
||||
nn.Linear(hidden_dim, hidden_dim // 2),
|
||||
nn.ReLU(inplace=True),
|
||||
nn.Linear(hidden_dim // 2, output_dim),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class ActionPredictor(nn.Module):
|
||||
def __init__(self, model_cfg: ModelConfig, data_cfg: DataConfig):
|
||||
super().__init__()
|
||||
if model_cfg.cnn_backbone == "mobilenet":
|
||||
self.image_encoder = MobileNetEncoder(model_cfg.cnn_feature_dim)
|
||||
else:
|
||||
self.image_encoder = SimpleCNN(model_cfg.cnn_feature_dim)
|
||||
|
||||
history_input_dim = data_cfg.history_len * data_cfg.action_dim
|
||||
if model_cfg.history_encoder == "gru":
|
||||
self.action_encoder = GRUHistoryEncoder(
|
||||
data_cfg.action_dim, model_cfg.action_hidden_dim, model_cfg.dropout)
|
||||
else:
|
||||
self.action_encoder = MLPHistoryEncoder(
|
||||
history_input_dim, model_cfg.action_hidden_dim, model_cfg.dropout)
|
||||
|
||||
fused_input = model_cfg.cnn_feature_dim + model_cfg.action_hidden_dim
|
||||
self.head = FusionHead(
|
||||
fused_input, model_cfg.fused_hidden_dim, data_cfg.action_dim, model_cfg.dropout)
|
||||
|
||||
self._use_gru = model_cfg.history_encoder == "gru"
|
||||
self._history_len = data_cfg.history_len
|
||||
self._action_dim = data_cfg.action_dim
|
||||
|
||||
def forward(self, image: torch.Tensor, action_history: torch.Tensor) -> torch.Tensor:
|
||||
img_feat = self.image_encoder(image)
|
||||
if self._use_gru:
|
||||
act_feat = self.action_encoder(action_history)
|
||||
else:
|
||||
act_feat = self.action_encoder(action_history.flatten(1))
|
||||
return self.head(torch.cat([img_feat, act_feat], dim=-1))
|
||||
@@ -0,0 +1,94 @@
|
||||
import argparse
|
||||
import json
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torchvision import transforms
|
||||
|
||||
from config import Config, ModelConfig, DataConfig
|
||||
from model import ActionPredictor
|
||||
|
||||
|
||||
class ActionPredictorInference:
|
||||
def __init__(self, checkpoint_path: str, device: str = None):
|
||||
checkpoint = torch.load(checkpoint_path, map_location="cpu", weights_only=True)
|
||||
cfg_dict = checkpoint["config"]
|
||||
self._data_cfg = DataConfig(**cfg_dict["data"])
|
||||
self._model_cfg = ModelConfig(**cfg_dict["model"])
|
||||
|
||||
if device is None:
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
self._device = torch.device(device)
|
||||
|
||||
self._model = ActionPredictor(self._model_cfg, self._data_cfg).to(self._device)
|
||||
self._model.load_state_dict(checkpoint["model_state_dict"])
|
||||
self._model.eval()
|
||||
|
||||
self._transform = transforms.Compose([
|
||||
transforms.Resize((self._data_cfg.image_size, self._data_cfg.image_size)),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
|
||||
])
|
||||
|
||||
self._action_mean = None
|
||||
self._action_std = None
|
||||
stats_path = checkpoint_path.rsplit("/", 1)[0] + "/action_stats.npy"
|
||||
try:
|
||||
stats = np.load(stats_path)
|
||||
self._action_mean = stats[0]
|
||||
self._action_std = stats[1]
|
||||
except (FileNotFoundError, ValueError):
|
||||
pass
|
||||
|
||||
@torch.no_grad()
|
||||
def predict(self, image: np.ndarray, action_history: np.ndarray) -> np.ndarray:
|
||||
img = Image.fromarray(image) if isinstance(image, np.ndarray) else image
|
||||
img_tensor = self._transform(img).unsqueeze(0).to(self._device)
|
||||
history = torch.tensor(action_history, dtype=torch.float32)
|
||||
if self._action_mean is not None:
|
||||
history = (history - self._action_mean) / self._action_std
|
||||
history = history.unsqueeze(0).to(self._device)
|
||||
pred = self._model(img_tensor, history).squeeze(0).cpu().numpy()
|
||||
if self._action_mean is not None:
|
||||
pred = pred * self._action_std + self._action_mean
|
||||
return pred
|
||||
|
||||
|
||||
def parse_action_history(s: str, action_dim: int) -> np.ndarray:
|
||||
steps = s.split(":")
|
||||
history = []
|
||||
for step in steps:
|
||||
values = [float(v) for v in step.split(",")]
|
||||
assert len(values) == action_dim, \
|
||||
f"Expected {action_dim} values per step, got {len(values)}"
|
||||
history.append(values)
|
||||
return np.array(history, dtype=np.float32)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="actpred inference")
|
||||
parser.add_argument("--model", required=True, help="checkpoint path")
|
||||
parser.add_argument("--image", required=True, help="image file path")
|
||||
parser.add_argument("--action-history", required=True,
|
||||
help="colon-separated action vectors, e.g. '45,0,45,0:45,10,45,10'")
|
||||
parser.add_argument("--action-dim", type=int, default=None,
|
||||
help="override action dim (auto-detected from checkpoint if omitted)")
|
||||
parser.add_argument("--output-format", choices=["json", "raw"], default="json")
|
||||
parser.add_argument("--device", type=str, default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
predictor = ActionPredictorInference(args.model, args.device)
|
||||
action_dim = args.action_dim or predictor._data_cfg.action_dim
|
||||
|
||||
image = Image.open(args.image).convert("RGB")
|
||||
history = parse_action_history(args.action_history, action_dim)
|
||||
predicted = predictor.predict(np.array(image), history)
|
||||
|
||||
if args.output_format == "json":
|
||||
print(json.dumps({"predicted_action": predicted.tolist()}, indent=2))
|
||||
else:
|
||||
print(" ".join(f"{v:.4f}" for v in predicted))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,8 @@
|
||||
# Core
|
||||
torch>=2.0
|
||||
torchvision>=0.15
|
||||
numpy>=1.24
|
||||
Pillow>=9.0
|
||||
|
||||
# Training utilities
|
||||
tqdm>=4.60
|
||||
@@ -0,0 +1,143 @@
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader, Subset
|
||||
from tqdm import tqdm
|
||||
|
||||
from config import Config, build_argparser, apply_overrides
|
||||
from model import ActionPredictor
|
||||
from dataset import ActionDataset
|
||||
|
||||
|
||||
def split_episodes(data_dir: str, val_ratio: float, seed: int = 42):
|
||||
n = ActionDataset.get_episode_count(data_dir)
|
||||
indices = list(range(n))
|
||||
rng = np.random.RandomState(seed)
|
||||
rng.shuffle(indices)
|
||||
val_count = max(1, int(n * val_ratio))
|
||||
return indices[val_count:], indices[:val_count]
|
||||
|
||||
|
||||
def train_one_epoch(model, loader, optimizer, device):
|
||||
model.train()
|
||||
total_loss = 0.0
|
||||
count = 0
|
||||
for batch in tqdm(loader, desc=" train", leave=False):
|
||||
image = batch["image"].to(device)
|
||||
history = batch["action_history"].to(device)
|
||||
target = batch["target"].to(device)
|
||||
optimizer.zero_grad()
|
||||
pred = model(image, history)
|
||||
loss = torch.nn.functional.mse_loss(pred, target)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total_loss += loss.item() * image.size(0)
|
||||
count += image.size(0)
|
||||
return total_loss / max(count, 1)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def validate(model, loader, device):
|
||||
model.eval()
|
||||
total_loss = 0.0
|
||||
count = 0
|
||||
for batch in tqdm(loader, desc=" val ", leave=False):
|
||||
image = batch["image"].to(device)
|
||||
history = batch["action_history"].to(device)
|
||||
target = batch["target"].to(device)
|
||||
pred = model(image, history)
|
||||
loss = torch.nn.functional.mse_loss(pred, target)
|
||||
total_loss += loss.item() * image.size(0)
|
||||
count += image.size(0)
|
||||
return total_loss / max(count, 1)
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_argparser("actpred training")
|
||||
args = parser.parse_args()
|
||||
cfg = apply_overrides(Config(), args)
|
||||
|
||||
os.makedirs(cfg.train.checkpoint_dir, exist_ok=True)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
print(f"Device: {device}")
|
||||
|
||||
train_indices, val_indices = split_episodes(
|
||||
cfg.data.data_dir, cfg.train.val_split)
|
||||
print(f"Episodes: {len(train_indices)} train, {len(val_indices)} val")
|
||||
|
||||
train_ds = ActionDataset(
|
||||
cfg.data.data_dir, cfg.data.history_len, cfg.data.action_dim,
|
||||
cfg.data.image_size, cfg.data.normalize_actions,
|
||||
cfg.data.action_stats_path, train_indices)
|
||||
val_ds = ActionDataset(
|
||||
cfg.data.data_dir, cfg.data.history_len, cfg.data.action_dim,
|
||||
cfg.data.image_size, cfg.data.normalize_actions,
|
||||
cfg.data.action_stats_path, val_indices)
|
||||
|
||||
print(f"Samples: {len(train_ds)} train, {len(val_ds)} val")
|
||||
|
||||
train_loader = DataLoader(train_ds, batch_size=cfg.train.batch_size,
|
||||
shuffle=True, num_workers=cfg.train.num_workers)
|
||||
val_loader = DataLoader(val_ds, batch_size=cfg.train.batch_size,
|
||||
shuffle=False, num_workers=cfg.train.num_workers)
|
||||
|
||||
model = ActionPredictor(cfg.model, cfg.data).to(device)
|
||||
n_params = sum(p.numel() for p in model.parameters())
|
||||
print(f"Model params: {n_params:,}")
|
||||
|
||||
optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.train.lr,
|
||||
weight_decay=cfg.train.weight_decay)
|
||||
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=cfg.train.epochs)
|
||||
|
||||
best_val_loss = float("inf")
|
||||
patience_counter = 0
|
||||
best_path = os.path.join(cfg.train.checkpoint_dir, "best.pt")
|
||||
|
||||
for epoch in range(1, cfg.train.epochs + 1):
|
||||
print(f"\nEpoch {epoch}/{cfg.train.epochs} lr={optimizer.param_groups[0]['lr']:.6f}")
|
||||
train_loss = train_one_epoch(model, train_loader, optimizer, device)
|
||||
val_loss = validate(model, val_loader, device)
|
||||
scheduler.step()
|
||||
print(f" train_loss={train_loss:.6f} val_loss={val_loss:.6f}")
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
patience_counter = 0
|
||||
torch.save({
|
||||
"epoch": epoch,
|
||||
"model_state_dict": model.state_dict(),
|
||||
"optimizer_state_dict": optimizer.state_dict(),
|
||||
"val_loss": val_loss,
|
||||
"config": cfg.to_dict(),
|
||||
}, best_path)
|
||||
inference_path = os.path.join(cfg.train.checkpoint_dir, "best_inference.pt")
|
||||
torch.save({
|
||||
"model_state_dict": model.state_dict(),
|
||||
"config": cfg.to_dict(),
|
||||
}, inference_path)
|
||||
print(f" -> saved best (val_loss={val_loss:.6f})")
|
||||
else:
|
||||
patience_counter += 1
|
||||
if patience_counter >= cfg.train.patience:
|
||||
print(f"\nEarly stopping at epoch {epoch} (patience={cfg.train.patience})")
|
||||
break
|
||||
|
||||
print(f"\nTraining done. Best val_loss={best_val_loss:.6f}")
|
||||
print(f"Best model saved to: {best_path}")
|
||||
|
||||
action_stats = train_ds.get_action_stats()
|
||||
if action_stats is not None:
|
||||
stats_path = os.path.join(cfg.train.checkpoint_dir, "action_stats.npy")
|
||||
np.save(stats_path, action_stats)
|
||||
print(f"Action stats saved to: {stats_path}")
|
||||
|
||||
if args.save_config:
|
||||
with open(args.save_config, "w") as f:
|
||||
json.dump(cfg.to_dict(), f, indent=2, ensure_ascii=False)
|
||||
print(f"Config saved to: {args.save_config}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,210 @@
|
||||
# fsys 安全审计报告
|
||||
|
||||
**审计日期**: 2026-08-08
|
||||
**审计范围**: `fsys/src/fsys.cpp`(1589 行,修复后)
|
||||
**对比基准**: `static/fsys.py`(792 行)+ `static_SECURITY_AUDIT_20260801.md`(static 24 项审计结论)+ `dnfs2_SECURITY_AUDIT_20260808.md`
|
||||
**审计轮次**: 第 1 轮(对照 static 审计逐项验证 + C++ 实现深度复查)
|
||||
**安全评级**: A-
|
||||
|
||||
---
|
||||
|
||||
## 1. 审计概述
|
||||
|
||||
fsys 是 Python `static` 模块的 C++20 重实现(libhttp + libmsg + boost::json),审计目标:
|
||||
|
||||
1. **逐条验证** static 审计(24 项)中的修复在 C++ 重写中是否保留,防止回归;
|
||||
2. **深度复查** C++ 实现特有的问题(内存模型、libcurl 使用、路径解析、路由机制)。
|
||||
|
||||
本轮发现 8 项问题(4 P1 / 1 P2 / 3 P3),**全部已修复**;另记录 6 项已知限制。
|
||||
|
||||
---
|
||||
|
||||
## 2. 本轮新发现问题(全部已修复)
|
||||
|
||||
### F-1: static_root/SSL 相对路径基于可执行文件目录解析(服务无法提供文件) ✅ 已修复
|
||||
|
||||
- **等级**: P1(功能性,启动即失效)
|
||||
- **位置**: `load_config()` 步骤 8(原 L347-369)
|
||||
- **问题**: 相对路径 `static_root` 通过 `/proc/self/exe` 基于可执行文件目录解析。二进制位于 `fsys/build/fsys`,而 `public` 符号链接位于 `fsys/public`,因此 `./public` 解析为 `fsys/build/./public`(不存在)。更严重的是构造函数 `fs::create_directories()` 会**静默创建空目录**,服务启动正常但所有路由返回 404/500,无告警。
|
||||
- **修复**: 相对路径改为基于 `fs::current_path()` 解析(`_run.sh` 以模块目录为 cwd 启动,与 Python 版 `BASE_DIR` 语义一致;Docker 使用绝对路径 `STATIC_ROOT=/app/public`,不受影响)。SSL 证书路径同样修复。
|
||||
|
||||
### F-2: favicon 代理 `CURLOPT_FOLLOWLOCATION` 盲跟重定向绕过 SSRF 校验(回归) ✅ 已修复
|
||||
|
||||
- **等级**: P1
|
||||
- **位置**: `_handle_favicon()`
|
||||
- **问题**: 原实现 `CURLOPT_FOLLOWLOCATION=1L` 让 libcurl 自动跟随重定向,**重定向目标不再经过私有 IP 校验**。攻击者可用公网主机返回 `302 Location: http://127.0.0.1:8005/...` 探测/访问内网服务。Python 版有 `_FaviconRedirectHandler` 逐跳重新校验(static 审计已有机制第 8 条),C++ 重写丢失了该防护,属于**安全回归**。
|
||||
- **修复**: 禁用自动跟随(`CURLOPT_FOLLOWLOCATION=0L`),提取 `validate_url_ssrf()` 校验函数(scheme/端口/私有 IP),手动逐跳跟随(最多 3 跳),每跳重新校验;同时设置 `CURLOPT_PROTOCOLS_STR="http,https"` 纵深防御。
|
||||
|
||||
### F-3: 查询参数与路径缺失 URL 解码(favicon/filelist/video/catch-all 功能性回归) ✅ 已修复
|
||||
|
||||
- **等级**: P1(功能性)
|
||||
- **位置**: `parse_query()`、`_handle_catch_all()`、`_handle_video()`、`_handle_favicon()`
|
||||
- **问题**: 前端全部使用 `encodeURIComponent`(已核实:`view.js:139` `/api/filelist?path=${encodeURIComponent(currentPath)}`、`dnfs.js:480` `/api/favicon?url=${encodeURIComponent(sp.url)}`、`view.js:73` 面包屑链接),Flask 自动解码。C++ 版不解码导致:
|
||||
1. `/api/favicon?url=https%3A%2F%2F...` → scheme 校验失败 → **favicon 功能完全失效**
|
||||
2. `/api/filelist?path=a%2Fb` → 按字面名 `a%2Fb` 查找 → **子目录导航失效**
|
||||
3. 中文/空格文件名的静态文件访问失效
|
||||
- **修复**: 新增 `url_decode()`(`%XX` + 查询串 `+`→空格,路径解码不转换 `+`);`parse_query` 键值均解码;catch-all/video 路径解码;`safe_path_join` 新增 NUL 字节拒绝(防解码 `%00` 截断)。解码后的遍历序列(如 `..%2F..`)仍被 `safe_path_join` 前缀校验拦截,仅解码一次无双解码风险。
|
||||
|
||||
### F-4: 静态文件/无 Range 视频全量读入内存(NEW-7 回归) ✅ 已修复(上限防护)
|
||||
|
||||
- **等级**: P1
|
||||
- **位置**: `_handle_catch_all()`、`_handle_root()`、`_handle_video()`
|
||||
- **问题**: libhttp 响应为 `string_body`,所有文件全量读入内存返回。Python 版 NEW-7 已用 `send_file` 流式修复该 DoS;C++ 重写回归。攻击者请求 public 下大文件(或 `Range: bytes=0-` 全量拉取大视频)可导致内存耗尽。
|
||||
- **修复**: 新增 `kMaxServeFileSize = 50MB`:
|
||||
- 普通文件 >50MB → 413
|
||||
- 视频无 Range 全量 >50MB → 413(浏览器拉取媒体必带 Range,实际无感)
|
||||
- Range 片段 >50MB → 416(防 `bytes=0-`)
|
||||
- 残留限制见 L-1(需 libhttp 支持 file_body/chunked 才能彻底解决)
|
||||
|
||||
### F-5: 限速表 `_rate_store` 无界增长 ✅ 已修复
|
||||
|
||||
- **等级**: P2
|
||||
- **位置**: `_check_rate_limit()`
|
||||
- **问题**: 限速键为 `"hash:" + client_ip()`,XFF 可伪造且无上限。攻击者旋转 XFF 值可令 map 无限增长耗尽内存(DoS)。Python 版有相同问题(未修),C++ 重写有机会修复。
|
||||
- **修复**: 键数超过 `kRateMaxKeys=65536` 时全局驱逐过期窗口键。
|
||||
|
||||
### F-6: Cookie `q4_token` 子串匹配误判 ✅ 已修复
|
||||
|
||||
- **等级**: P3
|
||||
- **位置**: `_handle_dnfs_proxy()`
|
||||
- **问题**: `cookies.find("q4_token=")` 子串搜索,名为 `x_q4_token=` 的 Cookie 也会命中,可能把非预期 Cookie 值作为 Bearer Token 转发给 dnfs2。
|
||||
- **修复**: 新增 `get_cookie()` 按 `;` 拆分、trim、精确按名匹配。
|
||||
|
||||
### F-7: `sha256_hex` EVP_MD 泄漏窗口 ✅ 已修复
|
||||
|
||||
- **等级**: P3
|
||||
- **位置**: `sha256_hex()`
|
||||
- **问题**: 释放条件 `hash_len > 0 && md != EVP_sha256()`,digest 失败(hash_len==0)时 `EVP_MD_fetch` 返回的 md 泄漏。
|
||||
- **修复**: 改用 `fetched` 标志,digest 成败均释放。
|
||||
|
||||
### F-8: `CURLOPT_PROTOCOLS` 已弃用(编译错误) ✅ 已修复
|
||||
|
||||
- **等级**: P3
|
||||
- **问题**: libcurl ≥7.85 弃用 `CURLOPT_PROTOCOLS`,`-Werror` 下编译失败。
|
||||
- **修复**: 改用 `CURLOPT_PROTOCOLS_STR`。
|
||||
|
||||
---
|
||||
|
||||
## 3. static 审计 24 项逐条验证(防回归检查)
|
||||
|
||||
| static 审计项 | fsys 状态 | 验证说明 |
|
||||
|------|------|------|
|
||||
| P0-1 无认证密码重置 | ✅ 未重实现 | 危险功能未移植 |
|
||||
| P0-2/3/4 yhdet OAuth(Token/SSL 跳过/硬编码密钥) | ✅ 未重实现 | 同上 |
|
||||
| P0-5 直连 SQLite | ✅ | 通过 `_call_db()` RpcClient 调 db2:8005 |
|
||||
| NEW-1/2 proxy_send_code / PRIVATE_SECRET | ✅ 未重实现 | 同上 |
|
||||
| P1-1 check_user/hash 限速 | ✅ | 滑动窗口 20/10 次每分钟,XFF 取真实 IP |
|
||||
| P1-2/NEW-5 路径遍历前缀 bypass | ✅ | `base_str + "/"` 边界校验 + `weakly_canonical`(symlink 逃逸同样拦截) |
|
||||
| NEW-3 dnfs 代理 CORS 通配 | ✅ | `_apply_cors` 白名单回显 + `Vary: Origin` |
|
||||
| NEW-6 代理错误泄露内部异常 | ✅ | 统一返回 `{"error": "upstream unavailable"}`,详情仅记日志 |
|
||||
| NEW-7 大文件全量读内存 | ⚠️→✅ | 本轮回归后由 F-4 上限防护覆盖,残留限制 L-1 |
|
||||
| NEW-8 api_hash 校验 | ✅ | `json::parse` 失败/非 object/password 非 string 均 400 + 限速 |
|
||||
| NEW-9 db 单例/retry/env | ✅ | 成员 RpcClient(retry_on_refused=true)+ DB_HOST/DB_PORT env |
|
||||
| NEW-10 favicon 端口校验 | ✅ | 端口 [1,65535] 校验,IPv6/userinfo 畸形 URL fail-closed 拒绝 |
|
||||
| NEW-11 CSP unsafe-inline | ⚠️ 继承 | 与 static/dnfs2 一致的已知限制(前端内联脚本依赖),见 L-3 |
|
||||
| NEW-12 SSL 回退开发服务器 | ✅ 消除 | libhttp 原生 TLS(F-2 改进项) |
|
||||
| NEW-13 日志单条失败弃批 | ✅ | `_send_log_batch` 单条 catch 后继续 |
|
||||
| NEW-14 端口 env 未捕获异常 | ✅ | `safe_stoi` 统一校验,main 捕获后打印 `[fsys] fatal` |
|
||||
| P2-1 Range 边界校验 | ✅ | start/end 范围 + end≥start + stoi 异常处理 |
|
||||
| P2-2 日志敏感信息 | ✅ | 无密码/Token 入日志 |
|
||||
| confs retry 强依赖 | ✅ | `retry_on_refused=true` |
|
||||
| 配置白名单合并 / 必需项校验 | ✅ | INI 解析 + cors_origins/static_root 必需校验 |
|
||||
| 安全响应头(6 项) | ✅ | `make_secured_response` 内联每个响应 |
|
||||
| `__` 前缀系统文件过滤 | ✅ | filelist 过滤 |
|
||||
| 请求体大小限制 | ✅ | `setMaxBodySize(10MB)`,超限 libhttp 返回 413 |
|
||||
| curl 全局初始化/清理 | ✅ | main 遵循 dnfs2 P3-N9 模式 |
|
||||
|
||||
**结论**: static 审计的全部有效修复在 fsys 中均得到保留;NEW-7(F-4)与 favicon 重定向校验(F-2)两处曾发生回归,已在本轮修复。
|
||||
|
||||
---
|
||||
|
||||
## 4. 行为差异说明(非缺陷)
|
||||
|
||||
### D-1: dnfs 代理透传上游状态码(改进)
|
||||
|
||||
Python 版 `_proxy_to_dnfs` 用 urllib,上游非 2xx 抛 HTTPError → 一律返回 502;因此前端 `dnfs.js` 中 `res.status === 401 → 跳转登录页` 的逻辑在 Python 版下是**死代码**(凭证过期时收到 502 而非 401)。fsys 透传上游真实状态码(400/401/404/500),使前端 401 跳转逻辑首次生效。前端其余错误处理读取 `body.error`,与状态码无关,无兼容性问题。
|
||||
|
||||
### D-2: libhttp 内置 OPTIONS 反射任意 Origin(继承,影响有限)
|
||||
|
||||
fsys 仅对 4 个 dnfs 路由注册了显式 OPTIONS handler(走白名单)。其余路由(如 `/api/hash`)的预检落入 libhttp 内置 OPTIONS 处理——该处理反射任意 Origin(libhttp 审计 P2-1,未修复)。由于 fsys 实际响应(POST 结果)不附带 `Access-Control-Allow-Origin` 头,浏览器仍会阻止跨域读取响应,泄露面被封堵;预检放行本身影响有限。根治依赖 libhttp 修复 P2-1。
|
||||
|
||||
### D-3: favicon URL 无路径时的构造
|
||||
|
||||
Python 版用 `urlparse().netloc`,fsys 用首个 `/` 截取 origin,等价;`http://host?query`(无路径直接带查询串)这类畸形输入在 fsys 中 fail-closed 拒绝(Python 可解析),不影响正常调用(前端始终发送完整 URL)。
|
||||
|
||||
---
|
||||
|
||||
## 5. 已知限制(记录,不修复)
|
||||
|
||||
### L-1: string_body 无流式传输(P2,架构性)
|
||||
|
||||
libhttp 响应体为 `string_body`,50MB 上限是缓解而非根治(50MB × 4 线程仍有 200MB 峰值)。彻底修复需 libhttp 支持 `file_body` 或 chunked 响应,与 Python 版 NEW-7 的 send_file 流式能力对齐。
|
||||
|
||||
### L-2: 限速依赖 XFF(P2,继承)
|
||||
|
||||
- libhttp 未向 handler 暴露 peer IP(`Session::remote_ip_` 仅内部可用),XFF 缺失时限速键回退 `"unknown"`,所有直连客户端共享窗口。Python 版有 `request.remote_addr` 兜底,此处为弱化。
|
||||
- 绕过 gateway 直连时可伪造 XFF 绕过限速(Python 相同)。生产依赖 gateway/waf 覆写 XFF。
|
||||
- 建议:libhttp 在 params 或 request 上下文暴露 peer IP。
|
||||
|
||||
### L-3: CSP `unsafe-inline`(P3,继承 static NEW-11 / dnfs2 L-7)
|
||||
|
||||
前端(dnfs 特殊节点注入的 HTML/JS)依赖内联脚本,nonce 化需前端架构改造。
|
||||
|
||||
### L-4: DNS rebinding 理论窗口(P3,继承 static NEW-10)
|
||||
|
||||
`is_private_host` 校验与实际 HTTP 连接之间存在 TOCTOU 窗口(重定向逐跳校验同样存在)。彻底修复需将校验所得 IP 绑定到连接(`CURLOPT_RESOLVE`),当前与 Python 版风险等级一致,可接受。
|
||||
|
||||
### L-5: favicon 代理无响应缓存
|
||||
|
||||
每次请求都实时拉取上游(Python 版同样无服务端缓存,仅有浏览器侧 `Cache-Control: max-age=86400`)。高频访问同一链接卡片会产生重复外联。
|
||||
|
||||
### L-6: Makefile `/opt/homebrew` 路径(继承 dnfs2 惯例)
|
||||
|
||||
macOS/Linux 双开发环境约定,Linux 编译需覆盖 `INCS/LIBS/RPATH`(本次审计编译即如此执行)。与 dnfs2 Makefile 保持逐字节一致,未单独修改。
|
||||
|
||||
---
|
||||
|
||||
## 6. 修复文件清单
|
||||
|
||||
| 文件 | 修改内容 |
|
||||
|------|---------|
|
||||
| `fsys/src/fsys.cpp` | F-1: static_root/SSL 相对路径基于 CWD 解析 |
|
||||
| `fsys/src/fsys.cpp` | F-2: `validate_url_ssrf()` + favicon 手动逐跳重定向校验 |
|
||||
| `fsys/src/fsys.cpp` | F-3: `url_decode()` + parse_query/catch-all/video 解码 + NUL 拒绝 |
|
||||
| `fsys/src/fsys.cpp` | F-4: `kMaxServeFileSize=50MB` 上限(413/416) |
|
||||
| `fsys/src/fsys.cpp` | F-5: `kRateMaxKeys=65536` 限速表驱逐 |
|
||||
| `fsys/src/fsys.cpp` | F-6: `get_cookie()` 精确按名解析 |
|
||||
| `fsys/src/fsys.cpp` | F-7: sha256_hex `fetched` 标志修复泄漏 |
|
||||
| `fsys/src/fsys.cpp` | F-8: `CURLOPT_PROTOCOLS_STR` |
|
||||
| `fsys/README.md` | `[directory] root` 相对路径语义说明修正 |
|
||||
|
||||
编译验证: `g++ -std=c++20 -O3 -Wall -Werror` 通过,零错误零警告(`-Werror=deprecated-declarations` 下通过)。
|
||||
|
||||
运行时冒烟测试(confs/db/dnfs 在线,临时端口 19200,与生产 Python 版 9200 对照):
|
||||
|
||||
| 用例 | 结果 |
|
||||
|------|------|
|
||||
| `GET /` / `/health` / filelist 根目录与子目录 | ✓ 200,`static_root` 正确解析到 `fsys/public`(F-1) |
|
||||
| filelist 编码路径 `path=js` / `games%2F` | ✓ 200(F-3 解码生效) |
|
||||
| 路径遍历 `/%2e%2e/%2e%2e/etc/passwd` | ✓ 403 |
|
||||
| favicon 编码公网 URL / 内网 URL | ✓ 200 image/x-icon / 403 forbidden host(F-2/F-3) |
|
||||
| `POST /api/hash` | ✓ 哈希值与 Python 版逐位一致 |
|
||||
| `POST /api/check_user` | ✓ 与 Python 版行为一致(当前环境 db Users 查询失败,两版同返 500 query failed,非 fsys 缺陷) |
|
||||
| dnfs 代理 `/get_notes` | ✓ 透传(上游自身 500,直连上游同果,非代理缺陷) |
|
||||
|
||||
---
|
||||
|
||||
## 7. 安全评级
|
||||
|
||||
### 评级: A-
|
||||
|
||||
**理由**:
|
||||
- static 审计 24 项修复全部保留,两处回归(F-2 SSRF 重定向、F-4 大文件内存)发现后立即修复
|
||||
- 4 项 P1 全部修复,编译零警告
|
||||
- 相比 Python 版新增:请求体限制、TLS 原生支持、限速表驱逐、Cookie 精确解析、401 透传使前端登录跳转生效
|
||||
- 扣分项:L-1 流式传输缺失(依赖 libhttp 演进)、L-2 限速依赖 XFF(依赖 libhttp 暴露 peer IP)、L-3 CSP unsafe-inline(前端架构限制)
|
||||
|
||||
**升级路径**(修复以下任一项可评 A):
|
||||
- libhttp 支持 file_body/chunked → 解除 L-1
|
||||
- libhttp 暴露 peer IP → 解除 L-2
|
||||
- libhttp 修复 P2-1(OPTIONS Origin 白名单)→ 消除 D-2
|
||||
@@ -4,6 +4,8 @@
|
||||
**审计范围**: `libhttp/inc/us/`、`libhttp/src/`(HttpServer 6 个拆分 TU + HttpClient + 遗留 WebSocketServer/WebSocketClient)
|
||||
**代码版本**: dev_qing 分支
|
||||
|
||||
**补充轮次**: 2026-08-08 — 下游 fsys 审计反馈(见下方“下游审计反馈”节,新增 P2-7/P2-8/P3-3)
|
||||
|
||||
---
|
||||
|
||||
## 修复进度
|
||||
@@ -12,8 +14,11 @@
|
||||
|--------|------|--------|----------|--------|
|
||||
| P0 | 0 | 0 | 0 | 0 |
|
||||
| P1 | 6 | 6 | 0 | 0 |
|
||||
| P2 | 4 | 0 | 0 | 4 |
|
||||
| **合计** | **10** | **8** | **0** | **2** |
|
||||
| P2 | 8 | 0 | 0 | 8 |
|
||||
| P3 | 3 | 2 | 0 | 1 |
|
||||
| **合计** | **17** | **8** | **0** | **9** |
|
||||
|
||||
> 勘误:原汇总表 P2 总数误记为 4(实际 P2 节含 P2-1~P2-6 共 6 项),本次随 2026-08-08 补充一并修正。
|
||||
|
||||
---
|
||||
|
||||
@@ -34,12 +39,14 @@
|
||||
|
||||
| # | 问题 | 状态 | 说明 |
|
||||
|---|------|------|------|
|
||||
| P2-1 | CORS 反射任意 Origin | ❌ 未修复 | OPTIONS 预检(`HttpServer_Request.cpp` L73-74)把 `Access-Control-Allow-Origin` 直接设为请求的 `Origin` 值,任意来源均被放行,无白名单校验(对应全局审计 P2-3 根因) |
|
||||
| P2-1 | CORS 反射任意 Origin | ❌ 未修复 | OPTIONS 预检(`HttpServer_Request.cpp` L73-74)把 `Access-Control-Allow-Origin` 直接设为请求的 `Origin` 值,任意来源均被放行,无白名单校验(对应全局审计 P2-3 根因)。**下游影响已确认**(fsys 审计 D-2):应用未显式注册 OPTIONS 的路由均落入此内置预检,白名单形同虚设;仅因实际应用响应不带 ACAO 头才封住跨域读取 |
|
||||
| P2-2 | TLS 客户端一律不校验证书 | ❌ 未修复 | `HttpClient.cpp` L131 与 WS 代理 `HttpServer_WsSession.cpp` L243 均 `set_verify_mode(asio::ssl::verify_none)`。对内部后端可接受,但库内不留逃生口,无法在需要校验时启用 |
|
||||
| P2-3 | WebSocket 握手无 Origin/CSWSH 校验 | ❌ 未修复 | libhttp 对任意 `ws`/`ws_proxy` upgrade 请求直接握手,不校验 `Origin`/`Sec-WebSocket-Protocol`。若后端依赖 Cookie 鉴权,存在跨站 WebSocket 劫持(CSWSH)风险(gateway 的 `ws_proxy` 依赖握手回调鉴权,属应用层缓解) |
|
||||
| P2-4 | HttpClient `ws_recv` 忽略 timeout 参数 | ❌ 未修复 | `HttpClient.cpp` L220-221 `(void)timeout_ms;` 丢弃超时参数,`ws_recv` 会**无限阻塞**。后端挂起时调用线程永久卡死 |
|
||||
| P2-5 | 反向代理后端连接建立后无超时 | ❌ 未修复 | 代理后端握手成功后 `proxy_tcp_stream_->expires_never()`(`HttpServer_WsSession.cpp` L262/L287),双向读写无超时,慢后端可长期占用连接 |
|
||||
| P2-6 | 无连接数限制 | ❌ 未修复 | `Listener::doAccept`(`HttpServer_Listener.cpp` L23-48)无限接受连接,每连接一个 Session,无最大连接数上限,可被连接耗尽 |
|
||||
| P2-7 | handler 无法获取客户端真实 IP | ❌ 未修复 | handler 签名仅接收 `req + params`,`Session::remote_ip_`/`getRemoteAddress()`(`HttpServerImpl.hpp` L63)不对外暴露。**下游影响**(fsys 审计 L-2):限速/审计日志只能信任 XFF 头,XFF 缺失时回退全局共享键,直连时 XFF 可伪造,限速防护弱化。建议:通过 params 注入或扩展 handler 上下文暴露 peer IP |
|
||||
| P2-8 | 响应仅支持 string_body,无 file_body/chunked | ❌ 未修复 | `handleRequest`/handler 返回类型固定为 `http::response<http::string_body>`,静态文件服务必须全量读入内存(内置静态分支 `HttpServer_Request.cpp` L95-119 同样如此)。**下游影响**(fsys 审计 L-1/F-4):fsys 只能用 50MB 上限缓解大文件 DoS,无法对齐 Python 版 `send_file` 流式能力。建议:支持 `file_body` 或 chunked 序列化响应 |
|
||||
|
||||
---
|
||||
|
||||
@@ -49,6 +56,22 @@
|
||||
|---|------|------|------|
|
||||
| P3-1 | 遗留 WebSocketServer/WebSocketClient 未构建 | ✅ 已修复 | 已删除 `WebSocketServer.cpp`/`WebSocketClient.cpp`/`WebSocketServer.hpp`/`WebSocketClient.hpp`/`WebSocketClientBase.hpp`(未构建、与 `HttpClient` 别名同名冲突、有缺陷),并清理 Makefile/README 引用。WebSocket 能力由 `HttpServer` + `HttpClient` 提供 |
|
||||
| P3-2 | CMakeLists.txt 过时 | ✅ 已修复 | CMakeLists 已重写对齐 Makefile:`libus_http.so`(6 个服务端 TU)+ `libus_httpclient.so`(HttpClient),移除遗留文件引用与错误库名 |
|
||||
| P3-3 | 预编译 .so 与源码脱节(set_timeout 符号缺失) | ❌ 未修复 | dnfs2 审计 P2-N5 已在源码新增 `HttpClient::set_timeout/set_connect_timeout`(`HttpClient.cpp` L45/L53),但 `build/` 目录的预编译产物为旧源码编译,`nm -D libus_httpclient.so` 无对应符号,下游 fsys 调用即链接失败(被迫回退默认 30s 超时)。且 `build/` 属主为 root,普通用户无法重建。建议:以 root 重编译发布与源码一致的 .so,或调整 build 目录权限 |
|
||||
|
||||
---
|
||||
|
||||
## 下游审计反馈(2026-08-08,fsys 第 1 轮审计)
|
||||
|
||||
fsys(static 的 C++ 重写,首个大规模使用 HttpServer + HttpClient 的业务模块)审计中确认/新发现以下 libhttp 侧问题,均已录入上方表格:
|
||||
|
||||
| 编号 | 反馈 | fsys 侧缓解 |
|
||||
|------|------|-------------|
|
||||
| P2-1 | 内置 OPTIONS 反射任意 Origin,下游未注册 OPTIONS 的路由白名单失效 | 应用响应不带 ACAO,跨域读取被浏览器拦截(影响有限但非根治) |
|
||||
| P2-7 | handler 无 peer IP,限速只能信 XFF | XFF 缺失回退 "unknown" 全局键 + 键数上限 65536 驱逐 |
|
||||
| P2-8 | 无流式响应,大文件全量读入内存 | 50MB 上限(413/416),残留限制记 fsys L-1 |
|
||||
| P3-3 | 预编译 .so 缺 set_timeout 符号 | 移除调用,使用默认 30s 超时 |
|
||||
|
||||
交叉引用: `docs/security/fsys_SECURITY_AUDIT_20260808.md`(D-2/L-1/L-2 节)
|
||||
|
||||
---
|
||||
|
||||
@@ -83,6 +106,9 @@
|
||||
- [ ] P2-5 代理后端读写超时 + 空闲保活
|
||||
- [ ] P2-6 最大连接数限制
|
||||
- [ ] P2-2 TLS 校验可配置(至少提供 `verify_peer` 选项)
|
||||
- [ ] P2-7 handler 暴露 peer IP(params 注入或 handler 上下文扩展)——下游限速/审计依赖
|
||||
- [ ] P2-8 支持 file_body/chunked 响应——下游静态服务流式传输依赖
|
||||
- [ ] P3-3 重编译发布与源码一致的 .so(补 set_timeout 符号),解决 build/ 目录 root 权限问题
|
||||
- [x] P3-1 删除或维护遗留 WebSocketServer/WebSocketClient → ✅ 已删除
|
||||
- [x] P3-2 同步 CMakeLists.txt 与 Makefile → ✅ 已重写对齐
|
||||
|
||||
@@ -107,4 +133,7 @@
|
||||
| HttpClient 响应无上限 | `src/HttpClient.cpp` | L63-81, L329-334 |
|
||||
| HttpClient TLS verify_none | `src/HttpClient.cpp` | L131 |
|
||||
| HttpClient ws_recv 忽略超时 | `src/HttpClient.cpp` | L220-221 |
|
||||
| HttpClient set_timeout(源码已有,.so 缺符号) | `src/HttpClient.cpp` / `inc/us/HttpClient.hpp` | L45, L53 / L70, L73 |
|
||||
| Session peer IP(未暴露给 handler) | `inc/us/detail/HttpServerImpl.hpp` | L63, L122 |
|
||||
| 内置静态文件(string_body 全量读) | `src/HttpServer_Request.cpp` | L95-119 |
|
||||
| ~~遗留 WS 实现(已删除)~~ | ~~`src/WebSocketServer.cpp`、`src/WebSocketClient.cpp`~~ | 已删除(见 P3-1),WebSocket 能力由 `HttpServer` + `HttpClient` 提供 |
|
||||
@@ -0,0 +1 @@
|
||||
build/
|
||||
@@ -0,0 +1,88 @@
|
||||
# fsys Dockerfile — C++20 静态文件托管服务
|
||||
# 多阶段构建:第一阶段编译 libhttp + libmsg + fsys,第二阶段精简运行时
|
||||
|
||||
# ============================================================================
|
||||
# Stage 1: 构建阶段
|
||||
# ============================================================================
|
||||
FROM ubuntu:24.04 AS builder
|
||||
|
||||
LABEL maintainer="qingsong"
|
||||
LABEL service="fsys"
|
||||
LABEL description="Static File Hosting Service (C++20) - 静态文件托管服务"
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
libboost-json-dev \
|
||||
libssl-dev \
|
||||
libcurl4-openssl-dev \
|
||||
make \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
COPY libhttp/ /build/libhttp/
|
||||
COPY libmsg/ /build/libmsg/
|
||||
COPY fsys/ /build/fsys/
|
||||
|
||||
# 编译 libmsg
|
||||
RUN cd /build/libmsg && make clean && make
|
||||
|
||||
# 编译 libhttp
|
||||
RUN cd /build/libhttp && make clean && make \
|
||||
CPP="g++" \
|
||||
INCS="-I inc" \
|
||||
SERVER_LIBS="-lssl -lcrypto -lpthread" \
|
||||
CLIENT_LIBS="-lssl -lcrypto -lcurl -lpthread"
|
||||
|
||||
# 编译 fsys
|
||||
RUN cd /build/fsys && make clean && make \
|
||||
CPP="g++" \
|
||||
INCS="-I ../libhttp/inc -I ../libmsg/inc" \
|
||||
LIBS="-L ../libhttp/build -L ../libmsg/build -lus_http -lus_httpclient -lus_msg -lboost_json -lssl -lcrypto -lcurl -lpthread" \
|
||||
RPATH="-Wl,-rpath,/app/lib"
|
||||
|
||||
# ============================================================================
|
||||
# Stage 2: 运行时阶段
|
||||
# ============================================================================
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libboost-json1.83.0 \
|
||||
libssl3 \
|
||||
libcurl4 \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd -m -u 1000 appuser
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 复制构建产物
|
||||
COPY --from=builder /build/fsys/build/fsys /app/fsys
|
||||
COPY --from=builder /build/libhttp/build/libus_http.so /app/lib/libus_http.so
|
||||
COPY --from=builder /build/libhttp/build/libus_httpclient.so /app/lib/libus_httpclient.so
|
||||
COPY --from=builder /build/libmsg/build/libus_msg.so /app/lib/libus_msg.so
|
||||
|
||||
# 复制静态文件(public 目录)
|
||||
COPY static/public/ /app/public/
|
||||
|
||||
ENV LD_LIBRARY_PATH=/app/lib
|
||||
|
||||
RUN chown -R appuser:appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 9200
|
||||
|
||||
# 健康检查:GET /health 期望 200
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD curl -sf http://localhost:9200/health -o /dev/null || exit 1
|
||||
|
||||
ENV LISTEN_ADDR=0.0.0.0 \
|
||||
LISTEN_PORT=9200 \
|
||||
STATIC_ROOT=/app/public
|
||||
|
||||
CMD ["/app/fsys"]
|
||||
@@ -0,0 +1,26 @@
|
||||
CPP = g++
|
||||
CPPFLAGS = -std=c++20 -O3 -Wall -Werror -pipe
|
||||
INCS = -I ../libhttp/inc -I ../libmsg/inc -I /opt/homebrew/include
|
||||
LIBS = -L ../libhttp/build -L ../libmsg/build -L /opt/homebrew/lib \
|
||||
-lus_http -lus_httpclient -lus_msg -lboost_json \
|
||||
-lssl -lcrypto -lcurl -lpthread
|
||||
RPATH = -Wl,-rpath,'$$ORIGIN/../../libhttp/build' \
|
||||
-Wl,-rpath,'$$ORIGIN/../../libmsg/build' \
|
||||
-Wl,-rpath,/opt/homebrew/lib
|
||||
|
||||
BUILD = build
|
||||
TARGET = $(BUILD)/fsys
|
||||
SRC = src/fsys.cpp
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
$(BUILD):
|
||||
mkdir -p $(BUILD)
|
||||
|
||||
$(TARGET): $(SRC) | $(BUILD)
|
||||
$(CPP) $(CPPFLAGS) $(INCS) -o $@ $< $(LIBS) $(RPATH)
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET)
|
||||
|
||||
.PHONY: all clean
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
# fsys — 静态文件托管服务 (C++20)
|
||||
|
||||
基于 libhttp + libmsg 的 C++20 静态文件托管与前端页面服务,监听 `127.0.0.1:9200`。
|
||||
|
||||
Python/Flask `static` 模块的生产级 C++ 重写,接口 100% 兼容,前端零改动。
|
||||
|
||||
## 架构
|
||||
|
||||
```
|
||||
waf :443 (TLS 终止 + 安全规则)
|
||||
│
|
||||
▼
|
||||
gateway :8443 (反向代理 + 认证)
|
||||
│
|
||||
▼
|
||||
fsys :9200 (libhttp 异步多线程 + CORS)
|
||||
│
|
||||
├── public/ 静态文件根目录
|
||||
├── dnfs2 :8006 笔记代理(反向代理)
|
||||
├── db2 :8005 用户查询 + 日志写入
|
||||
└── confs :8010 配置拉取
|
||||
```
|
||||
|
||||
## 路由
|
||||
|
||||
### 核心路由
|
||||
|
||||
| 路径 | 说明 |
|
||||
|------|------|
|
||||
| `GET /` | 首页(优先 index.html,回退 __view.html) |
|
||||
| `GET /*` | 静态文件 / 目录浏览 |
|
||||
| `GET /api/filelist?path=` | JSON 文件列表,过滤 `__` 前缀系统文件 |
|
||||
| `GET /video/*` | 视频/音频流媒体(HTTP Range 206 分段) |
|
||||
| `GET /health` | 健康检查 |
|
||||
|
||||
### 内部工具路由
|
||||
|
||||
| 路径 | 说明 |
|
||||
|------|------|
|
||||
| `POST /api/hash` | SHA-256 密码哈希(限速 20/min/IP) |
|
||||
| `POST /api/check_user` | 检查用户是否已注册(限速 10/min/IP) |
|
||||
| `GET /api/favicon?url=` | favicon SSRF 防护代理(512KB 上限,1 天缓存) |
|
||||
|
||||
### dnfs2 代理路由
|
||||
|
||||
| 路径 | 说明 |
|
||||
|------|------|
|
||||
| `POST /get_notes` | 获取树状笔记列表(代理至 dnfs2:8006) |
|
||||
| `POST /add_note` | 新增笔记(代理至 dnfs2:8006) |
|
||||
| `POST /delete_note` | 删除笔记(代理至 dnfs2:8006) |
|
||||
| `POST /update_note` | 更新笔记(代理至 dnfs2:8006) |
|
||||
|
||||
支持 HttpOnly Cookie 模式:`q4_token` Cookie 自动转为 `Authorization: Bearer` 头。
|
||||
|
||||
## 配置
|
||||
|
||||
与 `static` 模块完全兼容,使用相同的 `static.ini`(由 confs 服务分发)。
|
||||
|
||||
| Section | Key | 说明 |
|
||||
|---------|-----|------|
|
||||
| `[server]` | `host` | 监听地址(默认 127.0.0.1) |
|
||||
| `[server]` | `port` | 监听端口(默认 9200) |
|
||||
| `[directory]` | `root` | 静态文件根目录(相对路径基于工作目录,_run.sh 以模块目录启动) |
|
||||
| `[ssl]` | `enabled` | TLS 开关(默认 false) |
|
||||
| `[ssl]` | `cert_file` | 证书路径 |
|
||||
| `[ssl]` | `key_file` | 私钥路径 |
|
||||
| `[cors]` | `allowed_origins` | CORS 白名单,逗号分隔 |
|
||||
|
||||
### 环境变量
|
||||
|
||||
| 变量 | 说明 |
|
||||
|------|------|
|
||||
| `LISTEN_ADDR` | 覆盖监听地址 |
|
||||
| `LISTEN_PORT` | 覆盖监听端口 |
|
||||
| `STATIC_ROOT` | 覆盖静态文件根目录 |
|
||||
| `DB_HOST` / `DB_PORT` | 覆盖 db 服务地址 |
|
||||
| `DNFS_URL` | 覆盖 dnfs2 上游地址 |
|
||||
| `CONFS_HOST` / `CONFS_PORT` | 覆盖 confs 引导地址 |
|
||||
|
||||
### 命令行参数
|
||||
|
||||
| 参数 | 说明 |
|
||||
|------|------|
|
||||
| `--addr` / `-a` | 监听地址 |
|
||||
| `--port` / `-p` | 监听端口 |
|
||||
| `--threads` / `-t` | 工作线程数 |
|
||||
|
||||
## 相比 Python static 的改进
|
||||
|
||||
| 改进 | 说明 |
|
||||
|------|------|
|
||||
| 请求体大小限制 | `setMaxBodySize(10MB)` 防 OOM |
|
||||
| SSL 原生支持 | libhttp 构造器直接支持,无需回退开发服务器 |
|
||||
| 异步 IO | Boost.Beast 异步多线程替代 Flask 同步 |
|
||||
| 消除 Python 依赖 | 不再需要 venv/pip/Flask/waitress |
|
||||
| 命令行参数 | 支持 `--addr/-a --port/-p --threads/-t` |
|
||||
| 数值范围校验 | 所有端口/线程数有严格边界检查 |
|
||||
| 统一日志配置 | 复用主配置 confs 拉取的 db host/port |
|
||||
|
||||
## 运行
|
||||
|
||||
```bash
|
||||
bash _set.sh # 编译
|
||||
bash _run.sh # 启动
|
||||
```
|
||||
|
||||
## 依赖
|
||||
|
||||
libhttp, libmsg, boost::json, OpenSSL, libcurl
|
||||
|
||||
## 许可证
|
||||
|
||||
MIT License
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
exec ./build/fsys "$@"
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
cd "$(dirname "$0")"
|
||||
make
|
||||
@@ -0,0 +1,190 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>fsys 测试页</title>
|
||||
<style>
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body { font-family: system-ui, sans-serif; background: #0f1419; color: #e6e6e6; padding: 32px 16px; }
|
||||
.wrap { max-width: 860px; margin: 0 auto; }
|
||||
h1 { font-size: 22px; margin-bottom: 4px; }
|
||||
.sub { color: #8b98a5; font-size: 13px; margin-bottom: 24px; }
|
||||
.card { background: #1a2027; border: 1px solid #2a3441; border-radius: 10px; padding: 16px; margin-bottom: 14px; }
|
||||
.card h2 { font-size: 14px; color: #9fb3c8; margin-bottom: 10px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
||||
td { padding: 7px 8px; border-bottom: 1px solid #232d38; vertical-align: top; }
|
||||
td.name { width: 34%; color: #d7e2ec; }
|
||||
td.st { width: 90px; font-weight: 600; }
|
||||
.ok { color: #4ade80; } .fail { color: #f87171; } .pending { color: #8b98a5; }
|
||||
td.detail { color: #8b98a5; font-family: ui-monospace, monospace; font-size: 12px; word-break: break-all; }
|
||||
button { background: #2563eb; color: #fff; border: 0; border-radius: 8px; padding: 10px 22px;
|
||||
font-size: 14px; cursor: pointer; margin-bottom: 18px; }
|
||||
button:hover { background: #1d4ed8; }
|
||||
.links a { color: #60a5fa; text-decoration: none; margin-right: 14px; font-size: 13px; }
|
||||
.links a:hover { text-decoration: underline; }
|
||||
#summary { font-size: 14px; margin-bottom: 14px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<h1>fsys 测试页</h1>
|
||||
<div class="sub">C++ 静态文件托管服务 · 路由/安全功能自检</div>
|
||||
|
||||
<button onclick="runAll()">▶ 运行全部测试</button>
|
||||
<div id="summary"></div>
|
||||
|
||||
<div class="card">
|
||||
<h2>API 自检</h2>
|
||||
<table id="tbl">
|
||||
<tr data-t="health"><td class="name">GET /health</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="filelist_root"><td class="name">GET /api/filelist(根目录)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="filelist_sub"><td class="name">GET /api/filelist?path=test%2F(编码子目录)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="filelist_traverse"><td class="name">GET /api/filelist?path=..%2F..(遍历应 403)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="static_file"><td class="name">GET /test/hello.txt(静态文件)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="download"><td class="name">GET /test/hello.txt?download=true(附件头)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="utf8_path"><td class="name">GET 中文文件名(URL 解码)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="hash"><td class="name">POST /api/hash(sha256("test123"))</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="hash_bad"><td class="name">POST /api/hash 非法输入(应 400)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="check_user"><td class="name">POST /api/check_user(结构校验)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="favicon_private"><td class="name">GET /api/favicon 内网地址(SSRF 应 403)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="favicon_public"><td class="name">GET /api/favicon 公网地址(需外网)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="headers"><td class="name">安全响应头(CSP/X-Frame-Options…)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
<tr data-t="notfound"><td class="name">GET /不存在的路径(应 404)</td><td class="st pending">—</td><td class="detail"></td></tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="card links">
|
||||
<h2>手动验证链接</h2>
|
||||
<a href="/api/filelist?path=" target="_blank">filelist 根</a>
|
||||
<a href="/api/filelist?path=test" target="_blank">filelist test/</a>
|
||||
<a href="/test/hello.txt" target="_blank">hello.txt</a>
|
||||
<a href="/test/hello.txt?download=true" target="_blank">下载模式</a>
|
||||
<a href="/health" target="_blank">health</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const EXPECT_HASH = 'ecd71870d1963316a97e3ac3408c9835ad8cf0f3c1bc703527c30265534f75ae';
|
||||
|
||||
function setRow(key, ok, detail) {
|
||||
const tr = document.querySelector(`tr[data-t="${key}"]`);
|
||||
if (!tr) return;
|
||||
const st = tr.children[1], dt = tr.children[2];
|
||||
st.textContent = ok ? 'PASS' : 'FAIL';
|
||||
st.className = 'st ' + (ok ? 'ok' : 'fail');
|
||||
dt.textContent = detail || '';
|
||||
}
|
||||
|
||||
async function j(url, opt) {
|
||||
const res = await fetch(url, opt);
|
||||
const text = await res.text();
|
||||
let json = null;
|
||||
try { json = JSON.parse(text); } catch (e) {}
|
||||
return { res, text, json };
|
||||
}
|
||||
|
||||
const tests = {
|
||||
async health() {
|
||||
const { res, json } = await j('/health');
|
||||
const ok = res.status === 200 && json && json.status === 'ok';
|
||||
setRow('health', ok, `HTTP ${res.status} ${JSON.stringify(json)}`);
|
||||
},
|
||||
async filelist_root() {
|
||||
const { res, json } = await j('/api/filelist?path=');
|
||||
const ok = res.status === 200 && json && json.success === true && Array.isArray(json.items);
|
||||
setRow('filelist_root', ok, `HTTP ${res.status}, ${json && json.items ? json.items.length + ' 项' : '无 items'}`);
|
||||
},
|
||||
async filelist_sub() {
|
||||
const { res, json } = await j('/api/filelist?path=' + encodeURIComponent('test'));
|
||||
const hasHello = json && Array.isArray(json.items) && json.items.some(i => i.name === 'hello.txt');
|
||||
setRow('filelist_sub', res.status === 200 && hasHello, `HTTP ${res.status}, hello.txt ${hasHello ? '可见' : '缺失'}`);
|
||||
},
|
||||
async filelist_traverse() {
|
||||
const { res } = await j('/api/filelist?path=' + encodeURIComponent('../../'));
|
||||
setRow('filelist_traverse', res.status === 403, `HTTP ${res.status}(期望 403)`);
|
||||
},
|
||||
async static_file() {
|
||||
const { res, text } = await j('/test/hello.txt');
|
||||
const ok = res.status === 200 && text.includes('hello fsys');
|
||||
setRow('static_file', ok, `HTTP ${res.status}, ${(res.headers.get('content-type') || '').split(';')[0]}`);
|
||||
},
|
||||
async download() {
|
||||
const { res } = await j('/test/hello.txt?download=true');
|
||||
const cd = res.headers.get('content-disposition') || '';
|
||||
setRow('download', res.status === 200 && cd.includes('attachment'), `content-disposition: ${cd || '(无)'}`);
|
||||
},
|
||||
async utf8_path() {
|
||||
const { res, text } = await j('/test/' + encodeURIComponent('中文文件.txt'));
|
||||
const ok = res.status === 200 && text.includes('中文内容');
|
||||
setRow('utf8_path', ok, `HTTP ${res.status}(URL 解码生效=${ok})`);
|
||||
},
|
||||
async hash() {
|
||||
const { res, json } = await j('/api/hash', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ password: 'test123' })
|
||||
});
|
||||
const ok = res.status === 200 && json && json.hash === EXPECT_HASH;
|
||||
setRow('hash', ok, `HTTP ${res.status}, hash ${ok ? '匹配' : '不匹配'}`);
|
||||
},
|
||||
async hash_bad() {
|
||||
const { res } = await j('/api/hash', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ password: 12345 })
|
||||
});
|
||||
setRow('hash_bad', res.status === 400, `HTTP ${res.status}(期望 400)`);
|
||||
},
|
||||
async check_user() {
|
||||
const { res, json } = await j('/api/check_user', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ email: 'fsys-test@example.com' })
|
||||
});
|
||||
// db 可能不可用:只要返回结构合法(success 字段存在)即 PASS
|
||||
const ok = [200, 500].includes(res.status) && json && typeof json.success === 'boolean';
|
||||
setRow('check_user', ok, `HTTP ${res.status} ${JSON.stringify(json)}`);
|
||||
},
|
||||
async favicon_private() {
|
||||
const { res } = await j('/api/favicon?url=' + encodeURIComponent('http://127.0.0.1:8005/'));
|
||||
setRow('favicon_private', res.status === 403, `HTTP ${res.status}(期望 403 forbidden host)`);
|
||||
},
|
||||
async favicon_public() {
|
||||
try {
|
||||
const { res } = await j('/api/favicon?url=' + encodeURIComponent('https://www.baidu.com'));
|
||||
const ct = res.headers.get('content-type') || '';
|
||||
setRow('favicon_public', res.status === 200 && ct.startsWith('image/'), `HTTP ${res.status} ${ct}`);
|
||||
} catch (e) {
|
||||
setRow('favicon_public', false, '网络不可达: ' + e.message);
|
||||
}
|
||||
},
|
||||
async headers() {
|
||||
const res = await fetch('/health');
|
||||
const need = ['content-security-policy', 'x-frame-options', 'x-content-type-options',
|
||||
'referrer-policy', 'permissions-policy'];
|
||||
const missing = need.filter(h => !res.headers.get(h));
|
||||
setRow('headers', missing.length === 0, missing.length ? '缺失: ' + missing.join(', ') : '5 项安全头齐全');
|
||||
},
|
||||
async notfound() {
|
||||
const { res } = await j('/no/such/file-' + Date.now());
|
||||
setRow('notfound', res.status === 404, `HTTP ${res.status}(期望 404)`);
|
||||
}
|
||||
};
|
||||
|
||||
async function runAll() {
|
||||
const summary = document.getElementById('summary');
|
||||
summary.textContent = '测试运行中…';
|
||||
document.querySelectorAll('#tbl tr').forEach(tr => {
|
||||
tr.children[1].textContent = '…'; tr.children[1].className = 'st pending';
|
||||
tr.children[2].textContent = '';
|
||||
});
|
||||
let pass = 0, fail = 0;
|
||||
for (const [name, fn] of Object.entries(tests)) {
|
||||
try { await fn(); }
|
||||
catch (e) { setRow(name, false, '异常: ' + e.message); }
|
||||
const st = document.querySelector(`tr[data-t="${name}"] .st`);
|
||||
if (st && st.classList.contains('ok')) pass++; else fail++;
|
||||
}
|
||||
summary.innerHTML = `完成:<span class="ok">${pass} 通过</span> / <span class="fail">${fail} 失败</span>(共 ${pass + fail})`;
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1 @@
|
||||
hello fsys — 这是 fsys 静态文件服务的测试文件。
|
||||
@@ -0,0 +1 @@
|
||||
这是中文内容:用于验证 URL 解码(UTF-8 编码文件名)是否生效。
|
||||
+1589
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user