From 7e440256718c36e189481077392c19bf32d78e29 Mon Sep 17 00:00:00 2001 From: qing <1776177662@qq.com> Date: Fri, 21 Aug 2026 14:47:29 +0000 Subject: [PATCH] =?UTF-8?q?feat(robot=5Fagent):=20=E5=9F=BA=E4=BA=8E=20lib?= =?UTF-8?q?http=20=E7=9A=84=20DeepSeek=20=E8=A7=86=E8=A7=89=E6=9C=BA?= =?UTF-8?q?=E5=99=A8=E4=BA=BA=20agent=20=E9=AA=A8=E6=9E=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 工具循环：capture_image / get_robot_status / spawn_subagent - 子代理：技能超集（run_command 带超时截断、read_file / write_file） - V4L2 摄像头抓帧：MJPEG 优先零转码，YUYV+libjpeg 回退，/dev/video* 自动探测 - 时间戳注入：system prompt 会话级当前时间 + 工具结果执行时间 - CLI：--prompt-file / --camera / --camera-device / --model / --max-turns --- robot_agent/.gitignore | 1 + robot_agent/Makefile | 22 +++ robot_agent/README.md | 78 ++++++++++ robot_agent/agent.cpp | 153 ++++++++++++++++++ robot_agent/agent.hpp | 32 ++++ robot_agent/camera.cpp | 249 +++++++++++++++++++++++++++++ robot_agent/camera.hpp | 44 ++++++ robot_agent/main.cpp | 129 +++++++++++++++ robot_agent/subagent.cpp | 23 +++ robot_agent/subagent.hpp | 9 ++ robot_agent/tools.cpp | 329 +++++++++++++++++++++++++++++++++++++++ robot_agent/tools.hpp | 63 ++++++++ 12 files changed, 1132 insertions(+) create mode 100644 robot_agent/.gitignore create mode 100644 robot_agent/Makefile create mode 100644 robot_agent/README.md create mode 100644 robot_agent/agent.cpp create mode 100644 robot_agent/agent.hpp create mode 100644 robot_agent/camera.cpp create mode 100644 robot_agent/camera.hpp create mode 100644 robot_agent/main.cpp create mode 100644 robot_agent/subagent.cpp create mode 100644 robot_agent/subagent.hpp create mode 100644 robot_agent/tools.cpp create mode 100644 robot_agent/tools.hpp diff --git a/robot_agent/.gitignore b/robot_agent/.gitignore new file mode 100644 index 0000000..75652bc --- /dev/null +++ b/robot_agent/.gitignore @@ -0,0 +1 @@ +robot_agent diff --git a/robot_agent/Makefile b/robot_agent/Makefile new file mode 100644 index 0000000..670db64 --- /dev/null +++ b/robot_agent/Makefile @@ -0,0 +1,22 @@ +# robot_agent — 基于 libhttp 的 DeepSeek 视觉机器人 agent 骨架 +CXX := g++ +CPPFLAGS := --std=c++20 -O2 -Wall -Wextra + +LIBHTTP_DIR := ../libhttp +INCS := -I $(LIBHTTP_DIR)/inc +LIBS := -L $(LIBHTTP_DIR)/build -lus_httpclient -lboost_json -lssl -lcrypto -lcurl -ljpeg -lpthread +RPATH := -Wl,-rpath,$(LIBHTTP_DIR)/build + +TARGET := robot_agent + +SRCS := main.cpp agent.cpp subagent.cpp tools.cpp camera.cpp + +all: $(TARGET) + +$(TARGET): $(SRCS) tools.hpp + $(CXX) $(CPPFLAGS) $(INCS) $(SRCS) $(LIBS) $(RPATH) -o $@ + +clean: + rm -f $(TARGET) + +.PHONY: all clean diff --git a/robot_agent/README.md b/robot_agent/README.md new file mode 100644 index 0000000..0c05321 --- /dev/null +++ b/robot_agent/README.md @@ -0,0 +1,78 @@ +# robot_agent — 基于 libhttp 的 DeepSeek 视觉机器人 Agent + +C++20 机器人智能体骨架：命令行输入任务，调用 DeepSeek 视觉大模型（`deepseek-v4-flash-vision-exp`），通过**工具循环**驱动摄像头识图、状态查询、命令行执行与文件读写，后续可扩展硬件控制。 + +## 架构 + +``` +用户任务（CLI 参数或 --prompt-file 文件） + └─ 主代理（run_chat_loop 工具循环） + ├─ capture_image 拍照识图（真实摄像头 / 模拟图片） + ├─ get_robot_status 查询机器人状态 + └─ spawn_subagent 委派任务给子代理 + └─ 子代理（技能超集，独立工具循环） + ├─ run_command 执行 shell 命令（超时 + 输出截断） + ├─ read_file / write_file + ├─ capture_image / get_robot_status + └─ 返回结论 → 主代理综合回答 +``` + +- **工具循环**：模型返回 `tool_calls` → agent 执行工具 → 结果回传 → 继续，直到模型给出最终回答（`--max-turns` 限轮）。 +- **图片注入**：OpenAI 协议的 tool 消息只允许文本，工具返回的图片由主循环作为 `image_url` 注入下一轮。 +- **时间戳**：会话开始时向 system prompt 注入一次当前时间（会话内不变）；每个工具结果附 `工具执行时间`，供模型判断数据新旧。 +- **摄像头**：V4L2 直采，优先 MJPEG（硬件编码零转码，最快），回退 YUYV + libjpeg 编码；设备常开复用；`/dev/video*` 自动探测。 + +## 依赖与构建 + +- g++（C++20）、libhttp（`libus_httpclient.so`）、Boost.JSON、libcurl、OpenSSL、libjpeg、pthread +- libhttp 需先编译：`cd ../libhttp && make` + +```bash +make # 生成 ./robot_agent +make clean +``` + +## 使用 + +```bash +export DEEPSEEK_API_KEY=sk-xxx # 或用 --key + +./robot_agent "看看环境里有什么" # 真实摄像头识图 +./robot_agent "报告状态" # 状态查询 +./robot_agent "列出当前目录文件" # 自动委派子代理执行命令 +./robot_agent --prompt-file /tmp/task.txt # 从文件读取任务 +./robot_agent "描述图片" --camera ./test.png # 模拟摄像头（读文件，便于离线测试） +``` + +| 参数 | 说明 | 默认 | +|---|---|---| +| `--prompt-file <文件>` | 从文件读取任务（与位置参数互斥） | — | +| `--camera <文件>` | 模拟摄像头：拍照工具读该图片 | 空 = 真实摄像头 | +| `--camera-device <设备>` | 摄像头设备 | 自动探测 `/dev/video*` | +| `--model <名称>` | 模型名 | `deepseek-v4-flash-vision-exp` | +| `--key ` | API key | 环境变量 `DEEPSEEK_API_KEY` | +| `--endpoint ` | 接口地址 | `https://api.deepseek.com/chat/completions` | +| `--max-turns ` | 最大工具轮次 | 8 | + +工具日志输出到 stderr（`[主代理][工具]` / `[子代理][工具]`），stdout 只输出最终回答，便于脚本管道。 + +## 代码结构 + +| 文件 | 职责 | +|---|---| +| `main.cpp` | CLI 解析、主代理工具注册 | +| `agent.hpp/cpp` | `run_chat_loop` 工具循环（主/子代理共用）、消息构造、时间戳注入 | +| `subagent.hpp/cpp` | 子代理：技能超集 + 独立提示词 | +| `tools.hpp/cpp` | `Tool`/`ToolRegistry` 与全部工具实现 | +| `camera.hpp/cpp` | V4L2 抓帧（MJPEG/YUYV、mmap、超时保护） | + +## 扩展 + +- **加工具**：实现 `Tool{name, description, parameters(JSON Schema), execute}` 后 `registry.add(...)`。 +- **工具返回图片**：`ToolResult.image_data_url` 填 data URL，主循环自动注入下一轮。 +- **加子代理类型**：仿照 `subagent.cpp` 组装不同工具集与提示词。 +- **接真实硬件**：把 `capture_image`/`get_robot_status` 的实现换成设备驱动调用即可，接口不变。 + +## 安全注意 + +`run_command` 会执行任意 shell 命令且无白名单/审批，仅建议在可信环境（你自己的机器人主机）运行；后续如需对外暴露，应先加命令审批或沙箱。 diff --git a/robot_agent/agent.cpp b/robot_agent/agent.cpp new file mode 100644 index 0000000..9f3a7db --- /dev/null +++ b/robot_agent/agent.cpp @@ -0,0 +1,153 @@ +#include "agent.hpp" + +#include + +#include +#include +#include + +using pmc::net::HttpClient; + +namespace { + +// 本地时间 ISO8601，如 2026-08-21T14:30:05+08:00 +std::string now_iso8601() { + std::time_t tt = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + std::tm tm{}; + localtime_r(&tt, &tm); + char buf[64]; + strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S%z", &tm); + std::string s(buf); + if (s.size() >= 5 && (s[s.size() - 5] == '+' || s[s.size() - 5] == '-')) + s.insert(s.size() - 2, ":"); + return s; +} + +} // namespace + +json::object user_text_msg(const std::string& text) { + return json::object{{"role", "user"}, {"content", text}}; +} + +json::object user_image_msg(const std::string& text, const std::string& data_url) { + return json::object{ + {"role", "user"}, + {"content", json::array{ + json::object{{"type", "text"}, {"text", text}}, + json::object{{"type", "image_url"}, + {"image_url", json::object{{"url", data_url}}}}, + }}, + }; +} + +json::object tool_result_msg(const std::string& call_id, const std::string& text) { + return json::object{{"role", "tool"}, {"tool_call_id", call_id}, {"content", text}}; +} + +json::value build_tools_spec(const ToolRegistry& registry) { + json::array arr; + for (const auto& t : registry.all()) { + arr.push_back(json::object{ + {"type", "function"}, + {"function", json::object{ + {"name", t.name}, + {"description", t.description}, + {"parameters", t.parameters}, + }}, + }); + } + return arr; +} + +std::string run_chat_loop(const AgentConfig& cfg, + const std::string& system_prompt, + const std::string& user_question, + const ToolRegistry& registry, + const std::string& log_tag) { + json::array messages; + // 会话开始注入一次当前时间（会话内保持不变，避免模型混淆"现在"） + std::string timed_prompt = + system_prompt + "\n当前时间: " + now_iso8601() + "（本会话内保持不变）"; + messages.push_back(json::object{{"role", "system"}, {"content", timed_prompt}}); + messages.push_back(user_text_msg(user_question)); + + HttpClient client; + client.set_header("Authorization: Bearer " + cfg.api_key); + client.set_header("Content-Type: application/json"); + + for (int turn = 0; turn < cfg.max_turns; ++turn) { + json::object body; + body["model"] = cfg.model; + body["messages"] = messages; + body["tools"] = build_tools_spec(registry); + body["stream"] = false; + + std::string response_body; + client.post_stream_sync(cfg.endpoint, json::serialize(body), + [&](const std::string& chunk) { response_body += chunk; }); + + long code = client.get_response_code(); + if (code != 200) + throw std::runtime_error("HTTP " + std::to_string(code) + ": " + response_body); + + json::error_code ec; + json::value resp = json::parse(response_body, ec); + if (ec) + throw std::runtime_error("响应不是合法 JSON: " + response_body.substr(0, 200)); + if (resp.as_object().contains("error")) + throw std::runtime_error("API 错误: " + json::serialize(resp.at("error"))); + + auto& choices = resp.at("choices").as_array(); + if (choices.empty()) throw std::runtime_error("响应中没有 choices"); + auto& message = choices[0].as_object().at("message").as_object(); + + auto tc_it = message.find("tool_calls"); + bool has_tool_calls = tc_it != message.end() && !tc_it->value().is_null() && + !tc_it->value().as_array().empty(); + + if (has_tool_calls) { + messages.push_back(message); // 回传 assistant 的 tool_calls 消息 + + for (const auto& call : tc_it->value().as_array()) { + const auto& c = call.as_object(); + std::string call_id = + c.contains("id") ? json::value_to(c.at("id")) : ""; + const auto& fn = c.at("function").as_object(); + std::string name = json::value_to(fn.at("name")); + + json::value args; + auto args_it = fn.find("arguments"); + if (args_it != fn.end() && !args_it->value().is_null()) { + std::string args_str = json::value_to(args_it->value()); + json::error_code aec; + args = json::parse(args_str, aec); + if (aec) args = json::object(); + } else { + args = json::object(); + } + + ToolResult result; + if (!registry.execute(name, args, result)) + result = {"错误: 未知工具 " + name, ""}; + + // 工具结果统一附执行时间戳，供模型判断数据新旧 + std::string timed_result = "工具执行时间: " + now_iso8601() + "\n" + result.text; + std::cerr << "[" << log_tag << "][工具] " << name << " → " + << timed_result.substr(0, 120) << "\n"; + messages.push_back(tool_result_msg(call_id, timed_result)); + if (!result.image_data_url.empty()) + messages.push_back( + user_image_msg("工具返回的图片如下，请结合图片继续回答。", + result.image_data_url)); + } + continue; + } + + if (message.contains("content") && message.at("content").is_string()) + return json::value_to(message.at("content")); + throw std::runtime_error("模型没有返回文本内容"); + } + + throw std::runtime_error("达到最大工具轮次 (" + std::to_string(cfg.max_turns) + + ")，未获得最终回答"); +} diff --git a/robot_agent/agent.hpp b/robot_agent/agent.hpp new file mode 100644 index 0000000..41fcb22 --- /dev/null +++ b/robot_agent/agent.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "tools.hpp" + +#include + +namespace json = boost::json; + +struct AgentConfig { + std::string model; + std::string endpoint; + std::string api_key; + std::string camera_path; // 非空 = 模拟摄像头（读文件） + std::string camera_device; // 真实摄像头设备（默认 /dev/video0） + int max_turns = 8; +}; + +// 运行工具循环，直到模型给出最终回答并返回其文本。失败抛 std::runtime_error。 +// log_tag 用于区分不同代理的工具日志（输出到 stderr）。 +std::string run_chat_loop(const AgentConfig& cfg, + const std::string& system_prompt, + const std::string& user_question, + const ToolRegistry& registry, + const std::string& log_tag); + +// 消息构造（主代理与子代理共用） +json::object user_text_msg(const std::string& text); +json::object user_image_msg(const std::string& text, const std::string& data_url); +json::object tool_result_msg(const std::string& call_id, const std::string& text); +json::value build_tools_spec(const ToolRegistry& registry); diff --git a/robot_agent/camera.cpp b/robot_agent/camera.cpp new file mode 100644 index 0000000..688eaf1 --- /dev/null +++ b/robot_agent/camera.cpp @@ -0,0 +1,249 @@ +#include "camera.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +constexpr size_t CAPTURE_BUFFER_COUNT = 3; +constexpr int FRAME_TIMEOUT_MS = 1500; // 单帧出队超时 + +// 扫描 /dev/video*，返回第一个支持视频采集的设备 +std::string auto_detect_device() { + glob_t g; + std::vector candidates; + if (glob("/dev/video*", 0, nullptr, &g) == 0) { + for (size_t i = 0; i < g.gl_pathc; ++i) + candidates.emplace_back(g.gl_pathv[i]); + } + globfree(&g); + std::sort(candidates.begin(), candidates.end()); + + for (const auto& dev : candidates) { + int fd = open(dev.c_str(), O_RDWR | O_NONBLOCK); + if (fd < 0) continue; + v4l2_capability cap{}; + bool ok = ioctl(fd, VIDIOC_QUERYCAP, &cap) == 0 && + (cap.capabilities & V4L2_CAP_VIDEO_CAPTURE); + close(fd); + if (ok) return dev; + } + throw std::runtime_error("未找到可用的视频采集设备（已扫描 /dev/video*，" + "可用 --camera-device 指定）"); +} + +void xioctl(int fd, unsigned long request, void* arg, const char* what) { + if (ioctl(fd, request, arg) != 0) + throw std::runtime_error(std::string(what) + " 失败: " + strerror(errno)); +} + +// YUYV → RGB24 → JPEG（libjpeg 内存压缩） +std::vector yuyv_to_jpeg(const std::vector& yuyv, + unsigned w, unsigned h) { + std::vector rgb(static_cast(w) * h * 3); + const unsigned char* in = yuyv.data(); + unsigned char* out = rgb.data(); + for (unsigned y = 0; y < h; ++y) { + for (unsigned x = 0; x + 1 < w; x += 2) { + int y0 = in[0], u = in[1] - 128, y1 = in[2], v = in[3] - 128; + in += 4; + int c0 = y0 - 16, c1 = y1 - 16; + int r0 = (298 * c0 + 409 * v + 128) >> 8; + int g0 = (298 * c0 - 100 * u - 208 * v + 128) >> 8; + int b0 = (298 * c0 + 516 * u + 128) >> 8; + int r1 = (298 * c1 + 409 * v + 128) >> 8; + int g1 = (298 * c1 - 100 * u - 208 * v + 128) >> 8; + int b1 = (298 * c1 + 516 * u + 128) >> 8; + *out++ = static_cast(std::clamp(r0, 0, 255)); + *out++ = static_cast(std::clamp(g0, 0, 255)); + *out++ = static_cast(std::clamp(b0, 0, 255)); + *out++ = static_cast(std::clamp(r1, 0, 255)); + *out++ = static_cast(std::clamp(g1, 0, 255)); + *out++ = static_cast(std::clamp(b1, 0, 255)); + } + if (w % 2) { // 奇数宽度：最后一像素复用前一对的 U/V + int y0 = in[0], u = in[1] - 128, v = in[2] - 128; + in += 4; + int c0 = y0 - 16; + *out++ = static_cast(std::clamp((298 * c0 + 409 * v + 128) >> 8, 0, 255)); + *out++ = static_cast(std::clamp((298 * c0 - 100 * u - 208 * v + 128) >> 8, 0, 255)); + *out++ = static_cast(std::clamp((298 * c0 + 516 * u + 128) >> 8, 0, 255)); + } + } + + jpeg_compress_struct cinfo; + jpeg_error_mgr jerr; + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_compress(&cinfo); + unsigned char* outbuf = nullptr; + unsigned long outsize = 0; + jpeg_mem_dest(&cinfo, &outbuf, &outsize); + cinfo.image_width = w; + cinfo.image_height = h; + cinfo.input_components = 3; + cinfo.in_color_space = JCS_RGB; + jpeg_set_defaults(&cinfo); + jpeg_set_quality(&cinfo, 85, TRUE); + jpeg_start_compress(&cinfo, TRUE); + while (cinfo.next_scanline < h) { + JSAMPROW row = rgb.data() + static_cast(cinfo.next_scanline) * w * 3; + jpeg_write_scanlines(&cinfo, &row, 1); + } + jpeg_finish_compress(&cinfo); + std::vector result(outbuf, outbuf + outsize); + free(outbuf); + jpeg_destroy_compress(&cinfo); + return result; +} + +} // namespace + +CameraCapture::CameraCapture(const std::string& device) : device_(device) { + if (device_.empty()) device_ = auto_detect_device(); + open_device(); + init_format(); + request_buffers(); +} + +CameraCapture::~CameraCapture() { + std::lock_guard lock(mtx_); + if (fd_ < 0) return; + try { + stream_off(); + } catch (...) { + } + for (size_t i = 0; i < buf_count_; ++i) + munmap(buffers_[i], buffer_sizes_[i]); + close(fd_); +} + +void CameraCapture::open_device() { + fd_ = open(device_.c_str(), O_RDWR | O_NONBLOCK); + if (fd_ < 0) + throw std::runtime_error("无法打开摄像头 " + device_ + ": " + strerror(errno)); + + v4l2_capability cap{}; + if (ioctl(fd_, VIDIOC_QUERYCAP, &cap) != 0) + throw std::runtime_error("VIDIOC_QUERYCAP 失败: " + std::string(strerror(errno))); + if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE) || + !(cap.capabilities & V4L2_CAP_STREAMING)) + throw std::runtime_error(device_ + " 不支持视频捕获或 mmap 流式传输"); +} + +void CameraCapture::init_format() { + v4l2_format fmt{}; + fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + fmt.fmt.pix.width = 640; + fmt.fmt.pix.height = 480; + fmt.fmt.pix.field = V4L2_FIELD_ANY; + + // 优先 MJPEG：摄像头硬件编码，抓帧即用，无需转码 + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_MJPEG; + if (ioctl(fd_, VIDIOC_S_FMT, &fmt) == 0 && fmt.fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG) { + jpeg_format_ = true; + } else { + fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV; + xioctl(fd_, VIDIOC_S_FMT, &fmt, "设置 YUYV 格式"); + if (fmt.fmt.pix.pixelformat != V4L2_PIX_FMT_YUYV) + throw std::runtime_error(device_ + " 不支持 MJPEG 或 YUYV"); + jpeg_format_ = false; + } + width_ = fmt.fmt.pix.width; + height_ = fmt.fmt.pix.height; +} + +void CameraCapture::request_buffers() { + v4l2_requestbuffers req{}; + req.count = CAPTURE_BUFFER_COUNT; + req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.memory = V4L2_MEMORY_MMAP; + xioctl(fd_, VIDIOC_REQBUFS, &req, "申请缓冲"); + if (req.count < 2) + throw std::runtime_error("摄像头缓冲不足"); + buf_count_ = req.count; + + buffers_.resize(buf_count_); + buffer_sizes_.resize(buf_count_); + for (size_t i = 0; i < buf_count_; ++i) { + v4l2_buffer buf{}; + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = V4L2_MEMORY_MMAP; + buf.index = i; + xioctl(fd_, VIDIOC_QUERYBUF, &buf, "查询缓冲"); + buffers_[i] = mmap(nullptr, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, + buf.m.offset); + if (buffers_[i] == MAP_FAILED) + throw std::runtime_error("mmap 失败: " + std::string(strerror(errno))); + buffer_sizes_[i] = buf.length; + xioctl(fd_, VIDIOC_QBUF, &buf, "缓冲入队"); + } +} + +void CameraCapture::stream_on() { + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + xioctl(fd_, VIDIOC_STREAMON, &type, "开启视频流"); +} + +void CameraCapture::stream_off() { + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (ioctl(fd_, VIDIOC_STREAMOFF, &type) != 0 && errno != EINVAL) + throw std::runtime_error("关闭视频流失败: " + std::string(strerror(errno))); +} + +std::vector CameraCapture::dequeue_frame(int timeout_ms) { + auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeout_ms); + while (true) { + v4l2_buffer buf{}; + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = V4L2_MEMORY_MMAP; + if (ioctl(fd_, VIDIOC_DQBUF, &buf) == 0) { + std::vector frame(static_cast(buffers_[buf.index]), + static_cast(buffers_[buf.index]) + + buf.bytesused); + xioctl(fd_, VIDIOC_QBUF, &buf, "缓冲重新入队"); + return frame; + } + if (errno != EAGAIN) + throw std::runtime_error("出队帧失败: " + std::string(strerror(errno))); + auto now = std::chrono::steady_clock::now(); + if (now >= deadline) + throw std::runtime_error("等待摄像头画面超时"); + int remain_ms = static_cast( + std::chrono::duration_cast(deadline - now).count()); + struct pollfd pfd{fd_, POLLIN, 0}; + poll(&pfd, 1, std::min(remain_ms, 200)); + } +} + +std::string CameraCapture::grab_jpeg() { + std::lock_guard lock(mtx_); + stream_on(); + try { + dequeue_frame(FRAME_TIMEOUT_MS); // 丢弃第一帧（曝光未稳定） + auto frame = dequeue_frame(FRAME_TIMEOUT_MS); // 取第二帧 + stream_off(); + if (!jpeg_format_) { + auto jpeg = yuyv_to_jpeg(frame, width_, height_); + return std::string(reinterpret_cast(jpeg.data()), jpeg.size()); + } + return std::string(reinterpret_cast(frame.data()), frame.size()); + } catch (...) { + try { + stream_off(); + } catch (...) { + } + throw; + } +} diff --git a/robot_agent/camera.hpp b/robot_agent/camera.hpp new file mode 100644 index 0000000..160c978 --- /dev/null +++ b/robot_agent/camera.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include + +// V4L2 摄像头抓帧：优先 MJPEG（硬件编码，抓到的就是 JPEG，零转码，速度最快）， +// 设备不支持时回退 YUYV + libjpeg 软件编码。 +// 设备常开复用（构造时打开并建流），每次抓帧 = STREAMON → 取帧 → STREAMOFF。 +// device 为空时自动扫描 /dev/video* 探测第一个可用设备。失败抛 std::runtime_error。 +class CameraCapture { +public: + explicit CameraCapture(const std::string& device); + ~CameraCapture(); + + CameraCapture(const CameraCapture&) = delete; + CameraCapture& operator=(const CameraCapture&) = delete; + + // 抓一帧，返回 JPEG 数据 + std::string grab_jpeg(); + + unsigned width() const { return width_; } + unsigned height() const { return height_; } + +private: + void open_device(); + void init_format(); + void request_buffers(); + void stream_on(); + void stream_off(); + // 出队一帧（带超时）。skip_first=true 时丢弃第一帧再取第二帧（等待曝光稳定） + std::vector dequeue_frame(int timeout_ms); + + std::string device_; + int fd_ = -1; + bool jpeg_format_ = false; // true = MJPEG 直出，false = YUYV 需转码 + unsigned width_ = 0; + unsigned height_ = 0; + std::vector buffers_; + std::vector buffer_sizes_; + size_t buf_count_ = 0; + std::mutex mtx_; +}; diff --git a/robot_agent/main.cpp b/robot_agent/main.cpp new file mode 100644 index 0000000..eccfc84 --- /dev/null +++ b/robot_agent/main.cpp @@ -0,0 +1,129 @@ +#include "agent.hpp" +#include "tools.hpp" + +#include +#include +#include +#include +#include + +namespace { + +constexpr const char* DEFAULT_ENDPOINT = "https://api.deepseek.com/chat/completions"; +constexpr const char* DEFAULT_MODEL = "deepseek-v4-flash-vision-exp"; + +constexpr const char* MAIN_SYSTEM_PROMPT = + "你是控制硬件机器人的主智能体。当用户的问题涉及观察环境、看东西、识别物体时，" + "必须先调用 capture_image 工具获取照片再回答。" + "当任务需要执行 shell 命令、读写文件等操作时，调用 spawn_subagent 工具委派给子代理，" + "不要自己尝试执行。回答使用中文。"; + +struct Options { + std::string question; + std::string prompt_file; // 非空 = 从文件读取问题 + std::string camera_path; // 非空 = 模拟摄像头（读文件） + std::string camera_device; // 空 = 自动探测 /dev/video* + std::string model = DEFAULT_MODEL; + std::string endpoint = DEFAULT_ENDPOINT; + std::string api_key; + int max_turns = 8; +}; + +void print_usage(const char* prog) { + std::cerr << "用法: " << prog << " \"问题\" [选项]\n" + << " " << prog << " --prompt-file <文件> [选项]\n" + << "选项:\n" + << " --prompt-file <文件> 从文件读取问题（与位置参数互斥）\n" + << " --camera <文件> 模拟摄像头：拍照工具读该图片（不指定则用真实摄像头）\n" + << " --camera-device <设备> 摄像头设备（默认自动探测 /dev/video*）\n" + << " --model <名称> 模型名（默认 " << DEFAULT_MODEL << "）\n" + << " --key API key（默认读环境变量 DEEPSEEK_API_KEY）\n" + << " --endpoint 接口地址（默认 DeepSeek 官方）\n" + << " --max-turns 最大工具轮次（默认 8）\n"; +} + +Options parse_args(int argc, char** argv) { + Options opt; + if (const char* key_env = std::getenv("DEEPSEEK_API_KEY")) opt.api_key = key_env; + + for (int i = 1; i < argc; ++i) { + std::string arg = argv[i]; + auto next = [&](const char* name) -> std::string { + if (i + 1 >= argc) { + std::cerr << "错误: " << name << " 缺少参数\n"; + std::exit(2); + } + return argv[++i]; + }; + if (arg == "--camera") opt.camera_path = next("--camera"); + else if (arg == "--camera-device") opt.camera_device = next("--camera-device"); + else if (arg == "--prompt-file") opt.prompt_file = next("--prompt-file"); + else if (arg == "--model") opt.model = next("--model"); + else if (arg == "--key") opt.api_key = next("--key"); + else if (arg == "--endpoint") opt.endpoint = next("--endpoint"); + else if (arg == "--max-turns") opt.max_turns = std::stoi(next("--max-turns")); + else if (arg == "--help" || arg == "-h") { print_usage(argv[0]); std::exit(0); } + else if (!arg.empty() && arg[0] == '-') { + std::cerr << "错误: 未知选项 " << arg << "\n"; + print_usage(argv[0]); + std::exit(2); + } else { + opt.question = arg; + } + } + return opt; +} + +} // namespace + +int main(int argc, char** argv) { + Options opt = parse_args(argc, argv); + + if (!opt.prompt_file.empty()) { + if (!opt.question.empty()) { + std::cerr << "错误: --prompt-file 与位置参数问题不能同时使用\n"; + return 2; + } + std::ifstream f(opt.prompt_file); + if (!f) { + std::cerr << "错误: 无法读取提示词文件 " << opt.prompt_file << "\n"; + return 2; + } + std::ostringstream ss; + ss << f.rdbuf(); + opt.question = ss.str(); + } + + if (opt.question.empty()) { + std::cerr << "错误: 缺少问题参数（用位置参数或 --prompt-file）\n"; + print_usage(argv[0]); + return 2; + } + if (opt.api_key.empty()) { + std::cerr << "错误: 缺少 API key（用 --key 或设置环境变量 DEEPSEEK_API_KEY）\n"; + return 2; + } + + AgentConfig cfg; + cfg.model = opt.model; + cfg.endpoint = opt.endpoint; + cfg.api_key = opt.api_key; + cfg.camera_path = opt.camera_path; + cfg.camera_device = opt.camera_device; + cfg.max_turns = opt.max_turns; + + ToolRegistry registry; + registry.add(make_capture_image_tool(opt.camera_path, opt.camera_device)); + registry.add(make_robot_status_tool()); + registry.add(make_spawn_subagent_tool(cfg)); + + try { + std::string answer = + run_chat_loop(cfg, MAIN_SYSTEM_PROMPT, opt.question, registry, "主代理"); + std::cout << answer << "\n"; + } catch (const std::exception& e) { + std::cerr << "错误: " << e.what() << "\n"; + return 1; + } + return 0; +} diff --git a/robot_agent/subagent.cpp b/robot_agent/subagent.cpp new file mode 100644 index 0000000..9d0ce6f --- /dev/null +++ b/robot_agent/subagent.cpp @@ -0,0 +1,23 @@ +#include "subagent.hpp" + +#include "tools.hpp" + +namespace { + +constexpr const char* SUBAGENT_PROMPT = + "你是机器人的子代理（专家），拥有比主代理更丰富的技能：可以执行 shell 命令、" + "读写文件、拍照和查询状态。请利用这些技能完成主代理委派的任务，" + "返回简洁明确的结果，用中文。"; + +} // namespace + +std::string run_subagent(const AgentConfig& cfg, const std::string& task) { + ToolRegistry registry; + registry.add(make_run_command_tool()); + registry.add(make_read_file_tool()); + registry.add(make_write_file_tool()); + registry.add(make_capture_image_tool(cfg.camera_path, cfg.camera_device)); + registry.add(make_robot_status_tool()); + + return run_chat_loop(cfg, SUBAGENT_PROMPT, task, registry, "子代理"); +} diff --git a/robot_agent/subagent.hpp b/robot_agent/subagent.hpp new file mode 100644 index 0000000..4704e4b --- /dev/null +++ b/robot_agent/subagent.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "agent.hpp" + +#include + +// 子代理：拥有命令行执行、文件读写等扩展技能（主代理的超集）， +// 完成主代理委派的任务后返回结论文本。 +std::string run_subagent(const AgentConfig& cfg, const std::string& task); diff --git a/robot_agent/tools.cpp b/robot_agent/tools.cpp new file mode 100644 index 0000000..e96146c --- /dev/null +++ b/robot_agent/tools.cpp @@ -0,0 +1,329 @@ +#include "tools.hpp" + +#include "agent.hpp" +#include "camera.hpp" +#include "subagent.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::string base64_encode(const std::string& in) { + static const char table[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string out; + out.reserve((in.size() + 2) / 3 * 4); + for (size_t i = 0; i < in.size(); i += 3) { + unsigned n = static_cast(in[i]) << 16; + if (i + 1 < in.size()) n |= static_cast(in[i + 1]) << 8; + if (i + 2 < in.size()) n |= static_cast(in[i + 2]); + out.push_back(table[(n >> 18) & 0x3F]); + out.push_back(table[(n >> 12) & 0x3F]); + out.push_back(i + 1 < in.size() ? table[(n >> 6) & 0x3F] : '='); + out.push_back(i + 2 < in.size() ? table[n & 0x3F] : '='); + } + return out; +} + +std::string mime_from_path(const std::string& path) { + auto pos = path.find_last_of('.'); + std::string ext = (pos == std::string::npos) ? "" : path.substr(pos + 1); + if (ext == "jpg" || ext == "jpeg") return "image/jpeg"; + if (ext == "png") return "image/png"; + if (ext == "webp") return "image/webp"; + if (ext == "gif") return "image/gif"; + throw std::runtime_error("不支持的图片格式: " + ext); +} + +} // namespace + +Tool make_capture_image_tool(const std::string& simulated_image_path, + const std::string& camera_device) { + Tool t; + t.name = "capture_image"; + t.description = + "拍摄一张机器人当前环境的照片。当用户的问题涉及观察、看、识别物体或了解周围环境时，" + "必须先调用本工具获取照片，再基于照片回答。"; + t.parameters = json::object{{"type", "object"}, {"properties", json::object{}}}; + + // 摄像头实例懒加载并常开复用，避免每次抓帧都重新打开设备 + auto camera = std::make_shared>(); + + t.execute = [simulated_image_path, camera_device, camera](const json::value&) -> ToolResult { + if (!simulated_image_path.empty()) { + std::ifstream f(simulated_image_path, std::ios::binary); + if (!f) + return {"错误: 无法读取模拟摄像头图片 " + simulated_image_path, ""}; + std::ostringstream ss; + ss << f.rdbuf(); + std::string data_url; + try { + data_url = "data:" + mime_from_path(simulated_image_path) + ";base64," + + base64_encode(ss.str()); + } catch (const std::exception& e) { + return {std::string("错误: ") + e.what(), ""}; + } + return {"已拍摄当前环境照片", data_url}; + } + + try { + if (!*camera) + *camera = std::make_shared(camera_device); + std::string jpeg = (*camera)->grab_jpeg(); + return {"已拍摄当前环境照片（" + std::to_string((*camera)->width()) + "x" + + std::to_string((*camera)->height()) + "）", + "data:image/jpeg;base64," + base64_encode(jpeg)}; + } catch (const std::exception& e) { + camera->reset(); // 设备可能重新插拔，下次调用重试打开 + return {std::string("摄像头抓拍失败: ") + e.what(), ""}; + } + }; + return t; +} + +Tool make_robot_status_tool() { + Tool t; + t.name = "get_robot_status"; + t.description = "查询机器人当前状态（电量、温度、电机、传感器）。用户询问机器人状态时调用。"; + t.parameters = json::object{{"type", "object"}, {"properties", json::object{}}}; + t.execute = [](const json::value&) -> ToolResult { + return {"电量 85%，电机正常，温度 42°C，传感器无告警", ""}; + }; + return t; +} + +// ---- 命令行执行 ---- + +constexpr size_t MAX_STREAM_BYTES = 64 * 1024; // 单个流输出上限 +constexpr int MAX_CMD_TIMEOUT_S = 60; + +struct RunResult { + int exit_code = -1; + bool timed_out = false; + bool truncated = false; + std::string out; + std::string err; +}; + +// 追加输出，超出上限则丢弃并标记截断 +void append_capped(std::string& dst, const char* p, size_t n, bool& truncated) { + size_t room = MAX_STREAM_BYTES - dst.size(); + if (n > room) { + dst.append(p, room); + truncated = true; + } else { + dst.append(p, n); + } +} + +RunResult run_shell(const std::string& cmd, int timeout_s) { + int po[2], pe[2]; + if (pipe(po) != 0 || pipe(pe) != 0) throw std::runtime_error("pipe 失败"); + pid_t pid = fork(); + if (pid < 0) throw std::runtime_error("fork 失败"); + if (pid == 0) { + dup2(po[1], STDOUT_FILENO); + dup2(pe[1], STDERR_FILENO); + close(po[0]); + close(po[1]); + close(pe[0]); + close(pe[1]); + execl("/bin/sh", "sh", "-c", cmd.c_str(), static_cast(nullptr)); + _exit(127); + } + close(po[1]); + close(pe[1]); + fcntl(po[0], F_SETFL, O_NONBLOCK); + fcntl(pe[0], F_SETFL, O_NONBLOCK); + + RunResult r; + bool exited = false, out_eof = false, err_eof = false, killed = false; + int status = 0; + std::array buf; + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(timeout_s); + auto hard_deadline = deadline + std::chrono::seconds(3); + + while (!(exited && out_eof && err_eof)) { + struct pollfd fds[2] = { + {out_eof ? -1 : po[0], POLLIN, 0}, + {err_eof ? -1 : pe[0], POLLIN, 0}, + }; + if (poll(fds, 2, 100) > 0) { + for (const auto& fd : fds) { + if (fd.fd < 0 || !(fd.revents & (POLLIN | POLLHUP))) continue; + std::string& sink = (fd.fd == po[0]) ? r.out : r.err; + ssize_t n; + while ((n = read(fd.fd, buf.data(), buf.size())) > 0) + append_capped(sink, buf.data(), n, r.truncated); + if (n == 0 || (n < 0 && errno != EAGAIN && errno != EWOULDBLOCK)) { + close(fd.fd); + if (fd.fd == po[0]) out_eof = true; + else err_eof = true; + } + } + } + if (!exited) { + pid_t w = waitpid(pid, &status, WNOHANG); + if (w == pid) exited = true; + } + auto now = std::chrono::steady_clock::now(); + if (!killed && now >= deadline) { + kill(pid, SIGKILL); + killed = true; + r.timed_out = true; + } + if (now >= hard_deadline) break; + } + + if (!out_eof) close(po[0]); + if (!err_eof) close(pe[0]); + if (!exited) waitpid(pid, &status, 0); // 已被 SIGKILL，等待回收 + r.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; + return r; +} + +std::string format_run_result(const RunResult& r) { + std::string s = "exit_code=" + std::to_string(r.exit_code); + if (r.timed_out) s += "（超时，已终止）"; + s += "\nstdout:\n" + (r.out.empty() ? "(空)" : r.out); + s += "\nstderr:\n" + (r.err.empty() ? "(空)" : r.err); + if (r.truncated) s += "\n[输出超过 64KB，已截断]"; + return s; +} + +Tool make_run_command_tool() { + Tool t; + t.name = "run_command"; + t.description = + "在机器人主机上执行 shell 命令，返回退出码、标准输出和标准错误。" + "可用于查看系统状态、操作硬件等。命令有超时限制，输出超过 64KB 会被截断。"; + t.parameters = json::object{ + {"type", "object"}, + {"properties", json::object{ + {"command", json::object{{"type", "string"}, {"description", "要执行的 shell 命令"}}}, + {"timeout_seconds", json::object{{"type", "integer"}, + {"description", "超时秒数，默认 10，最大 60"}}}, + }}, + {"required", json::array{"command"}}, + }; + t.execute = [](const json::value& args) -> ToolResult { + const auto& obj = args.as_object(); + auto cit = obj.find("command"); + if (cit == obj.end() || !cit->value().is_string()) + return {"错误: 缺少 command 参数", ""}; + std::string cmd = json::value_to(cit->value()); + + int timeout_s = 10; + auto tit = obj.find("timeout_seconds"); + if (tit != obj.end() && tit->value().is_int64()) + timeout_s = static_cast(tit->value().as_int64()); + if (timeout_s < 1) timeout_s = 1; + if (timeout_s > MAX_CMD_TIMEOUT_S) timeout_s = MAX_CMD_TIMEOUT_S; + + try { + return {format_run_result(run_shell(cmd, timeout_s)), ""}; + } catch (const std::exception& e) { + return {std::string("执行失败: ") + e.what(), ""}; + } + }; + return t; +} + +// ---- 文件读写 ---- + +Tool make_read_file_tool() { + Tool t; + t.name = "read_file"; + t.description = "读取文本文件内容。超过 64KB 截断。"; + t.parameters = json::object{ + {"type", "object"}, + {"properties", json::object{ + {"path", json::object{{"type", "string"}, {"description", "文件路径"}}}, + }}, + {"required", json::array{"path"}}, + }; + t.execute = [](const json::value& args) -> ToolResult { + auto it = args.as_object().find("path"); + if (it == args.as_object().end() || !it->value().is_string()) + return {"错误: 缺少 path 参数", ""}; + std::string path = json::value_to(it->value()); + std::ifstream f(path, std::ios::binary); + if (!f) return {"错误: 无法读取文件 " + path, ""}; + std::string content((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); + bool truncated = content.size() > MAX_STREAM_BYTES; + if (truncated) content.resize(MAX_STREAM_BYTES); + return {content + (truncated ? "\n[内容已截断]" : ""), ""}; + }; + return t; +} + +Tool make_write_file_tool() { + Tool t; + t.name = "write_file"; + t.description = "写入文本文件（覆盖已有内容）。"; + t.parameters = json::object{ + {"type", "object"}, + {"properties", json::object{ + {"path", json::object{{"type", "string"}, {"description", "文件路径"}}}, + {"content", json::object{{"type", "string"}, {"description", "要写入的内容"}}}, + }}, + {"required", json::array{"path", "content"}}, + }; + t.execute = [](const json::value& args) -> ToolResult { + const auto& obj = args.as_object(); + auto pit = obj.find("path"); + auto cit = obj.find("content"); + if (pit == obj.end() || !pit->value().is_string() || cit == obj.end() || + !cit->value().is_string()) + return {"错误: 需要 path 和 content 参数", ""}; + std::string path = json::value_to(pit->value()); + std::string content = json::value_to(cit->value()); + std::ofstream f(path, std::ios::binary); + if (!f) return {"错误: 无法写入文件 " + path, ""}; + f.write(content.data(), static_cast(content.size())); + if (!f) return {"错误: 写入失败 " + path, ""}; + return {"已写入 " + std::to_string(content.size()) + " 字节到 " + path, ""}; + }; + return t; +} + +// ---- 子代理 ---- + +Tool make_spawn_subagent_tool(const AgentConfig& cfg) { + Tool t; + t.name = "spawn_subagent"; + t.description = + "委派任务给子代理执行。子代理拥有扩展技能：执行 shell 命令、读写文件、拍照、" + "查询状态。当任务需要运行命令、读写文件等主代理不具备的能力时调用。"; + t.parameters = json::object{ + {"type", "object"}, + {"properties", json::object{ + {"task", json::object{{"type", "string"}, {"description", "委派给子代理的具体任务"}}}, + }}, + {"required", json::array{"task"}}, + }; + t.execute = [cfg](const json::value& args) -> ToolResult { + auto it = args.as_object().find("task"); + if (it == args.as_object().end() || !it->value().is_string()) + return {"错误: 缺少 task 参数", ""}; + std::string task = json::value_to(it->value()); + try { + return {run_subagent(cfg, task), ""}; + } catch (const std::exception& e) { + return {std::string("子代理执行失败: ") + e.what(), ""}; + } + }; + return t; +} diff --git a/robot_agent/tools.hpp b/robot_agent/tools.hpp new file mode 100644 index 0000000..091c286 --- /dev/null +++ b/robot_agent/tools.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include + +#include +#include +#include + +namespace json = boost::json; + +struct AgentConfig; // 前向声明（定义见 agent.hpp），避免头文件循环依赖 + +// 工具执行结果。image_data_url 非空表示工具返回了一张图片， +// 由 agent 主循环把图片注入下一轮对话（OpenAI 协议的 tool 消息只允许文本）。 +struct ToolResult { + std::string text; + std::string image_data_url; +}; + +struct Tool { + std::string name; + std::string description; + json::value parameters; // JSON Schema（object） + std::function execute; +}; + +class ToolRegistry { +public: + void add(Tool tool) { tools_.push_back(std::move(tool)); } + + const std::vector& all() const { return tools_; } + + bool execute(const std::string& name, const json::value& args, ToolResult& out) const { + for (const auto& t : tools_) { + if (t.name == name) { + out = t.execute(args); + return true; + } + } + return false; + } + +private: + std::vector tools_; +}; + +// 拍照工具：simulated_image_path 非空时读取该文件（模拟模式）， +// 否则从真实摄像头设备（V4L2，默认 /dev/video0）抓帧。 +Tool make_capture_image_tool(const std::string& simulated_image_path, + const std::string& camera_device); + +// 状态查询工具：返回模拟的机器人状态。 +Tool make_robot_status_tool(); + +// 命令行工具：执行 shell 命令（带超时与输出截断）。 +Tool make_run_command_tool(); + +// 文件工具：读取/写入文本文件。 +Tool make_read_file_tool(); +Tool make_write_file_tool(); + +// 子代理工具：把任务委派给拥有扩展技能的子代理执行。 +Tool make_spawn_subagent_tool(const AgentConfig& cfg);