diff --git a/BotOps/README.md b/BotOps/README.md index 938f9c6..54e5948 100644 --- a/BotOps/README.md +++ b/BotOps/README.md @@ -71,6 +71,13 @@ BotOps/ ## 更新日志 +### v1.3 +- ✅ **适配服务端中断机制（Ctrl+C可靠停止）**： + - 收到 `interrupt_ack` 时立即收尾前端状态：调用 `abort_reasoning()` 将思考区域更新为"🛑 思考已中断"标签，调用 `finish_ai()` 结束AI回复状态 + - 新增 `ChatView.abort_reasoning()` 方法：中断时更新思考区域标签（保留已展示的思考内容） + - `append_reasoning()` 增加 `_ai_finalized` 检查，中断后丢弃残留的思考片段防止重新创建思考区域 + - 中断后服务端发来的残留 `chunk`/`reasoning`/`complete` 消息均被安全忽略，不会污染UI + ### v1.2 - ✅ **彻底修复思考区域顺序问题**：移除预创建空 AI widget 的逻辑，改为第一次收到 `chunk` 时才创建 widget。按消息到达顺序自然挂载（`reasoning_start`→`reasoning`→`chunk`），从根本上保证思考区域始终在正式回答之前 - ✅ 简化 `complete` 处理逻辑：仅作为收尾重置状态信号，不再进行补建逻辑 diff --git a/BotOps/app.py b/BotOps/app.py index 5ea6049..ca3170f 100644 --- a/BotOps/app.py +++ b/BotOps/app.py @@ -226,6 +226,9 @@ class BotOpsApp(App): f"❌ [{data.get('task_id', '?')}] {data.get('task_title', '?')}: " f"{data.get('error', '未知错误')}") elif t == "interrupt_ack": + # 服务端确认中断：立即收尾前端状态（更新思考区域为已中断、结束AI回复） + await self.chat.abort_reasoning() + await self.chat.finish_ai("") await self.notify_system(f"🛑 {content}") elif t == "error": await self.notify_system(f"❌ {content}", level="error") diff --git a/BotOps/widgets/chat_view.py b/BotOps/widgets/chat_view.py index 2dcf3a4..09c0bf8 100644 --- a/BotOps/widgets/chat_view.py +++ b/BotOps/widgets/chat_view.py @@ -75,6 +75,9 @@ class ChatView(VerticalScroll): """追加思考内容增量。""" if not delta: return + if self._ai_finalized: + # 已收尾（中断/完成），丢弃残留的思考片段 + return if self._reasoning_widget is None: await self.start_reasoning() self._reasoning_text += delta @@ -95,6 +98,17 @@ class ChatView(VerticalScroll): self._reasoning_widget.update(f"💭 思考过程（{len(self._reasoning_text)}字符）\n{display}\n{'─' * 30}") # 思考区域保留显示，不删除 + async def abort_reasoning(self): + """中断思考区域（用户Ctrl+C），更新提示文字为已中断。""" + if self._reasoning_widget is not None: + if self._reasoning_text: + display = self._reasoning_text + if len(display) > 3000: + display = "...(前略)\n" + display[-3000:] + self._reasoning_widget.update(f"🛑 思考已中断（{len(self._reasoning_text)}字符）\n{display}") + else: + self._reasoning_widget.update("🛑 思考已中断") + async def _ensure_ai_widget(self): """确保AI widget存在（第一次chunk到来时创建）。""" if self._ai_widget is None: diff --git a/aibox/README.md b/aibox/README.md index a1105a8..2283157 100644 --- a/aibox/README.md +++ b/aibox/README.md @@ -21,6 +21,7 @@ - 📦 **对话压缩**：消息数或 Token 数达到阈值时自动压缩，保持上下文质量 - 🛡️ **内存安全防护**：文件读取按需加载不全载、命令输出 1MB 限制、单行超长截断、工具结果多层防护防止 OOM - 🔒 **错误自动恢复**：API 请求失败自动回滚消息，工具结果安全清洗（控制字符过滤、非字符串转换） +- 🛑 **可靠中断机制**：支持 Ctrl+C 实时中断，通过 `threading.Event` + HTTP 连接关闭 + 子进程 kill 三层取消，流式响应和长时间运行命令均可立即终止 ## 🚀 快速开始 @@ -192,6 +193,15 @@ const ws = new WebSocket('ws://localhost:8765'); } ``` +#### 5. 中断（Ctrl+C） +```json +{ + "type": "interrupt", + "content": "用户按下了 Ctrl+C 中断当前操作" +} +``` +服务端收到后立即回复 `interrupt_ack`，并通过三层取消机制（Event标志 + HTTP连接关闭 + 子进程kill）终止正在进行的AI思考和工具执行。 + ### 支持的命令 > 注意：记忆搜索通过 `memory` 工具（在对话中由 AI 调用）完成，本系统**不存在** `/search` 斜杠命令。 @@ -706,6 +716,19 @@ elif command == "/mycommand": ## 📄 更新日志 +### v2.6 +- ✅ **修复 Ctrl+C 中断无法停止服务器的问题**： + - 新增 `interrupt` WebSocket 消息类型处理（BotOps 前端 Ctrl+C 发送此消息），立即回复 `interrupt_ack` 确认 + - `DeepSeekChat` 新增 `cancel()` 方法和 `_cancel_event`（threading.Event），线程安全可跨线程调用 + - **三层取消机制**： + 1. 设置取消标志，流式响应循环每轮检查，发现取消立即 break + 2. 关闭当前 HTTP response 对象，中断 `iter_lines()` 阻塞读取 + 3. 终止正在运行的子进程（`SubprocessCommandExecutor.cancel()` → `process.kill()`） + - 工具调用循环中每个工具执行前检查取消信号，中断后不再继续后续工具 + - API 请求异常/JSON解析异常/未知异常中均检查取消标志，若因取消导致异常则按中断处理而非报错 + - 中断后自动回滚本轮消息，防止污染对话历史 + - 新增 `EnhancedAIManager.cancel()` 统一入口，同时终止 AI 和命令执行器 + ### v2.5 - ✅ **新增 DeepSeek 思考模式支持**（Reasoning Mode）： - 新增 `api.reasoning_enabled` 配置项（默认 `false`），启用后在请求中添加 `thinking: {type: "enabled"}` 参数 diff --git a/aibox/aibox.py b/aibox/aibox.py index 2ee8098..86284a1 100644 --- a/aibox/aibox.py +++ b/aibox/aibox.py @@ -1168,7 +1168,7 @@ class ConfigManager: "api": { "deepseek_api_url": "https://api.deepseek.com/chat/completions", "deepseek_model": "deepseek-v4-flash", - "reasoning_enabled": false, + "reasoning_enabled": False, "api_key_env": "DEEPSEEK_API_KEY", "timeout": 120, "temperature": 0.7 @@ -3532,11 +3532,28 @@ class CommandExecutor(ABC): class SubprocessCommandExecutor(CommandExecutor): - """基于subprocess的命令执行器 - 支持超时、工作目录和输出大小限制""" + """基于subprocess的命令执行器 - 支持超时、工作目录、输出大小限制和取消""" # 命令输出最大字节数（1MB），超出截断 MAX_OUTPUT_BYTES = 1 * 1024 * 1024 + def __init__(self, logger=None): + super().__init__(logger) + self._current_process = None + self._process_lock = threading.Lock() + + def cancel(self): + """取消当前正在运行的命令（kill子进程）。线程安全。""" + with self._process_lock: + proc = self._current_process + if proc is not None: + try: + if proc.poll() is None: + proc.kill() + self.logger.info("已终止正在运行的子进程") + except Exception: + pass + def _read_with_limit(self, process, timeout: int, start_time: float, output_queue=None) -> Tuple[str, bool, bool]: """从进程读取输出，限制总大小和超时。 返回 (output_str, timeout_occurred, truncated) @@ -3612,6 +3629,9 @@ class SubprocessCommandExecutor(CommandExecutor): env=exec_env, cwd=os.path.expanduser(work_dir) ) + # 保存进程引用以便cancel()可以kill它 + with self._process_lock: + self._current_process = process start_time = time.time() output_str, timeout_occurred, truncated = self._read_with_limit(process, timeout, start_time) @@ -3654,6 +3674,9 @@ class SubprocessCommandExecutor(CommandExecutor): finally: if process and process.poll() is None: process.kill() + # 清除当前进程引用 + with self._process_lock: + self._current_process = None def execute_streaming(self, command: str, output_queue: queue.Queue, timeout: int = 30, work_dir: str = None, env: Dict = None, **kwargs): @@ -3694,6 +3717,9 @@ class SubprocessCommandExecutor(CommandExecutor): cwd=os.path.expanduser(work_dir) ) self.process = process + # 保存进程引用以便cancel()可以kill它 + with self._process_lock: + self._current_process = process start_time = time.time() output_lines = [] @@ -3788,6 +3814,8 @@ class SubprocessCommandExecutor(CommandExecutor): except: process.kill() self.process = None + with self._process_lock: + self._current_process = None # ==================== 工具基类 ==================== @@ -6750,11 +6778,51 @@ class DeepSeekChat: self.preserve_system = constants.PRESERVE_SYSTEM_PROMPTS self.stats = {"api_calls": 0, "tool_calls": 0, "errors": []} + # 取消机制：threading.Event 用于跨线程信号，_current_response 用于中断HTTP请求 + self._cancel_event = threading.Event() + self._current_response = None + self._command_executor = command_executor # 保存命令执行器引用，用于取消时kill子进程 + self._init_system_prompts(system_prompt) self.tool_registry = ToolRegistry(self.logger) self._register_tools(exit_callback, command_executor) + def cancel(self): + """中断当前正在进行的AI思考/工具执行。线程安全，可从任意线程调用。""" + self.logger.info("收到中断信号，正在取消当前操作...") + self._cancel_event.set() + # 强制关闭当前HTTP响应，立即中断 iter_lines() 的阻塞读取 + # 仅调用 response.close() 可能无法中断底层socket recv()，需要先关闭raw连接 + if self._current_response is not None: + try: + resp = self._current_response + self._current_response = None + # 先关闭底层raw socket连接，强制阻塞的recv()立即抛出异常 + if hasattr(resp, 'raw') and resp.raw is not None: + try: + resp.raw.close() + except Exception: + pass + resp.close() + except Exception: + pass + # 终止正在运行的子进程 + if self._command_executor is not None and hasattr(self._command_executor, 'cancel'): + try: + self._command_executor.cancel() + except Exception: + pass + + def _is_cancelled(self) -> bool: + """检查是否已被取消。""" + return self._cancel_event.is_set() + + def _check_cancelled(self): + """如果已取消，抛出异常中断当前流程。""" + if self._cancel_event.is_set(): + raise InterruptedError(i18n.get('user_interrupted', '用户中断')) + def _init_system_prompts(self, system_prompt: str = None): """初始化系统提示词 - 从 PromptManager 加载（按 identity → 技能 → soul → status 顺序）""" if system_prompt: @@ -6965,6 +7033,11 @@ class DeepSeekChat: MAX_RESULT_LENGTH = constants.TOOL_MAX_RESULT_LENGTH for tool_call in tool_calls: + # 每个工具调用前检查取消信号 + if self._is_cancelled(): + self.logger.info("工具调用被用户中断") + break + function = tool_call.get("function", {}) tool_name = function.get("name", "unknown") arguments = function.get("arguments", "{}") @@ -7076,6 +7149,13 @@ class DeepSeekChat: is_continuation: 是否为工具调用后的递归继续；为 True 时不注入冗长的工作流指令， 因为 system prompt 中已有完整行为规范，继续时只需传递工具结果 """ + # 首轮开始时清除取消标志 + if not is_continuation: + self._cancel_event.clear() + + # 立即检查是否已被取消 + self._check_cancelled() + if not self.conversation_active: self.logger.info("对话已结束，重置状态") self.conversation_active = True @@ -7217,6 +7297,7 @@ class DeepSeekChat: stream=True, proxies=proxies # 动态代理 ) + self._current_response = response # 保存引用以便cancel()关闭连接 if response.status_code != 200: error_msg = f"\n错误状态码: {response.status_code}\n错误响应: {response.text}" @@ -7225,6 +7306,9 @@ class DeepSeekChat: response.raise_for_status() for line in response.iter_lines(): + # 流式循环中检查取消信号 + if self._is_cancelled(): + break if line: line = line.decode('utf-8') if line.startswith('data: '): @@ -7315,6 +7399,16 @@ class DeepSeekChat: self.logger.error(f"JSON解析错误: {e}, 数据: {data}") continue + # 流式循环结束，检查是否被取消 + if self._is_cancelled(): + self._rollback_messages(msgs_before_turn) + interrupt_msg = i18n.get('user_interrupted', '用户中断') + if output_callback: + output_callback("line", "\n" + interrupt_msg + "\n") + output_callback("complete", interrupt_msg) + self.logger.info("AI响应被用户中断") + return interrupt_msg + # 处理工具调用 if current_tool_calls: tool_calls_buffer = list(current_tool_calls.values()) @@ -7382,7 +7476,24 @@ class DeepSeekChat: return full_response if full_response else "" + except InterruptedError: + # 用户主动中断：回滚消息，通知前端 + self._rollback_messages(msgs_before_turn) + interrupt_msg = i18n.get('user_interrupted', '用户中断') + if output_callback: + output_callback("line", "\n" + interrupt_msg + "\n") + output_callback("complete", interrupt_msg) + self.logger.info("AI响应被用户中断") + return interrupt_msg except requests.exceptions.RequestException as e: + # 如果是因为取消导致的连接关闭，按中断处理 + if self._is_cancelled(): + self._rollback_messages(msgs_before_turn) + interrupt_msg = i18n.get('user_interrupted', '用户中断') + if output_callback: + output_callback("line", "\n" + interrupt_msg + "\n") + output_callback("complete", interrupt_msg) + return interrupt_msg # 网络/HTTP 错误：回滚本轮添加的消息（user 消息 + 可能的 assistant/tool 消息）， # 防止格式错误的消息污染历史导致后续请求持续失败 self._rollback_messages(msgs_before_turn) @@ -7402,6 +7513,14 @@ class DeepSeekChat: self.logger.error(f"API响应解析失败，已回滚消息到位置 {msgs_before_turn}: {e}") return error_msg except Exception as e: + # 如果是因为取消导致的异常，按中断处理 + if self._is_cancelled(): + self._rollback_messages(msgs_before_turn) + interrupt_msg = i18n.get('user_interrupted', '用户中断') + if output_callback: + output_callback("line", "\n" + interrupt_msg + "\n") + output_callback("complete", interrupt_msg) + return interrupt_msg # 未知错误：回滚消息，防止对话状态被破坏 self._rollback_messages(msgs_before_turn) error_msg = f"\n{i18n.get('error_prefix')}{str(e)}" @@ -7410,6 +7529,9 @@ class DeepSeekChat: self.stats["errors"].append(str(e)) self.logger.error(f"未知错误，已回滚消息到位置 {msgs_before_turn}: {e}") return error_msg + finally: + # 清除当前响应引用 + self._current_response = None def _rollback_messages(self, to_length: int): """回滚 self.messages 到指定长度，用于 API 失败时恢复对话状态。""" @@ -7471,6 +7593,8 @@ class WebSocketHandler: self.logger = Logger("WebSocket") self.connected_clients = set() self.client_sessions = {} + self._processing_tasks = {} # websocket id -> 当前正在处理的asyncio Task + self._is_processing_message = False # 是否正在处理用户消息（防止并发） async def register(self, websocket): """注册新客户端""" @@ -7623,6 +7747,8 @@ class WebSocketHandler: await self.handle_command(websocket, data) elif msg_type == "message": await self.handle_user_message(websocket, data) + elif msg_type == "interrupt": + await self.handle_interrupt(websocket, data) elif msg_type == "task": await self.handle_task_message(websocket, data) elif msg_type == "ping": @@ -7648,6 +7774,18 @@ class WebSocketHandler: "content": f"处理失败: {str(e)}" }) + async def handle_interrupt(self, websocket, data): + """处理中断信号：取消正在进行的AI思考和工具执行""" + self.logger.info("收到客户端中断信号") + # 先发送确认，让前端立即得到反馈 + await self.send_message(websocket, { + "type": "interrupt_ack", + "content": "已收到中断信号，正在停止..." + }) + # 直接同步调用cancel()：cancel是线程安全的（只set Event + close连接 + kill进程），执行极快 + # 无需通过run_in_executor提交到线程池，避免额外调度延迟 + self.ai_manager.cancel() + async def handle_task_message(self, websocket, data): """处理任务相关消息 - 简化版，所有任务都是对话任务""" action = data.get("action", "").strip() @@ -7947,6 +8085,16 @@ class WebSocketHandler: if not content: return + # 防止并发处理用户消息：如果正在处理中，提示用户先中断 + if self._is_processing_message: + await self.send_message(websocket, { + "type": "info", + "content": "⏳ AI正在响应中，请等待完成或发送中断信号（Ctrl+C）" + }) + return + + self._is_processing_message = True + self.logger.info(f"收到用户消息: {content[:50]}...") # 确保AI已初始化 @@ -7988,7 +8136,6 @@ class WebSocketHandler: # 提交到事件循环 asyncio.run_coroutine_threadsafe(send_msg(), loop) - # 让AI思考并回应 - 不等待完整响应，通过回调实时发送 try: self.logger.info("开始AI思考...") @@ -8028,6 +8175,9 @@ class WebSocketHandler: }, ensure_ascii=False)) except: pass + finally: + # 无论成功、失败还是被中断，都清除处理标志 + self._is_processing_message = False # ==================== 对话管理器（修改版：添加重置对话功能） ==================== @@ -8109,6 +8259,17 @@ class EnhancedAIManager: self.logger.info("AI已初始化") return ai + def cancel(self): + """中断当前AI操作（线程安全，可从任意线程调用）""" + if self.ai: + self.ai.cancel() + # 同时确保命令执行器中的进程也被终止 + if self.command_executor and hasattr(self.command_executor, 'cancel'): + try: + self.command_executor.cancel() + except Exception: + pass + def show_welcome_message(self): """显示欢迎信息""" print("\n" + "="*70) @@ -8181,17 +8342,30 @@ class EnhancedAIManager: print(i18n.get('websocket_started', host, port)) async def handle_connection(self, websocket, path): - """处理WebSocket连接""" + """处理WebSocket连接 - 消息接收与处理解耦，确保中断信号能被即时处理""" self.logger.info(f"新连接: {websocket.remote_address}, path: {path}") await self.ws_handler.register(websocket) try: async for message in websocket: - await self.ws_handler.handle_message(websocket, message) + # 【关键修复】用create_task异步处理消息，不await阻塞接收循环 + # 这样当AI在后台生成响应时，interrupt消息仍能被立即接收和处理 + task = asyncio.create_task( + self.ws_handler.handle_message(websocket, message) + ) + self.ws_handler._processing_tasks[id(websocket)] = task + + def _cleanup_task(t, ws_id=id(websocket)): + self.ws_handler._processing_tasks.pop(ws_id, None) + task.add_done_callback(_cleanup_task) except websockets.exceptions.ConnectionClosed as e: self.logger.info(f"连接关闭: {e}") except Exception as e: self.logger.error(f"处理错误: {e}") finally: + # 连接关闭时取消正在进行的处理任务 + task = self.ws_handler._processing_tasks.pop(id(websocket), None) + if task and not task.done(): + task.cancel() await self.ws_handler.unregister(websocket) async def websocket_server(self): diff --git a/confs/README.md b/confs/README.md index e1b232f..7a6a4be 100644 --- a/confs/README.md +++ b/confs/README.md @@ -227,7 +227,7 @@ except RpcError as e: ## 集成关系 -- 由 `core`（`PMC`）通过 `init.txt` 中的 `bash /opt/micro_servers/confs/_run.sh` 拉起并守护（在 db 之后、ai_review 之前启动） +- 由 `core`（`PMC`）通过 `init.txt` 中的 `bash /opt/micro_servers/confs/_run.sh` 拉起并守护（confs 最先启动即可；各业务服务通过 pymsg `retry_on_refused=True` 在拉取配置时自动阻塞等待 confs 就绪，不严格要求启动顺序；confs 日志推送 db 为异步 fail-safe，db 不可用时静默丢弃，不阻塞 confs 启动） - 各内部服务通过 pymsg `RpcClient` 调 `get` 路由按需获取配置，替代本地散落的 ini/json/yaml - 自身日志通过 `log_client.py` 异步推送至 `db:8005` 的 Logs 表（与其他服务一致） @@ -235,21 +235,35 @@ except RpcError as e: | 服务 | 配置文件 | 接入方式 | |------|---------|---------| -| [static](../static/) | [static.ini](configs/static.ini) | 三层 fail-safe：内置默认值 → confs（优先）→ 本地 `config.ini` 回退 | -| [dnfs](../dnfs/) | [dnfs.ini](configs/dnfs.ini) | 三层 fail-safe：内置默认值 → confs（优先）→ 环境变量覆盖 | +| [static](../static/) | [static.ini](configs/static.ini) | confs 强依赖（pymsg 自动阻塞等待 confs 就绪）→ 环境变量覆盖（最高优先级，Docker 兼容） | +| [dnfs](../dnfs/) | [dnfs.ini](configs/dnfs.ini) | confs 强依赖（pymsg 自动阻塞等待 confs 就绪）→ 环境变量覆盖（最高优先级，Docker 兼容） | +| [db](../db/) | [db.ini](configs/db.ini) | confs 强依赖（pymsg 自动阻塞等待 confs 就绪）→ 环境变量覆盖（最高优先级，Docker 兼容） | static 的接入实现见 [static/fsys.py](../static/fsys.py)： -- `_fetch_from_confs()` 通过 `RpcClient` 调 `get` 路由取 `static.ini`，3 秒超时 -- `CONFS_HOST` / `CONFS_PORT` 环境变量定位 confs（默认 `127.0.0.1:8010`） -- `_merge_ini()` 仅覆盖已知的 section/key（白名单合并，不引入未预期字段） -- 配置来源会记录到日志（`logs.info/warning`），便于排查实际用了哪一层 +- 两层加载顺序：confs（static.ini，强依赖）→ 环境变量覆盖（优先级最高） +- pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪（每秒重试，每 5 秒打印等待提示），不会因启动顺序导致进程退出 +- static.ini 缺失必需配置项时仍然报错退出（配置错误，非启动时序问题） +- 配置项涵盖：`[server]` host/port、`[directory]` root、`[ssl]` enabled/cert_file/key_file（ssl 有默认值）、`[cors]` allowed_origins +- confs 连接地址为引导配置：环境变量 `CONFS_HOST`/`CONFS_PORT` 优先，默认 `127.0.0.1:8010` +- 支持的环境变量：`LISTEN_ADDR`/`LISTEN_PORT`/`STATIC_ROOT`/`CONFS_HOST`/`CONFS_PORT` +- 已删除 `--config` 命令行参数和本地 `config.ini` 回退逻辑 dnfs 的接入实现见 [dnfs/dnfs.py](../dnfs/dnfs.py)： -- 三层加载顺序：内置默认值 → confs（dnfs.ini）→ 环境变量覆盖（优先级最高） +- 两层加载顺序：confs（dnfs.ini，强依赖）→ 环境变量覆盖（优先级最高） +- pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪，不严格要求启动顺序 +- 配置项涵盖：`[server]` host/port、`[database]` host/port/timeout、`[ai_review]` url/timeout +- confs 连接地址为引导配置：环境变量 `CONFS_HOST`/`CONFS_PORT` 优先，默认 `127.0.0.1:8010` - 支持的环境变量：`LISTEN_ADDR`/`LISTEN_PORT`/`DB_HOST`/`DB_PORT`/`REVIEW_URL`/`CONFS_HOST`/`CONFS_PORT` -- Docker 环境通过环境变量传入服务名（db/ai_review），覆盖 confs 中的 127.0.0.1 默认值 -其他服务（gateway / waf / ai_review / smtp / db）尚未接入，仍读各自本地配置。 +db 的接入实现见 [db/dbv2.py](../db/dbv2.py)： +- 两层加载顺序：confs（db.ini，强依赖）→ 环境变量覆盖（优先级最高） +- pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪，不严格要求启动顺序 +- 配置项涵盖：`[server]` host/port/workers、`[database]` url、`[logs]` retention_days/max_records/cleanup_interval +- confs 连接地址为引导配置：环境变量 `CONFS_HOST`/`CONFS_PORT` 优先，默认 `127.0.0.1:8010` +- 支持的环境变量：`LISTEN_ADDR`/`LISTEN_PORT`/`WORKERS`/`DATABASE_URL`/`CONFS_HOST`/`CONFS_PORT` +- 独立运维工具 `db_tools.py`（backup/export/import 等）仍直接读 `DATABASE_URL` 环境变量，不依赖 confs，避免运维脚本依赖服务就绪 + +其他服务（gateway / waf / ai_review / smtp）尚未接入，仍读各自本地配置。 ## 项目结构 @@ -260,7 +274,8 @@ confs/ ├── configs/ # 配置文件存放目录（被分发对象） │ ├── example.ini # 示例配置 │ ├── static.ini # static 服务配置（已接入分发） -│ └── dnfs.ini # dnfs 服务配置（已接入分发） +│ ├── dnfs.ini # dnfs 服务配置（已接入分发） +│ └── db.ini # db 服务配置（已接入分发） ├── docs/ │ └── SECURITY_AUDIT_20260801.md # 安全审计报告 ├── _set.sh # 初始化脚本（建 venv + 安装依赖） diff --git a/confs/_run.sh b/confs/_run.sh index f8815d1..32c010d 100755 --- a/confs/_run.sh +++ b/confs/_run.sh @@ -2,5 +2,4 @@ cd "$(dirname "$0")" || exit source ./venv/bin/activate #exec ./smtp.bin -sleep 2 exec python3 confs.py diff --git a/confs/configs/db.ini b/confs/configs/db.ini new file mode 100644 index 0000000..bd2a2d3 --- /dev/null +++ b/confs/configs/db.ini @@ -0,0 +1,16 @@ +[server] +host = 0.0.0.0 +port = 8005 +workers = 4 + +[database] +url = sqlite:///database.db + +[logs] +retention_days = 30 +max_records = 1000000 +cleanup_interval = 3600 + +[confs] +host = 127.0.0.1 +port = 8010 diff --git a/confs/configs/dnfs.ini b/confs/configs/dnfs.ini index 8288444..f327890 100644 --- a/confs/configs/dnfs.ini +++ b/confs/configs/dnfs.ini @@ -11,6 +11,6 @@ timeout = 10 url = http://127.0.0.1:8007/review timeout = 10 -[confs] -host = 127.0.0.1 -port = 8010 +[gateway] +url = http://127.0.0.1:8443/whoami +timeout = 5 diff --git a/gateway/example.ini b/confs/configs/gateway.ini similarity index 84% rename from gateway/example.ini rename to confs/configs/gateway.ini index 495f72f..8f3ead1 100644 --- a/gateway/example.ini +++ b/confs/configs/gateway.ini @@ -23,7 +23,7 @@ max_refresh = 10 # 管理员账号白名单（分号分隔）。登录时命中即在 Token payload 写入 is_admin=true， # 下游服务（dnfs 等）凭该字段豁免所有权校验，无需各自维护名单。 # 修改后需重启 gateway 生效；旧 Token 无此字段，按非管理员处理。 -users = qing@qsont.me +users = qing@qsont.me;shufeng@qsont.me [CORS] # 允许跨域的 Origin 白名单（分号分隔） @@ -41,3 +41,13 @@ path = sqlite3.db # 支持 ws:// 和 wss:// 后端 # 示例：routes = /ws/aibox=ws://127.0.0.1:8765;/ws/other=wss://example.com/ws routes = /ws/aibox=ws://127.0.0.1:8765; + +[Downstream] +# 下游 pymsg 服务地址（其他微服务的 host:port） +# 环境变量可覆盖：DB_HOST/DB_PORT, CAPTCHA_HOST/CAPTCHA_PORT, SMTP_HOST/SMTP_PORT, CONFS_HOST/CONFS_PORT +db_host = 127.0.0.1 +db_port = 8005 +captcha_host = 127.0.0.1 +captcha_port = 8009 +smtp_host = 127.0.0.1 +smtp_port = 8008 diff --git a/confs/log_client.py b/confs/log_client.py index 0dd516f..515f1d6 100644 --- a/confs/log_client.py +++ b/confs/log_client.py @@ -1,13 +1,15 @@ -"""统一日志客户端 — 各服务共用（dnfs / smtp / static / confs 四份副本内容一致，改动请同步）。 +""" +统一日志客户端 — 各服务共用（基于 pymsg.AsyncRpcQueue）。 设计要点： - 通过 pymsg RPC 调用 dbv2 execute 路由，逐条 INSERT 到 Logs 表 - - 异步非阻塞：内存队列 + 后台线程批量推送，不拖慢业务 - - fail-safe：db 服务不可用时静默丢弃，绝不影响业务；后台线程任何异常都不会崩溃 + - 异步非阻塞：内存队列 + 后台线程攒批推送，不拖慢业务 + - fail-safe：db 服务不可用时静默丢弃，绝不影响业务 - 缓冲批量：队列满 50 条或超时 2 秒时自动 flush - - 时间戳：日志入队时即记录 ts，反映事件真实发生时间 + - 时间戳：日志入队时即记录 ts（UTC ISO 格式），反映事件真实发生时间 + - 底层复用 pymsg.AsyncRpcQueue 通用队列基础设施 -用法： +用法（与旧版完全兼容）： from log_client import logs logs.info("dnfs", "节点新增成功 id=42") @@ -19,28 +21,25 @@ import json import os as _os import sys as _sys -import threading -import queue -import time import atexit from datetime import datetime, timezone _sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), "..")) -from pymsg import RpcClient, RpcError # noqa: E402 +from pymsg import RpcClient, RpcError, AsyncRpcQueue # noqa: E402 # ── 配置 ────────────────────────────────────────────────────────────── _DBV2_ADDR = _os.environ.get('DB_HOST', '127.0.0.1') _DBV2_PORT = int(_os.environ.get('DB_PORT', '8005')) -_BATCH_SIZE = 50 # 队列满 50 条时 flush -_FLUSH_INTERVAL = 2.0 # 超时 2 秒强制 flush -_TIMEOUT = 5 # socket 超时（秒） -_QUEUE_MAXSIZE = 10000 # 队列上限，防止后台线程异常时内存无限增长；满则丢弃最旧条目 +_BATCH_SIZE = 50 +_FLUSH_INTERVAL = 2.0 +_TIMEOUT = 5 +_QUEUE_MAXSIZE = 10000 def _safe_json(obj): - """安全序列化 extra 字段，不可序列化对象转为 repr 字符串，保证不抛异常。""" + """安全序列化 extra 字段，不可序列化对象转为 repr 字符串。""" try: return json.dumps(obj, ensure_ascii=False, default=str) except Exception: @@ -48,163 +47,80 @@ def _safe_json(obj): def _now_iso(): - """生成当前 UTC 时间 ISO 字符串（在入队时调用，保证时间戳准确）。""" + """生成当前 UTC 时间 ISO 字符串（入队时调用，保证时间戳准确）。""" return datetime.now(timezone.utc).isoformat() class LogClient: - """异步批量日志推送客户端。""" + """异步批量日志推送客户端（API 与旧版完全兼容）。""" - def __init__(self, service_name="log", - batch_size=_BATCH_SIZE, - flush_interval=_FLUSH_INTERVAL): + def __init__(self, service_name: str = "log", + batch_size: int = _BATCH_SIZE, + flush_interval: float = _FLUSH_INTERVAL): self._service_name = service_name - self._batch_size = batch_size - self._flush_interval = flush_interval - self._queue = queue.Queue(maxsize=_QUEUE_MAXSIZE) - self._stop = threading.Event() - self._flush_signal = threading.Event() - self._flush_done = threading.Event() - self._thread = threading.Thread(target=self._flush_loop, - daemon=True, name='log-client') - self._thread.start() - atexit.register(self._shutdown) + self._client = RpcClient( + _DBV2_ADDR, _DBV2_PORT, + service_name + "-log", + timeout=_TIMEOUT, + ) + self._queue = AsyncRpcQueue( + handler=self._send_batch, + batch_size=batch_size, + flush_interval=flush_interval, + max_queue_size=_QUEUE_MAXSIZE, + on_error=self._on_error, + ) + self._queue.start() + atexit.register(self.close) # ── 公开接口 ──────────────────────────────────────────────────────── - def push(self, service, level, message, trace_id=None, extra=None): + def push(self, service: str, level: str, message: str, + trace_id: str = None, extra=None): """推送一条日志（非阻塞，立即返回）。 队列满时静默丢弃最旧条目再放入新条目，保证不阻塞业务线程、不导致 OOM。 """ entry = { - 'ts': _now_iso(), # 入队时即生成时间戳，反映真实事件时间 + 'ts': _now_iso(), 'service': service, 'level': level, 'message': message, 'trace_id': trace_id, 'extra': extra, } - try: - self._queue.put_nowait(entry) - except queue.Full: - # 队列满：丢弃最旧的一条，再放入新条目（drop-oldest 策略） - try: - self._queue.get_nowait() - except queue.Empty: - pass - try: - self._queue.put_nowait(entry) - except queue.Full: - pass # 极端竞争下仍失败则静默丢弃 + self._queue.put(entry) - def debug(self, service, message, **kw): + def debug(self, service: str, message: str, **kw): self.push(service, 'DEBUG', message, **kw) - def info(self, service, message, **kw): + def info(self, service: str, message: str, **kw): self.push(service, 'INFO', message, **kw) - def warning(self, service, message, **kw): + def warning(self, service: str, message: str, **kw): self.push(service, 'WARNING', message, **kw) - def error(self, service, message, **kw): + def warn(self, service: str, message: str, **kw): + self.push(service, 'WARNING', message, **kw) + + def error(self, service: str, message: str, **kw): self.push(service, 'ERROR', message, **kw) - def critical(self, service, message, **kw): + def critical(self, service: str, message: str, **kw): self.push(service, 'CRITICAL', message, **kw) def flush(self): - """手动 flush 队列中剩余的日志（含后台线程持有的局部 batch）。""" - self._flush_done.clear() - self._flush_signal.set() # 通知后台线程立即 flush 其局部 batch - # 等待后台线程完成 flush（最多等待 flush_interval + 2 秒余量） - self._flush_done.wait(timeout=self._flush_interval + 2.0) - # 主线程排空队列中剩余条目 - self._drain_queue_to_db() + """手动 flush 队列中剩余的日志。""" + self._queue.flush() - def _drain_queue_to_db(self): - """排空主线程可见的队列中剩余条目，直接发送到 db。""" - batch = [] - while True: - try: - batch.append(self._queue.get_nowait()) - if len(batch) >= self._batch_size: - self._send(batch) - batch = [] - except queue.Empty: - break - if batch: - self._send(batch) + def close(self): + """关闭客户端，flush 剩余日志。""" + self._queue.close() # ── 内部实现 ──────────────────────────────────────────────────────── - def _flush_loop(self): - """后台线程：积攒到 batch_size 或超时 flush_interval 时发送。 - - 外层捕获所有异常，保证后台线程在任何情况下（网络故障、序列化错误等） - 都不会死亡，避免日志队列无限增长导致 OOM。 - """ - batch = [] - last_flush = time.monotonic() - - while not self._stop.is_set(): - try: - timeout = self._flush_interval - (time.monotonic() - last_flush) - chk_timeout = max(0.1, min(timeout, 0.5)) - try: - item = self._queue.get(timeout=chk_timeout) - batch.append(item) - while len(batch) < self._batch_size: - try: - batch.append(self._queue.get_nowait()) - except queue.Empty: - break - except queue.Empty: - pass - - should_flush = ( - len(batch) >= self._batch_size or - self._flush_signal.is_set() or - (batch and time.monotonic() - last_flush >= self._flush_interval) - ) - if should_flush and batch: - try: - self._send(batch) - except Exception as e: - # _send 内部已有细粒度异常处理，此处是最终安全网 - print(f"[log_client] _send 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - batch = [] - last_flush = time.monotonic() - if self._flush_signal.is_set(): - self._flush_signal.clear() - self._flush_done.set() - except Exception as e: - # 最外层安全网：任何意外异常都不应杀死日志线程 - print(f"[log_client] _flush_loop 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - time.sleep(0.5) # 防止异常风暴导致 CPU 满载 - - # 退出前 flush 剩余 - if batch: - try: - self._send(batch) - except Exception as e: - print(f"[log_client] 退出时 flush 异常: {e}", file=_sys.stderr) - self._flush_done.set() - - def _send(self, batch): - """通过 pymsg RPC（execute 路由）批量发送到 dbv2 Logs 表。 - - 整个 batch 复用一个 TCP 连接；网络错误时放弃本批剩余条目（fail-safe）， - 不抛异常给调用方。 - """ - try: - client = RpcClient(_DBV2_ADDR, _DBV2_PORT, self._service_name + "-log") - except Exception as e: - print(f"[log_client] 创建 RpcClient 失败: {e}", file=_sys.stderr) - return - + def _send_batch(self, batch: list): + """后台线程回调：逐条 INSERT 到 db（与旧版行为一致：网络错误放弃本批剩余）。""" for entry in batch: extra_val = entry.get("extra") try: @@ -224,21 +140,21 @@ class LogClient: }, } try: - client.call("execute", payload, timeout=_TIMEOUT) + self._client.call("execute", payload, timeout=_TIMEOUT) except RpcError as e: - # 业务错误（如表结构不符）：打印后继续下一条 print(f"[log_client] INSERT 失败: {e}", file=_sys.stderr) except Exception as e: - # 网络错误：db 不可用，放弃本批剩余（fail-safe） - print(f"[log_client] 发送失败: {e}", file=_sys.stderr) + print(f"[log_client] 发送失败（放弃本批剩余）: {e}", file=_sys.stderr) break - def _shutdown(self): - """进程退出时 flush 剩余日志。""" - self._stop.set() - self.flush() + def _on_error(self, e: Exception, batch: list): + """异常安全网：handler 外的异常也不会崩溃线程。""" + print(f"[log_client] _send_batch 异常（已捕获，线程继续运行）: {e}", + file=_sys.stderr) # ── 全局单例 ────────────────────────────────────────────────────────── +# 各服务的 log_client.py 副本需将 service_name 改为自己的服务名。 +# 此文件（confs/log_client.py）的 service_name 为 "confs"。 -logs = LogClient() +logs = LogClient(service_name="confs") diff --git a/db/README.md b/db/README.md index 67b6baa..4cd8898 100644 --- a/db/README.md +++ b/db/README.md @@ -25,10 +25,64 @@ ./_run.sh # venv 中执行 python3 dbv2.py ``` -服务监听 `tcp://127.0.0.1:8005`。生产环境由 core（Nidus）按 init.txt 统一拉起，守护靠 systemd。 +服务监听 `tcp://0.0.0.0:8005`（默认）。生产环境由 core（Nidus）按 init.txt 统一拉起，守护靠 systemd。 **注意**：`execute` 路由依赖表已存在（通过反射获取结构）。建表见 `create_sqlite3.sql`（Users / Notes；Logs 表由服务启动时自建）。 +## 配置管理 + +db 已接入 [confs](../confs/) 统一配置服务，confs 是 db 启动的**强依赖**： + +1. **confs 统一配置服务** — 必须成功从 confs 拉取 `db.ini`，否则 db 启动失败直接退出（exit code 1） +2. **环境变量覆盖**（最高优先级）— Docker/部署环境通过环境变量覆盖 confs 中的值 + +### confs 配置文件（db.ini） + +配置文件托管于 [confs/configs/db.ini](../confs/configs/db.ini)： + +```ini +[server] +host = 0.0.0.0 +port = 8005 +workers = 4 + +[database] +url = sqlite:///database.db + +[logs] +retention_days = 30 +max_records = 1000000 +cleanup_interval = 3600 +``` + +### 引导配置（confs 连接地址） + +confs 自身的连接地址不通过 confs 获取（避免循环依赖），通过以下方式确定（优先级从高到低）： +- 环境变量 `CONFS_HOST` / `CONFS_PORT` +- 默认值 `127.0.0.1:8010` + +### 环境变量（最高优先级，覆盖 confs 配置值） + +| 环境变量 | 说明 | 对应 confs 配置项 | +|---------|------|------------------| +| `LISTEN_ADDR` | 监听地址 | `[server] host` | +| `LISTEN_PORT` | 监听端口 | `[server] port` | +| `WORKERS` | 回调并发线程数 | `[server] workers` | +| `DATABASE_URL` | 数据库连接 URL | `[database] url` | +| `CONFS_HOST` | confs 服务地址（引导配置） | — | +| `CONFS_PORT` | confs 服务端口（引导配置） | — | + +### 启动顺序 + +db 通过 pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪，**不严格要求启动顺序**： +- confs 先启动 → db 立即拉取配置，正常启动 +- db 先启动 → pymsg 自动重试（每秒一次，每 5 秒打印等待提示），confs 启动后自动连接成功 +- 配置完整性校验：若 db.ini 中缺少必需的 section/key 且无对应环境变量覆盖，仍然报错退出 +- 启动成功时控制台打印：`配置来源: confs (db.ini @ host:port)` + +> 独立运维工具 `db_tools.py`（backup/export/import/vacuum 等）仍直接读 `DATABASE_URL` 环境变量，不依赖 confs 服务就绪。 +> Docker 环境通过 `docker-entrypoint.sh` 传入环境变量（见 [Dockerfile](Dockerfile)），需确保 `CONFS_HOST`/`CONFS_PORT` 指向 confs 服务地址。 + ## 协议与 API ### 传输协议（pymsg） @@ -232,10 +286,10 @@ python3 _view_db.sqlite3.py -s "SELECT service, COUNT(*) FROM Logs GROUP BY serv ## 切换数据库 -只需修改环境变量 `DATABASE_URL`，并安装对应驱动： +推荐方式：修改 confs 的 [db.ini](../confs/configs/db.ini) 中 `[database] url`，然后通过 confs `reload` 路由重载（或重启 db）。也可通过环境变量 `DATABASE_URL` 覆盖（Docker 部署时常用）： -- SQLite（默认）：`sqlite:///database.db` -- PostgreSQL：`postgresql://user:pass@localhost/dbname`（需 `pip install psycopg2`） +- SQLite（默认）：`sqlite:///database.db`（相对路径）或 `sqlite:////app/data/database.db`（绝对路径，Docker 中使用） +- PostgreSQL：`postgresql://user:pass@localhost/dbname`（需 `pip install psycopg2-binary`） - MySQL：`mysql+pymysql://user:pass@localhost/dbname`（需 `pip install pymysql`） SQLAlchemy 会自动生成适配各数据库的 SQL 方言，无需修改接口调用。 diff --git a/db/dbv2.py b/db/dbv2.py index 771b206..efd124f 100644 --- a/db/dbv2.py +++ b/db/dbv2.py @@ -27,6 +27,7 @@ dbv2 — 基于 pymsg 协议（MsgServer）的数据库服务。 响应中的日期时间一律序列化为 ISO 8601（T 分隔）。 """ +import configparser import json import os import threading @@ -47,9 +48,91 @@ import sys as _sys _sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pymsg import RpcServer # noqa: E402 -# ── 数据库配置 ────────────────────────────────────────────────────────── +# ── 配置加载（两层：confs 强依赖 → 环境变量覆盖）────────────────────── +# confs 是 db 启动的强依赖：必须成功从 confs 拉取 db.ini，否则直接退出。 +# 环境变量优先级最高（Docker 兼容），可覆盖 confs 中的配置值。 +# confs 自身的连接地址（host/port）通过引导配置确定：环境变量 CONFS_HOST/CONFS_PORT +# 优先，缺省使用 127.0.0.1:8010（与其他服务一致的约定）。 -DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///database.db") +# 引导配置：confs 连接地址（不依赖 confs 本身，可通过环境变量覆盖） +_confs_host = os.environ.get('CONFS_HOST', '127.0.0.1') +_confs_port = int(os.environ.get('CONFS_PORT', '8010')) + +# 白名单结构：_merge_ini 仅覆盖此处列出的 section/key，防止意外字段 +_db_config = { + 'server': {'host': '', 'port': '', 'workers': ''}, + 'database': {'url': ''}, + 'logs': { + 'retention_days': '', + 'max_records': '', + 'cleanup_interval': '', + }, +} + + +def _merge_ini(text): + """把 ini 文本合并进 _db_config（仅覆盖已知的 section/key）。""" + ini = configparser.ConfigParser(interpolation=None) + ini.read_string(text) + for section in _db_config: + if ini.has_section(section): + for key, value in ini[section].items(): + if key in _db_config[section]: + _db_config[section][key] = value + + +def _fetch_from_confs(): + """从 confs 统一配置服务拉取 db.ini 全文。 + + 使用 retry_on_refused=True：confs 未就绪时 pymsg 自动阻塞重试（每秒一次， + 每 5 秒打印一次等待提示），直到 confs 启动后自动连接成功。 + RpcError（如配置文件不存在）仍直接报错退出——这是配置错误而非启动时序问题。 + """ + from pymsg import RpcClient, RpcError # noqa: E402 + client = RpcClient(_confs_host, _confs_port, 'db', timeout=5, + retry_on_refused=True) + try: + data = client.call('get', {'name': 'db.ini'}) + except RpcError as e: + print(f'[dbv2] 致命错误: 从 confs 获取 db.ini 失败: {e}', + file=_sys.stderr) + _sys.exit(1) + return data.get('content') + + +_confs_text = _fetch_from_confs() +_merge_ini(_confs_text) +_config_source = 'confs (db.ini)' +print(f'[dbv2] 配置来源: confs 统一配置服务 (db.ini @ {_confs_host}:{_confs_port})') + +# 环境变量覆盖（最高优先级，保持 Docker 兼容） +if os.environ.get('LISTEN_ADDR'): + _db_config['server']['host'] = os.environ['LISTEN_ADDR'] +if os.environ.get('LISTEN_PORT'): + _db_config['server']['port'] = os.environ['LISTEN_PORT'] +if os.environ.get('WORKERS'): + _db_config['server']['workers'] = os.environ['WORKERS'] +if os.environ.get('DATABASE_URL'): + _db_config['database']['url'] = os.environ['DATABASE_URL'] + +# 配置完整性校验 +_required = { + 'server': ['host', 'port', 'workers'], + 'database': ['url'], + 'logs': ['retention_days', 'max_records', 'cleanup_interval'], +} +for section, keys in _required.items(): + for key in keys: + if not _db_config[section][key]: + print(f'[dbv2] 致命错误: 配置项 [{section}] {key} 缺失（db.ini 中未配置且无环境变量覆盖）', + file=_sys.stderr) + _sys.exit(1) + +# 解析为最终常量 +DATABASE_URL = _db_config['database']['url'] +DEFAULT_RETENTION_DAYS = int(_db_config['logs']['retention_days']) +DEFAULT_MAX_RECORDS = int(_db_config['logs']['max_records']) +DEFAULT_CLEANUP_INTERVAL = int(_db_config['logs']['cleanup_interval']) # ── SQLite 连接配置（针对 Docker/多线程环境优化）────────────────────── @@ -76,10 +159,6 @@ def _configure_sqlite(dbapi_connection, connection_record): MAX_MESSAGE_LEN = 8192 # message 上限 8KB MAX_EXTRA_LEN = 4096 # extra 上限 4KB -DEFAULT_RETENTION_DAYS = 30 -DEFAULT_MAX_RECORDS = 1_000_000 -DEFAULT_CLEANUP_INTERVAL = 3600 # 秒 - #: where/values 中的哨兵值：服务端当前 UTC 时间 NOW_SENTINEL = "$now" @@ -456,17 +535,19 @@ class DBServer: if __name__ == "__main__": import argparse - default_addr = os.environ.get("LISTEN_ADDR", "0.0.0.0") - default_port = int(os.environ.get("LISTEN_PORT", "8005")) + default_addr = _db_config['server']['host'] + default_port = int(_db_config['server']['port']) + default_workers = int(_db_config['server']['workers']) parser = argparse.ArgumentParser(description="dbv2 — pymsg 数据库服务") parser.add_argument("--addr", default=default_addr, help="监听地址") parser.add_argument("--port", type=int, default=default_port, help="监听端口") - parser.add_argument("--workers", type=int, default=4, + parser.add_argument("--workers", type=int, default=default_workers, help="回调并发线程数（0=同步串行模式）") args = parser.parse_args() server = DBServer(args.addr, args.port, workers=args.workers) + print(f'[dbv2] 配置来源: {_config_source}') try: server.start() except KeyboardInterrupt: diff --git a/dnfs/README.md b/dnfs/README.md index 1413830..39afce2 100644 --- a/dnfs/README.md +++ b/dnfs/README.md @@ -14,13 +14,16 @@ Dendrachy Network File-System，基于父子关系（pid）组织的树状节点 ▼ dnfs :8006 (Flask + Waitress) │ - ├──► db :8005 数据持久化（SQL-over-HTTP，/execute） - └──► ai_review :8007 内容审核（仅 add_note / update_note 时调用） + ├──► confs :8010 启动时拉取配置（dnfs.ini，强依赖，retry_on_refused 等待就绪） + ├──► db :8005 数据持久化（pymsg TCP RPC，execute 路由） + └──► ai_review :8007 内容审核（HTTP POST /review，仅 add_note/update_note 时调用） ``` -- 不直连数据库，所有数据操作通过 HTTP 调用 `db` 服务（`http://127.0.0.1:8005/execute`）。 -- 新增/更新节点前调用 `ai_review` 服务（`http://127.0.0.1:8007/review`）进行内容审核，审核不通过则拒绝写入。 -- 通常部署在 gateway 之后（`Proxy.opened` 白名单中），由 gateway 完成 Token 验签后转发，dnfs 仅做所有权校验。 +- 启动时通过 pymsg 从 confs 拉取 `dnfs.ini` 配置，拉取失败直接退出（强依赖）；使用 `retry_on_refused=True` 自动等待 confs 就绪。 +- 不直连数据库，所有数据操作通过 pymsg TCP RPC 调用 `db` 服务（`_route: "execute"`），规避 SQL 注入风险。 +- 新增/更新节点前调用 `ai_review` 服务进行内容审核（HTTP POST `{content: "..."}`），审核不通过则拒绝写入。 +- 日志通过 pymsg `AsyncRpcQueue` 异步批量写入 db 的 `Logs` 表，fail-safe 不阻塞业务。 +- 通常部署在 gateway/WAF 之后（`Proxy.opened` 白名单中），由 gateway 完成 Token RSA 验签后转发。 ## 数据模型 @@ -42,7 +45,7 @@ Dendrachy Network File-System，基于父子关系（pid）组织的树状节点 Authorization: Bearer _ ``` -- Token 由 gateway 在 `/login` 签发，dnfs 通过 [dnfs.py:163](dnfs.py#L163) `decode_token` 仅解 Base64 取 payload，**不验签**（验签由 gateway 完成）。 +- Token 由 gateway 在 `/login` 签发，dnfs 通过 [dnfs.py:304](dnfs.py#L304) `decode_token` 仅解 Base64 取 payload，**当前代码不验签**（代码中标注 `# FIXME: 验证数字签名`，依赖部署上经 gateway 转发的网络隔离）。 - payload 中取 `user` 字段作为操作者，写入 `owner`。 - `delete_note` 与 `update_note` 校验所有权： - 节点 `owner` 必须等于当前 `user`，否则 `403`； @@ -208,34 +211,42 @@ curl -X POST http://localhost:8006/update_note \ ## 安全特性 -- **安全响应头**（[security_middleware.py](security_middleware.py)）：`X-Frame-Options: DENY`、`X-Content-Type-Options: nosniff`、`Referrer-Policy`、`Permissions-Policy`、CSP（`default-src 'self'`、`frame-ancestors 'none'`），并移除 `Server` 头。 +- **安全响应头**（[security_middleware.py](security_middleware.py)）：`X-Frame-Options: SAMEORIGIN`、`X-XSS-Protection`、`X-Content-Type-Options: nosniff`、`Referrer-Policy: strict-origin-when-cross-origin`、`Permissions-Policy`（禁用摄像头/麦克风/地理位置）、CSP（`default-src 'self'` 等），并移除 `Server` 头。 - **Token 鉴权**：写操作强制校验，payload 取 `user` 字段。 - **所有权隔离**：普通用户只能改/删自己的节点。 -- **内容审核**：写入与更新前调用 `ai_review`，色情/违禁/赌博/诈骗/暴力等内容会被拒绝。 +- **特殊节点管控**：link/html/image/file 类型有格式校验与大小限制（URL 前缀、文件名长度、内容大小上限）。 +- **内容审核**：写入与更新前调用 `ai_review`，非法内容会被拒绝（fail-closed 设计，见已知问题）。 +- **异步日志**：通过 pymsg `AsyncRpcQueue` 异步批量写日志，fail-safe 不阻塞业务线程。 +- **非 root 容器化运行**：Dockerfile 中使用 appuser 非特权用户运行服务。 ## 配置 -dnfs 已接入 confs 统一配置服务，采用**三层 fail-safe** 加载模式（与 static 服务一致）： +dnfs 已接入 confs 统一配置服务，confs 是 dnfs 启动的**强依赖**： -**加载顺序**：内置默认值 → confs 统一配置服务（dnfs.ini，优先）→ 环境变量覆盖（最高优先级） +1. **confs 统一配置服务** — 必须成功从 confs 拉取 `dnfs.ini`，否则 dnfs 启动失败直接退出（exit code 1） +2. **环境变量覆盖**（最高优先级）— Docker/部署环境通过环境变量覆盖 confs 中的值 ### 配置文件 集中托管在 [confs/configs/dnfs.ini](../confs/configs/dnfs.ini)，由 confs 服务分发： -| Section | Key | 默认值 | 说明 | -|---------|-----|--------|------| -| `[server]` | `host` | `127.0.0.1` | 监听地址 | -| `[server]` | `port` | `8006` | 监听端口 | -| `[database]` | `host` | `127.0.0.1` | db 服务地址 | -| `[database]` | `port` | `8005` | db 服务端口 | -| `[database]` | `timeout` | `10` | RPC 调用超时（秒） | -| `[ai_review]` | `url` | `http://127.0.0.1:8007/review` | AI 审核接口地址 | -| `[ai_review]` | `timeout` | `10` | AI 审核请求超时（秒） | -| `[confs]` | `host` | `127.0.0.1` | confs 服务地址 | -| `[confs]` | `port` | `8010` | confs 服务端口 | +| Section | Key | 说明 | +|---------|-----|------| +| `[server]` | `host` | 监听地址 | +| `[server]` | `port` | 监听端口 | +| `[database]` | `host` | db 服务地址 | +| `[database]` | `port` | db 服务端口 | +| `[database]` | `timeout` | RPC 调用超时（秒） | +| `[ai_review]` | `url` | AI 审核接口地址 | +| `[ai_review]` | `timeout` | AI 审核请求超时（秒） | -### 环境变量（Docker 环境必需，优先级最高） +### 引导配置（confs 连接地址） + +confs 自身的连接地址不通过 confs 获取（避免循环依赖），通过以下方式确定（优先级从高到低）： +- 环境变量 `CONFS_HOST` / `CONFS_PORT` +- 默认值 `127.0.0.1:8010` + +### 环境变量（最高优先级，覆盖 confs 配置值） 环境变量覆盖配置文件值，容器化部署时通过 docker-compose.yml 传入服务名： @@ -246,24 +257,32 @@ dnfs 已接入 confs 统一配置服务，采用**三层 fail-safe** 加载模 | `DB_HOST` | `db` | db 服务主机名（docker-compose 服务名） | | `DB_PORT` | `8005` | db 服务端口 | | `REVIEW_URL` | `http://ai_review:8007/review` | ai_review 服务地址 | -| `CONFS_HOST` | - | 可选，confs 容器化后使用 | -| `CONFS_PORT` | - | 可选，confs 容器化后使用 | +| `CONFS_HOST` | `confs` | confs 服务主机名（引导配置） | +| `CONFS_PORT` | `8010` | confs 服务端口（引导配置） | -配置来源会记录到日志（`logs.info/warning`），便于排查实际用了哪一层。本地裸机运行时 confs 不可用会自动回退到内置默认值 + 环境变量。 +### 启动顺序 + +dnfs 通过 pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪，**拉取配置阶段不严格要求启动顺序**： +- confs 先启动 → dnfs 立即拉取配置 +- dnfs 先启动 → pymsg 自动重试（每秒一次，每 5 秒打印等待提示），confs 启动后自动连接成功 +- 配置完整性校验：若 dnfs.ini 中缺少必需的 section/key 且无对应环境变量覆盖，仍然报错退出 +- 启动成功时控制台打印：`配置来源: confs (dnfs.ini @ host:port)` + +> 注意：dnfs 对 db/ai_review 的 RPC 调用是运行时按需连接（首次请求时才建立 TCP），db/ai_review 未就绪时相关 API 返回 502 但进程不退出；待依赖服务启动后自动恢复。 ## 前端 -三文件架构： +前端页面由 `static` 服务承载，不在 dnfs 模块内： ``` -templates/index.html 页面结构（Font Awesome 图标） -static/css/style.css 样式 -static/js/script.js 交互逻辑 +static/public/dnfs.html 页面结构 +static/public/css/ 样式（index.css / aibox.css 等） +static/public/js/dnfs.js 交互逻辑 ``` 功能：节点树浏览、新增/编辑/删除节点、文件 Base64 上传与下载、节点迁移（移动到其他父节点下）。 -> 注：首页路由 `@app.route('/')` 当前已注释，前端页面实际由 `static` 服务的 `dnfs.html` 承载。 +> 注：首页路由 `@app.route('/')` 和 `render_template` 相关代码已注释，dnfs 仅提供 JSON API，不直接提供 HTML 页面。 ## 依赖 @@ -271,12 +290,12 @@ static/js/script.js 交互逻辑 flask flask-cors waitress -nuitka # 可选，编译为独立二进制 +pymsg # 内部 RPC 通信库（../pymsg/） ``` -> `requirements.txt` 仍列有 `flask_sqlalchemy`，但代码已不使用（SQLAlchemy 直连代码已注释），属遗留依赖。 +运行时需要 `confs`（:8010，启动强依赖）、`db`（:8005）和 `ai_review`（:8007）服务可达；对外访问需经 gateway 转发并验签。 -运行时需要 `db`（:8005）和 `ai_review`（:8007）服务已启动；对外访问需经 gateway 转发并验签。 +> `requirements.txt` 中曾列有 `flask_sqlalchemy`，Dockerfile 中已添加 `pip uninstall -y flask_sqlalchemy` 清理遗留依赖；代码中的 SQLAlchemy 直连逻辑已全部注释，通过 pymsg RPC 调用 db 服务。 ## 运行 @@ -287,8 +306,46 @@ bash _run.sh # 启动服务（waitress） ## 已知问题与风险 -- **`decode_token` 未验签**（[dnfs.py:167](dnfs.py#L167)）：仅解 Base64 取 payload，存在 `# FIXME: 验证数字签名` 标记。当前安全性依赖"必须经 gateway 转发"这一部署假设；若 dnfs 被直接访问（绕过 gateway），任意伪造的 Token 都能通过。建议补 RSA 验签或在部署上确保仅 gateway 可达。 -- **`review_content` 返回类型不一致**（[dnfs.py:73-119](dnfs.py#L73)）：正常路径返回 `bool`（[L105](dnfs.py#L105)），HTTPError 路径返回 `dict`（[L113](dnfs.py#L113)）。调用处用 `True != review` 判断，HTTPError 时 `dict != True` 恒为真，会一律判为审核失败并吞掉具体错误信息。建议统一返回类型或在调用处 catch 异常。 -- **CORS 全开**：`CORS(app)` 未配置白名单，允许任意 Origin 跨域访问。考虑到写操作需 Token，风险可控，但仍建议按 gateway 的 `[CORS] allowed_origins` 方式收紧。 -- **`call_db_service` 异常未捕获**：调用处（[L208](dnfs.py#L208)、[L261](dnfs.py#L261)、[L318](dnfs.py#L318)、[L347](dnfs.py#L347)、[L440](dnfs.py#L440)）直接取 `result['success']` / `notes['data']`，若 db 服务返回非预期结构或网络异常会抛 500。 -- **`get_notes` 不鉴权**：任何能访问 dnfs 的人都能枚举所有节点内容。若内容含敏感 Base64 文件，需在 gateway 层做路径级鉴权。 +审计日期: 2026-08-01 + +### 🔴 P0 — 严重问题（需立即修复） + +1. **Token 无 RSA 签名验证，可伪造任意用户身份（含管理员）** — [dnfs.py:304-311](dnfs.py#L304-L311) + - `decode_token()` 直接 base64 解码 payload，完全跳过 RSA 签名验证（代码中有 `# FIXME: 验证数字签名` 注释） + - 若 dnfs 被直接访问（绕过 gateway），攻击者可构造 `{"user": "任意邮箱", "is_admin": true}` 的 payload，base64 编码后拼接下划线+任意字符串即可伪造管理员身份，绕过所有权限校验 + - **参考实现**: gateway `verifyWsAdminToken` 展示了正确流程：分割 `_` → base64 解码 → RSA 公钥验签 → 检查过期时间 + - **修复建议**: 实现 RSA 签名验证（持有 gateway 公钥）+ Token 过期校验（`iat`/`created_at` + `keep_time`） + +2. **`review_content()` 错误处理断裂，fail-closed 失效** — [dnfs.py:149-195](dnfs.py#L149-L195) + - 成功路径返回 `bool`，但 HTTPError/URLError/其他异常直接 `raise e`，没有返回 `False` 实现 fail-closed + - 内层 `except json.JSONDecodeError as e` 变量名遮蔽外层 `HTTPError as e` + - ai_review 不可用时异常向上传播导致 500，而非拒绝内容（违反 fail-closed 设计意图） + - **修复建议**: 所有异常路径返回 `False` 或抛 DnfsExp 拒绝写入 + +3. **`delete_note` 和 `update_note` 缺少 `@_handle_exp` 装饰器** — [dnfs.py:428](dnfs.py#L428), [dnfs.py:531](dnfs.py#L531) + - `add_note`/`get_notes` 使用了 `@_handle_exp` 统一异常处理，但 `delete_note`/`update_note` 没有 + - 抛出 `DnfsExp` 或其他异常时返回 Flask 默认 500 HTML 错误页，与其他接口 JSON 响应不一致 + +4. **有子节点的父节点可能被删除（树结构完整性保护缺陷）** + - **数据库层缺失外键约束**: 虽然 dbv2 启用了 `PRAGMA foreign_keys=ON`，但 Notes 表建表 SQL（[db/create_sqlite3.sql:13-19](../db/create_sqlite3.sql#L13-L19)）中 `pid` 列只是普通 INTEGER，没有定义 `FOREIGN KEY (pid) REFERENCES Notes(id) ON DELETE RESTRICT` + - **应用层 TOCTOU 竞态**: 子节点查询（SELECT）和删除（DELETE）是两个独立 RPC 调用，无事务保护，检查与删除之间可插入子节点导致父节点被删后子节点成为孤儿 + - **错误处理缺陷**: `_call_dbv2` 返回 `success=False` 时响应中无 `data` 键，直接访问 `notes['data']` 会抛 KeyError 而非正确处理 + - **修复建议**: (a) 添加数据库外键约束; (b) 应用层检查返回值 `success` 状态; (c) 使用数据库事务将检查+删除原子化 + +### 🟠 P1 — 高危问题 + +5. **Token 无过期校验** — payload 中包含 `iat`/`created_at`/`keep_time` 字段，但 `decode_token()` 完全不检查，泄露的 Token 可永久使用 +6. **请求体大小无限制** — Waitress 未配置 `max_request_body_size`，可能被大请求体 DoS；建议设置 10MB 上限与 WAF 对齐 +7. **`log_client.py` 配置与主模块不一致** — [log_client.py:32-33](log_client.py#L32-L33) 直接读 `DB_HOST`/`DB_PORT` 环境变量，不从 confs 拉取配置，可能连错地址 +8. **`update_note` 修改 pid 时未验证目标父节点存在** — `add_note` 会检查 pid≠0 时父节点是否存在，但 `update_note` 直接更新不检查，可将节点移动到不存在的父节点下 +9. **db RPC 客户端未启用启动等待** — `_rpc = RpcClient(...)` 未设置 `retry_on_refused=True`，db 未就绪时首次请求失败（虽然 confs 连接启用了重试） +10. **`delete_note`/`update_note` 中 `decode_token` 无 try-except** — `get_notes` 中包裹了异常处理，但这两个函数没有，base64 解码/JSON 解析错误直接抛 500 +11. **`note_id` 缺少强制类型转换** — `note_id: int = data.get('id')` 只是类型提示，不会实际转换；传入字符串 `"5"` 依赖 SQLite 类型亲和性隐式匹配 + +### 🟡 P2 — 中危问题 + +12. **错误响应泄露内部信息** — 多处错误消息拼接 `request.remote_addr` 和用户名返回给客户端（如"未授权访问，从xxx"），应仅记录到服务端日志 +13. **CORS 配置过于宽松** — `CORS(app)` 默认允许所有来源（`*`），建议限制为实际前端域名 +14. **`_call_dbv2` 返回值访问前未统一检查 success** — 多处直接访问 `notes['data']` 而不先确认 `success==True` +15. **`pid` 参数判断逻辑不严谨** — `if not pid: pid = 0` 会将空字符串/None 等非法输入默默转为 0，掩盖参数错误 +16. **遗留注释代码** — 文件中有大量注释掉的旧版 SQLAlchemy 代码和已废弃接口，影响可读性 diff --git a/dnfs/dnfs.py b/dnfs/dnfs.py index ff78c71..636d59b 100755 --- a/dnfs/dnfs.py +++ b/dnfs/dnfs.py @@ -19,10 +19,10 @@ from waitress import serve import os import sys import json -import base64 import configparser import urllib.request import urllib.error +import functools from typing import Dict, Any from security_middleware import apply_security_headers from log_client import logs @@ -33,15 +33,22 @@ from log_client import logs sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from pymsg import RpcClient, RpcError # noqa: E402 -# ==================== 配置加载（三层 fail-safe） ==================== -# 加载顺序：内置默认值 → confs 统一配置服务（dnfs.ini） → 环境变量覆盖 -# 环境变量优先级最高（Docker 环境使用服务名 db/ai_review 而非 127.0.0.1） +# ==================== 配置加载（两层：confs 强依赖 → 环境变量覆盖） ==================== +# confs 是 dnfs 启动的强依赖：必须成功从 confs 拉取 dnfs.ini，否则直接退出。 +# 环境变量优先级最高（Docker 环境使用服务名 db/ai_review 而非 127.0.0.1）。 +# confs 自身的连接地址通过引导配置确定：环境变量 CONFS_HOST/CONFS_PORT 优先， +# 缺省使用 127.0.0.1:8010（与其他服务一致的约定）。 +# 引导配置：confs 连接地址（不依赖 confs 本身） +_confs_host = os.environ.get('CONFS_HOST', '127.0.0.1') +_confs_port = int(os.environ.get('CONFS_PORT', '8010')) + +# 白名单结构：_merge_ini 仅覆盖此处列出的 section/key，防止意外字段 _dnfs_config = { - 'server': {'host': '127.0.0.1', 'port': '8006'}, - 'database': {'host': '127.0.0.1', 'port': '8005', 'timeout': '10'}, - 'ai_review': {'url': 'http://127.0.0.1:8007/review', 'timeout': '10'}, - 'confs': {'host': '127.0.0.1', 'port': '8010'}, + 'server': {'host': '', 'port': ''}, + 'database': {'host': '', 'port': '', 'timeout': ''}, + 'ai_review': {'url': '', 'timeout': ''}, + 'gateway': {'url': '', 'timeout': ''}, } @@ -57,23 +64,24 @@ def _merge_ini(text): def _fetch_from_confs(): - """从 confs 统一配置服务拉取 dnfs.ini 全文，失败返回 None（fail-safe）。""" + """从 confs 统一配置服务拉取 dnfs.ini 全文。 + + 使用 retry_on_refused=True：confs 未就绪时 pymsg 自动阻塞重试（每秒一次， + 每 5 秒打印一次等待提示），直到 confs 启动后自动连接成功。 + RpcError（如配置文件不存在）仍直接报错退出——这是配置错误而非启动时序问题。 + """ try: - host = os.environ.get('CONFS_HOST', _dnfs_config['confs']['host']) - port = int(os.environ.get('CONFS_PORT', _dnfs_config['confs']['port'])) - data = RpcClient(host, port, 'dnfs', timeout=3).call( - 'get', {'name': 'dnfs.ini'}) - return data.get('content') - except Exception: - return None + data = RpcClient(_confs_host, _confs_port, 'dnfs', timeout=5, + retry_on_refused=True).call('get', {'name': 'dnfs.ini'}) + except RpcError as e: + print(f'[dnfs] 致命错误: 从 confs 获取 dnfs.ini 失败: {e}', + file=sys.stderr) + sys.exit(1) + return data.get('content') _confs_text = _fetch_from_confs() -if _confs_text is not None: - _merge_ini(_confs_text) - logs.info("dnfs", "配置来源: confs 统一配置服务 (dnfs.ini)") -else: - logs.warning("dnfs", "confs 不可用，使用内置默认值 + 环境变量") +_merge_ini(_confs_text) # 环境变量覆盖（优先级最高，Docker 环境需要） if os.environ.get('LISTEN_ADDR'): @@ -86,20 +94,38 @@ if os.environ.get('DB_PORT'): _dnfs_config['database']['port'] = os.environ['DB_PORT'] if os.environ.get('REVIEW_URL'): _dnfs_config['ai_review']['url'] = os.environ['REVIEW_URL'] -if os.environ.get('CONFS_HOST'): - _dnfs_config['confs']['host'] = os.environ['CONFS_HOST'] -if os.environ.get('CONFS_PORT'): - _dnfs_config['confs']['port'] = os.environ['CONFS_PORT'] +if os.environ.get('GATEWAY_URL'): + _dnfs_config['gateway']['url'] = os.environ['GATEWAY_URL'] + +# 配置完整性校验 +_required = { + 'server': ['host', 'port'], + 'database': ['host', 'port', 'timeout'], + 'ai_review': ['url', 'timeout'], + 'gateway': ['url', 'timeout'], +} +for section, keys in _required.items(): + for key in keys: + if not _dnfs_config[section][key]: + print(f'[dnfs] 致命错误: 配置项 [{section}] {key} 缺失（dnfs.ini 中未配置且无环境变量覆盖）', + file=sys.stderr) + sys.exit(1) # 提取配置常量 +_DB_TIMEOUT = int(_dnfs_config['database']['timeout']) _DBV2_ADDR = _dnfs_config['database']['host'] _DBV2_PORT = int(_dnfs_config['database']['port']) -_DB_TIMEOUT = int(_dnfs_config['database']['timeout']) _REVIEW_SERVER_URL = _dnfs_config['ai_review']['url'] _REVIEW_TIMEOUT = int(_dnfs_config['ai_review']['timeout']) +_GATEWAY_URL = _dnfs_config['gateway']['url'] +_GATEWAY_TIMEOUT = int(_dnfs_config['gateway']['timeout']) _ADDR = _dnfs_config['server']['host'] _PORT = int(_dnfs_config['server']['port']) +_config_source = f'confs (dnfs.ini @ {_confs_host}:{_confs_port})' +print(f'[dnfs] 配置来源: {_config_source}') +print(f'[dnfs] Token 验签: gateway {_GATEWAY_URL}') + # 管理员判定已上移至 gateway：登录时命中 [Admin] users 白名单即在 Token payload # 写入 is_admin=true，本服务通过 info.get('is_admin') 读取，不再本地维护名单。 # 旧 Token 无此字段，按非管理员处理（安全方向）。 @@ -267,6 +293,7 @@ def _sp_review_text(title, sp): def _handle_exp(func): '''处理视图函数中抛出的异常，并在日志模块中进行记录''' + @functools.wraps(func) def wrapper(): try: return func() except DnfsExp as e: @@ -282,14 +309,46 @@ apply_security_headers(app) CORS(app) -def decode_token(token: str): - idx = token.find('_') - if idx != -1: - raw = token[:idx] - # FIXME: 验证数字签名 - decoded = base64.b64decode(raw) - jj = decoded.decode('utf-8') - return json.loads(jj) +def decode_token(token: str): + """通过 gateway /whoami 接口验证 Token 并返回用户信息。 + + gateway /whoami 内部完成 RSA 签名验证和过期检查，返回： + - 成功: {"login": true, "user": "...", "is_admin": true/false, ...} + - 失败: {"login": false} + 网络异常（gateway 不可达）按 fail-closed 处理，抛 DnfsExp 拒绝请求。 + """ + if not token: + return None + + req = urllib.request.Request( + _GATEWAY_URL, + data=b'{}', + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=_GATEWAY_TIMEOUT) as resp: + data = json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, OSError) as e: + # gateway 不可达或返回异常：fail-closed，拒绝请求 + logs.warning('dnfs', f'Token 验签失败（gateway 异常）: {e}', extra={}) + raise DnfsExp('认证服务暂时不可用', 503) + except Exception as e: + logs.warning('dnfs', f'Token 验签未知异常: {e}', extra={}) + raise DnfsExp('认证失败', 401) + + if not data.get('login'): + return None + + # gateway 返回 login=true 时 user 字段必定存在 + return { + 'user': data.get('user', ''), + 'is_admin': bool(data.get('is_admin', False)), + } @app.route('/add_note', methods=['POST']) @@ -407,7 +466,7 @@ def get_notes(): # eg: {"id": 1} @app.route('/delete_note', methods=['POST']) - +@_handle_exp def delete_note(): data = request.get_json() @@ -510,7 +569,7 @@ def delete_note(): # "content": "hello" #} @app.route('/update_note', methods=['POST']) - +@_handle_exp def update_note(): data = request.get_json() diff --git a/dnfs/log_client.py b/dnfs/log_client.py index 0dd516f..e61f68a 100644 --- a/dnfs/log_client.py +++ b/dnfs/log_client.py @@ -1,13 +1,15 @@ -"""统一日志客户端 — 各服务共用（dnfs / smtp / static / confs 四份副本内容一致，改动请同步）。 +""" +统一日志客户端 — 各服务共用（基于 pymsg.AsyncRpcQueue）。 设计要点： - 通过 pymsg RPC 调用 dbv2 execute 路由，逐条 INSERT 到 Logs 表 - - 异步非阻塞：内存队列 + 后台线程批量推送，不拖慢业务 - - fail-safe：db 服务不可用时静默丢弃，绝不影响业务；后台线程任何异常都不会崩溃 + - 异步非阻塞：内存队列 + 后台线程攒批推送，不拖慢业务 + - fail-safe：db 服务不可用时静默丢弃，绝不影响业务 - 缓冲批量：队列满 50 条或超时 2 秒时自动 flush - - 时间戳：日志入队时即记录 ts，反映事件真实发生时间 + - 时间戳：日志入队时即记录 ts（UTC ISO 格式），反映事件真实发生时间 + - 底层复用 pymsg.AsyncRpcQueue 通用队列基础设施 -用法： +用法（与旧版完全兼容）： from log_client import logs logs.info("dnfs", "节点新增成功 id=42") @@ -19,28 +21,25 @@ import json import os as _os import sys as _sys -import threading -import queue -import time import atexit from datetime import datetime, timezone _sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), "..")) -from pymsg import RpcClient, RpcError # noqa: E402 +from pymsg import RpcClient, RpcError, AsyncRpcQueue # noqa: E402 # ── 配置 ────────────────────────────────────────────────────────────── _DBV2_ADDR = _os.environ.get('DB_HOST', '127.0.0.1') _DBV2_PORT = int(_os.environ.get('DB_PORT', '8005')) -_BATCH_SIZE = 50 # 队列满 50 条时 flush -_FLUSH_INTERVAL = 2.0 # 超时 2 秒强制 flush -_TIMEOUT = 5 # socket 超时（秒） -_QUEUE_MAXSIZE = 10000 # 队列上限，防止后台线程异常时内存无限增长；满则丢弃最旧条目 +_BATCH_SIZE = 50 +_FLUSH_INTERVAL = 2.0 +_TIMEOUT = 5 +_QUEUE_MAXSIZE = 10000 def _safe_json(obj): - """安全序列化 extra 字段，不可序列化对象转为 repr 字符串，保证不抛异常。""" + """安全序列化 extra 字段，不可序列化对象转为 repr 字符串。""" try: return json.dumps(obj, ensure_ascii=False, default=str) except Exception: @@ -48,163 +47,80 @@ def _safe_json(obj): def _now_iso(): - """生成当前 UTC 时间 ISO 字符串（在入队时调用，保证时间戳准确）。""" + """生成当前 UTC 时间 ISO 字符串（入队时调用，保证时间戳准确）。""" return datetime.now(timezone.utc).isoformat() class LogClient: - """异步批量日志推送客户端。""" + """异步批量日志推送客户端（API 与旧版完全兼容）。""" - def __init__(self, service_name="log", - batch_size=_BATCH_SIZE, - flush_interval=_FLUSH_INTERVAL): + def __init__(self, service_name: str = "log", + batch_size: int = _BATCH_SIZE, + flush_interval: float = _FLUSH_INTERVAL): self._service_name = service_name - self._batch_size = batch_size - self._flush_interval = flush_interval - self._queue = queue.Queue(maxsize=_QUEUE_MAXSIZE) - self._stop = threading.Event() - self._flush_signal = threading.Event() - self._flush_done = threading.Event() - self._thread = threading.Thread(target=self._flush_loop, - daemon=True, name='log-client') - self._thread.start() - atexit.register(self._shutdown) + self._client = RpcClient( + _DBV2_ADDR, _DBV2_PORT, + service_name + "-log", + timeout=_TIMEOUT, + ) + self._queue = AsyncRpcQueue( + handler=self._send_batch, + batch_size=batch_size, + flush_interval=flush_interval, + max_queue_size=_QUEUE_MAXSIZE, + on_error=self._on_error, + ) + self._queue.start() + atexit.register(self.close) # ── 公开接口 ──────────────────────────────────────────────────────── - def push(self, service, level, message, trace_id=None, extra=None): + def push(self, service: str, level: str, message: str, + trace_id: str = None, extra=None): """推送一条日志（非阻塞，立即返回）。 队列满时静默丢弃最旧条目再放入新条目，保证不阻塞业务线程、不导致 OOM。 """ entry = { - 'ts': _now_iso(), # 入队时即生成时间戳，反映真实事件时间 + 'ts': _now_iso(), 'service': service, 'level': level, 'message': message, 'trace_id': trace_id, 'extra': extra, } - try: - self._queue.put_nowait(entry) - except queue.Full: - # 队列满：丢弃最旧的一条，再放入新条目（drop-oldest 策略） - try: - self._queue.get_nowait() - except queue.Empty: - pass - try: - self._queue.put_nowait(entry) - except queue.Full: - pass # 极端竞争下仍失败则静默丢弃 + self._queue.put(entry) - def debug(self, service, message, **kw): + def debug(self, service: str, message: str, **kw): self.push(service, 'DEBUG', message, **kw) - def info(self, service, message, **kw): + def info(self, service: str, message: str, **kw): self.push(service, 'INFO', message, **kw) - def warning(self, service, message, **kw): + def warning(self, service: str, message: str, **kw): self.push(service, 'WARNING', message, **kw) - def error(self, service, message, **kw): + def warn(self, service: str, message: str, **kw): + self.push(service, 'WARNING', message, **kw) + + def error(self, service: str, message: str, **kw): self.push(service, 'ERROR', message, **kw) - def critical(self, service, message, **kw): + def critical(self, service: str, message: str, **kw): self.push(service, 'CRITICAL', message, **kw) def flush(self): - """手动 flush 队列中剩余的日志（含后台线程持有的局部 batch）。""" - self._flush_done.clear() - self._flush_signal.set() # 通知后台线程立即 flush 其局部 batch - # 等待后台线程完成 flush（最多等待 flush_interval + 2 秒余量） - self._flush_done.wait(timeout=self._flush_interval + 2.0) - # 主线程排空队列中剩余条目 - self._drain_queue_to_db() + """手动 flush 队列中剩余的日志。""" + self._queue.flush() - def _drain_queue_to_db(self): - """排空主线程可见的队列中剩余条目，直接发送到 db。""" - batch = [] - while True: - try: - batch.append(self._queue.get_nowait()) - if len(batch) >= self._batch_size: - self._send(batch) - batch = [] - except queue.Empty: - break - if batch: - self._send(batch) + def close(self): + """关闭客户端，flush 剩余日志。""" + self._queue.close() # ── 内部实现 ──────────────────────────────────────────────────────── - def _flush_loop(self): - """后台线程：积攒到 batch_size 或超时 flush_interval 时发送。 - - 外层捕获所有异常，保证后台线程在任何情况下（网络故障、序列化错误等） - 都不会死亡，避免日志队列无限增长导致 OOM。 - """ - batch = [] - last_flush = time.monotonic() - - while not self._stop.is_set(): - try: - timeout = self._flush_interval - (time.monotonic() - last_flush) - chk_timeout = max(0.1, min(timeout, 0.5)) - try: - item = self._queue.get(timeout=chk_timeout) - batch.append(item) - while len(batch) < self._batch_size: - try: - batch.append(self._queue.get_nowait()) - except queue.Empty: - break - except queue.Empty: - pass - - should_flush = ( - len(batch) >= self._batch_size or - self._flush_signal.is_set() or - (batch and time.monotonic() - last_flush >= self._flush_interval) - ) - if should_flush and batch: - try: - self._send(batch) - except Exception as e: - # _send 内部已有细粒度异常处理，此处是最终安全网 - print(f"[log_client] _send 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - batch = [] - last_flush = time.monotonic() - if self._flush_signal.is_set(): - self._flush_signal.clear() - self._flush_done.set() - except Exception as e: - # 最外层安全网：任何意外异常都不应杀死日志线程 - print(f"[log_client] _flush_loop 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - time.sleep(0.5) # 防止异常风暴导致 CPU 满载 - - # 退出前 flush 剩余 - if batch: - try: - self._send(batch) - except Exception as e: - print(f"[log_client] 退出时 flush 异常: {e}", file=_sys.stderr) - self._flush_done.set() - - def _send(self, batch): - """通过 pymsg RPC（execute 路由）批量发送到 dbv2 Logs 表。 - - 整个 batch 复用一个 TCP 连接；网络错误时放弃本批剩余条目（fail-safe）， - 不抛异常给调用方。 - """ - try: - client = RpcClient(_DBV2_ADDR, _DBV2_PORT, self._service_name + "-log") - except Exception as e: - print(f"[log_client] 创建 RpcClient 失败: {e}", file=_sys.stderr) - return - + def _send_batch(self, batch: list): + """后台线程回调：逐条 INSERT 到 db（与旧版行为一致：网络错误放弃本批剩余）。""" for entry in batch: extra_val = entry.get("extra") try: @@ -224,21 +140,21 @@ class LogClient: }, } try: - client.call("execute", payload, timeout=_TIMEOUT) + self._client.call("execute", payload, timeout=_TIMEOUT) except RpcError as e: - # 业务错误（如表结构不符）：打印后继续下一条 print(f"[log_client] INSERT 失败: {e}", file=_sys.stderr) except Exception as e: - # 网络错误：db 不可用，放弃本批剩余（fail-safe） - print(f"[log_client] 发送失败: {e}", file=_sys.stderr) + print(f"[log_client] 发送失败（放弃本批剩余）: {e}", file=_sys.stderr) break - def _shutdown(self): - """进程退出时 flush 剩余日志。""" - self._stop.set() - self.flush() + def _on_error(self, e: Exception, batch: list): + """异常安全网：handler 外的异常也不会崩溃线程。""" + print(f"[log_client] _send_batch 异常（已捕获，线程继续运行）: {e}", + file=_sys.stderr) # ── 全局单例 ────────────────────────────────────────────────────────── +# 各服务的 log_client.py 副本需将 service_name 改为自己的服务名。 +# 此文件（confs/log_client.py）的 service_name 为 "confs"。 -logs = LogClient() +logs = LogClient(service_name="dnfs") diff --git a/docker-compose.yml b/docker-compose.yml index b1be30d..6401c09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,6 +83,11 @@ services: - DB_HOST=db - DB_PORT=8005 - REVIEW_URL=http://ai_review:8007/review + - GATEWAY_URL=http://host.docker.internal:8443/whoami + - CONFS_HOST=host.docker.internal + - CONFS_PORT=8010 + extra_hosts: + - "host.docker.internal:host-gateway" depends_on: db: condition: service_healthy diff --git a/gateway/README.md b/gateway/README.md index e3236db..844dafa 100644 --- a/gateway/README.md +++ b/gateway/README.md @@ -28,6 +28,30 @@ UniServe Framework 的应用层网关，C++20 实现。 | CAPTCHA | 一次性、5 分钟过期，登录与注册均可选启用 | | 速率限制 | 内存计数器，自动清理超过 1 万条的老条目 | | TLS 可选 | `Server.tls = off` 时纯 HTTP（配合 WAF），`on` 时用 `full_chain.crt` / `server.key` | +| 下游服务容错 | pymsg 客户端 `retry_on_refused` 机制：db 等强依赖服务未启动时自动阻塞重试，避免启动顺序问题；日志客户端 fail-safe 静默丢弃 | + +## pymsg 内部通信协议 + +gateway 通过项目内的 [pymsg](../pymsg/)（长度前缀帧 TCP 协议）与内部微服务通信，C++ 实现在 `inc/Msg.hpp`，与 Python 端字节级兼容。 + +**线协议**：`[4字节大端uint32长度][UTF-8载荷]`，最大 64 MiB。 + +**JSON RPC 约定**：请求形如 `{"_route": "<路由名>", ...}`，响应形如 `{"success": true, "data": ...}` 或 `{"success": false, "error": "..."}`。 + +**下游服务调用方式**： + +| 下游 | 默认端口 | `_route` | 调用位置 | 重试策略 | +|------|---------|----------|---------|---------| +| db | 8005 | `execute` | `DbC::call_msg()`（所有数据库 CRUD） | `retry_on_refused=true`，db 未就绪时阻塞等待 | +| db | 8005 | `execute` | `LogC::_send()`（日志异步批量推送） | `retry_on_refused=false`，fail-safe 静默丢弃 | +| captcha | 8009 | `verify` / `image` | `verify_captcha_remote()`、`/captcha/image` 代理 | 默认不重试，连接失败 fail-closed 返回 502/false | +| smtp | 8008 | `verify_email` / `send_email` | `verify_email_code_remote()`、`/register/send_code` | 默认不重试，连接失败 fail-closed 返回 502/false | +| confs | 8010 | `get` | 启动时拉取 `gateway.ini` 配置 | `retry_on_refused=true`，confs 未就绪时阻塞等待 | + +**retry_on_refused 机制**（`MsgClient::call()`，对应 Python 端 `MsgClient.call(retry_on_refused=True)`）： +- 仅对 `ECONNREFUSED`（目标端口无服务监听）重试，超时、连接重置、业务错误立即抛出 +- 固定 1 秒间隔重试；首次失败立即打印 stderr 提示，之后每 5 秒打印一次等待日志（节制日志，避免刷屏） +- 用于解决微服务启动顺序不确定问题：gateway 启动时 db 未就绪，首次 db 调用自动阻塞等待直到 db 上线 ## 接口 @@ -163,53 +187,24 @@ gateway 不直连数据库（早期直连本地 sqlite 的代码已注释保留 ## 配置 -[example.ini](example.ini)： +gateway 配置采用**两层加载模式**，与其他 Python 服务（dnfs/db/static）保持一致： -```ini -[Server] -# tls = off HTTP 模式（配合 waf 模块） -# tls = on HTTPS 模式（独立部署，证书为 full_chain.crt / server.key） -address = 127.0.0.1 -port = 8443 -tls = off +1. **confs 统一配置服务**：启动时从 confs（`127.0.0.1:8010`，可通过 `CONFS_HOST`/`CONFS_PORT` 环境变量覆盖）通过 pymsg `get` 路由拉取 `gateway.ini`，`retry_on_refused=true` 阻塞等待 confs 就绪。 +2. **环境变量覆盖**（最高优先级，Docker 兼容）：`LISTEN_ADDR`/`LISTEN_PORT`、`DB_HOST`/`DB_PORT`、`CAPTCHA_HOST`/`CAPTCHA_PORT`、`SMTP_HOST`/`SMTP_PORT` 覆盖 ini 中对应值。 +3. **完整性校验**：必需配置项缺失则启动失败。 -[Proxy] -default = http://127.0.0.1:9200 -closed = http://127.0.0.1:8005;http://127.0.0.1:8007; -opened = http://127.0.0.1:9200;http://127.0.0.1:8006; +配置文件统一存放于 [confs/configs/gateway.ini](../confs/configs/gateway.ini)，gateway 目录下不再保留本地配置文件。 -[Token] -# Token 有效期（秒），默认 3600 = 1 小时 -keep_time = 3600 - -[Admin] -# 管理员白名单（分号分隔邮箱），登录命中时 Token payload 写入 is_admin=true -# WebSocket 代理仅允许管理员访问，其他需管理员权限的下游同样凭此字段判断 -users = admin@example.com - -[CORS] -# 允许跨域的 Origin 白名单（分号分隔） -# 仅命中的 Origin 才会回填 Access-Control-Allow-Origin；缺省为空（不回填） -allowed_origins = https://qsont.me;https://www.qsont.me;http://127.0.0.1:8443;http://localhost:8443 - -[Database] -path = sqlite3.db - -[WsProxy] -# WebSocket 反向代理路由（仅管理员） -# 格式：/path=ws://host:port/target;/path2=wss://host2/target2 -# 支持 ws:// 和 wss:// 后端，握手阶段校验管理员 Token -# routes = /ws/aibox=ws://127.0.0.1:8765;/ws/log=wss://example.com/log -routes = -``` - -- `Proxy.default`：未带 `Target-Url` header 时的默认转发地址（不需 Token，主要服务前端静态资源）。 +配置项说明： +- `Server.address`/`port`/`tls`：监听地址、端口、TLS 开关；`address`/`port` 可被 `LISTEN_ADDR`/`LISTEN_PORT` 环境变量覆盖。 +- `Proxy.default`：未带 `Target-Url` header 时的默认转发地址（不需 Token）。 - `Proxy.opened`：允许外部带 Token 访问的内部服务白名单（分号分隔）。 -- `Proxy.closed`：仅记录用途，实际靠"不在 opened 中即拒绝"实现隔离。 -- `Token.keep_time`：登录签发 Token 的有效期（秒），缺省 3600。 -- `Admin.users`：管理员白名单（分号分隔邮箱），登录时命中即在 Token 中写入 `is_admin=true`。 -- `CORS.allowed_origins`：OPTIONS 预检允许跨域的 Origin 白名单（分号分隔），缺省为空（不回填 ACAO）。 -- `WsProxy.routes`：WebSocket 代理路由映射，格式 `/path=ws://host:port/target`，多个用分号分隔；留空则不开启 WS 代理。 +- `Proxy.closed`：仅文档记录用途，实际靠"不在 opened 中即拒绝"实现隔离。 +- `Token.*`：Token 有效期（`keep_time`）、刷新宽限期倍数（`refresh_grace_multiplier`）、最大刷新次数（`max_refresh`）。 +- `Admin.users`：管理员白名单（分号分隔邮箱），登录命中时 Token payload 写入 `is_admin=true`。 +- `CORS.allowed_origins`：允许跨域的 Origin 白名单（分号分隔）。 +- `WsProxy.routes`：WebSocket 代理路由，格式 `/path=ws://host:port/target`，分号分隔。 +- `Downstream.*`：下游 pymsg 服务地址（db/captcha/smtp），可被 `DB_HOST`/`DB_PORT`、`CAPTCHA_HOST`/`CAPTCHA_PORT`、`SMTP_HOST`/`SMTP_PORT` 环境变量覆盖。 ## 部署模式 @@ -252,38 +247,44 @@ tls = on - 项目内库：`libus_http`、`libus_crypt`（由 [libhttp](../libhttp/)、[libcrypt](../libcrypt/) 编译安装） ```bash -bash _set.sh # make clear && make -bash _run.sh # ./build/gateway.out -i example.ini +bash _set.sh # make clean && make +bash _run.sh # 从 confs 拉取配置启动 ``` -`gateway.out` 需以 `-i ` 传配置文件路径。其他参数：`-h/--help`、`-v/--version`。 +无需传参，配置统一由 confs 管理。命令行参数：`-h/--help`、`-v/--version`。 ## 目录结构 ``` gateway/ ├── src/ -│ └── main.cpp # 全部实现：PBKDF2 / PasswordValidator / RateLimiter / 路由 / handler +│ └── main.cpp # 全部实现：配置加载（confs → 环境变量覆盖）、PBKDF2、RateLimiter、路由/handler ├── inc/ -│ ├── Conf.hpp # ConfIni：基于 boost::property_tree 的 ini 解析 +│ ├── Msg.hpp # pymsg C++ 实现：长度前缀帧协议、MsgClient（含 retry_on_refused）、MsgException 族 +│ ├── DbC.hpp # db 服务客户端：通过 pymsg 调 dbv2（地址可配置，retry_on_refused=true） +│ ├── LogC.hpp # 统一日志客户端：异步批量队列 + pymsg INSERT（fail-safe） +│ ├── Conf.hpp # ConfIni：boost::property_tree ini 解析，支持 fromString/env/env_int │ ├── ExpBase.hpp # 异常基类（带 source_location） │ └── LocalTime.hpp # 本地时间工具（getCurrent / parse） ├── Makefile -├── example.ini ├── _set.sh / _run.sh └── README.md ``` +配置文件统一在 [confs/configs/gateway.ini](../confs/configs/gateway.ini)，gateway 目录无本地配置文件。 + ## 版本 -`0.6.0`（见 [src/main.cpp:31](src/main.cpp#L31)）。启动时打印： +`0.7.0`（见 [src/main.cpp:31](src/main.cpp#L31)）。启动时打印： ``` -Gateway 0.6.0 started +Gateway 0.7.0 started ``` 近期变更： +- `0.7.0`：配置体系完全迁移到 confs 统一配置服务——配置文件统一存放于 `confs/configs/gateway.ini`（原 example.ini 内容 + `[Downstream]` 段），删除本地 example.ini，移除 `-i/--input` 命令行参数（不再支持本地 ini 文件回退，与 static 模块一致）；启动时从 confs（`CONFS_HOST`/`CONFS_PORT`，默认 `127.0.0.1:8010`）通过 pymsg `get` 路由拉取配置（`retry_on_refused=true`）；`ConfIni` 增加 `fromString()`/`env()`/`env_int()`；环境变量覆盖支持 `LISTEN_ADDR`/`LISTEN_PORT`/`DB_HOST`/`DB_PORT`/`CAPTCHA_HOST`/`CAPTCHA_PORT`/`SMTP_HOST`/`SMTP_PORT`；配置完整性校验；`DbC` 增加 `init(host, port)`，main.cpp 所有硬编码 captcha/smtp/db 地址改为配置变量，`logs().set_endpoint()` 启动时初始化；`--version/-v` 选项补全实现；版本升至 0.7.0。 +- `0.6.1`：pymsg 客户端增加 `retry_on_refused` 重试机制——新增 `MsgConnectionRefusedException`；`MsgClient::call()` 新增 `retry_on_refused`/`retry_interval` 参数；`DbC::call_msg()` 启用重试等待 db 就绪；`LogC::_send()` 显式不重试（fail-safe）。 - `0.6.0`：WebSocket 反向代理（仅管理员）——新增 `[WsProxy]` 配置段，支持 `routes = /path=ws://host:port/target;...` 精确路径映射；握手阶段校验 Token 中 `is_admin=true`（基于 `Admin.users` 白名单），通过后双向透明桥接文本/二进制帧到后端，支持 `ws://` 和 `wss://`；后端连接错误/鉴权失败均优雅关闭。依赖 libhttp v3.1+ 的 `ws_proxy()` 接口。顺带修复 `LocalTime.hpp` 在 Apple clang 下 C++20 tzdb 不可用的编译问题（改用 `strftime`/`get_time`/`mktime`）。 - `0.5.4`：修复邮件轰炸漏洞（审计报告 P0-1）——`/register/send_code` 强制 CAPTCHA 校验（经 captcha:8009，Token 一次性消费），新增邮箱级双窗口限流（3 次/小时 + 10 次/天）；smtp 层同步增加邮箱级长期限流兜底 + 全局熔断（`SMTP_HOURLY_LIMIT` 环境变量，默认 100 封/小时）；前端 `register.html`/`register.js` 同步接入图形验证码（复用 `/captcha/image` 代理路由，方案 A：仅在「获取验证码」时要求 CAPTCHA，注册提交不加）。 - `0.5.3`：配合 smtp 同邮箱 60 秒发送冷却——`/register/send_code` 在 smtp 返回 `retry_after` 时按 429 透传其提示文案（其余发送失败仍用通用 500 文案，SMTP 内部错误细节不向外暴露）。 diff --git a/gateway/_run.sh b/gateway/_run.sh index dcbe665..9c34359 100644 --- a/gateway/_run.sh +++ b/gateway/_run.sh @@ -1,4 +1,10 @@ #!/bin/bash +# gateway 启动脚本 +# 配置统一由 confs 服务管理（confs/configs/gateway.ini）， +# 启动时从 confs（CONFS_HOST:CONFS_PORT，默认 127.0.0.1:8010）拉取配置。 cd "$(dirname "$0")" || exit + +# 等待依赖服务启动（confs/db） sleep 2 -exec ./build/gateway.out -i example.ini #1>/dev/null + +exec ./build/gateway.out "$@" diff --git a/gateway/inc/Conf.hpp b/gateway/inc/Conf.hpp index 6e8ce6c..a0b7d82 100644 --- a/gateway/inc/Conf.hpp +++ b/gateway/inc/Conf.hpp @@ -1,6 +1,9 @@ #pragma once #include #include +#include +#include +#include #include #include namespace property_tree = boost::property_tree; @@ -9,22 +12,54 @@ namespace qing { class ConfIni { public: + /// 默认构造（空配置，后续通过赋值覆盖，仅供 confs 加载流程内部使用） + ConfIni() = default; + + /// 从文件路径解析 ini（保留用于开发/调试模式） ConfIni(const std::string &filename) { + if (filename.empty()) return; // 空文件名不报错，后续会通过赋值覆盖 try { - std::cout << "open and parse ini file: " << filename << std::endl; + std::cout << "[conf] 从文件加载配置: " << filename << std::endl; property_tree::ini_parser::read_ini(filename, pt); } catch (const boost::property_tree::ini_parser_error& e) { - std::cerr << "INI parsing error: " << e.what() << std::endl; + std::cerr << "[conf] INI 解析错误: " << e.what() << std::endl; throw IniConfigureFileParseException("INI parsing error!"); } } + /// 从 ini 文本字符串解析（用于从 confs 拉取后解析） + static ConfIni fromString(const std::string &ini_text) { + ConfIni c; + try { + std::istringstream iss(ini_text); + property_tree::ini_parser::read_ini(iss, c.pt); + } + catch (const boost::property_tree::ini_parser_error& e) { + std::cerr << "[conf] INI 文本解析错误: " << e.what() << std::endl; + throw IniConfigureFileParseException("INI parsing error!"); + } + return c; + } + template T get(const std::string &path, const T &_defaut) { return pt.get(path, _defaut); } + /// 读取环境变量，不存在则返回默认值 + static std::string env(const char *name, const std::string &default_val = "") { + const char *v = std::getenv(name); + return v ? std::string(v) : default_val; + } + + static int env_int(const char *name, int default_val) { + const char *v = std::getenv(name); + if (!v || !*v) return default_val; + try { return std::stoi(v); } + catch (...) { return default_val; } + } + class IniConfigureFileParseException: public std::exception { public: explicit IniConfigureFileParseException(const std::string &msg) diff --git a/gateway/inc/DbC.hpp b/gateway/inc/DbC.hpp index 00ded14..796c468 100644 --- a/gateway/inc/DbC.hpp +++ b/gateway/inc/DbC.hpp @@ -10,6 +10,12 @@ namespace qing { * smtp/captcha 等其他下游的超时常量见 main.cpp。 */ static constexpr double DEFAULT_TIMEOUT = 5.0; + /// 初始化 db 服务地址（启动时从配置设置，默认 127.0.0.1:8005） + static void init(const std::string& host, int port) { + _host = host; + _port = port; + } + // 【保留】旧版 HTTP 接口：通过 HttpClient POST 到 dbv2 HTTP /execute 端点。 // 保留用于兼容性参考，新代码优先使用 call_msg()（pymsg TCP 协议，更可靠）。 static json::value call_http(json::value& params) { @@ -18,7 +24,8 @@ namespace qing { client.set_header("Content-Type: application/json"); std::string payload = json::serialize(params); std::string res = ""; - client.post_stream_sync(_DBURL, payload, [&res](const auto& ret) { + std::string url = "http://" + _host + ":" + std::to_string(_port) + "/execute"; + client.post_stream_sync(url, payload, [&res](const auto& ret) { res += ret; }); return json::parse(res); @@ -28,17 +35,23 @@ namespace qing { // 自动添加 _route = "execute"，返回 dbv2 pymsg 完整 JSON 响应 // {"success":..., "data":[...]}。 // 响应格式与 call_http 兼容，调用方可直接替换。 + // db 是强依赖，启用 retry_on_refused：db 未启动时阻塞等待，避免启动顺序问题。 static json::value call_msg(const json::value& params, double timeout = DEFAULT_TIMEOUT) { json::value jv_route = params; jv_route.as_object()["_route"] = "execute"; - pmc::net::MsgClient client("127.0.0.1", 8005, "gateway"); + pmc::net::MsgClient client(_host, _port, "gateway"); std::string resp = client.call( - json::serialize(jv_route), timeout); + json::serialize(jv_route), timeout, + /*retry_on_refused=*/true); return json::parse(resp); } + static const std::string& host() { return _host; } + static int port() { return _port; } + private: - static constexpr const char* _DBURL = "http://127.0.0.1:8005/execute"; + static inline std::string _host = "127.0.0.1"; + static inline int _port = 8005; }; } diff --git a/gateway/inc/LogC.hpp b/gateway/inc/LogC.hpp index a685d9e..66eaa88 100644 --- a/gateway/inc/LogC.hpp +++ b/gateway/inc/LogC.hpp @@ -196,7 +196,8 @@ private: payload["values"] = std::move(values); const std::string resp = client.call( - json::serialize(payload), _TIMEOUT); + json::serialize(payload), _TIMEOUT, + /*retry_on_refused=*/false); // 日志是 fail-safe，db 不可用时静默丢弃，绝不阻塞重试 auto obj = json::parse(resp).as_object(); if (!obj.at("success").as_bool()) { std::string err = "unknown"; diff --git a/gateway/inc/Msg.hpp b/gateway/inc/Msg.hpp index fd7a956..90c1151 100644 --- a/gateway/inc/Msg.hpp +++ b/gateway/inc/Msg.hpp @@ -17,11 +17,14 @@ */ #include +#include #include +#include #include #include #include #include +#include #include #include @@ -59,6 +62,13 @@ public: explicit MsgConnectionException(const std::string &msg) : MsgException(msg) {} }; +/// 目标服务未启动（connect 返回 ECONNREFUSED），用于 retry_on_refused 重试机制 +class MsgConnectionRefusedException : public MsgConnectionException { +public: + explicit MsgConnectionRefusedException(const std::string &msg) + : MsgConnectionException(msg) {} +}; + // ============================================================================ // Low-level protocol: 4-byte big-endian length prefix + UTF-8 payload // ============================================================================ @@ -155,10 +165,14 @@ inline int openConnection(const std::string& addr, int port, double timeout) { if (::connect(fd, reinterpret_cast(&serv_addr), sizeof(serv_addr)) < 0) { + int saved_errno = errno; ::close(fd); - throw MsgConnectionException( - "could not connect to " + addr + ":" + std::to_string(port) + - " - " + ::strerror(errno)); + std::string err_msg = "could not connect to " + addr + ":" + + std::to_string(port) + " - " + ::strerror(saved_errno); + if (saved_errno == ECONNREFUSED) { + throw MsgConnectionRefusedException(err_msg); + } + throw MsgConnectionException(err_msg); } return fd; @@ -242,19 +256,64 @@ public: /// /// 不会给消息添加 ``name:`` 前缀（适用于 RPC/API 调用）。 /// - /// @param msg 要发送的原始字符串。 - /// @param timeout 超时秒数，默认 10.0。 - /// @returns 服务器返回的字符串。 - std::string call(const std::string &msg, double timeout = 10.0) { - int fd = detail::openConnection(_addr, _port, timeout); - try { - pmsg_send(fd, msg); - std::string resp = pmsg_recv(fd); - ::close(fd); - return resp; - } catch (...) { - ::close(fd); - throw; + /// @param msg 要发送的原始字符串。 + /// @param timeout 超时秒数，默认 10.0。 + /// @param retry_on_refused 目标服务未启动（ECONNREFUSED）时是否阻塞重试， + /// 直到连接成功。适用于启动顺序不确定的服务发现场景。 + /// 默认 false（连接拒绝立即抛异常）。 + /// @param retry_interval 重试间隔秒数，默认 1.0。 + /// @returns 服务器返回的字符串。 + std::string call(const std::string &msg, double timeout = 10.0, + bool retry_on_refused = false, + double retry_interval = 1.0) { + if (!retry_on_refused) { + // 快速路径：不重试，直接执行一次，保持原有行为 + int fd = detail::openConnection(_addr, _port, timeout); + try { + pmsg_send(fd, msg); + std::string resp = pmsg_recv(fd); + ::close(fd); + return resp; + } catch (...) { + ::close(fd); + throw; + } + } + + // 重试路径：仅对 ECONNREFUSED 重试，其他错误立即抛出 + constexpr double kLogEvery = 5.0; // 重试期间打印等待提示的间隔（秒） + int attempt = 0; + auto last_log = std::chrono::steady_clock::now() - + std::chrono::duration_cast( + std::chrono::duration(kLogEvery)); + + while (true) { + try { + int fd = detail::openConnection(_addr, _port, timeout); + try { + pmsg_send(fd, msg); + std::string resp = pmsg_recv(fd); + ::close(fd); + return resp; + } catch (...) { + ::close(fd); + throw; + } + } catch (const MsgConnectionRefusedException &) { + ++attempt; + auto now = std::chrono::steady_clock::now(); + double elapsed = std::chrono::duration(now - last_log).count(); + if (elapsed >= kLogEvery || attempt == 1) { + std::fprintf(stderr, + "[pymsg] 等待服务 %s:%d 就绪... (已重试 %d 次)\n", + _addr.c_str(), _port, attempt); + std::fflush(stderr); + last_log = now; + } + std::this_thread::sleep_for( + std::chrono::duration(retry_interval)); + } + // 其他异常（MsgTimeoutException、普通MsgConnectionException等）不捕获，直接向上抛出 } } diff --git a/gateway/src/main.cpp b/gateway/src/main.cpp index 2437b17..0ad53da 100644 --- a/gateway/src/main.cpp +++ b/gateway/src/main.cpp @@ -28,7 +28,7 @@ using namespace qing; namespace json = boost::json; namespace po = boost::program_options; -constexpr const char *_VER = "0.6.0"; +constexpr const char *_VER = "0.7.0"; ////////////////////////////////// #include @@ -339,6 +339,17 @@ static int _TOKEN_MAX_REFRESH = 10; // Token 生命周期内最大刷新次数 static std::vector _CORS_ORIGINS; // CORS 允许的 Origin 白名单，可由 [CORS] allowed_origins 配置 static std::vector _ADMIN_USERS; // 管理员账号白名单，可由 [Admin] users 配置（分号分隔），登录时写入 Token payload 的 is_admin +/* 下游 pymsg 服务地址（从 confs [Downstream] 段加载，可被环境变量覆盖） */ +static std::string _DB_HOST = "127.0.0.1"; +static int _DB_PORT = 8005; +static std::string _CAPTCHA_HOST = "127.0.0.1"; +static int _CAPTCHA_PORT = 8009; +static std::string _SMTP_HOST = "127.0.0.1"; +static int _SMTP_PORT = 8008; +/* confs 引导配置（不依赖 confs 本身，环境变量 CONFS_HOST/CONFS_PORT 覆盖，默认 127.0.0.1:8010） */ +static std::string _CONFS_HOST = "127.0.0.1"; +static int _CONFS_PORT = 8010; + /* WebSocket 代理路由：path -> ws://backend-host:port/target * 仅管理员用户可访问，握手阶段由 ws_proxy 回调验签 Token 中的 is_admin */ static std::unordered_map _WS_PROXY_ROUTES; @@ -414,7 +425,7 @@ bool verify_captcha(const std::string& token, const std::string& answer) { * 与内置 verify_captcha 并存：登录/注册可任选其一接入。 */ bool verify_captcha_remote(const std::string& token, const std::string& answer) { try { - pmc::net::MsgClient client("127.0.0.1", 8009, "gateway"); + pmc::net::MsgClient client(_CAPTCHA_HOST, _CAPTCHA_PORT, "gateway"); json::value jv_req = { {"_route", "verify"}, {"token", token}, @@ -430,11 +441,11 @@ bool verify_captcha_remote(const std::string& token, const std::string& answer) } } -/* 调用 smtp 服务（pymsg/TCP :8008）校验邮箱验证码。 +/* 调用 smtp 服务（pymsg/TCP）校验邮箱验证码。 * 验证码一次性消费；服务不可用或响应异常一律视为校验失败（fail-closed）。 */ bool verify_email_code_remote(const std::string& email, const std::string& code) { try { - pmc::net::MsgClient client("127.0.0.1", 8008, "gateway"); + pmc::net::MsgClient client(_SMTP_HOST, _SMTP_PORT, "gateway"); json::value jv_req = { {"_route", "verify_email"}, {"email", email}, @@ -475,7 +486,6 @@ int main(int argc, char **argv) { desc.add_options() ("help,h", "output help message") ("version,v", "output version message") - ("input,i", po::value(), "input a configure text file") ; @@ -501,21 +511,67 @@ int main(int argc, char **argv) { return 0; } - - - /* 异常：没输入配置文件 */ - if (!vm.count("input")) { - std::cerr << "Please input configure file via -i" << std::endl; - return 1; + /* 分支：输出版本号 */ + if (vm.count("version")) { + std::cout << "Gateway " << _VER << std::endl; + return 0; } - { - try { + /* ==================== 配置加载（两层：confs 强依赖 → 环境变量覆盖） ==================== + * 与 Python 服务（dnfs/db/static）保持一致的加载模式： + * 1. 引导配置：confs 地址从 CONFS_HOST/CONFS_PORT 环境变量读取，缺省 127.0.0.1:8010 + * 2. 从 confs 拉取 gateway.ini 全文（retry_on_refused=true 等待 confs 就绪） + * 3. 用 boost::property_tree 解析 ini + * 4. 环境变量覆盖（最高优先级，Docker 环境需要服务名而非 127.0.0.1） + * 5. 配置完整性校验：必需字段缺失则启动失败 + * 不再支持本地 ini 文件回退（与 static 模块一致，配置统一由 confs 管理）。 + * ==================================================================================== */ - ConfIni cf(vm["input"].as()); + { + /* 1. 引导配置：confs 连接地址（不依赖 confs 本身，避免循环依赖） */ + _CONFS_HOST = ConfIni::env("CONFS_HOST", "127.0.0.1"); + _CONFS_PORT = ConfIni::env_int("CONFS_PORT", 8010); + + ConfIni cf; + try { + /* 从 confs 拉取 gateway.ini */ + std::cout << "[gateway] 从 confs (" << _CONFS_HOST << ":" << _CONFS_PORT + << ") 拉取 gateway.ini ..." << std::endl; + pmc::net::MsgClient confs_client(_CONFS_HOST, _CONFS_PORT, "gateway"); + json::object req; + req["_route"] = "get"; + req["name"] = "gateway.ini"; + std::string resp_str = confs_client.call( + json::serialize(req), /*timeout=*/5.0, + /*retry_on_refused=*/true); + auto resp = json::parse(resp_str).as_object(); + if (!resp.at("success").as_bool()) { + std::string err = "unknown"; + if (auto* ev = resp.if_contains("error")) { + if (ev->is_string()) err = ev->as_string().c_str(); + } + std::cerr << "[gateway] 致命错误: 从 confs 获取 gateway.ini 失败: " << err << std::endl; + return 1; + } + std::string ini_content = resp.at("data").as_object().at("content").as_string().c_str(); + cf = ConfIni::fromString(ini_content); + std::cout << "[gateway] 配置来源: confs (gateway.ini @ " + << _CONFS_HOST << ":" << _CONFS_PORT << ")" << std::endl; + } + catch (ConfIni::IniConfigureFileParseException &e) { + std::cerr << "ERROR: " << e.what() << std::endl; + std::cerr << "ini parse error" << std::endl; + return 1; + } + catch (const std::exception &e) { + std::cerr << "[gateway] 致命错误: 配置加载失败: " << e.what() << std::endl; + return 1; + } + + /* 2. 从 ini 读取配置 */ _ADDR = cf.get("Server.address", ""); - _PORT = cf.get ("Server.port", -1); - _TLS = cf.get ("Server.tls", false); + _PORT = cf.get("Server.port", -1); + _TLS = cf.get("Server.tls", false); _DEFT_URL = cf.get("Proxy.default", ""); _OPEN_URL = split(cf.get("Proxy.opened", ""), ';'); _DB_PATH = cf.get("Database.path", ""); @@ -525,8 +581,15 @@ int main(int argc, char **argv) { _CORS_ORIGINS = split(cf.get("CORS.allowed_origins", ""), ';'); _ADMIN_USERS = split(cf.get("Admin.users", ""), ';'); - /* [WsProxy] WebSocket 代理路由：格式 /path=ws://host:port/target;... - * 仅管理员（Token 中 is_admin=true）可访问，用于 aibox 等内部 WS 服务的网关暴露 */ + /* 下游服务地址 */ + _DB_HOST = cf.get("Downstream.db_host", "127.0.0.1"); + _DB_PORT = cf.get("Downstream.db_port", 8005); + _CAPTCHA_HOST = cf.get("Downstream.captcha_host", "127.0.0.1"); + _CAPTCHA_PORT = cf.get("Downstream.captcha_port", 8009); + _SMTP_HOST = cf.get("Downstream.smtp_host", "127.0.0.1"); + _SMTP_PORT = cf.get("Downstream.smtp_port", 8008); + + /* [WsProxy] WebSocket 代理路由：格式 /path=ws://host:port/target;... */ { std::string ws_cfg = cf.get("WsProxy.routes", ""); if (!ws_cfg.empty()) { @@ -536,7 +599,6 @@ int main(int argc, char **argv) { if (eq != std::string::npos) { std::string path = entry.substr(0, eq); std::string url = entry.substr(eq + 1); - // 去除首尾空白 auto trim = [](std::string& s) { while (!s.empty() && std::isspace((unsigned char)s.front())) s.erase(s.begin()); while (!s.empty() && std::isspace((unsigned char)s.back())) s.pop_back(); @@ -549,22 +611,53 @@ int main(int argc, char **argv) { } } } + + /* 3. 环境变量覆盖（优先级最高，Docker 环境使用服务名而非 127.0.0.1） */ + std::string env_listen_addr = ConfIni::env("LISTEN_ADDR"); + if (!env_listen_addr.empty()) _ADDR = env_listen_addr; + int env_listen_port = ConfIni::env_int("LISTEN_PORT", -1); + if (env_listen_port > 0) _PORT = env_listen_port; + + std::string env_db_host = ConfIni::env("DB_HOST"); + if (!env_db_host.empty()) _DB_HOST = env_db_host; + int env_db_port = ConfIni::env_int("DB_PORT", -1); + if (env_db_port > 0) _DB_PORT = env_db_port; + + std::string env_captcha_host = ConfIni::env("CAPTCHA_HOST"); + if (!env_captcha_host.empty()) _CAPTCHA_HOST = env_captcha_host; + int env_captcha_port = ConfIni::env_int("CAPTCHA_PORT", -1); + if (env_captcha_port > 0) _CAPTCHA_PORT = env_captcha_port; + + std::string env_smtp_host = ConfIni::env("SMTP_HOST"); + if (!env_smtp_host.empty()) _SMTP_HOST = env_smtp_host; + int env_smtp_port = ConfIni::env_int("SMTP_PORT", -1); + if (env_smtp_port > 0) _SMTP_PORT = env_smtp_port; + + /* 4. 配置完整性校验 */ + if (_ADDR.empty() || _PORT == -1) { + std::cerr << "[gateway] 致命错误: 配置项 [Server] address/port 缺失" + "（ini 中未配置且无 LISTEN_ADDR/LISTEN_PORT 环境变量覆盖）" << std::endl; + return 1; } - - - /* 常规输出 */ - catch (ConfIni::IniConfigureFileParseException &e) { - std::cerr << "ERROR: " << e.what() << std::endl; - std::cerr << "ini format input file error" << std::endl; + if (_DEFT_URL.empty()) { + std::cerr << "[gateway] 致命错误: 配置项 [Proxy] default 缺失" << std::endl; + return 1; + } + if (_DB_HOST.empty() || _DB_PORT <= 0) { + std::cerr << "[gateway] 致命错误: 配置项 [Downstream] db_host/db_port 缺失" << std::endl; return 1; } } - if (_ADDR == "" || _PORT == -1) { - std::cerr << "Please setup addree and port in Serve section in configure file"; - std::cerr << std::endl; - return 0; - } + /* 初始化下游客户端地址 */ + DbC::init(_DB_HOST, _DB_PORT); + logs().set_endpoint(_DB_HOST, _DB_PORT); + + std::cout << "[gateway] 监听: " << _ADDR << ":" << _PORT + << " (TLS: " << (_TLS ? "on" : "off") << ")" << std::endl; + std::cout << "[gateway] 下游服务: db=" << _DB_HOST << ":" << _DB_PORT + << ", captcha=" << _CAPTCHA_HOST << ":" << _CAPTCHA_PORT + << ", smtp=" << _SMTP_HOST << ":" << _SMTP_PORT << std::endl; // 根据 Server.tls 选择 HTTPS（带证书）或 HTTP（无证书）构造函数 // HTTP 模式用于 WAF 终止 TLS 后的内部转发链路 @@ -1373,7 +1466,7 @@ http::response handleCaptchaImage( res.version(req.version()); /* 调 captcha 服务（pymsg/TCP），连接失败/超时视为 502 */ - pmc::net::MsgClient client("127.0.0.1", 8009, "gateway"); + pmc::net::MsgClient client(_CAPTCHA_HOST, _CAPTCHA_PORT, "gateway"); std::string resp; try { resp = client.call(R"({"_route":"image"})", _TIMEOUT_CAPTCHA); @@ -1571,7 +1664,7 @@ http::response handleSendEmailCode( int retry_after = 0; // smtp 同邮箱发送冷却的剩余秒数，0 表示非冷却拒绝 std::string smtp_err; // 冷却拒绝时 smtp 返回的提示文案（仅冷却场景透传客户端） try { - pmc::net::MsgClient client("127.0.0.1", 8008, "gateway"); + pmc::net::MsgClient client(_SMTP_HOST, _SMTP_PORT, "gateway"); json::value jv_req = { {"_route", "send_email"}, {"email", email} diff --git a/libhttp/Makefile b/libhttp/Makefile index 41a1bb0..eaaa3ec 100644 --- a/libhttp/Makefile +++ b/libhttp/Makefile @@ -13,15 +13,38 @@ CLIENT_LIBS := -L/opt/homebrew/lib -lssl -lcrypto -lcurl -lpthread BUILD_TDIR := build $(shell mkdir -p $(BUILD_TDIR)) +# Server object files (HttpServer 拆分后) +SERVER_OBJS := $(BUILD_TDIR)/HttpServer.o \ + $(BUILD_TDIR)/HttpServer_Listener.o \ + $(BUILD_TDIR)/HttpServer_Routes.o \ + $(BUILD_TDIR)/HttpServer_Request.o \ + $(BUILD_TDIR)/HttpServer_Ws.o \ + $(BUILD_TDIR)/HttpServer_WsSession.o + #-------- all: $(BUILD_TDIR)/$(LIB_NAME).so $(BUILD_TDIR)/$(LIBC_NAME).so # ---- Server library (HttpServer, no curl) ---- -$(BUILD_TDIR)/$(LIB_NAME).so: $(BUILD_TDIR)/HttpServer.o +$(BUILD_TDIR)/$(LIB_NAME).so: $(SERVER_OBJS) $(CPP) -shared $^ $(SERVER_LIBS) -o $@ -$(BUILD_TDIR)/HttpServer.o: src/HttpServer.cpp - $(CPP) $(CPPFLAGS) -fPIC -c $^ $(INCS) -o $@ +$(BUILD_TDIR)/HttpServer.o: src/HttpServer.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ + +$(BUILD_TDIR)/HttpServer_Listener.o: src/HttpServer_Listener.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ + +$(BUILD_TDIR)/HttpServer_Routes.o: src/HttpServer_Routes.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ + +$(BUILD_TDIR)/HttpServer_Request.o: src/HttpServer_Request.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ + +$(BUILD_TDIR)/HttpServer_Ws.o: src/HttpServer_Ws.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ + +$(BUILD_TDIR)/HttpServer_WsSession.o: src/HttpServer_WsSession.cpp inc/us/detail/HttpServerImpl.hpp + $(CPP) $(CPPFLAGS) -fPIC -c $< $(INCS) -o $@ # ---- Client library (HttpClient, depends on curl + beast websocket) ---- $(BUILD_TDIR)/$(LIBC_NAME).so: $(BUILD_TDIR)/HttpClient.o @@ -36,8 +59,13 @@ clear: install: all @echo "install UniServe_libhttp..." @mkdir -vp $(INCLU_TDIR)/$(FRAMEWORK_NAME) + @mkdir -vp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/detail @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/HttpServer.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/ @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/HttpClient.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/ + @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/WebSocketServer.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/ + @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/WebSocketClient.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/ + @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/WebSocketClientBase.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/ + @cp -vf $(INCLU_SDIR)/$(FRAMEWORK_NAME)/detail/HttpServerImpl.hpp $(INCLU_TDIR)/$(FRAMEWORK_NAME)/detail/ @cp -vf $(BUILD_TDIR)/$(LIB_NAME).so $(INSTL_TDIR)/ @cp -vf $(BUILD_TDIR)/$(LIBC_NAME).so $(INSTL_TDIR)/ @echo "done." @@ -46,6 +74,10 @@ uninstall: @echo "uninstall UniServe_libhttp..." @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/HttpServer.hpp @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/HttpClient.hpp + @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/WebSocketServer.hpp + @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/WebSocketClient.hpp + @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/WebSocketClientBase.hpp + @rm -vf $(INCLU_TDIR)/$(FRAMEWORK_NAME)/detail/HttpServerImpl.hpp @rm -vf $(INSTL_TDIR)/$(LIB_NAME).so @rm -vf $(INSTL_TDIR)/$(LIBC_NAME).so @echo "done." diff --git a/libhttp/README.md b/libhttp/README.md index 4813da3..f96a371 100644 --- a/libhttp/README.md +++ b/libhttp/README.md @@ -46,6 +46,23 @@ Two shared libraries are produced: `libus_http.so` does **not** depend on libcurl. Link against `libus_httpclient.so` only when you need the client, and link both when you need both server and client (e.g. a proxy/gateway). +### Source Layout + +The server implementation is split into modular source files for faster incremental builds: + +| Source File | Responsibility | +|---|---| +| `src/HttpServer.cpp` | Core lifecycle (construct/destruct/start/stop/run), SSL init, session management, debug utilities | +| `src/HttpServer_Listener.cpp` | TCP connection acceptor (`Listener` class) | +| `src/HttpServer_Routes.cpp` | HTTP route registration, route matching, wildcard→regex conversion, query parameter parsing | +| `src/HttpServer_Request.cpp` | Request dispatch pipeline, static file serving, CORS preflight, Session HTTP read/write | +| `src/HttpServer_Ws.cpp` | WebSocket route registration, client registry, broadcast/send APIs | +| `src/HttpServer_WsSession.cpp` | Session WebSocket handling (regular WS frames + bidirectional WS proxy bridging) | + +Internal shared definitions (exception classes, full `Session`/`Listener` declarations) live in `inc/us/detail/HttpServerImpl.hpp` (installed to `/usr/include/us/detail/`). + +Modifying one module only requires recompiling its corresponding `.o` file — no need to rebuild the entire 1400-line translation unit. + ## Quick Start ### HTTP + WebSocket Server @@ -260,6 +277,7 @@ using WebSocketClientBase = HttpClient; ## Version History +- **v3.3.0**: Split HttpServer into 6 modular translation units by functional area (core/lifecycle, listener, routes, request dispatch, WS management, WS session/proxy) for dramatically faster incremental builds. Added internal detail header `inc/us/detail/HttpServerImpl.hpp` for shared definitions across TU boundaries. - **v3.2.0**: Split into two shared libraries — `libus_http.so` (server only, no curl) and `libus_httpclient.so` (HttpClient, depends on curl). HttpClient implementation moved to `src/HttpClient.cpp`; `HttpClient.hpp` is declaration-only. - **v3.1.0**: Added WebSocket reverse proxy (`ws_proxy()`) with bidirectional async bridging, ws:// and wss:// backend support, handshake-time authorization callback. - **v3.0.0**: Merged WebSocketServer into HttpServer and WebSocketClient into HttpClient. Single-port HTTP+WebSocket support. diff --git a/libhttp/inc/us/detail/HttpServerImpl.hpp b/libhttp/inc/us/detail/HttpServerImpl.hpp new file mode 100644 index 0000000..16393b0 --- /dev/null +++ b/libhttp/inc/us/detail/HttpServerImpl.hpp @@ -0,0 +1,180 @@ +#ifndef HTTPSERVER_IMPL_HPP +#define HTTPSERVER_IMPL_HPP + +#include "us/HttpServer.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace pmc { +namespace net { + +// ===================== 自定义异常 ===================== +class HttpServerException : public std::runtime_error { +public: + explicit HttpServerException(const std::string& msg) : runtime_error(msg) {} +}; +class HttpListenerException : public HttpServerException { +public: + explicit HttpListenerException(const std::string& msg) : HttpServerException(msg) {} +}; +class SSLInitException : public HttpServerException { +public: + explicit SSLInitException(const std::string& msg) : HttpServerException(msg) {} +}; +class CreateObjectFailed : public HttpServerException { +public: + explicit CreateObjectFailed(const std::string& msg) : HttpServerException(msg) {} +}; + +// ===================== Session (完整定义) ===================== +class HttpServer::Session : public std::enable_shared_from_this { + using tcp_stream = beast::tcp_stream; + using ssl_stream = asio::ssl::stream; + using ws_stream = websocket::stream; + using wss_stream = websocket::stream; + + // Proxy outbound types + using proxy_ws_stream = websocket::stream; + using proxy_wss_stream = websocket::stream&>; + +public: + Session(tcp::socket&& socket, HttpServer* server) + : stream_(std::move(socket)), server_(server), ssl_enabled_(false), + is_websocket_(false), is_ws_proxy_(false), client_id_("") {} + + Session(tcp::socket&& socket, asio::ssl::context& ssl_ctx, HttpServer* server) + : stream_(std::move(socket)), server_(server), ssl_enabled_(true), + is_websocket_(false), is_ws_proxy_(false), client_id_("") { + ssl_stream_ = std::make_unique(stream_, ssl_ctx); + } + + ~Session() { close(); } + + void run(); + void setRemoteInfo(const std::string& ip, unsigned short port); + void wsSend(const std::string& message); + std::string getClientId() const { return client_id_; } + std::string getRemoteAddress() const { return remote_ip_ + ":" + std::to_string(remote_port_); } + bool isWebSocket() const { return is_websocket_; } + +private: + void close(); + void doSSLHandshake(); + void doRead(); + void handleRead(beast::error_code ec); + bool isWebSocketUpgrade(const http::request& req); + void doWsUpgrade(); + void doWssUpgrade(); + void onWsAccept(beast::error_code ec); + + // Regular WebSocket + void doWsRead(); + void onWsRead(beast::error_code ec, std::size_t bytes_transferred); + void doWsWrite(); + void onWsClose(); + + // WebSocket Proxy + static void parseProxyUrl(const std::string& url, bool& ssl, std::string& host, + std::string& port, std::string& path); + void doProxyConnect(const std::string& target_url); + void onProxyResolve(beast::error_code ec, tcp::resolver::results_type results); + void onProxyConnect(beast::error_code ec, tcp::endpoint ep); + void onProxySslHandshake(beast::error_code ec); + void onProxyHandshake(beast::error_code ec); + + // Client → Backend + void doProxyReadClient(); + void onProxyReadClient(beast::error_code ec, std::size_t bytes_transferred); + void doProxyWriteBackend(); + + // Backend → Client + void doProxyReadBackend(); + void onProxyReadBackend(beast::error_code ec, std::size_t bytes_transferred); + void doProxyWriteClient(); + + void closeProxy(websocket::close_code code); + + // Regular HTTP + void processRequest(http::request req); + void doWrite(std::shared_ptr> response); + + // Client stream (inbound) + tcp_stream stream_; + std::unique_ptr ssl_stream_; + std::unique_ptr ws_stream_; + std::unique_ptr wss_stream_; + + beast::flat_buffer buffer_; + beast::flat_buffer ws_buffer_; + http::request request_; + + HttpServer* server_; + bool ssl_enabled_; + + std::string remote_ip_ = "unknown"; + unsigned short remote_port_ = 0; + + bool is_websocket_; + bool is_ws_proxy_; + std::string client_id_; + std::string ws_path_; + WebSocketMessageHandler ws_handler_; + std::deque ws_send_queue_; + + // Proxy state (outbound to backend) + std::string ws_proxy_target_; + bool proxy_ssl_ = false; + bool proxy_connected_ = false; + bool proxy_closed_ = false; + std::string proxy_host_, proxy_port_, proxy_path_; + + std::unique_ptr proxy_resolver_; + std::unique_ptr proxy_socket_; + std::unique_ptr proxy_tcp_stream_; + std::unique_ptr proxy_ssl_ctx_; + std::unique_ptr> proxy_ssl_stream_; + std::unique_ptr proxy_ws_stream_; + std::unique_ptr proxy_wss_stream_; + + beast::flat_buffer proxy_client_buffer_; + beast::flat_buffer proxy_backend_buffer_; + std::deque> proxy_to_backend_queue_; + std::deque> proxy_to_client_queue_; +}; + +// ===================== Listener (完整定义) ===================== +class HttpServer::Listener : public std::enable_shared_from_this { +public: + Listener(asio::io_context& ioc, tcp::endpoint endpoint, HttpServer* server) + : ioc_(ioc), acceptor_(ioc), server_(server), ssl_enabled_(false) { + initAcceptor(endpoint); + } + + Listener(asio::io_context& ioc, tcp::endpoint endpoint, + asio::ssl::context& ssl_ctx, HttpServer* server) + : ioc_(ioc), acceptor_(ioc), ssl_ctx_(&ssl_ctx), server_(server), ssl_enabled_(true) { + initAcceptor(endpoint); + } + + void run(); + +private: + void initAcceptor(tcp::endpoint ep); + void doAccept(); + + asio::io_context& ioc_; + tcp::acceptor acceptor_; + asio::ssl::context* ssl_ctx_ = nullptr; + HttpServer* server_; + bool ssl_enabled_; +}; + +} // namespace net +} // namespace pmc + +#endif // HTTPSERVER_IMPL_HPP diff --git a/libhttp/src/HttpServer.cpp b/libhttp/src/HttpServer.cpp index aaa9580..e16811e 100644 --- a/libhttp/src/HttpServer.cpp +++ b/libhttp/src/HttpServer.cpp @@ -1,860 +1,9 @@ -#include "us/HttpServer.hpp" -#include -#include -#include -#include -#include -#include -#include +#include "us/detail/HttpServerImpl.hpp" namespace pmc { namespace net { -// ===================== 自定义异常 ===================== -class HttpServerException : public std::runtime_error { -public: - explicit HttpServerException(const std::string& msg) : runtime_error(msg) {} -}; -class HttpListenerException : public HttpServerException { -public: - explicit HttpListenerException(const std::string& msg) : HttpServerException(msg) {} -}; -class SSLInitException : public HttpServerException { -public: - explicit SSLInitException(const std::string& msg) : HttpServerException(msg) {} -}; -class CreateObjectFailed : public HttpServerException { -public: - explicit CreateObjectFailed(const std::string& msg) : HttpServerException(msg) {} -}; - -// ===================== Session ===================== -class HttpServer::Session : public std::enable_shared_from_this { - using tcp_stream = beast::tcp_stream; - using ssl_stream = asio::ssl::stream; - using ws_stream = websocket::stream; - using wss_stream = websocket::stream; - - // Proxy outbound types - using proxy_ws_stream = websocket::stream; - using proxy_wss_stream = websocket::stream&>; - -public: - Session(tcp::socket&& socket, HttpServer* server) - : stream_(std::move(socket)), server_(server), ssl_enabled_(false), - is_websocket_(false), is_ws_proxy_(false), client_id_("") {} - - Session(tcp::socket&& socket, asio::ssl::context& ssl_ctx, HttpServer* server) - : stream_(std::move(socket)), server_(server), ssl_enabled_(true), - is_websocket_(false), is_ws_proxy_(false), client_id_("") { - ssl_stream_ = std::make_unique(stream_, ssl_ctx); - } - - ~Session() { close(); } - - void run() { - if (ssl_enabled_) { - doSSLHandshake(); - } else { - doRead(); - } - } - - void setRemoteInfo(const std::string& ip, unsigned short port) { - remote_ip_ = ip; - remote_port_ = port; - } - - void wsSend(const std::string& message) { - auto exec = stream_.get_executor(); - asio::post(exec, [self = shared_from_this(), message]() { - self->ws_send_queue_.push_back(message); - if (self->ws_send_queue_.size() == 1) { - self->doWsWrite(); - } - }); - } - - std::string getClientId() const { return client_id_; } - std::string getRemoteAddress() const { return remote_ip_ + ":" + std::to_string(remote_port_); } - bool isWebSocket() const { return is_websocket_; } - -private: - void close() { - beast::error_code ec; - closeProxy(websocket::close_code::going_away); - if (is_websocket_) { - if (ssl_enabled_ && wss_stream_) { - wss_stream_->close(websocket::close_code::normal, ec); - } else if (ws_stream_) { - ws_stream_->close(websocket::close_code::normal, ec); - } - } - if (ssl_stream_) { - ssl_stream_->shutdown(ec); - } - stream_.socket().shutdown(tcp::socket::shutdown_send, ec); - stream_.close(); - } - - void doSSLHandshake() { - auto self = shared_from_this(); - std::cout << "[SSL] Starting handshake for " - << remote_ip_ << ":" << remote_port_ << std::endl; - - ssl_stream_->async_handshake( - asio::ssl::stream_base::server, - [self](beast::error_code ec) { - if (ec) { - std::cerr << "[SSL] Handshake error: " - << self->remote_ip_ << ":" << self->remote_port_ - << " - " << ec.message() - << " (code=" << ec.value() << ")" << std::endl; - ERR_print_errors_fp(stderr); - return; - } - std::cout << "[SSL] Handshake succeeded: " - << self->remote_ip_ << ":" << self->remote_port_ << std::endl; - self->doRead(); - }); - } - - void doRead() { - auto self = shared_from_this(); - request_ = {}; - buffer_.consume(buffer_.size()); - - if (ssl_enabled_ && ssl_stream_) { - http::async_read(*ssl_stream_, buffer_, request_, - [self](beast::error_code ec, std::size_t) { - self->handleRead(ec); - }); - } else { - http::async_read(stream_, buffer_, request_, - [self](beast::error_code ec, std::size_t) { - self->handleRead(ec); - }); - } - } - - void handleRead(beast::error_code ec) { - if (!ec) { - if (isWebSocketUpgrade(request_)) { - std::string path = std::string(request_.target()); - auto qpos = path.find('?'); - if (qpos != std::string::npos) path = path.substr(0, qpos); - - // Check proxy routes first (higher priority) - auto proxy_handshake = server_->findWsProxyRoute(path); - if (proxy_handshake) { - std::string target = proxy_handshake(request_); - if (target.empty()) { - // Rejected by handshake callback → 403 - auto resp = std::make_shared>(); - resp->version(request_.version()); - resp->result(http::status::forbidden); - resp->set(http::field::content_type, "text/plain"); - resp->set(http::field::connection, "close"); - resp->body() = "403 Forbidden\n"; - resp->prepare_payload(); - doWrite(resp); - return; - } - ws_path_ = path; - is_ws_proxy_ = true; - ws_proxy_target_ = target; - if (ssl_enabled_) { - doWssUpgrade(); - } else { - doWsUpgrade(); - } - return; - } - - auto ws_handler = server_->findWsRoute(path); - if (ws_handler) { - ws_handler_ = ws_handler; - ws_path_ = path; - if (ssl_enabled_) { - doWssUpgrade(); - } else { - doWsUpgrade(); - } - return; - } - } - - auto req_copy = std::move(request_); - processRequest(std::move(req_copy)); - } else if (ec != http::error::end_of_stream) { - std::cerr << "[Read] Error from " << remote_ip_ << ":" << remote_port_ - << ": " << ec.message() << std::endl; - } - } - - bool isWebSocketUpgrade(const http::request& req) { - if (req.method() != http::verb::get) return false; - - auto upgrade = req.find(http::field::upgrade); - if (upgrade == req.end()) return false; - - std::string upgrade_val(upgrade->value()); - boost::to_lower(upgrade_val); - boost::trim(upgrade_val); - if (upgrade_val != "websocket") return false; - - auto connection = req.find(http::field::connection); - if (connection == req.end()) return false; - - std::string conn_val(connection->value()); - boost::to_lower(conn_val); - return conn_val.find("upgrade") != std::string::npos; - } - - void doWsUpgrade() { - auto self = shared_from_this(); - ws_stream_ = std::make_unique(stream_); - ws_stream_->text(true); - - ws_stream_->async_accept( - request_, - [self](beast::error_code ec) { - self->onWsAccept(ec); - }); - } - - void doWssUpgrade() { - auto self = shared_from_this(); - wss_stream_ = std::make_unique(*ssl_stream_); - wss_stream_->text(true); - - wss_stream_->async_accept( - request_, - [self](beast::error_code ec) { - self->onWsAccept(ec); - }); - } - - void onWsAccept(beast::error_code ec) { - if (ec) { - std::cerr << "[WS] Accept error from " << remote_ip_ << ":" << remote_port_ - << ": " << ec.message() << std::endl; - return; - } - - is_websocket_ = true; - client_id_ = server_->generateWsClientId(); - server_->addWsClient(client_id_, shared_from_this()); - - std::cout << "[WS] Client " << client_id_ << " connected from " - << remote_ip_ << ":" << remote_port_ << " at " << ws_path_ << std::endl; - - if (is_ws_proxy_) { - std::cout << "[WS-Proxy] Proxying " << ws_path_ << " -> " << ws_proxy_target_ << std::endl; - doProxyConnect(ws_proxy_target_); - return; - } - - if (server_->ws_connect_handler_) { - server_->ws_connect_handler_(client_id_, getRemoteAddress()); - } - - doWsRead(); - } - - // ======== Regular WebSocket message handling ======== - - void doWsRead() { - auto self = shared_from_this(); - ws_buffer_.consume(ws_buffer_.size()); - - if (ssl_enabled_ && wss_stream_) { - wss_stream_->async_read( - ws_buffer_, - [self](beast::error_code ec, std::size_t bytes_transferred) { - self->onWsRead(ec, bytes_transferred); - }); - } else if (ws_stream_) { - ws_stream_->async_read( - ws_buffer_, - [self](beast::error_code ec, std::size_t bytes_transferred) { - self->onWsRead(ec, bytes_transferred); - }); - } - } - - void onWsRead(beast::error_code ec, std::size_t bytes_transferred) { - if (ec == websocket::error::closed) { - onWsClose(); - return; - } - if (ec) { - std::cerr << "[WS] Read error from " << client_id_ << ": " << ec.message() << std::endl; - onWsClose(); - return; - } - - std::string message = beast::buffers_to_string(ws_buffer_.data()); - ws_buffer_.consume(bytes_transferred); - - if (ws_handler_) { - auto self = shared_from_this(); - ws_handler_(client_id_, message, - [self](const std::string& response) { - self->wsSend(response); - }); - } - - doWsRead(); - } - - void doWsWrite() { - if (ws_send_queue_.empty()) return; - - auto self = shared_from_this(); - std::string& message = ws_send_queue_.front(); - - auto write_handler = [self](beast::error_code ec, std::size_t) { - if (ec) { - std::cerr << "[WS] Write error to " << self->client_id_ << ": " << ec.message() << std::endl; - self->onWsClose(); - return; - } - self->ws_send_queue_.pop_front(); - if (!self->ws_send_queue_.empty()) { - self->doWsWrite(); - } - }; - - if (ssl_enabled_ && wss_stream_) { - wss_stream_->text(true); - wss_stream_->async_write(asio::buffer(message), write_handler); - } else if (ws_stream_) { - ws_stream_->text(true); - ws_stream_->async_write(asio::buffer(message), write_handler); - } - } - - void onWsClose() { - if (client_id_.empty()) return; - std::cout << "[WS] Client " << client_id_ << " disconnected" << std::endl; - - closeProxy(websocket::close_code::normal); - server_->removeWsClient(client_id_); - - if (server_->ws_disconnect_handler_) { - server_->ws_disconnect_handler_(client_id_); - } - client_id_.clear(); - is_websocket_ = false; - is_ws_proxy_ = false; - } - - // ======== WebSocket Proxy (bidirectional bridge) ======== - - static void parseProxyUrl(const std::string& url, bool& ssl, std::string& host, - std::string& port, std::string& path) { - std::string rest; - if (url.size() >= 6 && url.substr(0, 6) == "wss://") { - ssl = true; - rest = url.substr(6); - } else if (url.size() >= 5 && url.substr(0, 5) == "ws://") { - ssl = false; - rest = url.substr(5); - } else { - ssl = false; - rest = url; - } - - size_t path_pos = rest.find('/'); - std::string host_port; - if (path_pos != std::string::npos) { - host_port = rest.substr(0, path_pos); - path = rest.substr(path_pos); - } else { - host_port = rest; - path = "/"; - } - - size_t port_pos = host_port.find(':'); - if (port_pos != std::string::npos) { - host = host_port.substr(0, port_pos); - port = host_port.substr(port_pos + 1); - } else { - host = host_port; - port = ssl ? "443" : "80"; - } - } - - void doProxyConnect(const std::string& target_url) { - auto self = shared_from_this(); - bool ssl; - std::string host, port, path; - parseProxyUrl(target_url, ssl, host, port, path); - proxy_ssl_ = ssl; - proxy_host_ = host; - proxy_port_ = port; - proxy_path_ = path; - - auto exec = stream_.get_executor(); - proxy_resolver_ = std::make_unique(asio::make_strand(exec)); - - proxy_resolver_->async_resolve(host, port, - asio::bind_executor(exec, - [self](beast::error_code ec, tcp::resolver::results_type results) { - self->onProxyResolve(ec, results); - })); - } - - void onProxyResolve(beast::error_code ec, tcp::resolver::results_type results) { - if (ec) { - std::cerr << "[WS-Proxy] Resolve error: " << ec.message() << std::endl; - close(); - return; - } - - auto self = shared_from_this(); - auto exec = stream_.get_executor(); - - proxy_socket_ = std::make_unique(exec); - proxy_tcp_stream_ = std::make_unique(std::move(*proxy_socket_)); - proxy_socket_.reset(); - - proxy_tcp_stream_->expires_after(std::chrono::seconds(10)); - asio::async_connect(proxy_tcp_stream_->socket(), results, - asio::bind_executor(exec, - [self](beast::error_code ec, tcp::endpoint ep) { - self->onProxyConnect(ec, ep); - })); - } - - void onProxyConnect(beast::error_code ec, tcp::endpoint /*ep*/) { - if (ec) { - std::cerr << "[WS-Proxy] Connect error: " << ec.message() << std::endl; - close(); - return; - } - - auto self = shared_from_this(); - if (proxy_ssl_) { - proxy_ssl_ctx_ = std::make_unique(asio::ssl::context::tlsv12_client); - proxy_ssl_ctx_->set_verify_mode(asio::ssl::verify_none); - proxy_ssl_stream_ = std::make_unique>( - proxy_tcp_stream_->socket(), *proxy_ssl_ctx_); - - if (!SSL_set_tlsext_host_name(proxy_ssl_stream_->native_handle(), proxy_host_.c_str())) { - std::cerr << "[WS-Proxy] Failed to set SNI" << std::endl; - close(); - return; - } - - proxy_tcp_stream_->expires_after(std::chrono::seconds(10)); - auto exec = stream_.get_executor(); - proxy_ssl_stream_->async_handshake(asio::ssl::stream_base::client, - asio::bind_executor(exec, - [self](beast::error_code ec) { - self->onProxySslHandshake(ec); - })); - } else { - proxy_ws_stream_ = std::make_unique(proxy_tcp_stream_->socket()); - proxy_tcp_stream_->expires_never(); - - proxy_ws_stream_->set_option(websocket::stream_base::decorator( - [](websocket::request_type& req) { - req.set(http::field::user_agent, "UniServe-WsProxy"); - })); - - auto exec = self->stream_.get_executor(); - proxy_ws_stream_->async_handshake(proxy_host_, proxy_path_, - asio::bind_executor(exec, - [self](beast::error_code ec) { - self->onProxyHandshake(ec); - })); - } - } - - void onProxySslHandshake(beast::error_code ec) { - if (ec) { - std::cerr << "[WS-Proxy] SSL handshake error: " << ec.message() << std::endl; - close(); - return; - } - - auto self = shared_from_this(); - proxy_wss_stream_ = std::make_unique(*proxy_ssl_stream_); - proxy_tcp_stream_->expires_never(); - - proxy_wss_stream_->set_option(websocket::stream_base::decorator( - [](websocket::request_type& req) { - req.set(http::field::user_agent, "UniServe-WsProxy"); - })); - - auto exec = stream_.get_executor(); - proxy_wss_stream_->async_handshake(proxy_host_, proxy_path_, - asio::bind_executor(exec, - [self](beast::error_code ec) { - self->onProxyHandshake(ec); - })); - } - - void onProxyHandshake(beast::error_code ec) { - if (ec) { - std::cerr << "[WS-Proxy] Backend handshake error: " << ec.message() << std::endl; - close(); - return; - } - - proxy_connected_ = true; - proxy_closed_ = false; - std::cout << "[WS-Proxy] Connected to backend " << proxy_host_ << ":" << proxy_port_ - << proxy_path_ << std::endl; - - if (server_->ws_connect_handler_) { - server_->ws_connect_handler_(client_id_, getRemoteAddress()); - } - - // Start bidirectional bridging - doProxyReadClient(); - doProxyReadBackend(); - } - - // ---- Client → Backend direction ---- - - void doProxyReadClient() { - if (proxy_closed_) return; - auto self = shared_from_this(); - proxy_client_buffer_.consume(proxy_client_buffer_.size()); - - auto read_handler = [self](beast::error_code ec, std::size_t bytes_transferred) { - self->onProxyReadClient(ec, bytes_transferred); - }; - - if (ssl_enabled_ && wss_stream_) { - wss_stream_->async_read(proxy_client_buffer_, read_handler); - } else if (ws_stream_) { - ws_stream_->async_read(proxy_client_buffer_, read_handler); - } - } - - void onProxyReadClient(beast::error_code ec, std::size_t bytes_transferred) { - if (ec) { - if (ec != websocket::error::closed) { - std::cerr << "[WS-Proxy] Client read error: " << ec.message() << std::endl; - } - closeProxy(websocket::close_code::normal); - return; - } - - bool is_binary = false; - if (ssl_enabled_ && wss_stream_) { - is_binary = wss_stream_->got_binary(); - } else if (ws_stream_) { - is_binary = ws_stream_->got_binary(); - } - - std::string data = beast::buffers_to_string(proxy_client_buffer_.data()); - proxy_client_buffer_.consume(bytes_transferred); - - // Queue to backend - auto exec = stream_.get_executor(); - asio::post(exec, [self = shared_from_this(), data = std::move(data), is_binary]() { - self->proxy_to_backend_queue_.emplace_back(std::move(data), is_binary); - if (self->proxy_to_backend_queue_.size() == 1) { - self->doProxyWriteBackend(); - } - }); - - doProxyReadClient(); - } - - void doProxyWriteBackend() { - if (proxy_to_backend_queue_.empty() || !proxy_connected_ || proxy_closed_) return; - - auto self = shared_from_this(); - auto& [msg, is_binary] = proxy_to_backend_queue_.front(); - - auto write_handler = [self](beast::error_code ec, std::size_t) { - if (ec) { - std::cerr << "[WS-Proxy] Backend write error: " << ec.message() << std::endl; - self->closeProxy(websocket::close_code::internal_error); - return; - } - self->proxy_to_backend_queue_.pop_front(); - if (!self->proxy_to_backend_queue_.empty()) { - self->doProxyWriteBackend(); - } - }; - - if (proxy_ssl_ && proxy_wss_stream_) { - proxy_wss_stream_->binary(is_binary); - proxy_wss_stream_->async_write(asio::buffer(msg), write_handler); - } else if (proxy_ws_stream_) { - proxy_ws_stream_->binary(is_binary); - proxy_ws_stream_->async_write(asio::buffer(msg), write_handler); - } - } - - // ---- Backend → Client direction ---- - - void doProxyReadBackend() { - if (proxy_closed_) return; - auto self = shared_from_this(); - proxy_backend_buffer_.consume(proxy_backend_buffer_.size()); - - auto read_handler = [self](beast::error_code ec, std::size_t bytes_transferred) { - self->onProxyReadBackend(ec, bytes_transferred); - }; - - if (proxy_ssl_ && proxy_wss_stream_) { - proxy_wss_stream_->async_read(proxy_backend_buffer_, read_handler); - } else if (proxy_ws_stream_) { - proxy_ws_stream_->async_read(proxy_backend_buffer_, read_handler); - } - } - - void onProxyReadBackend(beast::error_code ec, std::size_t bytes_transferred) { - if (ec) { - if (ec != websocket::error::closed) { - std::cerr << "[WS-Proxy] Backend read error: " << ec.message() << std::endl; - } - closeProxy(websocket::close_code::normal); - return; - } - - bool is_binary = false; - if (proxy_ssl_ && proxy_wss_stream_) { - is_binary = proxy_wss_stream_->got_binary(); - } else if (proxy_ws_stream_) { - is_binary = proxy_ws_stream_->got_binary(); - } - - std::string data = beast::buffers_to_string(proxy_backend_buffer_.data()); - proxy_backend_buffer_.consume(bytes_transferred); - - // Queue to client - auto exec = stream_.get_executor(); - asio::post(exec, [self = shared_from_this(), data = std::move(data), is_binary]() { - self->proxy_to_client_queue_.emplace_back(std::move(data), is_binary); - if (self->proxy_to_client_queue_.size() == 1) { - self->doProxyWriteClient(); - } - }); - - doProxyReadBackend(); - } - - void doProxyWriteClient() { - if (proxy_to_client_queue_.empty() || !proxy_connected_ || proxy_closed_) return; - - auto self = shared_from_this(); - auto& [msg, is_binary] = proxy_to_client_queue_.front(); - - auto write_handler = [self](beast::error_code ec, std::size_t) { - if (ec) { - std::cerr << "[WS-Proxy] Client write error: " << ec.message() << std::endl; - self->closeProxy(websocket::close_code::internal_error); - return; - } - self->proxy_to_client_queue_.pop_front(); - if (!self->proxy_to_client_queue_.empty()) { - self->doProxyWriteClient(); - } - }; - - if (ssl_enabled_ && wss_stream_) { - wss_stream_->binary(is_binary); - wss_stream_->async_write(asio::buffer(msg), write_handler); - } else if (ws_stream_) { - ws_stream_->binary(is_binary); - ws_stream_->async_write(asio::buffer(msg), write_handler); - } - } - - void closeProxy(websocket::close_code code) { - if (proxy_closed_) return; - proxy_closed_ = true; - proxy_connected_ = false; - - beast::error_code ec; - // Close backend connection - if (proxy_ssl_ && proxy_wss_stream_) { - proxy_wss_stream_->close(code, ec); - } else if (proxy_ws_stream_) { - proxy_ws_stream_->close(code, ec); - } - if (proxy_ssl_stream_) { - proxy_ssl_stream_->shutdown(ec); - } - if (proxy_tcp_stream_) { - proxy_tcp_stream_->socket().shutdown(tcp::socket::shutdown_send, ec); - proxy_tcp_stream_->close(); - } - - proxy_to_backend_queue_.clear(); - proxy_to_client_queue_.clear(); - proxy_ws_stream_.reset(); - proxy_wss_stream_.reset(); - proxy_ssl_stream_.reset(); - proxy_ssl_ctx_.reset(); - proxy_tcp_stream_.reset(); - proxy_resolver_.reset(); - } - - // ======== Regular HTTP ======== - - void processRequest(http::request req) { - try { - auto resp = std::make_shared>( - server_->handleRequest(req)); - doWrite(resp); - } catch (const std::exception& e) { - std::cerr << "[Request] Handler exception: " << e.what() << std::endl; - auto err_resp = std::make_shared>(); - err_resp->version(req.version()); - err_resp->result(http::status::internal_server_error); - err_resp->set(http::field::content_type, "text/plain"); - err_resp->body() = "500 Internal Server Error\n"; - err_resp->prepare_payload(); - doWrite(err_resp); - } - } - - void doWrite(std::shared_ptr> response) { - auto self = shared_from_this(); - if (ssl_enabled_ && ssl_stream_) { - http::async_write(*ssl_stream_, *response, - [self, response](beast::error_code ec, std::size_t) { - if (ec) { - std::cerr << "[Write] Error to " << self->remote_ip_ << ":" << self->remote_port_ - << ": " << ec.message() << std::endl; - return; - } - if (!response->keep_alive()) self->close(); - else self->doRead(); - }); - } else { - http::async_write(stream_, *response, - [self, response](beast::error_code ec, std::size_t) { - if (ec) { - std::cerr << "[Write] Error to " << self->remote_ip_ << ":" << self->remote_port_ - << ": " << ec.message() << std::endl; - return; - } - if (!response->keep_alive()) self->close(); - else self->doRead(); - }); - } - } - - // ---- Client stream (inbound) ---- - tcp_stream stream_; - std::unique_ptr ssl_stream_; - std::unique_ptr ws_stream_; - std::unique_ptr wss_stream_; - - beast::flat_buffer buffer_; - beast::flat_buffer ws_buffer_; - http::request request_; - - HttpServer* server_; - bool ssl_enabled_; - - std::string remote_ip_ = "unknown"; - unsigned short remote_port_ = 0; - - bool is_websocket_; - bool is_ws_proxy_; - std::string client_id_; - std::string ws_path_; - WebSocketMessageHandler ws_handler_; - std::deque ws_send_queue_; - - // ---- Proxy state (outbound to backend) ---- - std::string ws_proxy_target_; - bool proxy_ssl_ = false; - bool proxy_connected_ = false; - bool proxy_closed_ = false; - std::string proxy_host_, proxy_port_, proxy_path_; - - std::unique_ptr proxy_resolver_; - std::unique_ptr proxy_socket_; - std::unique_ptr proxy_tcp_stream_; - std::unique_ptr proxy_ssl_ctx_; - std::unique_ptr> proxy_ssl_stream_; - std::unique_ptr proxy_ws_stream_; - std::unique_ptr proxy_wss_stream_; - - beast::flat_buffer proxy_client_buffer_; - beast::flat_buffer proxy_backend_buffer_; - std::deque> proxy_to_backend_queue_; - std::deque> proxy_to_client_queue_; -}; - -// ===================== Listener ===================== -class HttpServer::Listener : public std::enable_shared_from_this { -public: - Listener(asio::io_context& ioc, tcp::endpoint endpoint, HttpServer* server) - : ioc_(ioc), acceptor_(ioc), server_(server), ssl_enabled_(false) { - initAcceptor(endpoint); - } - - Listener(asio::io_context& ioc, tcp::endpoint endpoint, - asio::ssl::context& ssl_ctx, HttpServer* server) - : ioc_(ioc), acceptor_(ioc), ssl_ctx_(&ssl_ctx), server_(server), ssl_enabled_(true) { - initAcceptor(endpoint); - } - - void run() { - if (acceptor_.is_open()) doAccept(); - } - -private: - void initAcceptor(tcp::endpoint ep) { - beast::error_code ec; - acceptor_.open(ep.protocol(), ec); - if (!ec) { - acceptor_.set_option(asio::socket_base::reuse_address(true), ec); - acceptor_.bind(ep, ec); - if (!ec) acceptor_.listen(asio::socket_base::max_listen_connections, ec); - } - if (ec) throw HttpListenerException("Listener error: " + ec.message()); - } - - void doAccept() { - auto self = shared_from_this(); - acceptor_.async_accept( - asio::make_strand(ioc_), - [self](beast::error_code ec, tcp::socket socket) { - if (!ec) { - auto ep = socket.remote_endpoint(ec); - std::string ip = ec ? "unknown" : ep.address().to_string(); - unsigned short port = ec ? 0 : ep.port(); - std::cout << "[Listener] New connection from " << ip << ":" << port << std::endl; - - std::shared_ptr session; - if (self->ssl_enabled_ && self->ssl_ctx_) { - session = std::make_shared(std::move(socket), *self->ssl_ctx_, self->server_); - } else { - session = std::make_shared(std::move(socket), self->server_); - } - session->setRemoteInfo(ip, port); - self->server_->addSession(session); - session->run(); - } else { - std::cerr << "[Listener] Accept error: " << ec.message() << std::endl; - } - self->doAccept(); - }); - } - - asio::io_context& ioc_; - tcp::acceptor acceptor_; - asio::ssl::context* ssl_ctx_ = nullptr; - HttpServer* server_; - bool ssl_enabled_; -}; - -// ===================== HttpServer 实现 ===================== +// ===================== HttpServer 核心生命周期 ===================== HttpServer::HttpServer(const std::string& address, unsigned short port, unsigned int threads) : address_(address), port_(port), threads_(threads), ssl_enabled_(false), @@ -973,354 +122,34 @@ unsigned short HttpServer::getPort() const { return port_; } std::string HttpServer::getAddress() const { return address_; } bool HttpServer::isSSLEnabled() const { return ssl_enabled_; } -// ---- HTTP 路由注册 ---- -void HttpServer::get(const std::string& path, HttpRequestHandler handler) { - RouteEntry entry; - entry.path = path; - entry.handler = handler; - entry.is_wildcard = (path.find('*') != std::string::npos || - path.find('?') != std::string::npos); - if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); - get_handlers_.push_back(entry); +// ---- 会话管理 ---- +void HttpServer::addSession(const std::shared_ptr& session) { + std::lock_guard lock(sessions_mutex_); + sessions_.erase(std::remove_if(sessions_.begin(), sessions_.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), sessions_.end()); + sessions_.push_back(session); } -void HttpServer::post(const std::string& path, HttpRequestHandler handler) { - RouteEntry entry; - entry.path = path; - entry.handler = handler; - entry.is_wildcard = (path.find('*') != std::string::npos || - path.find('?') != std::string::npos); - if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); - post_handlers_.push_back(entry); +void HttpServer::removeSession(const Session* session) { + std::lock_guard lock(sessions_mutex_); + sessions_.erase(std::remove_if(sessions_.begin(), sessions_.end(), + [session](const std::weak_ptr& wp) { + auto sp = wp.lock(); + return !sp || sp.get() == session; + }), sessions_.end()); } -void HttpServer::put(const std::string& path, HttpRequestHandler handler) { - RouteEntry entry; - entry.path = path; - entry.handler = handler; - entry.is_wildcard = (path.find('*') != std::string::npos || - path.find('?') != std::string::npos); - if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); - put_handlers_.push_back(entry); +void HttpServer::closeAllSessions() { + std::lock_guard lock(sessions_mutex_); + sessions_.clear(); } -void HttpServer::del(const std::string& path, HttpRequestHandler handler) { - RouteEntry entry; - entry.path = path; - entry.handler = handler; - entry.is_wildcard = (path.find('*') != std::string::npos || - path.find('?') != std::string::npos); - if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); - delete_handlers_.push_back(entry); -} - -void HttpServer::options(const std::string& path, HttpRequestHandler handler) { - RouteEntry entry; - entry.path = path; - entry.handler = handler; - entry.is_wildcard = (path.find('*') != std::string::npos || - path.find('?') != std::string::npos); - if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); - options_handlers_.push_back(entry); -} - -void HttpServer::use(MiddlewareHandler m) { - middlewares_.push_back(m); -} - -void HttpServer::setStaticDirectory(const std::string& p) { - static_directory_ = p; -} - -// ---- WebSocket 路由注册 ---- -void HttpServer::ws(const std::string& path, WebSocketMessageHandler handler) { - WsRouteEntry entry; - entry.path = path; - entry.handler = handler; - ws_handlers_.push_back(entry); -} - -void HttpServer::ws_proxy(const std::string& path, WsProxyHandshakeHandler handshake) { - WsProxyRouteEntry entry; - entry.path = path; - entry.handshake = handshake; - ws_proxy_handlers_.push_back(entry); -} - -void HttpServer::onWsConnect(WebSocketConnectHandler handler) { - ws_connect_handler_ = handler; -} - -void HttpServer::onWsDisconnect(WebSocketDisconnectHandler handler) { - ws_disconnect_handler_ = handler; -} - -bool HttpServer::sendToWsClient(const std::string& clientId, const std::string& message) { - std::shared_ptr session; - { - std::lock_guard lock(ws_clients_mutex_); - auto it = ws_clients_.find(clientId); - if (it != ws_clients_.end()) { - session = it->second; - } +// ---- 调试工具 ---- +void HttpServer::debug::print_all_header_fields(const http::request& req) { + for (auto const& field : req) { + std::cout << "Name: " << field.name_string() + << ", Value: " << field.value() << std::endl; } - if (session) { - session->wsSend(message); - return true; - } - return false; -} - -size_t HttpServer::broadcastWs(const std::string& message) { - std::vector> clients; - { - std::lock_guard lock(ws_clients_mutex_); - for (auto& pair : ws_clients_) { - clients.push_back(pair.second); - } - } - for (auto& session : clients) { - session->wsSend(message); - } - return clients.size(); -} - -void HttpServer::disconnectWsClient(const std::string& clientId) { - std::lock_guard lock(ws_clients_mutex_); - ws_clients_.erase(clientId); -} - -size_t HttpServer::getWsClientCount() const { - std::lock_guard lock(ws_clients_mutex_); - return ws_clients_.size(); -} - -// ---- 通配符转换 ---- -std::regex HttpServer::wildcardToRegex(const std::string& pattern) { - std::string r; - r.reserve(pattern.size() * 2); - for (char c : pattern) { - if (c == '*') r += ".*"; - else if (c == '?') r += "."; - else if (c == '.' || c == '+' || c == '(' || c == ')' || c == '[' || c == ']' || - c == '{' || c == '}' || c == '^' || c == '$' || c == '|' || c == '\\') { - r += '\\'; - r += c; - } else r += c; - } - return std::regex(r, std::regex::icase); -} - -bool HttpServer::matchPath(const std::string& path, const std::regex& pattern) { - return std::regex_match(path, pattern); -} - -// ---- 查询参数解析 ---- -std::unordered_map HttpServer::parseQueryParams(const std::string& query) { - std::unordered_map params; - if (query.empty()) return params; - - std::vector pairs; - boost::split(pairs, query, boost::is_any_of("&")); - for (const auto& pair : pairs) { - std::vector kv; - boost::split(kv, pair, boost::is_any_of("=")); - if (kv.size() == 2) params[kv[0]] = kv[1]; - else if (kv.size() == 1) params[kv[0]] = ""; - } - return params; -} - -// ---- 路由查找 ---- -HttpRequestHandler HttpServer::findRoute(const std::string& method, const std::string& path) { - const std::vector* handlers = nullptr; - if (method == "GET") handlers = &get_handlers_; - else if (method == "POST") handlers = &post_handlers_; - else if (method == "PUT") handlers = &put_handlers_; - else if (method == "DELETE") handlers = &delete_handlers_; - else if (method == "OPTIONS") handlers = &options_handlers_; - else return nullptr; - - for (const auto& e : *handlers) { - if (!e.is_wildcard && e.path == path) return e.handler; - } - for (const auto& e : *handlers) { - if (e.is_wildcard && matchPath(path, e.pattern)) return e.handler; - } - return nullptr; -} - -bool HttpServer::routeExists(const std::string& method, const std::string& path) { - return findRoute(method, path) != nullptr; -} - -WebSocketMessageHandler HttpServer::findWsRoute(const std::string& path) { - for (const auto& e : ws_handlers_) { - if (e.path == path) return e.handler; - } - return nullptr; -} - -WsProxyHandshakeHandler HttpServer::findWsProxyRoute(const std::string& path) { - for (const auto& e : ws_proxy_handlers_) { - if (e.path == path) return e.handshake; - } - return nullptr; -} - -// ---- WebSocket 客户端管理 ---- -std::string HttpServer::generateWsClientId() { - return "ws_client_" + std::to_string(++ws_client_counter_); -} - -void HttpServer::addWsClient(const std::string& clientId, std::shared_ptr session) { - std::lock_guard lock(ws_clients_mutex_); - ws_clients_[clientId] = session; -} - -void HttpServer::removeWsClient(const std::string& clientId) { - std::lock_guard lock(ws_clients_mutex_); - ws_clients_.erase(clientId); -} - -std::shared_ptr HttpServer::getWsClient(const std::string& clientId) { - std::lock_guard lock(ws_clients_mutex_); - auto it = ws_clients_.find(clientId); - if (it != ws_clients_.end()) return it->second; - return nullptr; -} - -// ---- 请求处理核心 ---- -http::response HttpServer::handleRequest(const http::request& req) { - std::string target_path(req.target().data(), req.target().size()); - std::string query_string; - auto pos = target_path.find('?'); - if (pos != std::string::npos) { - query_string = target_path.substr(pos + 1); - target_path = target_path.substr(0, pos); - } - auto params = parseQueryParams(query_string); - - http::response res; - res.version(req.version()); - res.set(http::field::server, "Boost.Beast HttpServer"); - res.set(http::field::content_type, "text/plain"); - res.keep_alive(req.keep_alive()); - - bool middleware_handled = false; - for (auto& m : middlewares_) { - if (!m(req, res, params)) { - middleware_handled = true; - break; - } - } - if (middleware_handled) { - res.prepare_payload(); - return res; - } - - std::string method = http::to_string(req.method()); - - if (method == "OPTIONS") { - auto handler = findRoute("OPTIONS", target_path); - if (handler) { - try { - return handler(req, params); - } catch (const std::exception& e) { - std::cerr << "[Handler] Exception: " << e.what() << std::endl; - res.result(http::status::internal_server_error); - res.body() = "500 Internal Server Error\n"; - res.prepare_payload(); - return res; - } - } - - auto origin = req.find(http::field::origin); - if (origin != req.end()) { - std::vector all_methods = {"GET", "POST", "PUT", "DELETE", "OPTIONS"}; - std::vector allowed_methods; - - for (const auto& m : all_methods) { - if (m == "OPTIONS" || routeExists(m, target_path)) { - allowed_methods.push_back(m); - } - } - - if (!allowed_methods.empty() || findWsRoute(target_path)) { - if (allowed_methods.empty()) allowed_methods.push_back("GET"); - - std::string allow_methods; - for (size_t i = 0; i < allowed_methods.size(); ++i) { - if (i > 0) allow_methods += ", "; - allow_methods += allowed_methods[i]; - } - - res.result(http::status::ok); - res.set(http::field::allow, allow_methods); - res.set(http::field::access_control_allow_origin, - std::string(origin->value())); - res.set(http::field::access_control_allow_methods, allow_methods); - res.set(http::field::access_control_allow_headers, - "Content-Type, Authorization, X-Requested-With, Accept, Origin"); - res.set(http::field::access_control_max_age, "86400"); - res.body() = ""; - res.content_length(0); - res.prepare_payload(); - return res; - } - } - - res.result(http::status::not_found); - res.body() = "404 Not Found\n"; - res.prepare_payload(); - return res; - } - - auto handler = findRoute(method, target_path); - - if (!handler && !static_directory_.empty()) { - std::string file_path = static_directory_ + target_path; - std::ifstream file(file_path, std::ios::binary); - if (file.is_open()) { - std::stringstream buffer; - buffer << file.rdbuf(); - res.result(http::status::ok); - res.body() = buffer.str(); - - if (boost::algorithm::ends_with(file_path, ".html")) - res.set(http::field::content_type, "text/html"); - else if (boost::algorithm::ends_with(file_path, ".css")) - res.set(http::field::content_type, "text/css"); - else if (boost::algorithm::ends_with(file_path, ".js")) - res.set(http::field::content_type, "application/javascript"); - else if (boost::algorithm::ends_with(file_path, ".json")) - res.set(http::field::content_type, "application/json"); - else if (boost::algorithm::ends_with(file_path, ".png")) - res.set(http::field::content_type, "image/png"); - else if (boost::algorithm::ends_with(file_path, ".jpg") || - boost::algorithm::ends_with(file_path, ".jpeg")) - res.set(http::field::content_type, "image/jpeg"); - res.prepare_payload(); - return res; - } - } - - if (handler) { - try { - return handler(req, params); - } catch (const std::exception& e) { - std::cerr << "[Handler] Exception: " << e.what() << std::endl; - res.result(http::status::internal_server_error); - res.body() = "500 Internal Server Error\n"; - res.prepare_payload(); - return res; - } - } - - res.result(http::status::not_found); - res.body() = "404 Not Found\n"; - res.prepare_payload(); - return res; } void HttpServer::showStatus() const { @@ -1362,35 +191,5 @@ void HttpServer::showStatus() const { std::cout << "==========================" << std::endl; } -// ---- 会话管理 ---- -void HttpServer::addSession(const std::shared_ptr& session) { - std::lock_guard lock(sessions_mutex_); - sessions_.erase(std::remove_if(sessions_.begin(), sessions_.end(), - [](const std::weak_ptr& wp) { return wp.expired(); }), sessions_.end()); - sessions_.push_back(session); -} - -void HttpServer::removeSession(const Session* session) { - std::lock_guard lock(sessions_mutex_); - sessions_.erase(std::remove_if(sessions_.begin(), sessions_.end(), - [session](const std::weak_ptr& wp) { - auto sp = wp.lock(); - return !sp || sp.get() == session; - }), sessions_.end()); -} - -void HttpServer::closeAllSessions() { - std::lock_guard lock(sessions_mutex_); - sessions_.clear(); -} - -// ---- 调试工具 ---- -void HttpServer::debug::print_all_header_fields(const http::request& req) { - for (auto const& field : req) { - std::cout << "Name: " << field.name_string() - << ", Value: " << field.value() << std::endl; - } -} - } // namespace net } // namespace pmc diff --git a/libhttp/src/HttpServer_Listener.cpp b/libhttp/src/HttpServer_Listener.cpp new file mode 100644 index 0000000..6db6b0c --- /dev/null +++ b/libhttp/src/HttpServer_Listener.cpp @@ -0,0 +1,51 @@ +#include "us/detail/HttpServerImpl.hpp" + +namespace pmc { +namespace net { + +// ===================== Listener 实现 ===================== + +void HttpServer::Listener::run() { + if (acceptor_.is_open()) doAccept(); +} + +void HttpServer::Listener::initAcceptor(tcp::endpoint ep) { + beast::error_code ec; + acceptor_.open(ep.protocol(), ec); + if (!ec) { + acceptor_.set_option(asio::socket_base::reuse_address(true), ec); + acceptor_.bind(ep, ec); + if (!ec) acceptor_.listen(asio::socket_base::max_listen_connections, ec); + } + if (ec) throw HttpListenerException("Listener error: " + ec.message()); +} + +void HttpServer::Listener::doAccept() { + auto self = shared_from_this(); + acceptor_.async_accept( + asio::make_strand(ioc_), + [self](beast::error_code ec, tcp::socket socket) { + if (!ec) { + auto ep = socket.remote_endpoint(ec); + std::string ip = ec ? "unknown" : ep.address().to_string(); + unsigned short port = ec ? 0 : ep.port(); + std::cout << "[Listener] New connection from " << ip << ":" << port << std::endl; + + std::shared_ptr session; + if (self->ssl_enabled_ && self->ssl_ctx_) { + session = std::make_shared(std::move(socket), *self->ssl_ctx_, self->server_); + } else { + session = std::make_shared(std::move(socket), self->server_); + } + session->setRemoteInfo(ip, port); + self->server_->addSession(session); + session->run(); + } else { + std::cerr << "[Listener] Accept error: " << ec.message() << std::endl; + } + self->doAccept(); + }); +} + +} // namespace net +} // namespace pmc diff --git a/libhttp/src/HttpServer_Request.cpp b/libhttp/src/HttpServer_Request.cpp new file mode 100644 index 0000000..f5e5aa7 --- /dev/null +++ b/libhttp/src/HttpServer_Request.cpp @@ -0,0 +1,329 @@ +#include "us/detail/HttpServerImpl.hpp" + +namespace pmc { +namespace net { + +// ---- 请求处理核心 ---- +http::response HttpServer::handleRequest(const http::request& req) { + std::string target_path(req.target().data(), req.target().size()); + std::string query_string; + auto pos = target_path.find('?'); + if (pos != std::string::npos) { + query_string = target_path.substr(pos + 1); + target_path = target_path.substr(0, pos); + } + auto params = parseQueryParams(query_string); + + http::response res; + res.version(req.version()); + res.set(http::field::server, "Boost.Beast HttpServer"); + res.set(http::field::content_type, "text/plain"); + res.keep_alive(req.keep_alive()); + + bool middleware_handled = false; + for (auto& m : middlewares_) { + if (!m(req, res, params)) { + middleware_handled = true; + break; + } + } + if (middleware_handled) { + res.prepare_payload(); + return res; + } + + std::string method = http::to_string(req.method()); + + if (method == "OPTIONS") { + auto handler = findRoute("OPTIONS", target_path); + if (handler) { + try { + return handler(req, params); + } catch (const std::exception& e) { + std::cerr << "[Handler] Exception: " << e.what() << std::endl; + res.result(http::status::internal_server_error); + res.body() = "500 Internal Server Error\n"; + res.prepare_payload(); + return res; + } + } + + auto origin = req.find(http::field::origin); + if (origin != req.end()) { + std::vector all_methods = {"GET", "POST", "PUT", "DELETE", "OPTIONS"}; + std::vector allowed_methods; + + for (const auto& m : all_methods) { + if (m == "OPTIONS" || routeExists(m, target_path)) { + allowed_methods.push_back(m); + } + } + + if (!allowed_methods.empty() || findWsRoute(target_path)) { + if (allowed_methods.empty()) allowed_methods.push_back("GET"); + + std::string allow_methods; + for (size_t i = 0; i < allowed_methods.size(); ++i) { + if (i > 0) allow_methods += ", "; + allow_methods += allowed_methods[i]; + } + + res.result(http::status::ok); + res.set(http::field::allow, allow_methods); + res.set(http::field::access_control_allow_origin, + std::string(origin->value())); + res.set(http::field::access_control_allow_methods, allow_methods); + res.set(http::field::access_control_allow_headers, + "Content-Type, Authorization, X-Requested-With, Accept, Origin"); + res.set(http::field::access_control_max_age, "86400"); + res.body() = ""; + res.content_length(0); + res.prepare_payload(); + return res; + } + } + + res.result(http::status::not_found); + res.body() = "404 Not Found\n"; + res.prepare_payload(); + return res; + } + + auto handler = findRoute(method, target_path); + + if (!handler && !static_directory_.empty()) { + std::string file_path = static_directory_ + target_path; + std::ifstream file(file_path, std::ios::binary); + if (file.is_open()) { + std::stringstream buffer; + buffer << file.rdbuf(); + res.result(http::status::ok); + res.body() = buffer.str(); + + if (boost::algorithm::ends_with(file_path, ".html")) + res.set(http::field::content_type, "text/html"); + else if (boost::algorithm::ends_with(file_path, ".css")) + res.set(http::field::content_type, "text/css"); + else if (boost::algorithm::ends_with(file_path, ".js")) + res.set(http::field::content_type, "application/javascript"); + else if (boost::algorithm::ends_with(file_path, ".json")) + res.set(http::field::content_type, "application/json"); + else if (boost::algorithm::ends_with(file_path, ".png")) + res.set(http::field::content_type, "image/png"); + else if (boost::algorithm::ends_with(file_path, ".jpg") || + boost::algorithm::ends_with(file_path, ".jpeg")) + res.set(http::field::content_type, "image/jpeg"); + res.prepare_payload(); + return res; + } + } + + if (handler) { + try { + return handler(req, params); + } catch (const std::exception& e) { + std::cerr << "[Handler] Exception: " << e.what() << std::endl; + res.result(http::status::internal_server_error); + res.body() = "500 Internal Server Error\n"; + res.prepare_payload(); + return res; + } + } + + res.result(http::status::not_found); + res.body() = "404 Not Found\n"; + res.prepare_payload(); + return res; +} + +// ---- Session HTTP 处理 ---- +void HttpServer::Session::setRemoteInfo(const std::string& ip, unsigned short port) { + remote_ip_ = ip; + remote_port_ = port; +} + +void HttpServer::Session::doSSLHandshake() { + auto self = shared_from_this(); + std::cout << "[SSL] Starting handshake for " + << remote_ip_ << ":" << remote_port_ << std::endl; + + ssl_stream_->async_handshake( + asio::ssl::stream_base::server, + [self](beast::error_code ec) { + if (ec) { + std::cerr << "[SSL] Handshake error: " + << self->remote_ip_ << ":" << self->remote_port_ + << " - " << ec.message() + << " (code=" << ec.value() << ")" << std::endl; + ERR_print_errors_fp(stderr); + return; + } + std::cout << "[SSL] Handshake succeeded: " + << self->remote_ip_ << ":" << self->remote_port_ << std::endl; + self->doRead(); + }); +} + +void HttpServer::Session::doRead() { + auto self = shared_from_this(); + request_ = {}; + buffer_.consume(buffer_.size()); + + if (ssl_enabled_ && ssl_stream_) { + http::async_read(*ssl_stream_, buffer_, request_, + [self](beast::error_code ec, std::size_t) { + self->handleRead(ec); + }); + } else { + http::async_read(stream_, buffer_, request_, + [self](beast::error_code ec, std::size_t) { + self->handleRead(ec); + }); + } +} + +bool HttpServer::Session::isWebSocketUpgrade(const http::request& req) { + if (req.method() != http::verb::get) return false; + + auto upgrade = req.find(http::field::upgrade); + if (upgrade == req.end()) return false; + + std::string upgrade_val(upgrade->value()); + boost::to_lower(upgrade_val); + boost::trim(upgrade_val); + if (upgrade_val != "websocket") return false; + + auto connection = req.find(http::field::connection); + if (connection == req.end()) return false; + + std::string conn_val(connection->value()); + boost::to_lower(conn_val); + return conn_val.find("upgrade") != std::string::npos; +} + +void HttpServer::Session::handleRead(beast::error_code ec) { + if (!ec) { + if (isWebSocketUpgrade(request_)) { + std::string path = std::string(request_.target()); + auto qpos = path.find('?'); + if (qpos != std::string::npos) path = path.substr(0, qpos); + + // Check proxy routes first (higher priority) + auto proxy_handshake = server_->findWsProxyRoute(path); + if (proxy_handshake) { + std::string target = proxy_handshake(request_); + if (target.empty()) { + // Rejected by handshake callback → 403 + auto resp = std::make_shared>(); + resp->version(request_.version()); + resp->result(http::status::forbidden); + resp->set(http::field::content_type, "text/plain"); + resp->set(http::field::connection, "close"); + resp->body() = "403 Forbidden\n"; + resp->prepare_payload(); + doWrite(resp); + return; + } + ws_path_ = path; + is_ws_proxy_ = true; + ws_proxy_target_ = target; + if (ssl_enabled_) { + doWssUpgrade(); + } else { + doWsUpgrade(); + } + return; + } + + auto ws_handler = server_->findWsRoute(path); + if (ws_handler) { + ws_handler_ = ws_handler; + ws_path_ = path; + if (ssl_enabled_) { + doWssUpgrade(); + } else { + doWsUpgrade(); + } + return; + } + } + + auto req_copy = std::move(request_); + processRequest(std::move(req_copy)); + } else if (ec != http::error::end_of_stream) { + std::cerr << "[Read] Error from " << remote_ip_ << ":" << remote_port_ + << ": " << ec.message() << std::endl; + } +} + +void HttpServer::Session::processRequest(http::request req) { + try { + auto resp = std::make_shared>( + server_->handleRequest(req)); + doWrite(resp); + } catch (const std::exception& e) { + std::cerr << "[Request] Handler exception: " << e.what() << std::endl; + auto err_resp = std::make_shared>(); + err_resp->version(req.version()); + err_resp->result(http::status::internal_server_error); + err_resp->set(http::field::content_type, "text/plain"); + err_resp->body() = "500 Internal Server Error\n"; + err_resp->prepare_payload(); + doWrite(err_resp); + } +} + +void HttpServer::Session::doWrite(std::shared_ptr> response) { + auto self = shared_from_this(); + if (ssl_enabled_ && ssl_stream_) { + http::async_write(*ssl_stream_, *response, + [self, response](beast::error_code ec, std::size_t) { + if (ec) { + std::cerr << "[Write] Error to " << self->remote_ip_ << ":" << self->remote_port_ + << ": " << ec.message() << std::endl; + return; + } + if (!response->keep_alive()) self->close(); + else self->doRead(); + }); + } else { + http::async_write(stream_, *response, + [self, response](beast::error_code ec, std::size_t) { + if (ec) { + std::cerr << "[Write] Error to " << self->remote_ip_ << ":" << self->remote_port_ + << ": " << ec.message() << std::endl; + return; + } + if (!response->keep_alive()) self->close(); + else self->doRead(); + }); + } +} + +void HttpServer::Session::close() { + beast::error_code ec; + closeProxy(websocket::close_code::going_away); + if (is_websocket_) { + if (ssl_enabled_ && wss_stream_) { + wss_stream_->close(websocket::close_code::normal, ec); + } else if (ws_stream_) { + ws_stream_->close(websocket::close_code::normal, ec); + } + } + if (ssl_stream_) { + ssl_stream_->shutdown(ec); + } + stream_.socket().shutdown(tcp::socket::shutdown_send, ec); + stream_.close(); +} + +void HttpServer::Session::run() { + if (ssl_enabled_) { + doSSLHandshake(); + } else { + doRead(); + } +} + +} // namespace net +} // namespace pmc diff --git a/libhttp/src/HttpServer_Routes.cpp b/libhttp/src/HttpServer_Routes.cpp new file mode 100644 index 0000000..6683a46 --- /dev/null +++ b/libhttp/src/HttpServer_Routes.cpp @@ -0,0 +1,125 @@ +#include "us/detail/HttpServerImpl.hpp" + +namespace pmc { +namespace net { + +// ---- HTTP 路由注册 ---- +void HttpServer::get(const std::string& path, HttpRequestHandler handler) { + RouteEntry entry; + entry.path = path; + entry.handler = handler; + entry.is_wildcard = (path.find('*') != std::string::npos || + path.find('?') != std::string::npos); + if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); + get_handlers_.push_back(entry); +} + +void HttpServer::post(const std::string& path, HttpRequestHandler handler) { + RouteEntry entry; + entry.path = path; + entry.handler = handler; + entry.is_wildcard = (path.find('*') != std::string::npos || + path.find('?') != std::string::npos); + if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); + post_handlers_.push_back(entry); +} + +void HttpServer::put(const std::string& path, HttpRequestHandler handler) { + RouteEntry entry; + entry.path = path; + entry.handler = handler; + entry.is_wildcard = (path.find('*') != std::string::npos || + path.find('?') != std::string::npos); + if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); + put_handlers_.push_back(entry); +} + +void HttpServer::del(const std::string& path, HttpRequestHandler handler) { + RouteEntry entry; + entry.path = path; + entry.handler = handler; + entry.is_wildcard = (path.find('*') != std::string::npos || + path.find('?') != std::string::npos); + if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); + delete_handlers_.push_back(entry); +} + +void HttpServer::options(const std::string& path, HttpRequestHandler handler) { + RouteEntry entry; + entry.path = path; + entry.handler = handler; + entry.is_wildcard = (path.find('*') != std::string::npos || + path.find('?') != std::string::npos); + if (entry.is_wildcard) entry.pattern = wildcardToRegex(path); + options_handlers_.push_back(entry); +} + +void HttpServer::use(MiddlewareHandler m) { + middlewares_.push_back(m); +} + +void HttpServer::setStaticDirectory(const std::string& p) { + static_directory_ = p; +} + +// ---- 通配符转换 ---- +std::regex HttpServer::wildcardToRegex(const std::string& pattern) { + std::string r; + r.reserve(pattern.size() * 2); + for (char c : pattern) { + if (c == '*') r += ".*"; + else if (c == '?') r += "."; + else if (c == '.' || c == '+' || c == '(' || c == ')' || c == '[' || c == ']' || + c == '{' || c == '}' || c == '^' || c == '$' || c == '|' || c == '\\') { + r += '\\'; + r += c; + } else r += c; + } + return std::regex(r, std::regex::icase); +} + +bool HttpServer::matchPath(const std::string& path, const std::regex& pattern) { + return std::regex_match(path, pattern); +} + +// ---- 查询参数解析 ---- +std::unordered_map HttpServer::parseQueryParams(const std::string& query) { + std::unordered_map params; + if (query.empty()) return params; + + std::vector pairs; + boost::split(pairs, query, boost::is_any_of("&")); + for (const auto& pair : pairs) { + std::vector kv; + boost::split(kv, pair, boost::is_any_of("=")); + if (kv.size() == 2) params[kv[0]] = kv[1]; + else if (kv.size() == 1) params[kv[0]] = ""; + } + return params; +} + +// ---- 路由查找 ---- +HttpRequestHandler HttpServer::findRoute(const std::string& method, const std::string& path) { + const std::vector* handlers = nullptr; + if (method == "GET") handlers = &get_handlers_; + else if (method == "POST") handlers = &post_handlers_; + else if (method == "PUT") handlers = &put_handlers_; + else if (method == "DELETE") handlers = &delete_handlers_; + else if (method == "OPTIONS") handlers = &options_handlers_; + else return nullptr; + + for (const auto& e : *handlers) { + if (!e.is_wildcard && e.path == path) return e.handler; + } + for (const auto& e : *handlers) { + if (e.is_wildcard && matchPath(path, e.pattern)) return e.handler; + } + return nullptr; +} + +bool HttpServer::routeExists(const std::string& method, const std::string& path) { + return findRoute(method, path) != nullptr; +} + +} // namespace net +} // namespace pmc diff --git a/libhttp/src/HttpServer_Ws.cpp b/libhttp/src/HttpServer_Ws.cpp new file mode 100644 index 0000000..d037a97 --- /dev/null +++ b/libhttp/src/HttpServer_Ws.cpp @@ -0,0 +1,106 @@ +#include "us/detail/HttpServerImpl.hpp" + +namespace pmc { +namespace net { + +// ---- WebSocket 路由注册 ---- +void HttpServer::ws(const std::string& path, WebSocketMessageHandler handler) { + WsRouteEntry entry; + entry.path = path; + entry.handler = handler; + ws_handlers_.push_back(entry); +} + +void HttpServer::ws_proxy(const std::string& path, WsProxyHandshakeHandler handshake) { + WsProxyRouteEntry entry; + entry.path = path; + entry.handshake = handshake; + ws_proxy_handlers_.push_back(entry); +} + +void HttpServer::onWsConnect(WebSocketConnectHandler handler) { + ws_connect_handler_ = handler; +} + +void HttpServer::onWsDisconnect(WebSocketDisconnectHandler handler) { + ws_disconnect_handler_ = handler; +} + +WebSocketMessageHandler HttpServer::findWsRoute(const std::string& path) { + for (const auto& e : ws_handlers_) { + if (e.path == path) return e.handler; + } + return nullptr; +} + +WsProxyHandshakeHandler HttpServer::findWsProxyRoute(const std::string& path) { + for (const auto& e : ws_proxy_handlers_) { + if (e.path == path) return e.handshake; + } + return nullptr; +} + +// ---- WebSocket 客户端管理 ---- +std::string HttpServer::generateWsClientId() { + return "ws_client_" + std::to_string(++ws_client_counter_); +} + +void HttpServer::addWsClient(const std::string& clientId, std::shared_ptr session) { + std::lock_guard lock(ws_clients_mutex_); + ws_clients_[clientId] = session; +} + +void HttpServer::removeWsClient(const std::string& clientId) { + std::lock_guard lock(ws_clients_mutex_); + ws_clients_.erase(clientId); +} + +std::shared_ptr HttpServer::getWsClient(const std::string& clientId) { + std::lock_guard lock(ws_clients_mutex_); + auto it = ws_clients_.find(clientId); + if (it != ws_clients_.end()) return it->second; + return nullptr; +} + +bool HttpServer::sendToWsClient(const std::string& clientId, const std::string& message) { + std::shared_ptr session; + { + std::lock_guard lock(ws_clients_mutex_); + auto it = ws_clients_.find(clientId); + if (it != ws_clients_.end()) { + session = it->second; + } + } + if (session) { + session->wsSend(message); + return true; + } + return false; +} + +size_t HttpServer::broadcastWs(const std::string& message) { + std::vector> clients; + { + std::lock_guard lock(ws_clients_mutex_); + for (auto& pair : ws_clients_) { + clients.push_back(pair.second); + } + } + for (auto& session : clients) { + session->wsSend(message); + } + return clients.size(); +} + +void HttpServer::disconnectWsClient(const std::string& clientId) { + std::lock_guard lock(ws_clients_mutex_); + ws_clients_.erase(clientId); +} + +size_t HttpServer::getWsClientCount() const { + std::lock_guard lock(ws_clients_mutex_); + return ws_clients_.size(); +} + +} // namespace net +} // namespace pmc diff --git a/libhttp/src/HttpServer_WsSession.cpp b/libhttp/src/HttpServer_WsSession.cpp new file mode 100644 index 0000000..25eb7c9 --- /dev/null +++ b/libhttp/src/HttpServer_WsSession.cpp @@ -0,0 +1,506 @@ +#include "us/detail/HttpServerImpl.hpp" + +namespace pmc { +namespace net { + +// ===================== Session WebSocket 处理 ===================== + +void HttpServer::Session::wsSend(const std::string& message) { + auto exec = stream_.get_executor(); + asio::post(exec, [self = shared_from_this(), message]() { + self->ws_send_queue_.push_back(message); + if (self->ws_send_queue_.size() == 1) { + self->doWsWrite(); + } + }); +} + +void HttpServer::Session::doWsUpgrade() { + auto self = shared_from_this(); + ws_stream_ = std::make_unique(stream_); + ws_stream_->text(true); + + ws_stream_->async_accept( + request_, + [self](beast::error_code ec) { + self->onWsAccept(ec); + }); +} + +void HttpServer::Session::doWssUpgrade() { + auto self = shared_from_this(); + wss_stream_ = std::make_unique(*ssl_stream_); + wss_stream_->text(true); + + wss_stream_->async_accept( + request_, + [self](beast::error_code ec) { + self->onWsAccept(ec); + }); +} + +void HttpServer::Session::onWsAccept(beast::error_code ec) { + if (ec) { + std::cerr << "[WS] Accept error from " << remote_ip_ << ":" << remote_port_ + << ": " << ec.message() << std::endl; + return; + } + + is_websocket_ = true; + client_id_ = server_->generateWsClientId(); + server_->addWsClient(client_id_, shared_from_this()); + + std::cout << "[WS] Client " << client_id_ << " connected from " + << remote_ip_ << ":" << remote_port_ << " at " << ws_path_ << std::endl; + + if (is_ws_proxy_) { + std::cout << "[WS-Proxy] Proxying " << ws_path_ << " -> " << ws_proxy_target_ << std::endl; + doProxyConnect(ws_proxy_target_); + return; + } + + if (server_->ws_connect_handler_) { + server_->ws_connect_handler_(client_id_, getRemoteAddress()); + } + + doWsRead(); +} + +void HttpServer::Session::doWsRead() { + auto self = shared_from_this(); + ws_buffer_.consume(ws_buffer_.size()); + + if (ssl_enabled_ && wss_stream_) { + wss_stream_->async_read( + ws_buffer_, + [self](beast::error_code ec, std::size_t bytes_transferred) { + self->onWsRead(ec, bytes_transferred); + }); + } else if (ws_stream_) { + ws_stream_->async_read( + ws_buffer_, + [self](beast::error_code ec, std::size_t bytes_transferred) { + self->onWsRead(ec, bytes_transferred); + }); + } +} + +void HttpServer::Session::onWsRead(beast::error_code ec, std::size_t bytes_transferred) { + if (ec == websocket::error::closed) { + onWsClose(); + return; + } + if (ec) { + std::cerr << "[WS] Read error from " << client_id_ << ": " << ec.message() << std::endl; + onWsClose(); + return; + } + + std::string message = beast::buffers_to_string(ws_buffer_.data()); + ws_buffer_.consume(bytes_transferred); + + if (ws_handler_) { + auto self = shared_from_this(); + ws_handler_(client_id_, message, + [self](const std::string& response) { + self->wsSend(response); + }); + } + + doWsRead(); +} + +void HttpServer::Session::doWsWrite() { + if (ws_send_queue_.empty()) return; + + auto self = shared_from_this(); + std::string& message = ws_send_queue_.front(); + + auto write_handler = [self](beast::error_code ec, std::size_t) { + if (ec) { + std::cerr << "[WS] Write error to " << self->client_id_ << ": " << ec.message() << std::endl; + self->onWsClose(); + return; + } + self->ws_send_queue_.pop_front(); + if (!self->ws_send_queue_.empty()) { + self->doWsWrite(); + } + }; + + if (ssl_enabled_ && wss_stream_) { + wss_stream_->text(true); + wss_stream_->async_write(asio::buffer(message), write_handler); + } else if (ws_stream_) { + ws_stream_->text(true); + ws_stream_->async_write(asio::buffer(message), write_handler); + } +} + +void HttpServer::Session::onWsClose() { + if (client_id_.empty()) return; + std::cout << "[WS] Client " << client_id_ << " disconnected" << std::endl; + + closeProxy(websocket::close_code::normal); + server_->removeWsClient(client_id_); + + if (server_->ws_disconnect_handler_) { + server_->ws_disconnect_handler_(client_id_); + } + client_id_.clear(); + is_websocket_ = false; + is_ws_proxy_ = false; +} + +// ===================== WebSocket Proxy (双向桥接) ===================== + +void HttpServer::Session::parseProxyUrl(const std::string& url, bool& ssl, std::string& host, + std::string& port, std::string& path) { + std::string rest; + if (url.size() >= 6 && url.substr(0, 6) == "wss://") { + ssl = true; + rest = url.substr(6); + } else if (url.size() >= 5 && url.substr(0, 5) == "ws://") { + ssl = false; + rest = url.substr(5); + } else { + ssl = false; + rest = url; + } + + size_t path_pos = rest.find('/'); + std::string host_port; + if (path_pos != std::string::npos) { + host_port = rest.substr(0, path_pos); + path = rest.substr(path_pos); + } else { + host_port = rest; + path = "/"; + } + + size_t port_pos = host_port.find(':'); + if (port_pos != std::string::npos) { + host = host_port.substr(0, port_pos); + port = host_port.substr(port_pos + 1); + } else { + host = host_port; + port = ssl ? "443" : "80"; + } +} + +void HttpServer::Session::doProxyConnect(const std::string& target_url) { + auto self = shared_from_this(); + bool ssl; + std::string host, port, path; + parseProxyUrl(target_url, ssl, host, port, path); + proxy_ssl_ = ssl; + proxy_host_ = host; + proxy_port_ = port; + proxy_path_ = path; + + auto exec = stream_.get_executor(); + proxy_resolver_ = std::make_unique(asio::make_strand(exec)); + + proxy_resolver_->async_resolve(host, port, + asio::bind_executor(exec, + [self](beast::error_code ec, tcp::resolver::results_type results) { + self->onProxyResolve(ec, results); + })); +} + +void HttpServer::Session::onProxyResolve(beast::error_code ec, tcp::resolver::results_type results) { + if (ec) { + std::cerr << "[WS-Proxy] Resolve error: " << ec.message() << std::endl; + close(); + return; + } + + auto self = shared_from_this(); + auto exec = stream_.get_executor(); + + proxy_socket_ = std::make_unique(exec); + proxy_tcp_stream_ = std::make_unique(std::move(*proxy_socket_)); + proxy_socket_.reset(); + + proxy_tcp_stream_->expires_after(std::chrono::seconds(10)); + asio::async_connect(proxy_tcp_stream_->socket(), results, + asio::bind_executor(exec, + [self](beast::error_code ec, tcp::endpoint ep) { + self->onProxyConnect(ec, ep); + })); +} + +void HttpServer::Session::onProxyConnect(beast::error_code ec, tcp::endpoint /*ep*/) { + if (ec) { + std::cerr << "[WS-Proxy] Connect error: " << ec.message() << std::endl; + close(); + return; + } + + auto self = shared_from_this(); + if (proxy_ssl_) { + proxy_ssl_ctx_ = std::make_unique(asio::ssl::context::tlsv12_client); + proxy_ssl_ctx_->set_verify_mode(asio::ssl::verify_none); + proxy_ssl_stream_ = std::make_unique>( + proxy_tcp_stream_->socket(), *proxy_ssl_ctx_); + + if (!SSL_set_tlsext_host_name(proxy_ssl_stream_->native_handle(), proxy_host_.c_str())) { + std::cerr << "[WS-Proxy] Failed to set SNI" << std::endl; + close(); + return; + } + + proxy_tcp_stream_->expires_after(std::chrono::seconds(10)); + auto exec = stream_.get_executor(); + proxy_ssl_stream_->async_handshake(asio::ssl::stream_base::client, + asio::bind_executor(exec, + [self](beast::error_code ec) { + self->onProxySslHandshake(ec); + })); + } else { + proxy_ws_stream_ = std::make_unique(proxy_tcp_stream_->socket()); + proxy_tcp_stream_->expires_never(); + + proxy_ws_stream_->set_option(websocket::stream_base::decorator( + [](websocket::request_type& req) { + req.set(http::field::user_agent, "UniServe-WsProxy"); + })); + + auto exec = self->stream_.get_executor(); + proxy_ws_stream_->async_handshake(proxy_host_, proxy_path_, + asio::bind_executor(exec, + [self](beast::error_code ec) { + self->onProxyHandshake(ec); + })); + } +} + +void HttpServer::Session::onProxySslHandshake(beast::error_code ec) { + if (ec) { + std::cerr << "[WS-Proxy] SSL handshake error: " << ec.message() << std::endl; + close(); + return; + } + + auto self = shared_from_this(); + proxy_wss_stream_ = std::make_unique(*proxy_ssl_stream_); + proxy_tcp_stream_->expires_never(); + + proxy_wss_stream_->set_option(websocket::stream_base::decorator( + [](websocket::request_type& req) { + req.set(http::field::user_agent, "UniServe-WsProxy"); + })); + + auto exec = stream_.get_executor(); + proxy_wss_stream_->async_handshake(proxy_host_, proxy_path_, + asio::bind_executor(exec, + [self](beast::error_code ec) { + self->onProxyHandshake(ec); + })); +} + +void HttpServer::Session::onProxyHandshake(beast::error_code ec) { + if (ec) { + std::cerr << "[WS-Proxy] Backend handshake error: " << ec.message() << std::endl; + close(); + return; + } + + proxy_connected_ = true; + proxy_closed_ = false; + std::cout << "[WS-Proxy] Connected to backend " << proxy_host_ << ":" << proxy_port_ + << proxy_path_ << std::endl; + + if (server_->ws_connect_handler_) { + server_->ws_connect_handler_(client_id_, getRemoteAddress()); + } + + // Start bidirectional bridging + doProxyReadClient(); + doProxyReadBackend(); +} + +// ---- Client → Backend direction ---- + +void HttpServer::Session::doProxyReadClient() { + if (proxy_closed_) return; + auto self = shared_from_this(); + proxy_client_buffer_.consume(proxy_client_buffer_.size()); + + auto read_handler = [self](beast::error_code ec, std::size_t bytes_transferred) { + self->onProxyReadClient(ec, bytes_transferred); + }; + + if (ssl_enabled_ && wss_stream_) { + wss_stream_->async_read(proxy_client_buffer_, read_handler); + } else if (ws_stream_) { + ws_stream_->async_read(proxy_client_buffer_, read_handler); + } +} + +void HttpServer::Session::onProxyReadClient(beast::error_code ec, std::size_t bytes_transferred) { + if (ec) { + if (ec != websocket::error::closed) { + std::cerr << "[WS-Proxy] Client read error: " << ec.message() << std::endl; + } + closeProxy(websocket::close_code::normal); + return; + } + + bool is_binary = false; + if (ssl_enabled_ && wss_stream_) { + is_binary = wss_stream_->got_binary(); + } else if (ws_stream_) { + is_binary = ws_stream_->got_binary(); + } + + std::string data = beast::buffers_to_string(proxy_client_buffer_.data()); + proxy_client_buffer_.consume(bytes_transferred); + + // Queue to backend + auto exec = stream_.get_executor(); + asio::post(exec, [self = shared_from_this(), data = std::move(data), is_binary]() { + self->proxy_to_backend_queue_.emplace_back(std::move(data), is_binary); + if (self->proxy_to_backend_queue_.size() == 1) { + self->doProxyWriteBackend(); + } + }); + + doProxyReadClient(); +} + +void HttpServer::Session::doProxyWriteBackend() { + if (proxy_to_backend_queue_.empty() || !proxy_connected_ || proxy_closed_) return; + + auto self = shared_from_this(); + auto& [msg, is_binary] = proxy_to_backend_queue_.front(); + + auto write_handler = [self](beast::error_code ec, std::size_t) { + if (ec) { + std::cerr << "[WS-Proxy] Backend write error: " << ec.message() << std::endl; + self->closeProxy(websocket::close_code::internal_error); + return; + } + self->proxy_to_backend_queue_.pop_front(); + if (!self->proxy_to_backend_queue_.empty()) { + self->doProxyWriteBackend(); + } + }; + + if (proxy_ssl_ && proxy_wss_stream_) { + proxy_wss_stream_->binary(is_binary); + proxy_wss_stream_->async_write(asio::buffer(msg), write_handler); + } else if (proxy_ws_stream_) { + proxy_ws_stream_->binary(is_binary); + proxy_ws_stream_->async_write(asio::buffer(msg), write_handler); + } +} + +// ---- Backend → Client direction ---- + +void HttpServer::Session::doProxyReadBackend() { + if (proxy_closed_) return; + auto self = shared_from_this(); + proxy_backend_buffer_.consume(proxy_backend_buffer_.size()); + + auto read_handler = [self](beast::error_code ec, std::size_t bytes_transferred) { + self->onProxyReadBackend(ec, bytes_transferred); + }; + + if (proxy_ssl_ && proxy_wss_stream_) { + proxy_wss_stream_->async_read(proxy_backend_buffer_, read_handler); + } else if (proxy_ws_stream_) { + proxy_ws_stream_->async_read(proxy_backend_buffer_, read_handler); + } +} + +void HttpServer::Session::onProxyReadBackend(beast::error_code ec, std::size_t bytes_transferred) { + if (ec) { + if (ec != websocket::error::closed) { + std::cerr << "[WS-Proxy] Backend read error: " << ec.message() << std::endl; + } + closeProxy(websocket::close_code::normal); + return; + } + + bool is_binary = false; + if (proxy_ssl_ && proxy_wss_stream_) { + is_binary = proxy_wss_stream_->got_binary(); + } else if (proxy_ws_stream_) { + is_binary = proxy_ws_stream_->got_binary(); + } + + std::string data = beast::buffers_to_string(proxy_backend_buffer_.data()); + proxy_backend_buffer_.consume(bytes_transferred); + + // Queue to client + auto exec = stream_.get_executor(); + asio::post(exec, [self = shared_from_this(), data = std::move(data), is_binary]() { + self->proxy_to_client_queue_.emplace_back(std::move(data), is_binary); + if (self->proxy_to_client_queue_.size() == 1) { + self->doProxyWriteClient(); + } + }); + + doProxyReadBackend(); +} + +void HttpServer::Session::doProxyWriteClient() { + if (proxy_to_client_queue_.empty() || !proxy_connected_ || proxy_closed_) return; + + auto self = shared_from_this(); + auto& [msg, is_binary] = proxy_to_client_queue_.front(); + + auto write_handler = [self](beast::error_code ec, std::size_t) { + if (ec) { + std::cerr << "[WS-Proxy] Client write error: " << ec.message() << std::endl; + self->closeProxy(websocket::close_code::internal_error); + return; + } + self->proxy_to_client_queue_.pop_front(); + if (!self->proxy_to_client_queue_.empty()) { + self->doProxyWriteClient(); + } + }; + + if (ssl_enabled_ && wss_stream_) { + wss_stream_->binary(is_binary); + wss_stream_->async_write(asio::buffer(msg), write_handler); + } else if (ws_stream_) { + ws_stream_->binary(is_binary); + ws_stream_->async_write(asio::buffer(msg), write_handler); + } +} + +void HttpServer::Session::closeProxy(websocket::close_code code) { + if (proxy_closed_) return; + proxy_closed_ = true; + proxy_connected_ = false; + + beast::error_code ec; + // Close backend connection + if (proxy_ssl_ && proxy_wss_stream_) { + proxy_wss_stream_->close(code, ec); + } else if (proxy_ws_stream_) { + proxy_ws_stream_->close(code, ec); + } + if (proxy_ssl_stream_) { + proxy_ssl_stream_->shutdown(ec); + } + if (proxy_tcp_stream_) { + proxy_tcp_stream_->socket().shutdown(tcp::socket::shutdown_send, ec); + proxy_tcp_stream_->close(); + } + + proxy_to_backend_queue_.clear(); + proxy_to_client_queue_.clear(); + proxy_ws_stream_.reset(); + proxy_wss_stream_.reset(); + proxy_ssl_stream_.reset(); + proxy_ssl_ctx_.reset(); + proxy_tcp_stream_.reset(); + proxy_resolver_.reset(); +} + +} // namespace net +} // namespace pmc diff --git a/pymsg/README.md b/pymsg/README.md index ad76e58..3e84706 100644 --- a/pymsg/README.md +++ b/pymsg/README.md @@ -110,6 +110,20 @@ resp = client.call('{"action": "select", "table": "Users"}', timeout=10.0) 目标不可达时抛出 `ConnectionRefusedError`，超时无回复时抛出 `TimeoutError`。 +**启动等待模式（retry_on_refused）**： + +```python +# 服务启动时阻塞等待下游服务就绪，连接被拒绝时自动重试 +resp = client.call('{"_route": "get", "name": "db.ini"}', + timeout=5.0, + retry_on_refused=True, # 连接拒绝时阻塞重试 + retry_interval=1.0) # 重试间隔 1 秒（默认） +``` + +`retry_on_refused=True` 时，`ConnectionRefusedError` 不会抛出，而是按 `retry_interval`（默认 1.0 秒）间隔不断重试，直到连接成功。重试期间每 5 秒向 stderr 打印一次等待提示（如 `[pymsg] 等待服务 127.0.0.1:8010 就绪... (已重试 N 次)`），避免日志刷屏。仅对 `ConnectionRefusedError` 重试；`TimeoutError`、`ConnectionError`、业务错误等仍立即抛出。 + +适用场景：微服务启动顺序不确定时，服务进程可在不依赖外部进程管理器的情况下自动等待下游依赖就绪，无需手动 sys.exit + 外部重启。`log_client` 等 fail-safe 场景应保持默认 `retry_on_refused=False`，避免后台线程阻塞。 + **3. 长连接监听（回调模式）** ```python @@ -226,7 +240,23 @@ except RpcError as e: ... # 服务端业务错误（success=false） ``` -`call(route, payload=None, *, timeout=None)`：自动注入 `_route`、JSON 编解码、检查 `success`，返回 `data` 字段；业务错误抛 `RpcError`，网络故障仍抛 `ConnectionRefusedError` / `TimeoutError`（来自 `MsgClient`）。 +| 构造参数 | 含义 | +|----------|------| +| `addr` / `port` | 服务端地址与端口 | +| `name` | 客户端标识（透传 `MsgClient`） | +| `timeout` | 默认调用超时秒数（默认 `10.0`） | +| `retry_on_refused` | 连接被拒绝时是否阻塞重试（关键字参数，默认 `False`） | +| `retry_interval` | 重试间隔秒数（关键字参数，默认 `1.0`） | + +`call(route, payload=None, *, timeout=None, retry_on_refused=None, retry_interval=None)`：自动注入 `_route`、JSON 编解码、检查 `success`，返回 `data` 字段；业务错误抛 `RpcError`，网络故障仍抛 `ConnectionRefusedError` / `TimeoutError`（来自 `MsgClient`）。`retry_on_refused` 和 `retry_interval` 参数可在单次调用时覆盖构造时的设置。 + +**启动等待示例**（解决微服务启动顺序问题）： + +```python +# confs 客户端：启动时阻塞等待 confs 就绪，不依赖启动顺序 +rpc = RpcClient("127.0.0.1", 8010, "db", timeout=5, retry_on_refused=True) +data = rpc.call("get", {"name": "db.ini"}) # confs 未启动时自动阻塞等待 +``` ## 实际用例 diff --git a/pymsg/pymsg/__init__.py b/pymsg/pymsg/__init__.py index 48d4e7e..502d5b3 100644 --- a/pymsg/pymsg/__init__.py +++ b/pymsg/pymsg/__init__.py @@ -22,6 +22,7 @@ import select import socket as _socket import struct import threading +import time import traceback from concurrent.futures import ThreadPoolExecutor @@ -200,34 +201,56 @@ class MsgClient: ) # ---- 请求-应答（短连接）------------------------------------------------ - def call(self, msg: str, *, timeout: "float | None" = None) -> str: + def call(self, msg: str, *, timeout: "float | None" = None, + retry_on_refused: bool = False, + retry_interval: float = 1.0, + _log_every: float = 5.0) -> str: """发送一条消息并阻塞等待单条回复（短连接：connect → send → recv → close）。 适合请求-应答模式，如 RPC 调用。不会给消息添加 name 前缀。 Args: - msg: 要发送的原始字符串。 - timeout: 超时秒数，默认 10 秒。 + msg: 要发送的原始字符串。 + timeout: 超时秒数，默认 10 秒。 + retry_on_refused: 目标服务未启动（ConnectionRefusedError）时是否阻塞重试， + 直到连接成功。适用于启动顺序不确定的服务发现场景。 + 默认 False（连接拒绝立即抛异常）。 + retry_interval: 重试间隔秒数，默认 1.0。 + _log_every: 重试期间打印等待提示的间隔（秒），默认每 5 秒打印一次，避免刷屏。 Returns: str: 服务器返回的字符串。 Raises: - ConnectionRefusedError: 目标服务不可达。 + ConnectionRefusedError: 目标服务不可达且 ``retry_on_refused=False``。 TimeoutError: 等待响应超时。 ConnectionError: 连接在收全数据前断开。 """ _tout = timeout if timeout is not None else 10.0 - try: - with _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) as s: - s.settimeout(_tout) - s.connect((self.addr, self.port)) - send(s, msg) - return recv(s, timeout=_tout) - except ConnectionRefusedError: - raise ConnectionRefusedError( - f"Could not connect to {self.addr}:{self.port}" - ) + _last_log = 0.0 + _attempt = 0 + while True: + try: + with _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM) as s: + s.settimeout(_tout) + s.connect((self.addr, self.port)) + send(s, msg) + return recv(s, timeout=_tout) + except ConnectionRefusedError: + if not retry_on_refused: + raise ConnectionRefusedError( + f"Could not connect to {self.addr}:{self.port}" + ) + _attempt += 1 + now = time.monotonic() + if now - _last_log >= _log_every or _attempt == 1: + print( + f"[pymsg] 等待服务 {self.addr}:{self.port} 就绪... " + f"(已重试 {_attempt} 次)", + file=__import__('sys').stderr, + ) + _last_log = now + time.sleep(retry_interval) # ---- 持续监听（阻塞当前线程，回调模式）------------------------------- def listen(self, on_message: "callable") -> None: @@ -502,5 +525,6 @@ class MsgServer: # ── JSON RPC 约定层（依赖上面的 MsgClient / MsgServer，故放在末尾导入）── from .rpc import RpcClient, RpcError, RpcServer # noqa: E402 +from .async_queue import AsyncRpcQueue # noqa: E402 -__all__ += ["RpcClient", "RpcServer", "RpcError"] +__all__ += ["RpcClient", "RpcServer", "RpcError", "AsyncRpcQueue"] diff --git a/pymsg/pymsg/async_queue.py b/pymsg/pymsg/async_queue.py new file mode 100644 index 0000000..e8e8893 --- /dev/null +++ b/pymsg/pymsg/async_queue.py @@ -0,0 +1,202 @@ +""" +AsyncRpcQueue - 通用异步批量发送队列 + +提供内存队列 + 后台线程的异步批量处理机制，适用于日志上报、指标采集等 +非关键路径场景。业务线程put操作非阻塞，后台线程攒批处理，失败时通过错误回调 +通知，不影响主业务流程。 + +两种使用模式： + +1. 简单RPC模式（指定client + method + serialize）： + queue = AsyncRpcQueue( + client=rpc_client, + method='log_api.push', + batch_size=50, + flush_interval=2.0, + serialize=lambda items: {'logs': json.dumps(items)}, + ) + +2. 自定义处理模式（指定handler回调）： + def send_batch(batch): + for item in batch: + rpc_client.call('execute', item) + queue = AsyncRpcQueue( + handler=send_batch, + batch_size=50, + flush_interval=2.0, + ) +""" + +import threading +import time +from collections import deque +from typing import Any, Callable, Optional + + +class AsyncRpcQueue: + """通用异步批量发送队列""" + + def __init__( + self, + client: Any = None, + method: str = None, + *, + handler: Optional[Callable[[list], None]] = None, + batch_size: int = 100, + flush_interval: float = 2.0, + max_queue_size: int = 10000, + serialize: Optional[Callable[[list], dict]] = None, + on_error: Optional[Callable[[Exception, list], None]] = None, + retry_on_refused: bool = False, + retry_interval: float = 1.0, + ): + """ + 初始化异步队列 + + Args: + client: RpcClient实例（RPC模式必填） + method: RPC方法名（RPC模式必填） + handler: 自定义批量处理回调，接收一批items列表。提供此参数时 + 忽略client/method/serialize/retry_on_refused。 + batch_size: 批量发送的阈值，达到此数量立即发送 + flush_interval: 定时flush间隔（秒） + max_queue_size: 队列最大长度，超过则丢弃最旧数据 + serialize: 序列化函数，接收items列表返回RPC参数字典 + 默认: lambda items: {'items': items} + on_error: 错误回调，接收(exception, batch_items) + retry_on_refused: 是否在连接被拒绝时重试（RPC模式，透传给client.call） + retry_interval: 重试间隔（秒）（RPC模式，透传给client.call） + """ + if handler is None: + if client is None or method is None: + raise ValueError("必须提供 handler 或 (client + method)") + self._handler = self._make_rpc_handler( + client, method, + serialize or (lambda items: {'items': items}), + retry_on_refused, retry_interval, + ) + else: + self._handler = handler + + self._batch_size = batch_size + self._flush_interval = flush_interval + self._max_queue_size = max_queue_size + self._on_error = on_error + + self._queue: deque = deque() + self._lock = threading.Lock() + self._wakeup = threading.Event() + self._thread: Optional[threading.Thread] = None + self._closed = False + self._dropped_count = 0 + + @staticmethod + def _make_rpc_handler(client, method, serialize, retry_on_refused, retry_interval): + """创建RPC模式的默认handler""" + def handler(batch: list): + params = serialize(batch) + client.call( + method, params, + retry_on_refused=retry_on_refused, + retry_interval=retry_interval, + ) + return handler + + def start(self) -> None: + """启动后台发送线程""" + if self._thread is not None and self._thread.is_alive(): + return + self._closed = False + self._thread = threading.Thread( + target=self._run, + name=f'AsyncRpcQueue', + daemon=True, + ) + self._thread.start() + + def put(self, item: Any) -> bool: + """ + 放入一条数据到队列（非阻塞） + + Args: + item: 要发送的数据项 + + Returns: + bool: True表示放入成功，False表示队列已关闭 + """ + if self._closed: + return False + with self._lock: + if len(self._queue) >= self._max_queue_size: + self._queue.popleft() + self._dropped_count += 1 + self._queue.append(item) + self._wakeup.set() + return True + + def flush(self) -> None: + """立即触发一次发送（会阻塞直到处理完成，主要用于关闭时）""" + self._send_batch() + + def close(self) -> None: + """关闭队列，flush剩余数据后停止后台线程""" + if self._closed: + return + self._closed = True + self._wakeup.set() + if self._thread is not None: + self._thread.join(timeout=5.0) + self._send_batch() + + @property + def pending_count(self) -> int: + """当前队列中待发送的数据量""" + with self._lock: + return len(self._queue) + + @property + def dropped_count(self) -> int: + """因队列满而丢弃的数据量""" + return self._dropped_count + + def _run(self) -> None: + """后台线程主循环""" + last_flush = time.time() + while not self._closed: + self._wakeup.wait(timeout=0.1) + self._wakeup.clear() + + now = time.time() + should_flush = False + with self._lock: + if len(self._queue) >= self._batch_size: + should_flush = True + elif now - last_flush >= self._flush_interval and self._queue: + should_flush = True + if should_flush: + self._send_batch() + last_flush = time.time() + + def _send_batch(self) -> None: + """取一批数据处理""" + with self._lock: + if not self._queue: + return + batch = list(self._queue) + self._queue.clear() + try: + self._handler(batch) + except Exception as e: + if self._on_error: + try: + self._on_error(e, batch) + except Exception: + pass + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + return False diff --git a/pymsg/pymsg/rpc.py b/pymsg/pymsg/rpc.py index 74b2977..127e76a 100644 --- a/pymsg/pymsg/rpc.py +++ b/pymsg/pymsg/rpc.py @@ -36,21 +36,33 @@ class RpcClient: rpc = RpcClient("127.0.0.1", 8005, "dnfs") data = rpc.call("execute", {"action": "select", "table": "Users"}) + + # 启动时阻塞等待服务就绪（连接被拒绝时自动重试） + rpc = RpcClient("127.0.0.1", 8010, "static", retry_on_refused=True) """ def __init__(self, addr: str, port: int, name: str = "rpc", - timeout: float = 10.0) -> None: + timeout: float = 10.0, + *, retry_on_refused: bool = False, + retry_interval: float = 1.0) -> None: self._client = MsgClient(addr, port, name) self._timeout = timeout + self._retry_on_refused = retry_on_refused + self._retry_interval = retry_interval def call(self, route: str, payload: "dict | None" = None, *, - timeout: "float | None" = None): + timeout: "float | None" = None, + retry_on_refused: "bool | None" = None, + retry_interval: "float | None" = None): """调用指定路由，返回响应中的 ``data`` 字段。 Args: route: 路由名（注入为 ``_route`` 字段）。 payload: 业务字段 dict（``_route`` 键会被覆盖）。 timeout: 本次调用的超时秒数，缺省用构造时的值。 + retry_on_refused: 本次调用是否在连接被拒绝时阻塞重试， + None 表示使用构造时的值（默认 False）。 + retry_interval: 本次调用的重试间隔，None 表示使用构造时的值。 Returns: 服务端返回的 ``data``（可能为 None）。 @@ -58,13 +70,17 @@ class RpcClient: Raises: RpcError: 服务端返回 success=false，或响应非法。 pymsg.TimeoutError: 等待响应超时。 - ConnectionRefusedError: 目标服务不可达。 + ConnectionRefusedError: 目标服务不可达且未开启重试。 """ req = dict(payload or {}) req["_route"] = route + _retry = self._retry_on_refused if retry_on_refused is None else retry_on_refused + _interval = self._retry_interval if retry_interval is None else retry_interval resp = self._client.call( json.dumps(req, ensure_ascii=False), timeout=self._timeout if timeout is None else timeout, + retry_on_refused=_retry, + retry_interval=_interval, ) try: obj = json.loads(resp) diff --git a/static/README.md b/static/README.md index 06ac0d7..162c6ad 100644 --- a/static/README.md +++ b/static/README.md @@ -90,26 +90,52 @@ static :9200 (Flask + CORS) ## 配置 -`config.ini`： +static 已接入 confs 统一配置服务，confs 是 static 启动的**强依赖**： -```ini -[server] -host = 127.0.0.1 -port = 9200 +1. **confs 统一配置服务** — 必须成功从 confs 拉取 `static.ini`，否则 static 启动失败直接退出（exit code 1） +2. **环境变量覆盖**（最高优先级）— Docker/部署环境通过环境变量覆盖 confs 中的值 -[directory] -root = ./public +### 配置文件 -[ssl] -enabled = false -cert_file = cert.pem -key_file = key.pem +集中托管在 [confs/configs/static.ini](../confs/configs/static.ini)，由 confs 服务分发： -[cors] -# 允许跨域的 Origin 白名单（逗号分隔） -# 注意：生产环境请勿包含 localhost / 127.0.0.1 / 通配符 * -allowed_origins = https://qsont.me,https://yhdet.top -``` +| Section | Key | 说明 | +|---------|-----|------| +| `[server]` | `host` | 监听地址 | +| `[server]` | `port` | 监听端口 | +| `[directory]` | `root` | 静态文件根目录（相对 BASE_DIR） | +| `[ssl]` | `enabled` | 是否启用 TLS（`true`/`false`，默认 `false`） | +| `[ssl]` | `cert_file` | SSL 证书路径（默认 `cert.pem`） | +| `[ssl]` | `key_file` | SSL 私钥路径（默认 `key.pem`） | +| `[cors]` | `allowed_origins` | CORS 白名单，逗号分隔 | + +### 引导配置（confs 连接地址） + +confs 自身的连接地址不通过 confs 获取（避免循环依赖），通过以下方式确定（优先级从高到低）： +- 环境变量 `CONFS_HOST` / `CONFS_PORT` +- 默认值 `127.0.0.1:8010` + +### 环境变量（最高优先级，覆盖 confs 配置值） + +| 变量 | 说明 | +|------|------| +| `LISTEN_ADDR` | 监听地址（容器内需绑定 `0.0.0.0`） | +| `LISTEN_PORT` | 监听端口 | +| `STATIC_ROOT` | 静态文件根目录 | +| `CONFS_HOST` | confs 服务主机名（引导配置，Docker 环境使用服务名） | +| `CONFS_PORT` | confs 服务端口（引导配置） | + +### 启动顺序 + +static 通过 pymsg `retry_on_refused=True` 自动阻塞等待 confs 就绪，**拉取配置阶段不严格要求启动顺序**： +- confs 先启动 → static 立即拉取配置 +- static 先启动 → pymsg 自动重试（每秒一次，每 5 秒打印等待提示），confs 启动后自动连接成功 +- 配置完整性校验：若 static.ini 中缺少必需的 section/key（server.host/port、directory.root、cors.allowed_origins）且无对应环境变量覆盖，仍然报错退出 +- ssl 段为可选项，未配置时默认 `enabled=false`、cert/key 使用默认文件名 + +### 历史遗留：本地 config.ini 已删除 + +旧版本的 `static/config.ini` 本地回退逻辑已移除，配置统一由 confs 服务分发。 ## 依赖 diff --git a/static/_run.sh b/static/_run.sh index 1f44b63..dc49454 100644 --- a/static/_run.sh +++ b/static/_run.sh @@ -1,5 +1,5 @@ #!/bin/bash cd "$(dirname "$0")" || exit source ./venv/bin/activate -#exec ./fsys.bin --config config.ini 1>/dev/null -exec python3 ./fsys.py --config config.ini 1>/dev/null +#exec ./fsys.bin 1>/dev/null +exec python3 ./fsys.py 1>/dev/null diff --git a/static/config.ini b/static/config.ini deleted file mode 100644 index fbccc07..0000000 --- a/static/config.ini +++ /dev/null @@ -1,17 +0,0 @@ -[server] -host = 127.0.0.1 -port = 9200 - -[directory] -root = ./public - -[ssl] -enabled = false -cert_file = cert.pem -key_file = key.pem - -[cors] -# 允许跨域的 Origin 白名单（逗号分隔） -# 注意：生产环境请勿包含 localhost / 127.0.0.1 / 通配符 * -# 本地开发联调时可临时追加 http://localhost:9200，调试完务必移除 -allowed_origins = https://qsont.me,https://yhdet.top diff --git a/static/fsys.py b/static/fsys.py index f2542e2..9a9b201 100644 --- a/static/fsys.py +++ b/static/fsys.py @@ -36,33 +36,35 @@ file_handler.setLevel(logging.INFO) logger.addHandler(file_handler) parser = argparse.ArgumentParser(description='File Server') -parser.add_argument('--config', type=str, default='config.ini', help='Configuration file path') args = parser.parse_args() app = Flask(__name__) apply_security_headers(app) -# ==================== 配置文件加载 ==================== -# 加载顺序：内置默认值 → confs 统一配置服务（static.ini）； -# confs 不可用时 fail-safe 回退本地 config.ini -CONFIG_FILE = args.config -if not os.path.isabs(CONFIG_FILE): - CONFIG_FILE = os.path.join(BASE_DIR, CONFIG_FILE) +# ==================== 配置加载（两层：confs 强依赖 → 环境变量覆盖） ==================== +# confs 是 static 启动的强依赖：必须成功从 confs 拉取 static.ini，否则直接退出。 +# 环境变量优先级最高（Docker 兼容），可覆盖 confs 中的配置值。 +# confs 自身的连接地址通过引导配置确定：环境变量 CONFS_HOST/CONFS_PORT 优先， +# 缺省使用 127.0.0.1:8010（与其他服务一致的约定）。 +# 引导配置：confs 连接地址（不依赖 confs 本身） +_confs_host = os.environ.get('CONFS_HOST', '127.0.0.1') +_confs_port = int(os.environ.get('CONFS_PORT', '8010')) + +# 白名单结构：_merge_ini 仅覆盖此处列出的 section/key，防止意外字段 file_config = { - 'server': {'host': '0.0.0.0', 'port': '5000'}, - 'directory': {'root': './public'}, - 'ssl': {'enabled': 'false', 'cert_file': 'cert.pem', 'key_file': 'key.pem'}, - # CORS 白名单（逗号分隔），生产环境默认值；localhost 仅限本地开发时显式写入 config.ini - 'cors': {'allowed_origins': 'https://qsont.me,https://yhdet.top'}, + 'server': {'host': '', 'port': ''}, + 'directory': {'root': ''}, + 'ssl': {'enabled': '', 'cert_file': '', 'key_file': ''}, + 'cors': {'allowed_origins': ''}, } def _merge_ini(text): """把 ini 文本合并进 file_config（仅覆盖已知的 section/key）。""" - ini = configparser.ConfigParser() + ini = configparser.ConfigParser(interpolation=None) ini.read_string(text) - for section in ['server', 'directory', 'ssl', 'cors']: + for section in file_config: if ini.has_section(section): for key, value in ini[section].items(): if key in file_config[section]: @@ -70,28 +72,60 @@ def _merge_ini(text): def _fetch_from_confs(): - """从 confs 统一配置服务拉取 static.ini 全文，失败返回 None（fail-safe）。""" + """从 confs 统一配置服务拉取 static.ini 全文。 + + 使用 retry_on_refused=True：confs 未就绪时 pymsg 自动阻塞重试（每秒一次， + 每 5 秒打印一次等待提示），直到 confs 启动后自动连接成功。 + RpcError（如配置文件不存在）仍直接报错退出——这是配置错误而非启动时序问题。 + """ + from pymsg import RpcClient, RpcError # noqa: E402 + client = RpcClient(_confs_host, _confs_port, 'static', timeout=5, + retry_on_refused=True) try: - from pymsg import RpcClient - host = os.environ.get('CONFS_HOST', '127.0.0.1') - port = int(os.environ.get('CONFS_PORT', '8010')) - data = RpcClient(host, port, 'static', timeout=3).call( - 'get', {'name': 'static.ini'}) - return data.get('content') - except Exception: - return None + data = client.call('get', {'name': 'static.ini'}) + except RpcError as e: + print(f'[static] 致命错误: 从 confs 获取 static.ini 失败: {e}', + file=sys.stderr) + sys.exit(1) + return data.get('content') _confs_text = _fetch_from_confs() -if _confs_text is not None: - _merge_ini(_confs_text) - logger.info("配置来源: confs 统一配置服务 (static.ini)") - logs.info("static", "配置来源: confs 统一配置服务 (static.ini)") -elif os.path.exists(CONFIG_FILE): - with open(CONFIG_FILE, encoding='utf-8') as f: - _merge_ini(f.read()) - logger.warning(f"confs 不可用，回退本地配置文件 {CONFIG_FILE}") - logs.warning("static", f"confs 不可用，回退本地配置文件 {CONFIG_FILE}") +_merge_ini(_confs_text) + +# 环境变量覆盖（优先级最高，Docker 兼容） +if os.environ.get('LISTEN_ADDR'): + file_config['server']['host'] = os.environ['LISTEN_ADDR'] +if os.environ.get('LISTEN_PORT'): + file_config['server']['port'] = os.environ['LISTEN_PORT'] +if os.environ.get('STATIC_ROOT'): + file_config['directory']['root'] = os.environ['STATIC_ROOT'] + +# 配置完整性校验 +_required = { + 'server': ['host', 'port'], + 'directory': ['root'], + 'cors': ['allowed_origins'], +} +for section, keys in _required.items(): + for key in keys: + if not file_config[section][key]: + print(f'[static] 致命错误: 配置项 [{section}] {key} 缺失（static.ini 中未配置且无环境变量覆盖）', + file=sys.stderr) + sys.exit(1) + +# ssl 段有合理默认（enabled=false，cert/key 有默认文件名），不强制要求配置 +if not file_config['ssl']['enabled']: + file_config['ssl']['enabled'] = 'false' +if not file_config['ssl']['cert_file']: + file_config['ssl']['cert_file'] = 'cert.pem' +if not file_config['ssl']['key_file']: + file_config['ssl']['key_file'] = 'key.pem' + +_config_source = f'confs (static.ini @ {_confs_host}:{_confs_port})' +print(f'[static] 配置来源: {_config_source}') +logger.info(f"配置来源: {_config_source}") +logs.info("static", f"配置来源: {_config_source}") # ==================== CORS（配置加载完成后初始化） ==================== _cors_origins = [o.strip() for o in file_config['cors']['allowed_origins'].split(',') if o.strip()] diff --git a/static/log_client.py b/static/log_client.py index 0dd516f..f0b95b6 100644 --- a/static/log_client.py +++ b/static/log_client.py @@ -1,13 +1,15 @@ -"""统一日志客户端 — 各服务共用（dnfs / smtp / static / confs 四份副本内容一致，改动请同步）。 +""" +统一日志客户端 — 各服务共用（基于 pymsg.AsyncRpcQueue）。 设计要点： - 通过 pymsg RPC 调用 dbv2 execute 路由，逐条 INSERT 到 Logs 表 - - 异步非阻塞：内存队列 + 后台线程批量推送，不拖慢业务 - - fail-safe：db 服务不可用时静默丢弃，绝不影响业务；后台线程任何异常都不会崩溃 + - 异步非阻塞：内存队列 + 后台线程攒批推送，不拖慢业务 + - fail-safe：db 服务不可用时静默丢弃，绝不影响业务 - 缓冲批量：队列满 50 条或超时 2 秒时自动 flush - - 时间戳：日志入队时即记录 ts，反映事件真实发生时间 + - 时间戳：日志入队时即记录 ts（UTC ISO 格式），反映事件真实发生时间 + - 底层复用 pymsg.AsyncRpcQueue 通用队列基础设施 -用法： +用法（与旧版完全兼容）： from log_client import logs logs.info("dnfs", "节点新增成功 id=42") @@ -19,28 +21,25 @@ import json import os as _os import sys as _sys -import threading -import queue -import time import atexit from datetime import datetime, timezone _sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), "..")) -from pymsg import RpcClient, RpcError # noqa: E402 +from pymsg import RpcClient, RpcError, AsyncRpcQueue # noqa: E402 # ── 配置 ────────────────────────────────────────────────────────────── _DBV2_ADDR = _os.environ.get('DB_HOST', '127.0.0.1') _DBV2_PORT = int(_os.environ.get('DB_PORT', '8005')) -_BATCH_SIZE = 50 # 队列满 50 条时 flush -_FLUSH_INTERVAL = 2.0 # 超时 2 秒强制 flush -_TIMEOUT = 5 # socket 超时（秒） -_QUEUE_MAXSIZE = 10000 # 队列上限，防止后台线程异常时内存无限增长；满则丢弃最旧条目 +_BATCH_SIZE = 50 +_FLUSH_INTERVAL = 2.0 +_TIMEOUT = 5 +_QUEUE_MAXSIZE = 10000 def _safe_json(obj): - """安全序列化 extra 字段，不可序列化对象转为 repr 字符串，保证不抛异常。""" + """安全序列化 extra 字段，不可序列化对象转为 repr 字符串。""" try: return json.dumps(obj, ensure_ascii=False, default=str) except Exception: @@ -48,163 +47,80 @@ def _safe_json(obj): def _now_iso(): - """生成当前 UTC 时间 ISO 字符串（在入队时调用，保证时间戳准确）。""" + """生成当前 UTC 时间 ISO 字符串（入队时调用，保证时间戳准确）。""" return datetime.now(timezone.utc).isoformat() class LogClient: - """异步批量日志推送客户端。""" + """异步批量日志推送客户端（API 与旧版完全兼容）。""" - def __init__(self, service_name="log", - batch_size=_BATCH_SIZE, - flush_interval=_FLUSH_INTERVAL): + def __init__(self, service_name: str = "log", + batch_size: int = _BATCH_SIZE, + flush_interval: float = _FLUSH_INTERVAL): self._service_name = service_name - self._batch_size = batch_size - self._flush_interval = flush_interval - self._queue = queue.Queue(maxsize=_QUEUE_MAXSIZE) - self._stop = threading.Event() - self._flush_signal = threading.Event() - self._flush_done = threading.Event() - self._thread = threading.Thread(target=self._flush_loop, - daemon=True, name='log-client') - self._thread.start() - atexit.register(self._shutdown) + self._client = RpcClient( + _DBV2_ADDR, _DBV2_PORT, + service_name + "-log", + timeout=_TIMEOUT, + ) + self._queue = AsyncRpcQueue( + handler=self._send_batch, + batch_size=batch_size, + flush_interval=flush_interval, + max_queue_size=_QUEUE_MAXSIZE, + on_error=self._on_error, + ) + self._queue.start() + atexit.register(self.close) # ── 公开接口 ──────────────────────────────────────────────────────── - def push(self, service, level, message, trace_id=None, extra=None): + def push(self, service: str, level: str, message: str, + trace_id: str = None, extra=None): """推送一条日志（非阻塞，立即返回）。 队列满时静默丢弃最旧条目再放入新条目，保证不阻塞业务线程、不导致 OOM。 """ entry = { - 'ts': _now_iso(), # 入队时即生成时间戳，反映真实事件时间 + 'ts': _now_iso(), 'service': service, 'level': level, 'message': message, 'trace_id': trace_id, 'extra': extra, } - try: - self._queue.put_nowait(entry) - except queue.Full: - # 队列满：丢弃最旧的一条，再放入新条目（drop-oldest 策略） - try: - self._queue.get_nowait() - except queue.Empty: - pass - try: - self._queue.put_nowait(entry) - except queue.Full: - pass # 极端竞争下仍失败则静默丢弃 + self._queue.put(entry) - def debug(self, service, message, **kw): + def debug(self, service: str, message: str, **kw): self.push(service, 'DEBUG', message, **kw) - def info(self, service, message, **kw): + def info(self, service: str, message: str, **kw): self.push(service, 'INFO', message, **kw) - def warning(self, service, message, **kw): + def warning(self, service: str, message: str, **kw): self.push(service, 'WARNING', message, **kw) - def error(self, service, message, **kw): + def warn(self, service: str, message: str, **kw): + self.push(service, 'WARNING', message, **kw) + + def error(self, service: str, message: str, **kw): self.push(service, 'ERROR', message, **kw) - def critical(self, service, message, **kw): + def critical(self, service: str, message: str, **kw): self.push(service, 'CRITICAL', message, **kw) def flush(self): - """手动 flush 队列中剩余的日志（含后台线程持有的局部 batch）。""" - self._flush_done.clear() - self._flush_signal.set() # 通知后台线程立即 flush 其局部 batch - # 等待后台线程完成 flush（最多等待 flush_interval + 2 秒余量） - self._flush_done.wait(timeout=self._flush_interval + 2.0) - # 主线程排空队列中剩余条目 - self._drain_queue_to_db() + """手动 flush 队列中剩余的日志。""" + self._queue.flush() - def _drain_queue_to_db(self): - """排空主线程可见的队列中剩余条目，直接发送到 db。""" - batch = [] - while True: - try: - batch.append(self._queue.get_nowait()) - if len(batch) >= self._batch_size: - self._send(batch) - batch = [] - except queue.Empty: - break - if batch: - self._send(batch) + def close(self): + """关闭客户端，flush 剩余日志。""" + self._queue.close() # ── 内部实现 ──────────────────────────────────────────────────────── - def _flush_loop(self): - """后台线程：积攒到 batch_size 或超时 flush_interval 时发送。 - - 外层捕获所有异常，保证后台线程在任何情况下（网络故障、序列化错误等） - 都不会死亡，避免日志队列无限增长导致 OOM。 - """ - batch = [] - last_flush = time.monotonic() - - while not self._stop.is_set(): - try: - timeout = self._flush_interval - (time.monotonic() - last_flush) - chk_timeout = max(0.1, min(timeout, 0.5)) - try: - item = self._queue.get(timeout=chk_timeout) - batch.append(item) - while len(batch) < self._batch_size: - try: - batch.append(self._queue.get_nowait()) - except queue.Empty: - break - except queue.Empty: - pass - - should_flush = ( - len(batch) >= self._batch_size or - self._flush_signal.is_set() or - (batch and time.monotonic() - last_flush >= self._flush_interval) - ) - if should_flush and batch: - try: - self._send(batch) - except Exception as e: - # _send 内部已有细粒度异常处理，此处是最终安全网 - print(f"[log_client] _send 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - batch = [] - last_flush = time.monotonic() - if self._flush_signal.is_set(): - self._flush_signal.clear() - self._flush_done.set() - except Exception as e: - # 最外层安全网：任何意外异常都不应杀死日志线程 - print(f"[log_client] _flush_loop 异常（已捕获，线程继续运行）: {e}", - file=_sys.stderr) - time.sleep(0.5) # 防止异常风暴导致 CPU 满载 - - # 退出前 flush 剩余 - if batch: - try: - self._send(batch) - except Exception as e: - print(f"[log_client] 退出时 flush 异常: {e}", file=_sys.stderr) - self._flush_done.set() - - def _send(self, batch): - """通过 pymsg RPC（execute 路由）批量发送到 dbv2 Logs 表。 - - 整个 batch 复用一个 TCP 连接；网络错误时放弃本批剩余条目（fail-safe）， - 不抛异常给调用方。 - """ - try: - client = RpcClient(_DBV2_ADDR, _DBV2_PORT, self._service_name + "-log") - except Exception as e: - print(f"[log_client] 创建 RpcClient 失败: {e}", file=_sys.stderr) - return - + def _send_batch(self, batch: list): + """后台线程回调：逐条 INSERT 到 db（与旧版行为一致：网络错误放弃本批剩余）。""" for entry in batch: extra_val = entry.get("extra") try: @@ -224,21 +140,21 @@ class LogClient: }, } try: - client.call("execute", payload, timeout=_TIMEOUT) + self._client.call("execute", payload, timeout=_TIMEOUT) except RpcError as e: - # 业务错误（如表结构不符）：打印后继续下一条 print(f"[log_client] INSERT 失败: {e}", file=_sys.stderr) except Exception as e: - # 网络错误：db 不可用，放弃本批剩余（fail-safe） - print(f"[log_client] 发送失败: {e}", file=_sys.stderr) + print(f"[log_client] 发送失败（放弃本批剩余）: {e}", file=_sys.stderr) break - def _shutdown(self): - """进程退出时 flush 剩余日志。""" - self._stop.set() - self.flush() + def _on_error(self, e: Exception, batch: list): + """异常安全网：handler 外的异常也不会崩溃线程。""" + print(f"[log_client] _send_batch 异常（已捕获，线程继续运行）: {e}", + file=_sys.stderr) # ── 全局单例 ────────────────────────────────────────────────────────── +# 各服务的 log_client.py 副本需将 service_name 改为自己的服务名。 +# 此文件（confs/log_client.py）的 service_name 为 "confs"。 -logs = LogClient() +logs = LogClient(service_name="static") diff --git a/static/public/aibox.html b/static/public/aibox.html index 89fb608..e8b01da 100644 --- a/static/public/aibox.html +++ b/static/public/aibox.html @@ -20,10 +20,12 @@
- + +
+