From f9c02cd6adaa7335db9e7080fbd6b232e93381ff Mon Sep 17 00:00:00 2001 From: qing <1776177662@qq.com> Date: Sun, 2 Aug 2026 12:50:23 +0800 Subject: [PATCH] =?UTF-8?q?feat(redirect80):=20new=20HTTP=E2=86=92HTTPS=20?= =?UTF-8?q?redirect=20service=20on=20port=2080?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure Python stdlib http.server + ThreadingMixIn, zero web framework deps. - GET/HEAD → 301, POST/PUT/DELETE/PATCH → 308 - Reads Host header, strips port, redirects to https://{host}{path} - Config via confs (retry_on_refused, hard dependency) + env var override - Logs via log_client → db Logs table (fail-safe async batch) - Port 80 privilege via setcap cap_net_bind_service in _run.sh --- .gitignore | 1 + confs/configs/redirect80.ini | 5 + redirect80/.gitignore | 109 +++++++++++++++++++++ redirect80/README.md | 73 ++++++++++++++ redirect80/_run.sh | 16 ++++ redirect80/_set.sh | 5 + redirect80/log_client.py | 158 ++++++++++++++++++++++++++++++ redirect80/redirect80.py | 181 +++++++++++++++++++++++++++++++++++ redirect80/requirements.txt | 1 + 9 files changed, 549 insertions(+) create mode 100644 confs/configs/redirect80.ini create mode 100644 redirect80/.gitignore create mode 100644 redirect80/README.md create mode 100755 redirect80/_run.sh create mode 100755 redirect80/_set.sh create mode 100644 redirect80/log_client.py create mode 100644 redirect80/redirect80.py create mode 100644 redirect80/requirements.txt diff --git a/.gitignore b/.gitignore index 544d409..1c95536 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ gateway/shufeng.ini core/Nidus.service.shufeng ai_review/.env confs/__pycache__/ +.workbuddy/ libhttp/test_server *.pyc __pycache__/ diff --git a/confs/configs/redirect80.ini b/confs/configs/redirect80.ini new file mode 100644 index 0000000..8f122c3 --- /dev/null +++ b/confs/configs/redirect80.ini @@ -0,0 +1,5 @@ +[server] +host = 0.0.0.0 +port = 80 +redirect_port = 443 +redirect_code = 301 diff --git a/redirect80/.gitignore b/redirect80/.gitignore new file mode 100644 index 0000000..461da01 --- /dev/null +++ b/redirect80/.gitignore @@ -0,0 +1,109 @@ +#prjs +*.bin +*.build +*.dist +*.onefile-build +nuitka-crash-report.xml + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/redirect80/README.md b/redirect80/README.md new file mode 100644 index 0000000..52dfbd6 --- /dev/null +++ b/redirect80/README.md @@ -0,0 +1,73 @@ +# redirect80 — HTTP→HTTPS 重定向服务 + +基于 Python 内置 `http.server` 的极简 HTTP 重定向服务，监听 80 端口，将所有 HTTP 请求重定向到 HTTPS (443)，实现浏览器和搜索引擎友好的 HTTP→HTTPS 无缝升级。 + +## 设计 + +- **零 Web 框架依赖**：纯 Python 标准库 `http.server` + `ThreadingMixIn` 多线程并发 +- **智能重定向**：GET/HEAD → 301（搜索引擎友好永久重定向），POST/PUT/DELETE/PATCH → 308（保留方法和 body） +- **Host 保留**：读取请求 `Host` 头构造重定向 URL（剥离端口），回退用配置的监听地址 +- **日志集成**：通过 `log_client.py` 异步批量推送运行日志到 `db:8005` 的 Logs 表（fail-safe） +- **配置分层**：confs 强依赖 → 环境变量覆盖（`REDIRECT80_*`），confs 不可用时直接退出不做本地回退 + +## 配置 + +配置通过 confs 统一配置服务分发（`confs/configs/redirect80.ini`），启动时经 pymsg 阻塞等待拉取（retry_on_refused），confs 不可用时进程直接退出。环境变量优先级最高。 + +| 配置项 | 环境变量 | 默认值 | 说明 | +|--------|----------|--------|------| +| `host` | `REDIRECT80_HOST` | `0.0.0.0` | 监听地址 | +| `port` | `REDIRECT80_PORT` | `80` | 监听端口 | +| `redirect_port` | `REDIRECT80_REDIRECT_PORT` | `443` | 重定向目标 HTTPS 端口 | +| `redirect_code` | `REDIRECT80_REDIRECT_CODE` | `301` | 非 GET/HEAD 重定向状态码（308 vs 307） | + +## 安装 & 运行 + +```bash +cd redirect80 +bash _set.sh # 创建 venv 并安装依赖 +bash _run.sh # 运行（尝试自动设置 cap_net_bind_service） +``` + +## 端口权限 + +80 端口是 Linux 特权端口（<1024），需要 `CAP_NET_BIND_SERVICE` 能力或 root 权限。`_run.sh` 会自动尝试： + +```bash +sudo setcap 'cap_net_bind_service=+ep' venv/bin/python3 +``` + +如果 `setcap` 不可用或失败，建议使用 systemd 的 `AmbientCapabilities=CAP_NET_BIND_SERVICE`： + +```ini +[Service] +AmbientCapabilities=CAP_NET_BIND_SERVICE +ExecStart=/opt/micro_servers/redirect80/_run.sh +``` + +## 在 core 的 init.txt 中 + +``` +bash /opt/micro_servers/redirect80/_run.sh +``` + +建议放在 waf 之前： + +``` +bash /opt/micro_servers/db/_run.sh +bash /opt/micro_servers/confs/_run.sh +bash /opt/micro_servers/redirect80/_run.sh # ← 80→443 重定向 +bash /opt/micro_servers/waf/_run.sh +bash /opt/micro_servers/gateway/_run.sh +... +``` + +## 端口一览 + +| 端口 | 绑定地址 | 说明 | +|------|----------|------| +| 80 | 0.0.0.0 | HTTP 入口，301/308 → HTTPS :443 | + +## 许可证 + +MIT License diff --git a/redirect80/_run.sh b/redirect80/_run.sh new file mode 100755 index 0000000..d181884 --- /dev/null +++ b/redirect80/_run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +cd "$(dirname "$0")" || exit +source venv/bin/activate + +# 80 端口是特权端口（<1024），需要 CAP_NET_BIND_SERVICE 能力 +# 尝试自动设置 cap（失败不阻塞，用户可手动 sudo 或用 systemd AmbientCapabilities） +_python_bin="$(readlink -f venv/bin/python3)" +if command -v setcap &>/dev/null; then + if sudo setcap 'cap_net_bind_service=+ep' "$_python_bin" 2>/dev/null; then + echo "[redirect80] 已为 $_python_bin 设置 cap_net_bind_service" + else + echo "[redirect80] ⚠ 无法设置 cap_net_bind_service，可能需要 root 运行" + fi +fi + +exec python3 redirect80.py diff --git a/redirect80/_set.sh b/redirect80/_set.sh new file mode 100755 index 0000000..c7a534e --- /dev/null +++ b/redirect80/_set.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd "$(dirname "$0")" || exit +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple diff --git a/redirect80/log_client.py b/redirect80/log_client.py new file mode 100644 index 0000000..8529be7 --- /dev/null +++ b/redirect80/log_client.py @@ -0,0 +1,158 @@ +""" +统一日志客户端 — redirect80 服务副本（基于 pymsg.AsyncRpcQueue）。 + +设计要点： + - 通过 pymsg RPC 调用 db execute 路由，逐条 INSERT 到 Logs 表 + - 异步非阻塞：内存队列 + 后台线程攒批推送，不拖慢业务 + - fail-safe：db 服务不可用时静默丢弃，绝不影响业务 + - 缓冲批量：队列满 50 条或超时 2 秒时自动 flush + - 时间戳：日志入队时即记录 ts（UTC ISO 格式），反映事件真实发生时间 + - 底层复用 pymsg.AsyncRpcQueue 通用队列基础设施 + +用法（与旧版完全兼容）： + from log_client import logs + + logs.info("redirect80", "服务已启动") + logs.error("redirect80", "端口绑定失败", extra={"errno": 98}) + logs.push("redirect80", "WARNING", "confs 不可用，使用本地配置", + trace_id="startup", extra={"fallback": "redirect80.ini"}) +""" + +import json +import os as _os +import sys as _sys +import atexit +from datetime import datetime, timezone + +_sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), "..")) +from pymsg import RpcClient, RpcError, AsyncRpcQueue # noqa: E402 + +# ── 配置 ────────────────────────────────────────────────────────────── + +_DBV2_ADDR = _os.environ.get('DB_HOST', '127.0.0.1') +_DBV2_PORT = int(_os.environ.get('DB_PORT', '8005')) + +_BATCH_SIZE = 50 +_FLUSH_INTERVAL = 2.0 +_TIMEOUT = 5 +_QUEUE_MAXSIZE = 10000 + + +def _safe_json(obj): + """安全序列化 extra 字段，不可序列化对象转为 repr 字符串。""" + try: + return json.dumps(obj, ensure_ascii=False, default=str) + except Exception: + return repr(obj) + + +def _now_iso(): + """生成当前 UTC 时间 ISO 字符串（入队时调用，保证时间戳准确）。""" + return datetime.now(timezone.utc).isoformat() + + +class LogClient: + """异步批量日志推送客户端（API 与旧版完全兼容）。""" + + def __init__(self, service_name: str = "log", + batch_size: int = _BATCH_SIZE, + flush_interval: float = _FLUSH_INTERVAL): + self._service_name = service_name + self._client = RpcClient( + _DBV2_ADDR, _DBV2_PORT, + service_name + "-log", + timeout=_TIMEOUT, + ) + self._queue = AsyncRpcQueue( + handler=self._send_batch, + batch_size=batch_size, + flush_interval=flush_interval, + max_queue_size=_QUEUE_MAXSIZE, + on_error=self._on_error, + ) + self._queue.start() + atexit.register(self.close) + + # ── 公开接口 ──────────────────────────────────────────────────────── + + def push(self, service: str, level: str, message: str, + trace_id: str = None, extra=None): + """推送一条日志（非阻塞，立即返回）。 + + 队列满时静默丢弃最旧条目再放入新条目，保证不阻塞业务线程、不导致 OOM。 + """ + entry = { + 'ts': _now_iso(), + 'service': service, + 'level': level, + 'message': message, + 'trace_id': trace_id, + 'extra': extra, + } + self._queue.put(entry) + + def debug(self, service: str, message: str, **kw): + self.push(service, 'DEBUG', message, **kw) + + def info(self, service: str, message: str, **kw): + self.push(service, 'INFO', message, **kw) + + def warning(self, service: str, message: str, **kw): + self.push(service, 'WARNING', message, **kw) + + def warn(self, service: str, message: str, **kw): + self.push(service, 'WARNING', message, **kw) + + def error(self, service: str, message: str, **kw): + self.push(service, 'ERROR', message, **kw) + + def critical(self, service: str, message: str, **kw): + self.push(service, 'CRITICAL', message, **kw) + + def flush(self): + """手动 flush 队列中剩余的日志。""" + self._queue.flush() + + def close(self): + """关闭客户端，flush 剩余日志。""" + self._queue.close() + + # ── 内部实现 ──────────────────────────────────────────────────────── + + def _send_batch(self, batch: list): + """后台线程回调：逐条 INSERT 到 db（网络错误放弃本批剩余）。""" + for entry in batch: + extra_val = entry.get("extra") + try: + extra_json = _safe_json(extra_val) if extra_val is not None else None + except Exception: + extra_json = None + payload = { + "action": "insert", + "table": "Logs", + "values": { + "ts": entry.get("ts", _now_iso()), + "service": entry.get("service"), + "level": entry.get("level"), + "message": entry.get("message"), + "trace_id": entry.get("trace_id"), + "extra": extra_json, + }, + } + try: + self._client.call("execute", payload, timeout=_TIMEOUT) + except RpcError as e: + print(f"[log_client] INSERT 失败: {e}", file=_sys.stderr) + except Exception as e: + print(f"[log_client] 发送失败（放弃本批剩余）: {e}", file=_sys.stderr) + break + + def _on_error(self, e: Exception, batch: list): + """异常安全网：handler 外的异常也不会崩溃线程。""" + print(f"[log_client] _send_batch 异常（已捕获，线程继续运行）: {e}", + file=_sys.stderr) + + +# ── 全局单例 ────────────────────────────────────────────────────────── + +logs = LogClient(service_name="redirect80") diff --git a/redirect80/redirect80.py b/redirect80/redirect80.py new file mode 100644 index 0000000..95ee02e --- /dev/null +++ b/redirect80/redirect80.py @@ -0,0 +1,181 @@ +""" +redirect80 — HTTP 80 端口重定向服务。 + +监听 80 端口，所有 HTTP 请求返回 301/308 重定向到 HTTPS (443)， +保留原始 Host 和路径。为搜索引擎和浏览器提供 HTTP→HTTPS 无缝升级。 + +技术选型：Python 内置 http.server + 多线程混合模式，无外部 Web 框架依赖。 + 日志通过 log_client 异步推送到 db 的 Logs 表（fail-safe）。 + +配置（两层加载）： + 1. confs 强依赖拉取 redirect80.ini（retry_on_refused 阻塞等待） + 2. 环境变量覆盖（Docker/k8s 兼容） +confs 不可用时直接退出，不做本地回退。 +""" + +import configparser +import os +import sys as _sys +from http.server import HTTPServer, BaseHTTPRequestHandler +from socketserver import ThreadingMixIn + +# ── 路径设置 ────────────────────────────────────────────────────────── +_here = os.path.dirname(os.path.abspath(__file__)) +_sys.path.insert(0, os.path.join(_here, "..")) +from log_client import logs # noqa: E402 + +# ── 配置加载（confs 强依赖 → 环境变量覆盖）──────────────────────── +_config = { + "host": "0.0.0.0", + "port": 80, + "redirect_port": 443, + "redirect_code": 301, +} + +# 1. confs 强依赖拉取配置（阻塞等待 confs 就绪，失败则退出） +_sys.path.insert(0, os.path.join(_here, "..", "pymsg")) +from pymsg import RpcClient # noqa: E402 + +_confs_host = os.environ.get("CONFS_HOST", "127.0.0.1") +_confs_port = int(os.environ.get("CONFS_PORT", "8010")) +_confs_client = RpcClient( + _confs_host, _confs_port, "redirect80", + retry_on_refused=True, timeout=5, +) +_ini_text = _confs_client.call("get", {"name": "redirect80.ini"}) +_ini = configparser.ConfigParser(interpolation=None) +_ini.read_string(_ini_text) + +_merge_map = { + ("server", "host"): "host", + ("server", "port"): "port", + ("server", "redirect_port"): "redirect_port", + ("server", "redirect_code"): "redirect_code", +} +for (sec, key), var in _merge_map.items(): + if _ini.has_option(sec, key): + val = _ini.get(sec, key) + _config[var] = type(_config[var])(val) +logs.info("redirect80", "配置已从 confs 加载") + +# 2. 环境变量最高优先级覆盖 +_env_map = { + "REDIRECT80_HOST": "host", + "REDIRECT80_PORT": ("port", int), + "REDIRECT80_REDIRECT_PORT": ("redirect_port", int), + "REDIRECT80_REDIRECT_CODE": ("redirect_code", int), +} +for env_var, mapping in _env_map.items(): + val = os.environ.get(env_var) + if val is not None: + if isinstance(mapping, tuple): + _config[mapping[0]] = mapping[1](val) + else: + _config[mapping] = val + +# ── 请求处理器 ────────────────────────────────────────────────────────── + + +class RedirectHandler(BaseHTTPRequestHandler): + """HTTP 请求处理器：将所有请求重定向到 HTTPS。 + + GET/HEAD → 301 (Moved Permanently)，搜索引擎友好的永久重定向。 + POST/PUT/DELETE/PATCH → 308 (Permanent Redirect)，保留请求方法和 body。 + """ + + server_version = "UniServe/redirect80" + sys_version = "" + + def _redirect(self, code: int): + """构造 HTTPS 重定向 URL 并返回。""" + host = self.headers.get("Host", "") + # 剥离端口号（如果客户端带了 :80） + hostname = host.split(":")[0] if host else _config["host"] + target_port = _config["redirect_port"] + # 标准 443 端口不显式写 :443 + if target_port == 443: + location = f"https://{hostname}{self.path}" + else: + location = f"https://{hostname}:{target_port}{self.path}" + + self.send_response(code) + self.send_header("Location", location) + self.send_header("Connection", "close") + self.send_header("Cache-Control", "max-age=31536000") + self.end_headers() + + # 日志：仅记录摘要（不刷屏） + logs.info("redirect80", + f"{code} {self.command} {host} → {location}") + + def do_GET(self): + self._redirect(301) + + def do_HEAD(self): + self._redirect(301) + + def do_POST(self): + self._redirect(308) + + def do_PUT(self): + self._redirect(308) + + def do_DELETE(self): + self._redirect(308) + + def do_PATCH(self): + self._redirect(308) + + def do_OPTIONS(self): + self.send_response(204) + self.send_header("Allow", "GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONS") + self.send_header("Connection", "close") + self.end_headers() + + def log_message(self, format, *args): + """抑制默认的 stderr 日志（已通过 log_client 记录）。""" + pass + + +# ── 多线程 HTTP 服务器 ────────────────────────────────────────────────── + + +class ThreadingRedirectServer(ThreadingMixIn, HTTPServer): + """多线程混合 HTTP 服务器，处理并发连接。 + + ThreadingMixIn 为每个请求创建新线程，适合 redirect80 + 这种请求处理极快（无 I/O 阻塞）的场景。 + """ + daemon_threads = True + allow_reuse_address = True + + def handle_error(self, request, client_address): + """覆盖默认错误处理，异常写入 log_client 而非 stderr。""" + logs.error("redirect80", + f"处理 {client_address} 请求时异常", + extra={"client": str(client_address)}) + + +def main(): + host = _config["host"] + port = int(_config["port"]) + + server = ThreadingRedirectServer((host, port), RedirectHandler) + + logs.info("redirect80", + f"HTTP→HTTPS 重定向服务启动 " + f"http://{host}:{port} → https://*:{_config['redirect_port']}") + + print(f"[redirect80] 监听 http://{host}:{port} " + f"→ 重定向到 :{_config['redirect_port']}", file=_sys.stderr) + + try: + server.serve_forever() + except KeyboardInterrupt: + logs.info("redirect80", "收到 SIGINT，正在关闭...") + server.shutdown() + logs.info("redirect80", "服务已停止") + + +if __name__ == "__main__": + main() diff --git a/redirect80/requirements.txt b/redirect80/requirements.txt new file mode 100644 index 0000000..4d41e83 --- /dev/null +++ b/redirect80/requirements.txt @@ -0,0 +1 @@ +../pymsg