update

db/.dockerignore

@@ -0,0 +1,20 @@
+.env
+.env/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.so
+.git
+.gitignore
+*.md
+!README.md
+.envrc
+.venv/
+venv/
+ENV/
+*.db
+*.sqlite
+*.sqlite3
+.DS_Store

db/Dockerfile

@@ -0,0 +1,53 @@
+FROM python:3.11-slim
+
+LABEL maintainer="qingsong"
+LABEL service="db"
+LABEL description="dbv2 - pymsg协议数据库服务"
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+# 安装pymsg共享库
+COPY pymsg/ /app/pymsg/
+RUN pip install /app/pymsg/
+
+# 安装依赖
+COPY db/requirements.txt /app/
+RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+# 复制应用代码
+COPY db/ /app/db/
+
+# 入口脚本
+RUN chmod +x /app/db/docker-entrypoint.sh
+
+# 创建数据持久化目录并设置权限
+RUN mkdir -p /app/data && chown -R 1000:1000 /app/data
+VOLUME ["/app/data"]
+
+# 创建非root用户
+RUN useradd -m -u 1000 appuser && chown -R appuser:appuser /app
+USER appuser
+
+WORKDIR /app/db
+
+EXPOSE 8005
+
+# 健康检查：尝试连接TCP端口
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+    CMD python -c "import socket; s=socket.socket(); s.settimeout(2); s.connect(('localhost', 8005)); s.close()" || exit 1
+
+# 默认使用SQLite，数据持久化到/app/data/database.db
+ENV DATABASE_URL=sqlite:////app/data/database.db \
+    LISTEN_ADDR=0.0.0.0 \
+    LISTEN_PORT=8005 \
+    WORKERS=4
+
+ENTRYPOINT ["/app/db/docker-entrypoint.sh"]

db/db_tools.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+"""
+db_tools — SQLite 数据库备份、导出、导入工具
+
+用法:
+  python db_tools.py backup <output_path>     # 安全备份数据库（使用 sqlite3 backup API）
+  python db_tools.py export <output.json>     # 导出所有表数据为 JSON
+  python db_tools.py import <input.json>      # 从 JSON 导入数据
+  python db_tools.py vacuum                   # 执行 VACUUM 压缩数据库
+  python db_tools.py wal-checkpoint           # 强制 WAL 检查点（确保数据写入主库）
+  python db_tools.py info                     # 显示数据库信息和表统计
+"""
+
+import json
+import os
+import shutil
+import sys
+import sqlite3
+from datetime import datetime, timezone
+
+DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///database.db")
+
+# 从 sqlite:///path 提取文件路径
+def _get_db_path() -> str:
+    if DATABASE_URL.startswith("sqlite:////"):
+        return DATABASE_URL[10:]  # 绝对路径 sqlite:////app/data/xxx.db
+    elif DATABASE_URL.startswith("sqlite:///"):
+        return DATABASE_URL[9:]   # 相对路径
+    else:
+        return "database.db"
+
+
+def _connect() -> sqlite3.Connection:
+    """创建数据库连接（配置 PRAGMA）。"""
+    conn = sqlite3.connect(_get_db_path())
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def backup(output_path: str):
+    """
+    使用 SQLite 官方 backup API 进行在线热备份（不需要停机）。
+    这是最安全的备份方式，支持读写并发。
+    """
+    src = _get_db_path()
+    print(f"[backup] 正在备份数据库: {src} -> {output_path}")
+    
+    if not os.path.exists(src):
+        print(f"[backup] 错误: 数据库文件不存在: {src}", file=sys.stderr)
+        sys.exit(1)
+    
+    start = datetime.now(timezone.utc)
+    
+    # 使用 sqlite3 backup API（支持在线备份）
+    src_conn = sqlite3.connect(src)
+    dst_conn = sqlite3.connect(output_path)
+    try:
+        src_conn.backup(dst_conn)
+        dst_conn.commit()
+    finally:
+        dst_conn.close()
+        src_conn.close()
+    
+    size_mb = os.path.getsize(output_path) / (1024 * 1024)
+    elapsed = (datetime.now(timezone.utc) - start).total_seconds()
+    print(f"[backup] 备份完成: {output_path} ({size_mb:.2f} MB, 耗时 {elapsed:.2f}s)")
+
+
+def export_json(output_path: str):
+    """导出所有表数据为 JSON 格式（便于跨平台传输/迁移）。"""
+    print(f"[export] 正在导出数据到: {output_path}")
+    conn = _connect()
+    
+    result = {
+        "_meta": {
+            "exported_at": datetime.now(timezone.utc).isoformat(),
+            "db_path": _get_db_path(),
+            "sqlite_version": sqlite3.sqlite_version,
+        },
+        "tables": {}
+    }
+    
+    try:
+        # 获取所有用户表
+        tables = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
+        ).fetchall()
+        
+        for (table_name,) in tables:
+            # 获取列信息
+            columns = [desc[1] for desc in conn.execute(f"PRAGMA table_info({table_name})").fetchall()]
+            
+            # 获取行数
+            count = conn.execute(f"SELECT COUNT(*) FROM [{table_name}]").fetchone()[0]
+            
+            # 获取所有数据
+            rows = conn.execute(f"SELECT * FROM [{table_name}]").fetchall()
+            data = [dict(row) for row in rows]
+            
+            result["tables"][table_name] = {
+                "columns": columns,
+                "row_count": count,
+                "data": data,
+            }
+            print(f"[export]   表 {table_name}: {count} 行")
+        
+        # 同时导出 schema
+        schema = []
+        for (table_name,) in tables:
+            ddl = conn.execute(
+                "SELECT sql FROM sqlite_master WHERE type='table' AND name=?",
+                (table_name,)
+            ).fetchone()[0]
+            if ddl:
+                schema.append(ddl + ";")
+        
+        result["_meta"]["schema"] = schema
+        
+    finally:
+        conn.close()
+    
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(result, f, ensure_ascii=False, indent=2, default=str)
+    
+    size_mb = os.path.getsize(output_path) / (1024 * 1024)
+    print(f"[export] 导出完成: {output_path} ({size_mb:.2f} MB)")
+
+
+def import_json(input_path: str):
+    """从 JSON 文件导入数据（注意：会清空现有数据！）。"""
+    print(f"[import] 正在从 {input_path} 导入数据")
+    
+    if not os.path.exists(input_path):
+        print(f"[import] 错误: 文件不存在: {input_path}", file=sys.stderr)
+        sys.exit(1)
+    
+    with open(input_path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    
+    conn = _connect()
+    
+    try:
+        # 是否清空现有表（需要确认）
+        tables = data.get("tables", {})
+        if tables:
+            print(f"[import] 警告: 即将导入 {len(tables)} 个表，现有数据将被覆盖!")
+            # 为安全起见，不自动清空，由用户先备份
+            
+            # 按外键依赖顺序导入（先主表后子表，这里简单按名字排序）
+            for table_name in sorted(tables.keys()):
+                table_data = tables[table_name]
+                rows = table_data.get("data", [])
+                columns = table_data.get("columns", [])
+                
+                if not rows or not columns:
+                    print(f"[import]   跳表 {table_name}: 无数据或列定义")
+                    continue
+                
+                # 先清空表
+                conn.execute(f"DELETE FROM [{table_name}]")
+                
+                # 批量插入
+                placeholders = ", ".join(["?"] * len(columns))
+                col_names = ", ".join(f"[{c}]" for c in columns)
+                insert_sql = f"INSERT INTO [{table_name}] ({col_names}) VALUES ({placeholders})"
+                
+                conn.executemany(insert_sql, [tuple(row.get(col) for col in columns) for row in rows])
+                print(f"[import]   表 {table_name}: 导入 {len(rows)} 行")
+            
+            conn.commit()
+            print("[import] 导入完成")
+        else:
+            print("[import] 没有可导入的表数据")
+    finally:
+        conn.close()
+
+
+def vacuum():
+    """执行 VACUUM 压缩数据库，回收空间、重建索引。"""
+    print("[vacuum] 正在执行 VACUUM...")
+    db_path = _get_db_path()
+    before_size = os.path.getsize(db_path) / (1024 * 1024)
+    
+    conn = _connect()
+    try:
+        # WAL 检查点先将所有数据写入主库
+        conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        conn.execute("VACUUM")
+        conn.execute("ANALYZE")
+    finally:
+        conn.close()
+    
+    after_size = os.path.getsize(db_path) / (1024 * 1024)
+    saved = before_size - after_size
+    print(f"[vacuum] 完成: {before_size:.2f} MB -> {after_size:.2f} MB (节省 {saved:.2f} MB)")
+
+
+def wal_checkpoint():
+    """强制 WAL 检查点，确保所有写入都持久化到主数据库文件。"""
+    print("[wal-checkpoint] 执行 WAL 检查点...")
+    conn = _connect()
+    try:
+        result = conn.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
+        # busy, log, checkpointed
+        print(f"[wal-checkpoint] 结果: busy={result[0]}, log_pages={result[1]}, checkpointed={result[2]}")
+    finally:
+        conn.close()
+    print("[wal-checkpoint] 完成")
+
+
+def show_info():
+    """显示数据库信息和表统计。"""
+    db_path = _get_db_path()
+    print("=" * 50)
+    print("SQLite 数据库信息")
+    print("=" * 50)
+    print(f"文件路径: {os.path.abspath(db_path)}")
+    
+    if os.path.exists(db_path):
+        size_mb = os.path.getsize(db_path) / (1024 * 1024)
+        mtime = datetime.fromtimestamp(os.path.getmtime(db_path), tz=timezone.utc)
+        print(f"文件大小: {size_mb:.2f} MB")
+        print(f"最后修改: {mtime.isoformat()}")
+    else:
+        print("文件大小: (不存在)")
+    
+    # WAL 文件
+    wal_path = db_path + "-wal"
+    shm_path = db_path + "-shm"
+    if os.path.exists(wal_path):
+        wal_size = os.path.getsize(wal_path) / 1024
+        print(f"WAL 文件: {wal_size:.2f} KB")
+    if os.path.exists(shm_path):
+        shm_size = os.path.getsize(shm_path) / 1024
+        print(f"SHM 文件: {shm_size:.2f} KB")
+    
+    conn = _connect()
+    try:
+        print(f"SQLite 版本: {sqlite3.sqlite_version}")
+        
+        # PRAGMA 信息
+        journal_mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
+        synchronous = conn.execute("PRAGMA synchronous").fetchone()[0]
+        page_size = conn.execute("PRAGMA page_size").fetchone()[0]
+        page_count = conn.execute("PRAGMA page_count").fetchone()[0]
+        free_pages = conn.execute("PRAGMA freelist_count").fetchone()[0]
+        
+        print(f"Journal 模式: {journal_mode}")
+        print(f"同步级别: {synchronous}")
+        print(f"页大小: {page_size} bytes")
+        
+        # 表统计
+        tables = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
+        ).fetchall()
+        
+        print(f"\n表统计 ({len(tables)} 个表):")
+        print("-" * 40)
+        total_rows = 0
+        for (table_name,) in tables:
+            count = conn.execute(f"SELECT COUNT(*) FROM [{table_name}]").fetchone()[0]
+            total_rows += count
+            print(f"  {table_name:20s} {count:>8d} 行")
+        print("-" * 40)
+        print(f"  {'总计':20s} {total_rows:>8d} 行")
+        
+    finally:
+        conn.close()
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(__doc__)
+        sys.exit(1)
+    
+    cmd = sys.argv[1]
+    
+    if cmd == "backup":
+        output = sys.argv[2] if len(sys.argv) > 2 else f"backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.db"
+        backup(output)
+    elif cmd == "export":
+        output = sys.argv[2] if len(sys.argv) > 2 else f"export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+        export_json(output)
+    elif cmd == "import":
+        if len(sys.argv) < 3:
+            print("用法: python db_tools.py import <input.json>", file=sys.stderr)
+            sys.exit(1)
+        import_json(sys.argv[2])
+    elif cmd == "vacuum":
+        vacuum()
+    elif cmd == "wal-checkpoint":
+        wal_checkpoint()
+    elif cmd == "info":
+        show_info()
+    else:
+        print(f"未知命令: {cmd}", file=sys.stderr)
+        print(__doc__)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

db/dbv2.py

@@ -37,6 +37,7 @@ from datetime import datetime, date, time as dt_time, timezone, timedelta
 from sqlalchemy import (
     create_engine, MetaData, Table,
     select, insert, update, delete, func, and_,
+    event,
 )
 from sqlalchemy import types as sa_types
 from sqlalchemy.exc import SQLAlchemyError
@@ -44,12 +45,32 @@ from sqlalchemy.exc import SQLAlchemyError
 # pymsg 导入
 import sys as _sys
 _sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
-from pymsg import MsgServer  # noqa: E402
+from pymsg import RpcServer  # noqa: E402
 
 # ── 数据库配置 ──────────────────────────────────────────────────────────
 
 DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///database.db")
 
+# ── SQLite 连接配置（针对 Docker/多线程环境优化）──────────────────────
+
+def _configure_sqlite(dbapi_connection, connection_record):
+    """为每个 SQLite 连接设置 PRAGMA，优化并发与持久化性能。
+
+    - WAL 模式：读写不互斥，并发性能大幅提升，Docker 卷挂载下更稳定
+    - NORMAL 同步级别：在断电时可能丢失最近提交的事务（但不会损坏数据库）
+      配合 WAL checkpoint 足够安全，性能显著提升
+    - foreign_keys=ON：启用外键约束
+    - busy_timeout：锁等待超时 5 秒，避免 SQLITE_BUSY 错误
+    - mmap_size：使用内存映射 I/O 提升读取性能
+    """
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA journal_mode=WAL")
+    cursor.execute("PRAGMA synchronous=NORMAL")
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.execute("PRAGMA busy_timeout=5000")
+    cursor.execute("PRAGMA mmap_size=268435456")  # 256MB
+    cursor.close()
+
 # ── 日志模块常量 ────────────────────────────────────────────────────────
 
 MAX_MESSAGE_LEN = 8192   # message 上限 8KB
@@ -80,9 +101,20 @@ class DBServer:
     def __init__(self, addr="127.0.0.1", port=8005, workers=4):
         self.addr = addr
         self.port = port
-        self.engine = create_engine(DATABASE_URL)
+        self.engine = create_engine(
+            DATABASE_URL,
+            connect_args={"check_same_thread": False} if DATABASE_URL.startswith("sqlite") else {},
+        )
+        # 为 SQLite 连接应用 PRAGMA 优化
+        if DATABASE_URL.startswith("sqlite"):
+            event.listen(self.engine, "connect", _configure_sqlite)
         self.metadata = MetaData()
-        self._server = MsgServer(addr, port, workers=workers)
+        self._server = RpcServer(
+            addr, port, workers=workers,
+            json_default=_json_default,
+            on_error=lambda route, e: traceback.print_exc(),
+        )
+        self._server.add_route("execute", self._handle_execute)
 
         # 日志表缓存（避免重复反射）
         self._logs_table_cache = {}
@@ -98,54 +130,18 @@ class DBServer:
         # 抛 InvalidRequestError），反射操作需串行化
         self._reflect_lock = threading.Lock()
 
-        # 路由表
-        self._routes = {
-            "execute": self._handle_execute,
-        }
-
     # ── 启动 / 停止 ──────────────────────────────────────────────────
     def start(self) -> None:
         """启动服务（阻塞当前线程）。"""
         print(f"[dbv2] 监听 tcp://{self.addr}:{self.port}")
-        self._ensure_logs_table()   # 确保 Logs 表已建好（外部 MsgClient 写日志前）
+        self._ensure_logs_table()   # 确保 Logs 表已建好（外部写日志前）
         self._start_cleanup_thread()
-        self._server.start(self._dispatch)
+        self._server.start()
 
     def stop(self) -> None:
         """从另一线程停止服务。"""
         self._server.stop()
 
-    # ── 消息调度 -------------------------------------------------------
-    def _dispatch(self, sock, addr, text):
-        """解析 JSON 中的 _route，分发给对应的 handler，并回复结果。"""
-        try:
-            req = json.loads(text)
-        except json.JSONDecodeError as e:
-            self._reply(sock, False, error=f"JSON 解析失败: {e}")
-            return
-
-        route = req.pop("_route", "")
-        handler = self._routes.get(route)
-        if handler is None:
-            self._reply(sock, False, error=f"未知路由: {route}")
-            return
-
-        try:
-            result = handler(req, sock=sock)
-            self._reply(sock, True, data=result)
-        except Exception as e:
-            traceback.print_exc()
-            self._reply(sock, False, error=str(e))
-
-    def _reply(self, sock, success, data=None, error=None):
-        resp = {"success": success}
-        if data is not None:
-            resp["data"] = data
-        if error is not None:
-            resp["error"] = error
-        self._server.reply(sock, json.dumps(resp, ensure_ascii=False,
-                                            default=_json_default))
-
     # ── 表反射 ---------------------------------------------------------
     def _get_table(self, table_name):
         # double-check：_tables_cache 只在反射完整完成后写入（dict 读写 GIL
@@ -245,7 +241,7 @@ class DBServer:
     # 路由处理 — execute
     # ═══════════════════════════════════════════════════════════════════
 
-    def _handle_execute(self, data, **kw):
+    def _handle_execute(self, data):
         action = data.get("action", "select")
         table_name = data.get("table")
         if not table_name:
@@ -460,9 +456,12 @@ class DBServer:
 if __name__ == "__main__":
     import argparse
 
+    default_addr = os.environ.get("LISTEN_ADDR", "0.0.0.0")
+    default_port = int(os.environ.get("LISTEN_PORT", "8005"))
+
     parser = argparse.ArgumentParser(description="dbv2 — pymsg 数据库服务")
-    parser.add_argument("--addr", default="127.0.0.1", help="监听地址")
-    parser.add_argument("--port", type=int, default=8005, help="监听端口")
+    parser.add_argument("--addr", default=default_addr, help="监听地址")
+    parser.add_argument("--port", type=int, default=default_port, help="监听端口")
     parser.add_argument("--workers", type=int, default=4,
                         help="回调并发线程数（0=同步串行模式）")
     args = parser.parse_args()

db/docker-entrypoint.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")"
+
+DATA_DIR="/app/data"
+DB_PATH="${DATABASE_URL#sqlite:///}"
+if [ "$DB_PATH" = "$DATABASE_URL" ]; then
+    # URL 不是 sqlite:/// 格式，使用默认路径
+    DB_PATH="$DATA_DIR/database.db"
+fi
+
+echo "数据目录: $DATA_DIR"
+echo "数据库路径: $DB_PATH"
+
+# 确保数据目录存在并可写
+mkdir -p "$(dirname "$DB_PATH")"
+
+# 初始化数据库表（如果不存在）
+if [[ "$DATABASE_URL" == sqlite:* ]]; then
+    echo "初始化数据库 schema..."
+    python -c "
+import os
+from sqlalchemy import create_engine, text
+
+db_url = os.environ.get('DATABASE_URL', 'sqlite:////app/data/database.db')
+engine = create_engine(
+    db_url,
+    connect_args={'check_same_thread': False} if db_url.startswith('sqlite') else {},
+)
+# 应用 SQLite PRAGMA 优化
+from sqlalchemy import event
+@event.listens_for(engine, 'connect')
+def _configure_sqlite(dbapi_conn, _):
+    cur = dbapi_conn.cursor()
+    cur.execute('PRAGMA journal_mode=WAL')
+    cur.execute('PRAGMA synchronous=NORMAL')
+    cur.execute('PRAGMA foreign_keys=ON')
+    cur.execute('PRAGMA busy_timeout=5000')
+    cur.close()
+
+with open('create_sqlite3.sql', 'r') as f:
+    schema_sql = f.read()
+with engine.connect() as conn:
+    for stmt in schema_sql.split(';'):
+        stmt = stmt.strip()
+        if stmt:
+            try:
+                conn.execute(text(stmt))
+            except Exception as e:
+                pass
+    conn.commit()
+print('Schema 初始化完成.')
+"
+fi
+
+echo "启动 dbv2 服务 ${LISTEN_ADDR:-0.0.0.0}:${LISTEN_PORT:-8005}..."
+exec python dbv2.py --addr "${LISTEN_ADDR:-0.0.0.0}" --port "${LISTEN_PORT:-8005}" --workers "${WORKERS:-4}"