fix(aibox): FTS5 中文检索 + Token 压缩阈值生效——Compressor 抽离解耦

- db.py: FTS5 改用 trigram 分词（中文可检索），旧 unicode61 表自动迁移回灌；search_memories 增加 LIKE 兜底（%/_ 转义、MATCH 异常防护）
- conversation_compressor.py: should_compress 补齐 Token 阈值判断（原为死配置），Token 触发时按比例压缩早期消息
- ConversationCompressor 抽离至 subsystems，注入 llm_client 打破与 DeepSeekChat 循环引用；aibox.py 3517→3018 行
- 新增 21 项测试（中文检索/旧表迁移/LIKE 转义/token 估算/压缩流程/失败回退），全量 232 项通过

aibox/tests/test_conversation_compressor.py

@@ -0,0 +1,196 @@
+"""subsystems.conversation_compressor 单元测试
+
+覆盖：
+  - estimate_tokens: 中英文/工具调用 token 估算
+  - should_compress: 消息数阈值触发 / Token 阈值触发 / 均未触发 / 功能禁用
+  - compress_conversation: 消息数触发压缩 / Token 触发压缩 / LLM 失败保留原文
+  - _validate_message_structure: 移除孤立 tool 消息
+通过注入 FakeLLMClient 模拟 LLM 调用，不依赖真实 API。
+"""
+
+import pytest
+
+from subsystems import ConversationCompressor
+from common import constants, Logger
+
+
+class FakeLLMClient:
+    """模拟 llm_client 抽象层的 chat_completion"""
+
+    def __init__(self, reply="这是压缩摘要", error=None):
+        self.reply = reply
+        self.error = error
+        self.calls = []
+
+    def chat_completion(self, messages, temperature=0.7, max_tokens=None, **kwargs):
+        self.calls.append({"max_tokens": max_tokens, "n_msgs": len(messages)})
+        if self.error:
+            raise self.error
+        return self.reply
+
+
+def make_messages(n_pairs=15, long_content=False):
+    """构造 n_pairs 轮 user/assistant 对话（含 1 条前导 system）"""
+    msgs = [{"role": "system", "content": "SYS_PROMPT"}]
+    for i in range(n_pairs):
+        content = f"用户消息内容{i}"
+        if long_content:
+            content += "很长的文档内容" * 200
+        msgs.append({"role": "user", "content": content})
+        msgs.append({"role": "assistant", "content": f"助手回复{i}"})
+    return msgs
+
+
+@pytest.fixture
+def compressor():
+    return ConversationCompressor(llm_client=FakeLLMClient(), logger=Logger("TestCompressor", 0))
+
+
+def patch_const(monkeypatch, name, value):
+    """把 Constants 类属性 property 替换为固定值（参照 test_db.py 既有做法）"""
+    monkeypatch.setattr(type(constants), name, property(lambda self: value))
+
+
+# ==================== estimate_tokens ====================
+
+def test_estimate_tokens_chinese(compressor):
+    assert compressor.estimate_tokens([{"role": "user", "content": "你好"}]) == 13  # 2*1.5 + 10
+
+
+def test_estimate_tokens_english(compressor):
+    # 2 词 * 1.3 + (11 字符 - 2 词) * 0.8 + 10 = 19.8 → int 截断为 19
+    # （既有算法：other_chars 按词数抵扣，字母被计入 other，保持原行为）
+    assert compressor.estimate_tokens([{"role": "user", "content": "hello world"}]) == 19
+
+
+def test_estimate_tokens_tool_calls(compressor):
+    msgs = [{
+        "role": "assistant", "content": "",
+        "tool_calls": [{"function": {"name": "run_command", "arguments": "{\"a\":1}"}}],
+    }]
+    # name 11 字符*1.5=16.5 + arguments 8 字符*0.8=6.4 + 50 = 72.9 → 72
+    assert compressor.estimate_tokens(msgs) == 72
+
+
+def test_estimate_tokens_empty(compressor):
+    assert compressor.estimate_tokens([]) == 0
+
+
+# ==================== should_compress ====================
+
+def test_should_compress_below_thresholds(compressor, monkeypatch):
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 1000)
+    patch_const(monkeypatch, "COMPRESS_TOKEN_THRESHOLD", 10 ** 9)
+    should, reason = compressor.should_compress(make_messages(5))
+    assert should is False
+
+
+def test_should_compress_by_message_count(compressor, monkeypatch):
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 5)
+    should, reason = compressor.should_compress(make_messages(5))  # 10 条对话消息
+    assert should is True
+    assert "消息数" in reason
+
+
+def test_should_compress_by_token_count(compressor, monkeypatch):
+    """消息数未超阈值但 Token 数超阈值时也应触发（v3.1 修复项）。"""
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 1000)
+    patch_const(monkeypatch, "COMPRESS_TOKEN_THRESHOLD", 1)
+    should, reason = compressor.should_compress(make_messages(5))
+    assert should is True
+    assert "Token" in reason
+
+
+def test_should_compress_disabled(compressor, monkeypatch):
+    patch_const(monkeypatch, "COMPRESS_ENABLED", False)
+    should, _ = compressor.should_compress(make_messages(100))
+    assert should is False
+
+
+# ==================== compress_conversation ====================
+
+def test_compress_by_message_count(compressor, monkeypatch):
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 10)
+    messages = make_messages(15)  # 30 条对话消息 + 1 system
+    result = compressor.compress_conversation(messages)
+
+    non_system = [m for m in result if m["role"] != "system"]
+    assert len(non_system) < 30  # 消息数下降
+    # 原始 system 提示词保留
+    assert any(m["content"] == "SYS_PROMPT" for m in result)
+    # 压缩摘要已注入
+    assert any(m["role"] == "system" and "这是压缩摘要" in m["content"] for m in result)
+    # 最近消息保留
+    assert any(m.get("content") == "助手回复14" for m in non_system)
+    # LLM 被调用了一次
+    assert len(compressor.llm_client.calls) == 1
+
+
+def test_compress_by_token_count(compressor, monkeypatch):
+    """Token 触发（消息数远低于阈值）时也应实际执行压缩（v3.1 修复项）。"""
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 1000)
+    patch_const(monkeypatch, "COMPRESS_TOKEN_THRESHOLD", 1)
+    messages = make_messages(15, long_content=True)  # 30 条长消息
+    result = compressor.compress_conversation(messages)
+
+    non_system = [m for m in result if m["role"] != "system"]
+    assert len(non_system) < 30
+    assert any(m["role"] == "system" and "这是压缩摘要" in m["content"] for m in result)
+    assert len(compressor.llm_client.calls) == 1
+    # Token 数下降
+    assert compressor.estimate_tokens(non_system) < compressor.estimate_tokens(
+        [m for m in messages if m["role"] != "system"])
+
+
+def test_compress_llm_failure_keeps_original(monkeypatch):
+    comp = ConversationCompressor(
+        llm_client=FakeLLMClient(error=RuntimeError("api down")),
+        logger=Logger("TestCompressor", 0),
+    )
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 10)
+    messages = make_messages(15)
+    result = comp.compress_conversation(messages)
+    assert result == messages  # 失败时原样返回
+
+
+def test_compress_without_llm_client_keeps_original(monkeypatch):
+    comp = ConversationCompressor(llm_client=None)
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 10)
+    messages = make_messages(15)
+    assert comp.compress_conversation(messages) == messages
+
+
+def test_compress_empty_messages(compressor):
+    assert compressor.compress_conversation([]) == []
+
+
+def test_compress_too_few_messages_skips(compressor, monkeypatch):
+    patch_const(monkeypatch, "COMPRESS_MESSAGE_THRESHOLD", 10)
+    messages = make_messages(3)  # 6 条对话消息，压缩块必然 < 5
+    assert compressor.compress_conversation(messages) == messages
+    assert compressor.llm_client.calls == []
+
+
+# ==================== _validate_message_structure ====================
+
+def test_validate_removes_orphan_tool_messages(compressor):
+    msgs = [
+        {"role": "user", "content": "hi"},
+        {"role": "assistant", "content": "",
+         "tool_calls": [{"id": "t1", "function": {"name": "f", "arguments": "{}"}}]},
+        {"role": "tool", "tool_call_id": "t1", "content": "ok"},
+        {"role": "tool", "tool_call_id": "orphan", "content": "孤儿消息"},
+    ]
+    result = compressor._validate_message_structure(msgs)
+    assert len(result) == 3
+    assert all(m.get("tool_call_id") != "orphan" for m in result)
+
+
+def test_validate_keeps_paired_tool_messages(compressor):
+    msgs = [
+        {"role": "user", "content": "hi"},
+        {"role": "assistant", "content": "",
+         "tool_calls": [{"id": "t1", "function": {"name": "f", "arguments": "{}"}}]},
+        {"role": "tool", "tool_call_id": "t1", "content": "ok"},
+    ]
+    assert len(compressor._validate_message_structure(msgs)) == 3

aibox/tests/test_db.py

@@ -70,6 +70,87 @@ def test_memory_search_empty_keywords_returns_empty(mem_db):
     assert mem_db.search_memories([]) == []
 
 
+def test_memory_search_chinese_keyword(mem_db):
+    """中文关键词应可检索（trigram 分词或 LIKE 兜底）。"""
+    mem_db.add_memory(topic="天气查询", summary="今天天气晴朗", content="气温25度")
+    mem_db.add_memory(topic="会议安排", summary="下午三点开会", content="会议室A")
+
+    results = mem_db.search_memories(["天气"])
+    assert len(results) == 1
+    assert results[0]["topic"] == "天气查询"
+
+
+def test_memory_search_chinese_match_all(mem_db):
+    """中文多关键词 match_all：全部命中才返回。"""
+    mem_db.add_memory(topic="天气查询", summary="今天天气晴朗", content="气温25度")
+    mem_db.add_memory(topic="天气与会议", summary="开会前先查天气", content="会议室A")
+
+    results = mem_db.search_memories(["天气", "开会"], match_all=True)
+    assert len(results) == 1
+    assert results[0]["topic"] == "天气与会议"
+
+
+def test_memory_search_like_escape_wildcards(mem_db):
+    """LIKE 兜底路径中 %/_ 应被转义为字面字符，不当作通配符。"""
+    import sqlite3
+    mem_db.add_memory(topic="progress", summary="100% done")
+    mem_db.add_memory(topic="task_a", summary="plain text")
+    mem_db.add_memory(topic="taskXa", summary="wildcard trap")
+
+    def like_search(kw):
+        with sqlite3.connect(mem_db.db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            return mem_db._search_via_like(conn.cursor(), [kw], match_all=False, limit=100)
+
+    # "%" 为字面字符："100%" 只命中 progress，不当作任意串通配
+    results = like_search("100%")
+    assert [r["topic"] for r in results] == ["progress"]
+
+    # "_" 为字面字符："ask_" 只命中 task_a；若未转义，taskXa 也会被 %ask_% 误匹配
+    results = like_search("ask_")
+    assert [r["topic"] for r in results] == ["task_a"]
+
+
+def test_memory_fts_migration_from_unicode61(tmp_path):
+    """旧 unicode61 FTS 表在重新打开时应自动迁移为 trigram 并回灌数据。"""
+    import sqlite3
+    db_path = str(tmp_path / "legacy.db")
+
+    # 模拟旧库：unicode61 建表 + 写入数据
+    conn = sqlite3.connect(db_path)
+    conn.execute('''CREATE TABLE memories (
+        id INTEGER PRIMARY KEY AUTOINCREMENT, topic TEXT NOT NULL, summary TEXT NOT NULL,
+        content TEXT, start_time TEXT NOT NULL, end_time TEXT NOT NULL,
+        created_at TEXT NOT NULL, tags TEXT, importance INTEGER DEFAULT 1)''')
+    conn.execute('''CREATE VIRTUAL TABLE memories_fts
+        USING fts5(topic, summary, content, tags, content=memories)''')
+    conn.execute("INSERT INTO memories (topic, summary, start_time, end_time, created_at) "
+                 "VALUES ('天气查询', '今天天气晴朗', 't', 't', 't')")
+    conn.execute("INSERT INTO memories_fts (rowid, topic, summary) VALUES (1, '天气查询', '今天天气晴朗')")
+    conn.commit()
+    conn.close()
+
+    # 重新打开 → 自动迁移 → 中文可搜
+    db = MemoryDatabase(db_path=db_path)
+    results = db.search_memories(["天气"])
+    assert len(results) == 1
+    assert results[0]["topic"] == "天气查询"
+
+    # 迁移后 FTS 表应为 trigram 分词（环境支持时）
+    conn = sqlite3.connect(db_path)
+    sql = conn.execute("SELECT sql FROM sqlite_master WHERE name='memories_fts'").fetchone()[0]
+    fts_count = conn.execute("SELECT COUNT(*) FROM memories_fts").fetchone()[0]
+    conn.close()
+    assert fts_count == 1  # 数据已回灌
+    try:
+        probe = sqlite3.connect(":memory:")
+        probe.execute("CREATE VIRTUAL TABLE t USING fts5(x, tokenize='trigram')")
+        probe.close()
+        assert "trigram" in sql
+    except sqlite3.OperationalError:
+        pass  # 环境不支持 trigram，保持 unicode61 属预期行为
+
+
 def test_memory_delete(mem_db):
     mid = mem_db.add_memory(topic="待删除", summary="摘要")
     assert mem_db.delete_memory(mid) is True