fix(aibox): Batch N 波次1~2——第八轮审计 T 系列速赢与并发正确性修复

波次1 速赢批(7 项):
- T8 ProcessManager watermark 取号与回显记账原子化(entry.lock 改 RLock)
- T9 LLM 非 200 响应抛出前 close,流式路径清 _current_response
- T13 _sanitize_log_msg 增加裸 sk- key 脱敏,异常构造点统一过脱敏
- T12 流式判定改共享 saw_complete 标志(排空后 empty() 恒 False 死逻辑),
  超长截断前移堵住非流式直发绕过 N8 输出预算
- T16/T17 三处 Popen 显式 encoding=utf-8 + errors=replace
- T18 detach/log_file/force 统一过 _coerce_bool(字符串 "false" 误判)

波次2 并发正确性批(6 项):
- T4 registry 输出队列入口快照,不再向共享工具实例传播(跨会话串扰)
- T5 executor 单槽 _current_process 改令牌多槽注册表,_terminate_tree
  显式传 (proc, pgid),finally 只注销自己槽位(防并发误杀进程组)
- T6 FailoverClient 冷却键改 id(client),与 P6 去重键语义一致
- T7 stream_chat 拆普通方法+内部生成器,消除生成器延迟副作用清除取消标志
- T15 工具结果统一 TOOL_MAX_RESULT_LENGTH 截断(两条分支同口径)
- T24 非流式 execute() timeout 下限夹取 1s(该路径无 3600s 兜底)

测试:532 passed / 4 failed(平台环境项)/ 1 skipped,零回归
审计报告补记 §23 第八轮审计 + §24 Batch N 波次1~2 修复记录
This commit is contained in:
qing committed 2026-08-21 17:33:33 +08:00
1 parent 3e860c16e0
commit 5894e64287
8 files changed
+583 -110

No files matched your search

+25 -13
View File
@@ -1376,13 +1376,20 @@ class DeepSeekChat:
# 队列是否为空),闭包长期持有 output_callback(含 websocket 引用)
# 阻碍 GC,常驻服务下线程与内存持续累积。
drain_stop = threading.Event()
# T12:drain 线程与工具调用后的排空循环并发消费同一队列,complete
# 标记可能被任一方取走——用可变容器共享,两路见到都置位,供
# has_stream_output 判定流式工具(流式已逐行转发,勿重复整发结果)
saw_complete = [False]
def _drain_queue(q=output_queue, cb=output_callback, stop_event=drain_stop):
def _drain_queue(q=output_queue, cb=output_callback, stop_event=drain_stop,
flag=saw_complete):
"""持续排空 output_queue 并转发 line 输出(长时间工具执行期间防 WS 空闲断线)"""
while not stop_event.is_set():
try:
msg_type, content = q.get(timeout=0.1)
if cb and msg_type == "line":
if msg_type == "complete":
flag[0] = True
elif cb and msg_type == "line":
cb("line", str(content) if content is not None else "")
except queue.Empty:
continue
@@ -1423,6 +1430,9 @@ class DeepSeekChat:
)
self.logger.warning(f"编排者误调执行面工具 {tool_name},已拦截并引导改用 create_subagent")
else:
# T12:每个工具调用前重置流式标志(drain 线程与下方排空
# 循环并发消费,任一方见到 complete 都会置位)
saw_complete[0] = False
try:
result_raw = self.tool_registry.execute_tool(tool_name, **args)
# 排空队列中剩余标记(complete 标记供 has_stream_output 判定、
@@ -1430,7 +1440,9 @@ class DeepSeekChat:
while not output_queue.empty():
try:
msg_type, content = output_queue.get_nowait()
if output_callback and msg_type == "line":
if msg_type == "complete":
saw_complete[0] = True
elif output_callback and msg_type == "line":
output_callback("line", str(content) if content is not None else "")
except queue.Empty:
break
@@ -1448,20 +1460,20 @@ class DeepSeekChat:
# 安全转换并清理结果
result = self._sanitize_tool_result(result_raw)
# drain 线程持续转发 line 输出,这里通过 complete 标记判断是否流式工具
has_stream_output = not output_queue.empty()
# T12:流式工具的 line 输出已由 drain 线程逐行转发(registry 侧
# N8 预算已生效),勿再把完整 result 整发——此前排空后判定
# output_queue.empty() 恒为 False,导致重复推送 + 超长结果绕过预算
has_stream_output = saw_complete[0]
# 普通(非流式)工具没有队列输出,直接发送结果
# 长度预算前置:超长结果先替换为提示,再决定是否推送,
# 避免全量 result 经非流式直发路径绕过 N8 输出限制(T12)
if len(result) > MAX_RESULT_LENGTH:
result = f"⚠️ 工具调用结果长度超过 {MAX_RESULT_LENGTH} 字符,已丢弃。请使用更精确的查询获取所需信息。"
# 普通(非流式)工具没有队列输出,直接发送(已截断的)结果
if not has_stream_output and output_callback and result:
output_callback("line", result + "\n")
# 检查结果大小,如果超过阈值,不保留截断内容,仅返回提示信息
if len(result) > MAX_RESULT_LENGTH:
truncated_note = f"⚠️ 工具调用结果长度超过 {MAX_RESULT_LENGTH} 字符,已丢弃。请使用更精确的查询获取所需信息。"
result = truncated_note
if output_callback:
output_callback("line", truncated_note + "\n")
# 结束调用工具时发送换行
if output_callback:
output_callback("line", "\n\n")
+59 -16
View File
@@ -29,13 +29,17 @@ import requests
# L12(审计 2026-08-18):瞬时错误体可能回显网关/WAF 错误页中的
# Authorization 头,抹掉 Bearer 令牌后再进日志,防密钥落盘
_AUTH_HEADER_RE = re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]+")
# T13:错误响应体可能回显裸 API Key(如 sk-xxx),一并脱敏
_API_KEY_RE = re.compile(r"sk-[A-Za-z0-9_-]{16,}")
def _sanitize_log_msg(msg: str) -> str:
"""日志脱敏:替换错误消息中的 Bearer 令牌。"""
"""日志脱敏:替换错误消息中的 Bearer 令牌与裸 API Key。"""
if not msg:
return msg
return _AUTH_HEADER_RE.sub("Bearer ***", msg)
msg = _AUTH_HEADER_RE.sub("Bearer ***", msg)
msg = _API_KEY_RE.sub("sk-***", msg)
return msg
class LLMClient(ABC):
@@ -166,15 +170,23 @@ class DeepSeekClient(LLMClient):
proxies=self._resolve_proxies(),
)
if response.status_code == 200:
result = response.json()
return result["choices"][0]["message"]["content"].strip()
if response.status_code == 429 or response.status_code >= 500:
try:
result = response.json()
return result["choices"][0]["message"]["content"].strip()
finally:
response.close()
# T9+T13:非 200 立即关连接(stream=True 才需要,但统一防御),
# 错误体脱敏后再入异常,防 API Key 经错误页回显进历史/日志
status = response.status_code
body = _sanitize_log_msg(response.text[:200])
response.close()
if status == 429 or status >= 500:
# L4a:瞬时故障显式抛出——FailoverClient 据此转移下一档案;
# 无包装器时由调用方 catch-all 接住,下游仍走失败分支(语义等价)
raise TransientLLMError(
f"LLM 请求瞬时失败: HTTP {response.status_code}: {response.text[:200]}",
f"LLM 请求瞬时失败: HTTP {status}: {body}",
retry_after=_retry_after_of(response))
self._log("error", f"LLM 压缩请求失败: HTTP {response.status_code}")
self._log("error", f"LLM 压缩请求失败: HTTP {status}")
return None
except TransientLLMError:
raise # L4a:瞬时故障上抛供 FailoverClient 转移,不吞
@@ -219,13 +231,20 @@ class DeepSeekClient(LLMClient):
self._current_response = response
if response.status_code != 200:
if response.status_code == 429 or response.status_code >= 500:
# T9:非 200 立即释放连接(stream=True 下不关闭会阻塞同域名
# in-flight 请求直到 GC);T13:错误体脱敏后再入异常
status = response.status_code
retry_after = _retry_after_of(response)
body = _sanitize_log_msg(response.text[:200])
response.close()
self._current_response = None
if status == 429 or status >= 500:
# L4b:瞬时故障显式分类(此时尚未产出任何事件,可安全转移重放)
raise TransientLLMError(
f"LLM 流式请求瞬时失败: HTTP {response.status_code}: {response.text[:200]}",
retry_after=_retry_after_of(response))
f"LLM 流式请求瞬时失败: HTTP {status}: {body}",
retry_after=retry_after)
raise RuntimeError(
f"HTTP {response.status_code}: {response.text[:200]}")
f"HTTP {status}: {body}")
try:
for line in response.iter_lines():
@@ -368,8 +387,17 @@ class FailoverClient:
except Exception:
pass
@staticmethod
def _cool_key(client) -> int:
"""T6:冷却键按客户端实例(id)——client.name 取 display_name 优先且
无唯一性校验,两个 profile_name 不同但 display_name 相同的档案
(如主备都叫 "Kimi")冷却状态互相污染:A 失败冷却 → B 被误判
跳过(该转移时不转移)。P6 去重保证每档案恰好一个实例,实例被
_clients 持有生命周期内 id 稳定,与去重键语义一致。"""
return id(client)
def _cooling(self, client) -> bool:
entry = self._fail_at.get(client.name)
entry = self._fail_at.get(self._cool_key(client))
if entry is None:
return False
t, cd = entry
@@ -377,9 +405,9 @@ class FailoverClient:
def _mark_fail(self, client, cooldown: Optional[float] = None):
# L4c:429 Retry-After(经 TransientLLMError.retry_after)覆盖默认冷却时长
self._fail_at[client.name] = (time.monotonic(),
cooldown if cooldown and cooldown > 0
else self.cooldown_seconds)
self._fail_at[self._cool_key(client)] = (time.monotonic(),
cooldown if cooldown and cooldown > 0
else self.cooldown_seconds)
def _budget_exhausted(self) -> bool:
if self.timeout_budget is None or self._call_started is None:
@@ -451,10 +479,25 @@ class FailoverClient:
max_tokens: int = 8192, tools: Optional[List[Dict]] = None,
tool_choice: Optional[str] = None,
reasoning_enabled: bool = False, **extra) -> Iterator[Dict]:
"""L4b:流式转移——仅首事件前故障可安全重放(对用户不可见)。"""
"""L4b:流式转移——仅首事件前故障可安全重放(对用户不可见)。
T7:本方法是普通函数(非生成器)——生成器函数体延迟到首次 next()
才执行,若 cancel() 在创建生成器与首迭代之间被其他线程调用,
延迟执行的 `_cancelled = False` 会把用户的取消清掉(请求照常发出)。
标志重置与计数在返回生成器前同步完成,取消契约(类文档)恢复成立。
"""
self._cancelled = False # 新一轮请求,清除上一轮中断标志
self._call_started = time.monotonic()
self._stats["requests"] += 1
return self._stream_chat_impl(messages, temperature=temperature,
max_tokens=max_tokens, tools=tools,
tool_choice=tool_choice,
reasoning_enabled=reasoning_enabled, **extra)
def _stream_chat_impl(self, messages: List[Dict], temperature: float = None,
max_tokens: int = 8192, tools: Optional[List[Dict]] = None,
tool_choice: Optional[str] = None,
reasoning_enabled: bool = False, **extra) -> Iterator[Dict]:
last_exc = None
for idx, client in enumerate(self._clients):
if self._cooling(client):
+90 -46
View File
@@ -71,26 +71,63 @@ class SubprocessCommandExecutor(CommandExecutor):
def __init__(self, logger=None):
super().__init__(logger)
self._current_process = None
self._current_process = None # 兼容保留:最近一次注册的进程(诊断用)
self._current_pgid = None # L9:Popen 后立即捕获,shell 死亡后 getpgid 会失败
self._process_lock = threading.Lock()
# T5:单槽 _current_process 在并发执行时互相覆盖——先启动的执行
# 超时/取消会误杀后来者的进程组(killpg 连坐孙进程),先结束的
# finally 清引用导致后来者永远无法取消。改为令牌多槽注册表,
# 终止只作用于自己槽位内的 (proc, pgid)。
self._proc_slots: Dict[int, Tuple] = {}
self._token_seq = 0
def _terminate_tree(self, proc, force: bool = False):
def _register_proc(self, process):
"""T5:注册本次执行的进程槽,返回 (token, pgid)"""
with self._process_lock:
self._token_seq += 1
token = self._token_seq
pgid = None
if os.name == "posix":
try:
pgid = os.getpgid(process.pid)
except Exception:
pgid = None
self._proc_slots[token] = (process, pgid)
# 兼容属性指向最新注册(诊断/旧测试观察用,不参与终止逻辑)
self._current_process = process
self._current_pgid = pgid
return token, pgid
def _unregister_proc(self, token):
"""T5:仅移除自己的槽位;全部槽位清空时同步清兼容属性"""
with self._process_lock:
self._proc_slots.pop(token, None)
if not self._proc_slots:
self._current_process = None
self._current_pgid = None
def _terminate_tree(self, proc, force: bool = False, pgid=None):
"""终止整个进程组(L9:POSIX 下 shell=True 只杀 shell 时,孙进程
(`(x &)` 等)仍持管道写端——reader 阻塞无 EOF、fd/线程悬挂至孙
进程退出,写满 64KB 管道缓冲后孙进程也卡死。start_new_session 使
shell 成为会话首进程,按组信号可一次覆盖整棵进程树)。
组 ID 在 Popen 后立即保存:shell 死亡后 os.getpgid(pid) 会抛
ProcessLookupError,届时无法再从进程反查组。
T5:pgid 由调用方显式传入(注册槽位时捕获),不再读共享
_current_pgid——并发执行时那是别人的进程组。
Windows 无进程组概念,退回单进程 terminate/kill。"""
try:
if os.name == "posix":
sig = signal.SIGKILL if force else signal.SIGTERM
with self._process_lock:
pgid = self._current_pgid
if pgid is None:
pgid = os.getpgid(proc.pid)
os.killpg(pgid, sig)
try:
pgid = os.getpgid(proc.pid)
except Exception:
pgid = None
if pgid is not None:
os.killpg(pgid, sig)
else:
proc.kill() if force else proc.terminate()
elif force:
proc.kill()
else:
@@ -102,18 +139,21 @@ class SubprocessCommandExecutor(CommandExecutor):
pass
def cancel(self):
"""取消当前正在运行的命令(kill子进程组)。线程安全。"""
"""取消本执行器当前注册的全部运行中命令(kill 子进程组)。线程安全。
T5:多槽令牌注册表——并发执行时定向终止各自槽位,互不误杀;
无参调用语义为"终止本执行器全部在途命令"(中断/关闭路径)。"""
with self._process_lock:
proc = self._current_process
if proc is not None:
slots = list(self._proc_slots.values())
for proc, pgid in slots:
try:
if proc.poll() is None:
self._terminate_tree(proc, force=True)
self._terminate_tree(proc, force=True, pgid=pgid)
self.logger.info("已终止正在运行的子进程组")
except Exception:
pass
def _read_with_limit(self, process, timeout: int, start_time: float, output_queue=None) -> Tuple[str, bool, bool]:
def _read_with_limit(self, process, timeout: int, start_time: float, output_queue=None,
pgid=None) -> Tuple[str, bool, bool]:
"""从进程读取输出,限制总大小和超时。返回 (output_str, timeout_occurred, truncated)。
H2 修复:阻塞 readline 在独立 reader 线程执行,主循环带超时轮询队列。
@@ -147,14 +187,14 @@ class SubprocessCommandExecutor(CommandExecutor):
# J16(审计 2026-08-19):timeout<=0 语义为"不设超时"(有意为之,
# 调用方显式传 0 禁用);registry 层 3600s 流式兜底仍生效
if timeout > 0 and time.time() - start_time > timeout:
self._terminate_tree(process)
self._terminate_tree(process, pgid=pgid)
timeout_occurred = True
break
# 检查输出大小
if total_bytes >= self.MAX_OUTPUT_BYTES:
truncated = True
self._terminate_tree(process)
self._terminate_tree(process, pgid=pgid)
break
try:
@@ -175,7 +215,7 @@ class SubprocessCommandExecutor(CommandExecutor):
# 与管道 fd 至孙进程退出,写满 64KB 缓冲后孙进程自身
# 也卡死。此处强杀进程组收尾(nohup 重定向输出的
# 真正脱离场景不持管道,不会走到这里)。
self._terminate_tree(process, force=True)
self._terminate_tree(process, force=True, pgid=pgid)
break
continue
@@ -188,7 +228,7 @@ class SubprocessCommandExecutor(CommandExecutor):
output_chunks.append(line.encode('utf-8', errors='replace')[:allowed].decode('utf-8', errors='ignore'))
total_bytes = self.MAX_OUTPUT_BYTES
truncated = True
self._terminate_tree(process)
self._terminate_tree(process, pgid=pgid)
break
output_chunks.append(line)
@@ -224,6 +264,14 @@ class SubprocessCommandExecutor(CommandExecutor):
if not self.is_enabled():
return "❌ 命令执行功能已禁用"
# T24:非流式路径没有 registry 3600s 流式兜底,timeout<=0("不设
# 超时")会让挂起命令永久阻塞调用线程——下限夹取到 1s 快速失败。
# 流式路径保留 0=不限语义(registry 兜底在位)
try:
timeout = max(1, int(timeout))
except (TypeError, ValueError):
timeout = 30
# 默认工作目录为workspace,避免在启动目录散落文件
if work_dir is None or work_dir == ".":
work_dir = constants.WORKSPACE_DIR
@@ -236,6 +284,8 @@ class SubprocessCommandExecutor(CommandExecutor):
exec_env.update(env)
process = None
token = None
pgid = None
try:
process = subprocess.Popen(
command,
@@ -246,31 +296,28 @@ class SubprocessCommandExecutor(CommandExecutor):
text=True,
bufsize=1,
universal_newlines=True,
encoding="utf-8", # T16:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出
errors="replace", # T16:非法字节不抛异常中断 reader 线程
env=exec_env,
cwd=os.path.expanduser(work_dir),
# L9:POSIX 下 shell 成为会话首进程,超时/取消可按进程组
# 一次终止整棵进程树(含 & 后台孙进程),消除管道悬挂
start_new_session=(os.name == "posix")
)
# 保存进程引用以便cancel()可以kill它
with self._process_lock:
self._current_process = process
if os.name == "posix":
try:
self._current_pgid = os.getpgid(process.pid)
except Exception:
self._current_pgid = None
# T5:注册本执行专属槽位,终止/取消只作用于自己的 (proc, pgid)
token, pgid = self._register_proc(process)
start_time = time.time()
output_str, timeout_occurred, truncated = self._read_with_limit(process, timeout, start_time)
output_str, timeout_occurred, truncated = self._read_with_limit(
process, timeout, start_time, pgid=pgid)
# 确保进程终止
if process.poll() is None:
self._terminate_tree(process, force=timeout_occurred or truncated)
self._terminate_tree(process, force=timeout_occurred or truncated, pgid=pgid)
try:
process.wait(timeout=2)
except:
self._terminate_tree(process, force=True)
self._terminate_tree(process, force=True, pgid=pgid)
return_code = process.returncode if process.returncode is not None else -1
@@ -298,11 +345,10 @@ class SubprocessCommandExecutor(CommandExecutor):
return f"❌ 执行错误: {str(e)}"
finally:
if process and process.poll() is None:
self._terminate_tree(process, force=True)
# 清除当前进程引用
with self._process_lock:
self._current_process = None
self._current_pgid = None
self._terminate_tree(process, force=True, pgid=pgid)
# T5:仅注销自己的槽位,不再无差别清空共享引用
if token is not None:
self._unregister_proc(token)
def execute_streaming(self, command: str, output_queue: queue.Queue, timeout: int = 30,
work_dir: str = None, env: Dict = None, **kwargs):
@@ -329,6 +375,8 @@ class SubprocessCommandExecutor(CommandExecutor):
exec_env.update(env)
process = None
token = None
pgid = None
try:
process = subprocess.Popen(
command,
@@ -339,6 +387,8 @@ class SubprocessCommandExecutor(CommandExecutor):
text=True,
bufsize=1,
universal_newlines=True,
encoding="utf-8", # T16:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出
errors="replace", # T16:非法字节不抛异常中断 reader 线程
env=exec_env,
cwd=os.path.expanduser(work_dir),
# L9:POSIX 下 shell 成为会话首进程,超时/取消可按进程组
@@ -346,19 +396,13 @@ class SubprocessCommandExecutor(CommandExecutor):
start_new_session=(os.name == "posix")
)
self.process = process
# 保存进程引用以便cancel()可以kill它
with self._process_lock:
self._current_process = process
if os.name == "posix":
try:
self._current_pgid = os.getpgid(process.pid)
except Exception:
self._current_pgid = None
# T5:注册本执行专属槽位,终止/取消只作用于自己的 (proc, pgid)
token, pgid = self._register_proc(process)
start_time = time.time()
# H2 修复:复用线程化读取(reader 线程 + 超时轮询),行经 output_queue 流式转发
output_str, timeout_occurred, truncated = self._read_with_limit(
process, timeout, start_time, output_queue=output_queue)
process, timeout, start_time, output_queue=output_queue, pgid=pgid)
if timeout_occurred:
output_queue.put(("line", f"\n⏰ 命令执行超时({timeout}秒)\n"))
@@ -367,7 +411,7 @@ class SubprocessCommandExecutor(CommandExecutor):
# 如果超时或截断,确保进程被终止
if (timeout_occurred or truncated) and process.poll() is None:
self._terminate_tree(process, force=True)
self._terminate_tree(process, force=True, pgid=pgid)
try:
process.wait(timeout=2)
except:
@@ -404,12 +448,12 @@ class SubprocessCommandExecutor(CommandExecutor):
output_queue.put(("complete", f"❌ 执行错误: {str(e)}"))
finally:
if process and process.poll() is None:
self._terminate_tree(process)
self._terminate_tree(process, pgid=pgid)
try:
process.wait(timeout=2)
except:
self._terminate_tree(process, force=True)
self._terminate_tree(process, force=True, pgid=pgid)
self.process = None
with self._process_lock:
self._current_process = None
self._current_pgid = None
# T5:仅注销自己的槽位,不再无差别清空共享引用
if token is not None:
self._unregister_proc(token)
+8 -5
View File
@@ -76,7 +76,7 @@ class _ProcEntry:
self.max_buffer = max(1024, int(buffer_bytes))
self.next_seq = 1
self.evicted = 0
self.lock = threading.Lock()
self.lock = threading.RLock() # T8:可重入,write_input 持锁期间可调用 _record_line
self.reader_done = threading.Event()
self.reader = None
# 交互写入:单写锁保序(bg_write 串行化,防交错输入)
@@ -178,6 +178,8 @@ class ProcessManager:
text=True,
bufsize=1,
universal_newlines=True,
encoding="utf-8", # T17:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出致 reader 静默退出
errors="replace",
env=exec_env,
cwd=os.path.expanduser(work_dir),
# 对齐 L9:POSIX 下 shell 成为会话首进程,kill 按进程组
@@ -345,12 +347,13 @@ class ProcessManager:
if entry.status != "running":
return {"ok": False, "error": f"进程已结束({entry.status}),无法写入"}
# watermark:回显行将占用的序号(写入前先记账,响应必然在其后)
# watermark:取号与回显行记账必须在同一临界区,保证 watermark 等于
# 回显行真实序号,bg_output(since=watermark) 正确对齐本条响应(T8)
echo = "[stdin] " + text.replace("\n", "⏎")
with entry.lock:
watermark = entry.next_seq
# 回显行进环形缓冲 + log 镜像(多行输入折叠为单行展示)
echo = "[stdin] " + text.replace("\n", "⏎")
self._record_line(entry, echo)
# 回显行进环形缓冲 + log 镜像(多行输入折叠为单行展示)
self._record_line(entry, echo)
# 写入放 watchdog 线程:进程不读 stdin 且管道缓冲(~64KB)写满时
# write() 会阻塞,不能拖死工具线程;超时后 daemon 线程由系统兜底
+11 -9
View File
@@ -820,8 +820,9 @@ class TestL4aChatCompletionFailover:
assert fc.chat_completion([{"role": "user", "content": "q2"}]) == "r2"
assert mp.call_args_list[0].args[0].startswith("https://api.moonshot.cn")
# 冷却过期(手动回拨失败时间戳):恢复走主档案
fc._fail_at["deepseek"] = (_time.monotonic() - fc.cooldown_seconds - 1,
fc.cooldown_seconds)
# T6:冷却键改为客户端实例 id,种冷却态需走 _cool_key
fc._fail_at[fc._cool_key(fc._clients[0])] = (_time.monotonic() - fc.cooldown_seconds - 1,
fc.cooldown_seconds)
with mock.patch("llm_client.requests.post",
return_value=_resp(200, content="r3")) as mp:
assert fc.chat_completion([{"role": "user", "content": "q3"}]) == "r3"
@@ -998,8 +999,8 @@ class TestL4bStreamFailover:
primary = self._client("deepseek", stream=_fake_stream(events=self.EVENTS))
fallback = self._client("kimi", stream=_fake_stream(events=self.EVENTS))
fc = self._fc(primary, fallback)
fc._fail_at = {"deepseek": (_time.monotonic(), fc.cooldown_seconds),
"kimi": (_time.monotonic(), fc.cooldown_seconds)}
fc._fail_at = {fc._cool_key(primary): (_time.monotonic(), fc.cooldown_seconds),
fc._cool_key(fallback): (_time.monotonic(), fc.cooldown_seconds)}
with pytest.raises(RuntimeError, match="冷却期"):
list(fc.stream_chat([{"role": "user", "content": "q"}]))
primary.stream_chat.assert_not_called()
@@ -1010,7 +1011,7 @@ class TestL4bStreamFailover:
primary = self._client("deepseek", stream=_fake_stream(events=self.EVENTS))
fallback = self._client("kimi", stream=_fake_stream(events=self.EVENTS))
fc = self._fc(primary, fallback)
fc._fail_at["deepseek"] = (_time.monotonic(), fc.cooldown_seconds)
fc._fail_at[fc._cool_key(primary)] = (_time.monotonic(), fc.cooldown_seconds)
assert list(fc.stream_chat([{"role": "user", "content": "q"}])) == self.EVENTS
primary.stream_chat.assert_not_called()
@@ -1102,7 +1103,7 @@ class TestL4cObservabilityAndBudget:
stream=_fake_stream(events=self.EVENTS))
fallback = self._mk_client("kimi", stream=_fake_stream(events=self.EVENTS))
fc = self._fc(primary, fallback)
fc._fail_at["deepseek"] = (_time.monotonic(), fc.cooldown_seconds)
fc._fail_at[fc._cool_key(primary)] = (_time.monotonic(), fc.cooldown_seconds)
assert list(fc.stream_chat([{"role": "user", "content": "q"}])) == self.EVENTS
s = fc.get_stats()
assert s == {"requests": 1, "failovers": 0, "cooldown_skips": 1, "exhausted": 0}
@@ -1231,13 +1232,14 @@ class TestL4cObservabilityAndBudget:
# 第一次:主失败(retry_after=300)→ fallback
assert fc.chat_completion([{"role": "user", "content": "q1"}]) == "from-kimi"
# 冷却中(默认 60s 已过但 300s 未到):仍走 fallback
t, _ = fc._fail_at["deepseek"]
fc._fail_at["deepseek"] = (t - 100, 300)
_key = fc._cool_key(primary)
t, _ = fc._fail_at[_key]
fc._fail_at[_key] = (t - 100, 300)
primary.chat_completion = primary_ok
assert fc.chat_completion([{"role": "user", "content": "q2"}]) == "from-kimi"
primary_ok.assert_not_called()
# 300s 冷却过期:恢复主档案
fc._fail_at["deepseek"] = (t - 301, 300)
fc._fail_at[_key] = (t - 301, 300)
assert fc.chat_completion([{"role": "user", "content": "q3"}]) == "from-ds"
primary_ok.assert_called_once()
+8
View File
@@ -133,6 +133,11 @@ class BgRunTool(Tool):
if not _enabled():
return "❌ 后台进程功能已禁用(commands.enabled 或 background_processes.enabled 为 false)"
# T18:LLM 可能传字符串 "false",裸 bool("false") is True——
# detach 误开致进程脱离管理、log_file 误开,统一规整
detach = _coerce_bool(detach, False)
log_file = _coerce_bool(log_file, False)
result = self.manager.start(command, name=proc_name, work_dir=work_dir,
env=env, detach=detach, log_file=log_file)
if not result.get("ok"):
@@ -393,6 +398,9 @@ class BgKillTool(Tool):
if handle is None:
return "❌ 错误:handle 必须是整数句柄"
# T18:force 字符串 "false" 会被裸 bool 判为 True,跳过宽限直接 SIGKILL
force = _coerce_bool(force, False)
result = self.manager.kill(handle, force=force, grace=grace)
if not result.get("ok"):
return self.format_result(False, f"终止后台进程 #{handle} 失败",
+33 -15
View File
@@ -67,29 +67,42 @@ class ToolRegistry:
def get_tools_schemas(self) -> List[Dict]:
"""获取所有工具的OpenAI函数调用格式"""
return [tool.to_function_call_format() for tool in self._tools.values()]
@staticmethod
def _truncate_result(result) -> str:
"""T15:工具返回值统一长度预算(非流式与流式最终结果同口径)"""
if not isinstance(result, str):
return result
limit = constants.TOOL_MAX_RESULT_LENGTH
if len(result) > limit:
return (result[:limit] +
f"\n⚠️ 结果超过 {limit} 字符上限,已截断")
return result
def set_output_queue(self, queue: queue.Queue):
"""设置输出队列"""
"""设置输出队列(T4:仅记注册表级引用,不再传播到共享工具实例——
实例属性由 execute_streaming 每次调用自行注入局部队列,全局传播
会让并发会话互相覆盖对方的输出通路,造成跨会话输出串扰)"""
self.current_output_queue = queue
# 同时设置所有工具的队列
for tool in self._tools.values():
tool.output_queue = queue
def execute_tool(self, name: str, **kwargs) -> str:
"""执行工具"""
tool = self.get_tool(name)
if not tool:
return f"❌ 错误:找不到工具 '{name}'"
self.logger.debug(f"工具调用: {name}({_sanitize_kwargs(kwargs)})")
start_time = time.time()
try:
# T4:入口快照会话队列。转发循环可能持续数秒~小时,期间并发
# 会话的 set_output_queue 会覆盖单值引用——全程用快照,保证
# 本会话输出只进本会话队列,结束清理也不影响在途转发
session_queue = self.current_output_queue
# 检查是否有队列可用,并且工具支持流式输出
if self.current_output_queue and hasattr(tool, 'execute_streaming'):
if session_queue and hasattr(tool, 'execute_streaming'):
# 设置工具的流式输出属性
tool.streaming_output = True
tool.output_queue = self.current_output_queue
# 创建队列来收集流式输出
output_queue = queue.Queue()
@@ -134,8 +147,8 @@ class ToolRegistry:
# 预算耗尽:停止转发,仅提示一次
if not truncation_noted:
truncation_noted = True
if self.current_output_queue:
self.current_output_queue.put(
if session_queue:
session_queue.put(
("line", f"\n⚠️ 流式输出超过 {line_budget} 字符上限,已停止转发剩余内容\n"))
continue
remaining = line_budget - forwarded
@@ -146,13 +159,13 @@ class ToolRegistry:
else:
forwarded += len(piece)
# 发送到当前队列(会通过WebSocket发送)
if self.current_output_queue:
self.current_output_queue.put(("line", piece))
if session_queue:
session_queue.put(("line", piece))
elif msg_type == "complete":
final_result = content
# 也发送完整结果
if self.current_output_queue:
self.current_output_queue.put(("complete", content))
if session_queue:
session_queue.put(("complete", content))
except queue.Empty:
continue
@@ -173,6 +186,11 @@ class ToolRegistry:
else:
# 普通执行
result = tool.execute(**kwargs)
# T15:两条分支的最终返回值统一过 TOOL_MAX_RESULT_LENGTH——
# 此前仅流式转发有 line_budget(N8),非流式结果无上限直入
# 对话上下文(memory/memo get 超长 content,token/内存放大)
result = self._truncate_result(result)
elapsed = time.time() - start_time
self.logger.debug(f"工具结果 ({elapsed:.2f}s): {result[:100]}...")