fix(aibox): Batch N 波次1~2——第八轮审计 T 系列速赢与并发正确性修复
波次1 速赢批(7 项): - T8 ProcessManager watermark 取号与回显记账原子化(entry.lock 改 RLock) - T9 LLM 非 200 响应抛出前 close,流式路径清 _current_response - T13 _sanitize_log_msg 增加裸 sk- key 脱敏,异常构造点统一过脱敏 - T12 流式判定改共享 saw_complete 标志(排空后 empty() 恒 False 死逻辑), 超长截断前移堵住非流式直发绕过 N8 输出预算 - T16/T17 三处 Popen 显式 encoding=utf-8 + errors=replace - T18 detach/log_file/force 统一过 _coerce_bool(字符串 "false" 误判) 波次2 并发正确性批(6 项): - T4 registry 输出队列入口快照,不再向共享工具实例传播(跨会话串扰) - T5 executor 单槽 _current_process 改令牌多槽注册表,_terminate_tree 显式传 (proc, pgid),finally 只注销自己槽位(防并发误杀进程组) - T6 FailoverClient 冷却键改 id(client),与 P6 去重键语义一致 - T7 stream_chat 拆普通方法+内部生成器,消除生成器延迟副作用清除取消标志 - T15 工具结果统一 TOOL_MAX_RESULT_LENGTH 截断(两条分支同口径) - T24 非流式 execute() timeout 下限夹取 1s(该路径无 3600s 兜底) 测试:532 passed / 4 failed(平台环境项)/ 1 skipped,零回归 审计报告补记 §23 第八轮审计 + §24 Batch N 波次1~2 修复记录
This commit is contained in:
1 parent
3e860c16e0
commit
5894e64287
8 files changed
+583
-110
No files matched your search
+25
-13
@@ -1376,13 +1376,20 @@ class DeepSeekChat:
|
||||
# 队列是否为空),闭包长期持有 output_callback(含 websocket 引用)
|
||||
# 阻碍 GC,常驻服务下线程与内存持续累积。
|
||||
drain_stop = threading.Event()
|
||||
# T12:drain 线程与工具调用后的排空循环并发消费同一队列,complete
|
||||
# 标记可能被任一方取走——用可变容器共享,两路见到都置位,供
|
||||
# has_stream_output 判定流式工具(流式已逐行转发,勿重复整发结果)
|
||||
saw_complete = [False]
|
||||
|
||||
def _drain_queue(q=output_queue, cb=output_callback, stop_event=drain_stop):
|
||||
def _drain_queue(q=output_queue, cb=output_callback, stop_event=drain_stop,
|
||||
flag=saw_complete):
|
||||
"""持续排空 output_queue 并转发 line 输出(长时间工具执行期间防 WS 空闲断线)"""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
msg_type, content = q.get(timeout=0.1)
|
||||
if cb and msg_type == "line":
|
||||
if msg_type == "complete":
|
||||
flag[0] = True
|
||||
elif cb and msg_type == "line":
|
||||
cb("line", str(content) if content is not None else "")
|
||||
except queue.Empty:
|
||||
continue
|
||||
@@ -1423,6 +1430,9 @@ class DeepSeekChat:
|
||||
)
|
||||
self.logger.warning(f"编排者误调执行面工具 {tool_name},已拦截并引导改用 create_subagent")
|
||||
else:
|
||||
# T12:每个工具调用前重置流式标志(drain 线程与下方排空
|
||||
# 循环并发消费,任一方见到 complete 都会置位)
|
||||
saw_complete[0] = False
|
||||
try:
|
||||
result_raw = self.tool_registry.execute_tool(tool_name, **args)
|
||||
# 排空队列中剩余标记(complete 标记供 has_stream_output 判定、
|
||||
@@ -1430,7 +1440,9 @@ class DeepSeekChat:
|
||||
while not output_queue.empty():
|
||||
try:
|
||||
msg_type, content = output_queue.get_nowait()
|
||||
if output_callback and msg_type == "line":
|
||||
if msg_type == "complete":
|
||||
saw_complete[0] = True
|
||||
elif output_callback and msg_type == "line":
|
||||
output_callback("line", str(content) if content is not None else "")
|
||||
except queue.Empty:
|
||||
break
|
||||
@@ -1448,20 +1460,20 @@ class DeepSeekChat:
|
||||
# 安全转换并清理结果
|
||||
result = self._sanitize_tool_result(result_raw)
|
||||
|
||||
# drain 线程持续转发 line 输出,这里通过 complete 标记判断是否流式工具
|
||||
has_stream_output = not output_queue.empty()
|
||||
# T12:流式工具的 line 输出已由 drain 线程逐行转发(registry 侧
|
||||
# N8 预算已生效),勿再把完整 result 整发——此前排空后判定
|
||||
# output_queue.empty() 恒为 False,导致重复推送 + 超长结果绕过预算
|
||||
has_stream_output = saw_complete[0]
|
||||
|
||||
# 普通(非流式)工具没有队列输出,直接发送结果
|
||||
# 长度预算前置:超长结果先替换为提示,再决定是否推送,
|
||||
# 避免全量 result 经非流式直发路径绕过 N8 输出限制(T12)
|
||||
if len(result) > MAX_RESULT_LENGTH:
|
||||
result = f"⚠️ 工具调用结果长度超过 {MAX_RESULT_LENGTH} 字符,已丢弃。请使用更精确的查询获取所需信息。"
|
||||
|
||||
# 普通(非流式)工具没有队列输出,直接发送(已截断的)结果
|
||||
if not has_stream_output and output_callback and result:
|
||||
output_callback("line", result + "\n")
|
||||
|
||||
# 检查结果大小,如果超过阈值,不保留截断内容,仅返回提示信息
|
||||
if len(result) > MAX_RESULT_LENGTH:
|
||||
truncated_note = f"⚠️ 工具调用结果长度超过 {MAX_RESULT_LENGTH} 字符,已丢弃。请使用更精确的查询获取所需信息。"
|
||||
result = truncated_note
|
||||
if output_callback:
|
||||
output_callback("line", truncated_note + "\n")
|
||||
|
||||
# 结束调用工具时发送换行
|
||||
if output_callback:
|
||||
output_callback("line", "\n\n")
|
||||
|
||||
+59
-16
@@ -29,13 +29,17 @@ import requests
|
||||
# L12(审计 2026-08-18):瞬时错误体可能回显网关/WAF 错误页中的
|
||||
# Authorization 头,抹掉 Bearer 令牌后再进日志,防密钥落盘
|
||||
_AUTH_HEADER_RE = re.compile(r"Bearer\s+[A-Za-z0-9._~+/=-]+")
|
||||
# T13:错误响应体可能回显裸 API Key(如 sk-xxx),一并脱敏
|
||||
_API_KEY_RE = re.compile(r"sk-[A-Za-z0-9_-]{16,}")
|
||||
|
||||
|
||||
def _sanitize_log_msg(msg: str) -> str:
|
||||
"""日志脱敏:替换错误消息中的 Bearer 令牌。"""
|
||||
"""日志脱敏:替换错误消息中的 Bearer 令牌与裸 API Key。"""
|
||||
if not msg:
|
||||
return msg
|
||||
return _AUTH_HEADER_RE.sub("Bearer ***", msg)
|
||||
msg = _AUTH_HEADER_RE.sub("Bearer ***", msg)
|
||||
msg = _API_KEY_RE.sub("sk-***", msg)
|
||||
return msg
|
||||
|
||||
|
||||
class LLMClient(ABC):
|
||||
@@ -166,15 +170,23 @@ class DeepSeekClient(LLMClient):
|
||||
proxies=self._resolve_proxies(),
|
||||
)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
return result["choices"][0]["message"]["content"].strip()
|
||||
if response.status_code == 429 or response.status_code >= 500:
|
||||
try:
|
||||
result = response.json()
|
||||
return result["choices"][0]["message"]["content"].strip()
|
||||
finally:
|
||||
response.close()
|
||||
# T9+T13:非 200 立即关连接(stream=True 才需要,但统一防御),
|
||||
# 错误体脱敏后再入异常,防 API Key 经错误页回显进历史/日志
|
||||
status = response.status_code
|
||||
body = _sanitize_log_msg(response.text[:200])
|
||||
response.close()
|
||||
if status == 429 or status >= 500:
|
||||
# L4a:瞬时故障显式抛出——FailoverClient 据此转移下一档案;
|
||||
# 无包装器时由调用方 catch-all 接住,下游仍走失败分支(语义等价)
|
||||
raise TransientLLMError(
|
||||
f"LLM 请求瞬时失败: HTTP {response.status_code}: {response.text[:200]}",
|
||||
f"LLM 请求瞬时失败: HTTP {status}: {body}",
|
||||
retry_after=_retry_after_of(response))
|
||||
self._log("error", f"LLM 压缩请求失败: HTTP {response.status_code}")
|
||||
self._log("error", f"LLM 压缩请求失败: HTTP {status}")
|
||||
return None
|
||||
except TransientLLMError:
|
||||
raise # L4a:瞬时故障上抛供 FailoverClient 转移,不吞
|
||||
@@ -219,13 +231,20 @@ class DeepSeekClient(LLMClient):
|
||||
self._current_response = response
|
||||
|
||||
if response.status_code != 200:
|
||||
if response.status_code == 429 or response.status_code >= 500:
|
||||
# T9:非 200 立即释放连接(stream=True 下不关闭会阻塞同域名
|
||||
# in-flight 请求直到 GC);T13:错误体脱敏后再入异常
|
||||
status = response.status_code
|
||||
retry_after = _retry_after_of(response)
|
||||
body = _sanitize_log_msg(response.text[:200])
|
||||
response.close()
|
||||
self._current_response = None
|
||||
if status == 429 or status >= 500:
|
||||
# L4b:瞬时故障显式分类(此时尚未产出任何事件,可安全转移重放)
|
||||
raise TransientLLMError(
|
||||
f"LLM 流式请求瞬时失败: HTTP {response.status_code}: {response.text[:200]}",
|
||||
retry_after=_retry_after_of(response))
|
||||
f"LLM 流式请求瞬时失败: HTTP {status}: {body}",
|
||||
retry_after=retry_after)
|
||||
raise RuntimeError(
|
||||
f"HTTP {response.status_code}: {response.text[:200]}")
|
||||
f"HTTP {status}: {body}")
|
||||
|
||||
try:
|
||||
for line in response.iter_lines():
|
||||
@@ -368,8 +387,17 @@ class FailoverClient:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def _cool_key(client) -> int:
|
||||
"""T6:冷却键按客户端实例(id)——client.name 取 display_name 优先且
|
||||
无唯一性校验,两个 profile_name 不同但 display_name 相同的档案
|
||||
(如主备都叫 "Kimi")冷却状态互相污染:A 失败冷却 → B 被误判
|
||||
跳过(该转移时不转移)。P6 去重保证每档案恰好一个实例,实例被
|
||||
_clients 持有生命周期内 id 稳定,与去重键语义一致。"""
|
||||
return id(client)
|
||||
|
||||
def _cooling(self, client) -> bool:
|
||||
entry = self._fail_at.get(client.name)
|
||||
entry = self._fail_at.get(self._cool_key(client))
|
||||
if entry is None:
|
||||
return False
|
||||
t, cd = entry
|
||||
@@ -377,9 +405,9 @@ class FailoverClient:
|
||||
|
||||
def _mark_fail(self, client, cooldown: Optional[float] = None):
|
||||
# L4c:429 Retry-After(经 TransientLLMError.retry_after)覆盖默认冷却时长
|
||||
self._fail_at[client.name] = (time.monotonic(),
|
||||
cooldown if cooldown and cooldown > 0
|
||||
else self.cooldown_seconds)
|
||||
self._fail_at[self._cool_key(client)] = (time.monotonic(),
|
||||
cooldown if cooldown and cooldown > 0
|
||||
else self.cooldown_seconds)
|
||||
|
||||
def _budget_exhausted(self) -> bool:
|
||||
if self.timeout_budget is None or self._call_started is None:
|
||||
@@ -451,10 +479,25 @@ class FailoverClient:
|
||||
max_tokens: int = 8192, tools: Optional[List[Dict]] = None,
|
||||
tool_choice: Optional[str] = None,
|
||||
reasoning_enabled: bool = False, **extra) -> Iterator[Dict]:
|
||||
"""L4b:流式转移——仅首事件前故障可安全重放(对用户不可见)。"""
|
||||
"""L4b:流式转移——仅首事件前故障可安全重放(对用户不可见)。
|
||||
|
||||
T7:本方法是普通函数(非生成器)——生成器函数体延迟到首次 next()
|
||||
才执行,若 cancel() 在创建生成器与首迭代之间被其他线程调用,
|
||||
延迟执行的 `_cancelled = False` 会把用户的取消清掉(请求照常发出)。
|
||||
标志重置与计数在返回生成器前同步完成,取消契约(类文档)恢复成立。
|
||||
"""
|
||||
self._cancelled = False # 新一轮请求,清除上一轮中断标志
|
||||
self._call_started = time.monotonic()
|
||||
self._stats["requests"] += 1
|
||||
return self._stream_chat_impl(messages, temperature=temperature,
|
||||
max_tokens=max_tokens, tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
reasoning_enabled=reasoning_enabled, **extra)
|
||||
|
||||
def _stream_chat_impl(self, messages: List[Dict], temperature: float = None,
|
||||
max_tokens: int = 8192, tools: Optional[List[Dict]] = None,
|
||||
tool_choice: Optional[str] = None,
|
||||
reasoning_enabled: bool = False, **extra) -> Iterator[Dict]:
|
||||
last_exc = None
|
||||
for idx, client in enumerate(self._clients):
|
||||
if self._cooling(client):
|
||||
|
||||
@@ -71,26 +71,63 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
|
||||
def __init__(self, logger=None):
|
||||
super().__init__(logger)
|
||||
self._current_process = None
|
||||
self._current_process = None # 兼容保留:最近一次注册的进程(诊断用)
|
||||
self._current_pgid = None # L9:Popen 后立即捕获,shell 死亡后 getpgid 会失败
|
||||
self._process_lock = threading.Lock()
|
||||
# T5:单槽 _current_process 在并发执行时互相覆盖——先启动的执行
|
||||
# 超时/取消会误杀后来者的进程组(killpg 连坐孙进程),先结束的
|
||||
# finally 清引用导致后来者永远无法取消。改为令牌多槽注册表,
|
||||
# 终止只作用于自己槽位内的 (proc, pgid)。
|
||||
self._proc_slots: Dict[int, Tuple] = {}
|
||||
self._token_seq = 0
|
||||
|
||||
def _terminate_tree(self, proc, force: bool = False):
|
||||
def _register_proc(self, process):
|
||||
"""T5:注册本次执行的进程槽,返回 (token, pgid)"""
|
||||
with self._process_lock:
|
||||
self._token_seq += 1
|
||||
token = self._token_seq
|
||||
pgid = None
|
||||
if os.name == "posix":
|
||||
try:
|
||||
pgid = os.getpgid(process.pid)
|
||||
except Exception:
|
||||
pgid = None
|
||||
self._proc_slots[token] = (process, pgid)
|
||||
# 兼容属性指向最新注册(诊断/旧测试观察用,不参与终止逻辑)
|
||||
self._current_process = process
|
||||
self._current_pgid = pgid
|
||||
return token, pgid
|
||||
|
||||
def _unregister_proc(self, token):
|
||||
"""T5:仅移除自己的槽位;全部槽位清空时同步清兼容属性"""
|
||||
with self._process_lock:
|
||||
self._proc_slots.pop(token, None)
|
||||
if not self._proc_slots:
|
||||
self._current_process = None
|
||||
self._current_pgid = None
|
||||
|
||||
def _terminate_tree(self, proc, force: bool = False, pgid=None):
|
||||
"""终止整个进程组(L9:POSIX 下 shell=True 只杀 shell 时,孙进程
|
||||
(`(x &)` 等)仍持管道写端——reader 阻塞无 EOF、fd/线程悬挂至孙
|
||||
进程退出,写满 64KB 管道缓冲后孙进程也卡死。start_new_session 使
|
||||
shell 成为会话首进程,按组信号可一次覆盖整棵进程树)。
|
||||
组 ID 在 Popen 后立即保存:shell 死亡后 os.getpgid(pid) 会抛
|
||||
ProcessLookupError,届时无法再从进程反查组。
|
||||
T5:pgid 由调用方显式传入(注册槽位时捕获),不再读共享
|
||||
_current_pgid——并发执行时那是别人的进程组。
|
||||
Windows 无进程组概念,退回单进程 terminate/kill。"""
|
||||
try:
|
||||
if os.name == "posix":
|
||||
sig = signal.SIGKILL if force else signal.SIGTERM
|
||||
with self._process_lock:
|
||||
pgid = self._current_pgid
|
||||
if pgid is None:
|
||||
pgid = os.getpgid(proc.pid)
|
||||
os.killpg(pgid, sig)
|
||||
try:
|
||||
pgid = os.getpgid(proc.pid)
|
||||
except Exception:
|
||||
pgid = None
|
||||
if pgid is not None:
|
||||
os.killpg(pgid, sig)
|
||||
else:
|
||||
proc.kill() if force else proc.terminate()
|
||||
elif force:
|
||||
proc.kill()
|
||||
else:
|
||||
@@ -102,18 +139,21 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
pass
|
||||
|
||||
def cancel(self):
|
||||
"""取消当前正在运行的命令(kill子进程组)。线程安全。"""
|
||||
"""取消本执行器当前注册的全部运行中命令(kill 子进程组)。线程安全。
|
||||
T5:多槽令牌注册表——并发执行时定向终止各自槽位,互不误杀;
|
||||
无参调用语义为"终止本执行器全部在途命令"(中断/关闭路径)。"""
|
||||
with self._process_lock:
|
||||
proc = self._current_process
|
||||
if proc is not None:
|
||||
slots = list(self._proc_slots.values())
|
||||
for proc, pgid in slots:
|
||||
try:
|
||||
if proc.poll() is None:
|
||||
self._terminate_tree(proc, force=True)
|
||||
self._terminate_tree(proc, force=True, pgid=pgid)
|
||||
self.logger.info("已终止正在运行的子进程组")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _read_with_limit(self, process, timeout: int, start_time: float, output_queue=None) -> Tuple[str, bool, bool]:
|
||||
def _read_with_limit(self, process, timeout: int, start_time: float, output_queue=None,
|
||||
pgid=None) -> Tuple[str, bool, bool]:
|
||||
"""从进程读取输出,限制总大小和超时。返回 (output_str, timeout_occurred, truncated)。
|
||||
|
||||
H2 修复:阻塞 readline 在独立 reader 线程执行,主循环带超时轮询队列。
|
||||
@@ -147,14 +187,14 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
# J16(审计 2026-08-19):timeout<=0 语义为"不设超时"(有意为之,
|
||||
# 调用方显式传 0 禁用);registry 层 3600s 流式兜底仍生效
|
||||
if timeout > 0 and time.time() - start_time > timeout:
|
||||
self._terminate_tree(process)
|
||||
self._terminate_tree(process, pgid=pgid)
|
||||
timeout_occurred = True
|
||||
break
|
||||
|
||||
# 检查输出大小
|
||||
if total_bytes >= self.MAX_OUTPUT_BYTES:
|
||||
truncated = True
|
||||
self._terminate_tree(process)
|
||||
self._terminate_tree(process, pgid=pgid)
|
||||
break
|
||||
|
||||
try:
|
||||
@@ -175,7 +215,7 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
# 与管道 fd 至孙进程退出,写满 64KB 缓冲后孙进程自身
|
||||
# 也卡死。此处强杀进程组收尾(nohup 重定向输出的
|
||||
# 真正脱离场景不持管道,不会走到这里)。
|
||||
self._terminate_tree(process, force=True)
|
||||
self._terminate_tree(process, force=True, pgid=pgid)
|
||||
break
|
||||
continue
|
||||
|
||||
@@ -188,7 +228,7 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
output_chunks.append(line.encode('utf-8', errors='replace')[:allowed].decode('utf-8', errors='ignore'))
|
||||
total_bytes = self.MAX_OUTPUT_BYTES
|
||||
truncated = True
|
||||
self._terminate_tree(process)
|
||||
self._terminate_tree(process, pgid=pgid)
|
||||
break
|
||||
|
||||
output_chunks.append(line)
|
||||
@@ -224,6 +264,14 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
if not self.is_enabled():
|
||||
return "❌ 命令执行功能已禁用"
|
||||
|
||||
# T24:非流式路径没有 registry 3600s 流式兜底,timeout<=0("不设
|
||||
# 超时")会让挂起命令永久阻塞调用线程——下限夹取到 1s 快速失败。
|
||||
# 流式路径保留 0=不限语义(registry 兜底在位)
|
||||
try:
|
||||
timeout = max(1, int(timeout))
|
||||
except (TypeError, ValueError):
|
||||
timeout = 30
|
||||
|
||||
# 默认工作目录为workspace,避免在启动目录散落文件
|
||||
if work_dir is None or work_dir == ".":
|
||||
work_dir = constants.WORKSPACE_DIR
|
||||
@@ -236,6 +284,8 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
exec_env.update(env)
|
||||
|
||||
process = None
|
||||
token = None
|
||||
pgid = None
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
@@ -246,31 +296,28 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
text=True,
|
||||
bufsize=1,
|
||||
universal_newlines=True,
|
||||
encoding="utf-8", # T16:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出
|
||||
errors="replace", # T16:非法字节不抛异常中断 reader 线程
|
||||
env=exec_env,
|
||||
cwd=os.path.expanduser(work_dir),
|
||||
# L9:POSIX 下 shell 成为会话首进程,超时/取消可按进程组
|
||||
# 一次终止整棵进程树(含 & 后台孙进程),消除管道悬挂
|
||||
start_new_session=(os.name == "posix")
|
||||
)
|
||||
# 保存进程引用以便cancel()可以kill它
|
||||
with self._process_lock:
|
||||
self._current_process = process
|
||||
if os.name == "posix":
|
||||
try:
|
||||
self._current_pgid = os.getpgid(process.pid)
|
||||
except Exception:
|
||||
self._current_pgid = None
|
||||
# T5:注册本执行专属槽位,终止/取消只作用于自己的 (proc, pgid)
|
||||
token, pgid = self._register_proc(process)
|
||||
|
||||
start_time = time.time()
|
||||
output_str, timeout_occurred, truncated = self._read_with_limit(process, timeout, start_time)
|
||||
output_str, timeout_occurred, truncated = self._read_with_limit(
|
||||
process, timeout, start_time, pgid=pgid)
|
||||
|
||||
# 确保进程终止
|
||||
if process.poll() is None:
|
||||
self._terminate_tree(process, force=timeout_occurred or truncated)
|
||||
self._terminate_tree(process, force=timeout_occurred or truncated, pgid=pgid)
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except:
|
||||
self._terminate_tree(process, force=True)
|
||||
self._terminate_tree(process, force=True, pgid=pgid)
|
||||
|
||||
return_code = process.returncode if process.returncode is not None else -1
|
||||
|
||||
@@ -298,11 +345,10 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
return f"❌ 执行错误: {str(e)}"
|
||||
finally:
|
||||
if process and process.poll() is None:
|
||||
self._terminate_tree(process, force=True)
|
||||
# 清除当前进程引用
|
||||
with self._process_lock:
|
||||
self._current_process = None
|
||||
self._current_pgid = None
|
||||
self._terminate_tree(process, force=True, pgid=pgid)
|
||||
# T5:仅注销自己的槽位,不再无差别清空共享引用
|
||||
if token is not None:
|
||||
self._unregister_proc(token)
|
||||
|
||||
def execute_streaming(self, command: str, output_queue: queue.Queue, timeout: int = 30,
|
||||
work_dir: str = None, env: Dict = None, **kwargs):
|
||||
@@ -329,6 +375,8 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
exec_env.update(env)
|
||||
|
||||
process = None
|
||||
token = None
|
||||
pgid = None
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
@@ -339,6 +387,8 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
text=True,
|
||||
bufsize=1,
|
||||
universal_newlines=True,
|
||||
encoding="utf-8", # T16:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出
|
||||
errors="replace", # T16:非法字节不抛异常中断 reader 线程
|
||||
env=exec_env,
|
||||
cwd=os.path.expanduser(work_dir),
|
||||
# L9:POSIX 下 shell 成为会话首进程,超时/取消可按进程组
|
||||
@@ -346,19 +396,13 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
start_new_session=(os.name == "posix")
|
||||
)
|
||||
self.process = process
|
||||
# 保存进程引用以便cancel()可以kill它
|
||||
with self._process_lock:
|
||||
self._current_process = process
|
||||
if os.name == "posix":
|
||||
try:
|
||||
self._current_pgid = os.getpgid(process.pid)
|
||||
except Exception:
|
||||
self._current_pgid = None
|
||||
# T5:注册本执行专属槽位,终止/取消只作用于自己的 (proc, pgid)
|
||||
token, pgid = self._register_proc(process)
|
||||
|
||||
start_time = time.time()
|
||||
# H2 修复:复用线程化读取(reader 线程 + 超时轮询),行经 output_queue 流式转发
|
||||
output_str, timeout_occurred, truncated = self._read_with_limit(
|
||||
process, timeout, start_time, output_queue=output_queue)
|
||||
process, timeout, start_time, output_queue=output_queue, pgid=pgid)
|
||||
|
||||
if timeout_occurred:
|
||||
output_queue.put(("line", f"\n⏰ 命令执行超时({timeout}秒)\n"))
|
||||
@@ -367,7 +411,7 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
|
||||
# 如果超时或截断,确保进程被终止
|
||||
if (timeout_occurred or truncated) and process.poll() is None:
|
||||
self._terminate_tree(process, force=True)
|
||||
self._terminate_tree(process, force=True, pgid=pgid)
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except:
|
||||
@@ -404,12 +448,12 @@ class SubprocessCommandExecutor(CommandExecutor):
|
||||
output_queue.put(("complete", f"❌ 执行错误: {str(e)}"))
|
||||
finally:
|
||||
if process and process.poll() is None:
|
||||
self._terminate_tree(process)
|
||||
self._terminate_tree(process, pgid=pgid)
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except:
|
||||
self._terminate_tree(process, force=True)
|
||||
self._terminate_tree(process, force=True, pgid=pgid)
|
||||
self.process = None
|
||||
with self._process_lock:
|
||||
self._current_process = None
|
||||
self._current_pgid = None
|
||||
# T5:仅注销自己的槽位,不再无差别清空共享引用
|
||||
if token is not None:
|
||||
self._unregister_proc(token)
|
||||
@@ -76,7 +76,7 @@ class _ProcEntry:
|
||||
self.max_buffer = max(1024, int(buffer_bytes))
|
||||
self.next_seq = 1
|
||||
self.evicted = 0
|
||||
self.lock = threading.Lock()
|
||||
self.lock = threading.RLock() # T8:可重入,write_input 持锁期间可调用 _record_line
|
||||
self.reader_done = threading.Event()
|
||||
self.reader = None
|
||||
# 交互写入:单写锁保序(bg_write 串行化,防交错输入)
|
||||
@@ -178,6 +178,8 @@ class ProcessManager:
|
||||
text=True,
|
||||
bufsize=1,
|
||||
universal_newlines=True,
|
||||
encoding="utf-8", # T17:显式 UTF-8,避免 Windows cp936 解码 UTF-8 输出致 reader 静默退出
|
||||
errors="replace",
|
||||
env=exec_env,
|
||||
cwd=os.path.expanduser(work_dir),
|
||||
# 对齐 L9:POSIX 下 shell 成为会话首进程,kill 按进程组
|
||||
@@ -345,12 +347,13 @@ class ProcessManager:
|
||||
if entry.status != "running":
|
||||
return {"ok": False, "error": f"进程已结束({entry.status}),无法写入"}
|
||||
|
||||
# watermark:回显行将占用的序号(写入前先记账,响应必然在其后)
|
||||
# watermark:取号与回显行记账必须在同一临界区,保证 watermark 等于
|
||||
# 回显行真实序号,bg_output(since=watermark) 正确对齐本条响应(T8)
|
||||
echo = "[stdin] " + text.replace("\n", "⏎")
|
||||
with entry.lock:
|
||||
watermark = entry.next_seq
|
||||
# 回显行进环形缓冲 + log 镜像(多行输入折叠为单行展示)
|
||||
echo = "[stdin] " + text.replace("\n", "⏎")
|
||||
self._record_line(entry, echo)
|
||||
# 回显行进环形缓冲 + log 镜像(多行输入折叠为单行展示)
|
||||
self._record_line(entry, echo)
|
||||
|
||||
# 写入放 watchdog 线程:进程不读 stdin 且管道缓冲(~64KB)写满时
|
||||
# write() 会阻塞,不能拖死工具线程;超时后 daemon 线程由系统兜底
|
||||
|
||||
@@ -820,8 +820,9 @@ class TestL4aChatCompletionFailover:
|
||||
assert fc.chat_completion([{"role": "user", "content": "q2"}]) == "r2"
|
||||
assert mp.call_args_list[0].args[0].startswith("https://api.moonshot.cn")
|
||||
# 冷却过期(手动回拨失败时间戳):恢复走主档案
|
||||
fc._fail_at["deepseek"] = (_time.monotonic() - fc.cooldown_seconds - 1,
|
||||
fc.cooldown_seconds)
|
||||
# T6:冷却键改为客户端实例 id,种冷却态需走 _cool_key
|
||||
fc._fail_at[fc._cool_key(fc._clients[0])] = (_time.monotonic() - fc.cooldown_seconds - 1,
|
||||
fc.cooldown_seconds)
|
||||
with mock.patch("llm_client.requests.post",
|
||||
return_value=_resp(200, content="r3")) as mp:
|
||||
assert fc.chat_completion([{"role": "user", "content": "q3"}]) == "r3"
|
||||
@@ -998,8 +999,8 @@ class TestL4bStreamFailover:
|
||||
primary = self._client("deepseek", stream=_fake_stream(events=self.EVENTS))
|
||||
fallback = self._client("kimi", stream=_fake_stream(events=self.EVENTS))
|
||||
fc = self._fc(primary, fallback)
|
||||
fc._fail_at = {"deepseek": (_time.monotonic(), fc.cooldown_seconds),
|
||||
"kimi": (_time.monotonic(), fc.cooldown_seconds)}
|
||||
fc._fail_at = {fc._cool_key(primary): (_time.monotonic(), fc.cooldown_seconds),
|
||||
fc._cool_key(fallback): (_time.monotonic(), fc.cooldown_seconds)}
|
||||
with pytest.raises(RuntimeError, match="冷却期"):
|
||||
list(fc.stream_chat([{"role": "user", "content": "q"}]))
|
||||
primary.stream_chat.assert_not_called()
|
||||
@@ -1010,7 +1011,7 @@ class TestL4bStreamFailover:
|
||||
primary = self._client("deepseek", stream=_fake_stream(events=self.EVENTS))
|
||||
fallback = self._client("kimi", stream=_fake_stream(events=self.EVENTS))
|
||||
fc = self._fc(primary, fallback)
|
||||
fc._fail_at["deepseek"] = (_time.monotonic(), fc.cooldown_seconds)
|
||||
fc._fail_at[fc._cool_key(primary)] = (_time.monotonic(), fc.cooldown_seconds)
|
||||
assert list(fc.stream_chat([{"role": "user", "content": "q"}])) == self.EVENTS
|
||||
primary.stream_chat.assert_not_called()
|
||||
|
||||
@@ -1102,7 +1103,7 @@ class TestL4cObservabilityAndBudget:
|
||||
stream=_fake_stream(events=self.EVENTS))
|
||||
fallback = self._mk_client("kimi", stream=_fake_stream(events=self.EVENTS))
|
||||
fc = self._fc(primary, fallback)
|
||||
fc._fail_at["deepseek"] = (_time.monotonic(), fc.cooldown_seconds)
|
||||
fc._fail_at[fc._cool_key(primary)] = (_time.monotonic(), fc.cooldown_seconds)
|
||||
assert list(fc.stream_chat([{"role": "user", "content": "q"}])) == self.EVENTS
|
||||
s = fc.get_stats()
|
||||
assert s == {"requests": 1, "failovers": 0, "cooldown_skips": 1, "exhausted": 0}
|
||||
@@ -1231,13 +1232,14 @@ class TestL4cObservabilityAndBudget:
|
||||
# 第一次:主失败(retry_after=300)→ fallback
|
||||
assert fc.chat_completion([{"role": "user", "content": "q1"}]) == "from-kimi"
|
||||
# 冷却中(默认 60s 已过但 300s 未到):仍走 fallback
|
||||
t, _ = fc._fail_at["deepseek"]
|
||||
fc._fail_at["deepseek"] = (t - 100, 300)
|
||||
_key = fc._cool_key(primary)
|
||||
t, _ = fc._fail_at[_key]
|
||||
fc._fail_at[_key] = (t - 100, 300)
|
||||
primary.chat_completion = primary_ok
|
||||
assert fc.chat_completion([{"role": "user", "content": "q2"}]) == "from-kimi"
|
||||
primary_ok.assert_not_called()
|
||||
# 300s 冷却过期:恢复主档案
|
||||
fc._fail_at["deepseek"] = (t - 301, 300)
|
||||
fc._fail_at[_key] = (t - 301, 300)
|
||||
assert fc.chat_completion([{"role": "user", "content": "q3"}]) == "from-ds"
|
||||
primary_ok.assert_called_once()
|
||||
|
||||
|
||||
@@ -133,6 +133,11 @@ class BgRunTool(Tool):
|
||||
if not _enabled():
|
||||
return "❌ 后台进程功能已禁用(commands.enabled 或 background_processes.enabled 为 false)"
|
||||
|
||||
# T18:LLM 可能传字符串 "false",裸 bool("false") is True——
|
||||
# detach 误开致进程脱离管理、log_file 误开,统一规整
|
||||
detach = _coerce_bool(detach, False)
|
||||
log_file = _coerce_bool(log_file, False)
|
||||
|
||||
result = self.manager.start(command, name=proc_name, work_dir=work_dir,
|
||||
env=env, detach=detach, log_file=log_file)
|
||||
if not result.get("ok"):
|
||||
@@ -393,6 +398,9 @@ class BgKillTool(Tool):
|
||||
if handle is None:
|
||||
return "❌ 错误:handle 必须是整数句柄"
|
||||
|
||||
# T18:force 字符串 "false" 会被裸 bool 判为 True,跳过宽限直接 SIGKILL
|
||||
force = _coerce_bool(force, False)
|
||||
|
||||
result = self.manager.kill(handle, force=force, grace=grace)
|
||||
if not result.get("ok"):
|
||||
return self.format_result(False, f"终止后台进程 #{handle} 失败",
|
||||
|
||||
+33
-15
@@ -67,29 +67,42 @@ class ToolRegistry:
|
||||
def get_tools_schemas(self) -> List[Dict]:
|
||||
"""获取所有工具的OpenAI函数调用格式"""
|
||||
return [tool.to_function_call_format() for tool in self._tools.values()]
|
||||
|
||||
@staticmethod
|
||||
def _truncate_result(result) -> str:
|
||||
"""T15:工具返回值统一长度预算(非流式与流式最终结果同口径)"""
|
||||
if not isinstance(result, str):
|
||||
return result
|
||||
limit = constants.TOOL_MAX_RESULT_LENGTH
|
||||
if len(result) > limit:
|
||||
return (result[:limit] +
|
||||
f"\n⚠️ 结果超过 {limit} 字符上限,已截断")
|
||||
return result
|
||||
|
||||
def set_output_queue(self, queue: queue.Queue):
|
||||
"""设置输出队列"""
|
||||
"""设置输出队列(T4:仅记注册表级引用,不再传播到共享工具实例——
|
||||
实例属性由 execute_streaming 每次调用自行注入局部队列,全局传播
|
||||
会让并发会话互相覆盖对方的输出通路,造成跨会话输出串扰)"""
|
||||
self.current_output_queue = queue
|
||||
# 同时设置所有工具的队列
|
||||
for tool in self._tools.values():
|
||||
tool.output_queue = queue
|
||||
|
||||
|
||||
def execute_tool(self, name: str, **kwargs) -> str:
|
||||
"""执行工具"""
|
||||
tool = self.get_tool(name)
|
||||
if not tool:
|
||||
return f"❌ 错误:找不到工具 '{name}'"
|
||||
|
||||
|
||||
self.logger.debug(f"工具调用: {name}({_sanitize_kwargs(kwargs)})")
|
||||
start_time = time.time()
|
||||
|
||||
|
||||
try:
|
||||
# T4:入口快照会话队列。转发循环可能持续数秒~小时,期间并发
|
||||
# 会话的 set_output_queue 会覆盖单值引用——全程用快照,保证
|
||||
# 本会话输出只进本会话队列,结束清理也不影响在途转发
|
||||
session_queue = self.current_output_queue
|
||||
# 检查是否有队列可用,并且工具支持流式输出
|
||||
if self.current_output_queue and hasattr(tool, 'execute_streaming'):
|
||||
if session_queue and hasattr(tool, 'execute_streaming'):
|
||||
# 设置工具的流式输出属性
|
||||
tool.streaming_output = True
|
||||
tool.output_queue = self.current_output_queue
|
||||
|
||||
# 创建队列来收集流式输出
|
||||
output_queue = queue.Queue()
|
||||
@@ -134,8 +147,8 @@ class ToolRegistry:
|
||||
# 预算耗尽:停止转发,仅提示一次
|
||||
if not truncation_noted:
|
||||
truncation_noted = True
|
||||
if self.current_output_queue:
|
||||
self.current_output_queue.put(
|
||||
if session_queue:
|
||||
session_queue.put(
|
||||
("line", f"\n⚠️ 流式输出超过 {line_budget} 字符上限,已停止转发剩余内容\n"))
|
||||
continue
|
||||
remaining = line_budget - forwarded
|
||||
@@ -146,13 +159,13 @@ class ToolRegistry:
|
||||
else:
|
||||
forwarded += len(piece)
|
||||
# 发送到当前队列(会通过WebSocket发送)
|
||||
if self.current_output_queue:
|
||||
self.current_output_queue.put(("line", piece))
|
||||
if session_queue:
|
||||
session_queue.put(("line", piece))
|
||||
elif msg_type == "complete":
|
||||
final_result = content
|
||||
# 也发送完整结果
|
||||
if self.current_output_queue:
|
||||
self.current_output_queue.put(("complete", content))
|
||||
if session_queue:
|
||||
session_queue.put(("complete", content))
|
||||
except queue.Empty:
|
||||
continue
|
||||
|
||||
@@ -173,6 +186,11 @@ class ToolRegistry:
|
||||
else:
|
||||
# 普通执行
|
||||
result = tool.execute(**kwargs)
|
||||
|
||||
# T15:两条分支的最终返回值统一过 TOOL_MAX_RESULT_LENGTH——
|
||||
# 此前仅流式转发有 line_budget(N8),非流式结果无上限直入
|
||||
# 对话上下文(memory/memo get 超长 content,token/内存放大)
|
||||
result = self._truncate_result(result)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
self.logger.debug(f"工具结果 ({elapsed:.2f}s): {result[:100]}...")
|
||||
|
||||
Reference in new issue
Block a user