feat(liblogc): 统一集中式日志客户端库 + 全量迁移
新增: - liblogc/: C++ 统一日志库 (libus_logc.so)，支持 set_endpoint/set_service_name/set_level - pymsg/pymsg/logc.py: Python 统一日志模块，基于 AsyncRpcQueue C++ 迁移 (删除 6 份本地 LogC.hpp 副本): - ca, gateway: <us/LogC.hpp> + set_service_name - captcha2, confs2, smtp3: using qing::LogC + set_service_name - core: ConfsLogC → qing::LogC (ConfsTask.cpp) - 所有 Makefile 添加 -lus_logc Python 迁移 (6 个 log_client.py → 4 行薄包装): - smtp, redirect80, confs, captcha, dnfs, static - 全部委托到 pymsg.logc.LogClient Go (waf): 保持现状，自成体系
This commit is contained in:
qing committed
1 parent
01ae877673
commit
aa3741b861
32 files changed
+608
-2129
No files matched your search
@@ -0,0 +1 @@
|
||||
build/
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 建昌风物大顺脚本
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,53 @@
|
||||
# liblogc — 统一集中式日志客户端库
|
||||
#
|
||||
# 编译产物：libus_logc.so
|
||||
# 依赖：libus_msg（libmsg）、boost_json
|
||||
# 安装：头文件 → /usr/include/us/LogC.hpp
|
||||
# 库文件 → /lib/libus_logc.so
|
||||
|
||||
CC := g++
|
||||
STD := -std=c++20
|
||||
|
||||
VERSION := 0.0.1
|
||||
FRAMEWORK_NAME := us
|
||||
LIB_NAME := libus_logc
|
||||
INSTL_TDIR := /lib
|
||||
INCLU_TDIR := /usr/include
|
||||
|
||||
CFLAGS := $(STD) -O2 -fPIC -Wall
|
||||
LDFLAGS := -shared -lus_msg -lboost_json
|
||||
|
||||
SRC_DIR := src
|
||||
BUILD_DIR := build
|
||||
INC_DIR := inc
|
||||
|
||||
SRC := $(wildcard $(SRC_DIR)/*.cpp)
|
||||
OBJ := $(SRC:$(SRC_DIR)/%.cpp=$(BUILD_DIR)/%.o)
|
||||
|
||||
.PHONY: clear all install uninstall
|
||||
|
||||
all: $(BUILD_DIR) $(BUILD_DIR)/$(LIB_NAME).so
|
||||
|
||||
install: $(BUILD_DIR)/$(LIB_NAME).so
|
||||
mkdir -p $(INCLU_TDIR)/$(FRAMEWORK_NAME)
|
||||
cp -r $(INC_DIR)/$(FRAMEWORK_NAME)/* $(INCLU_TDIR)/$(FRAMEWORK_NAME)
|
||||
cp $(BUILD_DIR)/$(LIB_NAME).so $(INSTL_TDIR)/$(LIB_NAME).so.$(VERSION)
|
||||
ln -sf $(LIB_NAME).so.$(VERSION) $(INSTL_TDIR)/$(LIB_NAME).so
|
||||
ldconfig
|
||||
|
||||
uninstall:
|
||||
rm -f $(INCLU_TDIR)/$(FRAMEWORK_NAME)/LogC.hpp
|
||||
rm -f $(INSTL_TDIR)/$(LIB_NAME).so
|
||||
rm -f $(INSTL_TDIR)/$(LIB_NAME).so.$(VERSION)
|
||||
|
||||
clear:
|
||||
rm -rf $(BUILD_DIR)
|
||||
|
||||
$(BUILD_DIR):
|
||||
mkdir -p $(BUILD_DIR)
|
||||
|
||||
$(BUILD_DIR)/$(LIB_NAME).so: $(OBJ)
|
||||
$(CC) $(OBJ) -o $@ $(LDFLAGS)
|
||||
|
||||
$(BUILD_DIR)/%.o: $(SRC_DIR)/%.cpp
|
||||
$(CC) $(CFLAGS) -I$(INC_DIR) -c $< -o $@
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/bash
|
||||
cd "$(dirname "$0")" || exit
|
||||
make clear
|
||||
make
|
||||
make install
|
||||
@@ -0,0 +1,144 @@
|
||||
#pragma once
|
||||
|
||||
/** LogC — 统一集中式日志客户端（C++ 版）
|
||||
*
|
||||
* 通过 pymsg TCP 调用 dbv2 execute 路由，逐条 INSERT 到 Logs 表。
|
||||
*
|
||||
* 设计要点：
|
||||
* - 异步非阻塞：基于 pmc::net::AsyncQueue 内存队列 + 后台线程批量推送
|
||||
* - fail-safe：db 服务不可用时静默丢弃（仅 stderr 提示），绝不影响业务
|
||||
* - 缓冲批量：队列满 50 条或超时 2 秒时自动 flush
|
||||
* - 背压保护：队列上限 10000 条，满时丢最旧日志，防止内存无限增长
|
||||
* - 级别过滤：set_level() 设定阈值，低于阈值的日志直接丢弃
|
||||
* - 可配置 endpoint / service_name，各服务复用同一份库
|
||||
*
|
||||
* 用法：
|
||||
* #include <us/LogC.hpp>
|
||||
*
|
||||
* // 启动时配置（默认 127.0.0.1:8005）
|
||||
* qing::LogC::instance().set_endpoint("127.0.0.1", 8005);
|
||||
* qing::LogC::instance().set_service_name("gateway-log");
|
||||
* qing::LogC::instance().set_level("warn");
|
||||
*
|
||||
* qing::LogC::instance().info("gateway", "启动完成");
|
||||
* qing::LogC::instance().error("gateway", "连接失败",
|
||||
* std::nullopt, R"({"errno":111})");
|
||||
*
|
||||
* // 进程退出前调用（幂等）
|
||||
* qing::LogC::instance().shutdown();
|
||||
*/
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
// forward-declare: 避免在头文件中拉入 AsyncQueue 模板
|
||||
namespace pmc { namespace net {
|
||||
template <typename T> class AsyncQueue;
|
||||
}}
|
||||
|
||||
namespace qing {
|
||||
|
||||
class LogC {
|
||||
public:
|
||||
struct Entry {
|
||||
std::string service;
|
||||
std::string level;
|
||||
std::string message;
|
||||
std::optional<std::string> trace_id;
|
||||
std::optional<std::string> extra; // 已序列化的 JSON 字符串（存 Logs.extra TEXT）
|
||||
};
|
||||
|
||||
// 日志级别 → 数值（大写兼容）
|
||||
static const std::unordered_map<std::string, int>& level_order() {
|
||||
static const std::unordered_map<std::string, int> k = {
|
||||
{"debug", 0}, {"info", 1}, {"warn", 2}, {"warning", 2},
|
||||
{"error", 3}, {"critical", 3},
|
||||
{"DEBUG", 0}, {"INFO", 1}, {"WARN", 2}, {"WARNING", 2},
|
||||
{"ERROR", 3}, {"CRITICAL", 3},
|
||||
};
|
||||
return k;
|
||||
}
|
||||
|
||||
static LogC& instance() {
|
||||
static LogC inst;
|
||||
return inst;
|
||||
}
|
||||
|
||||
// ---- 配置（启动时调用，线程安全）-------------------------------------
|
||||
|
||||
void set_endpoint(const std::string& addr, int port);
|
||||
void set_service_name(const std::string& name);
|
||||
|
||||
// 设置日志级别阈值（debug|info|warn|error，大小写不限）
|
||||
void set_level(const std::string& level);
|
||||
|
||||
// ---- 推送接口（非阻塞，立即返回）-------------------------------------
|
||||
|
||||
void push(const std::string& service, const std::string& level,
|
||||
const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt);
|
||||
|
||||
void debug(const std::string& service, const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt) {
|
||||
push(service, "DEBUG", message, std::move(trace_id), std::move(extra));
|
||||
}
|
||||
void info(const std::string& service, const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt) {
|
||||
push(service, "INFO", message, std::move(trace_id), std::move(extra));
|
||||
}
|
||||
void warning(const std::string& service, const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt) {
|
||||
push(service, "WARNING", message, std::move(trace_id), std::move(extra));
|
||||
}
|
||||
void error(const std::string& service, const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt) {
|
||||
push(service, "ERROR", message, std::move(trace_id), std::move(extra));
|
||||
}
|
||||
void critical(const std::string& service, const std::string& message,
|
||||
std::optional<std::string> trace_id = std::nullopt,
|
||||
std::optional<std::string> extra = std::nullopt) {
|
||||
push(service, "CRITICAL", message, std::move(trace_id), std::move(extra));
|
||||
}
|
||||
|
||||
// ---- 生命周期 -------------------------------------------------------
|
||||
|
||||
void flush(); // 手动 flush 当前积攒批次，等待完成
|
||||
void shutdown(); // 停止后台线程并 flush 剩余日志（幂等）
|
||||
|
||||
private:
|
||||
LogC();
|
||||
~LogC();
|
||||
|
||||
LogC(const LogC&) = delete;
|
||||
LogC& operator=(const LogC&) = delete;
|
||||
|
||||
void _send_batch(std::vector<Entry>&& batch);
|
||||
static std::string _utc_now_iso();
|
||||
|
||||
// ---- 配置项 ---------------------------------------------------------
|
||||
std::string _db_addr = "127.0.0.1";
|
||||
int _db_port = 8005;
|
||||
std::string _service_name = "log";
|
||||
std::mutex _endpoint_mtx; // 保护 _db_addr/_db_port/_service_name
|
||||
std::atomic<int> _level_threshold{1}; // 默认 INFO(1)
|
||||
|
||||
static constexpr size_t _BATCH_SIZE = 50;
|
||||
static constexpr double _FLUSH_INTERVAL = 2.0;
|
||||
static constexpr size_t _MAX_QUEUE_SIZE = 10000;
|
||||
static constexpr double _TIMEOUT = 5.0;
|
||||
|
||||
// ---- 核心队列（unique_ptr 延迟到 .cpp 中实例化，保持头文件轻量）----
|
||||
std::unique_ptr<pmc::net::AsyncQueue<Entry>> _queue;
|
||||
};
|
||||
|
||||
} // namespace qing
|
||||
@@ -0,0 +1,141 @@
|
||||
/** LogC.cpp — 统一集中式日志客户端实现
|
||||
*
|
||||
* 包含 _send_batch（pymsg → db execute INSERT）和 _utc_now_iso（ISO 8601）。
|
||||
* 编译为 libus_logc.so，运行时依赖 libus_msg.so。
|
||||
*/
|
||||
|
||||
#include <us/LogC.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <ctime>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
|
||||
#include <boost/json.hpp>
|
||||
|
||||
#include <us/AsyncQueue.hpp>
|
||||
#include <us/Msg.hpp>
|
||||
|
||||
namespace json = boost::json;
|
||||
|
||||
namespace qing {
|
||||
|
||||
// ---- 生命周期 ---------------------------------------------------------------
|
||||
|
||||
LogC::LogC()
|
||||
: _queue(std::make_unique<pmc::net::AsyncQueue<Entry>>(
|
||||
[this](std::vector<Entry>&& batch) { this->_send_batch(std::move(batch)); },
|
||||
pmc::net::AsyncQueueConfig{_BATCH_SIZE, _FLUSH_INTERVAL, _MAX_QUEUE_SIZE, 0.5},
|
||||
[](const std::exception& e, std::vector<Entry>&&) {
|
||||
std::cerr << "[LogC] 发送失败: " << e.what() << std::endl;
|
||||
}))
|
||||
{
|
||||
_queue->start();
|
||||
}
|
||||
|
||||
LogC::~LogC() { shutdown(); }
|
||||
|
||||
// ---- 配置接口 ---------------------------------------------------------------
|
||||
|
||||
void LogC::set_endpoint(const std::string& addr, int port) {
|
||||
std::lock_guard<std::mutex> lock(_endpoint_mtx);
|
||||
_db_addr = addr;
|
||||
_db_port = port;
|
||||
}
|
||||
|
||||
void LogC::set_service_name(const std::string& name) {
|
||||
std::lock_guard<std::mutex> lock(_endpoint_mtx);
|
||||
_service_name = name;
|
||||
}
|
||||
|
||||
void LogC::set_level(const std::string& level) {
|
||||
auto it = level_order().find(level);
|
||||
if (it != level_order().end()) {
|
||||
_level_threshold.store(it->second, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 推送 -------------------------------------------------------------------
|
||||
|
||||
void LogC::push(const std::string& service, const std::string& level,
|
||||
const std::string& message,
|
||||
std::optional<std::string> trace_id,
|
||||
std::optional<std::string> extra) {
|
||||
auto it = level_order().find(level);
|
||||
int lvl = (it != level_order().end()) ? it->second : 1;
|
||||
if (lvl < _level_threshold.load(std::memory_order_relaxed)) return;
|
||||
_queue->push(Entry{service, level, message,
|
||||
std::move(trace_id), std::move(extra)});
|
||||
}
|
||||
|
||||
void LogC::flush() { _queue->flush(); }
|
||||
void LogC::shutdown() { _queue->shutdown(); }
|
||||
|
||||
// ---- 内部实现 ---------------------------------------------------------------
|
||||
|
||||
void LogC::_send_batch(std::vector<Entry>&& batch) {
|
||||
std::string addr;
|
||||
int port;
|
||||
std::string client_name;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(_endpoint_mtx);
|
||||
addr = _db_addr;
|
||||
port = _db_port;
|
||||
client_name = _service_name;
|
||||
}
|
||||
|
||||
const std::string now = _utc_now_iso();
|
||||
try {
|
||||
pmc::net::MsgClient client(addr, port, client_name);
|
||||
for (const auto& e : batch) {
|
||||
json::object values;
|
||||
values["ts"] = now;
|
||||
values["service"] = e.service;
|
||||
values["level"] = e.level;
|
||||
values["message"] = e.message;
|
||||
values["trace_id"] = e.trace_id ? json::value(*e.trace_id)
|
||||
: json::value(nullptr);
|
||||
values["extra"] = e.extra ? json::value(*e.extra)
|
||||
: json::value(nullptr);
|
||||
|
||||
json::object payload;
|
||||
payload["_route"] = "execute";
|
||||
payload["action"] = "insert";
|
||||
payload["table"] = "Logs";
|
||||
payload["values"] = std::move(values);
|
||||
|
||||
const std::string resp = client.call(
|
||||
json::serialize(payload), _TIMEOUT,
|
||||
/*retry_on_refused=*/false);
|
||||
auto obj = json::parse(resp).as_object();
|
||||
if (!obj.at("success").as_bool()) {
|
||||
std::string err = "unknown";
|
||||
if (auto* ev = obj.if_contains("error")) {
|
||||
if (ev->is_string()) err = ev->as_string().c_str();
|
||||
}
|
||||
std::cerr << "[LogC] INSERT 失败: " << err << std::endl;
|
||||
}
|
||||
}
|
||||
} catch (const std::exception& e) {
|
||||
// fail-safe：db 不可用时静默丢弃，绝不影响业务
|
||||
std::cerr << "[LogC] 发送失败: " << e.what() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::string LogC::_utc_now_iso() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto t = std::chrono::system_clock::to_time_t(now);
|
||||
const auto us = std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
now.time_since_epoch()).count() % 1000000;
|
||||
std::tm tm{};
|
||||
::gmtime_r(&t, &tm);
|
||||
char buf[40];
|
||||
std::snprintf(buf, sizeof buf,
|
||||
"%04d-%02d-%02dT%02d:%02d:%02d.%06d+00:00",
|
||||
tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday,
|
||||
tm.tm_hour, tm.tm_min, tm.tm_sec, static_cast<int>(us));
|
||||
return buf;
|
||||
}
|
||||
|
||||
} // namespace qing
|
||||
Reference in new issue
Block a user