# METADATA # { # "name": "duckduckgo", # # "display_name": { # "zh": "DuckDuckGo 搜索", # "en": "DuckDuckGo Search" # }, # "description": { "zh": "使用DuckDuckGo进行网络搜索和内容抓取。", "en": "Use DuckDuckGo for web search and content extraction." }, # "category": "Search", # "tools": [ # { # "name": "search", # "description": { "zh": "执行DuckDuckGo搜索并返回格式化的结果。", "en": "Run a DuckDuckGo search and return formatted results." }, # "parameters": [ # { # "name": "query", # "description": { "zh": "搜索查询字符串", "en": "Search query string." }, # "type": "string", # "required": true # }, # { # "name": "max_results", # "description": { "zh": "返回的最大结果数 (默认: 10)", "en": "Maximum number of results to return (default: 10)." }, # "type": "string", # "required": false # } # ] # }, # { # "name": "fetch_content", # "description": { "zh": "从网页URL抓取和解析内容。", "en": "Fetch and parse content from a webpage URL." }, # "parameters": [ # { # "name": "url", # "description": { "zh": "要抓取内容的网页URL", "en": "Webpage URL to fetch and parse." }, # "type": "string", # "required": true # } # ] # } # ] # } import re import time import asyncio import traceback from urllib.parse import unquote client = OkHttp.newClient() BASE_URL = "https://html.duckduckgo.com/html/" class RateLimiter: """A simple rate limiter to avoid sending too many requests.""" def __init__(self, requestsPerMinute=30): self.requestsPerMinute = requestsPerMinute self.requests = [] async def acquire(self): now = time.time() * 1000 # Remove requests older than 1 minute (60000 ms) self.requests = [reqTime for reqTime in self.requests if now - reqTime < 60000] if len(self.requests) >= self.requestsPerMinute: timeSinceFirstRequest = now - self.requests[0] waitTime = 60000 - timeSinceFirstRequest if waitTime > 0: await asyncio.sleep(waitTime / 1000) self.requests.append(time.time() * 1000) searchRateLimiter = RateLimiter(30) fetchRateLimiter = RateLimiter(20) def decodeHtmlEntities(text): """Decodes basic HTML entities.""" text = text.replace("&", "&") text = text.replace("<", "<") text = text.replace(">", ">") text = text.replace(""", '"') text = text.replace("'", "'") return text async def search(params): query = params.get("query") max_results = 10 if params.get("max_results"): try: parsedMaxResults = int(params["max_results"]) max_results = parsedMaxResults except (ValueError, TypeError): pass if not query: raise Exception("查询不能为空") await searchRateLimiter.acquire() console.log(f"正在从DuckDuckGo搜索: {query}") request = client.newRequest() request = request.url(BASE_URL) request = request.method("POST") request = request.body({"q": query, "b": "", "kl": ""}, "form") request = request.headers({ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" }) response = await request.build().execute() if not response.isSuccessful(): raise Exception(f"HTTP 错误! 状态码: {response.statusCode}") html = response.content results = [] resultRegex = re.compile( r'

.*?(.*?).*?

.*?(.*?)', re.DOTALL ) for match in resultRegex.finditer(html): if len(results) >= max_results: break link = match.group(1) if "y.js" in link: continue # Skip ads if link.startswith("//duckduckgo.com/l/?uddg="): try: link = unquote(link.split("uddg=")[1].split("&")[0]) except Exception as e: console.error(f"URL 解码失败: {link}") title = decodeHtmlEntities(re.sub(r'<[^>]*>', '', match.group(2)).strip()) snippet = decodeHtmlEntities(re.sub(r'<[^>]*>', '', match.group(3)).strip()) results.append({"title": title, "link": link, "snippet": snippet, "position": len(results) + 1}) console.log(f"成功找到 {len(results)} 个结果") return format_results_for_llm(results) async def fetch_content(params): url = params.get("url") if not url: raise Exception("URL不能为空") await fetchRateLimiter.acquire() console.log(f"正在抓取内容: {url}") try: request = client.newRequest() request = request.url(url) request = request.method("GET") request = request.headers({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}) response = await request.build().execute() if not response.isSuccessful(): raise Exception(f"无法访问网页 ({response.statusCode})") text = response.content text = re.sub(r']*>.*?', '', text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r']*>.*?', '', text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r']*>.*?', '', text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r']*>.*?', '', text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r']*>.*?', '', text, flags=re.IGNORECASE | re.DOTALL) text = re.sub(r'<[^>]+>', ' ', text) text = re.sub(r'\s+', ' ', text).strip() if len(text) > 8000: text = text[:8000] + "... [内容被截断]" console.log(f"成功抓取并解析内容 ({len(text)} 字符)") return text except Exception as error: msg = str(getattr(error, "message", error)) console.error(f"从 {url} 抓取内容时出错: {msg}") return f"错误: 从网页抓取内容时发生意外错误 ({msg})" def format_results_for_llm(results): if not results or len(results) == 0: return "没有为您的搜索查询找到结果。这可能是由于DuckDuckGo的机器人检测或查询没有匹配项。请尝试重新措辞您的搜索或在几分钟后重试。" output = [ f"{r['position']}. {r['title']}\n URL: {r['link']}\n 摘要: {r['snippet']}" for r in results ] return f"找到 {len(results)} 个搜索结果:\n\n" + "\n\n".join(output) async def duckduckgo_wrap(func, params, successMessage, failMessage): try: console.log(f"开始执行函数: {func.__name__ or '匿名函数'}") result = await func(params) complete({"success": True, "message": successMessage, "data": result}) except Exception as error: msg = str(getattr(error, "message", error)) console.error(f"函数 {func.__name__ or '匿名函数'} 执行失败! 错误: {msg}") complete({"success": False, "message": f"{failMessage}: {msg}", "error_stack": traceback.format_exc()}) async def search_wrapper(params): await duckduckgo_wrap(search, params, "搜索完成", "搜索失败") async def fetch_content_wrapper(params): await duckduckgo_wrap(fetch_content, params, "内容抓取完成", "内容抓取失败") exports.search = search_wrapper exports.fetch_content = fetch_content_wrapper