agentUniverse 0.0.17.dev6__tar.gz → 0.0.18.dev1__tar.gz

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: agentUniverse
-Version: 0.0.17.dev6
+Version: 0.0.18.dev1
 Summary: agentUniverse is a framework for developing applications powered by multi-agent base on large language model.
 Author: AntGroup
 Author-email: jerry.zzw@antgroup.com
@@ -27,8 +27,9 @@ Requires-Dist: flask_cors (==4.0.0)
 Requires-Dist: googleapis-common-protos (>=1.63.0,<2.0.0)
 Requires-Dist: grpcio (==1.63.0)
 Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
-Requires-Dist: httpx (==0.27.2)
+Requires-Dist: httpx (>=0.27.2)
 Requires-Dist: jieba (>=0.42.1,<0.43.0)
+Requires-Dist: jsonlines (>=4.0.0,<5.0.0)
 Requires-Dist: langchain (==0.1.20)
 Requires-Dist: langchain-anthropic (>=0.1.13,<0.2.0)
 Requires-Dist: langchain-community (==0.0.38)
@@ -39,9 +40,13 @@ Requires-Dist: myst-parser (>=2.0.0,<3.0.0)
 Requires-Dist: networkx (>=3.3,<4.0)
 Requires-Dist: numpy (>=1.26.0,<2.0.0)
 Requires-Dist: ollama (>=0.2.1,<0.3.0)
-Requires-Dist: openai (==1.13.3)
+Requires-Dist: openai (==1.55.3)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
+Requires-Dist: opentelemetry-api (>=1.25.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.25.0,<2.0.0)
+Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.25.0,<2.0.0)
+Requires-Dist: opentelemetry-sdk (>=1.25.0,<2.0.0)
+Requires-Dist: opentelemetry-semantic-conventions (>=0.48b0)
 Requires-Dist: opentracing (>=2.4.0,<3.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: pillow (>=10.4.0,<11.0.0)

agentuniverse-0.0.18.dev1/agentuniverse/agent/action/knowledge/reader/cloud_file_reader/yuque_reader.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2025/5/2 17:40
+# @Author  : KiteSoar
+# @Email   : hushihao2020x@163.com
+# @FileName: yuque_reader.py
+
+import re
+import time
+import random
+import json
+import urllib.parse
+import requests
+from pydantic import ConfigDict
+from requests.adapters import HTTPAdapter, Retry
+from bs4 import BeautifulSoup
+from typing import List, Any, Optional
+
+from agentuniverse.agent.action.knowledge.reader.reader import Reader
+from agentuniverse.agent.action.knowledge.store.document import Document
+
+
+class YuqueReader(Reader):
+    """
+    YuqueReader is a specialized reader designed to fetch and parse content from Yuque
+
+    Attributes:
+        cookies: Cookies for authentication with Yuque.
+        session: HTTP session with retry support.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    cookies: Optional[str] = None
+    session: Optional[requests.Session] = None
+    def __init__(self, cookies: str = None, **data: Any):
+        """Initialize HTTP session with retry support and optional cookies"""
+        super().__init__(**data)
+        self.cookies = cookies
+        self.session = requests.Session()
+        retries = Retry(total=5, backoff_factor=0.5, status_forcelist=[500,502,503,504])
+        self.session.mount('https://', HTTPAdapter(max_retries=retries))
+
+    def _fetch_url_title(self, url: str) -> str:
+        """Fetch page title and clean illegal filename characters"""
+        headers = {'Cookie': self.cookies} if self.cookies else {}
+        try:
+            resp = self.session.get(url, headers=headers, timeout=10)
+            resp.raise_for_status()
+            soup = BeautifulSoup(resp.text, 'html.parser')
+            tag = soup.title
+            if not tag or not tag.string:
+                return "Untitled"
+            title = tag.string.strip()
+            title = re.sub(r'[\\/:*?"<>|]', '-', title)
+            return title.replace(' · 语雀', '')
+        except requests.exceptions.RequestException:
+            return "Request Error"
+
+    def _fetch_page_markdown(self, book_id: str, slug: str) -> str:
+        """Fetch markdown source for a single document"""
+        headers = {'Cookie': self.cookies} if self.cookies else {}
+        url = f'https://www.yuque.com/api/docs/{slug}?book_id={book_id}&merge_dynamic_data=false&mode=markdown'
+        resp = self.session.get(url, headers=headers, timeout=20)
+        if resp.status_code != 200:
+            print("The document download failed. The page may have been deleted.", book_id, slug)
+            return ''
+        data = resp.json().get('data', {})
+        md = data.get('sourcecode', '')
+        # Process image references inline
+        def repl(m):
+            src = m.group(1)
+            return f'![]({src})'
+        return re.sub(r'!\[.*?\]\((.*?)\)', repl, md)
+
+    def _load_data(self, url: str) -> List[Document]:
+        """Fetch all docs in a Yuque book and return as List[Document]"""
+        headers = {'Cookie': self.cookies} if self.cookies else {}
+        try:
+            resp = self.session.get(url, headers=headers, timeout=10)
+            resp.raise_for_status()
+            encoded = re.findall(r'decodeURIComponent\("(.+)"\)\);', resp.text)[0]
+            docs = json.loads(urllib.parse.unquote(encoded))
+        except Exception as e:
+            print(f"Request failed: {e}")
+            return []
+
+        book_title = self._fetch_url_title(url)
+        # sanitize titles for metadata keys if needed
+        chars = '/:*?"<>|\n\r'
+        trans = str.maketrans({c: '_' for c in chars})
+
+        documents: List[Document] = []
+        for item in docs['book']['toc']:
+            if item['title'] != book_title:
+                continue
+            md = self._fetch_page_markdown(str(docs['book']['id']), item['url'])
+            if not md:
+                continue
+            metadata = {
+                'source': url,
+                'doc_title': item['title'],
+                'sanitized_title': item['title'].translate(trans)
+            }
+            documents.append(Document(text=md, metadata=metadata))
+            # Respectful delay
+            time.sleep(random.uniform(1, 3))
+        return documents
+
+    def __del__(self):
+        """Close HTTP session"""
+        try:
+            self.session.close()
+        except:
+            pass
+
+# if __name__ == '__main__':
+#     # Example usage
+#     reader = YuqueReader(cookies=None)
+#     test_url = "Fill in the publicly accessible Yuque document link"
+#     docs = reader.load_data(test_url)
+#     for doc in docs:
+#         print(f"Title: {doc.metadata['doc_title']}")
+#         print(doc.text)

agentuniverse-0.0.18.dev1/agentuniverse/agent/action/knowledge/reader/cloud_file_reader/yuque_reader.yaml

@@ -0,0 +1,6 @@
+name: 'default_yuque_reader'
+description: 'default yuque document reader'
+metadata:
+  type: 'READER'
+  module: 'agentuniverse.agent.action.knowledge.reader.cloud_document.yuque_reader'
+  class: 'YuqueReader'

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/agentuniverse/agent/agent.py

@@ -46,7 +46,7 @@ from agentuniverse.base.util.common_util import stream_output
 from agentuniverse.base.util.logging.logging_util import LOGGER
 from agentuniverse.base.util.memory_util import generate_messages, get_memory_string
 from agentuniverse.base.util.system_util import process_dict_with_funcs, is_system_builtin
-from agentuniverse.base.util.tracing.au_trace_manager import AuTraceManager
+from agentuniverse.base.tracing.au_trace_manager import AuTraceManager
 from agentuniverse.llm.llm import LLM
 from agentuniverse.llm.llm_manager import LLMManager
 from agentuniverse.prompt.chat_prompt import ChatPrompt

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/agentuniverse/agent_serve/web/gunicorn_server.py

@@ -13,7 +13,7 @@ from .flask_server import app
 from .post_fork_queue import POST_FORK_QUEUE
 from ...base.annotation.singleton import singleton
 from ...base.context.framework_context_manager import FrameworkContextManager
-from ...base.util.tracing.au_trace_manager import AuTraceManager
+from agentuniverse.base.tracing.au_trace_manager import AuTraceManager
 
 DEFAULT_GUNICORN_CONFIG = {
     'bind': '127.0.0.1:8888',

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/agentuniverse/agent_serve/web/mcp/mcp_server_manager.py

@@ -91,8 +91,10 @@ class MCPServerManager:
                     'tool': [],
                     'toolkit': []
                 }
-        else:
+        elif isinstance(configer_instance.as_mcp_tool, bool) and configer_instance.as_mcp_tool:
             server_name = DEFAULT_SERVER_NAME
+        else:
+            return
 
         if configer_type == ComponentEnum.TOOL.value:
             self.server_tool_map[server_name]['tool'].append(configer_instance.name)

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/agentuniverse/agent_serve/web/request_task.py

@@ -25,7 +25,7 @@ from .dal.entity.request_do import RequestDO
 from .thread_with_result import ThreadWithReturnValue
 from agentuniverse.base.util.logging.logging_util import LOGGER
 from ...agent.output_object import OutputObject
-from agentuniverse.base.util.tracing.au_trace_manager import AuTraceManager
+from agentuniverse.base.tracing.au_trace_manager import AuTraceManager
 from agentuniverse.base.util.logging.general_logger import get_context_prefix
 from agentuniverse.base.util.logging.log_type_enum import LogTypeEnum
 

agentuniverse-0.0.18.dev1/agentuniverse/agent_serve/web/thread_with_result.py

@@ -0,0 +1,110 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/3/26 14:31
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: thread_with_result.py
+
+from concurrent.futures import ThreadPoolExecutor, Future
+from threading import Thread
+from typing import Callable
+
+from opentelemetry import context as otel_context
+
+from agentuniverse.base.context.context_coordinator import ContextCoordinator
+
+
+class ThreadWithReturnValue(Thread):
+    """A thread can save the target func exec result."""
+
+    def __init__(self, group=None, target=None, name=None,
+                 args=(), kwargs=None):
+        super().__init__(group, target, name, args, kwargs)
+
+        if kwargs is None:
+            kwargs = {}
+        self.kwargs = kwargs
+        self.args = args
+        self.target = target
+        self._return = None
+        self.error = None
+        self._context_pack = ContextCoordinator.save_context()
+
+    def run(self):
+        """Run the target func and save result in _return."""
+        if self.target is not None:
+            # set the context values in the thread
+            otel_token = ContextCoordinator.recover_context(self._context_pack)
+            try:
+                self._return = self.target(*self.args, **self.kwargs)
+            except Exception as e:
+                self.error = e
+            finally:
+                if otel_token:
+                    otel_context.detach(otel_token)
+
+    def result(self):
+        """Wait for target func finished, then return the result or raise an
+        error."""
+        self.join()
+        if self.error is not None:
+            raise self.error
+        return self._return
+
+
+class ContextAwareFuture(Future):
+    """A Future that can store context and error information."""
+
+    def __init__(self):
+        super().__init__()
+        self.error = None
+        self._context_pack = None
+
+
+class ThreadPoolExecutorWithReturnValue(ThreadPoolExecutor):
+    """A ThreadPoolExecutor that preserves context across threads."""
+
+    def submit(self, fn: Callable, *args, **kwargs) -> ContextAwareFuture:
+        """Submit a callable to be executed with context preservation."""
+        context_pack = ContextCoordinator.save_context()
+        future = ContextAwareFuture()
+        future._context_pack = context_pack
+
+        def context_wrapper():
+            otel_token = None
+            try:
+                otel_token = ContextCoordinator.recover_context(context_pack)
+
+                result = fn(*args, **kwargs)
+                future.set_result(result)
+
+            except Exception as e:
+                future.error = e
+                future.set_exception(e)
+
+            finally:
+                if otel_token:
+                    otel_context.detach(otel_token)
+
+        super().submit(context_wrapper)
+
+        return future
+
+    def map(self, func: Callable, *iterables, timeout=None, chunksize=1):
+        """Apply func to each element of iterables with context preservation."""
+        context_pack = ContextCoordinator.save_context()
+
+        def context_aware_func(*args):
+            otel_token = None
+            try:
+                otel_token = ContextCoordinator.recover_context(context_pack)
+                return func(*args)
+            except Exception as e:
+                raise e
+            finally:
+                if otel_token:
+                    otel_context.detach(otel_token)
+
+        return super().map(context_aware_func, *iterables,
+                           timeout=timeout, chunksize=chunksize)

{agentuniverse-0.0.17.dev6 → agentuniverse-0.0.18.dev1}/agentuniverse/base/agentuniverse.py

@@ -33,7 +33,7 @@ from agentuniverse.agent_serve.web.rpc.grpc.grpc_server_booster import set_grpc_
 from agentuniverse.agent_serve.web.web_booster import ACTIVATE_OPTIONS
 from agentuniverse.agent_serve.web.post_fork_queue import POST_FORK_QUEUE
 from agentuniverse.agent_serve.web.web_util import FlaskServerManager
-
+from agentuniverse.base.tracing.otel.telemetry_manager import TelemetryManager
 
 @singleton
 class AgentUniverse(object):
@@ -93,6 +93,9 @@ class AgentUniverse(object):
             config_path)
         init_loggers(log_config_path)
 
+        # Init OTEL configs
+        TelemetryManager().init_from_config(configer.value.get('OTEL', {}))
+
         # init web request task database
         RequestLibrary(configer=configer)