agentUniverse 0.0.12.dev2__tar.gz → 0.0.13.dev0__tar.gz

{agentuniverse-0.0.12.dev2 → agentuniverse-0.0.13.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: agentUniverse
-Version: 0.0.12.dev2
+Version: 0.0.13.dev0
 Summary: agentUniverse is a framework for developing applications powered by multi-agent base on large language model.
 Home-page: https://github.com/alipay/agentUniverse
 Author: AntGroup
@@ -21,26 +21,30 @@ Requires-Dist: anthropic (>=0.26.0,<0.27.0)
 Requires-Dist: cffi (>=1.15.1,<2.0.0)
 Requires-Dist: chromadb (==0.4.24)
 Requires-Dist: dashscope (>=1.19.1,<2.0.0)
-Requires-Dist: duckduckgo-search (>=6.1.7,<7.0.0)
+Requires-Dist: duckduckgo-search (==6.1.7)
 Requires-Dist: flask (>=2.3.2,<3.0.0)
 Requires-Dist: flask_cors (==4.0.0)
 Requires-Dist: googleapis-common-protos (>=1.63.0,<2.0.0)
 Requires-Dist: grpcio (==1.63.0)
 Requires-Dist: gunicorn (>=22.0.0,<23.0.0)
+Requires-Dist: jieba (>=0.42.1,<0.43.0)
 Requires-Dist: langchain (==0.1.20)
 Requires-Dist: langchain-anthropic (>=0.1.13,<0.2.0)
 Requires-Dist: langchain-community (==0.0.38)
 Requires-Dist: langchain-core (==0.1.52)
 Requires-Dist: loguru (==0.7.2)
 Requires-Dist: myst-parser (>=2.0.0,<3.0.0)
+Requires-Dist: networkx (>=3.3,<4.0)
 Requires-Dist: numpy (>=1.26.0,<2.0.0)
 Requires-Dist: ollama (>=0.2.1,<0.3.0)
 Requires-Dist: openai (==1.13.3)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.25.0,<2.0.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: pillow (>=10.4.0,<11.0.0)
+Requires-Dist: primp (==0.5.5)
 Requires-Dist: pyarrow (>=16.1.0,<17.0.0)
-Requires-Dist: pydantic (>=2.6.4,<3.0.0)
+Requires-Dist: pydantic (>=2.6.4,<2.7.0)
 Requires-Dist: pymilvus (>=2.4.3,<3.0.0) ; extra == "store-ext"
 Requires-Dist: qianfan (>=0.3.12,<0.4.0)
 Requires-Dist: requests (>=2.32.0,<3.0.0)

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/__init__.py

@@ -0,0 +1,7 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/7/23 13:59
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: __init__.py.py

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/character_text_splitter.py

@@ -0,0 +1,50 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/8/5 14:37
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: text_splitter.py
+from typing import List, Optional
+from langchain.text_splitter import CharacterTextSplitter as Splitter
+
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import \
+    DocProcessor
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+
+class CharacterTextSplitter(DocProcessor):
+    chunk_size: int = 200
+    chunk_overlap: int = 20
+    separator: str = "/n/n"
+    splitter: Optional[Splitter] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.splitter = Splitter(separator=self.separator,
+                                 chunk_size=self.chunk_size,
+                                 chunk_overlap=self.chunk_overlap)
+
+    def _process_docs(self, origin_docs: List[Document], query: Query = None) -> \
+            List[Document]:
+        lc_doc_list = self.splitter.split_documents(Document.as_langchain_list(
+            origin_docs
+        ))
+        return Document.from_langchain_list(lc_doc_list)
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) -> 'DocProcessor':
+        super()._initialize_by_component_configer(doc_processor_configer)
+        if hasattr(doc_processor_configer, "chunk_size"):
+            self.chunk_size = doc_processor_configer.chunk_size
+        if hasattr(doc_processor_configer, "chunk_overlap"):
+            self.chunk_overlap = doc_processor_configer.chunk_overlap
+        if hasattr(doc_processor_configer, "separator"):
+            self.separator = doc_processor_configer.separator
+        self.splitter = Splitter(separator=self.separator,
+                                 chunk_size=self.chunk_size,
+                                 chunk_overlap=self.chunk_overlap)
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/character_text_splitter.yaml

@@ -0,0 +1,9 @@
+name: 'character_text_splitter'
+description: 'langchain character text splitter'
+chunk_size: 200
+chunk_overlap: 20
+separators: "/n/n"
+metadata:
+  type: 'DOC_PROCESSOR'
+  module: 'agentuniverse.agent.action.knowledge.doc_processor.character_text_splitter'
+  class: 'CharacterTextSplitter'

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/dashscope_reranker.py

@@ -0,0 +1,67 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/8/5 15:48
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: dashscope_reranker.py
+
+from typing import List, Optional
+import dashscope
+from http import HTTPStatus
+
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import \
+    DocProcessor
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+MODEL_NAME_MAP = {
+    "gte_rerank": dashscope.TextReRank.Models.gte_rerank
+}
+
+
+class DashscopeReranker(DocProcessor):
+    model_name: str = "gte_rerank"
+    top_n: int = 10
+
+    def _process_docs(self, origin_docs: List[Document], query: Query = None) -> \
+            List[Document]:
+        if not query or not query.query_str:
+            raise Exception("Dashscope reranker need an origin string query.")
+        if len(origin_docs)<1:
+            return origin_docs
+        documents_texts = []
+        for _doc in origin_docs:
+            documents_texts.append(_doc.text)
+        resp = dashscope.TextReRank.call(
+            model=MODEL_NAME_MAP.get(self.model_name),
+            query=query.query_str,
+            documents=documents_texts,
+            top_n=self.top_n,
+            return_documents=False
+        )
+        if resp.status_code == HTTPStatus.OK:
+            results = resp.output.results
+        else:
+            raise Exception(f"Dashscope rerank api call error: {resp}")
+        rerank_docs = []
+        for _result in results:
+            index = _result.index
+            if origin_docs[index].metadata:
+                origin_docs[index].metadata["relevance_score"] = _result.relevance_score
+            else:
+                origin_docs[index].metadata = {"relevance_score": _result.relevance_score}
+            rerank_docs.append(origin_docs[index])
+
+        return rerank_docs
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) -> 'DocProcessor':
+        super()._initialize_by_component_configer(doc_processor_configer)
+        if hasattr(doc_processor_configer, "model_name"):
+            self.model_name = doc_processor_configer.model_name
+        if hasattr(doc_processor_configer, "top_n"):
+            self.top_n = doc_processor_configer.top_n
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/dashscope_reranker.yaml

@@ -0,0 +1,6 @@
+name: 'dashscope_reranker'
+description: 'reranker use dashscope api'
+metadata:
+  type: 'DOC_PROCESSOR'
+  module: 'agentuniverse.agent.action.knowledge.doc_processor.dashscope_reranker'
+  class: 'DashscopeReranker'

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/doc_processor.py

@@ -0,0 +1,56 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/7/23 14:00
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: doc_processor.py
+
+from abc import abstractmethod
+from typing import List, Optional
+
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.base.component.component_base import ComponentEnum
+from agentuniverse.base.component.component_base import ComponentBase
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+
+class DocProcessor(ComponentBase):
+    """The basic class for doc processor.
+    """
+
+    component_type: ComponentEnum = ComponentEnum.DOC_PROCESSOR
+    name: Optional[str] = None
+    description: Optional[str] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def process_docs(self, origin_docs: List[Document], query: Query = None) -> \
+            List[Document]:
+        """Process input documents，return should also be a document list."""
+        return self._process_docs(origin_docs, query)
+
+    @abstractmethod
+    def _process_docs(self, origin_docs: List[Document],
+                      query: Query = None) -> \
+            List[Document]:
+        """Process input documents，return should also be a document list."""
+        pass
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) \
+            -> 'DocProcessor':
+        """Initialize the DocProcessor by the ComponentConfiger object.
+
+        Args:
+            doc_processor_configer(ComponentConfiger): A configer contains DocProcessor
+            basic info.
+        Returns:
+            DocProcessor: A DocProcessor instance.
+        """
+        self.name = doc_processor_configer.name
+        self.description = doc_processor_configer.description
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/doc_processor_manager.py

@@ -0,0 +1,20 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/7/23 14:12
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: doc_processor_manager.py
+
+from agentuniverse.base.annotation.singleton import singleton
+from agentuniverse.base.component.component_enum import ComponentEnum
+from agentuniverse.base.component.component_manager_base import ComponentManagerBase
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import DocProcessor
+
+
+@singleton
+class DocProcessorManager(ComponentManagerBase[DocProcessor]):
+    """A singleton manager class of the DocProcessor."""
+
+    def __init__(self):
+        super().__init__(ComponentEnum.DOC_PROCESSOR)

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/jieba_keyword_extractor.py

@@ -0,0 +1,74 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/8/6 10:44
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: jieba_keyword_extractor.py
+from typing import List
+
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import \
+    DocProcessor
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+import jieba
+import jieba.analyse
+
+# nltk english stopwords
+stop_words = {'or', "mustn't", 'how', 'their', 'again', 'few', 'other', 'who',
+              'being', 'theirs', 'during', 'if', 'on', 'she', 'wouldn', 'why',
+              'above', 'll', "weren't", 'your', 'are', 'an', 'over', 'his',
+              'hasn', 'off', 'you', 'he', 'was', "you'd", 'me', 'ain', 'any',
+              'what', 'most', 're', 'haven', "isn't", 'there', "it's", 'same',
+              'm', 'only', 'my', 'needn', 'too', 'into', 'in', 'by', 'between',
+              "that'll", "mightn't", "aren't", 'am', 'up', 'having', "you'll",
+              "you're", 'these', 'mustn', 'himself', 'down', 'such', 'wasn',
+              'ourselves', 'did', 'because', 'should', 'won', 'about', 'aren',
+              'don', 'while', 't', 'isn', 'have', 'whom', 'myself', 'itself',
+              'this', 'will', 'and', 'further', 'no', 'where', 'ma', 'yours',
+              'been', "didn't", 'that', 'had', 'when', 'we', 'herself', 'some',
+              'has', "she's", "needn't", "should've", 'of', "won't", 'both',
+              'which', "haven't", 'yourself', 'through', 'the', 'from',
+              "you've", 'for', 'then', 'hadn', 'a', 'them', 'as', 'after',
+              'themselves', "shouldn't", 'they', 'y', 'doesn', 'didn', 'here',
+              'ours', 'own', 'it', "hadn't", 'each', 'our', 'shouldn', 'all',
+              'out', 'before', 'couldn', 'd', "doesn't", 'hers', "hasn't",
+              'than', 'at', "don't", 'not', 'to', 'is', 'with', 'until',
+              'does', 'yourselves', 'under', 'below', 'i', 'those', "wouldn't",
+              'once', "couldn't", 'just', 's', 'shan', "wasn't", 'him', 'so',
+              'can', 'doing', 'o', 'her', 'were', 'now', 'very', 'weren',
+              'its', 'against', 'do', 've', 'be', 'mightn', 'but', "shan't",
+              'nor', 'more'}
+
+chinese_stopwords = {'的', '了', '在', '是', '我', '有', '和', '就', '不',
+                     '人', '都', '一', '一个', '上', '也', '很', '到', '说',
+                     '要', '去', '你', '会', '着', '没有', '看', '好', '自己',
+                     '请问', '您', '他', '于', '及', '即', '为', '最', '从', '以',
+                     '了', '将', '与', '吗', '吧', '中', '#', '什么', '怎么', '哪个',
+                     '哪些', '啥', '相关'}
+
+
+class JiebaKeywordExtractor(DocProcessor):
+    top_k: int = 3
+
+    def _process_docs(self, origin_docs: List[Document], query: Query = None) \
+            -> List[Document]:
+        for _doc in origin_docs:
+            words = jieba.lcut(_doc.text)
+            filtered_words = [word for word in words if word not in
+                              chinese_stopwords and word.lower() not in stop_words]
+            keywords = jieba.analyse.extract_tags(" ".join(filtered_words),
+                                                  topK=self.top_k)
+            _doc.keywords.update(keywords)
+
+        return origin_docs
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) -> 'DocProcessor':
+        super()._initialize_by_component_configer(doc_processor_configer)
+        if hasattr(doc_processor_configer, "top_k"):
+            self.top_k = doc_processor_configer.top_k
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/jieba_keyword_extractor.yaml

@@ -0,0 +1,7 @@
+name: 'jieba_keyword_extractor'
+description: 'extract keywords from text'
+top_k: 3
+metadata:
+  type: 'DOC_PROCESSOR'
+  module: 'agentuniverse.agent.action.knowledge.doc_processor.jieba_keyword_extractor'
+  class: 'JiebaKeywordExtractor'

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/recursive_character_text_splitter.py

@@ -0,0 +1,58 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+from typing import List
+
+# @Time    : 2024/7/31 16:19
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: recursive_character_text_splitter.py
+from typing import List, Optional
+from langchain.text_splitter import RecursiveCharacterTextSplitter as Splitter
+
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import \
+    DocProcessor
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+
+class RecursiveCharacterTextSplitter(DocProcessor):
+    chunk_size: int = 200
+    chunk_overlap: int = 20
+    separators: List[str] = ["\n\n", "\n", " ", ""]
+    splitter: Optional[Splitter] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.splitter = Splitter(separators=self.separators,
+                                 chunk_size=self.chunk_size,
+                                 chunk_overlap=self.chunk_overlap)
+
+    def _process_docs(self, origin_docs: List[Document], query: Query = None) -> \
+            List[Document]:
+        lc_doc_list = self.splitter.split_documents(Document.as_langchain_list(
+            origin_docs
+        ))
+        return Document.from_langchain_list(lc_doc_list)
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) -> 'DocProcessor':
+        super()._initialize_by_component_configer(doc_processor_configer)
+        if hasattr(doc_processor_configer, "chunk_size"):
+            self.chunk_size = doc_processor_configer.chunk_size
+        if hasattr(doc_processor_configer, "chunk_overlap"):
+            self.chunk_overlap = doc_processor_configer.chunk_overlap
+        if hasattr(doc_processor_configer, "separators"):
+            self.separators = doc_processor_configer.separators
+        self.splitter = Splitter(separators=self.separators,
+                                 chunk_size=self.chunk_size,
+                                 chunk_overlap=self.chunk_overlap)
+        return self
+
+
+
+
+
+
+

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/recursive_character_text_splitter.yaml

@@ -0,0 +1,11 @@
+name: 'recursive_character_text_splitter'
+description: 'langchain recursive character text splitter'
+chunk_size: 200
+chunk_overlap: 20
+separators:
+  - "\n\n"
+  - "\n"
+metadata:
+  type: 'DOC_PROCESSOR'
+  module: 'agentuniverse.agent.action.knowledge.doc_processor.recursive_character_text_splitter'
+  class: 'RecursiveCharacterTextSplitter'

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/token_text_splitter.py

@@ -0,0 +1,59 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/8/5 15:37
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: token_text_splitter.py
+from typing import List, Optional
+from langchain.text_splitter import TokenTextSplitter as Splitter
+
+from agentuniverse.agent.action.knowledge.doc_processor.doc_processor import \
+    DocProcessor
+from agentuniverse.agent.action.knowledge.store.document import Document
+from agentuniverse.agent.action.knowledge.store.query import Query
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+
+class TokenTextSplitter(DocProcessor):
+    chunk_size: int = 200
+    chunk_overlap: int = 20
+    encoding_name: str = 'gpt2'
+    model_name: Optional[str] = None
+    splitter: Optional[Splitter] = None
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.splitter = Splitter(
+            encoding_name=self.encoding_name,
+            model_name=self.model_name,
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap
+        )
+
+    def _process_docs(self, origin_docs: List[Document], query: Query = None) -> \
+            List[Document]:
+        lc_doc_list = self.splitter.split_documents(Document.as_langchain_list(
+            origin_docs
+        ))
+        return Document.from_langchain_list(lc_doc_list)
+
+    def _initialize_by_component_configer(self,
+                                         doc_processor_configer: ComponentConfiger) -> 'DocProcessor':
+        super()._initialize_by_component_configer(doc_processor_configer)
+        if hasattr(doc_processor_configer, "chunk_size"):
+            self.chunk_size = doc_processor_configer.chunk_size
+        if hasattr(doc_processor_configer, "chunk_overlap"):
+            self.chunk_overlap = doc_processor_configer.chunk_overlap
+        if hasattr(doc_processor_configer, "encoding_name"):
+            self.encoding_name = doc_processor_configer.encoding_name
+        if hasattr(doc_processor_configer, "model_name"):
+            self.model_name = doc_processor_configer.model_name
+        self.splitter = Splitter(
+            encoding_name=self.encoding_name,
+            model_name=self.model_name,
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap
+        )
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/doc_processor/token_text_splitter.yaml

@@ -0,0 +1,8 @@
+name: 'token_text_splitter'
+description: 'langchain token text splitter'
+chunk_size: 200
+chunk_overlap: 20
+metadata:
+  type: 'DOC_PROCESSOR'
+  module: 'agentuniverse.agent.action.knowledge.doc_processor.token_text_splitter'
+  class: 'TokenTextSplitter'

{agentuniverse-0.0.12.dev2 → agentuniverse-0.0.13.dev0}/agentuniverse/agent/action/knowledge/embedding/dashscope_embedding.py

@@ -32,8 +32,7 @@ class DashscopeEmbedding(Embedding):
         default_factory=lambda: get_from_env("DASHSCOPE_API_KEY")
     )
 
-
-    def get_embeddings(self, texts: List[str]) -> List[List[float]]:
+    def get_embeddings(self, texts: List[str], **kwargs) -> List[List[float]]:
         """
         Retrieve text embeddings for a list of input texts.
 
@@ -61,7 +60,7 @@ class DashscopeEmbedding(Embedding):
                 },
                 data=json.dumps(post_params, ensure_ascii=False).encode(
                     "utf-8"),
-                timeout=120
+                timeout=300
             )
             resp_json = response.json()
             return resp_json
@@ -72,9 +71,14 @@ class DashscopeEmbedding(Embedding):
             "model": self.embedding_model_name,
             "input": {},
             "parameters": {
-                "text_type": "query"
             }
         }
+        if self.embedding_model_name == "text-embedding-v3":
+            post_params["parameters"][
+                "dimension"] = self.embedding_dims if self.embedding_dims else 1024
+            post_params["parameters"][
+                "output_type"] = kwargs["output_type"] if "output_type" in kwargs else "dense"
+        post_params["parameters"]["text_type"] = kwargs["text_type"] if "text_type" in kwargs else "document"
 
         for batch in batched(texts):
             post_params["input"]["texts"] = batch
@@ -92,7 +96,7 @@ class DashscopeEmbedding(Embedding):
                                 f"error message:{error_message}")
         return result
 
-    async def async_get_embeddings(self, texts: List[str]) -> List[List[float]]:
+    async def async_get_embeddings(self, texts: List[str], **kwargs) -> List[List[float]]:
         """
         Async version of get_embeddings.
 
@@ -121,7 +125,7 @@ class DashscopeEmbedding(Embedding):
                         },
                         data=json.dumps(post_params, ensure_ascii=False).encode(
                             "utf-8"),
-                        timeout=120,
+                        timeout=300,
                 ) as resp:
                     resp_json = await resp.json()
             return resp_json
@@ -135,6 +139,14 @@ class DashscopeEmbedding(Embedding):
                 "text_type": "query"
             }
         }
+        if self.embedding_model_name == "text-embedding-v3":
+            post_params["parameters"][
+                "dimension"] = self.embedding_dims if self.embedding_dims else 1024
+            post_params["parameters"][
+                "output_type"] = kwargs[
+                "output_type"] if "output_type" in kwargs else "dense"
+        post_params["parameters"]["text_type"] = kwargs[
+            "text_type"] if "text_type" in kwargs else "document"
 
         for batch in batched(texts):
             post_params["input"]["texts"] = batch

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/embedding/dashscope_embedding.yaml

@@ -0,0 +1,7 @@
+name: 'dashscope_embedding'
+description: 'embedding use dashscope api'
+embedding_model_name: 'text-embedding-v2'
+metadata:
+  type: 'EMBEDDING'
+  module: 'agentuniverse.agent.action.knowledge.embedding.dashscope_embedding'
+  class: 'DashscopeEmbedding'

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/embedding/embedding.py

@@ -0,0 +1,64 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/3/18 19:18
+# @Author  : wangchongshi
+# @Email   : wangchongshi.wcs@antgroup.com
+# @FileName: embedding.py
+from abc import abstractmethod
+from typing import List, Optional
+
+from langchain_core.embeddings import Embeddings as LCEmbeddings
+
+from agentuniverse.base.component.component_base import ComponentEnum
+from agentuniverse.base.component.component_base import ComponentBase
+from agentuniverse.base.config.component_configer.component_configer import \
+    ComponentConfiger
+
+
+class Embedding(ComponentBase):
+    """The basic class for embedding.
+
+    Attributes:
+        embedding_model_name (Optional[str]): The name of the embedding model.
+    """
+
+    component_type: ComponentEnum = ComponentEnum.EMBEDDING
+    name: Optional[str] = None
+    description: Optional[str] = None
+    embedding_model_name: Optional[str] = None
+    embedding_dims: Optional[int] = None
+
+    @abstractmethod
+    def get_embeddings(self, text: List[str], **kwargs) -> List[List[float]]:
+        """Get embeddings."""
+
+    @abstractmethod
+    async def async_get_embeddings(self, texts: List[str], **kwargs) -> List[
+        List[float]]:
+        """Asynchronously get embeddings."""
+
+    def as_langchain(self) -> LCEmbeddings:
+        """Convert the agentUniverse(aU) embedding class to the langchain embedding class."""
+        pass
+
+    def _initialize_by_component_configer(self,
+                                          embedding_configer: ComponentConfiger) \
+            -> 'Embedding':
+        """Initialize the embedding by the ComponentConfiger object.
+
+        Args:
+            embedding_configer(ComponentConfiger): A configer contains embedding
+            basic info.
+        Returns:
+            Embedding: A embedding instance.
+        """
+        if embedding_configer.name:
+            self.name = embedding_configer.name
+        if embedding_configer.description:
+            self.description = embedding_configer.description
+        if hasattr(embedding_configer, "embedding_dims"):
+            self.embedding_dims = embedding_configer.embedding_dims
+        if hasattr(embedding_configer, "embedding_model_name"):
+            self.embedding_model_name = embedding_configer.embedding_model_name
+        return self

agentuniverse-0.0.13.dev0/agentuniverse/agent/action/knowledge/embedding/embedding_manager.py

@@ -0,0 +1,21 @@
+# !/usr/bin/env python3
+# -*- coding:utf-8 -*-
+
+# @Time    : 2024/7/23 11:28
+# @Author  : fanen.lhy
+# @Email   : fanen.lhy@antgroup.com
+# @FileName: embedding_manager.py
+
+from agentuniverse.base.annotation.singleton import singleton
+from agentuniverse.base.component.component_enum import ComponentEnum
+from agentuniverse.base.component.component_manager_base import ComponentManagerBase
+from agentuniverse.agent.action.knowledge.embedding.embedding import Embedding
+
+
+@singleton
+class EmbeddingManager(ComponentManagerBase[Embedding]):
+    """A singleton manager class of the embedding."""
+
+    def __init__(self):
+        super().__init__(ComponentEnum.EMBEDDING)
+