powermem 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- powermem/__init__.py +103 -0
- powermem/agent/__init__.py +35 -0
- powermem/agent/abstract/__init__.py +22 -0
- powermem/agent/abstract/collaboration.py +259 -0
- powermem/agent/abstract/context.py +187 -0
- powermem/agent/abstract/manager.py +232 -0
- powermem/agent/abstract/permission.py +217 -0
- powermem/agent/abstract/privacy.py +267 -0
- powermem/agent/abstract/scope.py +199 -0
- powermem/agent/agent.py +791 -0
- powermem/agent/components/__init__.py +18 -0
- powermem/agent/components/collaboration_coordinator.py +645 -0
- powermem/agent/components/permission_controller.py +586 -0
- powermem/agent/components/privacy_protector.py +767 -0
- powermem/agent/components/scope_controller.py +685 -0
- powermem/agent/factories/__init__.py +16 -0
- powermem/agent/factories/agent_factory.py +266 -0
- powermem/agent/factories/config_factory.py +308 -0
- powermem/agent/factories/memory_factory.py +229 -0
- powermem/agent/implementations/__init__.py +16 -0
- powermem/agent/implementations/hybrid.py +728 -0
- powermem/agent/implementations/multi_agent.py +1040 -0
- powermem/agent/implementations/multi_user.py +1020 -0
- powermem/agent/types.py +53 -0
- powermem/agent/wrappers/__init__.py +14 -0
- powermem/agent/wrappers/agent_memory_wrapper.py +427 -0
- powermem/agent/wrappers/compatibility_wrapper.py +520 -0
- powermem/config_loader.py +318 -0
- powermem/configs.py +249 -0
- powermem/core/__init__.py +19 -0
- powermem/core/async_memory.py +1493 -0
- powermem/core/audit.py +258 -0
- powermem/core/base.py +165 -0
- powermem/core/memory.py +1567 -0
- powermem/core/setup.py +162 -0
- powermem/core/telemetry.py +215 -0
- powermem/integrations/__init__.py +17 -0
- powermem/integrations/embeddings/__init__.py +13 -0
- powermem/integrations/embeddings/aws_bedrock.py +100 -0
- powermem/integrations/embeddings/azure_openai.py +55 -0
- powermem/integrations/embeddings/base.py +31 -0
- powermem/integrations/embeddings/config/base.py +132 -0
- powermem/integrations/embeddings/configs.py +31 -0
- powermem/integrations/embeddings/factory.py +48 -0
- powermem/integrations/embeddings/gemini.py +39 -0
- powermem/integrations/embeddings/huggingface.py +41 -0
- powermem/integrations/embeddings/langchain.py +35 -0
- powermem/integrations/embeddings/lmstudio.py +29 -0
- powermem/integrations/embeddings/mock.py +11 -0
- powermem/integrations/embeddings/ollama.py +53 -0
- powermem/integrations/embeddings/openai.py +49 -0
- powermem/integrations/embeddings/qwen.py +102 -0
- powermem/integrations/embeddings/together.py +31 -0
- powermem/integrations/embeddings/vertexai.py +54 -0
- powermem/integrations/llm/__init__.py +18 -0
- powermem/integrations/llm/anthropic.py +87 -0
- powermem/integrations/llm/base.py +132 -0
- powermem/integrations/llm/config/anthropic.py +56 -0
- powermem/integrations/llm/config/azure.py +56 -0
- powermem/integrations/llm/config/base.py +62 -0
- powermem/integrations/llm/config/deepseek.py +56 -0
- powermem/integrations/llm/config/ollama.py +56 -0
- powermem/integrations/llm/config/openai.py +79 -0
- powermem/integrations/llm/config/qwen.py +68 -0
- powermem/integrations/llm/config/qwen_asr.py +46 -0
- powermem/integrations/llm/config/vllm.py +56 -0
- powermem/integrations/llm/configs.py +26 -0
- powermem/integrations/llm/deepseek.py +106 -0
- powermem/integrations/llm/factory.py +118 -0
- powermem/integrations/llm/gemini.py +201 -0
- powermem/integrations/llm/langchain.py +65 -0
- powermem/integrations/llm/ollama.py +106 -0
- powermem/integrations/llm/openai.py +166 -0
- powermem/integrations/llm/openai_structured.py +80 -0
- powermem/integrations/llm/qwen.py +207 -0
- powermem/integrations/llm/qwen_asr.py +171 -0
- powermem/integrations/llm/vllm.py +106 -0
- powermem/integrations/rerank/__init__.py +20 -0
- powermem/integrations/rerank/base.py +43 -0
- powermem/integrations/rerank/config/__init__.py +7 -0
- powermem/integrations/rerank/config/base.py +27 -0
- powermem/integrations/rerank/configs.py +23 -0
- powermem/integrations/rerank/factory.py +68 -0
- powermem/integrations/rerank/qwen.py +159 -0
- powermem/intelligence/__init__.py +17 -0
- powermem/intelligence/ebbinghaus_algorithm.py +354 -0
- powermem/intelligence/importance_evaluator.py +361 -0
- powermem/intelligence/intelligent_memory_manager.py +284 -0
- powermem/intelligence/manager.py +148 -0
- powermem/intelligence/plugin.py +229 -0
- powermem/prompts/__init__.py +29 -0
- powermem/prompts/graph/graph_prompts.py +217 -0
- powermem/prompts/graph/graph_tools_prompts.py +469 -0
- powermem/prompts/importance_evaluation.py +246 -0
- powermem/prompts/intelligent_memory_prompts.py +163 -0
- powermem/prompts/templates.py +193 -0
- powermem/storage/__init__.py +14 -0
- powermem/storage/adapter.py +896 -0
- powermem/storage/base.py +109 -0
- powermem/storage/config/base.py +13 -0
- powermem/storage/config/oceanbase.py +58 -0
- powermem/storage/config/pgvector.py +52 -0
- powermem/storage/config/sqlite.py +27 -0
- powermem/storage/configs.py +159 -0
- powermem/storage/factory.py +59 -0
- powermem/storage/migration_manager.py +438 -0
- powermem/storage/oceanbase/__init__.py +8 -0
- powermem/storage/oceanbase/constants.py +162 -0
- powermem/storage/oceanbase/oceanbase.py +1384 -0
- powermem/storage/oceanbase/oceanbase_graph.py +1441 -0
- powermem/storage/pgvector/__init__.py +7 -0
- powermem/storage/pgvector/pgvector.py +420 -0
- powermem/storage/sqlite/__init__.py +0 -0
- powermem/storage/sqlite/sqlite.py +218 -0
- powermem/storage/sqlite/sqlite_vector_store.py +311 -0
- powermem/utils/__init__.py +35 -0
- powermem/utils/utils.py +605 -0
- powermem/version.py +23 -0
- powermem-0.1.0.dist-info/METADATA +187 -0
- powermem-0.1.0.dist-info/RECORD +123 -0
- powermem-0.1.0.dist-info/WHEEL +5 -0
- powermem-0.1.0.dist-info/licenses/LICENSE +206 -0
- powermem-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from abc import ABC
|
|
3
|
+
from typing import Dict, Optional, Union
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from pydantic import BaseModel, Field
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AzureConfig(BaseModel):
|
|
10
|
+
"""
|
|
11
|
+
Configuration settings for Azure.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
api_key (str): The API key used for authenticating with the Azure service.
|
|
15
|
+
azure_deployment (str): The name of the Azure deployment.
|
|
16
|
+
azure_endpoint (str): The endpoint URL for the Azure service.
|
|
17
|
+
api_version (str): The version of the Azure API being used.
|
|
18
|
+
default_headers (Dict[str, str]): Headers to include in requests to the Azure API.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
api_key: str = Field(
|
|
22
|
+
description="The API key used for authenticating with the Azure service.",
|
|
23
|
+
default=None,
|
|
24
|
+
)
|
|
25
|
+
azure_deployment: str = Field(description="The name of the Azure deployment.", default=None)
|
|
26
|
+
azure_endpoint: str = Field(description="The endpoint URL for the Azure service.", default=None)
|
|
27
|
+
api_version: str = Field(description="The version of the Azure API being used.", default=None)
|
|
28
|
+
default_headers: Optional[Dict[str, str]] = Field(
|
|
29
|
+
description="Headers to include in requests to the Azure API.", default=None
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
class BaseEmbedderConfig(ABC):
|
|
33
|
+
"""
|
|
34
|
+
Config for Embeddings.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
model: Optional[str] = None,
|
|
40
|
+
api_key: Optional[str] = None,
|
|
41
|
+
embedding_dims: Optional[int] = None,
|
|
42
|
+
# Ollama specific
|
|
43
|
+
ollama_base_url: Optional[str] = None,
|
|
44
|
+
# Openai specific
|
|
45
|
+
openai_base_url: Optional[str] = None,
|
|
46
|
+
# Huggingface specific
|
|
47
|
+
model_kwargs: Optional[dict] = None,
|
|
48
|
+
huggingface_base_url: Optional[str] = None,
|
|
49
|
+
# AzureOpenAI specific
|
|
50
|
+
azure_kwargs: Optional[AzureConfig] = {},
|
|
51
|
+
http_client_proxies: Optional[Union[Dict, str]] = None,
|
|
52
|
+
# VertexAI specific
|
|
53
|
+
vertex_credentials_json: Optional[str] = None,
|
|
54
|
+
memory_add_embedding_type: Optional[str] = None,
|
|
55
|
+
memory_update_embedding_type: Optional[str] = None,
|
|
56
|
+
memory_search_embedding_type: Optional[str] = None,
|
|
57
|
+
# Gemini specific
|
|
58
|
+
output_dimensionality: Optional[str] = None,
|
|
59
|
+
# LM Studio specific
|
|
60
|
+
lmstudio_base_url: Optional[str] = "http://localhost:1234/v1",
|
|
61
|
+
# AWS Bedrock specific
|
|
62
|
+
aws_access_key_id: Optional[str] = None,
|
|
63
|
+
aws_secret_access_key: Optional[str] = None,
|
|
64
|
+
aws_region: Optional[str] = None,
|
|
65
|
+
):
|
|
66
|
+
"""
|
|
67
|
+
Initializes a configuration class instance for the Embeddings.
|
|
68
|
+
|
|
69
|
+
:param model: Embedding model to use, defaults to None
|
|
70
|
+
:type model: Optional[str], optional
|
|
71
|
+
:param api_key: API key to be use, defaults to None
|
|
72
|
+
:type api_key: Optional[str], optional
|
|
73
|
+
:param embedding_dims: The number of dimensions in the embedding, defaults to None
|
|
74
|
+
:type embedding_dims: Optional[int], optional
|
|
75
|
+
:param ollama_base_url: Base URL for the Ollama API, defaults to None
|
|
76
|
+
:type ollama_base_url: Optional[str], optional
|
|
77
|
+
:param model_kwargs: key-value arguments for the huggingface embedding model, defaults a dict inside init
|
|
78
|
+
:type model_kwargs: Optional[Dict[str, Any]], defaults a dict inside init
|
|
79
|
+
:param huggingface_base_url: Huggingface base URL to be use, defaults to None
|
|
80
|
+
:type huggingface_base_url: Optional[str], optional
|
|
81
|
+
:param openai_base_url: Openai base URL to be use, defaults to "https://api.openai.com/v1"
|
|
82
|
+
:type openai_base_url: Optional[str], optional
|
|
83
|
+
:param azure_kwargs: key-value arguments for the AzureOpenAI embedding model, defaults a dict inside init
|
|
84
|
+
:type azure_kwargs: Optional[Dict[str, Any]], defaults a dict inside init
|
|
85
|
+
:param http_client_proxies: The proxy server settings used to create self.http_client, defaults to None
|
|
86
|
+
:type http_client_proxies: Optional[Dict | str], optional
|
|
87
|
+
:param vertex_credentials_json: The path to the Vertex AI credentials JSON file, defaults to None
|
|
88
|
+
:type vertex_credentials_json: Optional[str], optional
|
|
89
|
+
:param memory_add_embedding_type: The type of embedding to use for the add memory action, defaults to None
|
|
90
|
+
:type memory_add_embedding_type: Optional[str], optional
|
|
91
|
+
:param memory_update_embedding_type: The type of embedding to use for the update memory action, defaults to None
|
|
92
|
+
:type memory_update_embedding_type: Optional[str], optional
|
|
93
|
+
:param memory_search_embedding_type: The type of embedding to use for the search memory action, defaults to None
|
|
94
|
+
:type memory_search_embedding_type: Optional[str], optional
|
|
95
|
+
:param lmstudio_base_url: LM Studio base URL to be use, defaults to "http://localhost:1234/v1"
|
|
96
|
+
:type lmstudio_base_url: Optional[str], optional
|
|
97
|
+
"""
|
|
98
|
+
|
|
99
|
+
self.model = model
|
|
100
|
+
self.api_key = api_key
|
|
101
|
+
self.openai_base_url = openai_base_url
|
|
102
|
+
self.embedding_dims = embedding_dims
|
|
103
|
+
|
|
104
|
+
# AzureOpenAI specific
|
|
105
|
+
self.http_client = httpx.Client(proxies=http_client_proxies) if http_client_proxies else None
|
|
106
|
+
|
|
107
|
+
# Ollama specific
|
|
108
|
+
self.ollama_base_url = ollama_base_url
|
|
109
|
+
|
|
110
|
+
# Huggingface specific
|
|
111
|
+
self.model_kwargs = model_kwargs or {}
|
|
112
|
+
self.huggingface_base_url = huggingface_base_url
|
|
113
|
+
# AzureOpenAI specific
|
|
114
|
+
self.azure_kwargs = AzureConfig(**azure_kwargs) or {}
|
|
115
|
+
|
|
116
|
+
# VertexAI specific
|
|
117
|
+
self.vertex_credentials_json = vertex_credentials_json
|
|
118
|
+
self.memory_add_embedding_type = memory_add_embedding_type
|
|
119
|
+
self.memory_update_embedding_type = memory_update_embedding_type
|
|
120
|
+
self.memory_search_embedding_type = memory_search_embedding_type
|
|
121
|
+
|
|
122
|
+
# Gemini specific
|
|
123
|
+
self.output_dimensionality = output_dimensionality
|
|
124
|
+
|
|
125
|
+
# LM Studio specific
|
|
126
|
+
self.lmstudio_base_url = lmstudio_base_url
|
|
127
|
+
|
|
128
|
+
# AWS Bedrock specific
|
|
129
|
+
self.aws_access_key_id = aws_access_key_id
|
|
130
|
+
self.aws_secret_access_key = aws_secret_access_key
|
|
131
|
+
self.aws_region = aws_region or os.environ.get("AWS_REGION") or "us-west-2"
|
|
132
|
+
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, Field, field_validator
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class EmbedderConfig(BaseModel):
|
|
7
|
+
provider: str = Field(
|
|
8
|
+
description="Provider of the embedding model (e.g., 'ollama', 'openai')",
|
|
9
|
+
default="openai",
|
|
10
|
+
)
|
|
11
|
+
config: Optional[dict] = Field(description="Configuration for the specific embedding model", default={})
|
|
12
|
+
|
|
13
|
+
@field_validator("config")
|
|
14
|
+
def validate_config(cls, v, values):
|
|
15
|
+
provider = values.data.get("provider")
|
|
16
|
+
if provider in [
|
|
17
|
+
"openai",
|
|
18
|
+
"ollama",
|
|
19
|
+
"huggingface",
|
|
20
|
+
"azure_openai",
|
|
21
|
+
"gemini",
|
|
22
|
+
"vertexai",
|
|
23
|
+
"together",
|
|
24
|
+
"lmstudio",
|
|
25
|
+
"langchain",
|
|
26
|
+
"aws_bedrock",
|
|
27
|
+
"qwen",
|
|
28
|
+
]:
|
|
29
|
+
return v
|
|
30
|
+
else:
|
|
31
|
+
raise ValueError(f"Unsupported embedding provider: {provider}")
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Embedding factory for creating embedding instances
|
|
3
|
+
|
|
4
|
+
This module provides a factory for creating different embedding instances.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import importlib
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
11
|
+
from powermem.integrations.embeddings.mock import MockEmbeddings
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_class(class_type):
|
|
15
|
+
module_path, class_name = class_type.rsplit(".", 1)
|
|
16
|
+
module = importlib.import_module(module_path)
|
|
17
|
+
return getattr(module, class_name)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class EmbedderFactory:
|
|
21
|
+
provider_to_class = {
|
|
22
|
+
"openai": "powermem.integrations.embeddings.openai.OpenAIEmbedding",
|
|
23
|
+
"ollama": "powermem.integrations.embeddings.ollama.OllamaEmbedding",
|
|
24
|
+
"huggingface": "powermem.integrations.embeddings.huggingface.HuggingFaceEmbedding",
|
|
25
|
+
"azure_openai": "powermem.integrations.embeddings.azure_openai.AzureOpenAIEmbedding",
|
|
26
|
+
"gemini": "powermem.integrations.embeddings.gemini.GoogleGenAIEmbedding",
|
|
27
|
+
"vertexai": "powermem.integrations.embeddings.vertexai.VertexAIEmbedding",
|
|
28
|
+
"together": "powermem.integrations.embeddings.together.TogetherEmbedding",
|
|
29
|
+
"lmstudio": "powermem.integrations.embeddings.lmstudio.LMStudioEmbedding",
|
|
30
|
+
"langchain": "powermem.integrations.embeddings.langchain.LangchainEmbedding",
|
|
31
|
+
"aws_bedrock": "powermem.integrations.embeddings.aws_bedrock.AWSBedrockEmbedding",
|
|
32
|
+
"qwen": "powermem.integrations.embeddings.qwen.QwenEmbedding",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def create(cls, provider_name, config, vector_config: Optional[dict]):
|
|
37
|
+
# Handle mock provider directly
|
|
38
|
+
if provider_name == "mock":
|
|
39
|
+
return MockEmbeddings()
|
|
40
|
+
if provider_name == "upstash_vector" and vector_config and vector_config.enable_embeddings:
|
|
41
|
+
return MockEmbeddings()
|
|
42
|
+
class_type = cls.provider_to_class.get(provider_name)
|
|
43
|
+
if class_type:
|
|
44
|
+
embedder_instance = load_class(class_type)
|
|
45
|
+
base_config = BaseEmbedderConfig(**config)
|
|
46
|
+
return embedder_instance(base_config)
|
|
47
|
+
else:
|
|
48
|
+
raise ValueError(f"Unsupported Embedder provider: {provider_name}")
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Literal, Optional
|
|
3
|
+
|
|
4
|
+
from google import genai
|
|
5
|
+
from google.genai import types
|
|
6
|
+
|
|
7
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
8
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class GoogleGenAIEmbedding(EmbeddingBase):
|
|
12
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
13
|
+
super().__init__(config)
|
|
14
|
+
|
|
15
|
+
self.config.model = self.config.model or "models/text-embedding-004"
|
|
16
|
+
self.config.embedding_dims = self.config.embedding_dims or self.config.output_dimensionality or 768
|
|
17
|
+
|
|
18
|
+
api_key = self.config.api_key or os.getenv("GOOGLE_API_KEY")
|
|
19
|
+
|
|
20
|
+
self.client = genai.Client(api_key=api_key)
|
|
21
|
+
|
|
22
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
23
|
+
"""
|
|
24
|
+
Get the embedding for the given text using Google Generative AI.
|
|
25
|
+
Args:
|
|
26
|
+
text (str): The text to embed.
|
|
27
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
28
|
+
Returns:
|
|
29
|
+
list: The embedding vector.
|
|
30
|
+
"""
|
|
31
|
+
text = text.replace("\n", " ")
|
|
32
|
+
|
|
33
|
+
# Create config for embedding parameters
|
|
34
|
+
config = types.EmbedContentConfig(output_dimensionality=self.config.embedding_dims)
|
|
35
|
+
|
|
36
|
+
# Call the embed_content method with the correct parameters
|
|
37
|
+
response = self.client.models.embed_content(model=self.config.model, contents=text, config=config)
|
|
38
|
+
|
|
39
|
+
return response.embeddings[0].values
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Literal, Optional
|
|
3
|
+
|
|
4
|
+
from openai import OpenAI
|
|
5
|
+
from sentence_transformers import SentenceTransformer
|
|
6
|
+
|
|
7
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
8
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
9
|
+
|
|
10
|
+
logging.getLogger("transformers").setLevel(logging.WARNING)
|
|
11
|
+
logging.getLogger("sentence_transformers").setLevel(logging.WARNING)
|
|
12
|
+
logging.getLogger("huggingface_hub").setLevel(logging.WARNING)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class HuggingFaceEmbedding(EmbeddingBase):
|
|
16
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
17
|
+
super().__init__(config)
|
|
18
|
+
|
|
19
|
+
if config.huggingface_base_url:
|
|
20
|
+
self.client = OpenAI(base_url=config.huggingface_base_url)
|
|
21
|
+
else:
|
|
22
|
+
self.config.model = self.config.model or "multi-qa-MiniLM-L6-cos-v1"
|
|
23
|
+
|
|
24
|
+
self.model = SentenceTransformer(self.config.model, **self.config.model_kwargs)
|
|
25
|
+
|
|
26
|
+
self.config.embedding_dims = self.config.embedding_dims or self.model.get_sentence_embedding_dimension()
|
|
27
|
+
|
|
28
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
29
|
+
"""
|
|
30
|
+
Get the embedding for the given text using Hugging Face.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
text (str): The text to embed.
|
|
34
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
35
|
+
Returns:
|
|
36
|
+
list: The embedding vector.
|
|
37
|
+
"""
|
|
38
|
+
if self.config.huggingface_base_url:
|
|
39
|
+
return self.client.embeddings.create(input=text, model="tei").data[0].embedding
|
|
40
|
+
else:
|
|
41
|
+
return self.model.encode(text, convert_to_numpy=True).tolist()
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
from typing import Literal, Optional
|
|
2
|
+
|
|
3
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
4
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
from langchain.embeddings.base import Embeddings
|
|
8
|
+
except ImportError:
|
|
9
|
+
raise ImportError("langchain is not installed. Please install it using `pip install langchain`")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LangchainEmbedding(EmbeddingBase):
|
|
13
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
14
|
+
super().__init__(config)
|
|
15
|
+
|
|
16
|
+
if self.config.model is None:
|
|
17
|
+
raise ValueError("`model` parameter is required")
|
|
18
|
+
|
|
19
|
+
if not isinstance(self.config.model, Embeddings):
|
|
20
|
+
raise ValueError("`model` must be an instance of Embeddings")
|
|
21
|
+
|
|
22
|
+
self.langchain_model = self.config.model
|
|
23
|
+
|
|
24
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
25
|
+
"""
|
|
26
|
+
Get the embedding for the given text using Langchain.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
text (str): The text to embed.
|
|
30
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
31
|
+
Returns:
|
|
32
|
+
list: The embedding vector.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
return self.langchain_model.embed_query(text)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from typing import Literal, Optional
|
|
2
|
+
|
|
3
|
+
from openai import OpenAI
|
|
4
|
+
|
|
5
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
6
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class LMStudioEmbedding(EmbeddingBase):
|
|
10
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
11
|
+
super().__init__(config)
|
|
12
|
+
|
|
13
|
+
self.config.model = self.config.model or "nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.f16.gguf"
|
|
14
|
+
self.config.embedding_dims = self.config.embedding_dims or 1536
|
|
15
|
+
self.config.api_key = self.config.api_key or "lm-studio"
|
|
16
|
+
|
|
17
|
+
self.client = OpenAI(base_url=self.config.lmstudio_base_url, api_key=self.config.api_key)
|
|
18
|
+
|
|
19
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
20
|
+
"""
|
|
21
|
+
Get the embedding for the given text using LM Studio.
|
|
22
|
+
Args:
|
|
23
|
+
text (str): The text to embed.
|
|
24
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
25
|
+
Returns:
|
|
26
|
+
list: The embedding vector.
|
|
27
|
+
"""
|
|
28
|
+
text = text.replace("\n", " ")
|
|
29
|
+
return self.client.embeddings.create(input=[text], model=self.config.model).data[0].embedding
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from typing import Literal, Optional
|
|
2
|
+
|
|
3
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class MockEmbeddings(EmbeddingBase):
|
|
7
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
8
|
+
"""
|
|
9
|
+
Generate a mock embedding with dimension of 10.
|
|
10
|
+
"""
|
|
11
|
+
return [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import sys
|
|
3
|
+
from typing import Literal, Optional
|
|
4
|
+
|
|
5
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
6
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
from ollama import Client
|
|
10
|
+
except ImportError:
|
|
11
|
+
user_input = input("The 'ollama' library is required. Install it now? [y/N]: ")
|
|
12
|
+
if user_input.lower() == "y":
|
|
13
|
+
try:
|
|
14
|
+
subprocess.check_call([sys.executable, "-m", "pip", "install", "ollama"])
|
|
15
|
+
from ollama import Client
|
|
16
|
+
except subprocess.CalledProcessError:
|
|
17
|
+
print("Failed to install 'ollama'. Please install it manually using 'pip install ollama'.")
|
|
18
|
+
sys.exit(1)
|
|
19
|
+
else:
|
|
20
|
+
print("The required 'ollama' library is not installed.")
|
|
21
|
+
sys.exit(1)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class OllamaEmbedding(EmbeddingBase):
|
|
25
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
26
|
+
super().__init__(config)
|
|
27
|
+
|
|
28
|
+
self.config.model = self.config.model or "nomic-embed-text"
|
|
29
|
+
self.config.embedding_dims = self.config.embedding_dims or 512
|
|
30
|
+
|
|
31
|
+
self.client = Client(host=self.config.ollama_base_url)
|
|
32
|
+
self._ensure_model_exists()
|
|
33
|
+
|
|
34
|
+
def _ensure_model_exists(self):
|
|
35
|
+
"""
|
|
36
|
+
Ensure the specified model exists locally. If not, pull it from Ollama.
|
|
37
|
+
"""
|
|
38
|
+
local_models = self.client.list()["models"]
|
|
39
|
+
if not any(model.get("name") == self.config.model or model.get("model") == self.config.model for model in local_models):
|
|
40
|
+
self.client.pull(self.config.model)
|
|
41
|
+
|
|
42
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
43
|
+
"""
|
|
44
|
+
Get the embedding for the given text using Ollama.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
text (str): The text to embed.
|
|
48
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
49
|
+
Returns:
|
|
50
|
+
list: The embedding vector.
|
|
51
|
+
"""
|
|
52
|
+
response = self.client.embeddings(model=self.config.model, prompt=text)
|
|
53
|
+
return response["embedding"]
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import warnings
|
|
3
|
+
from typing import Literal, Optional
|
|
4
|
+
|
|
5
|
+
from openai import OpenAI
|
|
6
|
+
|
|
7
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
8
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class OpenAIEmbedding(EmbeddingBase):
|
|
12
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
13
|
+
super().__init__(config)
|
|
14
|
+
|
|
15
|
+
self.config.model = self.config.model or "text-embedding-3-small"
|
|
16
|
+
self.config.embedding_dims = self.config.embedding_dims or 1536
|
|
17
|
+
|
|
18
|
+
api_key = self.config.api_key or os.getenv("OPENAI_API_KEY")
|
|
19
|
+
base_url = (
|
|
20
|
+
self.config.openai_base_url
|
|
21
|
+
or os.getenv("OPENAI_API_BASE")
|
|
22
|
+
or os.getenv("OPENAI_BASE_URL")
|
|
23
|
+
or "https://api.openai.com/v1"
|
|
24
|
+
)
|
|
25
|
+
if os.environ.get("OPENAI_API_BASE"):
|
|
26
|
+
warnings.warn(
|
|
27
|
+
"The environment variable 'OPENAI_API_BASE' is deprecated and will be removed in the 0.1.80. "
|
|
28
|
+
"Please use 'OPENAI_BASE_URL' instead.",
|
|
29
|
+
DeprecationWarning,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
self.client = OpenAI(api_key=api_key, base_url=base_url)
|
|
33
|
+
|
|
34
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
35
|
+
"""
|
|
36
|
+
Get the embedding for the given text using OpenAI.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
text (str): The text to embed.
|
|
40
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
41
|
+
Returns:
|
|
42
|
+
list: The embedding vector.
|
|
43
|
+
"""
|
|
44
|
+
text = text.replace("\n", " ")
|
|
45
|
+
return (
|
|
46
|
+
self.client.embeddings.create(input=[text], model=self.config.model, dimensions=self.config.embedding_dims)
|
|
47
|
+
.data[0]
|
|
48
|
+
.embedding
|
|
49
|
+
)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Literal, Optional
|
|
3
|
+
|
|
4
|
+
try:
|
|
5
|
+
from dashscope import TextEmbedding
|
|
6
|
+
from dashscope.api_entities.dashscope_response import DashScopeAPIResponse
|
|
7
|
+
except ImportError:
|
|
8
|
+
TextEmbedding = None
|
|
9
|
+
DashScopeAPIResponse = None
|
|
10
|
+
import dashscope
|
|
11
|
+
|
|
12
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
13
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class QwenEmbedding(EmbeddingBase):
|
|
17
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
18
|
+
super().__init__(config)
|
|
19
|
+
|
|
20
|
+
# Set default model and dimensions
|
|
21
|
+
self.config.model = self.config.model or "text-embedding-v4"
|
|
22
|
+
self.config.embedding_dims = self.config.embedding_dims or 1536
|
|
23
|
+
|
|
24
|
+
# Check if dashscope is available
|
|
25
|
+
if TextEmbedding is None:
|
|
26
|
+
raise ImportError(
|
|
27
|
+
"DashScope SDK is not installed. Please install it with: pip install dashscope"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Set API key
|
|
31
|
+
api_key = self.config.api_key or os.getenv("DASHSCOPE_API_KEY")
|
|
32
|
+
if not api_key:
|
|
33
|
+
raise ValueError(
|
|
34
|
+
"API key is required. Set DASHSCOPE_API_KEY environment variable or pass api_key in config.")
|
|
35
|
+
|
|
36
|
+
# Set API key for DashScope SDK
|
|
37
|
+
dashscope.api_key = api_key
|
|
38
|
+
|
|
39
|
+
# Set base URL (if needed)
|
|
40
|
+
base_url = os.getenv("DASHSCOPE_BASE_URL") or "https://dashscope.aliyuncs.com/api/v1"
|
|
41
|
+
if base_url:
|
|
42
|
+
os.environ["DASHSCOPE_BASE_URL"] = base_url
|
|
43
|
+
|
|
44
|
+
def embed(self, text: str, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
45
|
+
"""
|
|
46
|
+
Get the embedding for the given text using Qwen.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
text (str): The text to embed.
|
|
50
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
51
|
+
Returns:
|
|
52
|
+
list: The embedding vector.
|
|
53
|
+
"""
|
|
54
|
+
# Clean text
|
|
55
|
+
text = text.replace("\n", " ").strip()
|
|
56
|
+
|
|
57
|
+
# Determine embedding type based on memory action
|
|
58
|
+
# Default values for DashScope text-embedding-v4:
|
|
59
|
+
# - "document" for add/update (RETRIEVAL_DOCUMENT equivalent)
|
|
60
|
+
# - "query" for search (RETRIEVAL_QUERY equivalent)
|
|
61
|
+
if memory_action == "add":
|
|
62
|
+
embedding_type = self.config.memory_add_embedding_type or "document"
|
|
63
|
+
elif memory_action == "search":
|
|
64
|
+
embedding_type = self.config.memory_search_embedding_type or "query"
|
|
65
|
+
elif memory_action == "update":
|
|
66
|
+
embedding_type = self.config.memory_update_embedding_type or "document"
|
|
67
|
+
else:
|
|
68
|
+
# Default to "document" if memory_action is None or unknown
|
|
69
|
+
embedding_type = "document"
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
# Prepare parameters
|
|
73
|
+
params = {
|
|
74
|
+
"model": self.config.model,
|
|
75
|
+
"input": text,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
# Add dimension parameter if specified
|
|
79
|
+
if hasattr(self.config, 'embedding_dims') and self.config.embedding_dims:
|
|
80
|
+
params["dimension"] = self.config.embedding_dims
|
|
81
|
+
|
|
82
|
+
# Add embedding type (always set, either from config or default)
|
|
83
|
+
params["text_type"] = embedding_type
|
|
84
|
+
|
|
85
|
+
# Call the API
|
|
86
|
+
response = TextEmbedding.call(**params)
|
|
87
|
+
|
|
88
|
+
if response.status_code != 200:
|
|
89
|
+
raise Exception(f"API request failed with status {response.status_code}: {response.message}")
|
|
90
|
+
|
|
91
|
+
# Extract embedding vector
|
|
92
|
+
# response.output is a dict, not an object with attributes
|
|
93
|
+
if isinstance(response.output, dict) and 'embeddings' in response.output:
|
|
94
|
+
embedding = response.output['embeddings'][0]['embedding']
|
|
95
|
+
else:
|
|
96
|
+
# Fallback for different response structures
|
|
97
|
+
embedding = response.output.get('embeddings', [{}])[0].get('embedding', [])
|
|
98
|
+
|
|
99
|
+
return embedding
|
|
100
|
+
|
|
101
|
+
except Exception as e:
|
|
102
|
+
raise Exception(f"Failed to generate embedding: {e}")
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Literal, Optional
|
|
3
|
+
|
|
4
|
+
from together import Together
|
|
5
|
+
|
|
6
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
7
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class TogetherEmbedding(EmbeddingBase):
|
|
11
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
12
|
+
super().__init__(config)
|
|
13
|
+
|
|
14
|
+
self.config.model = self.config.model or "togethercomputer/m2-bert-80M-8k-retrieval"
|
|
15
|
+
api_key = self.config.api_key or os.getenv("TOGETHER_API_KEY")
|
|
16
|
+
# TODO: check if this is correct
|
|
17
|
+
self.config.embedding_dims = self.config.embedding_dims or 768
|
|
18
|
+
self.client = Together(api_key=api_key)
|
|
19
|
+
|
|
20
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
21
|
+
"""
|
|
22
|
+
Get the embedding for the given text using OpenAI.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
text (str): The text to embed.
|
|
26
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
27
|
+
Returns:
|
|
28
|
+
list: The embedding vector.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
return self.client.embeddings.create(model=self.config.model, input=text).data[0].embedding
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import Literal, Optional
|
|
3
|
+
|
|
4
|
+
from vertexai.language_models import TextEmbeddingInput, TextEmbeddingModel
|
|
5
|
+
|
|
6
|
+
from powermem.integrations.embeddings.base import EmbeddingBase
|
|
7
|
+
from powermem.integrations.embeddings.config.base import BaseEmbedderConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class VertexAIEmbedding(EmbeddingBase):
|
|
11
|
+
def __init__(self, config: Optional[BaseEmbedderConfig] = None):
|
|
12
|
+
super().__init__(config)
|
|
13
|
+
|
|
14
|
+
self.config.model = self.config.model or "text-embedding-004"
|
|
15
|
+
self.config.embedding_dims = self.config.embedding_dims or 256
|
|
16
|
+
|
|
17
|
+
self.embedding_types = {
|
|
18
|
+
"add": self.config.memory_add_embedding_type or "RETRIEVAL_DOCUMENT",
|
|
19
|
+
"update": self.config.memory_update_embedding_type or "RETRIEVAL_DOCUMENT",
|
|
20
|
+
"search": self.config.memory_search_embedding_type or "RETRIEVAL_QUERY",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
credentials_path = self.config.vertex_credentials_json
|
|
24
|
+
|
|
25
|
+
if credentials_path:
|
|
26
|
+
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_path
|
|
27
|
+
elif not os.getenv("GOOGLE_APPLICATION_CREDENTIALS"):
|
|
28
|
+
raise ValueError(
|
|
29
|
+
"Google application credentials JSON is not provided. Please provide a valid JSON path or set the 'GOOGLE_APPLICATION_CREDENTIALS' environment variable."
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
self.model = TextEmbeddingModel.from_pretrained(self.config.model)
|
|
33
|
+
|
|
34
|
+
def embed(self, text, memory_action: Optional[Literal["add", "search", "update"]] = None):
|
|
35
|
+
"""
|
|
36
|
+
Get the embedding for the given text using Vertex AI.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
text (str): The text to embed.
|
|
40
|
+
memory_action (optional): The type of embedding to use. Must be one of "add", "search", or "update". Defaults to None.
|
|
41
|
+
Returns:
|
|
42
|
+
list: The embedding vector.
|
|
43
|
+
"""
|
|
44
|
+
embedding_type = "SEMANTIC_SIMILARITY"
|
|
45
|
+
if memory_action is not None:
|
|
46
|
+
if memory_action not in self.embedding_types:
|
|
47
|
+
raise ValueError(f"Invalid memory action: {memory_action}")
|
|
48
|
+
|
|
49
|
+
embedding_type = self.embedding_types[memory_action]
|
|
50
|
+
|
|
51
|
+
text_input = TextEmbeddingInput(text=text, task_type=embedding_type)
|
|
52
|
+
embeddings = self.model.get_embeddings(texts=[text_input], output_dimensionality=self.config.embedding_dims)
|
|
53
|
+
|
|
54
|
+
return embeddings[0].values
|