swarmauri-base 0.6.0.dev22__tar.gz → 0.6.0.dev23__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/PKG-INFO +2 -2
  2. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/pyproject.toml +2 -2
  3. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VectorStoreBase.py +135 -0
  4. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VectorStoreCloudMixin.py +23 -0
  5. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VectorStorePersistentMixin.py +33 -0
  6. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VectorStoreRetrieveMixin.py +21 -0
  7. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VectorStoreSaveLoadMixin.py +163 -0
  8. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/VisionVectorStoreBase.py +7 -0
  9. swarmauri_base-0.6.0.dev23/swarmauri_base/vector_stores/__init__.py +0 -0
  10. swarmauri_base-0.6.0.dev23/swarmauri_base/vectors/VectorBase.py +28 -0
  11. swarmauri_base-0.6.0.dev23/swarmauri_base/vectors/__init__.py +0 -0
  12. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/README.md +0 -0
  13. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentBase.py +0 -0
  14. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentConversationMixin.py +0 -0
  15. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentRetrieveMixin.py +0 -0
  16. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentSystemContextMixin.py +0 -0
  17. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentToolMixin.py +0 -0
  18. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/AgentVectorStoreMixin.py +0 -0
  19. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/agents/__init__.py +0 -0
  20. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chains/ChainBase.py +0 -0
  21. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chains/ChainContextBase.py +0 -0
  22. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chains/ChainStepBase.py +0 -0
  23. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chains/PromptContextChainBase.py +0 -0
  24. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chains/__init__.py +0 -0
  25. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chunkers/ChunkerBase.py +0 -0
  26. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/chunkers/__init__.py +0 -0
  27. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/control_panels/ControlPanelBase.py +0 -0
  28. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/control_panels/__init__.py +0 -0
  29. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/conversations/ConversationBase.py +0 -0
  30. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/conversations/ConversationSystemContextMixin.py +0 -0
  31. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/conversations/__init__.py +0 -0
  32. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/data_connectors/DataConnectorBase.py +0 -0
  33. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/data_connectors/__init__.py +0 -0
  34. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/distances/DistanceBase.py +0 -0
  35. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/distances/VisionDistanceBase.py +0 -0
  36. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/distances/__init__.py +0 -0
  37. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/document_stores/DocumentStoreBase.py +0 -0
  38. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/document_stores/DocumentStoreRetrieveBase.py +0 -0
  39. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/document_stores/__init__.py +0 -0
  40. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/documents/DocumentBase.py +0 -0
  41. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/documents/__init__.py +0 -0
  42. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/embeddings/EmbeddingBase.py +0 -0
  43. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/embeddings/VisionEmbeddingBase.py +0 -0
  44. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/embeddings/__init__.py +0 -0
  45. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/factories/FactoryBase.py +0 -0
  46. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/factories/_init__.py +0 -0
  47. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/image_gens/ImageGenBase.py +0 -0
  48. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/image_gens/__init__.py +0 -0
  49. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/llms/LLMBase.py +0 -0
  50. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/llms/__init__.py +0 -0
  51. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/measurements/MeasurementAggregateMixin.py +0 -0
  52. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/measurements/MeasurementBase.py +0 -0
  53. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/measurements/MeasurementCalculateMixin.py +0 -0
  54. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/measurements/MeasurementThresholdMixin.py +0 -0
  55. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/measurements/__init__.py +0 -0
  56. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/messages/MessageBase.py +0 -0
  57. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/messages/__init__.py +0 -0
  58. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/parsers/ParserBase.py +0 -0
  59. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/parsers/__init__.py +0 -0
  60. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/pipelines/PipelineBase.py +0 -0
  61. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/pipelines/__init__.py +0 -0
  62. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/prompts/PromptBase.py +0 -0
  63. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/prompts/PromptGeneratorBase.py +0 -0
  64. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/prompts/PromptMatrixBase.py +0 -0
  65. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/prompts/PromptTemplateBase.py +0 -0
  66. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/prompts/__init__.py +0 -0
  67. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/schema_converters/SchemaConverterBase.py +0 -0
  68. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/schema_converters/__init__.py +0 -0
  69. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/service_registries/ServiceRegistryBase.py +0 -0
  70. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/service_registries/__init__.py +0 -0
  71. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/state/StateBase.py +0 -0
  72. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/state/__init__.py +0 -0
  73. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/swarm/SwarmBase.py +0 -0
  74. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/swarm/__init__.py +0 -0
  75. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/task_mgmt_strategies/TaskMgtStrategyBase.py +0 -0
  76. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/task_mgmt_strategies/__init__.py +0 -0
  77. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/toolkits/ToolkitBase.py +0 -0
  78. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/toolkits/__init__.py +0 -0
  79. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/tools/ParameterBase.py +0 -0
  80. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/tools/ToolBase.py +0 -0
  81. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/tools/__init__.py +0 -0
  82. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/transports/TransportBase.py +0 -0
  83. {swarmauri_base-0.6.0.dev22 → swarmauri_base-0.6.0.dev23}/swarmauri_base/transports/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: swarmauri-base
3
- Version: 0.6.0.dev22
3
+ Version: 0.6.0.dev23
4
4
  Summary: This repository includes base classes and mixins for the Swarmauri framework.
5
5
  Home-page: http://github.com/swarmauri/swarmauri-sdk
6
6
  License: Apache-2.0
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.10
13
13
  Classifier: Programming Language :: Python :: 3.11
14
14
  Classifier: Programming Language :: Python :: 3.12
15
15
  Requires-Dist: pydantic (>=2.0,<3.0)
16
- Requires-Dist: swarmauri_core (>=0.6.0.dev22,<0.7.0)
16
+ Requires-Dist: swarmauri_core (>=0.6.0.dev23,<0.7.0)
17
17
  Project-URL: Repository, http://github.com/swarmauri/swarmauri-sdk
18
18
  Description-Content-Type: text/markdown
19
19
 
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "swarmauri-base"
3
- version = "0.6.0.dev22"
3
+ version = "0.6.0.dev23"
4
4
  description = "This repository includes base classes and mixins for the Swarmauri framework."
5
5
  authors = ["Jacob Stewart <jacob@swarmauri.com>"]
6
6
  license = "Apache-2.0"
@@ -16,7 +16,7 @@ classifiers = [
16
16
  [tool.poetry.dependencies]
17
17
  python = ">=3.10,<3.13"
18
18
 
19
- swarmauri_core = "^0.6.0.dev22"
19
+ swarmauri_core = "^0.6.0.dev23"
20
20
  pydantic = "^2.0"
21
21
 
22
22
  #numpy = "==1.26.4"
@@ -0,0 +1,135 @@
1
+ import json
2
+ from abc import ABC, abstractmethod
3
+ from typing import List, Optional, Literal
4
+ from pydantic import Field, PrivateAttr
5
+ from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes
6
+ from swarmauri_standard.documents.Document import Document
7
+ from swarmauri_core.vector_stores.IVectorStore import IVectorStore
8
+
9
+
10
+ class VectorStoreBase(IVectorStore, ComponentBase):
11
+ """
12
+ Abstract base class for document stores, implementing the IVectorStore interface.
13
+
14
+ This class provides a standard API for adding, updating, getting, and deleting documents in a vector store.
15
+ The specifics of storing (e.g., in a database, in-memory, or file system) are to be implemented by concrete subclasses.
16
+ """
17
+
18
+ documents: List[Document] = []
19
+ _embedder = PrivateAttr()
20
+ _distance = PrivateAttr()
21
+ resource: Optional[str] = Field(default=ResourceTypes.VECTOR_STORE.value)
22
+ type: Literal["VectorStoreBase"] = "VectorStoreBase"
23
+
24
+ @property
25
+ def embedder(self):
26
+ return self._embedder
27
+
28
+ @abstractmethod
29
+ def add_document(self, document: Document) -> None:
30
+ """
31
+ Add a single document to the document store.
32
+
33
+ Parameters:
34
+ - document (IDocument): The document to be added to the store.
35
+ """
36
+ pass
37
+
38
+ @abstractmethod
39
+ def add_documents(self, documents: List[Document]) -> None:
40
+ """
41
+ Add multiple documents to the document store in a batch operation.
42
+
43
+ Parameters:
44
+ - documents (List[IDocument]): A list of documents to be added to the store.
45
+ """
46
+ pass
47
+
48
+ @abstractmethod
49
+ def get_document(self, id: str) -> Optional[Document]:
50
+ """
51
+ Retrieve a single document by its identifier.
52
+
53
+ Parameters:
54
+ - doc_id (str): The unique identifier of the document to retrieve.
55
+
56
+ Returns:
57
+ - Optional[IDocument]: The requested document if found; otherwise, None.
58
+ """
59
+ pass
60
+
61
+ @abstractmethod
62
+ def get_all_documents(self) -> List[Document]:
63
+ """
64
+ Retrieve all documents stored in the document store.
65
+
66
+ Returns:
67
+ - List[IDocument]: A list of all documents in the store.
68
+ """
69
+ pass
70
+
71
+ @abstractmethod
72
+ def update_document(self, id: str, updated_document: Document) -> None:
73
+ """
74
+ Update a document in the document store.
75
+
76
+ Parameters:
77
+ - doc_id (str): The unique identifier of the document to update.
78
+ - updated_document (IDocument): The updated document instance.
79
+ """
80
+ pass
81
+
82
+ @abstractmethod
83
+ def delete_document(self, id: str) -> None:
84
+ """
85
+ Delete a document from the document store by its identifier.
86
+
87
+ Parameters:
88
+ - doc_id (str): The unique identifier of the document to delete.
89
+ """
90
+ pass
91
+
92
+ def clear_documents(self) -> None:
93
+ """
94
+ Deletes all documents from the vector store
95
+
96
+ """
97
+ self.documents = []
98
+
99
+ def document_count(self):
100
+ return len(self.documents)
101
+
102
+ def document_dumps(self) -> str:
103
+ """
104
+ Placeholder
105
+ """
106
+ return json.dumps([each.to_dict() for each in self.documents])
107
+
108
+ def document_dump(self, file_path: str) -> None:
109
+ """
110
+ Placeholder
111
+ """
112
+ with open(file_path, "w", encoding="utf-8") as f:
113
+ json.dump(
114
+ [each.to_dict() for each in self.documents],
115
+ f,
116
+ ensure_ascii=False,
117
+ indent=4,
118
+ )
119
+
120
+ def document_loads(self, json_data: str) -> None:
121
+ """
122
+ Placeholder
123
+ """
124
+ self.documents = [
125
+ globals()[each["type"]].from_dict(each) for each in json.loads(json_data)
126
+ ]
127
+
128
+ def document_load(self, file_path: str) -> None:
129
+ """
130
+ Placeholder
131
+ """
132
+ with open(file_path, "r", encoding="utf-8") as f:
133
+ self.documents = [
134
+ globals()[each["type"]].from_dict(each) for each in json.load(file_path)
135
+ ]
@@ -0,0 +1,23 @@
1
+ from typing import Optional
2
+ from pydantic import Field
3
+
4
+ from swarmauri_core.vector_stores.ICloudVectorStore import ICloudVectorStore
5
+
6
+
7
+ class VectorStoreCloudMixin(ICloudVectorStore):
8
+ """
9
+ Mixin class for cloud-based vector stores.
10
+ """
11
+
12
+ api_key: str
13
+ collection_name: str
14
+ url: Optional[str] = Field(
15
+ None, description="URL of the cloud-based store to connect to"
16
+ )
17
+
18
+ vector_size: Optional[int] = Field(
19
+ None, description="Size of the vectors used in the store"
20
+ )
21
+ client: Optional[object] = Field(
22
+ None, description="Client object for interacting with the cloud-based store"
23
+ )
@@ -0,0 +1,33 @@
1
+ from typing import Optional
2
+ from pydantic import Field
3
+
4
+ from swarmauri_core.vector_stores.IPersistentVectorStore import IPersistentVectorStore
5
+
6
+
7
+ class VectorStorePersistentMixin(IPersistentVectorStore):
8
+ """
9
+ Mixin class for persistent-based vector stores.
10
+ """
11
+
12
+ collection_name: str
13
+
14
+ collection: Optional[object] = Field(
15
+ None,
16
+ description="Collection object for interacting with the persistent-based store",
17
+ )
18
+
19
+ path: Optional[str] = Field(
20
+ None, description="URL of the persistent-based store to connect to"
21
+ )
22
+
23
+ vector_size: Optional[int] = Field(
24
+ None, description="Size of the vectors used in the store"
25
+ )
26
+ client: Optional[object] = Field(
27
+ None,
28
+ description="Client object for interacting with the persistent-based store",
29
+ )
30
+
31
+ vectorizer: Optional[object] = Field(
32
+ None, description="Vectorizer object for converting documents to vectors"
33
+ )
@@ -0,0 +1,21 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import List
3
+ from pydantic import BaseModel
4
+ from swarmauri_standard.documents.Document import Document
5
+ from swarmauri_core.vector_stores.IVectorStoreRetrieve import IVectorStoreRetrieve
6
+
7
+
8
+ class VectorStoreRetrieveMixin(IVectorStoreRetrieve, BaseModel):
9
+ @abstractmethod
10
+ def retrieve(self, query: str, top_k: int = 5) -> List[Document]:
11
+ """
12
+ Retrieve the top_k most relevant documents based on the given query.
13
+
14
+ Args:
15
+ query (str): The query string used for document retrieval.
16
+ top_k (int): The number of top relevant documents to retrieve.
17
+
18
+ Returns:
19
+ List[IDocument]: A list of the top_k most relevant documents.
20
+ """
21
+ pass
@@ -0,0 +1,163 @@
1
+ from typing import List
2
+ import os
3
+ from pydantic import BaseModel
4
+ import json
5
+ import glob
6
+ import importlib
7
+ from swarmauri_core.vector_stores.IVectorStoreSaveLoad import IVectorStoreSaveLoad
8
+ from swarmauri_standard.documents.Document import Document
9
+
10
+ class VectorStoreSaveLoadMixin(IVectorStoreSaveLoad, BaseModel):
11
+ """
12
+ Base class for vector stores with built-in support for saving and loading
13
+ the vectorizer's model and the documents.
14
+ """
15
+
16
+
17
+ def save_store(self, directory_path: str) -> None:
18
+ """
19
+ Saves both the vectorizer's model and the documents.
20
+ """
21
+ # Ensure the directory exists
22
+ if not os.path.exists(directory_path):
23
+ os.makedirs(directory_path)
24
+
25
+ # Save the vectorizer model
26
+ model_path = os.path.join(directory_path, "embedding_model")
27
+ self._vectorizer.save_model(model_path)
28
+
29
+ # Save documents
30
+ documents_path = os.path.join(directory_path, "documents.json")
31
+ with open(documents_path, 'w', encoding='utf-8') as f:
32
+ json.dump([each.to_dict() for each in self.documents],
33
+ f,
34
+ ensure_ascii=False,
35
+ indent=4)
36
+
37
+
38
+ def load_store(self, directory_path: str) -> None:
39
+ """
40
+ Loads both the vectorizer's model and the documents.
41
+ """
42
+ # Load the vectorizer model
43
+ model_path = os.path.join(directory_path, "embedding_model")
44
+ self.vectorizer.load_model(model_path)
45
+
46
+ # Load documents
47
+ documents_path = os.path.join(directory_path, "documents.json")
48
+ with open(documents_path, 'r', encoding='utf-8') as f:
49
+ self.documents = [self._load_document(each) for each in json.load(f)]
50
+
51
+ def _load_document(self, data):
52
+ document_type = data.pop("type")
53
+ if document_type:
54
+ module = importlib.import_module(f"swarmauri.documents.concrete.{document_type}")
55
+ document_class = getattr(module, document_type)
56
+ document = document_class.from_dict(data)
57
+ return document
58
+ else:
59
+ raise ValueError("Unknown document type")
60
+
61
+ def save_parts(self, directory_path: str, chunk_size: int = 10485760) -> None:
62
+ """
63
+ Splits the file into parts if it's too large and saves those parts individually.
64
+ """
65
+ file_number = 1
66
+ model_path = os.path.join(directory_path, "embedding_model")
67
+ parts_directory = os.path.join(directory_path, "parts")
68
+
69
+ if not os.path.exists(parts_directory):
70
+ os.makedirs(parts_directory)
71
+
72
+
73
+
74
+ with open(f"{model_path}/model.safetensors", 'rb') as f:
75
+ chunk = f.read(chunk_size)
76
+ while chunk:
77
+ with open(f"{parts_directory}/model.safetensors.part{file_number:03}", 'wb') as chunk_file:
78
+ chunk_file.write(chunk)
79
+ file_number += 1
80
+ chunk = f.read(chunk_size)
81
+
82
+ # Split the documents into parts and save them
83
+ documents_dir = os.path.join(directory_path, "documents")
84
+
85
+ self._split_json_file(directory_path, chunk_size=chunk_size)
86
+
87
+
88
+ def _split_json_file(self, directory_path: str, max_records=100, chunk_size: int = 10485760):
89
+ # Read the input JSON file
90
+ combined_documents_file_path = os.path.join(directory_path, "documents.json")
91
+
92
+ # load the master JSON file
93
+ with open(combined_documents_file_path , 'r') as file:
94
+ data = json.load(file)
95
+
96
+ # Set and Create documents parts folder if it does not exist
97
+ documents_dir = os.path.join(directory_path, "documents")
98
+ if not os.path.exists(documents_dir):
99
+ os.makedirs(documents_dir)
100
+ current_batch = []
101
+ file_index = 1
102
+ current_size = 0
103
+
104
+ for record in data:
105
+ current_batch.append(record)
106
+ current_size = len(json.dumps(current_batch).encode('utf-8'))
107
+
108
+ # Check if current batch meets the splitting criteria
109
+ if len(current_batch) >= max_records or current_size >= chunk_size:
110
+ # Write current batch to a new file
111
+ output_file = f'document_part_{file_index}.json'
112
+ output_file = os.path.join(documents_dir, output_file)
113
+ with open(output_file, 'w') as outfile:
114
+ json.dump(current_batch, outfile)
115
+
116
+ # Prepare for the next batch
117
+ current_batch = []
118
+ current_size = 0
119
+ file_index += 1
120
+
121
+ # Check if there's any remaining data to be written
122
+ if current_batch:
123
+ output_file = f'document_part_{file_index}.json'
124
+ output_file = os.path.join(documents_dir, output_file)
125
+ with open(output_file, 'w') as outfile:
126
+ json.dump(current_batch, outfile)
127
+
128
+ def load_parts(self, directory_path: str, file_pattern: str = '*.part*') -> None:
129
+ """
130
+ Combines file parts from a directory back into a single file and loads it.
131
+ """
132
+ model_path = os.path.join(directory_path, "embedding_model")
133
+ parts_directory = os.path.join(directory_path, "parts")
134
+ output_file_path = os.path.join(model_path, "model.safetensors")
135
+
136
+ parts = sorted(glob.glob(os.path.join(parts_directory, file_pattern)))
137
+ with open(output_file_path, 'wb') as output_file:
138
+ for part in parts:
139
+ with open(part, 'rb') as file_part:
140
+ output_file.write(file_part.read())
141
+
142
+ # Load the combined_model now
143
+ model_path = os.path.join(directory_path, "embedding_model")
144
+ self._vectorizer.load_model(model_path)
145
+
146
+ # Load document files
147
+ self._load_documents(directory_path)
148
+
149
+
150
+ def _load_documents(self, directory_path: str) -> None:
151
+ """
152
+ Loads the documents from parts stored in the given directory.
153
+ """
154
+ part_paths = glob.glob(os.path.join(directory_path, "documents/*.json"))
155
+ for part_path in part_paths:
156
+ with open(part_path, "r") as f:
157
+ part_documents = json.load(f)
158
+ for document_data in part_documents:
159
+ document_type = document_data.pop("type")
160
+ document_module = importlib.import_module(f"swarmauri.documents.concrete.{document_type}")
161
+ document_class = getattr(document_module, document_type)
162
+ document = document_class.from_dict(document_data)
163
+ self.documents.append(document)
@@ -0,0 +1,7 @@
1
+ from typing import Optional, Literal
2
+ from pydantic import Field
3
+ from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes
4
+
5
+ class VisionVectorStoreBase(ComponentBase):
6
+ resource: Optional[str] = Field(default=ResourceTypes.EMBEDDING.value, frozen=True)
7
+ type: Literal['VisionVectorStoreBase'] = 'VisionVectorStoreBase'
@@ -0,0 +1,28 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import List, Optional, Literal
3
+ import json
4
+ import numpy as np
5
+ from pydantic import Field
6
+ from swarmauri_core.vectors.IVector import IVector
7
+ from swarmauri_core.ComponentBase import ComponentBase, ResourceTypes
8
+
9
+ class VectorBase(IVector, ComponentBase):
10
+ value: List[float]
11
+ resource: Optional[str] = Field(default=ResourceTypes.VECTOR.value, frozen=True)
12
+ type: Literal['VectorBase'] = 'VectorBase'
13
+
14
+ def to_numpy(self) -> np.ndarray:
15
+ """
16
+ Converts the vector into a numpy array.
17
+
18
+ Returns:
19
+ np.ndarray: The numpy array representation of the vector.
20
+ """
21
+ return np.array(self.value)
22
+
23
+ @property
24
+ def shape(self):
25
+ return self.to_numpy().shape
26
+
27
+ def __len__(self):
28
+ return len(self.value)