syrtis-python-storage 1.0.1__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syrtis_python_storage-1.0.1/README.md → syrtis_python_storage-2.0.0/PKG-INFO +33 -1
- syrtis_python_storage-1.0.1/PKG-INFO → syrtis_python_storage-2.0.0/README.md +8 -15
- {syrtis_python_storage-1.0.1 → syrtis_python_storage-2.0.0}/pyproject.toml +17 -2
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/__init__.py +33 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_document_storage.py +36 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_file_storage.py +36 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_graph_storage.py +23 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_storage.py +36 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_vector_storage.py +49 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/filesystem_storage.py +57 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/mongo_document_storage.py +93 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/mongo_file_storage.py +96 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/neo4j_graph_storage.py +68 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_document_storage.py +133 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_file_storage.py +124 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_vector_storage.py +162 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/storage_registry.py +32 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/vector_match.py +12 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/capabilities_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/credential_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/database_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/label_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/path_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/storage_config_option.py +156 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/storages_config_option.py +26 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/table_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/type_config_option.py +15 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/const/capability.py +10 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/__init__.py +0 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/invalid_storage_config_exception.py +17 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/invalid_storage_key_exception.py +17 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/no_storage_for_capability_exception.py +20 -0
- syrtis_python_storage-2.0.0/src/syrtis_python_storage/py.typed +0 -0
- syrtis_python_storage-2.0.0/src/syrtis_syrtis_python_storage/__init__.py +0 -0
- syrtis_python_storage-2.0.0/src/syrtis_syrtis_python_storage/py.typed +0 -0
- syrtis_python_storage-2.0.0/tests/.gitkeep +0 -0
- syrtis_python_storage-2.0.0/tests/test_storage.py +229 -0
- {syrtis_python_storage-1.0.1/src/syrtis_syrtis_python_storage → syrtis_python_storage-2.0.0/src/syrtis_python_storage/common}/__init__.py +0 -0
- /syrtis_python_storage-1.0.1/src/syrtis_syrtis_python_storage/py.typed → /syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/__init__.py +0 -0
- /syrtis_python_storage-1.0.1/tests/.gitkeep → /syrtis_python_storage-2.0.0/src/syrtis_python_storage/const/__init__.py +0 -0
|
@@ -1,15 +1,47 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: syrtis-python-storage
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Author-Email: weeger <contact@syrtis.ai>
|
|
5
|
+
License: MIT
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
8
|
+
Classifier: Operating System :: OS Independent
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Requires-Dist: wexample-config
|
|
11
|
+
Requires-Dist: wexample-helpers>=20.8.0
|
|
12
|
+
Requires-Dist: wexample-remote>=1.2.0
|
|
13
|
+
Provides-Extra: mongo
|
|
14
|
+
Requires-Dist: pymongo>=4.6; extra == "mongo"
|
|
15
|
+
Provides-Extra: neo4j
|
|
16
|
+
Requires-Dist: neo4j>=5; extra == "neo4j"
|
|
17
|
+
Provides-Extra: postgres
|
|
18
|
+
Requires-Dist: psycopg2-binary; extra == "postgres"
|
|
19
|
+
Requires-Dist: sqlalchemy>=2; extra == "postgres"
|
|
20
|
+
Provides-Extra: dev
|
|
21
|
+
Requires-Dist: pytest; extra == "dev"
|
|
22
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
23
|
+
Requires-Dist: wexample-wallet; extra == "dev"
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
1
26
|
# python_storage
|
|
2
27
|
|
|
3
|
-
Version:
|
|
28
|
+
Version: 2.0.0
|
|
4
29
|
|
|
5
30
|
## Table of Contents
|
|
6
31
|
|
|
32
|
+
- [Introduction](#introduction)
|
|
7
33
|
- [Install](#install)
|
|
8
34
|
- [Tests](#tests)
|
|
9
35
|
- [Roadmap](#roadmap)
|
|
10
36
|
- [Status Compatibility](#status-compatibility)
|
|
11
37
|
- [Useful Links](#useful-links)
|
|
12
38
|
|
|
39
|
+
`syrtis-python-storage` gives an application the places it keeps contents in, behind one interface per kind of content: files written and read back by key, vectors searched by similarity, graphs queried in their own language, documents kept in collections and found by their fields. Each storage has a name and the capabilities it accepts (`files`, `vectors`, `graph`), and a registry picks the storage for a capability — so the code needing a place to write never chooses the backend itself.
|
|
40
|
+
|
|
41
|
+
Every storage is a `wexample-remote` remote, so it can be checked like any other service. Backends: a local directory, PostgreSQL with pgvector (the `postgres` extra), Neo4j (the `neo4j` extra), MongoDB (the `mongo` extra).
|
|
42
|
+
|
|
43
|
+
Its PHP twin is `syrtis/php-storage`: the same contracts on the same tables and collections, so that what one language wrote the other reads.
|
|
44
|
+
|
|
13
45
|
## Installation
|
|
14
46
|
|
|
15
47
|
```bash
|
|
@@ -1,29 +1,22 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: syrtis-python-storage
|
|
3
|
-
Version: 1.0.1
|
|
4
|
-
Author-Email: weeger <contact@syrtis.ai>
|
|
5
|
-
License: MIT
|
|
6
|
-
Classifier: Programming Language :: Python :: 3
|
|
7
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
8
|
-
Classifier: Operating System :: OS Independent
|
|
9
|
-
Requires-Python: >=3.10
|
|
10
|
-
Provides-Extra: dev
|
|
11
|
-
Requires-Dist: pytest; extra == "dev"
|
|
12
|
-
Requires-Dist: pytest-cov; extra == "dev"
|
|
13
|
-
Description-Content-Type: text/markdown
|
|
14
|
-
|
|
15
1
|
# python_storage
|
|
16
2
|
|
|
17
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
18
4
|
|
|
19
5
|
## Table of Contents
|
|
20
6
|
|
|
7
|
+
- [Introduction](#introduction)
|
|
21
8
|
- [Install](#install)
|
|
22
9
|
- [Tests](#tests)
|
|
23
10
|
- [Roadmap](#roadmap)
|
|
24
11
|
- [Status Compatibility](#status-compatibility)
|
|
25
12
|
- [Useful Links](#useful-links)
|
|
26
13
|
|
|
14
|
+
`syrtis-python-storage` gives an application the places it keeps contents in, behind one interface per kind of content: files written and read back by key, vectors searched by similarity, graphs queried in their own language, documents kept in collections and found by their fields. Each storage has a name and the capabilities it accepts (`files`, `vectors`, `graph`), and a registry picks the storage for a capability — so the code needing a place to write never chooses the backend itself.
|
|
15
|
+
|
|
16
|
+
Every storage is a `wexample-remote` remote, so it can be checked like any other service. Backends: a local directory, PostgreSQL with pgvector (the `postgres` extra), Neo4j (the `neo4j` extra), MongoDB (the `mongo` extra).
|
|
17
|
+
|
|
18
|
+
Its PHP twin is `syrtis/php-storage`: the same contracts on the same tables and collections, so that what one language wrote the other reads.
|
|
19
|
+
|
|
27
20
|
## Installation
|
|
28
21
|
|
|
29
22
|
```bash
|
|
@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "syrtis-python-storage"
|
|
9
|
-
version = "
|
|
9
|
+
version = "2.0.0"
|
|
10
10
|
authors = [
|
|
11
11
|
{ name = "weeger", email = "contact@syrtis.ai" },
|
|
12
12
|
]
|
|
@@ -16,7 +16,11 @@ classifiers = [
|
|
|
16
16
|
"License :: OSI Approved :: MIT License",
|
|
17
17
|
"Operating System :: OS Independent",
|
|
18
18
|
]
|
|
19
|
-
dependencies = [
|
|
19
|
+
dependencies = [
|
|
20
|
+
"wexample-config",
|
|
21
|
+
"wexample-helpers>=20.8.0",
|
|
22
|
+
"wexample-remote>=1.2.0",
|
|
23
|
+
]
|
|
20
24
|
|
|
21
25
|
[project.readme]
|
|
22
26
|
file = "README.md"
|
|
@@ -26,9 +30,20 @@ content-type = "text/markdown"
|
|
|
26
30
|
text = "MIT"
|
|
27
31
|
|
|
28
32
|
[project.optional-dependencies]
|
|
33
|
+
mongo = [
|
|
34
|
+
"pymongo>=4.6",
|
|
35
|
+
]
|
|
36
|
+
neo4j = [
|
|
37
|
+
"neo4j>=5",
|
|
38
|
+
]
|
|
39
|
+
postgres = [
|
|
40
|
+
"psycopg2-binary",
|
|
41
|
+
"sqlalchemy>=2",
|
|
42
|
+
]
|
|
29
43
|
dev = [
|
|
30
44
|
"pytest",
|
|
31
45
|
"pytest-cov",
|
|
46
|
+
"wexample-wallet",
|
|
32
47
|
]
|
|
33
48
|
|
|
34
49
|
[tool.setuptools.packages.find]
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
|
|
4
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
5
|
+
from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
|
|
6
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
7
|
+
from syrtis_python_storage.common.abstract_vector_storage import AbstractVectorStorage
|
|
8
|
+
from syrtis_python_storage.common.filesystem_storage import FilesystemStorage
|
|
9
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
10
|
+
from syrtis_python_storage.common.neo4j_graph_storage import Neo4jGraphStorage
|
|
11
|
+
from syrtis_python_storage.common.mongo_file_storage import MongoFileStorage
|
|
12
|
+
from syrtis_python_storage.common.postgres_document_storage import PostgresDocumentStorage
|
|
13
|
+
from syrtis_python_storage.common.postgres_file_storage import PostgresFileStorage
|
|
14
|
+
from syrtis_python_storage.common.postgres_vector_storage import PostgresVectorStorage
|
|
15
|
+
from syrtis_python_storage.common.storage_registry import StorageRegistry
|
|
16
|
+
from syrtis_python_storage.common.vector_match import VectorMatch
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"AbstractDocumentStorage",
|
|
20
|
+
"AbstractFileStorage",
|
|
21
|
+
"AbstractGraphStorage",
|
|
22
|
+
"AbstractStorage",
|
|
23
|
+
"AbstractVectorStorage",
|
|
24
|
+
"FilesystemStorage",
|
|
25
|
+
"MongoDocumentStorage",
|
|
26
|
+
"Neo4jGraphStorage",
|
|
27
|
+
"MongoFileStorage",
|
|
28
|
+
"PostgresDocumentStorage",
|
|
29
|
+
"PostgresFileStorage",
|
|
30
|
+
"PostgresVectorStorage",
|
|
31
|
+
"StorageRegistry",
|
|
32
|
+
"VectorMatch",
|
|
33
|
+
]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@base_class
|
|
12
|
+
class AbstractDocumentStorage(AbstractStorage):
|
|
13
|
+
"""Documents — dicts — kept by key in named collections, and found by
|
|
14
|
+
the values of their fields."""
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
18
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_DOCUMENTS
|
|
19
|
+
|
|
20
|
+
return [STORAGE_CAPABILITY_DOCUMENTS]
|
|
21
|
+
|
|
22
|
+
@abstract_method
|
|
23
|
+
def delete_document(self, collection: str, key: str) -> None:
|
|
24
|
+
"""Removes the document of the key; nothing when there is none."""
|
|
25
|
+
|
|
26
|
+
@abstract_method
|
|
27
|
+
def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
28
|
+
"""The documents whose fields hold the values given; all without filters."""
|
|
29
|
+
|
|
30
|
+
@abstract_method
|
|
31
|
+
def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
|
|
32
|
+
"""The document of the key, None when there is none."""
|
|
33
|
+
|
|
34
|
+
@abstract_method
|
|
35
|
+
def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
|
|
36
|
+
"""Writes the document under the key, replacing the one there."""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
4
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
5
|
+
|
|
6
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@base_class
|
|
10
|
+
class AbstractFileStorage(AbstractStorage):
|
|
11
|
+
"""Contents written and read back by key."""
|
|
12
|
+
|
|
13
|
+
@classmethod
|
|
14
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
15
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_FILES
|
|
16
|
+
|
|
17
|
+
return [STORAGE_CAPABILITY_FILES]
|
|
18
|
+
|
|
19
|
+
@abstract_method
|
|
20
|
+
def delete(self, key: str) -> None:
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
@abstract_method
|
|
24
|
+
def exists(self, key: str) -> bool:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
@abstract_method
|
|
28
|
+
def read(self, key: str) -> bytes:
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
def read_text(self, key: str, encoding: str = "utf-8") -> str:
|
|
32
|
+
return self.read(key).decode(encoding)
|
|
33
|
+
|
|
34
|
+
@abstract_method
|
|
35
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
36
|
+
pass
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@base_class
|
|
12
|
+
class AbstractGraphStorage(AbstractStorage):
|
|
13
|
+
"""Nodes and relationships, queried in the backend's own language."""
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
17
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_GRAPH
|
|
18
|
+
|
|
19
|
+
return [STORAGE_CAPABILITY_GRAPH]
|
|
20
|
+
|
|
21
|
+
@abstract_method
|
|
22
|
+
def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
23
|
+
"""The records the query returns, as dicts; none for a pure write."""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from wexample_helpers.classes.base_class import BaseClass
|
|
4
|
+
from wexample_helpers.classes.field import public_field
|
|
5
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
6
|
+
from wexample_remote.common.abstract_remote import AbstractRemote
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@base_class
|
|
10
|
+
class AbstractStorage(AbstractRemote, BaseClass):
|
|
11
|
+
"""A place contents are kept in — a directory, a bucket, a database —
|
|
12
|
+
whatever the backend. As a remote, it can be checked."""
|
|
13
|
+
|
|
14
|
+
capabilities: list[str] = public_field(
|
|
15
|
+
description="Kinds of content the storage accepts (e.g. `files`, `vectors`); "
|
|
16
|
+
"empty takes those of its class.",
|
|
17
|
+
factory=list,
|
|
18
|
+
)
|
|
19
|
+
label: str | None = public_field(default=None, description="Human-readable name.")
|
|
20
|
+
name: str = public_field(description="Name the storage is registered and chosen by.")
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
24
|
+
return []
|
|
25
|
+
|
|
26
|
+
def accepts(self, capability: str) -> bool:
|
|
27
|
+
return capability in self.get_capabilities()
|
|
28
|
+
|
|
29
|
+
def get_capabilities(self) -> list[str]:
|
|
30
|
+
return self.capabilities or self.get_default_capabilities()
|
|
31
|
+
|
|
32
|
+
def get_key(self) -> str:
|
|
33
|
+
return self.name
|
|
34
|
+
|
|
35
|
+
def get_label(self) -> str:
|
|
36
|
+
return self.label or self.name
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from syrtis_python_storage.common.vector_match import VectorMatch
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@base_class
|
|
15
|
+
class AbstractVectorStorage(AbstractStorage):
|
|
16
|
+
"""Vectors grouped in namespaces (e.g. one per document), each with a key
|
|
17
|
+
and the text it was made from, searched by cosine distance."""
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
21
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_VECTORS
|
|
22
|
+
|
|
23
|
+
return [STORAGE_CAPABILITY_VECTORS]
|
|
24
|
+
|
|
25
|
+
@abstract_method
|
|
26
|
+
def delete_vector(self, namespace: str, key: str) -> None:
|
|
27
|
+
"""Removes the vector of the key; nothing when there is none."""
|
|
28
|
+
|
|
29
|
+
@abstract_method
|
|
30
|
+
def delete_vectors(self, namespace: str) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@abstract_method
|
|
34
|
+
def get_vector_keys(self, namespace: str) -> set[str]:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
@abstract_method
|
|
38
|
+
def save_vector(self, namespace: str, key: str, vector: list[float], content: str | None) -> None:
|
|
39
|
+
"""Add the vector, or replace the one of the same key."""
|
|
40
|
+
|
|
41
|
+
@abstract_method
|
|
42
|
+
def search_vectors(
|
|
43
|
+
self,
|
|
44
|
+
namespace: str,
|
|
45
|
+
vector: list[float],
|
|
46
|
+
limit: int | None = None,
|
|
47
|
+
max_distance: float | None = None,
|
|
48
|
+
) -> list[VectorMatch]:
|
|
49
|
+
"""The closest vectors first."""
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from wexample_helpers.classes.field import public_field
|
|
8
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
9
|
+
|
|
10
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@base_class
|
|
17
|
+
class FilesystemStorage(AbstractFileStorage):
|
|
18
|
+
"""Contents as files of a directory, one per key; the directory is created
|
|
19
|
+
on first write."""
|
|
20
|
+
|
|
21
|
+
path: Path = public_field(description="Directory holding the files.")
|
|
22
|
+
|
|
23
|
+
def check_status(self) -> RemoteStatus:
|
|
24
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
25
|
+
|
|
26
|
+
directory = Path(self.path)
|
|
27
|
+
# The closest existing parent is the one a first write would create in.
|
|
28
|
+
existing = next(parent for parent in (directory, *directory.parents) if parent.exists())
|
|
29
|
+
if not existing.is_dir() or not os.access(existing, os.W_OK):
|
|
30
|
+
return RemoteStatus.down(f"Not a writable directory: {existing}")
|
|
31
|
+
return RemoteStatus.up()
|
|
32
|
+
|
|
33
|
+
def delete(self, key: str) -> None:
|
|
34
|
+
self.get_file_path(key).unlink(missing_ok=True)
|
|
35
|
+
|
|
36
|
+
def exists(self, key: str) -> bool:
|
|
37
|
+
return self.get_file_path(key).is_file()
|
|
38
|
+
|
|
39
|
+
def get_file_path(self, key: str) -> Path:
|
|
40
|
+
"""The file of a key, which may not leave the directory."""
|
|
41
|
+
directory = Path(self.path).resolve()
|
|
42
|
+
file_path = (directory / key).resolve()
|
|
43
|
+
if not file_path.is_relative_to(directory):
|
|
44
|
+
from syrtis_python_storage.exception.invalid_storage_key_exception import (
|
|
45
|
+
InvalidStorageKeyException,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
raise InvalidStorageKeyException(key=key, storage_name=self.name)
|
|
49
|
+
return file_path
|
|
50
|
+
|
|
51
|
+
def read(self, key: str) -> bytes:
|
|
52
|
+
return self.get_file_path(key).read_bytes()
|
|
53
|
+
|
|
54
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
55
|
+
file_path = self.get_file_path(key)
|
|
56
|
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
file_path.write_bytes(content.encode() if isinstance(content, str) else content)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
# The field a document's key is kept in.
|
|
15
|
+
MONGO_DOCUMENT_STORAGE_KEY_FIELD: str = "_id"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@base_class
|
|
19
|
+
class MongoDocumentStorage(AbstractDocumentStorage):
|
|
20
|
+
"""A MongoDB database, a collection per kind of document, the key as
|
|
21
|
+
`_id`. Needs the `mongo` extra. The server is reached at the host given:
|
|
22
|
+
a replica set announcing names its members alone resolve is not followed
|
|
23
|
+
(`directConnection`)."""
|
|
24
|
+
|
|
25
|
+
database: str = public_field(description="Database name.")
|
|
26
|
+
host: str = public_field(description="Server host.")
|
|
27
|
+
password: str | None = public_field(default=None, description="Password.")
|
|
28
|
+
port: int = public_field(default=27017, description="Server port.")
|
|
29
|
+
timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
|
|
30
|
+
user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
|
|
31
|
+
_client: Any = private_field(default=None, description="MongoDB client, created on first use.")
|
|
32
|
+
|
|
33
|
+
def __repr__(self) -> str:
|
|
34
|
+
# The password must never reach a log or a traceback.
|
|
35
|
+
return f"MongoDocumentStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> MongoDocumentStorage:
|
|
39
|
+
"""From a `wexample-wallet` credential of type `mongo` (host, port,
|
|
40
|
+
database, user, password), named after it unless `name` is given."""
|
|
41
|
+
return cls(
|
|
42
|
+
database=credential.get("database"),
|
|
43
|
+
host=credential.get("host"),
|
|
44
|
+
name=kwargs.pop("name", credential.name),
|
|
45
|
+
password=credential.get("password", default=None),
|
|
46
|
+
port=int(credential.get("port", default=27017)),
|
|
47
|
+
user=credential.get("user", default=None),
|
|
48
|
+
**kwargs,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def check_status(self) -> RemoteStatus:
|
|
52
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
53
|
+
|
|
54
|
+
self._get_database().command("ping")
|
|
55
|
+
return RemoteStatus.up(f"MongoDB {self._get_client().server_info()['version']}")
|
|
56
|
+
|
|
57
|
+
def delete_document(self, collection: str, key: str) -> None:
|
|
58
|
+
self._get_database()[collection].delete_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
|
|
59
|
+
|
|
60
|
+
def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
61
|
+
return [self._to_plain(document) for document in self._get_database()[collection].find(filters or {})]
|
|
62
|
+
|
|
63
|
+
def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
|
|
64
|
+
document = self._get_database()[collection].find_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
|
|
65
|
+
return self._to_plain(document) if document is not None else None
|
|
66
|
+
|
|
67
|
+
def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
|
|
68
|
+
self._get_database()[collection].replace_one(
|
|
69
|
+
{MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
|
|
70
|
+
{**document, MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
|
|
71
|
+
upsert=True,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def _get_client(self) -> Any:
|
|
75
|
+
if self._client is None:
|
|
76
|
+
from pymongo import MongoClient
|
|
77
|
+
|
|
78
|
+
self._client = MongoClient(
|
|
79
|
+
host=self.host,
|
|
80
|
+
port=self.port,
|
|
81
|
+
username=self.user,
|
|
82
|
+
password=self.password,
|
|
83
|
+
directConnection=True,
|
|
84
|
+
serverSelectionTimeoutMS=self.timeout * 1000,
|
|
85
|
+
)
|
|
86
|
+
return self._client
|
|
87
|
+
|
|
88
|
+
def _get_database(self) -> Any:
|
|
89
|
+
return self._get_client()[self.database]
|
|
90
|
+
|
|
91
|
+
def _to_plain(self, document: dict[str, Any]) -> dict[str, Any]:
|
|
92
|
+
"""The document without its key field, given back as the caller wrote it."""
|
|
93
|
+
return {field: value for field, value in document.items() if field != MONGO_DOCUMENT_STORAGE_KEY_FIELD}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
15
|
+
|
|
16
|
+
MONGO_FILE_STORAGE_COLLECTION_DEFAULT: str = "syrtis_storage_file"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@base_class
|
|
20
|
+
class MongoFileStorage(AbstractFileStorage):
|
|
21
|
+
"""Contents written and read back by key in a MongoDB collection — the
|
|
22
|
+
key as `_id`, the content as binary —, over the same connection as
|
|
23
|
+
`MongoDocumentStorage`. A content is one document: up to MongoDB's
|
|
24
|
+
16 MB. Needs the `mongo` extra."""
|
|
25
|
+
|
|
26
|
+
collection: str = public_field(default=MONGO_FILE_STORAGE_COLLECTION_DEFAULT, description="Collection holding the contents.")
|
|
27
|
+
database: str = public_field(description="Database name.")
|
|
28
|
+
host: str = public_field(description="Server host.")
|
|
29
|
+
password: str | None = public_field(default=None, description="Password.")
|
|
30
|
+
port: int = public_field(default=27017, description="Server port.")
|
|
31
|
+
timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
|
|
32
|
+
user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
|
|
33
|
+
_documents: Any = private_field(default=None, description="The document storage whose connection it uses, opened on first use.")
|
|
34
|
+
|
|
35
|
+
def __repr__(self) -> str:
|
|
36
|
+
# The password must never reach a log or a traceback.
|
|
37
|
+
return f"MongoFileStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> MongoFileStorage:
|
|
41
|
+
"""From a `wexample-wallet` credential of type `mongo` (host, port,
|
|
42
|
+
database, user, password), named after it unless `name` is given."""
|
|
43
|
+
return cls(
|
|
44
|
+
database=credential.get("database"),
|
|
45
|
+
host=credential.get("host"),
|
|
46
|
+
name=kwargs.pop("name", credential.name),
|
|
47
|
+
password=credential.get("password", default=None),
|
|
48
|
+
port=int(credential.get("port", default=27017)),
|
|
49
|
+
user=credential.get("user", default=None),
|
|
50
|
+
**kwargs,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def check_status(self) -> RemoteStatus:
|
|
54
|
+
return self._get_documents().check_status()
|
|
55
|
+
|
|
56
|
+
def delete(self, key: str) -> None:
|
|
57
|
+
self._get_collection().delete_one({"_id": key})
|
|
58
|
+
|
|
59
|
+
def exists(self, key: str) -> bool:
|
|
60
|
+
return self._get_collection().count_documents({"_id": key}, limit=1) > 0
|
|
61
|
+
|
|
62
|
+
def read(self, key: str) -> bytes:
|
|
63
|
+
document = self._get_collection().find_one({"_id": key})
|
|
64
|
+
if document is None:
|
|
65
|
+
raise FileNotFoundError(key)
|
|
66
|
+
|
|
67
|
+
return bytes(document["content"])
|
|
68
|
+
|
|
69
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
70
|
+
from bson.binary import Binary
|
|
71
|
+
|
|
72
|
+
self._get_collection().replace_one(
|
|
73
|
+
{"_id": key},
|
|
74
|
+
{"_id": key, "content": Binary(content.encode() if isinstance(content, str) else content)},
|
|
75
|
+
upsert=True,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def _get_collection(self) -> Any:
|
|
79
|
+
return self._get_documents()._get_database()[self.collection]
|
|
80
|
+
|
|
81
|
+
def _get_documents(self) -> MongoDocumentStorage:
|
|
82
|
+
"""The connection, as the document storage opens it."""
|
|
83
|
+
if self._documents is None:
|
|
84
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
85
|
+
|
|
86
|
+
self._documents = MongoDocumentStorage(
|
|
87
|
+
name=self.name,
|
|
88
|
+
database=self.database,
|
|
89
|
+
host=self.host,
|
|
90
|
+
password=self.password,
|
|
91
|
+
port=self.port,
|
|
92
|
+
timeout=self.timeout,
|
|
93
|
+
user=self.user,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
return self._documents
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@base_class
|
|
16
|
+
class Neo4jGraphStorage(AbstractGraphStorage):
|
|
17
|
+
"""A Neo4j database, queried in Cypher. Needs the `neo4j` extra."""
|
|
18
|
+
|
|
19
|
+
database: str | None = public_field(default=None, description="Database; the server's default when None.")
|
|
20
|
+
password: str = public_field(description="Password.")
|
|
21
|
+
url: str = public_field(description="Bolt url, e.g. `bolt://localhost:7687`.")
|
|
22
|
+
user: str = public_field(description="User name.")
|
|
23
|
+
_driver: Any = private_field(default=None, description="Neo4j driver, created on first use.")
|
|
24
|
+
|
|
25
|
+
def __repr__(self) -> str:
|
|
26
|
+
# The password must never reach a log or a traceback.
|
|
27
|
+
return f"Neo4jGraphStorage(name={self.name!r}, url={self.url!r})"
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> Neo4jGraphStorage:
|
|
31
|
+
"""From a `wexample-wallet` credential of type `neo4j` (url, user,
|
|
32
|
+
password), named after it unless `name` is given."""
|
|
33
|
+
return cls(
|
|
34
|
+
name=kwargs.pop("name", credential.name),
|
|
35
|
+
password=credential.get("password"),
|
|
36
|
+
url=credential.get("url"),
|
|
37
|
+
user=credential.get("user"),
|
|
38
|
+
**kwargs,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def check_status(self) -> RemoteStatus:
|
|
42
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
43
|
+
|
|
44
|
+
self._get_driver().verify_connectivity()
|
|
45
|
+
return RemoteStatus.up()
|
|
46
|
+
|
|
47
|
+
def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
48
|
+
records, _summary, _keys = self._get_driver().execute_query(
|
|
49
|
+
query, parameters or {}, database_=self.database
|
|
50
|
+
)
|
|
51
|
+
return [self._to_plain(record.data()) for record in records]
|
|
52
|
+
|
|
53
|
+
def _get_driver(self) -> Any:
|
|
54
|
+
if self._driver is None:
|
|
55
|
+
from neo4j import GraphDatabase
|
|
56
|
+
|
|
57
|
+
self._driver = GraphDatabase.driver(self.url, auth=(self.user, self.password))
|
|
58
|
+
return self._driver
|
|
59
|
+
|
|
60
|
+
def _to_plain(self, value: Any) -> Any:
|
|
61
|
+
# Dates and spatial values become text, so that records serialize.
|
|
62
|
+
if isinstance(value, dict):
|
|
63
|
+
return {key: self._to_plain(item) for key, item in value.items()}
|
|
64
|
+
if isinstance(value, list):
|
|
65
|
+
return [self._to_plain(item) for item in value]
|
|
66
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
67
|
+
return value
|
|
68
|
+
return str(value)
|