syrtis-python-storage 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syrtis_python_storage/__init__.py +33 -0
- syrtis_python_storage/common/__init__.py +0 -0
- syrtis_python_storage/common/abstract_document_storage.py +36 -0
- syrtis_python_storage/common/abstract_file_storage.py +36 -0
- syrtis_python_storage/common/abstract_graph_storage.py +23 -0
- syrtis_python_storage/common/abstract_storage.py +36 -0
- syrtis_python_storage/common/abstract_vector_storage.py +49 -0
- syrtis_python_storage/common/filesystem_storage.py +57 -0
- syrtis_python_storage/common/mongo_document_storage.py +93 -0
- syrtis_python_storage/common/mongo_file_storage.py +96 -0
- syrtis_python_storage/common/neo4j_graph_storage.py +68 -0
- syrtis_python_storage/common/postgres_document_storage.py +133 -0
- syrtis_python_storage/common/postgres_file_storage.py +124 -0
- syrtis_python_storage/common/postgres_vector_storage.py +162 -0
- syrtis_python_storage/common/storage_registry.py +32 -0
- syrtis_python_storage/common/vector_match.py +12 -0
- syrtis_python_storage/config_option/__init__.py +0 -0
- syrtis_python_storage/config_option/capabilities_config_option.py +15 -0
- syrtis_python_storage/config_option/credential_config_option.py +15 -0
- syrtis_python_storage/config_option/database_config_option.py +15 -0
- syrtis_python_storage/config_option/label_config_option.py +15 -0
- syrtis_python_storage/config_option/path_config_option.py +15 -0
- syrtis_python_storage/config_option/storage_config_option.py +156 -0
- syrtis_python_storage/config_option/storages_config_option.py +26 -0
- syrtis_python_storage/config_option/table_config_option.py +15 -0
- syrtis_python_storage/config_option/type_config_option.py +15 -0
- syrtis_python_storage/const/__init__.py +0 -0
- syrtis_python_storage/const/capability.py +10 -0
- syrtis_python_storage/exception/__init__.py +0 -0
- syrtis_python_storage/exception/invalid_storage_config_exception.py +17 -0
- syrtis_python_storage/exception/invalid_storage_key_exception.py +17 -0
- syrtis_python_storage/exception/no_storage_for_capability_exception.py +20 -0
- syrtis_python_storage/py.typed +0 -0
- {syrtis_python_storage-1.0.1.dist-info → syrtis_python_storage-2.0.0.dist-info}/METADATA +20 -2
- syrtis_python_storage-2.0.0.dist-info/RECORD +39 -0
- syrtis_python_storage-1.0.1.dist-info/RECORD +0 -6
- {syrtis_python_storage-1.0.1.dist-info → syrtis_python_storage-2.0.0.dist-info}/WHEEL +0 -0
- {syrtis_python_storage-1.0.1.dist-info → syrtis_python_storage-2.0.0.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
|
|
4
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
5
|
+
from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
|
|
6
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
7
|
+
from syrtis_python_storage.common.abstract_vector_storage import AbstractVectorStorage
|
|
8
|
+
from syrtis_python_storage.common.filesystem_storage import FilesystemStorage
|
|
9
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
10
|
+
from syrtis_python_storage.common.neo4j_graph_storage import Neo4jGraphStorage
|
|
11
|
+
from syrtis_python_storage.common.mongo_file_storage import MongoFileStorage
|
|
12
|
+
from syrtis_python_storage.common.postgres_document_storage import PostgresDocumentStorage
|
|
13
|
+
from syrtis_python_storage.common.postgres_file_storage import PostgresFileStorage
|
|
14
|
+
from syrtis_python_storage.common.postgres_vector_storage import PostgresVectorStorage
|
|
15
|
+
from syrtis_python_storage.common.storage_registry import StorageRegistry
|
|
16
|
+
from syrtis_python_storage.common.vector_match import VectorMatch
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"AbstractDocumentStorage",
|
|
20
|
+
"AbstractFileStorage",
|
|
21
|
+
"AbstractGraphStorage",
|
|
22
|
+
"AbstractStorage",
|
|
23
|
+
"AbstractVectorStorage",
|
|
24
|
+
"FilesystemStorage",
|
|
25
|
+
"MongoDocumentStorage",
|
|
26
|
+
"Neo4jGraphStorage",
|
|
27
|
+
"MongoFileStorage",
|
|
28
|
+
"PostgresDocumentStorage",
|
|
29
|
+
"PostgresFileStorage",
|
|
30
|
+
"PostgresVectorStorage",
|
|
31
|
+
"StorageRegistry",
|
|
32
|
+
"VectorMatch",
|
|
33
|
+
]
|
|
File without changes
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@base_class
|
|
12
|
+
class AbstractDocumentStorage(AbstractStorage):
|
|
13
|
+
"""Documents — dicts — kept by key in named collections, and found by
|
|
14
|
+
the values of their fields."""
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
18
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_DOCUMENTS
|
|
19
|
+
|
|
20
|
+
return [STORAGE_CAPABILITY_DOCUMENTS]
|
|
21
|
+
|
|
22
|
+
@abstract_method
|
|
23
|
+
def delete_document(self, collection: str, key: str) -> None:
|
|
24
|
+
"""Removes the document of the key; nothing when there is none."""
|
|
25
|
+
|
|
26
|
+
@abstract_method
|
|
27
|
+
def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
28
|
+
"""The documents whose fields hold the values given; all without filters."""
|
|
29
|
+
|
|
30
|
+
@abstract_method
|
|
31
|
+
def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
|
|
32
|
+
"""The document of the key, None when there is none."""
|
|
33
|
+
|
|
34
|
+
@abstract_method
|
|
35
|
+
def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
|
|
36
|
+
"""Writes the document under the key, replacing the one there."""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
4
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
5
|
+
|
|
6
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@base_class
|
|
10
|
+
class AbstractFileStorage(AbstractStorage):
|
|
11
|
+
"""Contents written and read back by key."""
|
|
12
|
+
|
|
13
|
+
@classmethod
|
|
14
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
15
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_FILES
|
|
16
|
+
|
|
17
|
+
return [STORAGE_CAPABILITY_FILES]
|
|
18
|
+
|
|
19
|
+
@abstract_method
|
|
20
|
+
def delete(self, key: str) -> None:
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
@abstract_method
|
|
24
|
+
def exists(self, key: str) -> bool:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
@abstract_method
|
|
28
|
+
def read(self, key: str) -> bytes:
|
|
29
|
+
pass
|
|
30
|
+
|
|
31
|
+
def read_text(self, key: str, encoding: str = "utf-8") -> str:
|
|
32
|
+
return self.read(key).decode(encoding)
|
|
33
|
+
|
|
34
|
+
@abstract_method
|
|
35
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
36
|
+
pass
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@base_class
|
|
12
|
+
class AbstractGraphStorage(AbstractStorage):
|
|
13
|
+
"""Nodes and relationships, queried in the backend's own language."""
|
|
14
|
+
|
|
15
|
+
@classmethod
|
|
16
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
17
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_GRAPH
|
|
18
|
+
|
|
19
|
+
return [STORAGE_CAPABILITY_GRAPH]
|
|
20
|
+
|
|
21
|
+
@abstract_method
|
|
22
|
+
def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
23
|
+
"""The records the query returns, as dicts; none for a pure write."""
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from wexample_helpers.classes.base_class import BaseClass
|
|
4
|
+
from wexample_helpers.classes.field import public_field
|
|
5
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
6
|
+
from wexample_remote.common.abstract_remote import AbstractRemote
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@base_class
|
|
10
|
+
class AbstractStorage(AbstractRemote, BaseClass):
|
|
11
|
+
"""A place contents are kept in — a directory, a bucket, a database —
|
|
12
|
+
whatever the backend. As a remote, it can be checked."""
|
|
13
|
+
|
|
14
|
+
capabilities: list[str] = public_field(
|
|
15
|
+
description="Kinds of content the storage accepts (e.g. `files`, `vectors`); "
|
|
16
|
+
"empty takes those of its class.",
|
|
17
|
+
factory=list,
|
|
18
|
+
)
|
|
19
|
+
label: str | None = public_field(default=None, description="Human-readable name.")
|
|
20
|
+
name: str = public_field(description="Name the storage is registered and chosen by.")
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
24
|
+
return []
|
|
25
|
+
|
|
26
|
+
def accepts(self, capability: str) -> bool:
|
|
27
|
+
return capability in self.get_capabilities()
|
|
28
|
+
|
|
29
|
+
def get_capabilities(self) -> list[str]:
|
|
30
|
+
return self.capabilities or self.get_default_capabilities()
|
|
31
|
+
|
|
32
|
+
def get_key(self) -> str:
|
|
33
|
+
return self.name
|
|
34
|
+
|
|
35
|
+
def get_label(self) -> str:
|
|
36
|
+
return self.label or self.name
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.abstract_method import abstract_method
|
|
6
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
7
|
+
|
|
8
|
+
from syrtis_python_storage.common.abstract_storage import AbstractStorage
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from syrtis_python_storage.common.vector_match import VectorMatch
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@base_class
|
|
15
|
+
class AbstractVectorStorage(AbstractStorage):
|
|
16
|
+
"""Vectors grouped in namespaces (e.g. one per document), each with a key
|
|
17
|
+
and the text it was made from, searched by cosine distance."""
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def get_default_capabilities(cls) -> list[str]:
|
|
21
|
+
from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_VECTORS
|
|
22
|
+
|
|
23
|
+
return [STORAGE_CAPABILITY_VECTORS]
|
|
24
|
+
|
|
25
|
+
@abstract_method
|
|
26
|
+
def delete_vector(self, namespace: str, key: str) -> None:
|
|
27
|
+
"""Removes the vector of the key; nothing when there is none."""
|
|
28
|
+
|
|
29
|
+
@abstract_method
|
|
30
|
+
def delete_vectors(self, namespace: str) -> None:
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@abstract_method
|
|
34
|
+
def get_vector_keys(self, namespace: str) -> set[str]:
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
@abstract_method
|
|
38
|
+
def save_vector(self, namespace: str, key: str, vector: list[float], content: str | None) -> None:
|
|
39
|
+
"""Add the vector, or replace the one of the same key."""
|
|
40
|
+
|
|
41
|
+
@abstract_method
|
|
42
|
+
def search_vectors(
|
|
43
|
+
self,
|
|
44
|
+
namespace: str,
|
|
45
|
+
vector: list[float],
|
|
46
|
+
limit: int | None = None,
|
|
47
|
+
max_distance: float | None = None,
|
|
48
|
+
) -> list[VectorMatch]:
|
|
49
|
+
"""The closest vectors first."""
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from wexample_helpers.classes.field import public_field
|
|
8
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
9
|
+
|
|
10
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
11
|
+
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@base_class
|
|
17
|
+
class FilesystemStorage(AbstractFileStorage):
|
|
18
|
+
"""Contents as files of a directory, one per key; the directory is created
|
|
19
|
+
on first write."""
|
|
20
|
+
|
|
21
|
+
path: Path = public_field(description="Directory holding the files.")
|
|
22
|
+
|
|
23
|
+
def check_status(self) -> RemoteStatus:
|
|
24
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
25
|
+
|
|
26
|
+
directory = Path(self.path)
|
|
27
|
+
# The closest existing parent is the one a first write would create in.
|
|
28
|
+
existing = next(parent for parent in (directory, *directory.parents) if parent.exists())
|
|
29
|
+
if not existing.is_dir() or not os.access(existing, os.W_OK):
|
|
30
|
+
return RemoteStatus.down(f"Not a writable directory: {existing}")
|
|
31
|
+
return RemoteStatus.up()
|
|
32
|
+
|
|
33
|
+
def delete(self, key: str) -> None:
|
|
34
|
+
self.get_file_path(key).unlink(missing_ok=True)
|
|
35
|
+
|
|
36
|
+
def exists(self, key: str) -> bool:
|
|
37
|
+
return self.get_file_path(key).is_file()
|
|
38
|
+
|
|
39
|
+
def get_file_path(self, key: str) -> Path:
|
|
40
|
+
"""The file of a key, which may not leave the directory."""
|
|
41
|
+
directory = Path(self.path).resolve()
|
|
42
|
+
file_path = (directory / key).resolve()
|
|
43
|
+
if not file_path.is_relative_to(directory):
|
|
44
|
+
from syrtis_python_storage.exception.invalid_storage_key_exception import (
|
|
45
|
+
InvalidStorageKeyException,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
raise InvalidStorageKeyException(key=key, storage_name=self.name)
|
|
49
|
+
return file_path
|
|
50
|
+
|
|
51
|
+
def read(self, key: str) -> bytes:
|
|
52
|
+
return self.get_file_path(key).read_bytes()
|
|
53
|
+
|
|
54
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
55
|
+
file_path = self.get_file_path(key)
|
|
56
|
+
file_path.parent.mkdir(parents=True, exist_ok=True)
|
|
57
|
+
file_path.write_bytes(content.encode() if isinstance(content, str) else content)
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
# The field a document's key is kept in.
|
|
15
|
+
MONGO_DOCUMENT_STORAGE_KEY_FIELD: str = "_id"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@base_class
|
|
19
|
+
class MongoDocumentStorage(AbstractDocumentStorage):
|
|
20
|
+
"""A MongoDB database, a collection per kind of document, the key as
|
|
21
|
+
`_id`. Needs the `mongo` extra. The server is reached at the host given:
|
|
22
|
+
a replica set announcing names its members alone resolve is not followed
|
|
23
|
+
(`directConnection`)."""
|
|
24
|
+
|
|
25
|
+
database: str = public_field(description="Database name.")
|
|
26
|
+
host: str = public_field(description="Server host.")
|
|
27
|
+
password: str | None = public_field(default=None, description="Password.")
|
|
28
|
+
port: int = public_field(default=27017, description="Server port.")
|
|
29
|
+
timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
|
|
30
|
+
user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
|
|
31
|
+
_client: Any = private_field(default=None, description="MongoDB client, created on first use.")
|
|
32
|
+
|
|
33
|
+
def __repr__(self) -> str:
|
|
34
|
+
# The password must never reach a log or a traceback.
|
|
35
|
+
return f"MongoDocumentStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> MongoDocumentStorage:
|
|
39
|
+
"""From a `wexample-wallet` credential of type `mongo` (host, port,
|
|
40
|
+
database, user, password), named after it unless `name` is given."""
|
|
41
|
+
return cls(
|
|
42
|
+
database=credential.get("database"),
|
|
43
|
+
host=credential.get("host"),
|
|
44
|
+
name=kwargs.pop("name", credential.name),
|
|
45
|
+
password=credential.get("password", default=None),
|
|
46
|
+
port=int(credential.get("port", default=27017)),
|
|
47
|
+
user=credential.get("user", default=None),
|
|
48
|
+
**kwargs,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
def check_status(self) -> RemoteStatus:
|
|
52
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
53
|
+
|
|
54
|
+
self._get_database().command("ping")
|
|
55
|
+
return RemoteStatus.up(f"MongoDB {self._get_client().server_info()['version']}")
|
|
56
|
+
|
|
57
|
+
def delete_document(self, collection: str, key: str) -> None:
|
|
58
|
+
self._get_database()[collection].delete_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
|
|
59
|
+
|
|
60
|
+
def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
61
|
+
return [self._to_plain(document) for document in self._get_database()[collection].find(filters or {})]
|
|
62
|
+
|
|
63
|
+
def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
|
|
64
|
+
document = self._get_database()[collection].find_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
|
|
65
|
+
return self._to_plain(document) if document is not None else None
|
|
66
|
+
|
|
67
|
+
def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
|
|
68
|
+
self._get_database()[collection].replace_one(
|
|
69
|
+
{MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
|
|
70
|
+
{**document, MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
|
|
71
|
+
upsert=True,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
def _get_client(self) -> Any:
|
|
75
|
+
if self._client is None:
|
|
76
|
+
from pymongo import MongoClient
|
|
77
|
+
|
|
78
|
+
self._client = MongoClient(
|
|
79
|
+
host=self.host,
|
|
80
|
+
port=self.port,
|
|
81
|
+
username=self.user,
|
|
82
|
+
password=self.password,
|
|
83
|
+
directConnection=True,
|
|
84
|
+
serverSelectionTimeoutMS=self.timeout * 1000,
|
|
85
|
+
)
|
|
86
|
+
return self._client
|
|
87
|
+
|
|
88
|
+
def _get_database(self) -> Any:
|
|
89
|
+
return self._get_client()[self.database]
|
|
90
|
+
|
|
91
|
+
def _to_plain(self, document: dict[str, Any]) -> dict[str, Any]:
|
|
92
|
+
"""The document without its key field, given back as the caller wrote it."""
|
|
93
|
+
return {field: value for field, value in document.items() if field != MONGO_DOCUMENT_STORAGE_KEY_FIELD}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
15
|
+
|
|
16
|
+
MONGO_FILE_STORAGE_COLLECTION_DEFAULT: str = "syrtis_storage_file"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@base_class
|
|
20
|
+
class MongoFileStorage(AbstractFileStorage):
|
|
21
|
+
"""Contents written and read back by key in a MongoDB collection — the
|
|
22
|
+
key as `_id`, the content as binary —, over the same connection as
|
|
23
|
+
`MongoDocumentStorage`. A content is one document: up to MongoDB's
|
|
24
|
+
16 MB. Needs the `mongo` extra."""
|
|
25
|
+
|
|
26
|
+
collection: str = public_field(default=MONGO_FILE_STORAGE_COLLECTION_DEFAULT, description="Collection holding the contents.")
|
|
27
|
+
database: str = public_field(description="Database name.")
|
|
28
|
+
host: str = public_field(description="Server host.")
|
|
29
|
+
password: str | None = public_field(default=None, description="Password.")
|
|
30
|
+
port: int = public_field(default=27017, description="Server port.")
|
|
31
|
+
timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
|
|
32
|
+
user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
|
|
33
|
+
_documents: Any = private_field(default=None, description="The document storage whose connection it uses, opened on first use.")
|
|
34
|
+
|
|
35
|
+
def __repr__(self) -> str:
|
|
36
|
+
# The password must never reach a log or a traceback.
|
|
37
|
+
return f"MongoFileStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> MongoFileStorage:
|
|
41
|
+
"""From a `wexample-wallet` credential of type `mongo` (host, port,
|
|
42
|
+
database, user, password), named after it unless `name` is given."""
|
|
43
|
+
return cls(
|
|
44
|
+
database=credential.get("database"),
|
|
45
|
+
host=credential.get("host"),
|
|
46
|
+
name=kwargs.pop("name", credential.name),
|
|
47
|
+
password=credential.get("password", default=None),
|
|
48
|
+
port=int(credential.get("port", default=27017)),
|
|
49
|
+
user=credential.get("user", default=None),
|
|
50
|
+
**kwargs,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def check_status(self) -> RemoteStatus:
|
|
54
|
+
return self._get_documents().check_status()
|
|
55
|
+
|
|
56
|
+
def delete(self, key: str) -> None:
|
|
57
|
+
self._get_collection().delete_one({"_id": key})
|
|
58
|
+
|
|
59
|
+
def exists(self, key: str) -> bool:
|
|
60
|
+
return self._get_collection().count_documents({"_id": key}, limit=1) > 0
|
|
61
|
+
|
|
62
|
+
def read(self, key: str) -> bytes:
|
|
63
|
+
document = self._get_collection().find_one({"_id": key})
|
|
64
|
+
if document is None:
|
|
65
|
+
raise FileNotFoundError(key)
|
|
66
|
+
|
|
67
|
+
return bytes(document["content"])
|
|
68
|
+
|
|
69
|
+
def write(self, key: str, content: bytes | str) -> None:
|
|
70
|
+
from bson.binary import Binary
|
|
71
|
+
|
|
72
|
+
self._get_collection().replace_one(
|
|
73
|
+
{"_id": key},
|
|
74
|
+
{"_id": key, "content": Binary(content.encode() if isinstance(content, str) else content)},
|
|
75
|
+
upsert=True,
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def _get_collection(self) -> Any:
|
|
79
|
+
return self._get_documents()._get_database()[self.collection]
|
|
80
|
+
|
|
81
|
+
def _get_documents(self) -> MongoDocumentStorage:
|
|
82
|
+
"""The connection, as the document storage opens it."""
|
|
83
|
+
if self._documents is None:
|
|
84
|
+
from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
|
|
85
|
+
|
|
86
|
+
self._documents = MongoDocumentStorage(
|
|
87
|
+
name=self.name,
|
|
88
|
+
database=self.database,
|
|
89
|
+
host=self.host,
|
|
90
|
+
password=self.password,
|
|
91
|
+
port=self.port,
|
|
92
|
+
timeout=self.timeout,
|
|
93
|
+
user=self.user,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
return self._documents
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING, Any
|
|
4
|
+
|
|
5
|
+
from wexample_helpers.classes.field import public_field
|
|
6
|
+
from wexample_helpers.classes.private_field import private_field
|
|
7
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
8
|
+
|
|
9
|
+
from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@base_class
|
|
16
|
+
class Neo4jGraphStorage(AbstractGraphStorage):
|
|
17
|
+
"""A Neo4j database, queried in Cypher. Needs the `neo4j` extra."""
|
|
18
|
+
|
|
19
|
+
database: str | None = public_field(default=None, description="Database; the server's default when None.")
|
|
20
|
+
password: str = public_field(description="Password.")
|
|
21
|
+
url: str = public_field(description="Bolt url, e.g. `bolt://localhost:7687`.")
|
|
22
|
+
user: str = public_field(description="User name.")
|
|
23
|
+
_driver: Any = private_field(default=None, description="Neo4j driver, created on first use.")
|
|
24
|
+
|
|
25
|
+
def __repr__(self) -> str:
|
|
26
|
+
# The password must never reach a log or a traceback.
|
|
27
|
+
return f"Neo4jGraphStorage(name={self.name!r}, url={self.url!r})"
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> Neo4jGraphStorage:
|
|
31
|
+
"""From a `wexample-wallet` credential of type `neo4j` (url, user,
|
|
32
|
+
password), named after it unless `name` is given."""
|
|
33
|
+
return cls(
|
|
34
|
+
name=kwargs.pop("name", credential.name),
|
|
35
|
+
password=credential.get("password"),
|
|
36
|
+
url=credential.get("url"),
|
|
37
|
+
user=credential.get("user"),
|
|
38
|
+
**kwargs,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def check_status(self) -> RemoteStatus:
|
|
42
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
43
|
+
|
|
44
|
+
self._get_driver().verify_connectivity()
|
|
45
|
+
return RemoteStatus.up()
|
|
46
|
+
|
|
47
|
+
def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
48
|
+
records, _summary, _keys = self._get_driver().execute_query(
|
|
49
|
+
query, parameters or {}, database_=self.database
|
|
50
|
+
)
|
|
51
|
+
return [self._to_plain(record.data()) for record in records]
|
|
52
|
+
|
|
53
|
+
def _get_driver(self) -> Any:
|
|
54
|
+
if self._driver is None:
|
|
55
|
+
from neo4j import GraphDatabase
|
|
56
|
+
|
|
57
|
+
self._driver = GraphDatabase.driver(self.url, auth=(self.user, self.password))
|
|
58
|
+
return self._driver
|
|
59
|
+
|
|
60
|
+
def _to_plain(self, value: Any) -> Any:
|
|
61
|
+
# Dates and spatial values become text, so that records serialize.
|
|
62
|
+
if isinstance(value, dict):
|
|
63
|
+
return {key: self._to_plain(item) for key, item in value.items()}
|
|
64
|
+
if isinstance(value, list):
|
|
65
|
+
return [self._to_plain(item) for item in value]
|
|
66
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
67
|
+
return value
|
|
68
|
+
return str(value)
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from wexample_helpers.classes.field import public_field
|
|
8
|
+
from wexample_helpers.classes.private_field import private_field
|
|
9
|
+
from wexample_helpers.decorator.base_class import base_class
|
|
10
|
+
|
|
11
|
+
from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
15
|
+
|
|
16
|
+
POSTGRES_DOCUMENT_STORAGE_TABLE_DEFAULT: str = "syrtis_storage_document"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@base_class
|
|
20
|
+
class PostgresDocumentStorage(AbstractDocumentStorage):
|
|
21
|
+
"""Documents in a PostgreSQL table — a row per collection and key, the
|
|
22
|
+
document as JSONB, indexed for containment (GIN) —, created on first
|
|
23
|
+
use. Found by the values of their fields (`@>`). Needs the `postgres`
|
|
24
|
+
extra."""
|
|
25
|
+
|
|
26
|
+
database: str = public_field(description="Database name.")
|
|
27
|
+
host: str = public_field(description="Server host.")
|
|
28
|
+
password: str | None = public_field(default=None, description="Password.")
|
|
29
|
+
port: int = public_field(default=5432, description="Server port.")
|
|
30
|
+
table: str = public_field(default=POSTGRES_DOCUMENT_STORAGE_TABLE_DEFAULT, description="Table holding the documents.")
|
|
31
|
+
user: str = public_field(description="User name.")
|
|
32
|
+
_engine: Any = private_field(default=None, description="SQLAlchemy engine, created on first use.")
|
|
33
|
+
_table_ready: bool = private_field(default=False, description="Whether the table exists.")
|
|
34
|
+
|
|
35
|
+
def __repr__(self) -> str:
|
|
36
|
+
# The password must never reach a log or a traceback.
|
|
37
|
+
return f"PostgresDocumentStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def from_credential(cls, credential: Any, **kwargs: Any) -> PostgresDocumentStorage:
|
|
41
|
+
"""From a `wexample-wallet` credential of type `postgres` (host, port,
|
|
42
|
+
database, user, password), named after it unless `name` is given."""
|
|
43
|
+
return cls(
|
|
44
|
+
database=credential.get("database"),
|
|
45
|
+
host=credential.get("host"),
|
|
46
|
+
name=kwargs.pop("name", credential.name),
|
|
47
|
+
password=credential.get("password", default=None),
|
|
48
|
+
port=int(credential.get("port", default=5432)),
|
|
49
|
+
user=credential.get("user"),
|
|
50
|
+
**kwargs,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
def check_status(self) -> RemoteStatus:
|
|
54
|
+
from sqlalchemy import text
|
|
55
|
+
from wexample_remote.common.remote_status import RemoteStatus
|
|
56
|
+
|
|
57
|
+
with self._get_engine().connect() as connection:
|
|
58
|
+
connection.execute(text("SELECT 1"))
|
|
59
|
+
|
|
60
|
+
return RemoteStatus.up()
|
|
61
|
+
|
|
62
|
+
def delete_document(self, collection: str, key: str) -> None:
|
|
63
|
+
self._execute(f"DELETE FROM {self._get_table()} WHERE collection = :collection AND key = :key", collection=collection, key=key)
|
|
64
|
+
|
|
65
|
+
def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
|
66
|
+
rows = self._execute(
|
|
67
|
+
f"SELECT document FROM {self._get_table()} WHERE collection = :collection AND document @> CAST(:filters AS jsonb) ORDER BY key",
|
|
68
|
+
collection=collection,
|
|
69
|
+
filters=json.dumps(filters or {}),
|
|
70
|
+
)
|
|
71
|
+
return [row.document for row in rows]
|
|
72
|
+
|
|
73
|
+
def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
|
|
74
|
+
rows = self._execute(
|
|
75
|
+
f"SELECT document FROM {self._get_table()} WHERE collection = :collection AND key = :key", collection=collection, key=key
|
|
76
|
+
)
|
|
77
|
+
return rows[0].document if rows else None
|
|
78
|
+
|
|
79
|
+
def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
|
|
80
|
+
self._execute(
|
|
81
|
+
f"INSERT INTO {self._get_table()} (collection, key, document) VALUES (:collection, :key, CAST(:document AS jsonb)) "
|
|
82
|
+
"ON CONFLICT (collection, key) DO UPDATE SET document = EXCLUDED.document",
|
|
83
|
+
collection=collection,
|
|
84
|
+
document=json.dumps(document),
|
|
85
|
+
key=key,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def _ensure_table(self) -> None:
|
|
89
|
+
from sqlalchemy import text
|
|
90
|
+
|
|
91
|
+
if self._table_ready:
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
table = self._get_table()
|
|
95
|
+
with self._get_engine().begin() as connection:
|
|
96
|
+
connection.execute(
|
|
97
|
+
text(f"CREATE TABLE IF NOT EXISTS {table} (collection TEXT NOT NULL, key TEXT NOT NULL, document JSONB NOT NULL, PRIMARY KEY (collection, key))")
|
|
98
|
+
)
|
|
99
|
+
connection.execute(text(f"CREATE INDEX IF NOT EXISTS {table}_document ON {table} USING GIN (document jsonb_path_ops)"))
|
|
100
|
+
self._table_ready = True
|
|
101
|
+
|
|
102
|
+
def _execute(self, statement: str, **parameters: Any) -> list[Any]:
|
|
103
|
+
from sqlalchemy import text
|
|
104
|
+
|
|
105
|
+
self._ensure_table()
|
|
106
|
+
with self._get_engine().begin() as connection:
|
|
107
|
+
result = connection.execute(text(statement), parameters)
|
|
108
|
+
return list(result) if result.returns_rows else []
|
|
109
|
+
|
|
110
|
+
def _get_engine(self) -> Any:
|
|
111
|
+
if self._engine is None:
|
|
112
|
+
from sqlalchemy import create_engine
|
|
113
|
+
from sqlalchemy.engine import URL
|
|
114
|
+
|
|
115
|
+
self._engine = create_engine(
|
|
116
|
+
URL.create(
|
|
117
|
+
"postgresql+psycopg2",
|
|
118
|
+
database=self.database,
|
|
119
|
+
host=self.host,
|
|
120
|
+
password=self.password,
|
|
121
|
+
port=int(self.port),
|
|
122
|
+
username=self.user,
|
|
123
|
+
)
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
return self._engine
|
|
127
|
+
|
|
128
|
+
def _get_table(self) -> str:
|
|
129
|
+
# Interpolated in the statements: an identifier only.
|
|
130
|
+
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", self.table):
|
|
131
|
+
raise ValueError(f"Invalid table name: {self.table!r}")
|
|
132
|
+
|
|
133
|
+
return self.table
|