syrtis-python-storage 1.0.1__tar.gz → 2.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. syrtis_python_storage-1.0.1/README.md → syrtis_python_storage-2.0.0/PKG-INFO +33 -1
  2. syrtis_python_storage-1.0.1/PKG-INFO → syrtis_python_storage-2.0.0/README.md +8 -15
  3. {syrtis_python_storage-1.0.1 → syrtis_python_storage-2.0.0}/pyproject.toml +17 -2
  4. syrtis_python_storage-2.0.0/src/syrtis_python_storage/__init__.py +33 -0
  5. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_document_storage.py +36 -0
  6. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_file_storage.py +36 -0
  7. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_graph_storage.py +23 -0
  8. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_storage.py +36 -0
  9. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/abstract_vector_storage.py +49 -0
  10. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/filesystem_storage.py +57 -0
  11. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/mongo_document_storage.py +93 -0
  12. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/mongo_file_storage.py +96 -0
  13. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/neo4j_graph_storage.py +68 -0
  14. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_document_storage.py +133 -0
  15. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_file_storage.py +124 -0
  16. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/postgres_vector_storage.py +162 -0
  17. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/storage_registry.py +32 -0
  18. syrtis_python_storage-2.0.0/src/syrtis_python_storage/common/vector_match.py +12 -0
  19. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/capabilities_config_option.py +15 -0
  20. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/credential_config_option.py +15 -0
  21. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/database_config_option.py +15 -0
  22. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/label_config_option.py +15 -0
  23. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/path_config_option.py +15 -0
  24. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/storage_config_option.py +156 -0
  25. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/storages_config_option.py +26 -0
  26. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/table_config_option.py +15 -0
  27. syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/type_config_option.py +15 -0
  28. syrtis_python_storage-2.0.0/src/syrtis_python_storage/const/capability.py +10 -0
  29. syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/__init__.py +0 -0
  30. syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/invalid_storage_config_exception.py +17 -0
  31. syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/invalid_storage_key_exception.py +17 -0
  32. syrtis_python_storage-2.0.0/src/syrtis_python_storage/exception/no_storage_for_capability_exception.py +20 -0
  33. syrtis_python_storage-2.0.0/src/syrtis_python_storage/py.typed +0 -0
  34. syrtis_python_storage-2.0.0/src/syrtis_syrtis_python_storage/__init__.py +0 -0
  35. syrtis_python_storage-2.0.0/src/syrtis_syrtis_python_storage/py.typed +0 -0
  36. syrtis_python_storage-2.0.0/tests/.gitkeep +0 -0
  37. syrtis_python_storage-2.0.0/tests/test_storage.py +229 -0
  38. {syrtis_python_storage-1.0.1/src/syrtis_syrtis_python_storage → syrtis_python_storage-2.0.0/src/syrtis_python_storage/common}/__init__.py +0 -0
  39. /syrtis_python_storage-1.0.1/src/syrtis_syrtis_python_storage/py.typed → /syrtis_python_storage-2.0.0/src/syrtis_python_storage/config_option/__init__.py +0 -0
  40. /syrtis_python_storage-1.0.1/tests/.gitkeep → /syrtis_python_storage-2.0.0/src/syrtis_python_storage/const/__init__.py +0 -0
@@ -1,15 +1,47 @@
1
+ Metadata-Version: 2.1
2
+ Name: syrtis-python-storage
3
+ Version: 2.0.0
4
+ Author-Email: weeger <contact@syrtis.ai>
5
+ License: MIT
6
+ Classifier: Programming Language :: Python :: 3
7
+ Classifier: License :: OSI Approved :: MIT License
8
+ Classifier: Operating System :: OS Independent
9
+ Requires-Python: >=3.10
10
+ Requires-Dist: wexample-config
11
+ Requires-Dist: wexample-helpers>=20.8.0
12
+ Requires-Dist: wexample-remote>=1.2.0
13
+ Provides-Extra: mongo
14
+ Requires-Dist: pymongo>=4.6; extra == "mongo"
15
+ Provides-Extra: neo4j
16
+ Requires-Dist: neo4j>=5; extra == "neo4j"
17
+ Provides-Extra: postgres
18
+ Requires-Dist: psycopg2-binary; extra == "postgres"
19
+ Requires-Dist: sqlalchemy>=2; extra == "postgres"
20
+ Provides-Extra: dev
21
+ Requires-Dist: pytest; extra == "dev"
22
+ Requires-Dist: pytest-cov; extra == "dev"
23
+ Requires-Dist: wexample-wallet; extra == "dev"
24
+ Description-Content-Type: text/markdown
25
+
1
26
  # python_storage
2
27
 
3
- Version: 1.0.1
28
+ Version: 2.0.0
4
29
 
5
30
  ## Table of Contents
6
31
 
32
+ - [Introduction](#introduction)
7
33
  - [Install](#install)
8
34
  - [Tests](#tests)
9
35
  - [Roadmap](#roadmap)
10
36
  - [Status Compatibility](#status-compatibility)
11
37
  - [Useful Links](#useful-links)
12
38
 
39
+ `syrtis-python-storage` gives an application the places it keeps contents in, behind one interface per kind of content: files written and read back by key, vectors searched by similarity, graphs queried in their own language, documents kept in collections and found by their fields. Each storage has a name and the capabilities it accepts (`files`, `vectors`, `graph`), and a registry picks the storage for a capability — so the code needing a place to write never chooses the backend itself.
40
+
41
+ Every storage is a `wexample-remote` remote, so it can be checked like any other service. Backends: a local directory, PostgreSQL with pgvector (the `postgres` extra), Neo4j (the `neo4j` extra), MongoDB (the `mongo` extra).
42
+
43
+ Its PHP twin is `syrtis/php-storage`: the same contracts on the same tables and collections, so that what one language wrote the other reads.
44
+
13
45
  ## Installation
14
46
 
15
47
  ```bash
@@ -1,29 +1,22 @@
1
- Metadata-Version: 2.1
2
- Name: syrtis-python-storage
3
- Version: 1.0.1
4
- Author-Email: weeger <contact@syrtis.ai>
5
- License: MIT
6
- Classifier: Programming Language :: Python :: 3
7
- Classifier: License :: OSI Approved :: MIT License
8
- Classifier: Operating System :: OS Independent
9
- Requires-Python: >=3.10
10
- Provides-Extra: dev
11
- Requires-Dist: pytest; extra == "dev"
12
- Requires-Dist: pytest-cov; extra == "dev"
13
- Description-Content-Type: text/markdown
14
-
15
1
  # python_storage
16
2
 
17
- Version: 1.0.1
3
+ Version: 2.0.0
18
4
 
19
5
  ## Table of Contents
20
6
 
7
+ - [Introduction](#introduction)
21
8
  - [Install](#install)
22
9
  - [Tests](#tests)
23
10
  - [Roadmap](#roadmap)
24
11
  - [Status Compatibility](#status-compatibility)
25
12
  - [Useful Links](#useful-links)
26
13
 
14
+ `syrtis-python-storage` gives an application the places it keeps contents in, behind one interface per kind of content: files written and read back by key, vectors searched by similarity, graphs queried in their own language, documents kept in collections and found by their fields. Each storage has a name and the capabilities it accepts (`files`, `vectors`, `graph`), and a registry picks the storage for a capability — so the code needing a place to write never chooses the backend itself.
15
+
16
+ Every storage is a `wexample-remote` remote, so it can be checked like any other service. Backends: a local directory, PostgreSQL with pgvector (the `postgres` extra), Neo4j (the `neo4j` extra), MongoDB (the `mongo` extra).
17
+
18
+ Its PHP twin is `syrtis/php-storage`: the same contracts on the same tables and collections, so that what one language wrote the other reads.
19
+
27
20
  ## Installation
28
21
 
29
22
  ```bash
@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
6
6
 
7
7
  [project]
8
8
  name = "syrtis-python-storage"
9
- version = "1.0.1"
9
+ version = "2.0.0"
10
10
  authors = [
11
11
  { name = "weeger", email = "contact@syrtis.ai" },
12
12
  ]
@@ -16,7 +16,11 @@ classifiers = [
16
16
  "License :: OSI Approved :: MIT License",
17
17
  "Operating System :: OS Independent",
18
18
  ]
19
- dependencies = []
19
+ dependencies = [
20
+ "wexample-config",
21
+ "wexample-helpers>=20.8.0",
22
+ "wexample-remote>=1.2.0",
23
+ ]
20
24
 
21
25
  [project.readme]
22
26
  file = "README.md"
@@ -26,9 +30,20 @@ content-type = "text/markdown"
26
30
  text = "MIT"
27
31
 
28
32
  [project.optional-dependencies]
33
+ mongo = [
34
+ "pymongo>=4.6",
35
+ ]
36
+ neo4j = [
37
+ "neo4j>=5",
38
+ ]
39
+ postgres = [
40
+ "psycopg2-binary",
41
+ "sqlalchemy>=2",
42
+ ]
29
43
  dev = [
30
44
  "pytest",
31
45
  "pytest-cov",
46
+ "wexample-wallet",
32
47
  ]
33
48
 
34
49
  [tool.setuptools.packages.find]
@@ -0,0 +1,33 @@
1
+ from __future__ import annotations
2
+
3
+ from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
4
+ from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
5
+ from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
6
+ from syrtis_python_storage.common.abstract_storage import AbstractStorage
7
+ from syrtis_python_storage.common.abstract_vector_storage import AbstractVectorStorage
8
+ from syrtis_python_storage.common.filesystem_storage import FilesystemStorage
9
+ from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
10
+ from syrtis_python_storage.common.neo4j_graph_storage import Neo4jGraphStorage
11
+ from syrtis_python_storage.common.mongo_file_storage import MongoFileStorage
12
+ from syrtis_python_storage.common.postgres_document_storage import PostgresDocumentStorage
13
+ from syrtis_python_storage.common.postgres_file_storage import PostgresFileStorage
14
+ from syrtis_python_storage.common.postgres_vector_storage import PostgresVectorStorage
15
+ from syrtis_python_storage.common.storage_registry import StorageRegistry
16
+ from syrtis_python_storage.common.vector_match import VectorMatch
17
+
18
+ __all__ = [
19
+ "AbstractDocumentStorage",
20
+ "AbstractFileStorage",
21
+ "AbstractGraphStorage",
22
+ "AbstractStorage",
23
+ "AbstractVectorStorage",
24
+ "FilesystemStorage",
25
+ "MongoDocumentStorage",
26
+ "Neo4jGraphStorage",
27
+ "MongoFileStorage",
28
+ "PostgresDocumentStorage",
29
+ "PostgresFileStorage",
30
+ "PostgresVectorStorage",
31
+ "StorageRegistry",
32
+ "VectorMatch",
33
+ ]
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ from wexample_helpers.classes.abstract_method import abstract_method
6
+ from wexample_helpers.decorator.base_class import base_class
7
+
8
+ from syrtis_python_storage.common.abstract_storage import AbstractStorage
9
+
10
+
11
+ @base_class
12
+ class AbstractDocumentStorage(AbstractStorage):
13
+ """Documents — dicts — kept by key in named collections, and found by
14
+ the values of their fields."""
15
+
16
+ @classmethod
17
+ def get_default_capabilities(cls) -> list[str]:
18
+ from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_DOCUMENTS
19
+
20
+ return [STORAGE_CAPABILITY_DOCUMENTS]
21
+
22
+ @abstract_method
23
+ def delete_document(self, collection: str, key: str) -> None:
24
+ """Removes the document of the key; nothing when there is none."""
25
+
26
+ @abstract_method
27
+ def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
28
+ """The documents whose fields hold the values given; all without filters."""
29
+
30
+ @abstract_method
31
+ def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
32
+ """The document of the key, None when there is none."""
33
+
34
+ @abstract_method
35
+ def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
36
+ """Writes the document under the key, replacing the one there."""
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+
3
+ from wexample_helpers.classes.abstract_method import abstract_method
4
+ from wexample_helpers.decorator.base_class import base_class
5
+
6
+ from syrtis_python_storage.common.abstract_storage import AbstractStorage
7
+
8
+
9
+ @base_class
10
+ class AbstractFileStorage(AbstractStorage):
11
+ """Contents written and read back by key."""
12
+
13
+ @classmethod
14
+ def get_default_capabilities(cls) -> list[str]:
15
+ from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_FILES
16
+
17
+ return [STORAGE_CAPABILITY_FILES]
18
+
19
+ @abstract_method
20
+ def delete(self, key: str) -> None:
21
+ pass
22
+
23
+ @abstract_method
24
+ def exists(self, key: str) -> bool:
25
+ pass
26
+
27
+ @abstract_method
28
+ def read(self, key: str) -> bytes:
29
+ pass
30
+
31
+ def read_text(self, key: str, encoding: str = "utf-8") -> str:
32
+ return self.read(key).decode(encoding)
33
+
34
+ @abstract_method
35
+ def write(self, key: str, content: bytes | str) -> None:
36
+ pass
@@ -0,0 +1,23 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ from wexample_helpers.classes.abstract_method import abstract_method
6
+ from wexample_helpers.decorator.base_class import base_class
7
+
8
+ from syrtis_python_storage.common.abstract_storage import AbstractStorage
9
+
10
+
11
+ @base_class
12
+ class AbstractGraphStorage(AbstractStorage):
13
+ """Nodes and relationships, queried in the backend's own language."""
14
+
15
+ @classmethod
16
+ def get_default_capabilities(cls) -> list[str]:
17
+ from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_GRAPH
18
+
19
+ return [STORAGE_CAPABILITY_GRAPH]
20
+
21
+ @abstract_method
22
+ def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
23
+ """The records the query returns, as dicts; none for a pure write."""
@@ -0,0 +1,36 @@
1
+ from __future__ import annotations
2
+
3
+ from wexample_helpers.classes.base_class import BaseClass
4
+ from wexample_helpers.classes.field import public_field
5
+ from wexample_helpers.decorator.base_class import base_class
6
+ from wexample_remote.common.abstract_remote import AbstractRemote
7
+
8
+
9
+ @base_class
10
+ class AbstractStorage(AbstractRemote, BaseClass):
11
+ """A place contents are kept in — a directory, a bucket, a database —
12
+ whatever the backend. As a remote, it can be checked."""
13
+
14
+ capabilities: list[str] = public_field(
15
+ description="Kinds of content the storage accepts (e.g. `files`, `vectors`); "
16
+ "empty takes those of its class.",
17
+ factory=list,
18
+ )
19
+ label: str | None = public_field(default=None, description="Human-readable name.")
20
+ name: str = public_field(description="Name the storage is registered and chosen by.")
21
+
22
+ @classmethod
23
+ def get_default_capabilities(cls) -> list[str]:
24
+ return []
25
+
26
+ def accepts(self, capability: str) -> bool:
27
+ return capability in self.get_capabilities()
28
+
29
+ def get_capabilities(self) -> list[str]:
30
+ return self.capabilities or self.get_default_capabilities()
31
+
32
+ def get_key(self) -> str:
33
+ return self.name
34
+
35
+ def get_label(self) -> str:
36
+ return self.label or self.name
@@ -0,0 +1,49 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING
4
+
5
+ from wexample_helpers.classes.abstract_method import abstract_method
6
+ from wexample_helpers.decorator.base_class import base_class
7
+
8
+ from syrtis_python_storage.common.abstract_storage import AbstractStorage
9
+
10
+ if TYPE_CHECKING:
11
+ from syrtis_python_storage.common.vector_match import VectorMatch
12
+
13
+
14
+ @base_class
15
+ class AbstractVectorStorage(AbstractStorage):
16
+ """Vectors grouped in namespaces (e.g. one per document), each with a key
17
+ and the text it was made from, searched by cosine distance."""
18
+
19
+ @classmethod
20
+ def get_default_capabilities(cls) -> list[str]:
21
+ from syrtis_python_storage.const.capability import STORAGE_CAPABILITY_VECTORS
22
+
23
+ return [STORAGE_CAPABILITY_VECTORS]
24
+
25
+ @abstract_method
26
+ def delete_vector(self, namespace: str, key: str) -> None:
27
+ """Removes the vector of the key; nothing when there is none."""
28
+
29
+ @abstract_method
30
+ def delete_vectors(self, namespace: str) -> None:
31
+ pass
32
+
33
+ @abstract_method
34
+ def get_vector_keys(self, namespace: str) -> set[str]:
35
+ pass
36
+
37
+ @abstract_method
38
+ def save_vector(self, namespace: str, key: str, vector: list[float], content: str | None) -> None:
39
+ """Add the vector, or replace the one of the same key."""
40
+
41
+ @abstract_method
42
+ def search_vectors(
43
+ self,
44
+ namespace: str,
45
+ vector: list[float],
46
+ limit: int | None = None,
47
+ max_distance: float | None = None,
48
+ ) -> list[VectorMatch]:
49
+ """The closest vectors first."""
@@ -0,0 +1,57 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+ from pathlib import Path
5
+ from typing import TYPE_CHECKING
6
+
7
+ from wexample_helpers.classes.field import public_field
8
+ from wexample_helpers.decorator.base_class import base_class
9
+
10
+ from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
11
+
12
+ if TYPE_CHECKING:
13
+ from wexample_remote.common.remote_status import RemoteStatus
14
+
15
+
16
+ @base_class
17
+ class FilesystemStorage(AbstractFileStorage):
18
+ """Contents as files of a directory, one per key; the directory is created
19
+ on first write."""
20
+
21
+ path: Path = public_field(description="Directory holding the files.")
22
+
23
+ def check_status(self) -> RemoteStatus:
24
+ from wexample_remote.common.remote_status import RemoteStatus
25
+
26
+ directory = Path(self.path)
27
+ # The closest existing parent is the one a first write would create in.
28
+ existing = next(parent for parent in (directory, *directory.parents) if parent.exists())
29
+ if not existing.is_dir() or not os.access(existing, os.W_OK):
30
+ return RemoteStatus.down(f"Not a writable directory: {existing}")
31
+ return RemoteStatus.up()
32
+
33
+ def delete(self, key: str) -> None:
34
+ self.get_file_path(key).unlink(missing_ok=True)
35
+
36
+ def exists(self, key: str) -> bool:
37
+ return self.get_file_path(key).is_file()
38
+
39
+ def get_file_path(self, key: str) -> Path:
40
+ """The file of a key, which may not leave the directory."""
41
+ directory = Path(self.path).resolve()
42
+ file_path = (directory / key).resolve()
43
+ if not file_path.is_relative_to(directory):
44
+ from syrtis_python_storage.exception.invalid_storage_key_exception import (
45
+ InvalidStorageKeyException,
46
+ )
47
+
48
+ raise InvalidStorageKeyException(key=key, storage_name=self.name)
49
+ return file_path
50
+
51
+ def read(self, key: str) -> bytes:
52
+ return self.get_file_path(key).read_bytes()
53
+
54
+ def write(self, key: str, content: bytes | str) -> None:
55
+ file_path = self.get_file_path(key)
56
+ file_path.parent.mkdir(parents=True, exist_ok=True)
57
+ file_path.write_bytes(content.encode() if isinstance(content, str) else content)
@@ -0,0 +1,93 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ from wexample_helpers.classes.field import public_field
6
+ from wexample_helpers.classes.private_field import private_field
7
+ from wexample_helpers.decorator.base_class import base_class
8
+
9
+ from syrtis_python_storage.common.abstract_document_storage import AbstractDocumentStorage
10
+
11
+ if TYPE_CHECKING:
12
+ from wexample_remote.common.remote_status import RemoteStatus
13
+
14
+ # The field a document's key is kept in.
15
+ MONGO_DOCUMENT_STORAGE_KEY_FIELD: str = "_id"
16
+
17
+
18
+ @base_class
19
+ class MongoDocumentStorage(AbstractDocumentStorage):
20
+ """A MongoDB database, a collection per kind of document, the key as
21
+ `_id`. Needs the `mongo` extra. The server is reached at the host given:
22
+ a replica set announcing names its members alone resolve is not followed
23
+ (`directConnection`)."""
24
+
25
+ database: str = public_field(description="Database name.")
26
+ host: str = public_field(description="Server host.")
27
+ password: str | None = public_field(default=None, description="Password.")
28
+ port: int = public_field(default=27017, description="Server port.")
29
+ timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
30
+ user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
31
+ _client: Any = private_field(default=None, description="MongoDB client, created on first use.")
32
+
33
+ def __repr__(self) -> str:
34
+ # The password must never reach a log or a traceback.
35
+ return f"MongoDocumentStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
36
+
37
+ @classmethod
38
+ def from_credential(cls, credential: Any, **kwargs: Any) -> MongoDocumentStorage:
39
+ """From a `wexample-wallet` credential of type `mongo` (host, port,
40
+ database, user, password), named after it unless `name` is given."""
41
+ return cls(
42
+ database=credential.get("database"),
43
+ host=credential.get("host"),
44
+ name=kwargs.pop("name", credential.name),
45
+ password=credential.get("password", default=None),
46
+ port=int(credential.get("port", default=27017)),
47
+ user=credential.get("user", default=None),
48
+ **kwargs,
49
+ )
50
+
51
+ def check_status(self) -> RemoteStatus:
52
+ from wexample_remote.common.remote_status import RemoteStatus
53
+
54
+ self._get_database().command("ping")
55
+ return RemoteStatus.up(f"MongoDB {self._get_client().server_info()['version']}")
56
+
57
+ def delete_document(self, collection: str, key: str) -> None:
58
+ self._get_database()[collection].delete_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
59
+
60
+ def find_documents(self, collection: str, filters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
61
+ return [self._to_plain(document) for document in self._get_database()[collection].find(filters or {})]
62
+
63
+ def get_document(self, collection: str, key: str) -> dict[str, Any] | None:
64
+ document = self._get_database()[collection].find_one({MONGO_DOCUMENT_STORAGE_KEY_FIELD: key})
65
+ return self._to_plain(document) if document is not None else None
66
+
67
+ def save_document(self, collection: str, key: str, document: dict[str, Any]) -> None:
68
+ self._get_database()[collection].replace_one(
69
+ {MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
70
+ {**document, MONGO_DOCUMENT_STORAGE_KEY_FIELD: key},
71
+ upsert=True,
72
+ )
73
+
74
+ def _get_client(self) -> Any:
75
+ if self._client is None:
76
+ from pymongo import MongoClient
77
+
78
+ self._client = MongoClient(
79
+ host=self.host,
80
+ port=self.port,
81
+ username=self.user,
82
+ password=self.password,
83
+ directConnection=True,
84
+ serverSelectionTimeoutMS=self.timeout * 1000,
85
+ )
86
+ return self._client
87
+
88
+ def _get_database(self) -> Any:
89
+ return self._get_client()[self.database]
90
+
91
+ def _to_plain(self, document: dict[str, Any]) -> dict[str, Any]:
92
+ """The document without its key field, given back as the caller wrote it."""
93
+ return {field: value for field, value in document.items() if field != MONGO_DOCUMENT_STORAGE_KEY_FIELD}
@@ -0,0 +1,96 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ from wexample_helpers.classes.field import public_field
6
+ from wexample_helpers.classes.private_field import private_field
7
+ from wexample_helpers.decorator.base_class import base_class
8
+
9
+ from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
10
+
11
+ if TYPE_CHECKING:
12
+ from wexample_remote.common.remote_status import RemoteStatus
13
+
14
+ from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
15
+
16
+ MONGO_FILE_STORAGE_COLLECTION_DEFAULT: str = "syrtis_storage_file"
17
+
18
+
19
+ @base_class
20
+ class MongoFileStorage(AbstractFileStorage):
21
+ """Contents written and read back by key in a MongoDB collection — the
22
+ key as `_id`, the content as binary —, over the same connection as
23
+ `MongoDocumentStorage`. A content is one document: up to MongoDB's
24
+ 16 MB. Needs the `mongo` extra."""
25
+
26
+ collection: str = public_field(default=MONGO_FILE_STORAGE_COLLECTION_DEFAULT, description="Collection holding the contents.")
27
+ database: str = public_field(description="Database name.")
28
+ host: str = public_field(description="Server host.")
29
+ password: str | None = public_field(default=None, description="Password.")
30
+ port: int = public_field(default=27017, description="Server port.")
31
+ timeout: int = public_field(default=10, description="Seconds a server is given to answer.")
32
+ user: str | None = public_field(default=None, description="User name; none for a server without authentication.")
33
+ _documents: Any = private_field(default=None, description="The document storage whose connection it uses, opened on first use.")
34
+
35
+ def __repr__(self) -> str:
36
+ # The password must never reach a log or a traceback.
37
+ return f"MongoFileStorage(name={self.name!r}, host={self.host!r}, database={self.database!r})"
38
+
39
+ @classmethod
40
+ def from_credential(cls, credential: Any, **kwargs: Any) -> MongoFileStorage:
41
+ """From a `wexample-wallet` credential of type `mongo` (host, port,
42
+ database, user, password), named after it unless `name` is given."""
43
+ return cls(
44
+ database=credential.get("database"),
45
+ host=credential.get("host"),
46
+ name=kwargs.pop("name", credential.name),
47
+ password=credential.get("password", default=None),
48
+ port=int(credential.get("port", default=27017)),
49
+ user=credential.get("user", default=None),
50
+ **kwargs,
51
+ )
52
+
53
+ def check_status(self) -> RemoteStatus:
54
+ return self._get_documents().check_status()
55
+
56
+ def delete(self, key: str) -> None:
57
+ self._get_collection().delete_one({"_id": key})
58
+
59
+ def exists(self, key: str) -> bool:
60
+ return self._get_collection().count_documents({"_id": key}, limit=1) > 0
61
+
62
+ def read(self, key: str) -> bytes:
63
+ document = self._get_collection().find_one({"_id": key})
64
+ if document is None:
65
+ raise FileNotFoundError(key)
66
+
67
+ return bytes(document["content"])
68
+
69
+ def write(self, key: str, content: bytes | str) -> None:
70
+ from bson.binary import Binary
71
+
72
+ self._get_collection().replace_one(
73
+ {"_id": key},
74
+ {"_id": key, "content": Binary(content.encode() if isinstance(content, str) else content)},
75
+ upsert=True,
76
+ )
77
+
78
+ def _get_collection(self) -> Any:
79
+ return self._get_documents()._get_database()[self.collection]
80
+
81
+ def _get_documents(self) -> MongoDocumentStorage:
82
+ """The connection, as the document storage opens it."""
83
+ if self._documents is None:
84
+ from syrtis_python_storage.common.mongo_document_storage import MongoDocumentStorage
85
+
86
+ self._documents = MongoDocumentStorage(
87
+ name=self.name,
88
+ database=self.database,
89
+ host=self.host,
90
+ password=self.password,
91
+ port=self.port,
92
+ timeout=self.timeout,
93
+ user=self.user,
94
+ )
95
+
96
+ return self._documents
@@ -0,0 +1,68 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import TYPE_CHECKING, Any
4
+
5
+ from wexample_helpers.classes.field import public_field
6
+ from wexample_helpers.classes.private_field import private_field
7
+ from wexample_helpers.decorator.base_class import base_class
8
+
9
+ from syrtis_python_storage.common.abstract_graph_storage import AbstractGraphStorage
10
+
11
+ if TYPE_CHECKING:
12
+ from wexample_remote.common.remote_status import RemoteStatus
13
+
14
+
15
+ @base_class
16
+ class Neo4jGraphStorage(AbstractGraphStorage):
17
+ """A Neo4j database, queried in Cypher. Needs the `neo4j` extra."""
18
+
19
+ database: str | None = public_field(default=None, description="Database; the server's default when None.")
20
+ password: str = public_field(description="Password.")
21
+ url: str = public_field(description="Bolt url, e.g. `bolt://localhost:7687`.")
22
+ user: str = public_field(description="User name.")
23
+ _driver: Any = private_field(default=None, description="Neo4j driver, created on first use.")
24
+
25
+ def __repr__(self) -> str:
26
+ # The password must never reach a log or a traceback.
27
+ return f"Neo4jGraphStorage(name={self.name!r}, url={self.url!r})"
28
+
29
+ @classmethod
30
+ def from_credential(cls, credential: Any, **kwargs: Any) -> Neo4jGraphStorage:
31
+ """From a `wexample-wallet` credential of type `neo4j` (url, user,
32
+ password), named after it unless `name` is given."""
33
+ return cls(
34
+ name=kwargs.pop("name", credential.name),
35
+ password=credential.get("password"),
36
+ url=credential.get("url"),
37
+ user=credential.get("user"),
38
+ **kwargs,
39
+ )
40
+
41
+ def check_status(self) -> RemoteStatus:
42
+ from wexample_remote.common.remote_status import RemoteStatus
43
+
44
+ self._get_driver().verify_connectivity()
45
+ return RemoteStatus.up()
46
+
47
+ def run_query(self, query: str, parameters: dict[str, Any] | None = None) -> list[dict[str, Any]]:
48
+ records, _summary, _keys = self._get_driver().execute_query(
49
+ query, parameters or {}, database_=self.database
50
+ )
51
+ return [self._to_plain(record.data()) for record in records]
52
+
53
+ def _get_driver(self) -> Any:
54
+ if self._driver is None:
55
+ from neo4j import GraphDatabase
56
+
57
+ self._driver = GraphDatabase.driver(self.url, auth=(self.user, self.password))
58
+ return self._driver
59
+
60
+ def _to_plain(self, value: Any) -> Any:
61
+ # Dates and spatial values become text, so that records serialize.
62
+ if isinstance(value, dict):
63
+ return {key: self._to_plain(item) for key, item in value.items()}
64
+ if isinstance(value, list):
65
+ return [self._to_plain(item) for item in value]
66
+ if value is None or isinstance(value, (bool, int, float, str)):
67
+ return value
68
+ return str(value)