spinta 0.2.dev19__py3-none-any.whl → 0.2.dev21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spinta/adapters/rc/__init__.py +1 -0
- spinta/adapters/rc/signature_adapter.py +118 -0
- spinta/adapters/soap_plugins.py +122 -0
- spinta/backends/helpers.py +234 -38
- spinta/backends/memory/commands/init.py +3 -3
- spinta/backends/memory/commands/manifest.py +1 -2
- spinta/backends/memory/commands/read.py +7 -7
- spinta/backends/postgresql/commands/bootstrap.py +16 -0
- spinta/backends/postgresql/commands/init.py +10 -11
- spinta/backends/postgresql/commands/manifest.py +5 -9
- spinta/backends/postgresql/commands/migrate/constants.py +1 -1
- spinta/backends/postgresql/commands/migrate/migrate.py +26 -118
- spinta/backends/postgresql/commands/migrate/model.py +233 -153
- spinta/backends/postgresql/commands/migrate/types/array.py +55 -29
- spinta/backends/postgresql/commands/migrate/types/datatype.py +45 -34
- spinta/backends/postgresql/commands/migrate/types/file.py +27 -18
- spinta/backends/postgresql/commands/migrate/types/ref.py +112 -88
- spinta/backends/postgresql/commands/migrate/types/string.py +20 -10
- spinta/backends/postgresql/commands/migrate/types/text.py +21 -9
- spinta/backends/postgresql/commands/redirect.py +5 -4
- spinta/backends/postgresql/commands/summary.py +22 -18
- spinta/backends/postgresql/commands/wipe.py +13 -10
- spinta/backends/postgresql/components.py +6 -4
- spinta/backends/postgresql/helpers/__init__.py +15 -0
- spinta/backends/postgresql/helpers/changes.py +5 -5
- spinta/backends/postgresql/helpers/migrate/actions.py +237 -121
- spinta/backends/postgresql/helpers/migrate/cast.py +67 -0
- spinta/backends/postgresql/helpers/migrate/migrate.py +367 -430
- spinta/backends/postgresql/helpers/migrate/name.py +219 -0
- spinta/backends/postgresql/helpers/name.py +18 -42
- spinta/backends/postgresql/helpers/redirect.py +5 -5
- spinta/backends/postgresql/types/array/init.py +9 -8
- spinta/backends/postgresql/types/file/init.py +7 -6
- spinta/backends/postgresql/types/ref/init.py +21 -6
- spinta/cli/admin.py +3 -0
- spinta/cli/config.py +13 -7
- spinta/cli/helpers/admin/scripts/changelog.py +16 -16
- spinta/cli/helpers/admin/scripts/deduplicate.py +7 -7
- spinta/cli/helpers/script/components.py +13 -0
- spinta/cli/helpers/script/core.py +27 -6
- spinta/cli/helpers/upgrade/components.py +1 -0
- spinta/cli/helpers/upgrade/registry.py +15 -0
- spinta/cli/helpers/upgrade/scripts/backends/postgresql/comments.py +37 -28
- spinta/cli/helpers/upgrade/scripts/backends/postgresql/schemas.py +313 -0
- spinta/cli/helpers/upgrade/scripts/redirect.py +3 -3
- spinta/cli/main.py +0 -1
- spinta/cli/migrate.py +0 -16
- spinta/cli/upgrade.py +4 -0
- spinta/commands/__init__.py +0 -23
- spinta/commands/read.py +2 -3
- spinta/components.py +1 -0
- spinta/config.py +2 -4
- spinta/config.yml +3 -0
- spinta/core/access.py +2 -1
- spinta/core/ufuncs.py +5 -1
- spinta/datasets/backends/dataframe/backends/soap/ufuncs/ufuncs.py +108 -15
- spinta/datasets/backends/dataframe/commands/check.py +33 -0
- spinta/datasets/backends/dataframe/ufuncs/query/components.py +10 -0
- spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py +77 -11
- spinta/datasets/backends/sql/ufuncs/query/ufuncs.py +0 -6
- spinta/datasets/backends/sql/ufuncs/result/ufuncs.py +0 -37
- spinta/datasets/commands/link.py +4 -2
- spinta/exceptions.py +5 -3
- spinta/manifests/backend/commands/load.py +1 -3
- spinta/manifests/backend/commands/manifest.py +0 -13
- spinta/manifests/components.py +1 -1
- spinta/manifests/dict/commands/load.py +0 -2
- spinta/manifests/dict/components.py +20 -14
- spinta/manifests/dict/helpers.py +296 -48
- spinta/manifests/helpers.py +1 -13
- spinta/manifests/internal/commands/load.py +13 -31
- spinta/manifests/internal_sql/commands/load.py +0 -2
- spinta/manifests/memory/commands/load.py +0 -2
- spinta/manifests/open_api/commands/load.py +0 -2
- spinta/manifests/rdf/commands/load.py +0 -2
- spinta/manifests/sql/commands/load.py +0 -4
- spinta/manifests/sql/helpers.py +11 -3
- spinta/manifests/tabular/commands/load.py +0 -4
- spinta/manifests/tabular/constants.py +1 -0
- spinta/manifests/xsd/commands/load.py +0 -4
- spinta/manifests/xsd/components.py +1 -1
- spinta/manifests/xsd/helpers.py +1076 -946
- spinta/manifests/xsd2/commands/load.py +0 -60
- spinta/manifests/yaml/commands/load.py +13 -34
- spinta/manifests/yaml/commands/manifest.py +0 -11
- spinta/manifests/yaml/helpers.py +1 -11
- spinta/nodes.py +0 -4
- spinta/testing/client.py +2 -0
- spinta/testing/migration.py +196 -75
- spinta/testing/pytest.py +21 -4
- spinta/types/config.py +9 -0
- spinta/types/datatype.py +6 -0
- spinta/types/model.py +33 -38
- spinta/types/text/helpers.py +1 -1
- spinta/ufuncs/querybuilder/ufuncs.py +10 -0
- spinta/ufuncs/resultbuilder/ufuncs.py +109 -2
- {spinta-0.2.dev19.dist-info → spinta-0.2.dev21.dist-info}/METADATA +2 -2
- {spinta-0.2.dev19.dist-info → spinta-0.2.dev21.dist-info}/RECORD +102 -110
- spinta/backends/mongo/commands/freeze.py +0 -15
- spinta/backends/postgresql/commands/freeze.py +0 -190
- spinta/backends/postgresql/types/array/freeze.py +0 -86
- spinta/backends/postgresql/types/file/freeze.py +0 -102
- spinta/backends/postgresql/types/object/freeze.py +0 -57
- spinta/backends/postgresql/types/ref/freeze.py +0 -83
- spinta/manifests/backend/commands/freeze.py +0 -10
- spinta/manifests/xsd2/commands/__init__.py +0 -0
- spinta/manifests/xsd2/commands/configure.py +0 -15
- spinta/manifests/xsd2/components.py +0 -9
- spinta/manifests/xsd2/helpers.py +0 -1310
- spinta/manifests/yaml/commands/freeze.py +0 -50
- spinta/migrations/__init__.py +0 -23
- spinta/migrations/schema/__init__.py +0 -0
- spinta/migrations/schema/alembic.py +0 -119
- /spinta/{manifests/xsd2 → adapters}/__init__.py +0 -0
- {spinta-0.2.dev19.dist-info → spinta-0.2.dev21.dist-info}/WHEEL +0 -0
- {spinta-0.2.dev19.dist-info → spinta-0.2.dev21.dist-info}/entry_points.txt +0 -0
- {spinta-0.2.dev19.dist-info → spinta-0.2.dev21.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Organization-specific SOAP helpers (e.g. RC broker request signing)."""
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import subprocess
|
|
3
|
+
|
|
4
|
+
from lxml import etree
|
|
5
|
+
|
|
6
|
+
from spinta.components import Context
|
|
7
|
+
from spinta.core.config import RawConfig
|
|
8
|
+
from spinta.core.ufuncs import Expr
|
|
9
|
+
from spinta.datasets.backends.dataframe.backends.soap.ufuncs.components import SoapQueryBuilder
|
|
10
|
+
from spinta.datasets.backends.dataframe.backends.soap.ufuncs.ufuncs import MakeCDATA
|
|
11
|
+
|
|
12
|
+
RC_ACTION_TYPE_FIELD = "ActionType"
|
|
13
|
+
RC_CALLER_CODE_FIELD = "CallerCode"
|
|
14
|
+
RC_END_USER_INFO_FIELD = "EndUserInfo"
|
|
15
|
+
RC_PARAMETERS_FIELD = "Parameters"
|
|
16
|
+
RC_TIME_FIELD = "Time"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def is_quoted(value: str) -> bool:
|
|
20
|
+
"""True if ``value`` has matching outer ``'`` or ``"`` quotes (Katalogas ``is_quoted``)."""
|
|
21
|
+
return len(value) > 1 and value[0] == value[-1] and value[0] in ('"', "'")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _require_rc_private_key_path(raw_config: RawConfig | None) -> str:
|
|
25
|
+
"""Return the configured PEM path, or raise if raw config / key path is missing."""
|
|
26
|
+
if raw_config is None:
|
|
27
|
+
raise RuntimeError("RC signature adapter was loaded but application configuration is missing.")
|
|
28
|
+
path = raw_config.get("rc_signature", "private_key_path", default=None)
|
|
29
|
+
if not path or not str(path).strip():
|
|
30
|
+
raise RuntimeError(
|
|
31
|
+
"When using the RC signature SOAP adapter, set `rc_signature.private_key_path` in configuration."
|
|
32
|
+
)
|
|
33
|
+
return str(path)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def validate_soap_adapter_config(raw_config: RawConfig | None) -> None:
|
|
37
|
+
"""Invoked by soap plugin loader when this module is listed in ``soap_adapter_modules``."""
|
|
38
|
+
_require_rc_private_key_path(raw_config)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _get_private_key_path(context: Context) -> str:
|
|
42
|
+
return _require_rc_private_key_path(context.get("rc"))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _compute_rc_signature(args: str, key_path: str) -> str:
|
|
46
|
+
"""Compute RC-style signature using openssl CLI.
|
|
47
|
+
This mirrors the legacy shell example:
|
|
48
|
+
echo -n "$ARGS" | openssl dgst -sha256 -sign $PRIVATE_KEY_FILE | base64 -w0
|
|
49
|
+
"""
|
|
50
|
+
proc = subprocess.run(
|
|
51
|
+
["openssl", "dgst", "-sha256", "-sign", key_path],
|
|
52
|
+
input=args.encode("utf-8"),
|
|
53
|
+
stdout=subprocess.PIPE,
|
|
54
|
+
stderr=subprocess.PIPE,
|
|
55
|
+
check=True,
|
|
56
|
+
)
|
|
57
|
+
sig_b64 = base64.b64encode(proc.stdout).decode("ascii")
|
|
58
|
+
|
|
59
|
+
return sig_b64.replace("\r", "").replace("\n", "")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def build_rc_string_to_sign(soap_body: dict[str, object]) -> str:
|
|
63
|
+
"""Return the plaintext that RC signs and verifies for SOAP calls.
|
|
64
|
+
|
|
65
|
+
The broker does not sign the whole XML; it expects an RSA-SHA256 signature over one
|
|
66
|
+
concatenated string, in this exact order with no delimiters: ActionType, CallerCode,
|
|
67
|
+
EndUserInfo, Parameters, Time. The server rebuilds the same string and checks the
|
|
68
|
+
signature, so we must assemble identical pieces from `soap_body`.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
input_block = soap_body.get("input")
|
|
72
|
+
input_dict = input_block if isinstance(input_block, dict) else None
|
|
73
|
+
|
|
74
|
+
def _get(name: str, default: str = "") -> str:
|
|
75
|
+
value = soap_body.get(f"input/{name}")
|
|
76
|
+
if value is None and input_dict is not None:
|
|
77
|
+
value = input_dict.get(name)
|
|
78
|
+
|
|
79
|
+
if value is None:
|
|
80
|
+
value = soap_body.get(name, default)
|
|
81
|
+
|
|
82
|
+
if isinstance(value, MakeCDATA):
|
|
83
|
+
value = value.data
|
|
84
|
+
|
|
85
|
+
if isinstance(value, etree.CDATA):
|
|
86
|
+
value = str(value)
|
|
87
|
+
|
|
88
|
+
if name == RC_PARAMETERS_FIELD and isinstance(value, str) and is_quoted(value):
|
|
89
|
+
value = value[1:-1]
|
|
90
|
+
|
|
91
|
+
return "" if value is None else str(value)
|
|
92
|
+
|
|
93
|
+
action_type = _get(RC_ACTION_TYPE_FIELD)
|
|
94
|
+
caller_code = _get(RC_CALLER_CODE_FIELD)
|
|
95
|
+
end_user_info = _get(RC_END_USER_INFO_FIELD, "")
|
|
96
|
+
parameters = _get(RC_PARAMETERS_FIELD, "")
|
|
97
|
+
time_value = _get(RC_TIME_FIELD)
|
|
98
|
+
|
|
99
|
+
return f"{action_type}{caller_code}{end_user_info}{parameters}{time_value}"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def compute_rc_signature_from_body(soap_body: dict[str, object], context: Context) -> str:
|
|
103
|
+
"""Build string-to-sign from soap_body and return base64 signature."""
|
|
104
|
+
key_path = _get_private_key_path(context)
|
|
105
|
+
args = build_rc_string_to_sign(soap_body)
|
|
106
|
+
|
|
107
|
+
return _compute_rc_signature(args, key_path)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def get_deferred_prepare_names() -> list[str]:
|
|
111
|
+
return ["rc_signature"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def get_body_resolvers() -> dict[str, object]:
|
|
115
|
+
def rc_signature_resolver(env: SoapQueryBuilder, expr: Expr | None = None) -> str:
|
|
116
|
+
return compute_rc_signature_from_body(env.soap_request_body, env.context)
|
|
117
|
+
|
|
118
|
+
return {"rc_signature": rc_signature_resolver}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Load optional SOAP adapter modules from ``soap_adapter_modules`` paths in config."""
|
|
2
|
+
|
|
3
|
+
import importlib.util
|
|
4
|
+
import logging
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from types import ModuleType
|
|
7
|
+
from typing import Any, Callable
|
|
8
|
+
|
|
9
|
+
from spinta.core.ufuncs import Expr, UFuncRegistry
|
|
10
|
+
from spinta.core.config import RawConfig
|
|
11
|
+
from spinta.ufuncs.loadbuilder.components import LoadBuilder
|
|
12
|
+
from spinta.datasets.backends.dataframe.backends.soap.ufuncs.components import SoapQueryBuilder
|
|
13
|
+
|
|
14
|
+
log = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
_deferred_prepare_names_cache: set[str] | None = None
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def get_deferred_prepare_names() -> set[str]:
|
|
21
|
+
"""Names of prepare ufuncs left unresolved until the SOAP body is built (empty before register)."""
|
|
22
|
+
return _deferred_prepare_names_cache or set()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _load_adapter_module_from_path(file_path: str, raw_config: RawConfig | None = None) -> ModuleType | None:
|
|
26
|
+
try:
|
|
27
|
+
path = Path(file_path)
|
|
28
|
+
if not path.exists():
|
|
29
|
+
log.warning("SOAP adapter module not found: %s", file_path)
|
|
30
|
+
return None
|
|
31
|
+
spec = importlib.util.spec_from_file_location(path.stem, path)
|
|
32
|
+
if spec is None or spec.loader is None:
|
|
33
|
+
log.warning("Failed to create module spec for: %s", file_path)
|
|
34
|
+
return None
|
|
35
|
+
module = importlib.util.module_from_spec(spec)
|
|
36
|
+
spec.loader.exec_module(module)
|
|
37
|
+
log.info("Loaded SOAP adapter module from: %s", file_path)
|
|
38
|
+
if hasattr(module, "validate_soap_adapter_config"):
|
|
39
|
+
module.validate_soap_adapter_config(raw_config)
|
|
40
|
+
return module
|
|
41
|
+
except Exception as e:
|
|
42
|
+
log.warning("Failed to load SOAP adapter module %s: %s", file_path, e)
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _load_adapters_from_config(raw_config: RawConfig | None) -> tuple[set[str], dict[str, Callable[..., Any]]]:
|
|
47
|
+
deferred: set[str] = set()
|
|
48
|
+
body_resolvers: dict[str, Callable[..., Any]] = {}
|
|
49
|
+
|
|
50
|
+
if raw_config is None:
|
|
51
|
+
return deferred, body_resolvers
|
|
52
|
+
|
|
53
|
+
module_paths = raw_config.get("soap_adapter_modules", default=[])
|
|
54
|
+
if not module_paths:
|
|
55
|
+
return deferred, body_resolvers
|
|
56
|
+
|
|
57
|
+
if isinstance(module_paths, str):
|
|
58
|
+
module_paths = [module_paths]
|
|
59
|
+
|
|
60
|
+
for module_path in module_paths:
|
|
61
|
+
module = _load_adapter_module_from_path(module_path, raw_config)
|
|
62
|
+
if module is None:
|
|
63
|
+
continue
|
|
64
|
+
|
|
65
|
+
if hasattr(module, "get_deferred_prepare_names"):
|
|
66
|
+
try:
|
|
67
|
+
result = module.get_deferred_prepare_names()
|
|
68
|
+
except Exception as e:
|
|
69
|
+
log.warning("Failed to call get_deferred_prepare_names in %s: %s", module_path, e)
|
|
70
|
+
else:
|
|
71
|
+
if isinstance(result, (list, tuple)):
|
|
72
|
+
deferred.update(result)
|
|
73
|
+
else:
|
|
74
|
+
deferred.add(str(result))
|
|
75
|
+
|
|
76
|
+
if hasattr(module, "get_body_resolvers"):
|
|
77
|
+
try:
|
|
78
|
+
result = module.get_body_resolvers()
|
|
79
|
+
except Exception as e:
|
|
80
|
+
log.warning("Failed to call get_body_resolvers in %s: %s", module_path, e)
|
|
81
|
+
else:
|
|
82
|
+
if isinstance(result, dict):
|
|
83
|
+
body_resolvers.update(result)
|
|
84
|
+
|
|
85
|
+
return deferred, body_resolvers
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _deferred_prepare_resolver(env, expr: Expr) -> Expr:
|
|
89
|
+
env.param.soap_body = {env.this: expr}
|
|
90
|
+
return expr
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _make_body_resolver(callable_fn: Callable[..., Any]) -> Callable[[SoapQueryBuilder, Expr], Any]:
|
|
94
|
+
"""Adapter may implement ``fn(env)`` only; fall back if ``fn(env, expr)`` raises ``TypeError``."""
|
|
95
|
+
|
|
96
|
+
def body_resolver(env: SoapQueryBuilder, expr: Expr) -> Any:
|
|
97
|
+
try:
|
|
98
|
+
return callable_fn(env, expr)
|
|
99
|
+
except TypeError:
|
|
100
|
+
return callable_fn(env)
|
|
101
|
+
|
|
102
|
+
return body_resolver
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def register_soap_ufuncs(registry: UFuncRegistry, raw_config: RawConfig | None = None) -> None:
|
|
106
|
+
"""Register SOAP adapter resolvers from config (`soap_adapter_modules`).
|
|
107
|
+
Adapters are loaded from local Python files listed in:
|
|
108
|
+
soap_adapter_modules:
|
|
109
|
+
- /path/to/adapter.py
|
|
110
|
+
Call this after resolver.collect().
|
|
111
|
+
"""
|
|
112
|
+
global _deferred_prepare_names_cache
|
|
113
|
+
|
|
114
|
+
deferred, body_resolvers = _load_adapters_from_config(raw_config)
|
|
115
|
+
|
|
116
|
+
_deferred_prepare_names_cache = deferred
|
|
117
|
+
|
|
118
|
+
for name in deferred:
|
|
119
|
+
registry.register(name, _deferred_prepare_resolver, (LoadBuilder, Expr))
|
|
120
|
+
|
|
121
|
+
for name, callable_fn in body_resolvers.items():
|
|
122
|
+
registry.register(name, _make_body_resolver(callable_fn), (SoapQueryBuilder, Expr))
|
spinta/backends/helpers.py
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import dataclasses
|
|
1
4
|
from typing import Any
|
|
2
|
-
from typing import Dict
|
|
3
5
|
from typing import Iterable
|
|
4
6
|
from typing import Iterator
|
|
5
|
-
from typing import List
|
|
6
|
-
from typing import Optional
|
|
7
|
-
from typing import Tuple
|
|
8
7
|
from typing import TypeVar
|
|
9
|
-
|
|
8
|
+
|
|
9
|
+
import sqlalchemy as sa
|
|
10
|
+
|
|
11
|
+
from multipledispatch import dispatch
|
|
10
12
|
|
|
11
13
|
from spinta import commands
|
|
12
14
|
from spinta import exceptions
|
|
@@ -14,6 +16,7 @@ from spinta import spyna
|
|
|
14
16
|
from spinta.auth import authorized
|
|
15
17
|
from spinta.backends import Backend
|
|
16
18
|
from spinta.backends.components import SelectTree
|
|
19
|
+
from spinta.backends.postgresql.helpers import get_pg_name
|
|
17
20
|
from spinta.commands import build_full_response
|
|
18
21
|
from spinta.components import Config, DataItem
|
|
19
22
|
from spinta.core.enums import Action
|
|
@@ -28,6 +31,11 @@ from spinta.utils.data import take
|
|
|
28
31
|
from spinta.backends.constants import TableType, BackendOrigin
|
|
29
32
|
|
|
30
33
|
|
|
34
|
+
from sqlalchemy.dialects import postgresql
|
|
35
|
+
|
|
36
|
+
pg_identifier_preparer = postgresql.dialect().identifier_preparer
|
|
37
|
+
|
|
38
|
+
|
|
31
39
|
def validate_and_return_transaction(context: Context, backend: Backend, **kwargs):
|
|
32
40
|
if not backend.available:
|
|
33
41
|
backend.available = commands.wait(context, backend)
|
|
@@ -51,7 +59,7 @@ def validate_and_return_begin(context: Context, backend: Backend, **kwargs):
|
|
|
51
59
|
|
|
52
60
|
|
|
53
61
|
def load_backend(
|
|
54
|
-
context: Context, component: Component, name: str, origin: BackendOrigin, data:
|
|
62
|
+
context: Context, component: Component, name: str, origin: BackendOrigin, data: dict[str, str]
|
|
55
63
|
) -> Backend:
|
|
56
64
|
config = context.get("config")
|
|
57
65
|
type_ = data.get("type")
|
|
@@ -77,7 +85,7 @@ def load_backend(
|
|
|
77
85
|
def get_select_tree(
|
|
78
86
|
context: Context,
|
|
79
87
|
action: Action,
|
|
80
|
-
select:
|
|
88
|
+
select: list[str] | None,
|
|
81
89
|
) -> SelectTree:
|
|
82
90
|
if isinstance(select, dict):
|
|
83
91
|
select = list(select.keys())
|
|
@@ -92,8 +100,8 @@ def get_select_tree(
|
|
|
92
100
|
def _apply_always_show_id(
|
|
93
101
|
context: Context,
|
|
94
102
|
action: Action,
|
|
95
|
-
select:
|
|
96
|
-
) ->
|
|
103
|
+
select: list[str] | None,
|
|
104
|
+
) -> list[str] | None:
|
|
97
105
|
if action in (Action.GETALL, Action.SEARCH):
|
|
98
106
|
config = context.get("config")
|
|
99
107
|
if config.always_show_id:
|
|
@@ -106,18 +114,18 @@ def _apply_always_show_id(
|
|
|
106
114
|
|
|
107
115
|
def get_select_prop_names(
|
|
108
116
|
context: Context,
|
|
109
|
-
node:
|
|
110
|
-
props:
|
|
117
|
+
node: Model | Property | DataType,
|
|
118
|
+
props: dict[str, Property],
|
|
111
119
|
action: Action,
|
|
112
120
|
select: SelectTree,
|
|
113
121
|
*,
|
|
114
122
|
# If False, do not check if client has access to this property.
|
|
115
123
|
auth: bool = True,
|
|
116
124
|
# Allowed reserved property names.
|
|
117
|
-
reserved:
|
|
125
|
+
reserved: list[str] = None,
|
|
118
126
|
# If False, do not include Denorm type props
|
|
119
127
|
include_denorm_props: bool = True,
|
|
120
|
-
) ->
|
|
128
|
+
) -> list[str]:
|
|
121
129
|
known = set(reserved or []) | set(take(props))
|
|
122
130
|
check_unknown_props(node, select, known)
|
|
123
131
|
|
|
@@ -138,13 +146,13 @@ def get_select_prop_names(
|
|
|
138
146
|
|
|
139
147
|
def select_model_props(
|
|
140
148
|
model: Model,
|
|
141
|
-
prop_names:
|
|
149
|
+
prop_names: list[str],
|
|
142
150
|
value: dict,
|
|
143
151
|
select: SelectTree,
|
|
144
|
-
reserved:
|
|
152
|
+
reserved: list[str],
|
|
145
153
|
) -> Iterator[
|
|
146
|
-
|
|
147
|
-
|
|
154
|
+
tuple[
|
|
155
|
+
Property | str,
|
|
148
156
|
Any,
|
|
149
157
|
SelectTree,
|
|
150
158
|
]
|
|
@@ -170,16 +178,16 @@ T = TypeVar("T")
|
|
|
170
178
|
|
|
171
179
|
|
|
172
180
|
def select_props(
|
|
173
|
-
node:
|
|
181
|
+
node: Namespace | Model | Property,
|
|
174
182
|
keys: Iterable[str],
|
|
175
|
-
props:
|
|
176
|
-
value:
|
|
183
|
+
props: dict[str, Property],
|
|
184
|
+
value: dict[str, T],
|
|
177
185
|
select: SelectTree,
|
|
178
186
|
*,
|
|
179
187
|
reserved: bool = True,
|
|
180
188
|
) -> Iterator[
|
|
181
|
-
|
|
182
|
-
|
|
189
|
+
tuple[
|
|
190
|
+
Property | str,
|
|
183
191
|
T,
|
|
184
192
|
SelectTree,
|
|
185
193
|
]
|
|
@@ -191,15 +199,15 @@ def select_props(
|
|
|
191
199
|
|
|
192
200
|
|
|
193
201
|
def select_only_props(
|
|
194
|
-
node:
|
|
202
|
+
node: Namespace | Model | Property,
|
|
195
203
|
keys: Iterable[str],
|
|
196
|
-
props:
|
|
204
|
+
props: dict[str, Property],
|
|
197
205
|
select: SelectTree,
|
|
198
206
|
*,
|
|
199
207
|
reserved: bool = True,
|
|
200
208
|
) -> Iterator[
|
|
201
|
-
|
|
202
|
-
|
|
209
|
+
tuple[
|
|
210
|
+
Property | str,
|
|
203
211
|
SelectTree,
|
|
204
212
|
]
|
|
205
213
|
]:
|
|
@@ -211,9 +219,9 @@ def select_only_props(
|
|
|
211
219
|
|
|
212
220
|
def _select_prop(
|
|
213
221
|
key: str,
|
|
214
|
-
props:
|
|
215
|
-
node:
|
|
216
|
-
) ->
|
|
222
|
+
props: dict[str, Property],
|
|
223
|
+
node: Namespace | Model | Property,
|
|
224
|
+
) -> Property | None:
|
|
217
225
|
if not (prop := props.get(key)) or prop.hidden:
|
|
218
226
|
return None
|
|
219
227
|
|
|
@@ -222,12 +230,12 @@ def _select_prop(
|
|
|
222
230
|
|
|
223
231
|
def select_keys(
|
|
224
232
|
keys: Iterable[str],
|
|
225
|
-
value:
|
|
233
|
+
value: dict[str, T],
|
|
226
234
|
select: SelectTree,
|
|
227
235
|
*,
|
|
228
236
|
reserved: bool = True,
|
|
229
237
|
) -> Iterator[
|
|
230
|
-
|
|
238
|
+
tuple[
|
|
231
239
|
str,
|
|
232
240
|
T,
|
|
233
241
|
SelectTree,
|
|
@@ -253,7 +261,7 @@ def select_only_keys(
|
|
|
253
261
|
*,
|
|
254
262
|
reserved: bool = True,
|
|
255
263
|
) -> Iterator[
|
|
256
|
-
|
|
264
|
+
tuple[
|
|
257
265
|
str,
|
|
258
266
|
SelectTree,
|
|
259
267
|
]
|
|
@@ -280,8 +288,8 @@ def select_only_keys(
|
|
|
280
288
|
# FIXME: We should check select list at the very beginning of
|
|
281
289
|
# request, not when returning results.
|
|
282
290
|
def check_unknown_props(
|
|
283
|
-
node:
|
|
284
|
-
select:
|
|
291
|
+
node: Model | Property | DataType,
|
|
292
|
+
select: Iterable[str] | None,
|
|
285
293
|
known: Iterable[str],
|
|
286
294
|
):
|
|
287
295
|
unknown_properties = set(select or []) - set(known) - {"*"}
|
|
@@ -291,7 +299,7 @@ def check_unknown_props(
|
|
|
291
299
|
)
|
|
292
300
|
|
|
293
301
|
|
|
294
|
-
def flat_select_to_nested(select:
|
|
302
|
+
def flat_select_to_nested(select: list[str] | None) -> SelectTree:
|
|
295
303
|
"""
|
|
296
304
|
>>> flat_select_to_nested(None)
|
|
297
305
|
|
|
@@ -316,7 +324,7 @@ def flat_select_to_nested(select: Optional[List[str]]) -> SelectTree:
|
|
|
316
324
|
return res
|
|
317
325
|
|
|
318
326
|
|
|
319
|
-
def get_model_reserved_props(action: Action, include_page: bool) ->
|
|
327
|
+
def get_model_reserved_props(action: Action, include_page: bool) -> list[str]:
|
|
320
328
|
if action == Action.GETALL:
|
|
321
329
|
reserved = ["_type", "_id", "_revision"]
|
|
322
330
|
elif action == Action.SEARCH:
|
|
@@ -332,12 +340,145 @@ def get_model_reserved_props(action: Action, include_page: bool) -> List[str]:
|
|
|
332
340
|
return reserved
|
|
333
341
|
|
|
334
342
|
|
|
335
|
-
def get_ns_reserved_props(action: Action) ->
|
|
343
|
+
def get_ns_reserved_props(action: Action) -> list[str]:
|
|
336
344
|
return []
|
|
337
345
|
|
|
338
346
|
|
|
347
|
+
@dataclasses.dataclass
|
|
348
|
+
class TableIdentifier:
|
|
349
|
+
"""
|
|
350
|
+
Represents a table identifier across logical (app) and PostgreSQL layers.
|
|
351
|
+
|
|
352
|
+
It builds derived names used for internal logic and SQL queries, including
|
|
353
|
+
schema-qualified and escaped identifiers.
|
|
354
|
+
|
|
355
|
+
Attributes:
|
|
356
|
+
schema (str | None): Logical schema/namespace (e.g. "datasets/gov/rc").
|
|
357
|
+
base_name (str): Base table name (e.g. "Building").
|
|
358
|
+
table_type (TableType): Table type suffix (default: TableType.MAIN).
|
|
359
|
+
table_arg (str | None): Optional argument appended used for table types that require property.
|
|
360
|
+
default_pg_schema (str | None): Fallback PG schema if schema is not given.
|
|
361
|
+
|
|
362
|
+
logical_name (str): Computed name (base + type + optional arg).
|
|
363
|
+
Example: "Building/:list/apartments"
|
|
364
|
+
logical_qualified_name (str): Logical name with schema (dataset).
|
|
365
|
+
Example: "datasets/gov/rc/Building/:list/apartments"
|
|
366
|
+
|
|
367
|
+
pg_table_name (str): PG-safe (compressed) table name from logical_name.
|
|
368
|
+
pg_schema_name (str | None): PG-safe (compressed) schema name.
|
|
369
|
+
pg_qualified_name (str): PG-safe (compressed) schema with table name.
|
|
370
|
+
Example: "datasets/gov/rc.Building/:list/apartments" (unescaped).
|
|
371
|
+
pg_escaped_qualified_name (str): Quoted version of pg_qualified_name, used for queries.
|
|
372
|
+
Example: '"datasets/gov/rc"."Building/:list/apartments"' (escaped).
|
|
373
|
+
|
|
374
|
+
Example:
|
|
375
|
+
>>> TableIdentifier("datasets/gov/rc", "Buildings", TableType.LIST, "apartments")
|
|
376
|
+
# logical_qualified_name: "datasets/gov/rc/Buildings/:list/apartments"
|
|
377
|
+
# pg_qualified_name: "datasets/gov/rc.Building/:list/apartments"
|
|
378
|
+
|
|
379
|
+
>>> TableIdentifier("datasets/gov/rc", "Buildings")
|
|
380
|
+
# logical_qualified_name: "datasets/gov/rc/Buildings"
|
|
381
|
+
# pg_qualified_name: "datasets/gov/rc.Building"
|
|
382
|
+
|
|
383
|
+
>>> TableIdentifier("datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits", "Buildings")
|
|
384
|
+
# logical_qualified_name: "datasets/gov/rc/very/long/dataset/name/that/does/not/fit/withing/limits/Buildings"
|
|
385
|
+
# pg_qualified_name: "datasets/gov/rc/very/long/dataset/nam_e5985b69_t/withing/limits.Building"
|
|
386
|
+
"""
|
|
387
|
+
|
|
388
|
+
schema: str | None
|
|
389
|
+
base_name: str
|
|
390
|
+
table_type: TableType = dataclasses.field(default=TableType.MAIN)
|
|
391
|
+
table_arg: str | None = dataclasses.field(default=None)
|
|
392
|
+
default_pg_schema: str | None = dataclasses.field(default=None)
|
|
393
|
+
|
|
394
|
+
logical_name: str = dataclasses.field(init=False)
|
|
395
|
+
# Name with namespace connected with '/', like it is used with Model class
|
|
396
|
+
logical_qualified_name: str = dataclasses.field(init=False)
|
|
397
|
+
|
|
398
|
+
pg_table_name: str = dataclasses.field(init=False)
|
|
399
|
+
pg_schema_name: str | None = dataclasses.field(init=False)
|
|
400
|
+
# Used for hashed schema and table names
|
|
401
|
+
pg_qualified_name: str = dataclasses.field(init=False)
|
|
402
|
+
# Escaped qualified name, used for queries
|
|
403
|
+
pg_escaped_qualified_name: str = dataclasses.field(init=False)
|
|
404
|
+
|
|
405
|
+
def __post_init__(self):
|
|
406
|
+
self.logical_name = self.base_name + self.table_type.value
|
|
407
|
+
if self.table_arg:
|
|
408
|
+
self.logical_name += "/" + self.table_arg
|
|
409
|
+
|
|
410
|
+
self.logical_qualified_name = f"{self.schema}/{self.logical_name}" if self.schema else self.logical_name
|
|
411
|
+
|
|
412
|
+
self.pg_table_name = get_pg_name(self.logical_name)
|
|
413
|
+
self.pg_schema_name = get_pg_name(self.schema) if self.schema else self.default_pg_schema
|
|
414
|
+
self.pg_qualified_name = (
|
|
415
|
+
f"{self.pg_schema_name}.{self.pg_table_name}" if self.pg_schema_name else self.pg_table_name
|
|
416
|
+
)
|
|
417
|
+
self.pg_escaped_qualified_name = (
|
|
418
|
+
f"{pg_identifier_preparer.quote(self.pg_schema_name)}.{pg_identifier_preparer.quote(self.pg_table_name)}"
|
|
419
|
+
if self.pg_schema_name
|
|
420
|
+
else pg_identifier_preparer.quote(self.pg_table_name)
|
|
421
|
+
)
|
|
422
|
+
|
|
423
|
+
def change_table_type(self, new_type: TableType, table_arg: str | None = None) -> "TableIdentifier":
|
|
424
|
+
return dataclasses.replace(self, table_type=new_type, table_arg=table_arg)
|
|
425
|
+
|
|
426
|
+
def apply_removed_prefix(self, remove_model_only: bool = False) -> "TableIdentifier":
|
|
427
|
+
if remove_model_only or not self.table_arg:
|
|
428
|
+
if not self.base_name.startswith("__"):
|
|
429
|
+
return dataclasses.replace(self, base_name=f"__{self.base_name}")
|
|
430
|
+
return self
|
|
431
|
+
|
|
432
|
+
if not self.table_arg.startswith("__"):
|
|
433
|
+
return dataclasses.replace(self, table_arg=f"__{self.table_arg}")
|
|
434
|
+
return self
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
@dispatch(str)
|
|
438
|
+
def get_table_identifier(item: str, **kwargs) -> TableIdentifier:
|
|
439
|
+
schema, model_name, table_type, table_arg = split_logical_name(item)
|
|
440
|
+
return TableIdentifier(schema=schema, base_name=model_name, table_type=table_type, table_arg=table_arg, **kwargs)
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
@dispatch(sa.Table)
|
|
444
|
+
def get_table_identifier(item: sa.Table, **kwargs) -> TableIdentifier:
|
|
445
|
+
if not item.comment:
|
|
446
|
+
return TableIdentifier(schema=item.schema, base_name=item.name, **kwargs)
|
|
447
|
+
if item.schema not in ("public", None):
|
|
448
|
+
return get_table_identifier(item.comment, **kwargs)
|
|
449
|
+
|
|
450
|
+
schema, model_name, table_type, table_arg = split_logical_name(item.comment)
|
|
451
|
+
return TableIdentifier(
|
|
452
|
+
schema=None,
|
|
453
|
+
base_name=f"{schema}/{model_name}" if schema else model_name,
|
|
454
|
+
table_type=table_type,
|
|
455
|
+
table_arg=table_arg,
|
|
456
|
+
**kwargs,
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
@dispatch((Model, Property))
|
|
461
|
+
def get_table_identifier(node: Model | Property, **kwargs) -> TableIdentifier:
|
|
462
|
+
return get_table_identifier(node, TableType.MAIN, **kwargs)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
@dispatch((Model, Property), TableType)
|
|
466
|
+
def get_table_identifier(
|
|
467
|
+
node: Model | Property, table_type: TableType, table_arg: str = None, **kwargs
|
|
468
|
+
) -> TableIdentifier:
|
|
469
|
+
model = node if isinstance(node, Model) else node.model
|
|
470
|
+
|
|
471
|
+
schema = model.ns.name if model.ns else None
|
|
472
|
+
base_name = model.get_name_without_ns()
|
|
473
|
+
|
|
474
|
+
if isinstance(node, Property) and table_type in (TableType.LIST, TableType.FILE):
|
|
475
|
+
table_arg = node.place
|
|
476
|
+
|
|
477
|
+
return TableIdentifier(schema, base_name, table_type, table_arg, **kwargs)
|
|
478
|
+
|
|
479
|
+
|
|
339
480
|
def get_table_name(
|
|
340
|
-
node:
|
|
481
|
+
node: Model | Property,
|
|
341
482
|
ttype: TableType = TableType.MAIN,
|
|
342
483
|
) -> str:
|
|
343
484
|
if isinstance(node, Model):
|
|
@@ -351,6 +492,27 @@ def get_table_name(
|
|
|
351
492
|
return name
|
|
352
493
|
|
|
353
494
|
|
|
495
|
+
def split_table_name(full_name: str) -> tuple[str | None, str]:
|
|
496
|
+
parts = full_name.split(".", maxsplit=1)
|
|
497
|
+
if len(parts) == 1:
|
|
498
|
+
return None, parts[0]
|
|
499
|
+
return parts[0], parts[1]
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def split_logical_name(full_name: str) -> tuple[str | None, str, TableType, str | None]:
|
|
503
|
+
base_name, table_type, property_name = extract_table_data_from_logical_name(full_name)
|
|
504
|
+
parts = base_name.split("/")
|
|
505
|
+
if len(parts) == 1:
|
|
506
|
+
return None, parts[0], table_type, property_name
|
|
507
|
+
|
|
508
|
+
for i, part in enumerate(parts):
|
|
509
|
+
if part[0].isupper() or (part[:2] == "__" and part[2].isupper()):
|
|
510
|
+
namespace = "/".join(parts[:i])
|
|
511
|
+
model = "/".join(parts[i:])
|
|
512
|
+
return namespace, model, table_type, property_name
|
|
513
|
+
return None, base_name, table_type, property_name
|
|
514
|
+
|
|
515
|
+
|
|
354
516
|
def load_query_builder_class(config: Config, backend: Backend):
|
|
355
517
|
if backend.query_builder_type is None:
|
|
356
518
|
return
|
|
@@ -400,3 +562,37 @@ def prepare_response(
|
|
|
400
562
|
else:
|
|
401
563
|
resp = {}
|
|
402
564
|
return resp
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def extract_table_data_from_logical_name(table_name: str) -> tuple[str | None, TableType | None, str | None]:
|
|
568
|
+
"""
|
|
569
|
+
Extracts the main table name, table type, and an optional property suffix from a logical
|
|
570
|
+
table name string. It parses the given logical table name and determines whether it belongs
|
|
571
|
+
to the main table or some specific table type. If a specific type is found, it splits the
|
|
572
|
+
table name into its components.
|
|
573
|
+
|
|
574
|
+
Parameters:
|
|
575
|
+
table_name (str): The logical table name string that needs to be processed.
|
|
576
|
+
|
|
577
|
+
Returns:
|
|
578
|
+
tuple: A tuple containing:
|
|
579
|
+
- str | None: The main table name.
|
|
580
|
+
- TableType | None: The type of the table, which can be `MAIN` or other enum members of
|
|
581
|
+
`TableType`.
|
|
582
|
+
- str | None: A property suffix string if present in the logical table name, or
|
|
583
|
+
None otherwise.
|
|
584
|
+
"""
|
|
585
|
+
if "/:" not in table_name:
|
|
586
|
+
return table_name, TableType.MAIN, None
|
|
587
|
+
|
|
588
|
+
for table_type in TableType:
|
|
589
|
+
if table_type is TableType.MAIN:
|
|
590
|
+
continue
|
|
591
|
+
|
|
592
|
+
if table_type.value in table_name:
|
|
593
|
+
data = table_name.split(table_type.value, 1)
|
|
594
|
+
if data[1]:
|
|
595
|
+
return data[0], table_type, data[1][1:] # skip /property slash
|
|
596
|
+
return data[0], table_type, None
|
|
597
|
+
|
|
598
|
+
return None, None, None
|
|
@@ -2,12 +2,12 @@ from spinta import commands
|
|
|
2
2
|
from spinta.components import Context
|
|
3
3
|
from spinta.manifests.components import Manifest
|
|
4
4
|
from spinta.backends.constants import TableType
|
|
5
|
-
from spinta.backends.helpers import
|
|
5
|
+
from spinta.backends.helpers import get_table_identifier
|
|
6
6
|
from spinta.backends.memory.components import Memory
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
@commands.prepare.register(Context, Memory, Manifest)
|
|
10
10
|
def prepare(context: Context, backend: Memory, manifest: Manifest, **kwargs):
|
|
11
11
|
for model in commands.get_models(context, manifest).values():
|
|
12
|
-
backend.create(
|
|
13
|
-
backend.create(
|
|
12
|
+
backend.create(get_table_identifier(model).logical_qualified_name)
|
|
13
|
+
backend.create(get_table_identifier(model, TableType.CHANGELOG).logical_qualified_name)
|