spinta 0.2.dev19__py3-none-any.whl → 0.2.dev20__py3-none-any.whl

spinta/adapters/rc/__init__.py

@@ -0,0 +1 @@
+"""Organization-specific SOAP helpers (e.g. RC broker request signing)."""

spinta/adapters/rc/signature_adapter.py

@@ -0,0 +1,118 @@
+import base64
+import subprocess
+
+from lxml import etree
+
+from spinta.components import Context
+from spinta.core.config import RawConfig
+from spinta.core.ufuncs import Expr
+from spinta.datasets.backends.dataframe.backends.soap.ufuncs.components import SoapQueryBuilder
+from spinta.datasets.backends.dataframe.backends.soap.ufuncs.ufuncs import MakeCDATA
+
+RC_ACTION_TYPE_FIELD = "ActionType"
+RC_CALLER_CODE_FIELD = "CallerCode"
+RC_END_USER_INFO_FIELD = "EndUserInfo"
+RC_PARAMETERS_FIELD = "Parameters"
+RC_TIME_FIELD = "Time"
+
+
+def is_quoted(value: str) -> bool:
+    """True if ``value`` has matching outer ``'`` or ``"`` quotes (Katalogas ``is_quoted``)."""
+    return len(value) > 1 and value[0] == value[-1] and value[0] in ('"', "'")
+
+
+def _require_rc_private_key_path(raw_config: RawConfig | None) -> str:
+    """Return the configured PEM path, or raise if raw config / key path is missing."""
+    if raw_config is None:
+        raise RuntimeError("RC signature adapter was loaded but application configuration is missing.")
+    path = raw_config.get("rc_signature", "private_key_path", default=None)
+    if not path or not str(path).strip():
+        raise RuntimeError(
+            "When using the RC signature SOAP adapter, set `rc_signature.private_key_path` in configuration."
+        )
+    return str(path)
+
+
+def validate_soap_adapter_config(raw_config: RawConfig | None) -> None:
+    """Invoked by soap plugin loader when this module is listed in ``soap_adapter_modules``."""
+    _require_rc_private_key_path(raw_config)
+
+
+def _get_private_key_path(context: Context) -> str:
+    return _require_rc_private_key_path(context.get("rc"))
+
+
+def _compute_rc_signature(args: str, key_path: str) -> str:
+    """Compute RC-style signature using openssl CLI.
+    This mirrors the legacy shell example:
+        echo -n "$ARGS" | openssl dgst -sha256 -sign $PRIVATE_KEY_FILE | base64 -w0
+    """
+    proc = subprocess.run(
+        ["openssl", "dgst", "-sha256", "-sign", key_path],
+        input=args.encode("utf-8"),
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=True,
+    )
+    sig_b64 = base64.b64encode(proc.stdout).decode("ascii")
+
+    return sig_b64.replace("\r", "").replace("\n", "")
+
+
+def build_rc_string_to_sign(soap_body: dict[str, object]) -> str:
+    """Return the plaintext that RC signs and verifies for SOAP calls.
+
+    The broker does not sign the whole XML; it expects an RSA-SHA256 signature over one
+    concatenated string, in this exact order with no delimiters: ActionType, CallerCode,
+    EndUserInfo, Parameters, Time. The server rebuilds the same string and checks the
+    signature, so we must assemble identical pieces from `soap_body`.
+    """
+
+    input_block = soap_body.get("input")
+    input_dict = input_block if isinstance(input_block, dict) else None
+
+    def _get(name: str, default: str = "") -> str:
+        value = soap_body.get(f"input/{name}")
+        if value is None and input_dict is not None:
+            value = input_dict.get(name)
+
+        if value is None:
+            value = soap_body.get(name, default)
+
+        if isinstance(value, MakeCDATA):
+            value = value.data
+
+        if isinstance(value, etree.CDATA):
+            value = str(value)
+
+        if name == RC_PARAMETERS_FIELD and isinstance(value, str) and is_quoted(value):
+            value = value[1:-1]
+
+        return "" if value is None else str(value)
+
+    action_type = _get(RC_ACTION_TYPE_FIELD)
+    caller_code = _get(RC_CALLER_CODE_FIELD)
+    end_user_info = _get(RC_END_USER_INFO_FIELD, "")
+    parameters = _get(RC_PARAMETERS_FIELD, "")
+    time_value = _get(RC_TIME_FIELD)
+
+    return f"{action_type}{caller_code}{end_user_info}{parameters}{time_value}"
+
+
+def compute_rc_signature_from_body(soap_body: dict[str, object], context: Context) -> str:
+    """Build string-to-sign from soap_body and return base64 signature."""
+    key_path = _get_private_key_path(context)
+    args = build_rc_string_to_sign(soap_body)
+
+    return _compute_rc_signature(args, key_path)
+
+
+def get_deferred_prepare_names() -> list[str]:
+    return ["rc_signature"]
+
+
+def get_body_resolvers() -> dict[str, object]:
+    def rc_signature_resolver(env: SoapQueryBuilder, expr: Expr | None = None) -> str:
+        return compute_rc_signature_from_body(env.soap_request_body, env.context)
+
+    return {"rc_signature": rc_signature_resolver}

spinta/adapters/soap_plugins.py

@@ -0,0 +1,122 @@
+"""Load optional SOAP adapter modules from ``soap_adapter_modules`` paths in config."""
+
+import importlib.util
+import logging
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Callable
+
+from spinta.core.ufuncs import Expr, UFuncRegistry
+from spinta.core.config import RawConfig
+from spinta.ufuncs.loadbuilder.components import LoadBuilder
+from spinta.datasets.backends.dataframe.backends.soap.ufuncs.components import SoapQueryBuilder
+
+log = logging.getLogger(__name__)
+
+
+_deferred_prepare_names_cache: set[str] | None = None
+
+
+def get_deferred_prepare_names() -> set[str]:
+    """Names of prepare ufuncs left unresolved until the SOAP body is built (empty before register)."""
+    return _deferred_prepare_names_cache or set()
+
+
+def _load_adapter_module_from_path(file_path: str, raw_config: RawConfig | None = None) -> ModuleType | None:
+    try:
+        path = Path(file_path)
+        if not path.exists():
+            log.warning("SOAP adapter module not found: %s", file_path)
+            return None
+        spec = importlib.util.spec_from_file_location(path.stem, path)
+        if spec is None or spec.loader is None:
+            log.warning("Failed to create module spec for: %s", file_path)
+            return None
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        log.info("Loaded SOAP adapter module from: %s", file_path)
+        if hasattr(module, "validate_soap_adapter_config"):
+            module.validate_soap_adapter_config(raw_config)
+        return module
+    except Exception as e:
+        log.warning("Failed to load SOAP adapter module %s: %s", file_path, e)
+        return None
+
+
+def _load_adapters_from_config(raw_config: RawConfig | None) -> tuple[set[str], dict[str, Callable[..., Any]]]:
+    deferred: set[str] = set()
+    body_resolvers: dict[str, Callable[..., Any]] = {}
+
+    if raw_config is None:
+        return deferred, body_resolvers
+
+    module_paths = raw_config.get("soap_adapter_modules", default=[])
+    if not module_paths:
+        return deferred, body_resolvers
+
+    if isinstance(module_paths, str):
+        module_paths = [module_paths]
+
+    for module_path in module_paths:
+        module = _load_adapter_module_from_path(module_path, raw_config)
+        if module is None:
+            continue
+
+        if hasattr(module, "get_deferred_prepare_names"):
+            try:
+                result = module.get_deferred_prepare_names()
+            except Exception as e:
+                log.warning("Failed to call get_deferred_prepare_names in %s: %s", module_path, e)
+            else:
+                if isinstance(result, (list, tuple)):
+                    deferred.update(result)
+                else:
+                    deferred.add(str(result))
+
+        if hasattr(module, "get_body_resolvers"):
+            try:
+                result = module.get_body_resolvers()
+            except Exception as e:
+                log.warning("Failed to call get_body_resolvers in %s: %s", module_path, e)
+            else:
+                if isinstance(result, dict):
+                    body_resolvers.update(result)
+
+    return deferred, body_resolvers
+
+
+def _deferred_prepare_resolver(env, expr: Expr) -> Expr:
+    env.param.soap_body = {env.this: expr}
+    return expr
+
+
+def _make_body_resolver(callable_fn: Callable[..., Any]) -> Callable[[SoapQueryBuilder, Expr], Any]:
+    """Adapter may implement ``fn(env)`` only; fall back if ``fn(env, expr)`` raises ``TypeError``."""
+
+    def body_resolver(env: SoapQueryBuilder, expr: Expr) -> Any:
+        try:
+            return callable_fn(env, expr)
+        except TypeError:
+            return callable_fn(env)
+
+    return body_resolver
+
+
+def register_soap_ufuncs(registry: UFuncRegistry, raw_config: RawConfig | None = None) -> None:
+    """Register SOAP adapter resolvers from config (`soap_adapter_modules`).
+    Adapters are loaded from local Python files listed in:
+        soap_adapter_modules:
+          - /path/to/adapter.py
+    Call this after resolver.collect().
+    """
+    global _deferred_prepare_names_cache
+
+    deferred, body_resolvers = _load_adapters_from_config(raw_config)
+
+    _deferred_prepare_names_cache = deferred
+
+    for name in deferred:
+        registry.register(name, _deferred_prepare_resolver, (LoadBuilder, Expr))
+
+    for name, callable_fn in body_resolvers.items():
+        registry.register(name, _make_body_resolver(callable_fn), (SoapQueryBuilder, Expr))

spinta/cli/config.py

@@ -1,16 +1,15 @@
-from typing import List
-from typing import Optional
+from typing import List, Optional
 
-from typer import Argument
+from typer import Argument, Option, echo
 from typer import Context as TyperContext
-from typer import Option
-from typer import echo
 
+from spinta import commands
 from spinta.cli.helpers.manifest import convert_str_to_manifest_path
 from spinta.cli.helpers.store import prepare_manifest
-from spinta.core.enums import Mode
+from spinta.components import Context, Store
 from spinta.core.config import KeyFormat
 from spinta.core.context import configure_context
+from spinta.core.enums import Mode
 
 
 def config(
@@ -32,9 +31,16 @@ def check(
 ):
     """Check configuration and manifests"""
     manifests = convert_str_to_manifest_path(manifests)
-    context = configure_context(ctx.obj, manifests, mode=mode, check_names=check_names)
+    context: Context = configure_context(ctx.obj, manifests, mode=mode, check_names=check_names)
     prepare_manifest(context, ensure_config_dir=True, full_load=True)
     manager = context.get("error_manager")
+
+    store: Store = context.get("store")
+    if store.manifest:
+        for model in commands.get_models(context, store.manifest).values():
+            if model.external and model.external.resource and model.external.resource.backend:
+                commands.check(context, model, model.external.resource.backend)
+
     handler = manager.handler
 
     if handler.get_counts():

spinta/commands/read.py

@@ -123,11 +123,10 @@ def getall(context: Context, model: Model, page: Page, *, query: Expr = None, li
     page.size = size + 1
 
     page_meta = PaginationMetaData(page_size=size, limit=limit)
+    paginated_query = add_page_expr(query, page)
     while not page_meta.is_finished:
         page_meta.is_finished = True
-        query = add_page_expr(query, page)
-        rows = commands.getall(context, model, backend, query=query, **kwargs)
-
+        rows = commands.getall(context, model, backend, query=paginated_query, **kwargs)
         yield from get_paginated_values(page, page_meta, rows, extract_source_page_keys)
 
 

spinta/components.py

@@ -1095,6 +1095,7 @@ class Config:
     scope_log: bool
     check_contract_scopes: bool
     default_auth_client: str
+    default_access_level: str
     http_basic_auth: bool
     token_validation_key: dict | None = None
     token_validation_keys_download_url: str | None = None

spinta/config.py

@@ -54,7 +54,6 @@ CONFIG = {
             "xml": "spinta.manifests.dict.components:XmlManifest",
             "internal": "spinta.manifests.internal_sql.components:InternalSQLManifest",
             "xsd": "spinta.manifests.xsd.components:XsdManifest",
-            "xsd2": "spinta.manifests.xsd2.components:XsdManifest2",
             "openapi": "spinta.manifests.open_api.components:OpenAPIManifest",
         },
         "backends": {
@@ -347,6 +346,7 @@ CONFIG = {
             },
             "config_path": pathlib.Path("tests/config"),
             "default_auth_client": "baa448a8-205c-4faa-a048-a10e4b32a136",
+            "default_access_level": "protected",
             "sync_retry_count": 0,
         },
     },

spinta/config.yml

@@ -195,6 +195,9 @@ always_show_id:
 default_auth_client:
   type: string
 
+default_access_level:
+  type: string
+
 check:
   type: object
   items:

spinta/core/access.py

@@ -73,6 +73,7 @@ def link_access_param(
     ] = (),
     *,
     use_given: bool = True,
+    default_access: Access = Access.private,
 ) -> None:
     if component.access is None:
         for parent in parents:
@@ -83,4 +84,4 @@ def link_access_param(
                 component.access = candidate
                 break
         else:
-            component.access = Access.protected
+            component.access = default_access

spinta/core/ufuncs.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import dataclasses
 import functools
 import importlib
-from typing import Any, Optional, List, Union
+from typing import Any, Optional, List, Union, Callable
 from typing import Dict
 from typing import TYPE_CHECKING
 from typing import Tuple
@@ -140,6 +140,10 @@ class UFuncRegistry:
             # automatically by __call__.
             importlib.import_module(module)
 
+    def register(self, name: str, func: Callable, types: tuple) -> None:
+        """Register a single (name, func, types) for plugin-discovered resolvers."""
+        self._ufuncs.append((name, func, types))
+
     def ufuncs(self):
         # Create commands from collected ufuncs.
         ufuncs = {}

spinta/datasets/backends/dataframe/backends/soap/ufuncs/ufuncs.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import logging
 from typing import Any
 
 from lxml import etree
 
+from spinta.adapters.soap_plugins import get_deferred_prepare_names
 from spinta.auth import authorized, query_client
 from spinta.components import Property
 from spinta.core.enums import Action
@@ -21,6 +23,10 @@ from spinta.utils.config import get_clients_path
 from spinta.utils.data import take
 from spinta.utils.schema import NA
 
+log = logging.getLogger(__name__)
+
+SOAP_BODY_VALUE_TYPE_CDATA = "cdata"
+
 
 class MakeCDATA:
     """
@@ -47,7 +53,7 @@ def eq_(env: SoapQueryBuilder, field: Bind, value: object) -> Expr | None:
     try:
         prop = env.resolve_property(field)
     except PropertyNotFound:
-        # leave query parameter for other query builders to resolve
+        # leave query parameter for other resolvers
         return Expr("eq", field, value)
 
     if not isinstance(prop.external.prepare, Expr):
@@ -59,7 +65,7 @@ def eq_(env: SoapQueryBuilder, field: Bind, value: object) -> Expr | None:
 @ufunc.resolver(SoapQueryBuilder, Expr, name="and")
 def and_(env: SoapQueryBuilder, expr: Expr) -> list[Any]:
     args, kwargs = expr.resolve(env)
-    args = [a for a in args if a is not None]
+    args = [arg_item for arg_item in args if arg_item is not None]
     return env.call("and", args)
 
 
@@ -72,16 +78,88 @@ def and_(env: SoapQueryBuilder, args: list) -> Any:
 
 
 def _finalize_soap_request_body_resolve(env: SoapQueryBuilder) -> None:
+    deferred_names = get_deferred_prepare_names()
+
     for param_body_key, param_body_value in env.soap_request_body.items():
         if not isinstance(param_body_value, Expr):
             continue
+
+        if getattr(param_body_value, "name", None) in deferred_names:
+            continue
+
         env.soap_request_body[param_body_key] = env.resolve(param_body_value)
 
 
+def _param_for_soap_body_key(env: SoapQueryBuilder, param_source: str, expr: Expr | None = None) -> Param | None:
+    """Find the resource Param that owns this SOAP body slot (flat key like ``input/Signature``)."""
+    param_lower = param_source.lower()
+
+    for resource_param in env.params.values():
+        soap_body = getattr(resource_param, "soap_body", None)
+        if not soap_body:
+            continue
+
+        if param_source in soap_body:
+            return resource_param
+
+        if any(soap_key.lower() == param_lower for soap_key in soap_body):
+            return resource_param
+
+        if expr is not None:
+            for prepared_value in soap_body.values():
+                if prepared_value is expr:
+                    return resource_param
+
+    return None
+
+
+def _property_value_key_for_deferred(matched_param: Param | None, param_source: str) -> str:
+    if matched_param is not None:
+        return matched_param.name
+    path_suffix = param_source.rsplit("/", 1)[-1]
+    return path_suffix.lower() if path_suffix else param_source
+
+
+def _resolve_deferred_soap_request_body_exprs(env: SoapQueryBuilder) -> None:
+    deferred_names = get_deferred_prepare_names()
+    if not deferred_names:
+        return
+
+    for param_source, deferred_expr in list(env.soap_request_body.items()):
+        if not isinstance(deferred_expr, Expr):
+            continue
+
+        if getattr(deferred_expr, "name", None) not in deferred_names:
+            continue
+
+        resolved = env.resolve(deferred_expr)
+        matched_param = _param_for_soap_body_key(env, param_source, expr=deferred_expr)
+
+        if (
+            matched_param
+            and getattr(matched_param, "soap_body_value_type", None) == SOAP_BODY_VALUE_TYPE_CDATA
+            and resolved
+        ):
+            env.soap_request_body[param_source] = MakeCDATA(resolved)
+        else:
+            env.soap_request_body[param_source] = resolved
+
+        property_value_key = _property_value_key_for_deferred(matched_param, param_source)
+        env.property_values[property_value_key] = resolved
+
+        if matched_param is None:
+            log.warning(
+                "SOAP deferred resolve: no Param matched for %r; property_values updated under %r",
+                param_source,
+                property_value_key,
+            )
+
+
 def _populate_soap_request_body_with_url_values(env: SoapQueryBuilder) -> None:
     for prop in take(env.model.properties).values():
         if not authorized(env.context, prop, Action.GETALL):
             continue
+
         env.call("soap_request_body", prop)
 
 
@@ -93,25 +171,24 @@ def soap_request_body(env: SoapQueryBuilder) -> None:
     for param in env.params.values():
         if not hasattr(param, "soap_body"):
             continue
+
         env.soap_request_body.update(param.soap_body)
 
     _finalize_soap_request_body_resolve(env)
     _populate_soap_request_body_with_url_values(env)
+    _resolve_deferred_soap_request_body_exprs(env)
 
 
 @ufunc.resolver(SoapQueryBuilder, Property)
 def soap_request_body(env: SoapQueryBuilder, prop: Property) -> None:
     """
-    Only care about properties that describe URL query parameters:
+    Only care for properties that describe URL query parameters:
         properties without `source` and with `prepare`.
-    We ignore the rest.
     """
     if prop.external.name:
-        # Ignore properties with `source`
         return None
 
     if not isinstance(prop.external.prepare, Expr):
-        # Ignore properties without `prepare` expression
         return None
 
     resource_param = env(this=prop).resolve(prop.external.prepare)
@@ -129,25 +206,41 @@ def _get_final_soap_request_body_value(env: SoapQueryBuilder, property_name: str
 
 @ufunc.resolver(SoapQueryBuilder, Property, Param)
 def soap_request_body(env: SoapQueryBuilder, prop: Property, param: Param) -> None:
-    """Replace default param.soap_body values with url_param values"""
-    param_source = next(iter(param.soap_body))
+    """Merge URL query params into the SOAP body; resolve non-deferred Expr; keep deferred Expr for a later pass."""
+    soap_body = getattr(param, "soap_body", None)
+    if not soap_body:
+        return
+
+    param_source = next(iter(soap_body))
+    deferred_names = get_deferred_prepare_names()
+
     final_value = _get_final_soap_request_body_value(env, prop.place, param_source)
 
+    if isinstance(final_value, Expr) and getattr(final_value, "name", None) not in deferred_names:
+        final_value = env.resolve(final_value)
+
+    is_deferred_expr = isinstance(final_value, Expr) and getattr(final_value, "name", None) in deferred_names
+
     if final_value is NA:
-        # If value not in URL and DSA has no default - remove it from SOAP request completely
         env.soap_request_body.pop(param_source, None)
         final_value = None
-    elif final_value:
-        # If value should be sent as CDATA - change it to etree.CDATA
-        soap_final_value = MakeCDATA(final_value) if param.soap_body_value_type == "cdata" else final_value
+
+    elif is_deferred_expr:
+        env.soap_request_body[param_source] = final_value
+
+    else:
+        if param.soap_body_value_type == SOAP_BODY_VALUE_TYPE_CDATA and final_value:
+            soap_final_value = MakeCDATA(final_value)
+        else:
+            soap_final_value = final_value
+
         env.soap_request_body[param_source] = soap_final_value
 
-    # Check if required property values exist
     if prop.dtype.required and param_source not in env.soap_request_body:
         raise MissingRequiredProperty(prop, prop=prop.name)
 
-    # Update property values. Even if value is not sent via SOAP, it should have None value when displayed by Spinta
-    env.property_values.update({param.name: final_value})
+    if not is_deferred_expr:
+        env.property_values.update({param.name: final_value})
 
 
 @ufunc.resolver(SoapQueryBuilder, str)

spinta/datasets/backends/dataframe/commands/check.py

@@ -0,0 +1,33 @@
+from spinta import commands, exceptions
+from spinta.components import Context, Model
+from spinta.core.ufuncs import Expr
+from spinta.datasets.backends.dataframe.components import DaskBackend
+from spinta.datasets.backends.dataframe.ufuncs.query.ufuncs import COMPARE
+from spinta.handlers import ErrorManager
+
+
+def _get_compare_operators(expr: Expr) -> list[str]:
+    """Recursively collect compare operator (filters) names from an Expr tree."""
+    if not isinstance(expr, Expr):
+        return []
+    filters: list[str] = []
+    if expr.name in COMPARE:
+        filters.append(expr.name)
+    for arg in expr.args:
+        filters.extend(_get_compare_operators(arg))
+    for val in expr.kwargs.values():
+        filters.extend(_get_compare_operators(val))
+    return filters
+
+
+@commands.check.register(Context, Model, DaskBackend)
+def check(context: Context, model: Model, backend: DaskBackend) -> None:
+    # If there are any compare operators in the prepare expression,
+    # raise an error since they are not supported for Dask backend.
+    if (
+        model.external
+        and model.external.prepare
+        and (compare_operators := list(dict.fromkeys(_get_compare_operators(model.external.prepare))))
+    ):
+        manager: ErrorManager = context.get("error_manager")
+        manager.handle_error(exceptions.DaskBackendCompareNotSupported(model, operators=compare_operators))

spinta/datasets/backends/dataframe/ufuncs/query/components.py

@@ -8,6 +8,7 @@ from spinta.components import Model, Property
 from spinta.core.ufuncs import Env, Expr
 
 from spinta.exceptions import UnknownMethod
+from spinta.ufuncs.propertyresolver.components import PropertyResolver
 from spinta.ufuncs.querybuilder.components import Selected
 from spinta.utils.schema import NA
 from spinta.datasets.backends.dataframe.components import DaskBackend
@@ -19,6 +20,7 @@ class DaskDataFrameQueryBuilder(Env):
     dataframe: DataFrame
     params: dict
     url_query_params: Expr | None
+    property_resolver: PropertyResolver = None
 
     def init(self, backend: DaskBackend, dataframe: DataFrame, params: dict) -> DaskDataFrameQueryBuilder:
         return self(
@@ -55,6 +57,14 @@ class DaskDataFrameQueryBuilder(Env):
     def default_resolver(self, expr, *args, **kwargs):
         raise UnknownMethod(name=expr.name, expr=str(expr(*args, **kwargs)))
 
+    def resolve_property(self, *args, **kwargs) -> Property:
+        if self.property_resolver is None:
+            resolver = PropertyResolver(self.context)
+            resolver = resolver.init(model=self.model, ufunc_types=True)
+            self.property_resolver = resolver
+        result = self.property_resolver.resolve_property(*args, **kwargs)
+        return result
+
 
 class DaskSelected(Selected):
     # Item name in select list.