spinta 0.2.dev27__tar.gz → 0.2.dev29__tar.gz

{spinta-0.2.dev27 → spinta-0.2.dev29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev27
+Version: 0.2.dev29
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE
@@ -73,7 +73,7 @@ Requires-Dist: sqlalchemy (>=1.4,<1.5)
 Requires-Dist: sqlalchemy-utils (<=0.41.2)
 Requires-Dist: sqlean-py (<=3.49.1)
 Requires-Dist: sqlparse
-Requires-Dist: starlette (>=0.52.1,<1.0.0)
+Requires-Dist: starlette (>=1.3.1,<2.0.0)
 Requires-Dist: tabulate
 Requires-Dist: toposort
 Requires-Dist: tqdm

{spinta-0.2.dev27 → spinta-0.2.dev29}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev27"
+version = "0.2dev29"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"
@@ -57,7 +57,7 @@ requests = "*"
 setuptools = "*"
 setuptools-scm = "*"
 # https://github.com/encode/starlette/security/advisories/GHSA-f96h-pmfr-66vw
-starlette = ">=0.52.1,<1.0.0"
+starlette = ">=1.3.1,<2.0.0"
 pydantic = "*"
 toposort = "*"
 tqdm = "*"
@@ -141,8 +141,8 @@ docutils = "*"
 requests-mock = "^1.11.0"
 ruff = "^0.12.10"
 
-# Starlette
-httpx = "*"
+# Starlette test client (starlette >=1.0 requires httpx2)
+httpx2 = "*"
 
 # Docs
 sphinx = "*"

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/api/__init__.py

@@ -50,7 +50,7 @@ from spinta.exceptions import (
     error_response,
 )
 from spinta.formats.html.helpers import get_templates
-from spinta.middlewares import ContextMiddleware, StrictTransportSecurityMiddleware
+from spinta.middlewares import ContextMiddleware, PathNormalizationMiddleware, StrictTransportSecurityMiddleware
 from spinta.urlparams import Version, get_response_type
 
 log = logging.getLogger(__name__)
@@ -370,7 +370,7 @@ async def error(request, exc):
     elif isinstance(exc, BaseError):
         status_code = exc.status_code
         errors = [error_response(exc)]
-        headers = exc.headers
+        headers = dict(exc.headers)
     else:
         if isinstance(exc, HTTPException):
             status_code = exc.status_code
@@ -400,6 +400,11 @@ async def error(request, exc):
 
     response = {"errors": errors}
 
+    # Error responses can reflect request input (e.g. the requested path) and
+    # must never be stored by shared caches, otherwise they can be used for
+    # web cache poisoning.
+    headers["Cache-Control"] = "no-store"
+
     fmt = get_response_type(request.state.context, request)
     if fmt == "json" or fmt is None:
         return JSONResponse(
@@ -414,6 +419,7 @@ async def error(request, exc):
         }
 
         return templates.TemplateResponse(
+            request,
             "error.html",
             response,
             status_code=status_code,
@@ -467,6 +473,7 @@ def init(context: Context):
             StrictTransportSecurityMiddleware,
             value=config.http_strict_transport_security,
         ),
+        Middleware(PathNormalizationMiddleware),
         Middleware(ContextMiddleware, context=context),
     ]
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/auth.py

@@ -538,7 +538,7 @@ class AdminToken(rfc6749.TokenMixin):
         return True
 
     def check_scope(self, scope: SCOPE_TYPE, **kwargs) -> bool:
-        pass
+        return True
 
     def get_sub(self) -> str:  # User.
         return "admin"
@@ -864,7 +864,6 @@ def authorized(
 
     # Unauthenticated clients can only access nodes if spinta config.access is open.
     unauthenticated = token.get_client_id() == get_default_auth_client_id(context)
-
     if unauthenticated and config.access < Access.open:
         if throw:
             raise AuthorizedClientsOnly()

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/backends/__init__.py

@@ -590,7 +590,7 @@ def is_object_id(context: Context, value: str):
 
 @is_object_id.register(Context, Backend, Model, str)
 def is_object_id(context: Context, backend: Backend, model: Model, value: str):
-    return is_object_id(context, backend, model.properties["_id"].dtype, value)
+    return is_object_id(context, backend, model.id_prop.dtype, value)
 
 
 @is_object_id.register(Context, Backend, PrimaryKey, str)
@@ -1940,6 +1940,31 @@ def cast_backend_to_python(context: Context, dtype: Ref, backend: Backend, data:
     return processed_data
 
 
+@commands.cast_backend_to_python.register(Context, ExternalRef, Backend, dict)
+def cast_backend_to_python(
+    context: Context, dtype: ExternalRef, backend: Backend, data: Dict[str, Any], **kwargs
+) -> Any:
+    if not data:
+        return data
+
+    processed_data = {}
+    for key in data:
+        prop = commands.resolve_property(dtype.prop.model, f"{dtype.prop.place}.{key}")
+        if prop is not None:
+            processed_data[key] = commands.cast_backend_to_python(context, prop, backend, data[key], **kwargs)
+
+    for prop in dtype.refprops:
+        if prop.name not in processed_data and prop.name in data:
+            processed_data[prop.name] = commands.cast_backend_to_python(
+                context, prop, backend, data[prop.name], **kwargs
+            )
+
+    if not processed_data or all(value is None for value in processed_data.values()):
+        return None
+
+    return processed_data
+
+
 @commands.cast_backend_to_python.register(Context, Object, Backend, dict)
 def cast_backend_to_python(
     context: Context, dtype: Object, backend: Backend, data: Dict[str, Any], **kwargs

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/backends/components.py

@@ -36,6 +36,9 @@ class Backend:
     result_builder_type: str = ""
     result_builder_class: Type[ResultBuilder]
 
+    # Some backends might only work with source given to models and some might not need it
+    model_requires_source: bool = False
+
     def __repr__(self):
         return f"<{self.__class__.__module__}.{self.__class__.__name__}(name={self.name!r}) at 0x{id(self):02x}>"
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/backends/postgresql/helpers/exceptions.py

@@ -19,9 +19,7 @@ def create_exception(item: DataItem, error: psy_errors.ForeignKeyViolation):
     error_message = error.diag.message_detail
     if item.action is Action.DELETE:
         error_model = extractor.extract_error_model(error_message)
-        return exceptions.ReferringObjectFound(
-            item.model.properties.get("_id"), model=error_model, id=item.saved.get("_id")
-        )
+        return exceptions.ReferringObjectFound(item.model.id_prop, model=error_model, id=item.saved.get("_id"))
     else:
         error_property_names = extractor.extract_error_property_names(error_message)
         error_ref_id = extractor.extract_error_ref_id(error_message)

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/backends/postgresql/ufuncs/query/components.py

@@ -57,11 +57,9 @@ class PgQueryBuilder(QueryBuilder):
             self.call("select", Expr("select"))
 
         if not self.aggregate:
-            self.selected["_id"] = Selected(
-                item=self.add_column(self.table.c["_id"]), prop=self.model.properties["_id"]
-            )
+            self.selected["_id"] = Selected(item=self.add_column(self.table.c["_id"]), prop=self.model.id_prop)
             self.selected["_revision"] = Selected(
-                item=self.add_column(self.table.c["_revision"]), prop=self.model.properties["_revision"]
+                item=self.add_column(self.table.c["_revision"]), prop=self.model.revision_prop
             )
         merged_selected = merge_with_page_selected_list(self.columns, self.page)
         merged_sorted = merge_with_page_sort(self.sort, self.page)
@@ -116,7 +114,7 @@ class PgQueryBuilder(QueryBuilder):
             else:
                 rtable = self.backend.get_table(rmodel).alias()
 
-            rpkey = self.backend.get_column(rtable, rmodel.properties["_id"])
+            rpkey = self.backend.get_column(rtable, rmodel.id_prop)
 
             if isinstance(lrkey, list) and not isinstance(rpkey, list):
                 if len(lrkey) == 1:
@@ -199,10 +197,10 @@ class PgQueryBuilder(QueryBuilder):
             raise PropertyNotFound(model, property=prop)
 
         ltable = self.backend.get_table(inherit_model)
-        lrkey = self.backend.get_column(ltable, inherit_model.properties["_id"])
+        lrkey = self.backend.get_column(ltable, inherit_model.id_prop)
 
         rtable = self.backend.get_table(base_model).alias()
-        rpkey = self.backend.get_column(rtable, base_model.properties["_id"])
+        rpkey = self.backend.get_column(rtable, base_model.id_prop)
 
         if base_model.name in self.joins:
             if self.joins[base_model.name] == rtable:

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/backends/postgresql/ufuncs/query/ufuncs.py

@@ -287,7 +287,7 @@ def select(env, dtype: Ref):
     if not dtype.inherited:
         name = "_id"
         if env.query_params.prioritize_uri and uri is not None:
-            fpr = ForeignProperty(None, dtype, dtype.model.properties["_id"].dtype)
+            fpr = ForeignProperty(None, dtype, dtype.model.id_prop.dtype)
             table = env.get_joined_table(fpr)
             column = table.c[uri.place]
             name = "_uri"
@@ -311,7 +311,7 @@ def select(env, dtype: ExternalRef):
         if dtype.model.given.pkeys or dtype.explicit:
             props = dtype.refprops
         else:
-            props = [dtype.model.properties["_id"]]
+            props = [dtype.model.id_prop]
         for prop in props:
             name = f"{dtype.prop.place}.{prop.place}"
             column = table.c[name]
@@ -350,7 +350,7 @@ def _select_backref(env, dtype, is_array=False):
     return_columns = {}
 
     if commands.identifiable(refprop):
-        id_ = fpr.right.prop.model.properties["_id"]
+        id_ = fpr.right.prop.model.id_prop
         column_name = id_.name
         label = f"{dtype.prop.name}.{column_name}"
         required_columns.append((column_name, label))
@@ -431,7 +431,7 @@ def _denorm_to_foreign_property(env: PgQueryBuilder, dtype: Denorm):
                 and isinstance(root_ref_parent, Property)
                 and isinstance(root_ref_parent.dtype, (Ref, BackRef))
             ):
-                fpr = ForeignProperty(fpr, root_ref_parent.dtype, root_ref_parent.dtype.model.properties["_id"].dtype)
+                fpr = ForeignProperty(fpr, root_ref_parent.dtype, root_ref_parent.dtype.model.id_prop.dtype)
 
                 if not root_ref_parent.dtype.inherited:
                     break

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/cli/helpers/admin/scripts/add_local_ids.py

@@ -25,7 +25,7 @@ def add_explicit_id_properties(context: Context, manifest: str, verbose: bool =
             if verbose:
                 echo(f'Model "{model.model_type()}" has no `ref`, skipping.', err=True)
             continue
-        prop = model.properties.get("_id")
+        prop = model.id_prop
         if prop is None or prop.explicitly_given:
             continue
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/cli/helpers/admin/scripts/remove_local_ids.py

@@ -19,7 +19,7 @@ def remove_explicit_id_properties(context: Context, manifest: str) -> int:
     for model in commands.get_models(context, manifest).values():
         if model.model_type().startswith(RESERVED_MODEL_PREFIX):
             continue
-        prop = model.properties.get("_id")
+        prop = model.id_prop
         if prop is None or not prop.explicitly_given:
             continue
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/cli/helpers/push/read.py

@@ -55,9 +55,9 @@ def _iter_model_rows(
         else:
             stream = read_model_data(context, model, limit, stop_on_error, params=params)
             for item in stream:
-                if push_counter:
+                if push_counter is not None:
                     push_counter.update(1)
-                if model_push_counter:
+                if model_push_counter is not None:
                     model_push_counter.update(1)
                 yield PushRow(model, item)
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/cli/helpers/store.py

@@ -166,4 +166,4 @@ def attach_backends(context: Context, store: Store, manifest: Manifest) -> None:
 
 def attach_keymaps(context: Context, store: Store) -> None:
     for keymap in store.keymaps.values():
-        context.attach(f"keymap.{keymap.name}", lambda: keymap)
+        context.attach(f"keymap.{keymap.name}", lambda: keymap.copy())

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/cli/pii.py

@@ -220,7 +220,7 @@ def detect(
                         f"transaction.{resource.backend.name}", validate_and_return_begin, context, resource.backend
                     )
         for keymap in store.keymaps.values():
-            context.attach(f"keymap.{keymap.name}", lambda: keymap)
+            context.attach(f"keymap.{keymap.name}", lambda: keymap.copy())
 
         ns = commands.get_namespace(context, manifest, "")
         models = commands.traverse_ns_models(context, ns, manifest, Action.SEARCH)

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/components.py

@@ -662,7 +662,6 @@ class Model(MetaData):
     comments: List[Comment] = None
     base: Base = None
     uri: str = None
-    uri_prop: Property = None
     page: PageInfo = None
     features: str = None
     status: Status | None = None
@@ -675,6 +674,11 @@ class Model(MetaData):
     distribution_strategy: DistributionStrategy | None = None
     required_keymap_properties = None
 
+    # Quick access properties
+    uri_prop: Property | None = None
+    id_prop: Property
+    revision_prop: Property
+
     schema = {
         "keymap": {"type": "string"},
         "backend": {"type": "string"},
@@ -741,7 +745,7 @@ class Model(MetaData):
         # return self.name.split('/')[-1]
 
     def add_keymap_property_combination(self, given_props: List[Property]):
-        extract_names = list([prop.name for prop in given_props])
+        extract_names = tuple([prop.place for prop in given_props])
         if extract_names not in self.required_keymap_properties:
             self.required_keymap_properties.append(extract_names)
 

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/datasets/backends/dataframe/backends/csv/components.py

@@ -6,6 +6,9 @@ from spinta.datasets.backends.dataframe.components import DaskBackend
 class Csv(DaskBackend):
     type: str = "dask/csv"
 
+    # Csv data is mainly one table, meaning model.source is usually not given
+    model_requires_source: bool = False
+
     @contextlib.contextmanager
     def begin(self):
         yield

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/datasets/backends/dataframe/backends/memory/components.py

@@ -6,6 +6,9 @@ from spinta.datasets.backends.dataframe.components import DaskBackend
 class MemoryDaskBackend(DaskBackend):
     type: str = "dask/memory"
 
+    # Memory backends can set model.source, but it can also leave it open and still work
+    model_requires_source: bool = False
+
     @contextlib.contextmanager
     def begin(self):
         yield

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/datasets/backends/dataframe/commands/read.py

@@ -16,16 +16,14 @@ from spinta.core.ufuncs import Env, Expr
 from spinta.datasets.backends.dataframe.backends.memory.components import MemoryDaskBackend
 from spinta.datasets.backends.dataframe.components import DaskBackend
 from spinta.datasets.backends.dataframe.ufuncs.query.components import DaskDataFrameQueryBuilder
-from spinta.datasets.backends.helpers import handle_ref_key_assignment
 from spinta.datasets.components import Resource
 from spinta.datasets.helpers import encode_composite_string_id, get_enum_filters, get_ref_filters
-from spinta.datasets.keymaps.components import KeyMap
 from spinta.datasets.utils import iterparams
 from spinta.dimensions.enum.helpers import get_prop_enum
 from spinta.dimensions.param.components import ResolvedParams
 from spinta.exceptions import NoExternalName, PropertyNotFound, ValueNotInEnum
 from spinta.manifests.components import Manifest
-from spinta.types.datatype import Base32, Boolean, DataType, DateTime, Integer, Number, PrimaryKey, Ref
+from spinta.types.datatype import Base32, Boolean, DataType, DateTime, Integer, Number, Ref
 from spinta.typing import ObjectData
 from spinta.ufuncs.helpers import merge_formulas
 from spinta.ufuncs.querybuilder.components import Selected
@@ -182,7 +180,7 @@ def _select_primary_key(model: Model, df: DataFrame):
         result = _select(prop, df)
     else:
         result = [_select(prop, df) for prop in pkeys]
-    return Selected(prop=model.properties["_id"], prep=result)
+    return Selected(prop=model.id_prop, prep=result)
 
 
 def get_pkeys_if_ref(prop: Property) -> list[str]:
@@ -258,7 +256,6 @@ def dask_get_all(
     params: dict | None = None,
 ):
     params = params or {}
-    keymap: KeyMap = context.get(f"keymap.{model.keymap.name}")
 
     query = merge_formulas(model.external.prepare, query)
     query = merge_formulas(query, get_enum_filters(context, model))
@@ -279,16 +276,7 @@ def dask_get_all(
         for key, sel in env_selected.items():
             val = _get_row_value(context, row, sel, env.params)
             if sel.prop:
-                if isinstance(sel.prop.dtype, PrimaryKey):
-                    if isinstance(val, list):
-                        val = [
-                            list_value.get("_id", list_value) if isinstance(list_value, dict) else list_value
-                            for list_value in val
-                        ]
-                    val = keymap.encode(sel.prop.model.model_type(), val)
-                elif isinstance(sel.prop.dtype, Ref):
-                    val = handle_ref_key_assignment(context, keymap, env, val, sel.prop.dtype)
-                elif is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
+                if is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
                     val = encode_composite_string_id(val, model.external.pkeys)
             res[key] = val
         res = flat_dicts_to_nested(res, list_keys=list_keys)

{spinta-0.2.dev27 → spinta-0.2.dev29}/spinta/datasets/backends/dataframe/ufuncs/query/components.py

@@ -15,6 +15,8 @@ from spinta.ufuncs.propertyresolver.components import PropertyResolver
 from spinta.ufuncs.querybuilder.components import Func, Selected
 from spinta.utils.schema import NA
 
+RESERVED_COUNT_PROP = "__dask_count"
+
 
 class DaskDataFrameQueryBuilder(Env):
     backend: DaskBackend
@@ -90,9 +92,6 @@ class DaskSelected(Selected):
     prep: Any = NA
 
 
-RESERVED_COUNT_PROP = "__dask_count"
-
-
 @dataclasses.dataclass
 class Count(Func):
     pass

spinta-0.2.dev29/spinta/datasets/backends/dataframe/ufuncs/query/helpers.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from spinta.datasets.backends.dataframe.ufuncs.query.components import (
+    DaskDataFrameQueryBuilder,
+)
+from spinta.datasets.backends.dataframe.ufuncs.query.components import (
+    DaskSelected as Selected,
+)
+from spinta.exceptions import (
+    GivenValueCountMissmatch,
+    PropertyNotPartOfRefProps,
+)
+from spinta.types.datatype import ExternalRef, Ref
+
+
+def select_ref_foreign_key_properties(
+    env: DaskDataFrameQueryBuilder, dtype: Ref, *, properties: list = None
+) -> dict[str, Selected]:
+    if properties is None:
+        if dtype.prop.external and dtype.prop.external.prepare:
+            properties = env(this=dtype.prop).resolve(dtype.prop.external.prepare)
+
+    prep = {}
+    given_mapping_count = len(properties) if properties else 1
+    if len(dtype.refprops) != given_mapping_count:
+        raise GivenValueCountMissmatch(dtype, given_count=given_mapping_count, expected_count=len(dtype.refprops))
+
+    if properties is None:
+        refprop = dtype.refprops[0]
+        # Currently, dask can only access its own dataframe columns, so we we have to use dtype.prop.external.name as source.
+        prep[refprop.name] = Selected(item=dtype.prop.external.name, prop=refprop)
+    else:
+        for i, prop in enumerate(dtype.refprops):
+            sel = env.call("select", properties[i])
+            if sel.prop is None:
+                sel.prop = prop
+            prep[prop.name] = sel
+    return prep
+
+
+def select_external_ref_foreign_key_properties(
+    env: DaskDataFrameQueryBuilder, dtype: ExternalRef, *, properties: list = None, target: str = None
+) -> dict[str, Selected]:
+    if properties is None:
+        if dtype.prop.external and dtype.prop.external.prepare:
+            properties = env(this=dtype.prop).resolve(dtype.prop.external.prepare)
+
+    if target is not None:
+        value = next((prop for prop in dtype.refprops if prop.name == target), None)
+        if value is None:
+            raise PropertyNotPartOfRefProps(dtype, prop=target, refprops=list(prop_.name for prop_ in dtype.refprops))
+        target = value
+
+    prep = {}
+    given_mapping_count = len(properties) if properties else 1
+    if len(dtype.refprops) != given_mapping_count:
+        raise GivenValueCountMissmatch(dtype, given_count=given_mapping_count, expected_count=len(dtype.refprops))
+
+    if properties is None:
+        refprop = dtype.refprops[0]
+        # Currently, dask can only access its own dataframe columns, so we we have to use dtype.prop.external.name as source.
+        prep[refprop.name] = Selected(item=dtype.prop.external.name, prop=refprop)
+    else:
+        if target is not None:
+            pos = dtype.refprops.index(target)
+            sel = env.call("select", properties[pos])
+            if sel.prop is None:
+                sel.prop = target
+            prep[target.name] = sel
+        else:
+            for i, prop in enumerate(dtype.refprops):
+                sel = env.call("select", properties[i])
+                if sel.prop is None:
+                    sel.prop = prop
+                prep[prop.name] = sel
+    return prep