spinta 0.2.dev21__tar.gz → 0.2.dev22__tar.gz

{spinta-0.2.dev21 → spinta-0.2.dev22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev21
+Version: 0.2.dev22
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev21 → spinta-0.2.dev22}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev21"
+version = "0.2dev22"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/auth.py

@@ -58,6 +58,7 @@ from spinta.exceptions import (
     InvalidClientsKeymapStructure,
     InvalidScopes,
     InvalidClientFileFormat,
+    ModelNotFound,
 )
 from spinta.utils import passwords
 from spinta.utils.config import get_clients_path, get_keymap_path, get_id_path, get_helpers_path
@@ -858,40 +859,46 @@ def authorized(
     scope_formatter: ScopeFormatterFunc = None,
 ):
     config: Config = context.get("config")
+
+    # Disable access to nodes that have lower access level than config.access
+    if config.access > node.access:
+        if throw:
+            raise ModelNotFound(model=node.name)
+        return False
+
     token = context.get("auth.token")
-    # Unauthorized clients can only access open nodes.
-    unauthorized = token.get_client_id() == get_default_auth_client_id(context)
-    open_node = node.access >= Access.open
-    if unauthorized and not open_node:
+
+    # Unauthenticated clients can only access nodes if spinta config.access is open.
+    unauthenticated = token.get_client_id() == get_default_auth_client_id(context)
+
+    if unauthenticated and config.access < Access.open:
         if throw:
             raise AuthorizedClientsOnly()
         else:
             return False
 
-    # Private nodes can only be accessed with explicit node scope.
+    # Add explicit node scope
     scopes = [node]
 
-    # Protected and higher level nodes can be accessed with parent nodes scopes.
-    if node.access > Access.private:
-        ns = None
-
-        if isinstance(node, Property):
-            # Hidden nodes also require explicit scope.
-            # XXX: `hidden` parameter should only be used for API control, not
-            #      access control. See docs.
-            if not node.hidden:
-                scopes.append(node.model)
-                scopes.append(node.model.ns)
-                ns = node.model.ns
-        elif isinstance(node, Model):
-            scopes.append(node.ns)
-            ns = node.ns
-        elif isinstance(node, Namespace):
-            ns = node
-
-        # Add all parent namespace scopes too.
-        if ns:
-            scopes.extend(ns.parents())
+    # Add parent node scopes
+    ns = None
+    if isinstance(node, Property):
+        # Hidden nodes also require explicit scope.
+        # XXX: `hidden` parameter should only be used for API control, not
+        #      access control. See docs.
+        if not node.hidden:
+            scopes.append(node.model)
+            scopes.append(node.model.ns)
+            ns = node.model.ns
+    elif isinstance(node, Model):
+        scopes.append(node.ns)
+        ns = node.ns
+    elif isinstance(node, Namespace):
+        ns = node
+
+    # Add all parent namespace scopes too.
+    if ns:
+        scopes.extend(ns.parents())
 
     # Build scope names.
     scope_formatter = scope_formatter or config.scope_formatter

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/cli/helpers/upgrade/scripts/backends/postgresql/schemas.py

@@ -1,8 +1,6 @@
 from contextlib import ExitStack
 from typing import Generator
 
-from alembic.operations import Operations
-from alembic.runtime.migration import MigrationContext
 from click import echo
 from sqlalchemy.engine import Inspector
 from tqdm import tqdm
@@ -105,6 +103,9 @@ def gather_associated_tables(backend: PostgreSQL, table_identifier: TableIdentif
 
 
 def migrate_schemas(context: Context, **kwargs):
+    from alembic.operations import Operations
+    from alembic.runtime.migration import MigrationContext
+
     ensure_store_is_loaded(context)
     store = context.get("store")
 

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/components.py

@@ -1095,7 +1095,8 @@ class Config:
     scope_log: bool
     check_contract_scopes: bool
     default_auth_client: str
-    default_access_level: str
+    default_access_level: Access
+    access: Access
     http_basic_auth: bool
     token_validation_key: dict | None = None
     token_validation_keys_download_url: str | None = None

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/config.py

@@ -344,7 +344,8 @@ CONFIG = {
             },
             "config_path": pathlib.Path("tests/config"),
             "default_auth_client": "baa448a8-205c-4faa-a048-a10e4b32a136",
-            "default_access_level": "protected",
+            "default_access_level": "open",
+            "access": "open",
             "sync_retry_count": 0,
         },
     },

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/datasets/backends/dataframe/backends/json/commands/read.py

@@ -164,7 +164,7 @@ def getall(
     builder.update(model=model, params={param.name: param for param in resource.params}, url_query_params=query)
 
     props = {}
-    for prop in model.properties.values():
+    for prop in model.flatprops.values():
         if prop.external and prop.external.name:
             root_source = _get_prop_full_source(model.external.name, prop.external.name)
             props[prop.external.name] = {

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/datasets/backends/dataframe/commands/read.py

@@ -276,6 +276,11 @@ def dask_get_all(
             val = _get_row_value(context, row, sel, env.params)
             if sel.prop:
                 if isinstance(sel.prop.dtype, PrimaryKey):
+                    if isinstance(val, list):
+                        val = [
+                            list_value.get("_id", list_value) if isinstance(list_value, dict) else list_value
+                            for list_value in val
+                        ]
                     val = keymap.encode(sel.prop.model.model_type(), val)
                 elif isinstance(sel.prop.dtype, Ref):
                     val = handle_ref_key_assignment(context, keymap, env, val, sel.prop.dtype)

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -336,12 +336,15 @@ def select(env: DaskDataFrameQueryBuilder, selected: Selected) -> Selected:
 
 @ufunc.resolver(DaskDataFrameQueryBuilder, Ref, GetAttr)
 def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: GetAttr) -> Selected | None:
-    prep = env.call("select", prep)
-    if prep is not None:
-        return Selected(
-            prop=dtype.prop,
-            prep=prep,
-        )
+    resolved_prep = env.call("select", prep)
+
+    result = {}
+    result["_id"] = Selected(prop=dtype.prop, prep=resolved_prep)
+    for prop in dtype.properties.values():
+        sel = env.call("select", prop)
+        result[prop.name] = sel
+
+    return Selected(prop=dtype.prop, prep=result)
 
 
 @ufunc.resolver(DaskDataFrameQueryBuilder, Ref, object)
@@ -353,6 +356,18 @@ def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: Any) -> Selected:
     )
 
 
+@ufunc.resolver(DaskDataFrameQueryBuilder, Ref)
+def select(env: DaskDataFrameQueryBuilder, dtype: Ref) -> Selected:
+    prep = {}
+    prep["_id"] = Selected(item=dtype.prop.external.name, prop=dtype.prop)
+
+    for prop in dtype.properties.values():
+        sel = env.call("select", prop)
+        prep[prop.name] = sel
+
+    return Selected(prop=dtype.prop, prep=prep)
+
+
 @ufunc.resolver(DaskDataFrameQueryBuilder, GetAttr)
 def select(env: DaskDataFrameQueryBuilder, attr: GetAttr) -> Selected:
     resolved = env.resolve_property(attr)

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/datasets/backends/helpers.py

@@ -14,6 +14,10 @@ from spinta.utils.schema import NA
 
 
 def handle_ref_key_assignment(context: Context, keymap: KeyMap, env: Env, value: Any, ref: Ref) -> dict:
+    original_value = value
+    if isinstance(value, dict):
+        value = value["_id"]
+
     keymap_name = ref.model.model_type()
     if ref.refprops != ref.model.external.pkeys:
         keymap_name = f"{keymap_name}.{'_'.join(prop.name for prop in ref.refprops)}"
@@ -55,7 +59,16 @@ def handle_ref_key_assignment(context: Context, keymap: KeyMap, env: Env, value:
 
             # FIXME Quick hack when trying to get `Internal` model keys while running in `External` mode (should probably return error, or None)
             if ref_model.mode == Mode.external and not check_if_model_has_backend_and_source(ref_model):
-                return {"_id": keymap.encode(keymap_name, target_value)}
+                val = {"_id": keymap.encode(keymap_name, target_value)}
+                if isinstance(original_value, dict):
+                    for nested_prop_name, nested_prop in ref.properties.items():
+                        nested_value = original_value[nested_prop_name]
+                        if isinstance(nested_value, dict):
+                            nested_value = handle_ref_key_assignment(
+                                context, keymap, env, nested_value, nested_prop.dtype
+                            )
+                        val[nested_prop_name] = nested_value
+                return val
 
             expr_parts = ["select()"]
             for i, prop in enumerate(ref.refprops):
@@ -67,7 +80,10 @@ def handle_ref_key_assignment(context: Context, keymap: KeyMap, env: Env, value:
             for row in rows:
                 if val is not None:
                     raise MultiplePrimaryKeyCandidatesFound(ref, values=target_value)
-                val = row["_id"]
+                if "_id" in row:
+                    val = row["_id"]
+                else:
+                    val = keymap.encode(keymap_name, target_value)
                 found_value = True
 
             if not found_value:
@@ -84,6 +100,13 @@ def handle_ref_key_assignment(context: Context, keymap: KeyMap, env: Env, value:
                 values = values[0]
             val[prop] = values
             i = i + count
+
+    if isinstance(original_value, dict):
+        for nested_prop_name, nested_prop in ref.properties.items():
+            nested_value = original_value[nested_prop_name]
+            if isinstance(nested_value, dict):
+                nested_value = handle_ref_key_assignment(context, keymap, env, nested_value, nested_prop.dtype)
+            val[nested_prop_name] = nested_value
     return val
 
 

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/exceptions.py

@@ -1223,3 +1223,9 @@ class SourceOrPrepareNotAllowed(UserError):
     template = """
         The source {source} was not expected. Delete it from the manifest or update the prepare function to allow it.    
     """
+
+
+class PartialIncorrectProperty(BaseError):
+    template = (
+        "The composite property {property} is not correct. Check if all parts of the composite property are present."
+    )

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/testing/client.py

@@ -94,7 +94,6 @@ def create_remote_server(
         {
             "config_path": confdir,
             "default_auth_client": None,
-            "default_access_level": "protected",
         }
     )
     context = create_test_context(rc)
@@ -294,7 +293,6 @@ def create_rc(
             },
             # tests/config/clients/3388ea36-4a4f-4821-900a-b574c8829d52.yml
             "default_auth_client": "3388ea36-4a4f-4821-900a-b574c8829d52",
-            "default_access_level": "protected",
         }
     )
 

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/types/config.py

@@ -83,6 +83,11 @@ def load(context: Context, config: Config) -> Config:
         default="private",
         cast=lambda name: get_enum_by_name(Access, name),
     )
+    config.access = rc.get(
+        "access",
+        default="open",
+        cast=lambda name: get_enum_by_name(Access, name),
+    )
     config.http_basic_auth = rc.get("http_basic_auth", default=False, cast=asbool)
     config.token_validation_key = rc.get("token_validation_key", cast=json.loads) or None
     config.token_validation_keys_download_url = rc.get("token_validation_keys_download_url")

spinta-0.2.dev22/spinta/types/partial/link.py

@@ -0,0 +1,63 @@
+from copy import copy
+
+from spinta import commands
+from spinta.components import Context, Property
+from spinta.core.enums import Mode
+from spinta.exceptions import PartialTypeNotFound, ParentNodeNotFound, PartialIncorrectProperty
+from spinta.types.datatype import Partial, Ref
+
+
+def get_ref_value(context: Context, prop: Property) -> Property | None:
+    parent = prop.parent
+    if not (isinstance(parent, Property) and isinstance(parent.dtype, Ref)):
+        return None
+    if isinstance(parent.dtype.model, str):
+        parent.dtype.model = commands.get_model(context, prop.model.manifest, parent.dtype.model)
+    model = parent.dtype.model
+
+    if prop.name not in model.properties:
+        return None
+
+    model_property_value = model.properties[prop.name]
+    model_dtype = model_property_value.dtype
+    if hasattr(model_dtype, "model") and isinstance(model_dtype.model, str):
+        model_property_value.dtype.model = commands.get_model(
+            context, prop.model.manifest, model_property_value.dtype.model
+        )
+    return model_property_value
+
+
+@commands.link.register(Context, Partial)
+def link(context: Context, dtype: Partial):
+    prop = dtype.prop
+    parent = prop.parent
+    if parent and isinstance(parent, Property):
+        if isinstance(parent.dtype, Ref):
+            props = dtype.properties
+            result = get_ref_value(context, prop)
+            if not result:
+                raise PartialIncorrectProperty(dtype)
+            prop.dtype = copy(result.dtype)
+            prop.dtype.properties = props
+            prop.dtype.inherited = True
+            prop.given.explicit = False
+            prop.given.name = ""
+            prop.dtype.prop = prop
+            if prop.level is None:
+                prop.level = result.level
+            # For external mode copy the external mapping
+            # For internal mode leave it as is, because it breaks multi-level denormalization
+            if result.external and prop.model.mode == Mode.external:
+                prop.external = copy(result.external)
+            if isinstance(prop.dtype, Ref):
+                prop.dtype.refprops = []
+            commands.link(context, prop.dtype)
+        else:
+            raise PartialTypeNotFound(dtype)
+    else:
+        raise ParentNodeNotFound(
+            dtype,
+            model_name=prop.model.name,
+            property_names=",".join([f"{prop.name}.{i}" for i in dtype.properties.keys()]),
+            missing_property_name=prop.name,
+        )

{spinta-0.2.dev21 → spinta-0.2.dev22}/spinta/types/ref/link.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from spinta import commands
-from spinta.components import Context
+from spinta.components import Context, Model
 from spinta.types.datatype import Ref
 from spinta.exceptions import ModelReferenceNotFound, MissingRefModel
 from spinta.exceptions import ModelReferenceKeyNotFound
@@ -25,6 +25,8 @@ def link(context: Context, dtype: Ref) -> None:
         # Self reference.
         dtype.model = dtype.prop.model
     else:
+        if isinstance(rmodel, Model):
+            rmodel = rmodel.name
         if not commands.has_model(context, dtype.prop.model.manifest, rmodel):
             raise ModelReferenceNotFound(dtype, ref=rmodel)
         dtype.model = commands.get_model(context, dtype.prop.model.manifest, rmodel)

spinta-0.2.dev21/spinta/types/partial/link.py

@@ -1,48 +0,0 @@
-from copy import copy
-
-from spinta import commands
-from spinta.components import Context, Property
-from spinta.exceptions import PartialTypeNotFound, ParentNodeNotFound
-from spinta.types.datatype import Partial, Ref
-
-
-def get_ref_value(context: Context, prop: Property):
-    parent = prop.parent
-    if isinstance(parent, Property) and isinstance(parent.dtype, Ref):
-        parent_parent = parent.parent
-        if isinstance(parent_parent, Property) and isinstance(parent_parent.dtype, Ref):
-            model = parent_parent.dtype.model
-            first_level = model.properties[parent.name]
-            if prop.name in first_level.dtype.properties:
-                return first_level.dtype.properties[prop.name]
-        else:
-            model = parent.dtype.model
-            if prop.name in model.properties:
-                return model.properties[prop.name]
-
-
-@commands.link.register(Context, Partial)
-def link(context: Context, dtype: Partial):
-    prop = dtype.prop
-    parent = prop.parent
-    if parent and isinstance(parent, Property):
-        if isinstance(parent.dtype, Ref):
-            props = dtype.properties
-            result = get_ref_value(context, prop)
-            prop.dtype = copy(result.dtype)
-            prop.dtype.properties = props
-            prop.dtype.inherited = True
-            prop.given.explicit = False
-            prop.given.name = ""
-            prop.dtype.prop = prop
-            for partial_prop in props.values():
-                commands.link(context, partial_prop)
-        else:
-            raise PartialTypeNotFound(dtype)
-    else:
-        raise ParentNodeNotFound(
-            dtype,
-            model_name=prop.model.name,
-            property_names=",".join([f"{prop.name}.{i}" for i in dtype.properties.keys()]),
-            missing_property_name=prop.name,
-        )