spinta 0.2.dev22__tar.gz → 0.2.dev24__tar.gz

{spinta-0.2.dev22 → spinta-0.2.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev22
+Version: 0.2.dev24
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE
@@ -36,6 +36,7 @@ Requires-Dist: alembic (>=1.11.0,<1.12.0) ; extra == "migrations" or extra == "a
 Requires-Dist: asyncpg
 Requires-Dist: authlib (>=1.0.0)
 Requires-Dist: cachetools
+Requires-Dist: cbor2 (>=5.9.0,<6.0.0)
 Requires-Dist: click
 Requires-Dist: dask[dataframe]
 Requires-Dist: docutils (>=0.21.2,<0.22.0)

{spinta-0.2.dev22 → spinta-0.2.dev24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev22"
+version = "0.2dev24"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"
@@ -116,6 +116,7 @@ cachetools = "*"
 docutils = "^0.21.2"
 zeep = "^4.3.1"
 redis = "^6.4.0"
+cbor2 = "^5.9.0"
 
 [tool.poetry.extras]
 http = ["gunicorn", "uvicorn"]

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/backends/__init__.py

@@ -13,6 +13,7 @@ from typing import Iterable
 from typing import List
 from typing import Optional
 
+from cbor2 import dumps as cbor_dumps
 import dateutil
 import shapely.geometry.base
 from geoalchemy2.elements import WKTElement, WKBElement
@@ -23,7 +24,7 @@ from spinta import commands
 from spinta import exceptions
 from spinta.backends.components import Backend
 from spinta.backends.components import SelectTree
-from spinta.backends.helpers import check_unknown_props, get_select_tree, prepare_response
+from spinta.backends.helpers import check_unknown_props, get_select_tree, prepare_response, is_accessible_by_equals_sign
 from spinta.backends.helpers import flat_select_to_nested
 from spinta.backends.helpers import get_model_reserved_props
 from spinta.backends.helpers import get_select_prop_names
@@ -58,7 +59,19 @@ from spinta.exceptions import (
 from spinta.exceptions import NoItemRevision
 from spinta.formats.components import Format
 from spinta.manifests.components import Manifest
-from spinta.types.datatype import Array, ExternalRef, Inherit, PageType, BackRef, ArrayBackRef, Integer, Boolean, Denorm
+from spinta.types.datatype import (
+    Array,
+    ExternalRef,
+    Inherit,
+    PageType,
+    BackRef,
+    ArrayBackRef,
+    Integer,
+    Boolean,
+    Denorm,
+    Base32,
+    String,
+)
 from spinta.types.datatype import Binary
 from spinta.types.datatype import DataType
 from spinta.types.datatype import Date
@@ -590,12 +603,31 @@ def is_object_id(context: Context, value: str):
 
 @is_object_id.register(Context, Backend, Model, str)
 def is_object_id(context: Context, backend: Backend, model: Model, value: str):
+    return is_object_id(context, backend, model.properties["_id"].dtype, value)
+
+
+@is_object_id.register(Context, Backend, PrimaryKey, str)
+def is_object_id(context: Context, backend: Backend, dtype: PrimaryKey, value: str):
     try:
         return uuid.UUID(value).version == 4
     except ValueError:
         return False
 
 
+@is_object_id.register(Context, Backend, DataType, str)
+def is_object_id(context: Context, backend: Backend, dtype: DataType, value: str):
+    candidate = value
+    if is_accessible_by_equals_sign(dtype.prop, value):
+        if not value.startswith("="):
+            return False
+        candidate = value[1:]
+    try:
+        dtype.load(candidate)
+    except exceptions.InvalidValue:
+        return False
+    return True
+
+
 @is_object_id.register(Context, Backend, Model, uuid.UUID)
 def is_object_id(context: Context, backend: Backend, model: Model, value: uuid.UUID):
     return value.version == 4
@@ -1750,6 +1782,13 @@ def cast_backend_to_python(context: Context, dtype: DataType, backend: Backend,
     return data
 
 
+@commands.cast_backend_to_python.register(Context, String, Backend, object)
+def cast_backend_to_python(context: Context, dtype: String, backend: Backend, data: Any, **kwargs) -> Any:
+    if data is None or is_nan(data):
+        return None
+    return str(data)
+
+
 @commands.cast_backend_to_python.register(Context, UUID, Backend, object)
 def cast_backend_to_python(context: Context, dtype: UUID, backend: Backend, data: Any, **kwargs) -> Any:
     if is_nan(data):
@@ -1891,6 +1930,13 @@ def cast_backend_to_python(context: Context, dtype: Ref, backend: Backend, data:
 
     processed_data = {}
     for key in data:
+        if key == "_id":
+            # _id reaches this dispatch already in its final form — produced by
+            # handle_ref_key_assignment for external readers, or read directly
+            # from the storage column for internal backends. Re-applying the
+            # referenced model's _id cast double-encodes Base32 ids.
+            processed_data[key] = data[key]
+            continue
         prop = commands.resolve_property(dtype.prop.model, f"{dtype.prop.place}.{key}")
         if prop is not None:
             processed_data[key] = commands.cast_backend_to_python(context, prop, backend, data[key], **kwargs)
@@ -1939,6 +1985,18 @@ def cast_backend_to_python(context: Context, dtype: Denorm, backend: Backend, da
     return commands.cast_backend_to_python(context, dtype.rel_prop, backend, data, **kwargs)
 
 
+@commands.cast_backend_to_python.register(Context, Base32, Backend, object)
+def cast_backend_to_python(context: Context, dtype: Base32, backend: Backend, data: Any, **kwargs) -> Any:
+    if is_nan(data):
+        return None
+    if isinstance(data, (list, tuple)):
+        data = cbor_dumps(list(data))
+    else:
+        data = str(data).encode("utf-8")
+    encoded = base64.b32encode(data)
+    return encoded.rstrip(b"=").decode("utf-8")
+
+
 @commands.reload_backend_metadata.register(Context, Manifest, Backend)
 def reload_backend_metadata(context, manifest, backend):
     pass

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/backends/helpers.py

@@ -26,7 +26,7 @@ from spinta.components import Model
 from spinta.components import Namespace
 from spinta.components import Property
 from spinta.exceptions import BackendUnavailable
-from spinta.types.datatype import DataType, Denorm
+from spinta.types.datatype import DataType, Denorm, String, Base32, PrimaryKey
 from spinta.utils.data import take
 from spinta.backends.constants import TableType, BackendOrigin
 
@@ -596,3 +596,28 @@ def extract_table_data_from_logical_name(table_name: str) -> tuple[str | None, T
             return data[0], table_type, None
 
     return None, None, None
+
+
+def is_custom_id_prop(prop: Property) -> bool:
+    return prop.name == "_id" and not isinstance(prop.dtype, PrimaryKey)
+
+
+def is_custom_revision_prop(prop: Property) -> bool:
+    return prop.name == "_revision" and prop.explicitly_given
+
+
+def is_accessible_by_equals_sign(id_prop: Property, value: str | int) -> bool:
+    if isinstance(id_prop.dtype, Base32):
+        return True
+
+    if isinstance(id_prop.dtype, String):
+        return not check_if_model_primary_key_is_composite(id_prop.model)
+
+    return False
+
+
+def check_if_model_primary_key_is_composite(model: Model) -> bool:
+    pkeys_count = len(model.external.pkeys)
+    if pkeys_count > 1:
+        return True
+    return False

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/admin.py

@@ -2,15 +2,14 @@ from __future__ import annotations
 
 import logging
 import pathlib
-import sys
 from typing import Optional, List
 
 from typer import Context as TyperContext, Argument
 from typer import Option
-from typer import echo
 
 from spinta.cli.helpers.admin.components import ADMIN_SCRIPT_TYPE
 from spinta.cli.helpers.admin.registry import admin_script_registry
+from spinta.cli.helpers.message import cli_error
 from spinta.cli.helpers.script.components import ScriptStatusCache
 from spinta.cli.helpers.script.core import run_specific_script
 from spinta.cli.helpers.script.helpers import sort_scripts_by_required
@@ -48,26 +47,30 @@ def admin(
         False,
         "-c",
         "--check",
-        help=("Only runs script checks, skipping execution part (used to find out what scripts are needed to run)."),
+        help="Only runs script checks, skipping execution part (used to find out what scripts are needed to run).",
     ),
     input_path: Optional[pathlib.Path] = Option(
         None,
         "-i",
         "--input",
-        help=("Path to input file (some scripts might require extra data). If not given, script will read from stdin."),
+        help="Path to input file (some scripts might require extra data). If not given, script will read from stdin.",
+    ),
+    output_path: Optional[pathlib.Path] = Option(
+        None,
+        "-o",
+        "--output",
+        help="Path to output file (some scripts might write extra data). If not given, script will write to stdout.",
     ),
 ):
     context = configure_context(ctx.obj)
 
     if force and check_only:
-        echo("Cannot run force mode with check only mode", err=True)
-        sys.exit(1)
+        cli_error("Cannot run force mode with check only mode")
 
     load_config(context, ensure_config_dir=ensure_config_dir)
 
     if not scripts:
-        echo("At least one script needs to be specified", err=True)
-        sys.exit(1)
+        cli_error("At least one script needs to be specified")
 
     script_objects = {}
     for script in scripts:
@@ -86,5 +89,6 @@ def admin(
             script_name=script,
             check_only=check_only,
             input_path=input_path,
+            output_path=output_path,
             status_cache=status_cache,
         )

spinta-0.2.dev24/spinta/cli/comment.py

@@ -0,0 +1,113 @@
+from typer import Argument
+from typer import Context as TyperContext
+from typer import Option
+from typer import echo
+
+from spinta.cli.helpers.enums import CommentPart
+from spinta.cli.helpers.store import load_manifest
+from spinta.components import Context, Property
+from spinta.core.context import configure_context
+from spinta.core.enums import Access
+from spinta.manifests.components import Manifest
+from spinta.manifests.tabular.helpers import datasets_to_tabular
+from spinta.manifests.tabular.helpers import render_tabular_manifest_rows
+from spinta.manifests.tabular.helpers import write_tabular_manifest
+
+
+def comment(
+    ctx: TyperContext,
+    part: CommentPart = Argument(None, help="Part to comment"),
+    author: str | None = Option(None, "--author", help="Comment author"),
+    uri: str | None = Option(
+        None, "--uri", help="URI tag stored on each comment (use with uncomment --uri to restore selectively)"
+    ),
+    description: str | None = Option(None, "--description", help="Additional description stored on each comment"),
+    output: str | None = Option(None, "-o", "--output", help="Output tabular manifest in a specified file"),
+    manifests: list[str] = Argument(None, help="Source manifest files"),
+) -> None:
+    """Comment out specific manifest parts so the manifest can be processed further."""
+    context: Context = ctx.obj
+    context = configure_context(context, manifests)
+
+    comment_manifest(
+        context,
+        author=author,
+        uri=uri,
+        description=description,
+        output=output,
+        manifests=manifests,
+    )
+
+
+def comment_manifest(
+    context: Context,
+    author: str | None = None,
+    uri: str | None = None,
+    description: str | None = None,
+    output: str | None = None,
+    manifests: list[str] | None = None,
+) -> None:
+    verbose = bool(output)
+    context = configure_context(context, manifests)
+    store = load_manifest(
+        context,
+        load_internal=False,
+        verbose=verbose,
+        full_load=True,
+    )
+
+    _update_systemic_comments(context, store.manifest, author=author, uri=uri, description=description)
+    rows = datasets_to_tabular(context, store.manifest, external=False, access=Access.private)
+
+    if output:
+        write_tabular_manifest(context, output, rows)
+    else:
+        manager = context.get("error_manager")
+        handler = manager.handler
+        table = render_tabular_manifest_rows(rows)
+        if handler.get_counts():
+            handler.post_process()
+        else:
+            echo(table)
+
+
+def _update_systemic_comments(
+    context: Context,
+    manifest: Manifest,
+    *,
+    author: str | None = None,
+    uri: str | None = None,
+    description: str | None = None,
+) -> None:
+    """Update author/uri/description on comment type properties."""
+    from spinta import commands
+
+    if all(argument is None for argument in [author, uri, description]):
+        return
+
+    models = commands.get_models(context, manifest)
+    for model in models.values():
+        for prop in model.properties.values():
+            _patch_property_restore_comments(prop, author=author, uri=uri, description=description)
+
+
+def _patch_property_restore_comments(
+    prop: Property,
+    *,
+    author: str | None,
+    uri: str | None,
+    description: str | None,
+) -> None:
+    if not prop.comments:
+        return
+
+    for comment in prop.comments:
+        if not comment.prepare or not comment.prepare.startswith("update"):
+            continue
+
+        if author is not None:
+            comment.author = author
+        if uri is not None:
+            comment.uri = uri
+        if description is not None:
+            comment.comment = description

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/admin/components.py

@@ -15,3 +15,4 @@ class AdminScript(ScriptBase):
 class Script(enum.Enum):
     DEDUPLICATE = "deduplicate"
     CHANGELOG = "changelog"
+    ENUM_LIST = "enum_list"

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/admin/registry.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from spinta.cli.helpers.admin.components import AdminScript, Script, ADMIN_SCRIPT_TYPE
 from spinta.cli.helpers.admin.scripts.changelog import migrate_changelog_duplicates, cli_requires_changelog_migrations
 from spinta.cli.helpers.admin.scripts.deduplicate import migrate_duplicates, cli_requires_deduplicate_migrations
+from spinta.cli.helpers.admin.scripts.enums import gather_invalid_enum_values
 from spinta.cli.helpers.script.components import ScriptTarget, ScriptTag
 from spinta.cli.helpers.script.registry import script_registry
 from spinta.cli.helpers.upgrade.components import Script as UpgradeScript, UPGRADE_SCRIPT_TYPE
@@ -29,3 +30,6 @@ script_registry.register(
         targets={ScriptTarget.BACKEND.value},
     )
 )
+script_registry.register(
+    AdminScript(name=Script.ENUM_LIST.value, run=gather_invalid_enum_values, targets={ScriptTarget.BACKEND.value})
+)

spinta-0.2.dev24/spinta/cli/helpers/admin/scripts/enums.py

@@ -0,0 +1,148 @@
+import csv
+import pathlib
+import sys
+from contextlib import nullcontext
+from typing import Iterator
+
+from multipledispatch import dispatch
+from dataclasses import dataclass, field
+
+from spinta import commands
+from spinta.backends import Backend
+from spinta.backends.postgresql.components import PostgreSQL
+from spinta.cli.helpers.script.helpers import ensure_store_is_loaded
+from spinta.components import Context, Model, Property
+from spinta.core.ufuncs import Expr
+from spinta.manifests.components import Manifest
+
+import sqlalchemy as sa
+
+from spinta.ufuncs.resultbuilder.components import EnumResultBuilder
+
+
+@dispatch(Backend, Property)
+def gather_unique_property_values() -> list:
+    raise NotImplementedError
+
+
+@dispatch(PostgreSQL, Property)
+def gather_unique_property_values(backend: PostgreSQL, prop: Property) -> list:
+    table = backend.get_table(prop)
+    column = backend.get_column(table, prop)
+    result = backend.engine.execute(sa.select(column).distinct()).scalars().all()
+    return result
+
+
+@dispatch(Property)
+def gather_unique_property_values(prop: Property) -> list:
+    return gather_unique_property_values(prop.dtype.backend, prop)
+
+
+def get_models_with_enums(context: Context, manifest: Manifest) -> Iterator[Model]:
+    for model in commands.get_models(context, manifest).values():
+        for prop in model.flatprops.values():
+            if prop.enum:
+                yield model
+                break
+
+
+@dataclass
+class InvalidEnumProperty:
+    prop: Property
+    invalid_values: list = field(default_factory=list)
+
+    def add_invalid_value(self, value: object):
+        if value not in self.invalid_values:
+            self.invalid_values.append(value)
+
+
+@dataclass
+class InvalidEnumModel:
+    model: Model
+    enum_props: dict[str, InvalidEnumProperty] = field(default_factory=dict)
+
+    def add_invalid_value(self, prop: Property, value: object):
+        self.get_prop(prop).add_invalid_value(value)
+
+    def get_prop(self, prop: Property) -> InvalidEnumProperty:
+        if prop.place not in self.enum_props:
+            self.enum_props[prop.place] = InvalidEnumProperty(prop=prop)
+        return self.enum_props[prop.place]
+
+
+def gather_invalid_enum_values(context: Context, output_path: pathlib.Path | None = None, **kwargs):
+    store = ensure_store_is_loaded(context)
+    manifest = store.manifest
+    invalid_models = {}
+    with context:
+        for model in get_models_with_enums(context, manifest):
+            enum_model = InvalidEnumModel(model=model)
+            for prop in model.flatprops.values():
+                if not prop.enum:
+                    continue
+
+                enum_contains_expr = any(isinstance(enum.prepare, Expr) for enum in prop.enum.values())
+                values = gather_unique_property_values(prop)
+                for value in values:
+                    if value is None and not prop.dtype.required:
+                        continue
+                    elif value is None:
+                        enum_model.add_invalid_value(prop, value)
+                        continue
+
+                    if str(value) in prop.enum:
+                        continue
+
+                    check_value = value
+                    if enum_contains_expr:
+                        for enum in prop.enum.values():
+                            env = EnumResultBuilder(context).init(value)
+                            val = env.resolve(enum.prepare)
+                            if env.has_value_changed:
+                                check_value = val
+                                break
+
+                    if str(check_value) not in prop.enum:
+                        enum_model.add_invalid_value(prop, value)
+
+            if enum_model.enum_props:
+                invalid_models[model.model_type()] = enum_model
+
+    output_invalid_enums_to_csv(invalid_models, output_path)
+
+
+def output_invalid_enums_to_csv(invalid_models: dict, output_path: pathlib.Path | None = None):
+    stream_ctx = open(output_path, "w") if output_path else nullcontext(sys.stdout)
+    with stream_ctx as stream:
+        writer = csv.DictWriter(stream, fieldnames=["model", "property", "invalid_value"], lineterminator="\n")
+        writer.writeheader()
+        for model_key, model in sorted(invalid_models.items()):
+            model_written = False
+
+            for prop_name, prop in model.enum_props.items():
+                it = iter(prop.invalid_values)
+                first = next(it, None)
+                if first is None:
+                    continue
+
+                model_name = ""
+                if not model_written:
+                    model_name = model_key
+                    model_written = True
+
+                writer.writerow(
+                    {
+                        "model": model_name,
+                        "property": prop_name,
+                        "invalid_value": first,
+                    }
+                )
+
+                for value in it:
+                    writer.writerow(
+                        {
+                            "model": "",
+                            "property": "",
+                            "invalid_value": value,
+                        }
+                    )

spinta-0.2.dev24/spinta/cli/helpers/enums.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+
+class CommentPart(str, Enum):
+    missing_external_refs = "missing-external-refs"

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/message.py

@@ -1,3 +1,5 @@
+import sys
+
 import tqdm
 from click import echo
 from click.exceptions import Exit
@@ -9,7 +11,9 @@ def cli_error(message: str):
 
 
 def cli_message(message: str, progress_bar: tqdm.tqdm = None):
+    # https://pubs.opengroup.org/onlinepubs/9799919799/
+    # This documentation states that `stderr` should be used for diagnostic messages (in our case status)
     if progress_bar is not None:
-        progress_bar.write(message)
+        progress_bar.write(message, file=sys.stderr)
     else:
-        echo(message)
+        echo(message, err=True)

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/script/components.py

@@ -31,7 +31,7 @@ class _ScriptMeta(type):
 
 class ScriptBase(metaclass=_ScriptMeta):
     """
-    Represents an script with optional preconditions and target constraints.
+    Represents a script with optional preconditions and target constraints.
 
     Targets and tags are mainly used to be able to filter specific scripts. Main use case would be to set target as
     specific object, like sqlalchemy keymap and tag as migration, so then you could filter all database migration

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/script/core.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from click import echo
-
+from spinta.cli.helpers.message import cli_message
 from spinta.cli.helpers.script.components import ScriptStatus, ScriptBase, ScriptStatusCache
 from spinta.cli.helpers.script.helpers import sort_scripts_by_required, script_check_status_message
 from spinta.cli.helpers.script.registry import script_registry
@@ -54,7 +53,7 @@ def run_specific_script(
     if force:
         status = ScriptStatus.FORCED
 
-    echo(script_check_status_message(script_name, status))
+    cli_message(script_check_status_message(script_name, status))
     if status in (ScriptStatus.FORCED, ScriptStatus.REQUIRED) and not check_only:
         script.run(context, destructive=destructive, **kwargs)
         if status_cache is not None:
@@ -73,7 +72,7 @@ def check_script(
             script = script.value
 
         if not script_registry.contains(script_type, script):
-            echo(f"Warning: {script_type!r} script {script!r} was not found", err=True)
+            cli_message(f"Warning: {script_type!r} script {script!r} was not found")
             return ScriptStatus.SKIPPED
 
         script = script_registry.get(script_type, script)
@@ -92,9 +91,8 @@ def check_script(
                 ScriptStatus.REQUIRED,
                 ScriptStatus.SKIPPED,
             ):
-                echo(
-                    f"Warning: {required_script_type!r} script {required_script!r} requirement is not met for {script.name!r} script",
-                    err=True,
+                cli_message(
+                    f"Warning: {script_type!r} script {required_script!r} requirement is not met for {script.name!r} script",
                 )
                 return ScriptStatus.SKIPPED
 

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/helpers/script/helpers.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from collections import defaultdict, deque
 
-from typer import echo
 
+from spinta.cli.helpers.message import cli_message
 from spinta.cli.helpers.script.components import ScriptStatus
 from spinta.cli.helpers.upgrade.components import UpgradeScript
 from spinta.components import Context, Store
@@ -44,7 +44,7 @@ def sort_scripts_by_required(scripts: dict[str, UpgradeScript]) -> dict:
 
     if len(result) != len(data):
         unresolved = set(data) - set(result)
-        echo(f"Warning: Dependency cycle detected or unresolved dependencies in: {unresolved}", err=True)
+        cli_message(f"Warning: Dependency cycle detected or unresolved dependencies in: {unresolved}")
         # Extend results, potentially might cause errors, because of cycles
         result.extend(unresolved)
 

{spinta-0.2.dev22 → spinta-0.2.dev24}/spinta/cli/main.py

@@ -17,6 +17,8 @@ from spinta.cli import data
 from spinta.cli import inspect
 from spinta.cli import manifest
 from spinta.cli import migrate
+from spinta.cli.comment import comment
+from spinta.cli.uncomment import uncomment
 from spinta.cli import pii
 from spinta.cli import pull
 from spinta.cli import push
@@ -45,6 +47,8 @@ add(app, "inspect", inspect.inspect, short_help=("Update manifest schema from an
 add(app, "pii", pii.app, short_help="Manage Person Identifying Information")
 
 add(app, "copy", manifest.copy, short_help="Copy only specified metadata from a manifest")
+add(app, "comment", comment, short_help="Comment unsupported functionality in a manifest")
+add(app, "uncomment", uncomment, short_help="Restore commented parts of a manifest")
 add(app, "show", show, short_help="Show manifest as ascii table")
 
 add(app, "bootstrap", migrate.bootstrap, short_help="Initialize backends")