spinta 0.2.dev22__tar.gz → 0.2.dev23__tar.gz

{spinta-0.2.dev22 → spinta-0.2.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev22
+Version: 0.2.dev23
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev22 → spinta-0.2.dev23}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev22"
+version = "0.2dev23"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/admin.py

@@ -2,15 +2,14 @@ from __future__ import annotations
 
 import logging
 import pathlib
-import sys
 from typing import Optional, List
 
 from typer import Context as TyperContext, Argument
 from typer import Option
-from typer import echo
 
 from spinta.cli.helpers.admin.components import ADMIN_SCRIPT_TYPE
 from spinta.cli.helpers.admin.registry import admin_script_registry
+from spinta.cli.helpers.message import cli_error
 from spinta.cli.helpers.script.components import ScriptStatusCache
 from spinta.cli.helpers.script.core import run_specific_script
 from spinta.cli.helpers.script.helpers import sort_scripts_by_required
@@ -48,26 +47,30 @@ def admin(
         False,
         "-c",
         "--check",
-        help=("Only runs script checks, skipping execution part (used to find out what scripts are needed to run)."),
+        help="Only runs script checks, skipping execution part (used to find out what scripts are needed to run).",
     ),
     input_path: Optional[pathlib.Path] = Option(
         None,
         "-i",
         "--input",
-        help=("Path to input file (some scripts might require extra data). If not given, script will read from stdin."),
+        help="Path to input file (some scripts might require extra data). If not given, script will read from stdin.",
+    ),
+    output_path: Optional[pathlib.Path] = Option(
+        None,
+        "-o",
+        "--output",
+        help="Path to output file (some scripts might write extra data). If not given, script will write to stdout.",
     ),
 ):
     context = configure_context(ctx.obj)
 
     if force and check_only:
-        echo("Cannot run force mode with check only mode", err=True)
-        sys.exit(1)
+        cli_error("Cannot run force mode with check only mode")
 
     load_config(context, ensure_config_dir=ensure_config_dir)
 
     if not scripts:
-        echo("At least one script needs to be specified", err=True)
-        sys.exit(1)
+        cli_error("At least one script needs to be specified")
 
     script_objects = {}
     for script in scripts:
@@ -86,5 +89,6 @@ def admin(
             script_name=script,
             check_only=check_only,
             input_path=input_path,
+            output_path=output_path,
             status_cache=status_cache,
         )

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/admin/components.py

@@ -15,3 +15,4 @@ class AdminScript(ScriptBase):
 class Script(enum.Enum):
     DEDUPLICATE = "deduplicate"
     CHANGELOG = "changelog"
+    ENUM_LIST = "enum_list"

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/admin/registry.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from spinta.cli.helpers.admin.components import AdminScript, Script, ADMIN_SCRIPT_TYPE
 from spinta.cli.helpers.admin.scripts.changelog import migrate_changelog_duplicates, cli_requires_changelog_migrations
 from spinta.cli.helpers.admin.scripts.deduplicate import migrate_duplicates, cli_requires_deduplicate_migrations
+from spinta.cli.helpers.admin.scripts.enums import gather_invalid_enum_values
 from spinta.cli.helpers.script.components import ScriptTarget, ScriptTag
 from spinta.cli.helpers.script.registry import script_registry
 from spinta.cli.helpers.upgrade.components import Script as UpgradeScript, UPGRADE_SCRIPT_TYPE
@@ -29,3 +30,6 @@ script_registry.register(
         targets={ScriptTarget.BACKEND.value},
     )
 )
+script_registry.register(
+    AdminScript(name=Script.ENUM_LIST.value, run=gather_invalid_enum_values, targets={ScriptTarget.BACKEND.value})
+)

spinta-0.2.dev23/spinta/cli/helpers/admin/scripts/enums.py

@@ -0,0 +1,148 @@
+import csv
+import pathlib
+import sys
+from contextlib import nullcontext
+from typing import Iterator
+
+from multipledispatch import dispatch
+from dataclasses import dataclass, field
+
+from spinta import commands
+from spinta.backends import Backend
+from spinta.backends.postgresql.components import PostgreSQL
+from spinta.cli.helpers.script.helpers import ensure_store_is_loaded
+from spinta.components import Context, Model, Property
+from spinta.core.ufuncs import Expr
+from spinta.manifests.components import Manifest
+
+import sqlalchemy as sa
+
+from spinta.ufuncs.resultbuilder.components import EnumResultBuilder
+
+
+@dispatch(Backend, Property)
+def gather_unique_property_values() -> list:
+    raise NotImplementedError
+
+
+@dispatch(PostgreSQL, Property)
+def gather_unique_property_values(backend: PostgreSQL, prop: Property) -> list:
+    table = backend.get_table(prop)
+    column = backend.get_column(table, prop)
+    result = backend.engine.execute(sa.select(column).distinct()).scalars().all()
+    return result
+
+
+@dispatch(Property)
+def gather_unique_property_values(prop: Property) -> list:
+    return gather_unique_property_values(prop.dtype.backend, prop)
+
+
+def get_models_with_enums(context: Context, manifest: Manifest) -> Iterator[Model]:
+    for model in commands.get_models(context, manifest).values():
+        for prop in model.flatprops.values():
+            if prop.enum:
+                yield model
+                break
+
+
+@dataclass
+class InvalidEnumProperty:
+    prop: Property
+    invalid_values: list = field(default_factory=list)
+
+    def add_invalid_value(self, value: object):
+        if value not in self.invalid_values:
+            self.invalid_values.append(value)
+
+
+@dataclass
+class InvalidEnumModel:
+    model: Model
+    enum_props: dict[str, InvalidEnumProperty] = field(default_factory=dict)
+
+    def add_invalid_value(self, prop: Property, value: object):
+        self.get_prop(prop).add_invalid_value(value)
+
+    def get_prop(self, prop: Property) -> InvalidEnumProperty:
+        if prop.place not in self.enum_props:
+            self.enum_props[prop.place] = InvalidEnumProperty(prop=prop)
+        return self.enum_props[prop.place]
+
+
+def gather_invalid_enum_values(context: Context, output_path: pathlib.Path | None = None, **kwargs):
+    store = ensure_store_is_loaded(context)
+    manifest = store.manifest
+    invalid_models = {}
+    with context:
+        for model in get_models_with_enums(context, manifest):
+            enum_model = InvalidEnumModel(model=model)
+            for prop in model.flatprops.values():
+                if not prop.enum:
+                    continue
+
+                enum_contains_expr = any(isinstance(enum.prepare, Expr) for enum in prop.enum.values())
+                values = gather_unique_property_values(prop)
+                for value in values:
+                    if value is None and not prop.dtype.required:
+                        continue
+                    elif value is None:
+                        enum_model.add_invalid_value(prop, value)
+                        continue
+
+                    if str(value) in prop.enum:
+                        continue
+
+                    check_value = value
+                    if enum_contains_expr:
+                        for enum in prop.enum.values():
+                            env = EnumResultBuilder(context).init(value)
+                            val = env.resolve(enum.prepare)
+                            if env.has_value_changed:
+                                check_value = val
+                                break
+
+                    if str(check_value) not in prop.enum:
+                        enum_model.add_invalid_value(prop, value)
+
+            if enum_model.enum_props:
+                invalid_models[model.model_type()] = enum_model
+
+    output_invalid_enums_to_csv(invalid_models, output_path)
+
+
+def output_invalid_enums_to_csv(invalid_models: dict, output_path: pathlib.Path | None = None):
+    stream_ctx = open(output_path, "w") if output_path else nullcontext(sys.stdout)
+    with stream_ctx as stream:
+        writer = csv.DictWriter(stream, fieldnames=["model", "property", "invalid_value"], lineterminator="\n")
+        writer.writeheader()
+        for model_key, model in sorted(invalid_models.items()):
+            model_written = False
+
+            for prop_name, prop in model.enum_props.items():
+                it = iter(prop.invalid_values)
+                first = next(it, None)
+                if first is None:
+                    continue
+
+                model_name = ""
+                if not model_written:
+                    model_name = model_key
+                    model_written = True
+
+                writer.writerow(
+                    {
+                        "model": model_name,
+                        "property": prop_name,
+                        "invalid_value": first,
+                    }
+                )
+
+                for value in it:
+                    writer.writerow(
+                        {
+                            "model": "",
+                            "property": "",
+                            "invalid_value": value,
+                        }
+                    )

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/message.py

@@ -1,3 +1,5 @@
+import sys
+
 import tqdm
 from click import echo
 from click.exceptions import Exit
@@ -9,7 +11,9 @@ def cli_error(message: str):
 
 
 def cli_message(message: str, progress_bar: tqdm.tqdm = None):
+    # https://pubs.opengroup.org/onlinepubs/9799919799/
+    # This documentation states that `stderr` should be used for diagnostic messages (in our case status)
     if progress_bar is not None:
-        progress_bar.write(message)
+        progress_bar.write(message, file=sys.stderr)
     else:
-        echo(message)
+        echo(message, err=True)

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/script/components.py

@@ -31,7 +31,7 @@ class _ScriptMeta(type):
 
 class ScriptBase(metaclass=_ScriptMeta):
     """
-    Represents an script with optional preconditions and target constraints.
+    Represents a script with optional preconditions and target constraints.
 
     Targets and tags are mainly used to be able to filter specific scripts. Main use case would be to set target as
     specific object, like sqlalchemy keymap and tag as migration, so then you could filter all database migration

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/script/core.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from click import echo
-
+from spinta.cli.helpers.message import cli_message
 from spinta.cli.helpers.script.components import ScriptStatus, ScriptBase, ScriptStatusCache
 from spinta.cli.helpers.script.helpers import sort_scripts_by_required, script_check_status_message
 from spinta.cli.helpers.script.registry import script_registry
@@ -54,7 +53,7 @@ def run_specific_script(
     if force:
         status = ScriptStatus.FORCED
 
-    echo(script_check_status_message(script_name, status))
+    cli_message(script_check_status_message(script_name, status))
     if status in (ScriptStatus.FORCED, ScriptStatus.REQUIRED) and not check_only:
         script.run(context, destructive=destructive, **kwargs)
         if status_cache is not None:
@@ -73,7 +72,7 @@ def check_script(
             script = script.value
 
         if not script_registry.contains(script_type, script):
-            echo(f"Warning: {script_type!r} script {script!r} was not found", err=True)
+            cli_message(f"Warning: {script_type!r} script {script!r} was not found")
             return ScriptStatus.SKIPPED
 
         script = script_registry.get(script_type, script)
@@ -92,9 +91,8 @@ def check_script(
                 ScriptStatus.REQUIRED,
                 ScriptStatus.SKIPPED,
             ):
-                echo(
-                    f"Warning: {required_script_type!r} script {required_script!r} requirement is not met for {script.name!r} script",
-                    err=True,
+                cli_message(
+                    f"Warning: {script_type!r} script {required_script!r} requirement is not met for {script.name!r} script",
                 )
                 return ScriptStatus.SKIPPED
 

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/cli/helpers/script/helpers.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 from collections import defaultdict, deque
 
-from typer import echo
 
+from spinta.cli.helpers.message import cli_message
 from spinta.cli.helpers.script.components import ScriptStatus
 from spinta.cli.helpers.upgrade.components import UpgradeScript
 from spinta.components import Context, Store
@@ -44,7 +44,7 @@ def sort_scripts_by_required(scripts: dict[str, UpgradeScript]) -> dict:
 
     if len(result) != len(data):
         unresolved = set(data) - set(result)
-        echo(f"Warning: Dependency cycle detected or unresolved dependencies in: {unresolved}", err=True)
+        cli_message(f"Warning: Dependency cycle detected or unresolved dependencies in: {unresolved}")
         # Extend results, potentially might cause errors, because of cycles
         result.extend(unresolved)
 

{spinta-0.2.dev22 → spinta-0.2.dev23}/spinta/datasets/backends/dataframe/backends/soap/commands/read.py

@@ -21,7 +21,9 @@ from spinta.typing import ObjectData
 from spinta.ufuncs.querybuilder.components import QueryParams
 
 
-def _get_data_soap(url: str, backend: Soap, soap_request_body: dict, extra_headers: dict) -> list[dict]:
+def _get_data_soap(
+    url: str, backend: Soap, soap_request_body: dict, extra_headers: dict, source: str | None = None
+) -> list[dict]:
     for key, value in soap_request_body.items():
         if isinstance(value, MakeCDATA):
             soap_request_body[key] = value()
@@ -38,6 +40,15 @@ def _get_data_soap(url: str, backend: Soap, soap_request_body: dict, extra_heade
     except zeep.exceptions.Error as e:
         raise UnexpectedErrorReadingData(exception=type(e).__name__, message=str(e))
 
+    if isinstance(response_data, dict):
+        for part in source.split("/"):
+            if not part:
+                continue
+            if part in response_data:
+                response_data = response_data[part]
+            else:
+                response_data = []
+                break
     if response_data and not isinstance(response_data, list):
         response_data = [response_data]
 
@@ -137,6 +148,7 @@ def getall(
             backend=backend,
             soap_request_body=builder.soap_request_body,
             extra_headers=http_headers,
+            source=model.external.name,
         )
         .flatten()
         .to_dataframe(meta=meta)

spinta-0.2.dev23/spinta/testing/csv.py

@@ -0,0 +1,13 @@
+import pathlib
+
+from requests.models import Response
+
+
+def parse_csv(resp: Response) -> list[list[str]]:
+    resp.raise_for_status()
+    return [line.strip().split(",") for line in resp.text.splitlines()]
+
+
+def read_csv(path: pathlib.Path) -> list[list[str]]:
+    with path.open("r") as f:
+        return [line.strip().split(",") for line in f.readlines()]

spinta-0.2.dev22/spinta/testing/csv.py

@@ -1,8 +0,0 @@
-from typing import List
-
-from requests.models import Response
-
-
-def parse_csv(resp: Response) -> List[List[str]]:
-    resp.raise_for_status()
-    return [line.strip().split(",") for line in resp.text.splitlines()]