spinta 0.2.dev13__tar.gz → 0.2.dev15__tar.gz

{spinta-0.2.dev13 → spinta-0.2.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev13
+Version: 0.2.dev15
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE
@@ -49,6 +49,7 @@ Requires-Dist: msgpack
 Requires-Dist: multipledispatch
 Requires-Dist: numpy
 Requires-Dist: openpyxl
+Requires-Dist: pandas (>=2.3.3,<3)
 Requires-Dist: phonenumbers
 Requires-Dist: pprintpp
 Requires-Dist: psutil

{spinta-0.2.dev13 → spinta-0.2.dev15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev13"
+version = "0.2dev15"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"
@@ -35,6 +35,7 @@ spinta = "spinta.cli.main:app"
 
 [tool.poetry.dependencies]
 python = "^3.10"
+pandas = ">=2.3.3,<3"
 aiofiles = "*"
 # https://github.com/atviriduomenys/spinta/issues/675
 # 0.x -> 1.x upgrade changed their implementation API, so we can no longer support 0.x.

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/backends/postgresql/ufuncs/query/ufuncs.py

@@ -28,7 +28,7 @@ from spinta.core.ufuncs import ufunc, GetAttr
 from spinta.datasets.backends.sql.ufuncs.components import Selected
 from spinta.exceptions import EmptyStringSearch, NoneValueComparison, NotImplementedFeature
 from spinta.exceptions import FieldNotInResource
-from spinta.types.datatype import Array
+from spinta.types.datatype import Array, Boolean
 from spinta.types.datatype import DataType, ExternalRef, Inherit, BackRef, Time, ArrayBackRef, Denorm
 from spinta.types.datatype import Date
 from spinta.types.datatype import DateTime
@@ -559,6 +559,12 @@ COMPARE_STRING = [
 ]
 
 
+def _compare(env: PgQueryBuilder, op: str, dtype: DataType, value: Any):
+    column = env.backend.get_column(env.table, dtype.prop)
+    cond = _sa_compare(env, dtype.prop, op, column, value)
+    return _prepare_condition(env, dtype.prop, cond)
+
+
 @ufunc.resolver(PgQueryBuilder, GetAttr, object, names=COMPARE)
 def compare(env: PgQueryBuilder, op: str, attr: GetAttr, value: Any):
     resolved = env.resolve_property(attr)
@@ -570,15 +576,13 @@ def compare(env, op: str, reserved: ReservedProperty, value: Any):
     table = env.backend.get_table(reserved.dtype.prop.model)
     column = table.c[reserved.dtype.prop.place + "." + reserved.param]
 
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, reserved.dtype.prop, op, column, value)
     return _prepare_condition(env, reserved.dtype.prop, cond)
 
 
 @ufunc.resolver(PgQueryBuilder, PrimaryKey, object, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, ForeignProperty, object, names=COMPARE)
@@ -594,84 +598,67 @@ def compare(env, op, dtype, value):
     except ValueError:
         raise exceptions.InvalidValue(dtype, op=op, arg=type(value).__name__)
 
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, str_value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, str_value)
 
 
 @ufunc.resolver(PgQueryBuilder, PrimaryKey, type(None), names=COMPARE)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, UUID_dtype, str, names=COMPARE)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, String, str, names=COMPARE)
 def compare(env, op, dtype, value):
     if op in ("startswith", "contains"):
         _ensure_non_empty(op, value)
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, (Integer, Number), (int, float), names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, DateTime, str, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
     value = datetime.datetime.fromisoformat(value)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, DateTime, datetime.datetime, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, Date, str, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
     value = datetime.date.fromisoformat(value)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, Date, datetime.date, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, Time, str, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
     value = datetime.time.fromisoformat(value)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, Time, datetime.time, names=COMPARE_EQUATIONS)
 def compare(env, op, dtype, value):
-    column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare(op, column, value)
-    return _prepare_condition(env, dtype.prop, cond)
+    return _compare(env, op, dtype, value)
+
+
+@ufunc.resolver(PgQueryBuilder, Boolean, bool, names=COMPARE_EQUATIONS)
+def compare(env, op, dtype, value):
+    return _compare(env, op, dtype, value)
 
 
 @ufunc.resolver(PgQueryBuilder, DataType, object, names=COMPARE)
@@ -693,21 +680,21 @@ def compare(
 @ufunc.resolver(PgQueryBuilder, UUID_dtype, str)
 def eq(env, dtype, value):
     column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare("eq", column, value)
+    cond = _sa_compare(env, dtype.prop, "eq", column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
 @ufunc.resolver(PgQueryBuilder, DataType, type(None))
 def eq(env, dtype, value):
     column = env.backend.get_column(env.table, dtype.prop)
-    cond = _sa_compare("eq", column, value)
+    cond = _sa_compare(env, dtype.prop, "eq", column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
 @ufunc.resolver(PgQueryBuilder, Text, (str, Bind, type(None)))
 def eq(env, dtype, value):
     column = get_language_column(env, dtype.prop)
-    cond = _sa_compare("eq", column, value)
+    cond = _sa_compare(env, dtype.prop, "eq", column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -720,7 +707,7 @@ def eq(
 ):
     table = env.get_joined_table(fpr)
     column = env.backend.get_column(table, fpr.right.prop)
-    cond = _sa_compare("eq", column, value)
+    cond = _sa_compare(env, fpr.right.prop, "eq", column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -733,7 +720,7 @@ def eq(
 ):
     table = env.backend.get_table(fpr.left.model)
     column = env.backend.get_column(table, fpr.left.prop)
-    cond = _sa_compare("eq", column, value)
+    cond = _sa_compare(env, fpr.left.prop, "eq", column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -743,11 +730,11 @@ def _ensure_non_empty(op, s):
 
 
 @ufunc.resolver(PgQueryBuilder, UUID_dtype, str, names=COMPARE_STRING)
-def compare(env: PgQueryBuilder, op: str, dtype: UUID, value: str):
+def compare(env: PgQueryBuilder, op: str, dtype: UUID_dtype, value: str):
     if op in ("startswith", "contains"):
         _ensure_non_empty(op, value)
     column = env.backend.get_column(env.table, dtype.prop).cast(sa.String)
-    return _sa_compare(op, column, value)
+    return _sa_compare(env, dtype.prop, op, column, value)
 
 
 @ufunc.resolver(PgQueryBuilder, ForeignProperty, String, str, names=COMPARE_STRING)
@@ -762,7 +749,7 @@ def compare(
         _ensure_non_empty(op, value)
     table = env.get_joined_table(fpr)
     column = table.c[fpr.right.prop.place]
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, fpr.right.prop, op, column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -771,7 +758,7 @@ def compare(env, op, dtype, value):
     if op in ("startswith", "contains"):
         _ensure_non_empty(op, value)
     column = env.backend.get_column(env.table, dtype.prop)
-    return _sa_compare(op, column, value)
+    return _sa_compare(env, dtype.prop, op, column, value)
 
 
 @ufunc.resolver(PgQueryBuilder, ForeignProperty, (Integer, Number), (int, float), names=COMPARE_EQUATIONS)
@@ -784,7 +771,7 @@ def compare(
 ):
     table = env.get_joined_table(fpr)
     column = table.c[fpr.right.prop.place]
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, fpr.right.prop, op, column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -799,7 +786,7 @@ def compare(
     table = env.get_joined_table(fpr)
     column = table.c[fpr.right.prop.place]
     value = datetime.datetime.fromisoformat(value)
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, fpr.right.prop, op, column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -814,7 +801,7 @@ def compare(
     table = env.get_joined_table(fpr)
     column = table.c[fpr.right.prop.place]
     value = datetime.date.fromisoformat(value)
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, fpr.right.prop, op, column, value)
     return _prepare_condition(env, dtype.prop, cond)
 
 
@@ -834,11 +821,11 @@ def compare(env, op, fn, value):
         _ensure_non_empty(op, value)
     column = get_column_with_extra(env, fn.dtype.prop)
     column = sa.func.lower(column)
-    cond = _sa_compare(op, column, value)
+    cond = _sa_compare(env, fn.dtype.prop, op, column, value)
     return _prepare_condition(env, fn.dtype.prop, cond)
 
 
-def _sa_compare(op, column, value):
+def _sa_compare(env, prop, op, column, value):
     if value is None and op not in ["eq", "ne"]:
         raise NoneValueComparison(op=op)
 
@@ -849,8 +836,13 @@ def _sa_compare(op, column, value):
         column = column.astext
 
     if op == "eq":
+        if isinstance(value, bool):
+            return column.is_(value)
         return column == value
 
+    if op == "ne":
+        return _ne_compare(env, prop, column, value)
+
     if op == "lt":
         return column < value
 
@@ -1036,6 +1028,8 @@ def _ne_compare(env: PgQueryBuilder, prop: Property, column, value):
     """
 
     if prop.list is None:
+        if isinstance(value, bool):
+            return column.is_not(value)
         return column != value
 
     main_table = env.backend.get_table(prop.model)

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/cli/helpers/sync/api_helpers.py

@@ -9,7 +9,7 @@ from spinta.client import get_access_token, RemoteClientCredentials
 from spinta.exceptions import UnexpectedAPIResponse, UnexpectedAPIResponseData
 
 
-STATIC_BASE_PATH_TAIL = "/uapi/datasets/org/vssa/isris/dcat"
+STATIC_BASE_PATH_TAIL = "/uapi/datasets/gov/vssa/ror/dcat"
 
 
 def validate_api_response(response: Response, expected_status_codes: set[HTTPStatus], operation: str) -> None:

spinta-0.2.dev13/spinta/cli/helpers/sync/controllers/synchronization/catalog_to_agent.py → spinta-0.2.dev15/spinta/cli/helpers/sync/controllers/synchronization/manifest_catalog_to_agent.py

@@ -20,7 +20,7 @@ from spinta.manifests.tabular.helpers import write_tabular_manifest, datasets_to
 PRIVATE_PROPERTY_PREFIX = "_"
 
 
-def execute_synchronization_catalog_to_agent(
+def execute_manifest_synchronization_catalog_to_agent(
     context: Context,
     base_path: str,
     headers: dict[str, str],

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/cli/sync.py

@@ -4,8 +4,8 @@ from typer import echo, Context as TyperContext
 
 from spinta.cli.helpers.sync.api_helpers import get_base_path_and_headers
 from spinta.cli.helpers.sync.controllers.data_service import get_data_service_children_dataset_ids
-from spinta.cli.helpers.sync.controllers.synchronization.catalog_to_agent import (
-    execute_synchronization_catalog_to_agent,
+from spinta.cli.helpers.sync.controllers.synchronization.manifest_catalog_to_agent import (
+    execute_manifest_synchronization_catalog_to_agent,
 )
 from spinta.cli.helpers.sync.helpers import (
     get_configuration_credentials,
@@ -44,7 +44,9 @@ def sync(ctx: TyperContext) -> None:
     base_path, headers, agent_name = prepare_api_context(context)
 
     dataset_ids = get_data_service_children_dataset_ids(base_path, headers, agent_name)
-    manifest = execute_synchronization_catalog_to_agent(context, base_path, headers, manifest_path, dataset_ids)
+    manifest = execute_manifest_synchronization_catalog_to_agent(
+        context, base_path, headers, manifest_path, dataset_ids
+    )
     echo(render_tabular_manifest(context, manifest))
 
     # TODO: Part 2: Source -> Agent: https://github.com/atviriduomenys/spinta/issues/1489.

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/datasets/backends/dataframe/commands/read.py

@@ -258,7 +258,8 @@ def dask_get_all(
 
     query = merge_formulas(model.external.prepare, query)
     query = merge_formulas(query, get_enum_filters(context, model))
-    query = merge_formulas(query, get_ref_filters(context, model))
+    if context.get("config").check_ref_filters:
+        query = merge_formulas(query, get_ref_filters(context, model))
     env = env.init(backend, df, params)
     expr = env.resolve(query)
     where = env.execute(expr)

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/manifests/open_api/openapi_config.py

@@ -527,6 +527,8 @@ COMMON_SCHEMAS = {
                     "version": {"type": "string", "examples": [0.1]},
                 },
             },
+            "dsa": {"type": "object", "properties": {"version": {"type": "string", "examples": ["0.1.0"]}}},
+            "uapi": {"type": "object", "properties": {"version": {"type": "string", "examples": ["0.1.0"]}}},
             "build": {"type": "object", "properties": {"version": {"type": "string", "examples": ["0.0.1"]}}},
         },
     },

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/manifests/open_api/openapi_generator.py

@@ -568,10 +568,10 @@ class OpenAPIGenerator:
 
         spec = {
             "openapi": VERSION,
-            "info": INFO,
-            "externalDocs": EXTERNAL_DOCS,
-            "servers": SERVERS,
-            "tags": BASE_TAGS,
+            "info": copy.deepcopy(INFO),
+            "externalDocs": copy.deepcopy(EXTERNAL_DOCS),
+            "servers": copy.deepcopy(SERVERS),
+            "tags": copy.deepcopy(BASE_TAGS),
             "components": {},
         }
 

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/types/backref/link.py

@@ -3,7 +3,6 @@ from spinta.components import Context, Model, Property
 from spinta.exceptions import (
     ModelReferenceNotFound,
     MultipleBackRefReferencesFound,
-    NoBackRefReferencesFound,
     NoReferencesFound,
     OneToManyBackRefNotSupported,
 )
@@ -83,21 +82,23 @@ def _link_backref(context: Context, dtype: BackRef):
     if not result:
         if given_refprop is not None:
             raise NoReferencesFound(dtype, prop_name=given_refprop, model=dtype.model)
-        raise NoBackRefReferencesFound(dtype, model=dtype.model.name)
-    for prop in result:
-        if count < 1:
-            if given_refprop and prop.name == given_refprop or given_refprop is None:
-                dtype.refprop = prop
-                count += 1
-        else:
-            raise MultipleBackRefReferencesFound(dtype, model=dtype.model.name)
+        # Allow backref without a corresponding ref property
+        dtype.refprop = None
+    else:
+        for prop in result:
+            if count < 1:
+                if given_refprop and prop.name == given_refprop or given_refprop is None:
+                    dtype.refprop = prop
+                    count += 1
+            else:
+                raise MultipleBackRefReferencesFound(dtype, model=dtype.model.name)
 
 
 @commands.link.register(Context, BackRef)
 def link(context: Context, dtype: BackRef) -> None:
     _link_backref(context, dtype)
 
-    if _is_parent_array_backref(dtype) or dtype.refprop.list is None:
+    if _is_parent_array_backref(dtype) or dtype.refprop is None or dtype.refprop.list is None:
         return
 
     raise OneToManyBackRefNotSupported(dtype)

{spinta-0.2.dev13 → spinta-0.2.dev15}/spinta/types/config.py

@@ -91,6 +91,7 @@ def load(context: Context, config: Config) -> Config:
     config.sync_retry_delay_range = rc.get("sync_retry_delay_range", default=(1, 5, 10, 30, 60), cast=tuple)
     config.languages = rc.get("languages", default=[])
     config.check_names = rc.get("check", "names", default=False)
+    config.check_ref_filters = rc.get("check_ref_filters", default=True, cast=asbool)
     config.root = rc.get("root", default=None)
     config.max_api_file_size = rc.get("max_file_size", default=100)
     config.max_error_count_on_insert = rc.get("max_error_count_on_insert", default=100)