spinta 0.2.dev23__tar.gz → 0.2.dev24__tar.gz

{spinta-0.2.dev23 → spinta-0.2.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev23
+Version: 0.2.dev24
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE
@@ -36,6 +36,7 @@ Requires-Dist: alembic (>=1.11.0,<1.12.0) ; extra == "migrations" or extra == "a
 Requires-Dist: asyncpg
 Requires-Dist: authlib (>=1.0.0)
 Requires-Dist: cachetools
+Requires-Dist: cbor2 (>=5.9.0,<6.0.0)
 Requires-Dist: click
 Requires-Dist: dask[dataframe]
 Requires-Dist: docutils (>=0.21.2,<0.22.0)

{spinta-0.2.dev23 → spinta-0.2.dev24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev23"
+version = "0.2dev24"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"
@@ -116,6 +116,7 @@ cachetools = "*"
 docutils = "^0.21.2"
 zeep = "^4.3.1"
 redis = "^6.4.0"
+cbor2 = "^5.9.0"
 
 [tool.poetry.extras]
 http = ["gunicorn", "uvicorn"]

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/backends/__init__.py

@@ -13,6 +13,7 @@ from typing import Iterable
 from typing import List
 from typing import Optional
 
+from cbor2 import dumps as cbor_dumps
 import dateutil
 import shapely.geometry.base
 from geoalchemy2.elements import WKTElement, WKBElement
@@ -23,7 +24,7 @@ from spinta import commands
 from spinta import exceptions
 from spinta.backends.components import Backend
 from spinta.backends.components import SelectTree
-from spinta.backends.helpers import check_unknown_props, get_select_tree, prepare_response
+from spinta.backends.helpers import check_unknown_props, get_select_tree, prepare_response, is_accessible_by_equals_sign
 from spinta.backends.helpers import flat_select_to_nested
 from spinta.backends.helpers import get_model_reserved_props
 from spinta.backends.helpers import get_select_prop_names
@@ -58,7 +59,19 @@ from spinta.exceptions import (
 from spinta.exceptions import NoItemRevision
 from spinta.formats.components import Format
 from spinta.manifests.components import Manifest
-from spinta.types.datatype import Array, ExternalRef, Inherit, PageType, BackRef, ArrayBackRef, Integer, Boolean, Denorm
+from spinta.types.datatype import (
+    Array,
+    ExternalRef,
+    Inherit,
+    PageType,
+    BackRef,
+    ArrayBackRef,
+    Integer,
+    Boolean,
+    Denorm,
+    Base32,
+    String,
+)
 from spinta.types.datatype import Binary
 from spinta.types.datatype import DataType
 from spinta.types.datatype import Date
@@ -590,12 +603,31 @@ def is_object_id(context: Context, value: str):
 
 @is_object_id.register(Context, Backend, Model, str)
 def is_object_id(context: Context, backend: Backend, model: Model, value: str):
+    return is_object_id(context, backend, model.properties["_id"].dtype, value)
+
+
+@is_object_id.register(Context, Backend, PrimaryKey, str)
+def is_object_id(context: Context, backend: Backend, dtype: PrimaryKey, value: str):
     try:
         return uuid.UUID(value).version == 4
     except ValueError:
         return False
 
 
+@is_object_id.register(Context, Backend, DataType, str)
+def is_object_id(context: Context, backend: Backend, dtype: DataType, value: str):
+    candidate = value
+    if is_accessible_by_equals_sign(dtype.prop, value):
+        if not value.startswith("="):
+            return False
+        candidate = value[1:]
+    try:
+        dtype.load(candidate)
+    except exceptions.InvalidValue:
+        return False
+    return True
+
+
 @is_object_id.register(Context, Backend, Model, uuid.UUID)
 def is_object_id(context: Context, backend: Backend, model: Model, value: uuid.UUID):
     return value.version == 4
@@ -1750,6 +1782,13 @@ def cast_backend_to_python(context: Context, dtype: DataType, backend: Backend,
     return data
 
 
+@commands.cast_backend_to_python.register(Context, String, Backend, object)
+def cast_backend_to_python(context: Context, dtype: String, backend: Backend, data: Any, **kwargs) -> Any:
+    if data is None or is_nan(data):
+        return None
+    return str(data)
+
+
 @commands.cast_backend_to_python.register(Context, UUID, Backend, object)
 def cast_backend_to_python(context: Context, dtype: UUID, backend: Backend, data: Any, **kwargs) -> Any:
     if is_nan(data):
@@ -1891,6 +1930,13 @@ def cast_backend_to_python(context: Context, dtype: Ref, backend: Backend, data:
 
     processed_data = {}
     for key in data:
+        if key == "_id":
+            # _id reaches this dispatch already in its final form — produced by
+            # handle_ref_key_assignment for external readers, or read directly
+            # from the storage column for internal backends. Re-applying the
+            # referenced model's _id cast double-encodes Base32 ids.
+            processed_data[key] = data[key]
+            continue
         prop = commands.resolve_property(dtype.prop.model, f"{dtype.prop.place}.{key}")
         if prop is not None:
             processed_data[key] = commands.cast_backend_to_python(context, prop, backend, data[key], **kwargs)
@@ -1939,6 +1985,18 @@ def cast_backend_to_python(context: Context, dtype: Denorm, backend: Backend, da
     return commands.cast_backend_to_python(context, dtype.rel_prop, backend, data, **kwargs)
 
 
+@commands.cast_backend_to_python.register(Context, Base32, Backend, object)
+def cast_backend_to_python(context: Context, dtype: Base32, backend: Backend, data: Any, **kwargs) -> Any:
+    if is_nan(data):
+        return None
+    if isinstance(data, (list, tuple)):
+        data = cbor_dumps(list(data))
+    else:
+        data = str(data).encode("utf-8")
+    encoded = base64.b32encode(data)
+    return encoded.rstrip(b"=").decode("utf-8")
+
+
 @commands.reload_backend_metadata.register(Context, Manifest, Backend)
 def reload_backend_metadata(context, manifest, backend):
     pass

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/backends/helpers.py

@@ -26,7 +26,7 @@ from spinta.components import Model
 from spinta.components import Namespace
 from spinta.components import Property
 from spinta.exceptions import BackendUnavailable
-from spinta.types.datatype import DataType, Denorm
+from spinta.types.datatype import DataType, Denorm, String, Base32, PrimaryKey
 from spinta.utils.data import take
 from spinta.backends.constants import TableType, BackendOrigin
 
@@ -596,3 +596,28 @@ def extract_table_data_from_logical_name(table_name: str) -> tuple[str | None, T
             return data[0], table_type, None
 
     return None, None, None
+
+
+def is_custom_id_prop(prop: Property) -> bool:
+    return prop.name == "_id" and not isinstance(prop.dtype, PrimaryKey)
+
+
+def is_custom_revision_prop(prop: Property) -> bool:
+    return prop.name == "_revision" and prop.explicitly_given
+
+
+def is_accessible_by_equals_sign(id_prop: Property, value: str | int) -> bool:
+    if isinstance(id_prop.dtype, Base32):
+        return True
+
+    if isinstance(id_prop.dtype, String):
+        return not check_if_model_primary_key_is_composite(id_prop.model)
+
+    return False
+
+
+def check_if_model_primary_key_is_composite(model: Model) -> bool:
+    pkeys_count = len(model.external.pkeys)
+    if pkeys_count > 1:
+        return True
+    return False

spinta-0.2.dev24/spinta/cli/comment.py

@@ -0,0 +1,113 @@
+from typer import Argument
+from typer import Context as TyperContext
+from typer import Option
+from typer import echo
+
+from spinta.cli.helpers.enums import CommentPart
+from spinta.cli.helpers.store import load_manifest
+from spinta.components import Context, Property
+from spinta.core.context import configure_context
+from spinta.core.enums import Access
+from spinta.manifests.components import Manifest
+from spinta.manifests.tabular.helpers import datasets_to_tabular
+from spinta.manifests.tabular.helpers import render_tabular_manifest_rows
+from spinta.manifests.tabular.helpers import write_tabular_manifest
+
+
+def comment(
+    ctx: TyperContext,
+    part: CommentPart = Argument(None, help="Part to comment"),
+    author: str | None = Option(None, "--author", help="Comment author"),
+    uri: str | None = Option(
+        None, "--uri", help="URI tag stored on each comment (use with uncomment --uri to restore selectively)"
+    ),
+    description: str | None = Option(None, "--description", help="Additional description stored on each comment"),
+    output: str | None = Option(None, "-o", "--output", help="Output tabular manifest in a specified file"),
+    manifests: list[str] = Argument(None, help="Source manifest files"),
+) -> None:
+    """Comment out specific manifest parts so the manifest can be processed further."""
+    context: Context = ctx.obj
+    context = configure_context(context, manifests)
+
+    comment_manifest(
+        context,
+        author=author,
+        uri=uri,
+        description=description,
+        output=output,
+        manifests=manifests,
+    )
+
+
+def comment_manifest(
+    context: Context,
+    author: str | None = None,
+    uri: str | None = None,
+    description: str | None = None,
+    output: str | None = None,
+    manifests: list[str] | None = None,
+) -> None:
+    verbose = bool(output)
+    context = configure_context(context, manifests)
+    store = load_manifest(
+        context,
+        load_internal=False,
+        verbose=verbose,
+        full_load=True,
+    )
+
+    _update_systemic_comments(context, store.manifest, author=author, uri=uri, description=description)
+    rows = datasets_to_tabular(context, store.manifest, external=False, access=Access.private)
+
+    if output:
+        write_tabular_manifest(context, output, rows)
+    else:
+        manager = context.get("error_manager")
+        handler = manager.handler
+        table = render_tabular_manifest_rows(rows)
+        if handler.get_counts():
+            handler.post_process()
+        else:
+            echo(table)
+
+
+def _update_systemic_comments(
+    context: Context,
+    manifest: Manifest,
+    *,
+    author: str | None = None,
+    uri: str | None = None,
+    description: str | None = None,
+) -> None:
+    """Update author/uri/description on comment type properties."""
+    from spinta import commands
+
+    if all(argument is None for argument in [author, uri, description]):
+        return
+
+    models = commands.get_models(context, manifest)
+    for model in models.values():
+        for prop in model.properties.values():
+            _patch_property_restore_comments(prop, author=author, uri=uri, description=description)
+
+
+def _patch_property_restore_comments(
+    prop: Property,
+    *,
+    author: str | None,
+    uri: str | None,
+    description: str | None,
+) -> None:
+    if not prop.comments:
+        return
+
+    for comment in prop.comments:
+        if not comment.prepare or not comment.prepare.startswith("update"):
+            continue
+
+        if author is not None:
+            comment.author = author
+        if uri is not None:
+            comment.uri = uri
+        if description is not None:
+            comment.comment = description

spinta-0.2.dev24/spinta/cli/helpers/enums.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+
+class CommentPart(str, Enum):
+    missing_external_refs = "missing-external-refs"

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/cli/main.py

@@ -17,6 +17,8 @@ from spinta.cli import data
 from spinta.cli import inspect
 from spinta.cli import manifest
 from spinta.cli import migrate
+from spinta.cli.comment import comment
+from spinta.cli.uncomment import uncomment
 from spinta.cli import pii
 from spinta.cli import pull
 from spinta.cli import push
@@ -45,6 +47,8 @@ add(app, "inspect", inspect.inspect, short_help=("Update manifest schema from an
 add(app, "pii", pii.app, short_help="Manage Person Identifying Information")
 
 add(app, "copy", manifest.copy, short_help="Copy only specified metadata from a manifest")
+add(app, "comment", comment, short_help="Comment unsupported functionality in a manifest")
+add(app, "uncomment", uncomment, short_help="Restore commented parts of a manifest")
 add(app, "show", show, short_help="Show manifest as ascii table")
 
 add(app, "bootstrap", migrate.bootstrap, short_help="Initialize backends")

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/cli/manifest.py

@@ -1,6 +1,5 @@
 from typing import Iterator, Union
 from typing import List
-from typing import Optional
 
 from typer import Argument
 from typer import Context as TyperContext
@@ -36,9 +35,10 @@ def copy(
     #       https://github.com/tiangolo/typer/issues/151
     access: str = Option("private", help=("Copy properties with at least specified access")),
     format_names: bool = Option(False, help=("Reformat model and property names.")),
-    output: Optional[str] = Option(None, "-o", "--output", help=("Output tabular manifest in a specified file")),
-    columns: Optional[str] = Option(None, "-c", "--columns", help=("Comma separated list of columns")),
-    order_by: Optional[str] = Option(
+    output: str | None = Option(None, "-o", "--output", help=("Output tabular manifest in a specified file")),
+    dataset: str | None = Option(None, "-d", "--dataset", help=("Main dataset name")),
+    columns: str | None = Option(None, "-c", "--columns", help=("Comma separated list of columns")),
+    order_by: str | None = Option(
         None, help=("Order by a specified column (currently only access column is supported)")
     ),
     rename_duplicates: bool = Option(False, help=("Rename duplicate model names by adding number suffix")),
@@ -57,6 +57,7 @@ def copy(
         order_by=order_by,
         rename_duplicates=rename_duplicates,
         manifests=manifests,
+        dataset=dataset,
     )
 
 
@@ -65,12 +66,13 @@ def copy_manifest(
     source: bool = True,
     access: str = "private",
     format_names: bool = False,
-    output: Optional[str] = None,
-    columns: Optional[str] = None,
-    order_by: Optional[str] = None,
+    output: str | None = None,
+    columns: str | None = None,
+    order_by: str | None = None,
     rename_duplicates: bool = False,
     manifests: List[str] = None,
-    output_type: Optional[str] = None,
+    output_type: str | None = None,
+    dataset: str | None = None,
 ):
     """Copy models from CSV manifest files into another CSV manifest file"""
     access = get_enum_by_name(Access, access)
@@ -113,7 +115,7 @@ def copy_manifest(
         )
     if output:
         if output_type == "mermaid":
-            write_mermaid_manifest(context, output, rows)
+            write_mermaid_manifest(context, rows, dataset, output)
         elif internal:
             write_internal_sql_manifest(context, output, rows)
         else:

spinta-0.2.dev24/spinta/cli/uncomment.py

@@ -0,0 +1,110 @@
+import csv
+import pathlib
+import re
+
+from typer import Argument
+from typer import Context as TyperContext
+from typer import Option
+from typer import echo
+
+from spinta.components import Context
+from spinta.manifests.tabular.components import MANIFEST_COLUMNS
+from spinta.manifests.tabular.components import ManifestRow
+from spinta.manifests.tabular.helpers import render_tabular_manifest_rows
+from spinta.manifests.tabular.helpers import torow
+from spinta.manifests.tabular.helpers import write_tabular_manifest
+
+
+UPDATE_FUNCTION = "update"
+COLUMN_TYPE_COMMENT = "comment"
+
+
+def uncomment(
+    ctx: TyperContext,
+    uri: str | None = Option(None, "--uri", help="Only restore comments that carry this URI tag"),
+    output: str | None = Option(None, "-o", "--output", help="Output tabular manifest in a specified file"),
+    manifests: list[str] = Argument(None, help="Source manifest files"),
+):
+    """Restore commented-out properties."""
+    context: Context = ctx.obj
+    uncomment_manifest(context, uri=uri, output=output, manifests=manifests)
+
+
+def uncomment_manifest(
+    context: Context,
+    uri: str | None = None,
+    output: str | None = None,
+    manifests: list[str] | None = None,
+) -> None:
+    """Read raw CSV rows, remove `update` comments, and write the result."""
+    rows = _read_raw_manifest_rows(manifests or [])
+    result = _uncomment_rows(rows, uri_filter=uri)
+    if output:
+        write_tabular_manifest(context, output, iter(result))
+    else:
+        echo(render_tabular_manifest_rows(iter(result)))
+
+
+def _read_raw_manifest_rows(manifests: list[str]) -> list[ManifestRow]:
+    """Read CSV manifest files as raw dicts, normalising to MANIFEST_COLUMNS keys."""
+    rows = []
+    for path in manifests:
+        with pathlib.Path(path).open(encoding="utf-8-sig") as file:
+            reader = csv.DictReader(file)
+            for raw in reader:
+                rows.append(torow(MANIFEST_COLUMNS, raw))
+    return rows
+
+
+def _parse_update_fields(prepare: str) -> dict[str, str]:
+    """Parse 'update(key1:val1, key2:val2, ...)' into {'key1': 'val1', 'key2': 'val2', ...}.
+
+    Splits on commas only when followed by a word-colon pattern, so values containing
+    commas inside brackets (e.g. ref:example2/City[id, name]) are handled correctly.
+    """
+    match = re.match(rf"{UPDATE_FUNCTION}\((.+)\)\s*$", (prepare or "").strip())
+    if not match:
+        return {}
+    parts = re.split(r",\s*(?=\w+:)", match.group(1))
+    result = {}
+    for part in parts:
+        key, _, value = part.partition(":")
+        result[key.strip()] = value.strip().strip('"')
+    return result
+
+
+def _is_restore_comment(row: ManifestRow) -> bool:
+    prepare = row.get("prepare") or ""
+    return row.get("type") == COLUMN_TYPE_COMMENT and prepare.startswith(UPDATE_FUNCTION)
+
+
+def _uncomment_rows(rows: list[ManifestRow], uri_filter: str | None) -> list[ManifestRow]:
+    """
+    Walk rows in order. When an `update` comment is found (and passes the URI filter),
+    patch the most recent property row in result and drop the comment row.
+    """
+    result: list[ManifestRow] = []
+    last_property_index: int | None = None
+
+    for row in rows:
+        if _is_restore_comment(row):
+            # Restore only rows that match the `uri` if it is given by input;
+            if uri_filter is not None and row.get("uri") != uri_filter:
+                result.append(row)
+                continue
+
+            # Parse the expression and apply field to the last seen row (comments go after the row they change);
+            fields = _parse_update_fields(row.get("prepare", ""))
+            if fields and last_property_index is not None:
+                prop_row: dict = dict(result[last_property_index])
+                for key, value in fields.items():
+                    prop_row[key] = value
+                prop_row["level"] = row.get("level") or ""
+                result[last_property_index] = prop_row
+        else:
+            # If it's not an `update` comment, append to the result as-is;
+            if row.get("property"):
+                last_property_index = len(result)
+            result.append(row)
+
+    return result

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/components.py

@@ -38,6 +38,7 @@ if TYPE_CHECKING:
     from spinta.accesslog import AccessLog
     from spinta.formats.components import Format
     from spinta.dimensions.comments.components import Comment
+    from spinta.dimensions.scope.components import Scope
 
 
 class Context:
@@ -620,6 +621,14 @@ def page_in_data(data: dict) -> bool:
     return "_page" in data
 
 
+def revision_in_data(data: dict) -> bool:
+    return "_revision" in data
+
+
+def check_if_revision_explicit(prop: Property) -> bool:
+    return prop.name == "_revision" and prop.explicitly_given
+
+
 class ParamsPage:
     values: List[Any]
     size: int
@@ -642,6 +651,7 @@ class Model(MetaData):
     description: str
     ns: Namespace
     external: Entity = None
+    scopes: Dict[str, Scope]
     properties: Dict[str, Property]
     mode: Mode = None
     given: ModelGiven
@@ -666,6 +676,7 @@ class Model(MetaData):
         "unique": {"default": []},
         "base": {},
         "link": {},
+        "scopes": {"default": {}},
         "properties": {"default": {}},
         "external": {},
         "level": {
@@ -884,6 +895,7 @@ class Attachment:
 class UrlParseNode(TypedDict):
     name: str
     args: List[Any]
+    id_prop: Property
 
 
 class UrlParams:

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/config.py

@@ -165,6 +165,7 @@ CONFIG = {
             "partial_array": "spinta.types.datatype:PartialArray",
             "uuid": "spinta.types.datatype:UUID",
             "unknown": "spinta.types.datatype:Unknown",
+            "base32": "spinta.types.datatype:Base32",
         },
         "urlparams": {
             "component": "spinta.urlparams:UrlParams",

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/core/access.py

@@ -9,6 +9,7 @@ from spinta.utils.enums import enum_by_name, get_enum_by_name
 
 if TYPE_CHECKING:
     from spinta.dimensions.enum.components import EnumItem
+    from spinta.dimensions.scope.components import Scope
     from spinta.components import Model
     from spinta.components import Namespace
     from spinta.components import Property
@@ -25,6 +26,7 @@ def load_access_param(
         Model,
         Property,
         EnumItem,
+        Scope,
     ],
     given_access: Union[str, Access],
     parents: Iterable[

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/datasets/backends/dataframe/commands/read.py

@@ -15,16 +15,17 @@ from spinta.core.ufuncs import Expr, Env
 from spinta.datasets.backends.dataframe.components import DaskBackend
 from spinta.datasets.backends.dataframe.backends.memory.components import MemoryDaskBackend
 from spinta.datasets.backends.dataframe.ufuncs.query.components import DaskDataFrameQueryBuilder
+from spinta.backends.helpers import is_custom_id_prop
 from spinta.datasets.backends.helpers import handle_ref_key_assignment
 from spinta.datasets.components import Resource
-from spinta.datasets.helpers import get_enum_filters, get_ref_filters
+from spinta.datasets.helpers import get_enum_filters, get_ref_filters, encode_composite_string_id
 from spinta.datasets.keymaps.components import KeyMap
 from spinta.datasets.utils import iterparams
 from spinta.dimensions.enum.helpers import get_prop_enum
 from spinta.dimensions.param.components import ResolvedParams
 from spinta.exceptions import PropertyNotFound, NoExternalName, ValueNotInEnum
 from spinta.manifests.components import Manifest
-from spinta.types.datatype import PrimaryKey, Ref, DataType, Boolean, Number, Integer, DateTime
+from spinta.types.datatype import PrimaryKey, Ref, DataType, Boolean, Number, Integer, DateTime, Base32
 from spinta.typing import ObjectData
 from spinta.ufuncs.querybuilder.components import Selected
 from spinta.ufuncs.helpers import merge_formulas
@@ -284,6 +285,8 @@ def dask_get_all(
                     val = keymap.encode(sel.prop.model.model_type(), val)
                 elif isinstance(sel.prop.dtype, Ref):
                     val = handle_ref_key_assignment(context, keymap, env, val, sel.prop.dtype)
+                elif is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
+                    val = encode_composite_string_id(val, model.external.pkeys)
             res[key] = val
         res = commands.cast_backend_to_python(context, model, backend, res, extra_properties=extra_properties)
         yield res

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -6,6 +6,7 @@ from typing import Dict, Any, Tuple, List, Iterator
 from dask.dataframe import Series
 
 from spinta.auth import authorized
+from spinta.backends.helpers import is_custom_id_prop, is_custom_revision_prop
 from spinta.components import Property
 from spinta.core.enums import Action
 from spinta.core.ufuncs import Expr, ufunc, Bind, Unresolved, GetAttr
@@ -166,7 +167,9 @@ def select(env: DaskDataFrameQueryBuilder, expr: Expr):
             else:
                 raise PropertyNotFound(env.model, property=resolved[selected_key])
     else:
-        for prop in take(["_id", all], env.model.properties).values():
+        for prop in take(["_id", "_revision", all], env.model.properties).values():
+            if prop.name == "_revision" and not is_custom_revision_prop(prop):
+                continue
             if authorized(env.context, prop, Action.GETALL):
                 env.selected[prop.place] = env.call("select", prop)
 
@@ -238,6 +241,13 @@ def select(env: DaskDataFrameQueryBuilder, prop: Property) -> Selected:
         elif prop.external and prop.external.name:
             # If prepare is not given, then take value from `source`.
             result = env.call("select", prop.dtype)
+        elif is_custom_id_prop(prop):
+            pkeys = prop.model.external.pkeys or list(take(prop.model.properties).values())
+            if len(pkeys) == 1:
+                prep = env.call("select", pkeys[0])
+            else:
+                prep = [env.call("select", pk) for pk in pkeys]
+            result = Selected(prop=prop, prep=prep)
         elif prop.is_reserved():
             # Reserved properties never have external source.
             result = env.call("select", prop.dtype)

{spinta-0.2.dev23 → spinta-0.2.dev24}/spinta/datasets/backends/helpers.py

@@ -3,6 +3,7 @@ import pathlib
 from typing import Any
 
 from spinta import commands, spyna
+from spinta.backends.helpers import is_custom_id_prop
 from spinta.components import Property, Model, Context
 from spinta.core.enums import Mode
 from spinta.core.ufuncs import Env, asttoexpr
@@ -49,7 +50,7 @@ def handle_ref_key_assignment(context: Context, keymap: KeyMap, env: Env, value:
 
         val = None
         contains = None
-        if ref.refprops[0].name != "_id":
+        if ref.refprops[0].name != "_id" and not is_custom_id_prop(ref.model.properties["_id"]):
             contains = keymap.contains(keymap_name, target_value)
         if not contains:
             if target_value is None: