spinta 0.2.dev25__tar.gz → 0.2.dev26__tar.gz

{spinta-0.2.dev25 → spinta-0.2.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spinta
-Version: 0.2.dev25
+Version: 0.2.dev26
 Summary: A platform for describing, extracting, transforming, loading and serving open data.
 License: MIT
 License-File: LICENSE

{spinta-0.2.dev25 → spinta-0.2.dev26}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "spinta"
-version = "0.2dev25"
+version = "0.2dev26"
 description = "A platform for describing, extracting, transforming, loading and serving open data."
 authors = ["Mantas Zimnickas <sirexas@gmail.com>"]
 license = "MIT"

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/backends/helpers.py

@@ -91,7 +91,7 @@ def get_select_tree(
         select = list(select.keys())
 
     select = _apply_always_show_id(context, action, select)
-    if select is None and action in (Action.GETALL, Action.SEARCH):
+    if select is None and action in (Action.GETALL, Action.SEARCH, Action.GETONE):
         # If select is not given, select everything.
         select = {"*": {}}
     return flat_select_to_nested(select)

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/backends/postgresql/commands/load.py

@@ -5,6 +5,7 @@ import sqlalchemy as sa
 from spinta import commands
 from spinta.backends.postgresql.components import PostgreSQL
 from spinta.backends.postgresql.helpers.name import PG_NAMING_CONVENTION
+from spinta.backends.postgresql.sqlalchemy import create_postgresql_engine
 from spinta.components import Context
 from spinta.utils.sqlalchemy import get_metadata_naming_convention
 
@@ -12,7 +13,7 @@ from spinta.utils.sqlalchemy import get_metadata_naming_convention
 @commands.load.register(Context, PostgreSQL, dict)
 def load(context: Context, backend: PostgreSQL, config: Dict[str, Any]):
     backend.dsn = config["dsn"]
-    backend.engine = sa.create_engine(backend.dsn, echo=False)
+    backend.engine = create_postgresql_engine(backend.dsn, echo=False)
     backend.schema = sa.MetaData(backend.engine, naming_convention=get_metadata_naming_convention(PG_NAMING_CONVENTION))
     backend.tables = {}
 

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/backends/postgresql/commands/wait.py

@@ -1,7 +1,7 @@
-import sqlalchemy as sa
 import sqlalchemy.exc
 
 from spinta import commands
+from spinta.backends.postgresql.sqlalchemy import create_postgresql_engine
 from spinta.components import Context
 from spinta.backends.postgresql.components import PostgreSQL
 
@@ -10,7 +10,7 @@ from spinta.backends.postgresql.components import PostgreSQL
 def wait(context: Context, backend: PostgreSQL, *, fail: bool = False) -> bool:
     rc = context.get("rc")
     dsn = rc.get("backends", backend.name, "dsn", required=True)
-    engine = sa.create_engine(dsn, connect_args={"connect_timeout": 0})
+    engine = create_postgresql_engine(dsn, connect_args={"connect_timeout": 0})
     try:
         conn = engine.connect()
     except sqlalchemy.exc.OperationalError:

spinta-0.2.dev26/spinta/backends/postgresql/sqlalchemy.py

@@ -0,0 +1,38 @@
+import sqlalchemy as sa
+
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.sql.expression import FunctionElement
+from sqlalchemy.dialects.mssql.base import ischema_names
+from geoalchemy2.types import Geometry
+
+
+ischema_names["geometry"] = Geometry
+
+
+class utcnow(FunctionElement):
+    inherit_cache = True
+    type = sa.DateTime()
+
+
+@compiles(utcnow, "postgresql")
+def pg_utcnow(element, compiler, **kw):
+    return "TIMEZONE('utc', CURRENT_TIMESTAMP)"
+
+
+def create_postgresql_engine(uri: str, connect_args: dict | None = None, **kwargs) -> sa.engine.Engine:
+    default_kwargs = {
+        "pool_pre_ping": True,
+        "pool_recycle": 900,
+    }
+
+    default_kwargs.update(kwargs)
+    default_connect_args = {
+        "keepalives": 1,
+        "keepalives_idle": 60,
+        "keepalives_interval": 10,
+        "keepalives_count": 5,
+    }
+    if connect_args:
+        default_connect_args.update(connect_args)
+
+    return sa.create_engine(uri, connect_args=default_connect_args, **default_kwargs)

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/cli/comment.py

@@ -5,15 +5,19 @@ from typer import echo
 
 from spinta.cli.helpers.enums import CommentPart
 from spinta.cli.helpers.store import load_manifest
-from spinta.components import Context, Property
+from spinta.components import Context
 from spinta.core.context import configure_context
 from spinta.core.enums import Access
+from spinta.dimensions.comments.components import Comment
 from spinta.manifests.components import Manifest
 from spinta.manifests.tabular.helpers import datasets_to_tabular
 from spinta.manifests.tabular.helpers import render_tabular_manifest_rows
 from spinta.manifests.tabular.helpers import write_tabular_manifest
 
 
+RESTORE_COMMENT_PREFIXES = ("update", "insert")
+
+
 def comment(
     ctx: TyperContext,
     part: CommentPart = Argument(None, help="Part to comment"),
@@ -79,7 +83,7 @@ def _update_systemic_comments(
     uri: str | None = None,
     description: str | None = None,
 ) -> None:
-    """Update author/uri/description on comment type properties."""
+    """Update author/uri/description on systemic restore comments (refs and bases)."""
     from spinta import commands
 
     if all(argument is None for argument in [author, uri, description]):
@@ -87,22 +91,24 @@ def _update_systemic_comments(
 
     models = commands.get_models(context, manifest)
     for model in models.values():
+        _patch_restore_comments(model.comments, author=author, uri=uri, description=description)
         for prop in model.properties.values():
-            _patch_property_restore_comments(prop, author=author, uri=uri, description=description)
+            _patch_restore_comments(prop.comments, author=author, uri=uri, description=description)
 
 
-def _patch_property_restore_comments(
-    prop: Property,
+def _patch_restore_comments(
+    comments: list[Comment] | None,
     *,
     author: str | None,
     uri: str | None,
     description: str | None,
 ) -> None:
-    if not prop.comments:
+    if not comments:
         return
 
-    for comment in prop.comments:
-        if not comment.prepare or not comment.prepare.startswith("update"):
+    for comment in comments:
+        prepare = comment.prepare or ""
+        if not prepare.startswith(RESTORE_COMMENT_PREFIXES):
             continue
 
         if author is not None:

spinta-0.2.dev26/spinta/cli/uncomment.py

@@ -0,0 +1,172 @@
+import csv
+import pathlib
+import re
+
+from typer import Argument
+from typer import Context as TyperContext
+from typer import Option
+from typer import echo
+
+from spinta.components import Context
+from spinta.manifests.tabular.components import MANIFEST_COLUMNS
+from spinta.manifests.tabular.components import ManifestRow
+from spinta.manifests.tabular.helpers import render_tabular_manifest_rows
+from spinta.manifests.tabular.helpers import torow
+from spinta.manifests.tabular.helpers import write_tabular_manifest
+
+
+UPDATE_FUNCTION = "update"
+COLUMN_TYPE_COMMENT = "comment"
+INSERT_FUNCTION = "insert"
+
+
+def uncomment(
+    ctx: TyperContext,
+    uri: str | None = Option(None, "--uri", help="Only restore comments that carry this URI tag"),
+    output: str | None = Option(None, "-o", "--output", help="Output tabular manifest in a specified file"),
+    manifests: list[str] = Argument(None, help="Source manifest files"),
+):
+    """Restore commented-out properties."""
+    context: Context = ctx.obj
+    uncomment_manifest(context, uri=uri, output=output, manifests=manifests)
+
+
+def uncomment_manifest(
+    context: Context,
+    uri: str | None = None,
+    output: str | None = None,
+    manifests: list[str] | None = None,
+) -> None:
+    """Read raw CSV rows, remove `update`, `insert` comments, and write the result."""
+    rows = _read_raw_manifest_rows(manifests or [])
+    result = _uncomment_rows(rows, uri_filter=uri)
+    if output:
+        write_tabular_manifest(context, output, iter(result))
+    else:
+        echo(render_tabular_manifest_rows(iter(result)))
+
+
+def _read_raw_manifest_rows(manifests: list[str]) -> list[ManifestRow]:
+    """Read CSV manifest files as raw dicts, normalising to MANIFEST_COLUMNS keys."""
+    rows = []
+    for path in manifests:
+        with pathlib.Path(path).open(encoding="utf-8-sig") as file:
+            reader = csv.DictReader(file)
+            for raw in reader:
+                rows.append(torow(MANIFEST_COLUMNS, raw))
+    return rows
+
+
+def _parse_function_fields(prepare: str, func_name: str) -> dict[str, str]:
+    """Parse 'func_name(key1:val1, key2:val2, ...)' into {'key1': 'val1', 'key2': 'val2', ...}.
+
+    Splits on commas only when followed by a word-colon pattern, so values containing
+    commas inside brackets (e.g. ref:example2/City[id, name]) are handled correctly.
+    """
+    match = re.match(rf"{func_name}\((.+)\)\s*$", (prepare or "").strip())
+    if not match:
+        return {}
+    parts = re.split(r",\s*(?=\w+:)", match.group(1))
+    result = {}
+    for part in parts:
+        key, _, value = part.partition(":")
+        result[key.strip()] = value.strip().strip('"')
+    return result
+
+
+def _parse_update_fields(prepare: str) -> dict[str, str]:
+    return _parse_function_fields(prepare, UPDATE_FUNCTION)
+
+
+def _parse_insert_fields(prepare: str) -> dict[str, str]:
+    return _parse_function_fields(prepare, INSERT_FUNCTION)
+
+
+def _is_restore_comment(row: ManifestRow, func_name: str) -> bool:
+    prepare = row.get("prepare") or ""
+    return row.get("type") == COLUMN_TYPE_COMMENT and prepare.startswith(func_name)
+
+
+def _insert_base_resets(result: list[ManifestRow]) -> list[ManifestRow]:
+    """Insert `/` reset rows where base context would leak from a previous
+    model into a model that has no base row directly above it."""
+    fixed: list[ManifestRow] = []
+    active_base: str | None = None
+    base_seen_since_last_model: bool = False
+
+    for row in result:
+        is_base_row = row.get("base") and not row.get("model") and not row.get("property")
+        is_model_row = bool(row.get("model"))
+
+        if is_base_row:
+            active_base = row["base"]
+            base_seen_since_last_model = True
+            fixed.append(row)
+        elif is_model_row:
+            if not base_seen_since_last_model and active_base and active_base != "/":
+                # No base row immediately above this model, but a base context
+                # is still active from a previous model. Insert `/` to break it.
+                reset_row = {col: "" for col in row.keys()}
+                reset_row["base"] = "/"
+                fixed.append(reset_row)
+                active_base = "/"
+            base_seen_since_last_model = False
+            fixed.append(row)
+        else:
+            fixed.append(row)
+
+    return fixed
+
+
+def _uncomment_rows(rows: list[ManifestRow], uri_filter: str | None) -> list[ManifestRow]:
+    """
+    Walk rows in order. When an `update` or `insert` comment is found (and passes the URI filter),
+    patch the most recent property row in `update` case, insert a new base row before the model in `insert` case and drop the comment row.
+    """
+    result: list[ManifestRow] = []
+    last_property_index: int | None = None
+    last_model_index: int | None = None
+    base_was_inserted: bool = False
+
+    for row in rows:
+        if _is_restore_comment(row, UPDATE_FUNCTION) or _is_restore_comment(row, INSERT_FUNCTION):
+            # Restore only rows that match the `uri` if it is given by input;
+            if uri_filter is not None and row.get("uri") != uri_filter:
+                result.append(row)
+                continue
+            if _is_restore_comment(row, UPDATE_FUNCTION):
+                fields = _parse_update_fields(row.get("prepare", ""))
+                if fields and last_property_index is not None:
+                    prop_row: dict = dict(result[last_property_index])
+                    for key, value in fields.items():
+                        prop_row[key] = value
+                    prop_row["level"] = row.get("level") or ""
+                    result[last_property_index] = prop_row
+
+            elif _is_restore_comment(row, INSERT_FUNCTION):
+                fields = _parse_insert_fields(row.get("prepare", ""))
+                if fields and last_model_index is not None:
+                    base_row: dict = {col: "" for col in row.keys()}
+                    for key, value in fields.items():
+                        base_row[key] = value
+                    if row.get("level"):
+                        model_row = result[last_model_index]
+                        model_row["level"] = row.get("level")
+                        result[last_model_index] = model_row
+                    result.insert(last_model_index, base_row)
+                    last_model_index += 1
+                    base_was_inserted = True
+
+        else:
+            # If it's not an `update` or `insert` comment, append as-is;
+            if row.get("model"):
+                last_model_index = len(result)
+                last_property_index = None
+            if row.get("property"):
+                last_property_index = len(result)
+            result.append(row)
+
+    if base_was_inserted:
+        result = _insert_base_resets(result)
+
+    return result

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/components.py

@@ -929,6 +929,8 @@ class UrlParams:
     select_props: Optional[Dict[str, Union[Expr, Bind]]] = None
     select_funcs: Optional[Dict[str, FuncProperty]] = None
 
+    custom_scope: str | None = None
+
     sort: List[dict] = None
     limit: Optional[int] = None
     offset: Optional[int] = None

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/datasets/backends/dataframe/commands/read.py

@@ -31,6 +31,7 @@ from spinta.ufuncs.querybuilder.components import Selected
 from spinta.ufuncs.helpers import merge_formulas
 from spinta.ufuncs.resultbuilder.components import ResultBuilder
 from spinta.utils.data import take
+from spinta.utils.nestedstruct import flat_dicts_to_nested, extract_list_property_names
 from spinta.utils.schema import NA
 
 OBJECT_DTYPE = "object"
@@ -268,12 +269,14 @@ def dask_get_all(
     where = env.execute(expr)
     qry = env.build(where)
 
+    env_selected = env.selected
+    list_keys = extract_list_property_names(model, env_selected.keys())
     for i, row in qry.iterrows():
         row = row.to_dict()
         res = {
             "_type": model.model_type(),
         }
-        for key, sel in env.selected.items():
+        for key, sel in env_selected.items():
             val = _get_row_value(context, row, sel, env.params)
             if sel.prop:
                 if isinstance(sel.prop.dtype, PrimaryKey):
@@ -288,6 +291,7 @@ def dask_get_all(
                 elif is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
                     val = encode_composite_string_id(val, model.external.pkeys)
             res[key] = val
+        res = flat_dicts_to_nested(res, list_keys=list_keys)
         res = commands.cast_backend_to_python(context, model, backend, res, extra_properties=extra_properties)
         yield res
 

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/datasets/backends/dataframe/ufuncs/query/components.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
 
+import dataclasses
 from typing import Any
 
-from dask.dataframe import DataFrame
+import pandas as pd
+from dask.dataframe import DataFrame, from_delayed
+from dask import delayed
 
 from spinta.components import Model, Property
 from spinta.core.ufuncs import Env, Expr
 
 from spinta.exceptions import UnknownMethod
 from spinta.ufuncs.propertyresolver.components import PropertyResolver
-from spinta.ufuncs.querybuilder.components import Selected
+from spinta.ufuncs.querybuilder.components import Selected, Func
 from spinta.utils.schema import NA
 from spinta.datasets.backends.dataframe.components import DaskBackend
 
@@ -33,6 +36,7 @@ class DaskDataFrameQueryBuilder(Env):
             offset=None,
             params=params,
             url_query=None,
+            count=False,
         )
 
     def build(self, where):
@@ -48,6 +52,18 @@ class DaskDataFrameQueryBuilder(Env):
 
         if where is not None:
             df = df[where]
+
+        if self.count:
+            # To only return one row, we need to calculate the count first and then transform it back to dataframe
+            # Otherwise iterrows will duplicate the result with the number of rows equal to count.
+            # Dask allows delayed calculations which ar lazy.
+
+            # Create a delayed scalar function that returns the count of the dataframe as a new one-column dataframe
+            count_scalar = delayed(lambda value: pd.DataFrame({RESERVED_COUNT_PROP: [value]}))(
+                df.map_partitions(len).sum()
+            )
+            df = from_delayed([count_scalar], meta=pd.DataFrame({RESERVED_COUNT_PROP: pd.Series(dtype="int64")}))
+
         return df
 
     def execute(self, expr: Any):
@@ -73,3 +89,11 @@ class DaskSelected(Selected):
     prop: Property = None
     # A value or an Expr for further processing on selected value.
     prep: Any = NA
+
+
+RESERVED_COUNT_PROP = "__dask_count"
+
+
+@dataclasses.dataclass
+class Count(Func):
+    pass

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -13,6 +13,8 @@ from spinta.core.ufuncs import Expr, ufunc, Bind, Unresolved, GetAttr
 from spinta.datasets.backends.dataframe.ufuncs.query.components import (
     DaskDataFrameQueryBuilder,
     DaskSelected as Selected,
+    Count,
+    RESERVED_COUNT_PROP,
 )
 from spinta.datasets.components import Param
 from spinta.datasets.utils import iterparams
@@ -22,7 +24,7 @@ from spinta.exceptions import (
     SourceOrPrepareNotAllowed,
     InvalidArgumentInExpression,
 )
-from spinta.types.datatype import DataType, PrimaryKey, Ref
+from spinta.types.datatype import DataType, Integer, Number, Boolean, PrimaryKey, Ref
 from spinta.types.text.components import Text
 from spinta.ufuncs.components import ForeignProperty
 from spinta.utils.data import take
@@ -127,8 +129,8 @@ def _resolve_unresolved(env: DaskDataFrameQueryBuilder, field: Bind) -> str:
 
 
 @ufunc.resolver(DaskDataFrameQueryBuilder)
-def count(env: DaskDataFrameQueryBuilder):
-    return len(env.dataframe.index)
+def count(env: DaskDataFrameQueryBuilder) -> Count:
+    return Count()
 
 
 @ufunc.resolver(DaskDataFrameQueryBuilder, Expr)
@@ -142,30 +144,8 @@ def select(env: DaskDataFrameQueryBuilder, expr: Expr):
 
     env.selected = {}
     if args:
-        resolved = {}
-        selected_keys = set()
-        selected_languages = {}
         for key, arg in args:
-            resolved[key] = env.call("_resolve_property", arg)
-        for key, prop in resolved.items():
-            prop_parent = getattr(prop, "parent", None)
-            if prop_parent and isinstance(prop_parent.dtype, Text):
-                selected_keys.add(prop_parent.place)
-                selected_languages.setdefault(prop_parent.place, set()).add(prop.name)
-            else:
-                selected_keys.add(prop.name)
-        for selected_key in selected_keys:
-            prop = env.model.flatprops.get(selected_key)
-            if isinstance(prop.dtype, Text) and authorized(env.context, prop, Action.GETALL):
-                selected_language = selected_languages.get(prop.place)
-                if not selected_language:
-                    env.selected[prop.place] = env.call("select", prop)
-                else:
-                    env.selected[prop.place] = env.call("select", prop, selected_language)
-            elif authorized(env.context, prop, Action.GETALL):
-                env.selected[resolved[selected_key].place] = env.call("select", resolved[selected_key])
-            else:
-                raise PropertyNotFound(env.model, property=resolved[selected_key])
+            env.selected[key] = env.call("select", arg)
     else:
         for prop in take(["_id", "_revision", all], env.model.properties).values():
             if prop.name == "_revision" and not is_custom_revision_prop(prop):
@@ -502,6 +482,15 @@ def select(
     return {k: env.call("select", v) for k, v in prep.items()}
 
 
+@ufunc.resolver(DaskDataFrameQueryBuilder, Count)
+def select(
+    env: DaskDataFrameQueryBuilder,
+    value: Count,
+) -> Selected:
+    env.count = True
+    return Selected(item=RESERVED_COUNT_PROP)
+
+
 @ufunc.resolver(DaskDataFrameQueryBuilder, Expr)
 def base64(env: DaskDataFrameQueryBuilder, expr: Expr) -> Expr:
     return expr  # Expression will be resolved in ResultBuilder
@@ -533,6 +522,24 @@ def eq_(env: DaskDataFrameQueryBuilder, dtype: DataType, obj: object) -> Series:
     return env.dataframe[name] == str(obj)
 
 
+@ufunc.resolver(DaskDataFrameQueryBuilder, Integer, object, name="eq")
+def eq_(env: DaskDataFrameQueryBuilder, dtype: Integer, obj: object) -> Series:
+    name = dtype.prop.external.name
+    return env.dataframe[name] == obj
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Number, object, name="eq")
+def eq_(env: DaskDataFrameQueryBuilder, dtype: Number, obj: object) -> Series:
+    name = dtype.prop.external.name
+    return env.dataframe[name] == obj
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Boolean, object, name="eq")
+def eq_(env: DaskDataFrameQueryBuilder, dtype: Boolean, obj: object) -> Series:
+    name = dtype.prop.external.name
+    return env.dataframe[name] == obj
+
+
 @ufunc.resolver(DaskDataFrameQueryBuilder, Param, name="eval")
 def eval_(env: DaskDataFrameQueryBuilder, param: Param) -> Iterator[str]:
     resolved_values = (

{spinta-0.2.dev25 → spinta-0.2.dev26}/spinta/datasets/backends/sql/commands/cast.py

@@ -4,14 +4,15 @@ from typing import List, Any
 
 from spinta import commands
 from spinta.backends import Backend
-from spinta.backends.helpers import is_custom_id_prop
+from spinta.backends.helpers import check_if_model_primary_key_is_composite, is_custom_id_prop
 from spinta.components import Context, Model
 from spinta.core.enums import Mode
 from spinta.datasets.backends.helpers import generate_ref_id_using_select, flatten_keymap_encoding_values
 from spinta.datasets.backends.sql.components import Sql
+from spinta.datasets.helpers import encode_composite_string_id
 from spinta.datasets.keymaps.components import KeyMap
 from spinta.exceptions import GivenValueCountMissmatch, KeymapValueNotFound
-from spinta.types.datatype import Ref, ExternalRef, Array
+from spinta.types.datatype import Base32, Ref, ExternalRef, Array
 from spinta.types.namespace import check_if_model_has_backend_and_source
 
 
@@ -26,6 +27,80 @@ def cast_backend_to_python(
     return super_(context, model, backend, data, keymap=keymap, **kwargs)
 
 
+def _cast_custom_ref_id(
+    context: Context,
+    dtype: Ref,
+    backend: Sql,
+    ref_model: Model,
+    ref_id_prop,
+    values: dict,
+    ref_filters_disabled: bool,
+):
+    # A custom `_id` is derived from the referenced model's primary keys. When
+    # implicit ref filters are disabled, the referenced row might be filtered
+    # out at the source (e.g. by an explicit filter on the ref model), so
+    # selecting it could find nothing. Since the refprop values already give us
+    # those primary keys, rebuild the id the same way the referenced model would
+    # (mirrors `build_row_result`) instead of requiring the row to be selectable
+    # -- but only when the refprops are exactly those primary keys.
+    if ref_filters_disabled and dtype.refprops == ref_model.external.pkeys:
+        pk_values = list(values.values())
+        is_composite = check_if_model_primary_key_is_composite(ref_model)
+        if is_composite and not isinstance(ref_id_prop.dtype, Base32):
+            # `build_row_result` joins composite primary keys into a single
+            # string for non Base32 ids.
+            return encode_composite_string_id(pk_values, ref_model.external.pkeys)
+        # Single primary key ids (and Base32 ids, which cbor encode composite
+        # keys) are produced by casting the raw primary key value(s) through the
+        # `_id` data type.
+        data = pk_values if is_composite else pk_values[0]
+        return commands.cast_backend_to_python(context, ref_id_prop.dtype, backend, data)
+
+    return generate_ref_id_using_select(context, dtype, values)
+
+
+def _cast_keymap_ref_id(
+    context: Context,
+    dtype: Ref,
+    ref_model: Model,
+    keymap: KeyMap,
+    keymap_name: str,
+    encoding_values,
+    values: dict,
+    ref_filters_disabled: bool,
+):
+    if keymap.contains(keymap_name, encoding_values):
+        return keymap.encode(keymap_name, encoding_values)
+
+    if encoding_values is None:
+        return None
+
+    if ref_model.mode == Mode.external and not check_if_model_has_backend_and_source(ref_model):
+        raise KeymapValueNotFound(
+            dtype,
+            keymap=keymap.name,
+            model_name=dtype.model.name,
+            values=encoding_values,
+        )
+
+    # When implicit ref filters are disabled, the referenced row might be
+    # filtered out at the source (e.g. by an explicit filter on the ref model),
+    # so selecting it could find nothing. The `_id` of a keymap based model is
+    # deterministic given its primary keys, so encode it directly instead of
+    # requiring the row to be selectable.
+    #
+    # Models with an identifiable `base` are excluded: their `_id` is seeded by
+    # the base's key (see `generate_pk_for_row`), which cannot be reconstructed
+    # from the foreign key alone. Encoding directly would mint a base-unaware id
+    # that diverges from -- and later clashes with -- the value the referenced
+    # model produces, so fall back to selecting.
+    base_seeded = ref_model.base and commands.identifiable(ref_model.base)
+    if ref_filters_disabled and not base_seeded:
+        return keymap.encode(keymap_name, encoding_values)
+
+    return generate_ref_id_using_select(context, dtype, values)
+
+
 @commands.cast_backend_to_python.register(Context, Ref, Sql, dict)
 def cast_backend_to_python(context: Context, dtype: Ref, backend: Sql, data: dict, *, keymap: KeyMap = None, **kwargs):
     processed_data = {}
@@ -71,24 +146,19 @@ def cast_backend_to_python(context: Context, dtype: Ref, backend: Sql, data: dic
     # Backwards compatibility, all nested values are converted to list values without keys
     encoding_values = flatten_keymap_encoding_values(encoding_values)
 
-    if is_custom_id_prop(ref_model.properties["_id"]):
-        id_value = generate_ref_id_using_select(context, dtype, values)
+    # First decide where the referenced model's `_id` comes from: a custom `_id`
+    # is derived from its own primary keys, while a default `_id` is resolved
+    # through the keymap. The two are produced differently, so handle each in its
+    # own helper (each also decides whether the id can be rebuilt locally or
+    # whether the referenced row must be selected).
+    ref_id_prop = ref_model.properties["_id"]
+    ref_filters_disabled = not context.get("config").check_ref_filters
+    if is_custom_id_prop(ref_id_prop):
+        id_value = _cast_custom_ref_id(context, dtype, backend, ref_model, ref_id_prop, values, ref_filters_disabled)
     else:
-        contains = keymap.contains(keymap_name, encoding_values)
-
-        if contains:
-            id_value = keymap.encode(keymap_name, encoding_values)
-        elif encoding_values is None:
-            id_value = None
-        elif ref_model.mode == Mode.external and not check_if_model_has_backend_and_source(ref_model):
-            raise KeymapValueNotFound(
-                dtype,
-                keymap=keymap.name,
-                model_name=dtype.model.name,
-                values=encoding_values,
-            )
-        else:
-            id_value = generate_ref_id_using_select(context, dtype, values)
+        id_value = _cast_keymap_ref_id(
+            context, dtype, ref_model, keymap, keymap_name, encoding_values, values, ref_filters_disabled
+        )
 
     processed_data["_id"] = id_value