spinta 0.2.dev27__py3-none-any.whl → 0.2.dev29__py3-none-any.whl

spinta/api/__init__.py

@@ -50,7 +50,7 @@ from spinta.exceptions import (
     error_response,
 )
 from spinta.formats.html.helpers import get_templates
-from spinta.middlewares import ContextMiddleware, StrictTransportSecurityMiddleware
+from spinta.middlewares import ContextMiddleware, PathNormalizationMiddleware, StrictTransportSecurityMiddleware
 from spinta.urlparams import Version, get_response_type
 
 log = logging.getLogger(__name__)
@@ -370,7 +370,7 @@ async def error(request, exc):
     elif isinstance(exc, BaseError):
         status_code = exc.status_code
         errors = [error_response(exc)]
-        headers = exc.headers
+        headers = dict(exc.headers)
     else:
         if isinstance(exc, HTTPException):
             status_code = exc.status_code
@@ -400,6 +400,11 @@ async def error(request, exc):
 
     response = {"errors": errors}
 
+    # Error responses can reflect request input (e.g. the requested path) and
+    # must never be stored by shared caches, otherwise they can be used for
+    # web cache poisoning.
+    headers["Cache-Control"] = "no-store"
+
     fmt = get_response_type(request.state.context, request)
     if fmt == "json" or fmt is None:
         return JSONResponse(
@@ -414,6 +419,7 @@ async def error(request, exc):
         }
 
         return templates.TemplateResponse(
+            request,
             "error.html",
             response,
             status_code=status_code,
@@ -467,6 +473,7 @@ def init(context: Context):
             StrictTransportSecurityMiddleware,
             value=config.http_strict_transport_security,
         ),
+        Middleware(PathNormalizationMiddleware),
         Middleware(ContextMiddleware, context=context),
     ]
 

spinta/auth.py

@@ -538,7 +538,7 @@ class AdminToken(rfc6749.TokenMixin):
         return True
 
     def check_scope(self, scope: SCOPE_TYPE, **kwargs) -> bool:
-        pass
+        return True
 
     def get_sub(self) -> str:  # User.
         return "admin"
@@ -864,7 +864,6 @@ def authorized(
 
     # Unauthenticated clients can only access nodes if spinta config.access is open.
     unauthenticated = token.get_client_id() == get_default_auth_client_id(context)
-
     if unauthenticated and config.access < Access.open:
         if throw:
             raise AuthorizedClientsOnly()

spinta/backends/__init__.py

@@ -590,7 +590,7 @@ def is_object_id(context: Context, value: str):
 
 @is_object_id.register(Context, Backend, Model, str)
 def is_object_id(context: Context, backend: Backend, model: Model, value: str):
-    return is_object_id(context, backend, model.properties["_id"].dtype, value)
+    return is_object_id(context, backend, model.id_prop.dtype, value)
 
 
 @is_object_id.register(Context, Backend, PrimaryKey, str)
@@ -1940,6 +1940,31 @@ def cast_backend_to_python(context: Context, dtype: Ref, backend: Backend, data:
     return processed_data
 
 
+@commands.cast_backend_to_python.register(Context, ExternalRef, Backend, dict)
+def cast_backend_to_python(
+    context: Context, dtype: ExternalRef, backend: Backend, data: Dict[str, Any], **kwargs
+) -> Any:
+    if not data:
+        return data
+
+    processed_data = {}
+    for key in data:
+        prop = commands.resolve_property(dtype.prop.model, f"{dtype.prop.place}.{key}")
+        if prop is not None:
+            processed_data[key] = commands.cast_backend_to_python(context, prop, backend, data[key], **kwargs)
+
+    for prop in dtype.refprops:
+        if prop.name not in processed_data and prop.name in data:
+            processed_data[prop.name] = commands.cast_backend_to_python(
+                context, prop, backend, data[prop.name], **kwargs
+            )
+
+    if not processed_data or all(value is None for value in processed_data.values()):
+        return None
+
+    return processed_data
+
+
 @commands.cast_backend_to_python.register(Context, Object, Backend, dict)
 def cast_backend_to_python(
     context: Context, dtype: Object, backend: Backend, data: Dict[str, Any], **kwargs

spinta/backends/components.py

@@ -36,6 +36,9 @@ class Backend:
     result_builder_type: str = ""
     result_builder_class: Type[ResultBuilder]
 
+    # Some backends might only work with source given to models and some might not need it
+    model_requires_source: bool = False
+
     def __repr__(self):
         return f"<{self.__class__.__module__}.{self.__class__.__name__}(name={self.name!r}) at 0x{id(self):02x}>"
 

spinta/backends/postgresql/helpers/exceptions.py

@@ -19,9 +19,7 @@ def create_exception(item: DataItem, error: psy_errors.ForeignKeyViolation):
     error_message = error.diag.message_detail
     if item.action is Action.DELETE:
         error_model = extractor.extract_error_model(error_message)
-        return exceptions.ReferringObjectFound(
-            item.model.properties.get("_id"), model=error_model, id=item.saved.get("_id")
-        )
+        return exceptions.ReferringObjectFound(item.model.id_prop, model=error_model, id=item.saved.get("_id"))
     else:
         error_property_names = extractor.extract_error_property_names(error_message)
         error_ref_id = extractor.extract_error_ref_id(error_message)

spinta/backends/postgresql/ufuncs/query/components.py

@@ -57,11 +57,9 @@ class PgQueryBuilder(QueryBuilder):
             self.call("select", Expr("select"))
 
         if not self.aggregate:
-            self.selected["_id"] = Selected(
-                item=self.add_column(self.table.c["_id"]), prop=self.model.properties["_id"]
-            )
+            self.selected["_id"] = Selected(item=self.add_column(self.table.c["_id"]), prop=self.model.id_prop)
             self.selected["_revision"] = Selected(
-                item=self.add_column(self.table.c["_revision"]), prop=self.model.properties["_revision"]
+                item=self.add_column(self.table.c["_revision"]), prop=self.model.revision_prop
             )
         merged_selected = merge_with_page_selected_list(self.columns, self.page)
         merged_sorted = merge_with_page_sort(self.sort, self.page)
@@ -116,7 +114,7 @@ class PgQueryBuilder(QueryBuilder):
             else:
                 rtable = self.backend.get_table(rmodel).alias()
 
-            rpkey = self.backend.get_column(rtable, rmodel.properties["_id"])
+            rpkey = self.backend.get_column(rtable, rmodel.id_prop)
 
             if isinstance(lrkey, list) and not isinstance(rpkey, list):
                 if len(lrkey) == 1:
@@ -199,10 +197,10 @@ class PgQueryBuilder(QueryBuilder):
             raise PropertyNotFound(model, property=prop)
 
         ltable = self.backend.get_table(inherit_model)
-        lrkey = self.backend.get_column(ltable, inherit_model.properties["_id"])
+        lrkey = self.backend.get_column(ltable, inherit_model.id_prop)
 
         rtable = self.backend.get_table(base_model).alias()
-        rpkey = self.backend.get_column(rtable, base_model.properties["_id"])
+        rpkey = self.backend.get_column(rtable, base_model.id_prop)
 
         if base_model.name in self.joins:
             if self.joins[base_model.name] == rtable:

spinta/backends/postgresql/ufuncs/query/ufuncs.py

@@ -287,7 +287,7 @@ def select(env, dtype: Ref):
     if not dtype.inherited:
         name = "_id"
         if env.query_params.prioritize_uri and uri is not None:
-            fpr = ForeignProperty(None, dtype, dtype.model.properties["_id"].dtype)
+            fpr = ForeignProperty(None, dtype, dtype.model.id_prop.dtype)
             table = env.get_joined_table(fpr)
             column = table.c[uri.place]
             name = "_uri"
@@ -311,7 +311,7 @@ def select(env, dtype: ExternalRef):
         if dtype.model.given.pkeys or dtype.explicit:
             props = dtype.refprops
         else:
-            props = [dtype.model.properties["_id"]]
+            props = [dtype.model.id_prop]
         for prop in props:
             name = f"{dtype.prop.place}.{prop.place}"
             column = table.c[name]
@@ -350,7 +350,7 @@ def _select_backref(env, dtype, is_array=False):
     return_columns = {}
 
     if commands.identifiable(refprop):
-        id_ = fpr.right.prop.model.properties["_id"]
+        id_ = fpr.right.prop.model.id_prop
         column_name = id_.name
         label = f"{dtype.prop.name}.{column_name}"
         required_columns.append((column_name, label))
@@ -431,7 +431,7 @@ def _denorm_to_foreign_property(env: PgQueryBuilder, dtype: Denorm):
                 and isinstance(root_ref_parent, Property)
                 and isinstance(root_ref_parent.dtype, (Ref, BackRef))
             ):
-                fpr = ForeignProperty(fpr, root_ref_parent.dtype, root_ref_parent.dtype.model.properties["_id"].dtype)
+                fpr = ForeignProperty(fpr, root_ref_parent.dtype, root_ref_parent.dtype.model.id_prop.dtype)
 
                 if not root_ref_parent.dtype.inherited:
                     break

spinta/cli/helpers/admin/scripts/add_local_ids.py

@@ -25,7 +25,7 @@ def add_explicit_id_properties(context: Context, manifest: str, verbose: bool =
             if verbose:
                 echo(f'Model "{model.model_type()}" has no `ref`, skipping.', err=True)
             continue
-        prop = model.properties.get("_id")
+        prop = model.id_prop
         if prop is None or prop.explicitly_given:
             continue
 

spinta/cli/helpers/admin/scripts/remove_local_ids.py

@@ -19,7 +19,7 @@ def remove_explicit_id_properties(context: Context, manifest: str) -> int:
     for model in commands.get_models(context, manifest).values():
         if model.model_type().startswith(RESERVED_MODEL_PREFIX):
             continue
-        prop = model.properties.get("_id")
+        prop = model.id_prop
         if prop is None or not prop.explicitly_given:
             continue
 

spinta/cli/helpers/push/read.py

@@ -55,9 +55,9 @@ def _iter_model_rows(
         else:
             stream = read_model_data(context, model, limit, stop_on_error, params=params)
             for item in stream:
-                if push_counter:
+                if push_counter is not None:
                     push_counter.update(1)
-                if model_push_counter:
+                if model_push_counter is not None:
                     model_push_counter.update(1)
                 yield PushRow(model, item)
 

spinta/cli/helpers/store.py

@@ -166,4 +166,4 @@ def attach_backends(context: Context, store: Store, manifest: Manifest) -> None:
 
 def attach_keymaps(context: Context, store: Store) -> None:
     for keymap in store.keymaps.values():
-        context.attach(f"keymap.{keymap.name}", lambda: keymap)
+        context.attach(f"keymap.{keymap.name}", lambda: keymap.copy())

spinta/cli/pii.py

@@ -220,7 +220,7 @@ def detect(
                         f"transaction.{resource.backend.name}", validate_and_return_begin, context, resource.backend
                     )
         for keymap in store.keymaps.values():
-            context.attach(f"keymap.{keymap.name}", lambda: keymap)
+            context.attach(f"keymap.{keymap.name}", lambda: keymap.copy())
 
         ns = commands.get_namespace(context, manifest, "")
         models = commands.traverse_ns_models(context, ns, manifest, Action.SEARCH)

spinta/components.py

@@ -662,7 +662,6 @@ class Model(MetaData):
     comments: List[Comment] = None
     base: Base = None
     uri: str = None
-    uri_prop: Property = None
     page: PageInfo = None
     features: str = None
     status: Status | None = None
@@ -675,6 +674,11 @@ class Model(MetaData):
     distribution_strategy: DistributionStrategy | None = None
     required_keymap_properties = None
 
+    # Quick access properties
+    uri_prop: Property | None = None
+    id_prop: Property
+    revision_prop: Property
+
     schema = {
         "keymap": {"type": "string"},
         "backend": {"type": "string"},
@@ -741,7 +745,7 @@ class Model(MetaData):
         # return self.name.split('/')[-1]
 
     def add_keymap_property_combination(self, given_props: List[Property]):
-        extract_names = list([prop.name for prop in given_props])
+        extract_names = tuple([prop.place for prop in given_props])
         if extract_names not in self.required_keymap_properties:
             self.required_keymap_properties.append(extract_names)
 

spinta/datasets/backends/dataframe/backends/csv/components.py

@@ -6,6 +6,9 @@ from spinta.datasets.backends.dataframe.components import DaskBackend
 class Csv(DaskBackend):
     type: str = "dask/csv"
 
+    # Csv data is mainly one table, meaning model.source is usually not given
+    model_requires_source: bool = False
+
     @contextlib.contextmanager
     def begin(self):
         yield

spinta/datasets/backends/dataframe/backends/memory/components.py

@@ -6,6 +6,9 @@ from spinta.datasets.backends.dataframe.components import DaskBackend
 class MemoryDaskBackend(DaskBackend):
     type: str = "dask/memory"
 
+    # Memory backends can set model.source, but it can also leave it open and still work
+    model_requires_source: bool = False
+
     @contextlib.contextmanager
     def begin(self):
         yield

spinta/datasets/backends/dataframe/commands/read.py

@@ -16,16 +16,14 @@ from spinta.core.ufuncs import Env, Expr
 from spinta.datasets.backends.dataframe.backends.memory.components import MemoryDaskBackend
 from spinta.datasets.backends.dataframe.components import DaskBackend
 from spinta.datasets.backends.dataframe.ufuncs.query.components import DaskDataFrameQueryBuilder
-from spinta.datasets.backends.helpers import handle_ref_key_assignment
 from spinta.datasets.components import Resource
 from spinta.datasets.helpers import encode_composite_string_id, get_enum_filters, get_ref_filters
-from spinta.datasets.keymaps.components import KeyMap
 from spinta.datasets.utils import iterparams
 from spinta.dimensions.enum.helpers import get_prop_enum
 from spinta.dimensions.param.components import ResolvedParams
 from spinta.exceptions import NoExternalName, PropertyNotFound, ValueNotInEnum
 from spinta.manifests.components import Manifest
-from spinta.types.datatype import Base32, Boolean, DataType, DateTime, Integer, Number, PrimaryKey, Ref
+from spinta.types.datatype import Base32, Boolean, DataType, DateTime, Integer, Number, Ref
 from spinta.typing import ObjectData
 from spinta.ufuncs.helpers import merge_formulas
 from spinta.ufuncs.querybuilder.components import Selected
@@ -182,7 +180,7 @@ def _select_primary_key(model: Model, df: DataFrame):
         result = _select(prop, df)
     else:
         result = [_select(prop, df) for prop in pkeys]
-    return Selected(prop=model.properties["_id"], prep=result)
+    return Selected(prop=model.id_prop, prep=result)
 
 
 def get_pkeys_if_ref(prop: Property) -> list[str]:
@@ -258,7 +256,6 @@ def dask_get_all(
     params: dict | None = None,
 ):
     params = params or {}
-    keymap: KeyMap = context.get(f"keymap.{model.keymap.name}")
 
     query = merge_formulas(model.external.prepare, query)
     query = merge_formulas(query, get_enum_filters(context, model))
@@ -279,16 +276,7 @@ def dask_get_all(
         for key, sel in env_selected.items():
             val = _get_row_value(context, row, sel, env.params)
             if sel.prop:
-                if isinstance(sel.prop.dtype, PrimaryKey):
-                    if isinstance(val, list):
-                        val = [
-                            list_value.get("_id", list_value) if isinstance(list_value, dict) else list_value
-                            for list_value in val
-                        ]
-                    val = keymap.encode(sel.prop.model.model_type(), val)
-                elif isinstance(sel.prop.dtype, Ref):
-                    val = handle_ref_key_assignment(context, keymap, env, val, sel.prop.dtype)
-                elif is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
+                if is_custom_id_prop(sel.prop) and isinstance(val, list) and not isinstance(sel.prop.dtype, Base32):
                     val = encode_composite_string_id(val, model.external.pkeys)
             res[key] = val
         res = flat_dicts_to_nested(res, list_keys=list_keys)

spinta/datasets/backends/dataframe/ufuncs/query/components.py

@@ -15,6 +15,8 @@ from spinta.ufuncs.propertyresolver.components import PropertyResolver
 from spinta.ufuncs.querybuilder.components import Func, Selected
 from spinta.utils.schema import NA
 
+RESERVED_COUNT_PROP = "__dask_count"
+
 
 class DaskDataFrameQueryBuilder(Env):
     backend: DaskBackend
@@ -90,9 +92,6 @@ class DaskSelected(Selected):
     prep: Any = NA
 
 
-RESERVED_COUNT_PROP = "__dask_count"
-
-
 @dataclasses.dataclass
 class Count(Func):
     pass

spinta/datasets/backends/dataframe/ufuncs/query/helpers.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from spinta.datasets.backends.dataframe.ufuncs.query.components import (
+    DaskDataFrameQueryBuilder,
+)
+from spinta.datasets.backends.dataframe.ufuncs.query.components import (
+    DaskSelected as Selected,
+)
+from spinta.exceptions import (
+    GivenValueCountMissmatch,
+    PropertyNotPartOfRefProps,
+)
+from spinta.types.datatype import ExternalRef, Ref
+
+
+def select_ref_foreign_key_properties(
+    env: DaskDataFrameQueryBuilder, dtype: Ref, *, properties: list = None
+) -> dict[str, Selected]:
+    if properties is None:
+        if dtype.prop.external and dtype.prop.external.prepare:
+            properties = env(this=dtype.prop).resolve(dtype.prop.external.prepare)
+
+    prep = {}
+    given_mapping_count = len(properties) if properties else 1
+    if len(dtype.refprops) != given_mapping_count:
+        raise GivenValueCountMissmatch(dtype, given_count=given_mapping_count, expected_count=len(dtype.refprops))
+
+    if properties is None:
+        refprop = dtype.refprops[0]
+        # Currently, dask can only access its own dataframe columns, so we we have to use dtype.prop.external.name as source.
+        prep[refprop.name] = Selected(item=dtype.prop.external.name, prop=refprop)
+    else:
+        for i, prop in enumerate(dtype.refprops):
+            sel = env.call("select", properties[i])
+            if sel.prop is None:
+                sel.prop = prop
+            prep[prop.name] = sel
+    return prep
+
+
+def select_external_ref_foreign_key_properties(
+    env: DaskDataFrameQueryBuilder, dtype: ExternalRef, *, properties: list = None, target: str = None
+) -> dict[str, Selected]:
+    if properties is None:
+        if dtype.prop.external and dtype.prop.external.prepare:
+            properties = env(this=dtype.prop).resolve(dtype.prop.external.prepare)
+
+    if target is not None:
+        value = next((prop for prop in dtype.refprops if prop.name == target), None)
+        if value is None:
+            raise PropertyNotPartOfRefProps(dtype, prop=target, refprops=list(prop_.name for prop_ in dtype.refprops))
+        target = value
+
+    prep = {}
+    given_mapping_count = len(properties) if properties else 1
+    if len(dtype.refprops) != given_mapping_count:
+        raise GivenValueCountMissmatch(dtype, given_count=given_mapping_count, expected_count=len(dtype.refprops))
+
+    if properties is None:
+        refprop = dtype.refprops[0]
+        # Currently, dask can only access its own dataframe columns, so we we have to use dtype.prop.external.name as source.
+        prep[refprop.name] = Selected(item=dtype.prop.external.name, prop=refprop)
+    else:
+        if target is not None:
+            pos = dtype.refprops.index(target)
+            sel = env.call("select", properties[pos])
+            if sel.prop is None:
+                sel.prop = target
+            prep[target.name] = sel
+        else:
+            for i, prop in enumerate(dtype.refprops):
+                sel = env.call("select", properties[i])
+                if sel.prop is None:
+                    sel.prop = prop
+                prep[prop.name] = sel
+    return prep

spinta/datasets/backends/dataframe/ufuncs/query/ufuncs.py

@@ -5,7 +5,6 @@ from typing import Any, Dict, Iterator, List, Tuple
 
 from dask.dataframe import Series
 
-from spinta.auth import authorized
 from spinta.backends.helpers import is_custom_id_prop, is_custom_revision_prop
 from spinta.components import Property
 from spinta.core.enums import Action
@@ -18,7 +17,13 @@ from spinta.datasets.backends.dataframe.ufuncs.query.components import (
 from spinta.datasets.backends.dataframe.ufuncs.query.components import (
     DaskSelected as Selected,
 )
+from spinta.datasets.backends.dataframe.ufuncs.query.helpers import (
+    select_external_ref_foreign_key_properties,
+    select_ref_foreign_key_properties,
+)
 from spinta.datasets.components import Param
+from spinta.datasets.enums import ExternalIdPattern
+from spinta.datasets.helpers import authorized_or_system_request
 from spinta.datasets.utils import iterparams
 from spinta.exceptions import (
     InvalidArgumentInExpression,
@@ -26,13 +31,23 @@ from spinta.exceptions import (
     SourceCannotBeList,
     SourceOrPrepareNotAllowed,
 )
-from spinta.types.datatype import Boolean, DataType, Integer, Number, PrimaryKey, Ref
+from spinta.types.datatype import Boolean, DataType, ExternalRef, Integer, Number, PrimaryKey, Ref
 from spinta.types.text.components import Text
 from spinta.ufuncs.components import ForeignProperty
+from spinta.ufuncs.querybuilder.helpers import process_literal_value
 from spinta.utils.data import take
 from spinta.utils.schema import NA
 
 
+@ufunc.resolver(DaskDataFrameQueryBuilder, Expr)
+def testlist(env: DaskDataFrameQueryBuilder, expr: Expr) -> tuple:
+    args, kwargs = expr.resolve(env)
+    result = []
+    for arg in args:
+        result.append(process_literal_value(arg))
+    return tuple(result)
+
+
 @ufunc.resolver(DaskDataFrameQueryBuilder, Expr, name="and")
 def and_(env, expr):
     args, kwargs = expr.resolve(env)
@@ -152,7 +167,7 @@ def select(env: DaskDataFrameQueryBuilder, expr: Expr):
         for prop in take(["_id", "_revision", all], env.model.properties).values():
             if prop.name == "_revision" and not is_custom_revision_prop(prop):
                 continue
-            if authorized(env.context, prop, Action.GETALL):
+            if authorized_or_system_request(env.context, prop, Action.GETALL):
                 env.selected[prop.place] = env.call("select", prop)
 
 
@@ -182,7 +197,7 @@ def _get_property_for_select(
         #      then how prepare context should be defined? Probably resolvers
         #      should be called with a different env class?
         #      tag:resolving_private_properties_in_prepare_context
-        nested or authorized(env.context, prop, Action.SEARCH)
+        nested or authorized_or_system_request(env.context, prop, Action.SEARCH)
     ):
         return prop
     else:
@@ -199,7 +214,7 @@ def select(env: DaskDataFrameQueryBuilder, prop: Property) -> Selected:
     if prop.place not in env.resolved:
         if isinstance(prop.external, list):
             raise SourceCannotBeList(prop)
-        if prop.external.name and prop.external.prepare is not NA:
+        if prop.external and prop.external.prepare is not NA:
             # If property doesn't have external name - it describes query parameter
             # If `prepare` formula is given, evaluate formula.
             if isinstance(prop.external.prepare, Expr):
@@ -213,13 +228,6 @@ def select(env: DaskDataFrameQueryBuilder, prop: Property) -> Selected:
             #      properties.
             #      tag:resolving_private_properties_in_prepare_context
             result = env.call("select", prop.dtype, result)
-        elif prop.external.prepare is not NA:
-            # property without external name may be evaluated already, if it is use the value
-            if isinstance(prop.external.prepare, Expr):
-                result = env(this=prop).resolve(prop.external.prepare)
-                result = env.call("select", prop.dtype, result)
-            else:
-                result = Selected(prop=prop, prep=prop.external.prepare)
         elif prop.external and prop.external.name:
             # If prepare is not given, then take value from `source`.
             result = env.call("select", prop.dtype)
@@ -306,18 +314,22 @@ def select(
     dtype: PrimaryKey,
 ) -> Selected:
     model = dtype.prop.model
-    pkeys = model.external.pkeys
-
+    pkeys = (model.base and model.base.pk) or model.external.pkeys
     if not pkeys:
         # If primary key is not specified use all properties to uniquely
         # identify row.
         pkeys = take(model.properties).values()
 
-    if len(pkeys) == 1:
-        prop = pkeys[0]
-        result = env.call("select", prop)
-    else:
-        result = [env.call("select", prop) for prop in pkeys]
+    result = {
+        ExternalIdPattern.ID_KEY.value: {prop.name: env.call("select", prop) for prop in pkeys},
+        ExternalIdPattern.COMBINATIONS_KEY.value: {},
+    }
+    for required_pk_combination in model.required_keymap_properties:
+        combination_result = {}
+        for prop_name in required_pk_combination:
+            prop = env.call("_resolve_property", prop_name)
+            combination_result[prop.name] = env.call("select", prop)
+        result[ExternalIdPattern.COMBINATIONS_KEY.value][required_pk_combination] = combination_result
     return Selected(prop=dtype.prop, prep=result)
 
 
@@ -331,7 +343,10 @@ def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: GetAttr) -> Selecte
     resolved_prep = env.call("select", prep)
 
     result = {}
-    result["_id"] = Selected(prop=dtype.prop, prep=resolved_prep)
+    if not dtype.inherited:
+        refprop = dtype.refprops[0]
+        result[ExternalIdPattern.ID_KEY.value] = Selected(prop=dtype.prop, prep={refprop.name: resolved_prep})
+
     for prop in dtype.properties.values():
         sel = env.call("select", prop)
         result[prop.name] = sel
@@ -341,7 +356,7 @@ def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: GetAttr) -> Selecte
 
 @ufunc.resolver(DaskDataFrameQueryBuilder, Ref, object)
 def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: Any) -> Selected:
-    fpr = ForeignProperty(None, dtype, dtype.model.properties["_id"].dtype)
+    fpr = ForeignProperty(None, dtype, dtype.model.id_prop.dtype)
     return Selected(
         prop=dtype.prop,
         prep=env.call("select", fpr, fpr.right.prop),
@@ -351,7 +366,51 @@ def select(env: DaskDataFrameQueryBuilder, dtype: Ref, prep: Any) -> Selected:
 @ufunc.resolver(DaskDataFrameQueryBuilder, Ref)
 def select(env: DaskDataFrameQueryBuilder, dtype: Ref) -> Selected:
     prep = {}
-    prep["_id"] = Selected(item=dtype.prop.external.name, prop=dtype.prop)
+    if not dtype.inherited:
+        prep[ExternalIdPattern.ID_KEY.value] = Selected(
+            prop=dtype.prop, prep=select_ref_foreign_key_properties(env, dtype)
+        )
+
+    for prop in dtype.properties.values():
+        sel = env.call("select", prop)
+        prep[prop.name] = sel
+
+    return Selected(prop=dtype.prop, prep=prep)
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, Ref, (list, tuple))
+def select(env: DaskDataFrameQueryBuilder, dtype: Ref, data: list | tuple) -> Selected:
+    prep = {}
+    if not dtype.inherited:
+        prep[ExternalIdPattern.ID_KEY.value] = Selected(
+            prop=dtype.prop, prep=select_ref_foreign_key_properties(env, dtype, properties=data)
+        )
+
+    for prop in dtype.properties.values():
+        sel = env.call("select", prop)
+        prep[prop.name] = sel
+
+    return Selected(prop=dtype.prop, prep=prep)
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, ExternalRef)
+def select(env: DaskDataFrameQueryBuilder, dtype: ExternalRef) -> Selected:
+    prep = {}
+    if not dtype.inherited:
+        prep.update(select_external_ref_foreign_key_properties(env, dtype))
+
+    for prop in dtype.properties.values():
+        sel = env.call("select", prop)
+        prep[prop.name] = sel
+
+    return Selected(prop=dtype.prop, prep=prep)
+
+
+@ufunc.resolver(DaskDataFrameQueryBuilder, ExternalRef, (list, tuple))
+def select(env: DaskDataFrameQueryBuilder, dtype: ExternalRef, data: list | tuple) -> Selected:
+    prep = {}
+    if not dtype.inherited:
+        prep.update(select_external_ref_foreign_key_properties(env, dtype, properties=data))
 
     for prop in dtype.properties.values():
         sel = env.call("select", prop)
@@ -434,7 +493,7 @@ def select(
 def select(env: DaskDataFrameQueryBuilder, fpr: ForeignProperty, item: Bind):
     model = fpr.right.prop.model
     prop = env.call("_resolve_property", item)
-    if authorized(env.context, prop, Action.SEARCH):
+    if authorized_or_system_request(env.context, prop, Action.SEARCH):
         return env.call("select", fpr, prop)
     else:
         raise PropertyNotFound(model, property=item.name)