chalkruntime 3.43.87__tar.gz → 3.43.89__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/PKG-INFO +1 -1
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph.py +26 -21
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_impl.py +1 -2
- chalkruntime-3.43.89/chalkruntime/graph/lazy_graph.py +474 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/protograph_deserializer.py +119 -51
- chalkruntime-3.43.89/chalkruntime/has_many_join_keys.py +98 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/vectorized_hasmany_sampler.py +31 -34
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/PKG-INFO +1 -1
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/SOURCES.txt +3 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/pyproject.toml +2 -2
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/README.md +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/constants.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/dataframe.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/lazyframe.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/failed_argument.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/resolver_errors.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/wrapped_resolver_exception.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/_sql_source_pickle_patch.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/chalk_overload.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/convert_chalkpy_underscore.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/dataframe_resolver_correlation.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/feature.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/feature_alias.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/filter_conversion.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/global_graph.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_proxy.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_state.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/jinja_parser.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/mat_agg_filter_expression.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/materializations.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/materialized_feature_view.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/maybe_named_collection.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/named_query.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/nearest_neighbor.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/overlay_graph.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/protograph_serializer.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/pseudofeatures.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/resolver.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/singletons.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/sklearn_model_parser.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/source_secrets.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/stream_resolver.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore_codec_info.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore_operation_registry.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/variables.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/heaptrack_launcher.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/group_incrementalizer.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/incrementalizer.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/batch_result_collector.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/bound_invoker.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/bound_invoker_cache.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/general_bound_invoker.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/no_arg_scalar_invoker.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/one_to_one_invoker.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/overlay_features.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/parse_external_resolver.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/partition_batch.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/query_execution_parameters.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_args_builder.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_input.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_input_upload.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_output_metadata.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_raw_output_parsing.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_result.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_runner.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/sample.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/validator.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/converter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/importer.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/memray_launcher.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/metadata.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/py.typed +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/branch_model_endpoint_config.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/config.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/entrypoint.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/env_helper.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/remote_python_function_registry_client.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/server_context_snapshot.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/service.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/subprocess_hook_service.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/composed_rewriter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/contextual_query_rewriter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/filter_injection_site.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/filter_query_rewriter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/identity_rewriter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/query_rewriter.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/query_rewriter_helper.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/sqlglot_import_lock.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/sqlglot_parsing.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sqlalchemy_mysql_keepalive.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/converter_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/exc.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/message_parsing.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/resolver_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/streaming_invoker_kernel/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/streaming_invoker_kernel/local_streaming_invoker_kernel.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/types.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/window_keys.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/__init__.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/async_helpers.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/contextvars.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/datadog.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/internal_pl_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/posix_file_lock.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/proto_bytes_cache.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/pyarrow_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/tracing.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/viztracer_profiling.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/valgrind_launcher.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/dependency_links.txt +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/requires.txt +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/top_level.txt +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/setup.cfg +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/setup.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_branch_model_endpoint_config.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_captured_global_function_refs.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_datadog.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_feature_aliases.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_graph_cache.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_internal_pl_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_mat_agg_filter_expressions.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_mat_agg_projection_expressions.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_materialized_feature_view.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_posix_file_lock.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_pyarrow_utils.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_server_context_snapshot.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sql_max_row_version_lookback_deserialization.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sql_retry_policy_deserialization.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sqlalchemy_mysql_keepalive.py +0 -0
- {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_subprocess_hook_service.py +0 -0
|
@@ -319,28 +319,33 @@ class ResolvedGraph(Protocol):
|
|
|
319
319
|
return self.resolver_id_to_resolver[resolver_id]
|
|
320
320
|
|
|
321
321
|
@cached_property
|
|
322
|
-
def composite_pkey_resolvers(self) ->
|
|
323
|
-
from
|
|
324
|
-
|
|
322
|
+
def composite_pkey_resolvers(self) -> Mapping[ScalarFeatureType, list[ResolverParsed]]:
|
|
323
|
+
"""The resolvers computing each primary key from other features of its namespace, in graph
|
|
324
|
+
order. Indexing a primary key no resolver computes gives an empty list."""
|
|
325
325
|
output = defaultdict(list)
|
|
326
326
|
for resolver in self.resolvers:
|
|
327
|
-
|
|
328
|
-
if
|
|
329
|
-
continue
|
|
330
|
-
if resolver.output_is_df or resolver.input_is_df:
|
|
331
|
-
continue
|
|
332
|
-
if isinstance(resolver, OfflineResolverParsed):
|
|
333
|
-
# only online resolvers may compute pkeys
|
|
334
|
-
continue
|
|
335
|
-
pkey_feature = self.primary_feature_for_namespace(get_unique_item(namespaces))
|
|
336
|
-
if pkey_feature is None:
|
|
337
|
-
continue
|
|
338
|
-
if (
|
|
339
|
-
len(resolver.inputs) >= 1
|
|
340
|
-
and pkey_feature in resolver.output_features
|
|
341
|
-
and pkey_feature not in resolver.inputs
|
|
342
|
-
):
|
|
343
|
-
if pkey_feature not in output:
|
|
344
|
-
output[pkey_feature] = []
|
|
327
|
+
pkey_feature = composite_pkey_computed_by(resolver, self)
|
|
328
|
+
if pkey_feature is not None:
|
|
345
329
|
output[pkey_feature].append(resolver)
|
|
346
330
|
return output
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def composite_pkey_computed_by(resolver: ResolverParsed, graph: ResolvedGraph) -> ScalarFeatureType | None:
|
|
334
|
+
"""The primary key ``resolver`` computes from other features of its namespace, which makes it one of
|
|
335
|
+
``composite_pkey_resolvers[pkey]``; ``None`` when it computes none."""
|
|
336
|
+
from chalkruntime.graph.resolver import OfflineResolverParsed
|
|
337
|
+
|
|
338
|
+
namespaces = resolver.input_root_namespaces
|
|
339
|
+
if len(namespaces) != 1:
|
|
340
|
+
return None
|
|
341
|
+
if resolver.output_is_df or resolver.input_is_df:
|
|
342
|
+
return None
|
|
343
|
+
if isinstance(resolver, OfflineResolverParsed):
|
|
344
|
+
# only online resolvers may compute pkeys
|
|
345
|
+
return None
|
|
346
|
+
pkey_feature = graph.primary_feature_for_namespace(get_unique_item(namespaces))
|
|
347
|
+
if pkey_feature is None:
|
|
348
|
+
return None
|
|
349
|
+
if len(resolver.inputs) >= 1 and pkey_feature in resolver.output_features and pkey_feature not in resolver.inputs:
|
|
350
|
+
return pkey_feature
|
|
351
|
+
return None
|
|
@@ -19,7 +19,6 @@ from typing import (
|
|
|
19
19
|
Sequence,
|
|
20
20
|
TypeVar,
|
|
21
21
|
cast,
|
|
22
|
-
final,
|
|
23
22
|
overload,
|
|
24
23
|
)
|
|
25
24
|
|
|
@@ -721,7 +720,7 @@ def _split_top_level_commas(s: str) -> list[str]:
|
|
|
721
720
|
return [stripped for p in parts if (stripped := p.strip())]
|
|
722
721
|
|
|
723
722
|
|
|
724
|
-
|
|
723
|
+
# Not final: LazyResolvedGraph (lazy_graph.py) subclasses it to convert resolvers on first use.
|
|
725
724
|
class ResolvedGraphImpl(ResolvedGraph):
|
|
726
725
|
def __init__(
|
|
727
726
|
self,
|
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
"""A ``ResolvedGraph`` that converts stream resolvers and expression resolvers on first use.
|
|
2
|
+
|
|
3
|
+
``FromProtoConverter.convert_graph`` converts every resolver at boot. On graphs with many stream
|
|
4
|
+
resolvers and expression features nearly all of that time goes to converting their underscore
|
|
5
|
+
expressions, which a pod reads for only the few resolvers its queries plan with.
|
|
6
|
+
``FromProtoConverter.convert_graph_lazily`` builds this graph instead: features and declared
|
|
7
|
+
resolvers convert eagerly as before, while each stream resolver converts from its proto, and each
|
|
8
|
+
feature's expression resolvers from the feature, the first time someone asks for them. Every object
|
|
9
|
+
is the one ``convert_graph`` would build; only the time it is built changes.
|
|
10
|
+
|
|
11
|
+
Lookups by name (``resolver_by_name``, ``get_stream_resolver``, ``resolver_id_to_resolver[id]``) and
|
|
12
|
+
``composite_pkey_resolvers[pkey]`` convert just what they return. ``lazy_resolver_index`` hands the
|
|
13
|
+
planner what it needs to find a feature's candidate resolvers without walking the graph. Anything that
|
|
14
|
+
reads every resolver (``resolvers``, ``stream_resolvers``, iterating ``resolver_id_to_resolver``)
|
|
15
|
+
converts all of them once, upserts them in ``convert_graph``'s order, logs its caller, and from then on
|
|
16
|
+
the graph behaves exactly as an eager one.
|
|
17
|
+
|
|
18
|
+
A conversion error that ``convert_graph`` would raise at boot is raised when the item is first used.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import dataclasses
|
|
24
|
+
import logging
|
|
25
|
+
import threading
|
|
26
|
+
import traceback
|
|
27
|
+
from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
|
|
28
|
+
from functools import cached_property
|
|
29
|
+
from typing import TYPE_CHECKING, Any, TypeVar, overload
|
|
30
|
+
|
|
31
|
+
from chalk._gen.chalk.graph.v1 import graph_pb2 as pb
|
|
32
|
+
from chalkruntime.graph.graph import ResolvedGraph, composite_pkey_computed_by
|
|
33
|
+
from chalkruntime.graph.graph_impl import ResolvedGraphImpl
|
|
34
|
+
from chalkruntime.metadata import compute_resolver_id
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from chalk._gen.chalk.symbolic_value.v1.symbolic_value_pb2 import SymbolicValue
|
|
38
|
+
from chalk.sql._internal.sql_source import BaseSQLSource
|
|
39
|
+
from chalk.sql._internal.sql_source_group import SQLSourceGroup
|
|
40
|
+
from chalk.streams import StreamSource
|
|
41
|
+
|
|
42
|
+
from chalkruntime.graph.feature import ScalarFeatureType
|
|
43
|
+
from chalkruntime.graph.materialized_feature_view import MaterializedFeatureViewParsed
|
|
44
|
+
from chalkruntime.graph.protograph_deserializer import OnlineStoreConfigParsed
|
|
45
|
+
from chalkruntime.graph.resolver import ResolverParsed
|
|
46
|
+
from chalkruntime.graph.stream_resolver import StreamResolverParsed
|
|
47
|
+
|
|
48
|
+
_logger = logging.getLogger(__name__)
|
|
49
|
+
|
|
50
|
+
# The suffix ``underscore_to_online_resolver`` gives a materialized aggregation's unmaterialized resolver.
|
|
51
|
+
_UNMATERIALIZED_TAG = "unit_testing"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclasses.dataclass(frozen=True)
|
|
55
|
+
class LazyResolverIndex:
|
|
56
|
+
"""What a planner needs to find candidate resolvers on a graph whose stream and expression
|
|
57
|
+
resolvers are not all converted yet.
|
|
58
|
+
|
|
59
|
+
``held_resolvers`` are the converted resolvers in ``resolvers`` order: the declared resolvers, then
|
|
60
|
+
the stream resolvers once ``streams_held``. A stream resolver is never a planning candidate
|
|
61
|
+
(``is_valid_candidate_resolver``) unless a query prefers it, and an expression resolver outputs
|
|
62
|
+
only its own feature, so a feature's candidates are the held resolvers' outputs plus
|
|
63
|
+
``expression_resolvers_for(feature)``, which come after them in ``resolvers`` order.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
held_resolvers: tuple[ResolverParsed, ...]
|
|
67
|
+
streams_held: bool
|
|
68
|
+
expression_resolvers_for: Callable[[ScalarFeatureType], tuple[ResolverParsed, ...]]
|
|
69
|
+
# A scalar feature's position among the graph's scalar features, which orders expression
|
|
70
|
+
# resolvers of different features as ``resolvers`` does.
|
|
71
|
+
feature_position: Callable[[ScalarFeatureType], int]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
_T = TypeVar("_T")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _LazyCompositePkeyResolvers(Mapping["ScalarFeatureType", "list[ResolverParsed]"]):
|
|
78
|
+
"""``composite_pkey_resolvers`` of a lazy graph, computed per primary key.
|
|
79
|
+
|
|
80
|
+
A primary key's composite resolvers are among the held resolvers and the key's own expression
|
|
81
|
+
resolvers: a stream resolver has no feature inputs, and any other expression resolver outputs a
|
|
82
|
+
different feature. It behaves as the eager ``defaultdict``: indexing a key no resolver computes
|
|
83
|
+
gives an empty list and from then on holds the key.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
def __init__(self, graph: LazyResolvedGraph):
|
|
87
|
+
super().__init__()
|
|
88
|
+
self._graph = graph
|
|
89
|
+
self._computed: dict[ScalarFeatureType, list[ResolverParsed]] = {}
|
|
90
|
+
# Keys indexed so far, as the eager defaultdict holds them.
|
|
91
|
+
self._indexed: dict[ScalarFeatureType, None] = {}
|
|
92
|
+
|
|
93
|
+
def _compute(self, pkey: ScalarFeatureType) -> list[ResolverParsed]:
|
|
94
|
+
computed = self._computed.get(pkey)
|
|
95
|
+
if computed is None:
|
|
96
|
+
index = self._graph.lazy_resolver_index()
|
|
97
|
+
resolvers: Collection[ResolverParsed] = (
|
|
98
|
+
self._graph.resolvers
|
|
99
|
+
if index is None
|
|
100
|
+
else (*index.held_resolvers, *index.expression_resolvers_for(pkey))
|
|
101
|
+
)
|
|
102
|
+
computed = [r for r in resolvers if composite_pkey_computed_by(r, self._graph) == pkey]
|
|
103
|
+
self._computed[pkey] = computed
|
|
104
|
+
return computed
|
|
105
|
+
|
|
106
|
+
def _holds(self, pkey: ScalarFeatureType) -> bool:
|
|
107
|
+
return pkey in self._indexed or len(self._compute(pkey)) > 0
|
|
108
|
+
|
|
109
|
+
def __getitem__(self, pkey: ScalarFeatureType) -> list[ResolverParsed]:
|
|
110
|
+
self._indexed[pkey] = None
|
|
111
|
+
return self._compute(pkey)
|
|
112
|
+
|
|
113
|
+
# As the eager defaultdict's ``get``, a lookup does not hold the key, which ``Mapping.get``
|
|
114
|
+
# (``__getitem__`` with a fallback) would.
|
|
115
|
+
@overload
|
|
116
|
+
def get(self, pkey: object, /) -> list[ResolverParsed] | None: ...
|
|
117
|
+
@overload
|
|
118
|
+
def get(self, pkey: object, /, default: _T) -> list[ResolverParsed] | _T: ...
|
|
119
|
+
def get(self, pkey: object, /, default: object = None) -> object:
|
|
120
|
+
from chalkruntime.graph.feature import ScalarFeatureType
|
|
121
|
+
|
|
122
|
+
if isinstance(pkey, ScalarFeatureType) and self._holds(pkey):
|
|
123
|
+
return self._compute(pkey)
|
|
124
|
+
return default
|
|
125
|
+
|
|
126
|
+
def __contains__(self, pkey: object) -> bool:
|
|
127
|
+
from chalkruntime.graph.feature import ScalarFeatureType
|
|
128
|
+
|
|
129
|
+
return isinstance(pkey, ScalarFeatureType) and self._holds(pkey)
|
|
130
|
+
|
|
131
|
+
def _all(self) -> dict[ScalarFeatureType, list[ResolverParsed]]:
|
|
132
|
+
output: dict[ScalarFeatureType, list[ResolverParsed]] = {}
|
|
133
|
+
for resolver in self._graph.resolvers:
|
|
134
|
+
pkey = composite_pkey_computed_by(resolver, self._graph)
|
|
135
|
+
if pkey is not None:
|
|
136
|
+
output.setdefault(pkey, []).append(resolver)
|
|
137
|
+
for pkey in self._indexed:
|
|
138
|
+
output.setdefault(pkey, [])
|
|
139
|
+
return output
|
|
140
|
+
|
|
141
|
+
def __iter__(self) -> Iterator[ScalarFeatureType]:
|
|
142
|
+
return iter(self._all())
|
|
143
|
+
|
|
144
|
+
def __len__(self) -> int:
|
|
145
|
+
return len(self._all())
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class _LazyResolverIdToResolver(Mapping[int, "ResolverParsed"]):
|
|
149
|
+
"""``resolver_id_to_resolver`` of a lazy graph: a lookup by id converts just that resolver, while
|
|
150
|
+
iterating converts every resolver."""
|
|
151
|
+
|
|
152
|
+
def __init__(self, graph: LazyResolvedGraph):
|
|
153
|
+
super().__init__()
|
|
154
|
+
self._graph = graph
|
|
155
|
+
|
|
156
|
+
def __getitem__(self, resolver_id: int) -> ResolverParsed:
|
|
157
|
+
resolver = self._graph.resolver_by_id_lazily(resolver_id)
|
|
158
|
+
if resolver is None:
|
|
159
|
+
raise KeyError(resolver_id)
|
|
160
|
+
return resolver
|
|
161
|
+
|
|
162
|
+
def __iter__(self) -> Iterator[int]:
|
|
163
|
+
return iter(self._graph.materialized_resolver_id_to_resolver("resolver_id_to_resolver iteration"))
|
|
164
|
+
|
|
165
|
+
def __len__(self) -> int:
|
|
166
|
+
return len(self._graph.materialized_resolver_id_to_resolver("resolver_id_to_resolver length"))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class LazyResolvedGraph(ResolvedGraphImpl):
|
|
170
|
+
"""See the module docstring. Built by ``FromProtoConverter.convert_graph_lazily``."""
|
|
171
|
+
|
|
172
|
+
def __init__(
|
|
173
|
+
self,
|
|
174
|
+
*,
|
|
175
|
+
sql_sources: Sequence[BaseSQLSource],
|
|
176
|
+
sql_source_groups: Sequence[SQLSourceGroup],
|
|
177
|
+
online_store_configs: Sequence[OnlineStoreConfigParsed],
|
|
178
|
+
symbolic_value_explicits: Sequence[SymbolicValue],
|
|
179
|
+
materialized_feature_views: Sequence[MaterializedFeatureViewParsed],
|
|
180
|
+
stream_resolver_protos: Sequence[pb.StreamResolver],
|
|
181
|
+
stream_sources: dict[tuple[str, str | None], StreamSource],
|
|
182
|
+
captured_global_values: Mapping[str, pb.FunctionReferenceCapturedGlobal],
|
|
183
|
+
):
|
|
184
|
+
super().__init__(
|
|
185
|
+
sql_sources=sql_sources,
|
|
186
|
+
sql_source_groups=sql_source_groups,
|
|
187
|
+
online_store_configs=online_store_configs,
|
|
188
|
+
symbolic_value_explicits=symbolic_value_explicits,
|
|
189
|
+
materialized_feature_views=materialized_feature_views,
|
|
190
|
+
)
|
|
191
|
+
# Materialization may convert while it holds the lock, and conversion reads the graph.
|
|
192
|
+
self._lazy_lock = threading.RLock()
|
|
193
|
+
self._stream_protos: dict[str, pb.StreamResolver] = {
|
|
194
|
+
r.fqn.rsplit(".", maxsplit=1)[-1]: r for r in stream_resolver_protos
|
|
195
|
+
}
|
|
196
|
+
self._stream_sources = stream_sources
|
|
197
|
+
self._captured_global_values = captured_global_values
|
|
198
|
+
self._converted_streams: dict[str, StreamResolverParsed] = {}
|
|
199
|
+
self._streams_held = False
|
|
200
|
+
# Filled by ``index_expression_features`` once the features are converted.
|
|
201
|
+
self._scalar_positions: dict[str, int] = {}
|
|
202
|
+
self._expression_features: list[ScalarFeatureType] = []
|
|
203
|
+
self._expression_feature_by_resolver_name: dict[str, ScalarFeatureType] = {}
|
|
204
|
+
self._lazy_short_name_by_id: dict[int, str] | None = None
|
|
205
|
+
self._converted_expressions: dict[str, tuple[ResolverParsed, ...]] = {}
|
|
206
|
+
self._fully_materialized = False
|
|
207
|
+
self._composite_pkey_resolvers = _LazyCompositePkeyResolvers(self)
|
|
208
|
+
self._resolver_id_to_resolver = _LazyResolverIdToResolver(self)
|
|
209
|
+
self._forcing_callers_logged: set[str] = set()
|
|
210
|
+
|
|
211
|
+
# An unpickled graph is exactly as lazy as the original: every resolver converted so far travels as
|
|
212
|
+
# it is, and the held protos travel serialized, since pickle cannot import the generated message
|
|
213
|
+
# classes by their registered module name. The lock is recreated.
|
|
214
|
+
def __getstate__(self) -> dict[str, object]:
|
|
215
|
+
state = dict(self.__dict__)
|
|
216
|
+
del state["_lazy_lock"]
|
|
217
|
+
state["_stream_protos"] = {short: r.SerializeToString() for short, r in self._stream_protos.items()}
|
|
218
|
+
state["_captured_global_values"] = {
|
|
219
|
+
ref: value.SerializeToString() for ref, value in self._captured_global_values.items()
|
|
220
|
+
}
|
|
221
|
+
return state
|
|
222
|
+
|
|
223
|
+
def __setstate__(self, state: dict[str, Any]) -> None:
|
|
224
|
+
self.__dict__.update(state)
|
|
225
|
+
self._stream_protos = {short: pb.StreamResolver.FromString(r) for short, r in state["_stream_protos"].items()}
|
|
226
|
+
self._captured_global_values = {
|
|
227
|
+
ref: pb.FunctionReferenceCapturedGlobal.FromString(value)
|
|
228
|
+
for ref, value in state["_captured_global_values"].items()
|
|
229
|
+
}
|
|
230
|
+
self._lazy_lock = threading.RLock()
|
|
231
|
+
|
|
232
|
+
def index_expression_features(self) -> None:
|
|
233
|
+
"""Record the scalar features whose expressions ``scalar_feature_types_to_resolvers`` turns into
|
|
234
|
+
resolvers, and the names those resolvers can take, without converting them. Called once the
|
|
235
|
+
features are converted."""
|
|
236
|
+
from chalkruntime.graph.convert_chalkpy_underscore import get_underscore_fqn_from_feature_fqn
|
|
237
|
+
from chalkruntime.graph.feature import ScalarFeatureType
|
|
238
|
+
|
|
239
|
+
scalars = [f for f in self.features if isinstance(f, ScalarFeatureType)]
|
|
240
|
+
self._scalar_positions = {f.fqn: i for i, f in enumerate(scalars)}
|
|
241
|
+
self._expression_features = [
|
|
242
|
+
f for f in scalars if f.underscore_expression is not None or f.offline_underscore_expression is not None
|
|
243
|
+
]
|
|
244
|
+
names: dict[str, ScalarFeatureType] = {}
|
|
245
|
+
for f in self._expression_features:
|
|
246
|
+
for fqn in (
|
|
247
|
+
get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online"),
|
|
248
|
+
get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online", tag=_UNMATERIALIZED_TAG),
|
|
249
|
+
get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="offline"),
|
|
250
|
+
):
|
|
251
|
+
names[fqn.rsplit(".", maxsplit=1)[-1]] = f
|
|
252
|
+
self._expression_feature_by_resolver_name = names
|
|
253
|
+
self._check_unique_names()
|
|
254
|
+
|
|
255
|
+
def _check_unique_names(self) -> None:
|
|
256
|
+
"""Raise on the short-name and id collisions ``upsert_resolvers`` raises on at boot for an eager
|
|
257
|
+
graph, for every name a lazy resolver is sure to take."""
|
|
258
|
+
from chalkruntime.graph.convert_chalkpy_underscore import get_underscore_fqn_from_feature_fqn
|
|
259
|
+
|
|
260
|
+
seen: dict[str, str] = {r.short_name: r.fqn for r in super().resolvers}
|
|
261
|
+
sure_names = [r.fqn for r in self._stream_protos.values()]
|
|
262
|
+
for f in self._expression_features:
|
|
263
|
+
if f.underscore_expression is not None:
|
|
264
|
+
sure_names.append(get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online"))
|
|
265
|
+
if f.offline_underscore_expression is not None or f.underscore_expression is not None:
|
|
266
|
+
sure_names.append(get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="offline"))
|
|
267
|
+
ids: dict[int, str] = {compute_resolver_id(short): fqn for short, fqn in seen.items()}
|
|
268
|
+
for fqn in sure_names:
|
|
269
|
+
short = fqn.rsplit(".", maxsplit=1)[-1]
|
|
270
|
+
if short in seen:
|
|
271
|
+
raise ValueError(
|
|
272
|
+
f"Multiple resolvers with the same short name when parsing the graph: '{seen[short]}' and "
|
|
273
|
+
+ f"'{fqn}' both have the same short name '{short}'"
|
|
274
|
+
)
|
|
275
|
+
seen[short] = fqn
|
|
276
|
+
resolver_id = compute_resolver_id(short)
|
|
277
|
+
if resolver_id in ids:
|
|
278
|
+
raise ValueError(
|
|
279
|
+
f"Multiple resolvers with the same ID when parsing the graph: '{ids[resolver_id]}' and "
|
|
280
|
+
+ f"'{fqn}' both have the same id {resolver_id}"
|
|
281
|
+
)
|
|
282
|
+
ids[resolver_id] = fqn
|
|
283
|
+
|
|
284
|
+
# --- per-item conversion -------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
def _stream_resolver(self, short_name: str) -> StreamResolverParsed:
|
|
287
|
+
from chalkruntime.graph.protograph_deserializer import FromProtoConverter
|
|
288
|
+
|
|
289
|
+
with self._lazy_lock:
|
|
290
|
+
converted = self._converted_streams.get(short_name)
|
|
291
|
+
if converted is None:
|
|
292
|
+
converted = FromProtoConverter.convert_stream_resolver(
|
|
293
|
+
self._stream_protos[short_name],
|
|
294
|
+
graph=self,
|
|
295
|
+
stream_sources=self._stream_sources,
|
|
296
|
+
captured_global_values=self._captured_global_values,
|
|
297
|
+
)
|
|
298
|
+
self._converted_streams[short_name] = converted
|
|
299
|
+
return converted
|
|
300
|
+
|
|
301
|
+
def _expression_resolvers(self, feature: ScalarFeatureType) -> tuple[ResolverParsed, ...]:
|
|
302
|
+
"""The resolvers ``scalar_feature_types_to_resolvers`` builds from ``feature``'s expressions, in
|
|
303
|
+
its order; empty for a feature without expressions."""
|
|
304
|
+
from chalkruntime.graph.convert_chalkpy_underscore import scalar_feature_types_to_resolvers
|
|
305
|
+
|
|
306
|
+
if feature.underscore_expression is None and feature.offline_underscore_expression is None:
|
|
307
|
+
return ()
|
|
308
|
+
with self._lazy_lock:
|
|
309
|
+
converted = self._converted_expressions.get(feature.fqn)
|
|
310
|
+
if converted is None:
|
|
311
|
+
converted = tuple(
|
|
312
|
+
scalar_feature_types_to_resolvers(scalars=[feature], graph=self, include_definition_strings=False)
|
|
313
|
+
)
|
|
314
|
+
# The planner's lazy candidate search relies on an expression resolver outputting only
|
|
315
|
+
# its feature (``LazyResolverIndex``).
|
|
316
|
+
for resolver in converted:
|
|
317
|
+
if tuple(resolver.output_features) != (feature,):
|
|
318
|
+
raise ValueError(
|
|
319
|
+
f"Expression resolver '{resolver.fqn}' outputs {list(resolver.output_features)}, not "
|
|
320
|
+
+ f"just its feature '{feature.fqn}'"
|
|
321
|
+
)
|
|
322
|
+
self._converted_expressions[feature.fqn] = converted
|
|
323
|
+
return converted
|
|
324
|
+
|
|
325
|
+
def _feature_position(self, feature: ScalarFeatureType) -> int:
|
|
326
|
+
return self._scalar_positions[feature.fqn]
|
|
327
|
+
|
|
328
|
+
# --- whole-graph materialization ------------------------------------------------------------
|
|
329
|
+
|
|
330
|
+
def _log_forcing_caller(self, what: str) -> None:
|
|
331
|
+
frames = traceback.extract_stack()[:-3]
|
|
332
|
+
caller = next(
|
|
333
|
+
(f"{f.filename}:{f.lineno} {f.name}" for f in reversed(frames) if not f.filename.endswith("lazy_graph.py")),
|
|
334
|
+
"unknown",
|
|
335
|
+
)
|
|
336
|
+
key = f"{what} @ {caller}"
|
|
337
|
+
if key not in self._forcing_callers_logged:
|
|
338
|
+
self._forcing_callers_logged.add(key)
|
|
339
|
+
_logger.info(f"Converting every lazily held resolver for {what}, read at {caller}")
|
|
340
|
+
|
|
341
|
+
def _hold_streams(self, what: str) -> None:
|
|
342
|
+
with self._lazy_lock:
|
|
343
|
+
if self._streams_held:
|
|
344
|
+
return
|
|
345
|
+
self._log_forcing_caller(what)
|
|
346
|
+
streams = [self._stream_resolver(short) for short in self._stream_protos]
|
|
347
|
+
with self.ensure_transaction() as ctx:
|
|
348
|
+
ctx.upsert_resolvers(streams, replace_on_conflict=False)
|
|
349
|
+
self._streams_held = True
|
|
350
|
+
|
|
351
|
+
def _materialize_all(self, what: str) -> None:
|
|
352
|
+
with self._lazy_lock:
|
|
353
|
+
if self._fully_materialized:
|
|
354
|
+
return
|
|
355
|
+
self._log_forcing_caller(what)
|
|
356
|
+
self._hold_streams(what)
|
|
357
|
+
expression_resolvers = [r for f in self._expression_features for r in self._expression_resolvers(f)]
|
|
358
|
+
with self.ensure_transaction() as ctx:
|
|
359
|
+
ctx.upsert_resolvers(expression_resolvers, replace_on_conflict=False)
|
|
360
|
+
self._fully_materialized = True
|
|
361
|
+
|
|
362
|
+
@property
|
|
363
|
+
def fully_materialized(self) -> bool:
|
|
364
|
+
return self._fully_materialized
|
|
365
|
+
|
|
366
|
+
def lazy_resolver_index(self) -> LazyResolverIndex | None:
|
|
367
|
+
"""The planner's view of the not-yet-converted resolvers; ``None`` once every resolver is held."""
|
|
368
|
+
if self._fully_materialized:
|
|
369
|
+
return None
|
|
370
|
+
return LazyResolverIndex(
|
|
371
|
+
held_resolvers=tuple(super().resolvers),
|
|
372
|
+
streams_held=self._streams_held,
|
|
373
|
+
expression_resolvers_for=self._expression_resolvers,
|
|
374
|
+
feature_position=self._feature_position,
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
@property
|
|
378
|
+
def expression_features(self) -> Sequence[ScalarFeatureType]:
|
|
379
|
+
"""The scalar features with an expression, in the order ``resolvers`` holds their resolvers."""
|
|
380
|
+
return tuple(self._expression_features)
|
|
381
|
+
|
|
382
|
+
def stream_resolvers_where(self, predicate: Callable[[pb.StreamResolver], bool]) -> list[StreamResolverParsed]:
|
|
383
|
+
"""The stream resolvers whose proto satisfies ``predicate``, in ``stream_resolvers`` order,
|
|
384
|
+
converting only those."""
|
|
385
|
+
return [self._stream_resolver(short) for short, proto in self._stream_protos.items() if predicate(proto)]
|
|
386
|
+
|
|
387
|
+
def materialized_resolver_id_to_resolver(self, what: str) -> Mapping[int, ResolverParsed]:
|
|
388
|
+
self._materialize_all(what)
|
|
389
|
+
return super().resolver_id_to_resolver
|
|
390
|
+
|
|
391
|
+
def resolver_by_id_lazily(self, resolver_id: int) -> ResolverParsed | None:
|
|
392
|
+
held = super().resolver_id_to_resolver.get(resolver_id)
|
|
393
|
+
if held is not None or self._fully_materialized:
|
|
394
|
+
return held
|
|
395
|
+
with self._lazy_lock:
|
|
396
|
+
if self._lazy_short_name_by_id is None:
|
|
397
|
+
self._lazy_short_name_by_id = {
|
|
398
|
+
compute_resolver_id(short): short
|
|
399
|
+
for short in (*self._stream_protos, *self._expression_feature_by_resolver_name)
|
|
400
|
+
}
|
|
401
|
+
short_name = self._lazy_short_name_by_id.get(resolver_id)
|
|
402
|
+
return None if short_name is None else self.resolver_by_name(short_name)
|
|
403
|
+
|
|
404
|
+
# --- ResolvedGraph -------------------------------------------------------------------------
|
|
405
|
+
|
|
406
|
+
@property
|
|
407
|
+
def resolvers(self):
|
|
408
|
+
self._materialize_all("resolvers")
|
|
409
|
+
return super().resolvers
|
|
410
|
+
|
|
411
|
+
@property
|
|
412
|
+
def resolver_id_to_resolver(self) -> Mapping[int, ResolverParsed]:
|
|
413
|
+
if self._fully_materialized:
|
|
414
|
+
return super().resolver_id_to_resolver
|
|
415
|
+
return self._resolver_id_to_resolver
|
|
416
|
+
|
|
417
|
+
@property
|
|
418
|
+
def stream_resolvers(self):
|
|
419
|
+
self._hold_streams("stream_resolvers")
|
|
420
|
+
return super().stream_resolvers
|
|
421
|
+
|
|
422
|
+
def resolver_by_name(self, short_name_or_fqn: str) -> ResolverParsed | None:
|
|
423
|
+
held = super().resolver_by_name(short_name_or_fqn)
|
|
424
|
+
if held is not None or self._fully_materialized:
|
|
425
|
+
return held
|
|
426
|
+
short_name = short_name_or_fqn.rsplit(".", maxsplit=1)[-1]
|
|
427
|
+
if short_name in self._stream_protos:
|
|
428
|
+
return self._stream_resolver(short_name)
|
|
429
|
+
feature = self._expression_feature_by_resolver_name.get(short_name)
|
|
430
|
+
if feature is None:
|
|
431
|
+
return None
|
|
432
|
+
return next((r for r in self._expression_resolvers(feature) if r.short_name == short_name), None)
|
|
433
|
+
|
|
434
|
+
@cached_property
|
|
435
|
+
def composite_pkey_resolvers(self) -> Mapping[ScalarFeatureType, list[ResolverParsed]]:
|
|
436
|
+
# A cached_property like the base's, so the override keeps its attribute kind. The mapping is
|
|
437
|
+
# built once in __init__ and indexes lazily itself, so caching it changes nothing; it is exact
|
|
438
|
+
# before and after every resolver is converted, and keeps the keys indexed so far.
|
|
439
|
+
return self._composite_pkey_resolvers
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def lazy_resolver_index(graph: object) -> LazyResolverIndex | None:
|
|
443
|
+
"""``graph``'s lazy resolver index when it is a ``LazyResolvedGraph`` that has not converted every
|
|
444
|
+
resolver yet; otherwise ``None``, and the caller walks ``graph.resolvers``."""
|
|
445
|
+
if isinstance(graph, LazyResolvedGraph):
|
|
446
|
+
return graph.lazy_resolver_index()
|
|
447
|
+
return None
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def held_resolvers_or_all(graph: ResolvedGraph) -> Collection[ResolverParsed]:
|
|
451
|
+
"""The held resolvers of a lazy graph that has not converted every resolver; otherwise
|
|
452
|
+
``graph.resolvers``. For callers that look only at resolvers a graph declares, which a stream or
|
|
453
|
+
expression resolver never is (a static or ChalkSQL resolver); each caller states why that holds."""
|
|
454
|
+
index = lazy_resolver_index(graph)
|
|
455
|
+
if index is None:
|
|
456
|
+
return graph.resolvers
|
|
457
|
+
return index.held_resolvers
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def dataframe_output_candidates(graph: ResolvedGraph) -> Collection[ResolverParsed]:
|
|
461
|
+
"""Every resolver of ``graph`` that can have ``output_is_df``, in ``resolvers`` order, for callers
|
|
462
|
+
that keep only those. On a lazy graph that has not converted every resolver, these are the held
|
|
463
|
+
resolvers plus the stream resolvers whose proto declares a DataFrame output: ``output_is_df`` is
|
|
464
|
+
set exactly by a ``df`` output, and an expression resolver outputs only its own scalar feature
|
|
465
|
+
(``LazyResolvedGraph._expression_resolvers`` checks it). The held resolvers precede the remaining
|
|
466
|
+
stream resolvers in ``resolvers``, so the order is kept."""
|
|
467
|
+
index = lazy_resolver_index(graph)
|
|
468
|
+
if index is None:
|
|
469
|
+
return graph.resolvers
|
|
470
|
+
if index.streams_held:
|
|
471
|
+
return index.held_resolvers
|
|
472
|
+
assert isinstance(graph, LazyResolvedGraph)
|
|
473
|
+
streams = graph.stream_resolvers_where(lambda proto: any(o.HasField("df") for o in proto.outputs))
|
|
474
|
+
return (*index.held_resolvers, *streams)
|