chalkruntime 3.43.87__tar.gz → 3.43.89__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/PKG-INFO +1 -1
  2. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph.py +26 -21
  3. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_impl.py +1 -2
  4. chalkruntime-3.43.89/chalkruntime/graph/lazy_graph.py +474 -0
  5. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/protograph_deserializer.py +119 -51
  6. chalkruntime-3.43.89/chalkruntime/has_many_join_keys.py +98 -0
  7. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/vectorized_hasmany_sampler.py +31 -34
  8. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/PKG-INFO +1 -1
  9. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/SOURCES.txt +3 -0
  10. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/pyproject.toml +2 -2
  11. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/README.md +0 -0
  12. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/__init__.py +0 -0
  13. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/constants.py +0 -0
  14. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/__init__.py +0 -0
  15. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/dataframe.py +0 -0
  16. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/dataframe/lazyframe.py +0 -0
  17. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/__init__.py +0 -0
  18. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/failed_argument.py +0 -0
  19. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/resolver_errors.py +0 -0
  20. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/exc/wrapped_resolver_exception.py +0 -0
  21. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/__init__.py +0 -0
  22. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/_sql_source_pickle_patch.py +0 -0
  23. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/chalk_overload.py +0 -0
  24. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/convert_chalkpy_underscore.py +0 -0
  25. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/dataframe_resolver_correlation.py +0 -0
  26. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/feature.py +0 -0
  27. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/feature_alias.py +0 -0
  28. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/filter_conversion.py +0 -0
  29. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/global_graph.py +0 -0
  30. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_proxy.py +0 -0
  31. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/graph_state.py +0 -0
  32. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/jinja_parser.py +0 -0
  33. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/mat_agg_filter_expression.py +0 -0
  34. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/materializations.py +0 -0
  35. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/materialized_feature_view.py +0 -0
  36. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/maybe_named_collection.py +0 -0
  37. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/named_query.py +0 -0
  38. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/nearest_neighbor.py +0 -0
  39. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/overlay_graph.py +0 -0
  40. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/protograph_serializer.py +0 -0
  41. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/pseudofeatures.py +0 -0
  42. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/resolver.py +0 -0
  43. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/singletons.py +0 -0
  44. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/sklearn_model_parser.py +0 -0
  45. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/source_secrets.py +0 -0
  46. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/stream_resolver.py +0 -0
  47. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore.py +0 -0
  48. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore_codec_info.py +0 -0
  49. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/underscore_operation_registry.py +0 -0
  50. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/graph/variables.py +0 -0
  51. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/heaptrack_launcher.py +0 -0
  52. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/__init__.py +0 -0
  53. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/group_incrementalizer.py +0 -0
  54. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/incrementalization/incrementalizer.py +0 -0
  55. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/__init__.py +0 -0
  56. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/batch_result_collector.py +0 -0
  57. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/bound_invoker.py +0 -0
  58. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/bound_invoker_cache.py +0 -0
  59. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/general_bound_invoker.py +0 -0
  60. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/no_arg_scalar_invoker.py +0 -0
  61. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/one_to_one_invoker.py +0 -0
  62. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/overlay_features.py +0 -0
  63. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/parse_external_resolver.py +0 -0
  64. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/partition_batch.py +0 -0
  65. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/query_execution_parameters.py +0 -0
  66. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_args_builder.py +0 -0
  67. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_input.py +0 -0
  68. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_input_upload.py +0 -0
  69. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_output_metadata.py +0 -0
  70. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_raw_output_parsing.py +0 -0
  71. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_result.py +0 -0
  72. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/resolver_runner.py +0 -0
  73. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/sample.py +0 -0
  74. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/invoker/validator.py +0 -0
  75. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/__init__.py +0 -0
  76. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/converter.py +0 -0
  77. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/loader/importer.py +0 -0
  78. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/memray_launcher.py +0 -0
  79. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/metadata.py +0 -0
  80. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/py.typed +0 -0
  81. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/__init__.py +0 -0
  82. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/branch_model_endpoint_config.py +0 -0
  83. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/config.py +0 -0
  84. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/entrypoint.py +0 -0
  85. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/env_helper.py +0 -0
  86. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/remote_python_function_registry_client.py +0 -0
  87. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/server_context_snapshot.py +0 -0
  88. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/service.py +0 -0
  89. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/server/subprocess_hook_service.py +0 -0
  90. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/__init__.py +0 -0
  91. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/composed_rewriter.py +0 -0
  92. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/contextual_query_rewriter.py +0 -0
  93. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/filter_injection_site.py +0 -0
  94. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/filter_query_rewriter.py +0 -0
  95. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/identity_rewriter.py +0 -0
  96. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/query_rewriter.py +0 -0
  97. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/query_rewriter_helper.py +0 -0
  98. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/sqlglot_import_lock.py +0 -0
  99. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sql_rewriter/sqlglot_parsing.py +0 -0
  100. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/sqlalchemy_mysql_keepalive.py +0 -0
  101. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/__init__.py +0 -0
  102. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/converter_utils.py +0 -0
  103. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/exc.py +0 -0
  104. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/message_parsing.py +0 -0
  105. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/resolver_utils.py +0 -0
  106. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/streaming_invoker_kernel/__init__.py +0 -0
  107. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/streaming_invoker_kernel/local_streaming_invoker_kernel.py +0 -0
  108. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/types.py +0 -0
  109. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/streaming/window_keys.py +0 -0
  110. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/__init__.py +0 -0
  111. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/async_helpers.py +0 -0
  112. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/contextvars.py +0 -0
  113. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/datadog.py +0 -0
  114. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/internal_pl_utils.py +0 -0
  115. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/posix_file_lock.py +0 -0
  116. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/proto_bytes_cache.py +0 -0
  117. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/pyarrow_utils.py +0 -0
  118. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/tracing.py +0 -0
  119. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/utils/viztracer_profiling.py +0 -0
  120. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime/valgrind_launcher.py +0 -0
  121. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/dependency_links.txt +0 -0
  122. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/requires.txt +0 -0
  123. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/chalkruntime.egg-info/top_level.txt +0 -0
  124. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/setup.cfg +0 -0
  125. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/setup.py +0 -0
  126. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_branch_model_endpoint_config.py +0 -0
  127. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_captured_global_function_refs.py +0 -0
  128. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_datadog.py +0 -0
  129. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_feature_aliases.py +0 -0
  130. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_graph_cache.py +0 -0
  131. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_internal_pl_utils.py +0 -0
  132. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_mat_agg_filter_expressions.py +0 -0
  133. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_mat_agg_projection_expressions.py +0 -0
  134. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_materialized_feature_view.py +0 -0
  135. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_posix_file_lock.py +0 -0
  136. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_pyarrow_utils.py +0 -0
  137. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_server_context_snapshot.py +0 -0
  138. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sql_max_row_version_lookback_deserialization.py +0 -0
  139. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sql_retry_policy_deserialization.py +0 -0
  140. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_sqlalchemy_mysql_keepalive.py +0 -0
  141. {chalkruntime-3.43.87 → chalkruntime-3.43.89}/tests/test_subprocess_hook_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chalkruntime
3
- Version: 3.43.87
3
+ Version: 3.43.89
4
4
  Summary: Runtime support library for Chalk AI
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -319,28 +319,33 @@ class ResolvedGraph(Protocol):
319
319
  return self.resolver_id_to_resolver[resolver_id]
320
320
 
321
321
  @cached_property
322
- def composite_pkey_resolvers(self) -> dict[ScalarFeatureType, list[ResolverParsed]]:
323
- from chalkruntime.graph.resolver import OfflineResolverParsed
324
-
322
+ def composite_pkey_resolvers(self) -> Mapping[ScalarFeatureType, list[ResolverParsed]]:
323
+ """The resolvers computing each primary key from other features of its namespace, in graph
324
+ order. Indexing a primary key no resolver computes gives an empty list."""
325
325
  output = defaultdict(list)
326
326
  for resolver in self.resolvers:
327
- namespaces = resolver.input_root_namespaces
328
- if len(namespaces) != 1:
329
- continue
330
- if resolver.output_is_df or resolver.input_is_df:
331
- continue
332
- if isinstance(resolver, OfflineResolverParsed):
333
- # only online resolvers may compute pkeys
334
- continue
335
- pkey_feature = self.primary_feature_for_namespace(get_unique_item(namespaces))
336
- if pkey_feature is None:
337
- continue
338
- if (
339
- len(resolver.inputs) >= 1
340
- and pkey_feature in resolver.output_features
341
- and pkey_feature not in resolver.inputs
342
- ):
343
- if pkey_feature not in output:
344
- output[pkey_feature] = []
327
+ pkey_feature = composite_pkey_computed_by(resolver, self)
328
+ if pkey_feature is not None:
345
329
  output[pkey_feature].append(resolver)
346
330
  return output
331
+
332
+
333
+ def composite_pkey_computed_by(resolver: ResolverParsed, graph: ResolvedGraph) -> ScalarFeatureType | None:
334
+ """The primary key ``resolver`` computes from other features of its namespace, which makes it one of
335
+ ``composite_pkey_resolvers[pkey]``; ``None`` when it computes none."""
336
+ from chalkruntime.graph.resolver import OfflineResolverParsed
337
+
338
+ namespaces = resolver.input_root_namespaces
339
+ if len(namespaces) != 1:
340
+ return None
341
+ if resolver.output_is_df or resolver.input_is_df:
342
+ return None
343
+ if isinstance(resolver, OfflineResolverParsed):
344
+ # only online resolvers may compute pkeys
345
+ return None
346
+ pkey_feature = graph.primary_feature_for_namespace(get_unique_item(namespaces))
347
+ if pkey_feature is None:
348
+ return None
349
+ if len(resolver.inputs) >= 1 and pkey_feature in resolver.output_features and pkey_feature not in resolver.inputs:
350
+ return pkey_feature
351
+ return None
@@ -19,7 +19,6 @@ from typing import (
19
19
  Sequence,
20
20
  TypeVar,
21
21
  cast,
22
- final,
23
22
  overload,
24
23
  )
25
24
 
@@ -721,7 +720,7 @@ def _split_top_level_commas(s: str) -> list[str]:
721
720
  return [stripped for p in parts if (stripped := p.strip())]
722
721
 
723
722
 
724
- @final
723
+ # Not final: LazyResolvedGraph (lazy_graph.py) subclasses it to convert resolvers on first use.
725
724
  class ResolvedGraphImpl(ResolvedGraph):
726
725
  def __init__(
727
726
  self,
@@ -0,0 +1,474 @@
1
+ """A ``ResolvedGraph`` that converts stream resolvers and expression resolvers on first use.
2
+
3
+ ``FromProtoConverter.convert_graph`` converts every resolver at boot. On graphs with many stream
4
+ resolvers and expression features nearly all of that time goes to converting their underscore
5
+ expressions, which a pod reads for only the few resolvers its queries plan with.
6
+ ``FromProtoConverter.convert_graph_lazily`` builds this graph instead: features and declared
7
+ resolvers convert eagerly as before, while each stream resolver converts from its proto, and each
8
+ feature's expression resolvers from the feature, the first time someone asks for them. Every object
9
+ is the one ``convert_graph`` would build; only the time it is built changes.
10
+
11
+ Lookups by name (``resolver_by_name``, ``get_stream_resolver``, ``resolver_id_to_resolver[id]``) and
12
+ ``composite_pkey_resolvers[pkey]`` convert just what they return. ``lazy_resolver_index`` hands the
13
+ planner what it needs to find a feature's candidate resolvers without walking the graph. Anything that
14
+ reads every resolver (``resolvers``, ``stream_resolvers``, iterating ``resolver_id_to_resolver``)
15
+ converts all of them once, upserts them in ``convert_graph``'s order, logs its caller, and from then on
16
+ the graph behaves exactly as an eager one.
17
+
18
+ A conversion error that ``convert_graph`` would raise at boot is raised when the item is first used.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import dataclasses
24
+ import logging
25
+ import threading
26
+ import traceback
27
+ from collections.abc import Callable, Collection, Iterator, Mapping, Sequence
28
+ from functools import cached_property
29
+ from typing import TYPE_CHECKING, Any, TypeVar, overload
30
+
31
+ from chalk._gen.chalk.graph.v1 import graph_pb2 as pb
32
+ from chalkruntime.graph.graph import ResolvedGraph, composite_pkey_computed_by
33
+ from chalkruntime.graph.graph_impl import ResolvedGraphImpl
34
+ from chalkruntime.metadata import compute_resolver_id
35
+
36
+ if TYPE_CHECKING:
37
+ from chalk._gen.chalk.symbolic_value.v1.symbolic_value_pb2 import SymbolicValue
38
+ from chalk.sql._internal.sql_source import BaseSQLSource
39
+ from chalk.sql._internal.sql_source_group import SQLSourceGroup
40
+ from chalk.streams import StreamSource
41
+
42
+ from chalkruntime.graph.feature import ScalarFeatureType
43
+ from chalkruntime.graph.materialized_feature_view import MaterializedFeatureViewParsed
44
+ from chalkruntime.graph.protograph_deserializer import OnlineStoreConfigParsed
45
+ from chalkruntime.graph.resolver import ResolverParsed
46
+ from chalkruntime.graph.stream_resolver import StreamResolverParsed
47
+
48
+ _logger = logging.getLogger(__name__)
49
+
50
+ # The suffix ``underscore_to_online_resolver`` gives a materialized aggregation's unmaterialized resolver.
51
+ _UNMATERIALIZED_TAG = "unit_testing"
52
+
53
+
54
+ @dataclasses.dataclass(frozen=True)
55
+ class LazyResolverIndex:
56
+ """What a planner needs to find candidate resolvers on a graph whose stream and expression
57
+ resolvers are not all converted yet.
58
+
59
+ ``held_resolvers`` are the converted resolvers in ``resolvers`` order: the declared resolvers, then
60
+ the stream resolvers once ``streams_held``. A stream resolver is never a planning candidate
61
+ (``is_valid_candidate_resolver``) unless a query prefers it, and an expression resolver outputs
62
+ only its own feature, so a feature's candidates are the held resolvers' outputs plus
63
+ ``expression_resolvers_for(feature)``, which come after them in ``resolvers`` order.
64
+ """
65
+
66
+ held_resolvers: tuple[ResolverParsed, ...]
67
+ streams_held: bool
68
+ expression_resolvers_for: Callable[[ScalarFeatureType], tuple[ResolverParsed, ...]]
69
+ # A scalar feature's position among the graph's scalar features, which orders expression
70
+ # resolvers of different features as ``resolvers`` does.
71
+ feature_position: Callable[[ScalarFeatureType], int]
72
+
73
+
74
+ _T = TypeVar("_T")
75
+
76
+
77
+ class _LazyCompositePkeyResolvers(Mapping["ScalarFeatureType", "list[ResolverParsed]"]):
78
+ """``composite_pkey_resolvers`` of a lazy graph, computed per primary key.
79
+
80
+ A primary key's composite resolvers are among the held resolvers and the key's own expression
81
+ resolvers: a stream resolver has no feature inputs, and any other expression resolver outputs a
82
+ different feature. It behaves as the eager ``defaultdict``: indexing a key no resolver computes
83
+ gives an empty list and from then on holds the key.
84
+ """
85
+
86
+ def __init__(self, graph: LazyResolvedGraph):
87
+ super().__init__()
88
+ self._graph = graph
89
+ self._computed: dict[ScalarFeatureType, list[ResolverParsed]] = {}
90
+ # Keys indexed so far, as the eager defaultdict holds them.
91
+ self._indexed: dict[ScalarFeatureType, None] = {}
92
+
93
+ def _compute(self, pkey: ScalarFeatureType) -> list[ResolverParsed]:
94
+ computed = self._computed.get(pkey)
95
+ if computed is None:
96
+ index = self._graph.lazy_resolver_index()
97
+ resolvers: Collection[ResolverParsed] = (
98
+ self._graph.resolvers
99
+ if index is None
100
+ else (*index.held_resolvers, *index.expression_resolvers_for(pkey))
101
+ )
102
+ computed = [r for r in resolvers if composite_pkey_computed_by(r, self._graph) == pkey]
103
+ self._computed[pkey] = computed
104
+ return computed
105
+
106
+ def _holds(self, pkey: ScalarFeatureType) -> bool:
107
+ return pkey in self._indexed or len(self._compute(pkey)) > 0
108
+
109
+ def __getitem__(self, pkey: ScalarFeatureType) -> list[ResolverParsed]:
110
+ self._indexed[pkey] = None
111
+ return self._compute(pkey)
112
+
113
+ # As the eager defaultdict's ``get``, a lookup does not hold the key, which ``Mapping.get``
114
+ # (``__getitem__`` with a fallback) would.
115
+ @overload
116
+ def get(self, pkey: object, /) -> list[ResolverParsed] | None: ...
117
+ @overload
118
+ def get(self, pkey: object, /, default: _T) -> list[ResolverParsed] | _T: ...
119
+ def get(self, pkey: object, /, default: object = None) -> object:
120
+ from chalkruntime.graph.feature import ScalarFeatureType
121
+
122
+ if isinstance(pkey, ScalarFeatureType) and self._holds(pkey):
123
+ return self._compute(pkey)
124
+ return default
125
+
126
+ def __contains__(self, pkey: object) -> bool:
127
+ from chalkruntime.graph.feature import ScalarFeatureType
128
+
129
+ return isinstance(pkey, ScalarFeatureType) and self._holds(pkey)
130
+
131
+ def _all(self) -> dict[ScalarFeatureType, list[ResolverParsed]]:
132
+ output: dict[ScalarFeatureType, list[ResolverParsed]] = {}
133
+ for resolver in self._graph.resolvers:
134
+ pkey = composite_pkey_computed_by(resolver, self._graph)
135
+ if pkey is not None:
136
+ output.setdefault(pkey, []).append(resolver)
137
+ for pkey in self._indexed:
138
+ output.setdefault(pkey, [])
139
+ return output
140
+
141
+ def __iter__(self) -> Iterator[ScalarFeatureType]:
142
+ return iter(self._all())
143
+
144
+ def __len__(self) -> int:
145
+ return len(self._all())
146
+
147
+
148
+ class _LazyResolverIdToResolver(Mapping[int, "ResolverParsed"]):
149
+ """``resolver_id_to_resolver`` of a lazy graph: a lookup by id converts just that resolver, while
150
+ iterating converts every resolver."""
151
+
152
+ def __init__(self, graph: LazyResolvedGraph):
153
+ super().__init__()
154
+ self._graph = graph
155
+
156
+ def __getitem__(self, resolver_id: int) -> ResolverParsed:
157
+ resolver = self._graph.resolver_by_id_lazily(resolver_id)
158
+ if resolver is None:
159
+ raise KeyError(resolver_id)
160
+ return resolver
161
+
162
+ def __iter__(self) -> Iterator[int]:
163
+ return iter(self._graph.materialized_resolver_id_to_resolver("resolver_id_to_resolver iteration"))
164
+
165
+ def __len__(self) -> int:
166
+ return len(self._graph.materialized_resolver_id_to_resolver("resolver_id_to_resolver length"))
167
+
168
+
169
+ class LazyResolvedGraph(ResolvedGraphImpl):
170
+ """See the module docstring. Built by ``FromProtoConverter.convert_graph_lazily``."""
171
+
172
+ def __init__(
173
+ self,
174
+ *,
175
+ sql_sources: Sequence[BaseSQLSource],
176
+ sql_source_groups: Sequence[SQLSourceGroup],
177
+ online_store_configs: Sequence[OnlineStoreConfigParsed],
178
+ symbolic_value_explicits: Sequence[SymbolicValue],
179
+ materialized_feature_views: Sequence[MaterializedFeatureViewParsed],
180
+ stream_resolver_protos: Sequence[pb.StreamResolver],
181
+ stream_sources: dict[tuple[str, str | None], StreamSource],
182
+ captured_global_values: Mapping[str, pb.FunctionReferenceCapturedGlobal],
183
+ ):
184
+ super().__init__(
185
+ sql_sources=sql_sources,
186
+ sql_source_groups=sql_source_groups,
187
+ online_store_configs=online_store_configs,
188
+ symbolic_value_explicits=symbolic_value_explicits,
189
+ materialized_feature_views=materialized_feature_views,
190
+ )
191
+ # Materialization may convert while it holds the lock, and conversion reads the graph.
192
+ self._lazy_lock = threading.RLock()
193
+ self._stream_protos: dict[str, pb.StreamResolver] = {
194
+ r.fqn.rsplit(".", maxsplit=1)[-1]: r for r in stream_resolver_protos
195
+ }
196
+ self._stream_sources = stream_sources
197
+ self._captured_global_values = captured_global_values
198
+ self._converted_streams: dict[str, StreamResolverParsed] = {}
199
+ self._streams_held = False
200
+ # Filled by ``index_expression_features`` once the features are converted.
201
+ self._scalar_positions: dict[str, int] = {}
202
+ self._expression_features: list[ScalarFeatureType] = []
203
+ self._expression_feature_by_resolver_name: dict[str, ScalarFeatureType] = {}
204
+ self._lazy_short_name_by_id: dict[int, str] | None = None
205
+ self._converted_expressions: dict[str, tuple[ResolverParsed, ...]] = {}
206
+ self._fully_materialized = False
207
+ self._composite_pkey_resolvers = _LazyCompositePkeyResolvers(self)
208
+ self._resolver_id_to_resolver = _LazyResolverIdToResolver(self)
209
+ self._forcing_callers_logged: set[str] = set()
210
+
211
+ # An unpickled graph is exactly as lazy as the original: every resolver converted so far travels as
212
+ # it is, and the held protos travel serialized, since pickle cannot import the generated message
213
+ # classes by their registered module name. The lock is recreated.
214
+ def __getstate__(self) -> dict[str, object]:
215
+ state = dict(self.__dict__)
216
+ del state["_lazy_lock"]
217
+ state["_stream_protos"] = {short: r.SerializeToString() for short, r in self._stream_protos.items()}
218
+ state["_captured_global_values"] = {
219
+ ref: value.SerializeToString() for ref, value in self._captured_global_values.items()
220
+ }
221
+ return state
222
+
223
+ def __setstate__(self, state: dict[str, Any]) -> None:
224
+ self.__dict__.update(state)
225
+ self._stream_protos = {short: pb.StreamResolver.FromString(r) for short, r in state["_stream_protos"].items()}
226
+ self._captured_global_values = {
227
+ ref: pb.FunctionReferenceCapturedGlobal.FromString(value)
228
+ for ref, value in state["_captured_global_values"].items()
229
+ }
230
+ self._lazy_lock = threading.RLock()
231
+
232
+ def index_expression_features(self) -> None:
233
+ """Record the scalar features whose expressions ``scalar_feature_types_to_resolvers`` turns into
234
+ resolvers, and the names those resolvers can take, without converting them. Called once the
235
+ features are converted."""
236
+ from chalkruntime.graph.convert_chalkpy_underscore import get_underscore_fqn_from_feature_fqn
237
+ from chalkruntime.graph.feature import ScalarFeatureType
238
+
239
+ scalars = [f for f in self.features if isinstance(f, ScalarFeatureType)]
240
+ self._scalar_positions = {f.fqn: i for i, f in enumerate(scalars)}
241
+ self._expression_features = [
242
+ f for f in scalars if f.underscore_expression is not None or f.offline_underscore_expression is not None
243
+ ]
244
+ names: dict[str, ScalarFeatureType] = {}
245
+ for f in self._expression_features:
246
+ for fqn in (
247
+ get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online"),
248
+ get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online", tag=_UNMATERIALIZED_TAG),
249
+ get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="offline"),
250
+ ):
251
+ names[fqn.rsplit(".", maxsplit=1)[-1]] = f
252
+ self._expression_feature_by_resolver_name = names
253
+ self._check_unique_names()
254
+
255
+ def _check_unique_names(self) -> None:
256
+ """Raise on the short-name and id collisions ``upsert_resolvers`` raises on at boot for an eager
257
+ graph, for every name a lazy resolver is sure to take."""
258
+ from chalkruntime.graph.convert_chalkpy_underscore import get_underscore_fqn_from_feature_fqn
259
+
260
+ seen: dict[str, str] = {r.short_name: r.fqn for r in super().resolvers}
261
+ sure_names = [r.fqn for r in self._stream_protos.values()]
262
+ for f in self._expression_features:
263
+ if f.underscore_expression is not None:
264
+ sure_names.append(get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="online"))
265
+ if f.offline_underscore_expression is not None or f.underscore_expression is not None:
266
+ sure_names.append(get_underscore_fqn_from_feature_fqn(f.fqn, resolver_kind="offline"))
267
+ ids: dict[int, str] = {compute_resolver_id(short): fqn for short, fqn in seen.items()}
268
+ for fqn in sure_names:
269
+ short = fqn.rsplit(".", maxsplit=1)[-1]
270
+ if short in seen:
271
+ raise ValueError(
272
+ f"Multiple resolvers with the same short name when parsing the graph: '{seen[short]}' and "
273
+ + f"'{fqn}' both have the same short name '{short}'"
274
+ )
275
+ seen[short] = fqn
276
+ resolver_id = compute_resolver_id(short)
277
+ if resolver_id in ids:
278
+ raise ValueError(
279
+ f"Multiple resolvers with the same ID when parsing the graph: '{ids[resolver_id]}' and "
280
+ + f"'{fqn}' both have the same id {resolver_id}"
281
+ )
282
+ ids[resolver_id] = fqn
283
+
284
+ # --- per-item conversion -------------------------------------------------------------------
285
+
286
+ def _stream_resolver(self, short_name: str) -> StreamResolverParsed:
287
+ from chalkruntime.graph.protograph_deserializer import FromProtoConverter
288
+
289
+ with self._lazy_lock:
290
+ converted = self._converted_streams.get(short_name)
291
+ if converted is None:
292
+ converted = FromProtoConverter.convert_stream_resolver(
293
+ self._stream_protos[short_name],
294
+ graph=self,
295
+ stream_sources=self._stream_sources,
296
+ captured_global_values=self._captured_global_values,
297
+ )
298
+ self._converted_streams[short_name] = converted
299
+ return converted
300
+
301
+ def _expression_resolvers(self, feature: ScalarFeatureType) -> tuple[ResolverParsed, ...]:
302
+ """The resolvers ``scalar_feature_types_to_resolvers`` builds from ``feature``'s expressions, in
303
+ its order; empty for a feature without expressions."""
304
+ from chalkruntime.graph.convert_chalkpy_underscore import scalar_feature_types_to_resolvers
305
+
306
+ if feature.underscore_expression is None and feature.offline_underscore_expression is None:
307
+ return ()
308
+ with self._lazy_lock:
309
+ converted = self._converted_expressions.get(feature.fqn)
310
+ if converted is None:
311
+ converted = tuple(
312
+ scalar_feature_types_to_resolvers(scalars=[feature], graph=self, include_definition_strings=False)
313
+ )
314
+ # The planner's lazy candidate search relies on an expression resolver outputting only
315
+ # its feature (``LazyResolverIndex``).
316
+ for resolver in converted:
317
+ if tuple(resolver.output_features) != (feature,):
318
+ raise ValueError(
319
+ f"Expression resolver '{resolver.fqn}' outputs {list(resolver.output_features)}, not "
320
+ + f"just its feature '{feature.fqn}'"
321
+ )
322
+ self._converted_expressions[feature.fqn] = converted
323
+ return converted
324
+
325
+ def _feature_position(self, feature: ScalarFeatureType) -> int:
326
+ return self._scalar_positions[feature.fqn]
327
+
328
+ # --- whole-graph materialization ------------------------------------------------------------
329
+
330
+ def _log_forcing_caller(self, what: str) -> None:
331
+ frames = traceback.extract_stack()[:-3]
332
+ caller = next(
333
+ (f"{f.filename}:{f.lineno} {f.name}" for f in reversed(frames) if not f.filename.endswith("lazy_graph.py")),
334
+ "unknown",
335
+ )
336
+ key = f"{what} @ {caller}"
337
+ if key not in self._forcing_callers_logged:
338
+ self._forcing_callers_logged.add(key)
339
+ _logger.info(f"Converting every lazily held resolver for {what}, read at {caller}")
340
+
341
+ def _hold_streams(self, what: str) -> None:
342
+ with self._lazy_lock:
343
+ if self._streams_held:
344
+ return
345
+ self._log_forcing_caller(what)
346
+ streams = [self._stream_resolver(short) for short in self._stream_protos]
347
+ with self.ensure_transaction() as ctx:
348
+ ctx.upsert_resolvers(streams, replace_on_conflict=False)
349
+ self._streams_held = True
350
+
351
+ def _materialize_all(self, what: str) -> None:
352
+ with self._lazy_lock:
353
+ if self._fully_materialized:
354
+ return
355
+ self._log_forcing_caller(what)
356
+ self._hold_streams(what)
357
+ expression_resolvers = [r for f in self._expression_features for r in self._expression_resolvers(f)]
358
+ with self.ensure_transaction() as ctx:
359
+ ctx.upsert_resolvers(expression_resolvers, replace_on_conflict=False)
360
+ self._fully_materialized = True
361
+
362
+ @property
363
+ def fully_materialized(self) -> bool:
364
+ return self._fully_materialized
365
+
366
+ def lazy_resolver_index(self) -> LazyResolverIndex | None:
367
+ """The planner's view of the not-yet-converted resolvers; ``None`` once every resolver is held."""
368
+ if self._fully_materialized:
369
+ return None
370
+ return LazyResolverIndex(
371
+ held_resolvers=tuple(super().resolvers),
372
+ streams_held=self._streams_held,
373
+ expression_resolvers_for=self._expression_resolvers,
374
+ feature_position=self._feature_position,
375
+ )
376
+
377
+ @property
378
+ def expression_features(self) -> Sequence[ScalarFeatureType]:
379
+ """The scalar features with an expression, in the order ``resolvers`` holds their resolvers."""
380
+ return tuple(self._expression_features)
381
+
382
+ def stream_resolvers_where(self, predicate: Callable[[pb.StreamResolver], bool]) -> list[StreamResolverParsed]:
383
+ """The stream resolvers whose proto satisfies ``predicate``, in ``stream_resolvers`` order,
384
+ converting only those."""
385
+ return [self._stream_resolver(short) for short, proto in self._stream_protos.items() if predicate(proto)]
386
+
387
+ def materialized_resolver_id_to_resolver(self, what: str) -> Mapping[int, ResolverParsed]:
388
+ self._materialize_all(what)
389
+ return super().resolver_id_to_resolver
390
+
391
+ def resolver_by_id_lazily(self, resolver_id: int) -> ResolverParsed | None:
392
+ held = super().resolver_id_to_resolver.get(resolver_id)
393
+ if held is not None or self._fully_materialized:
394
+ return held
395
+ with self._lazy_lock:
396
+ if self._lazy_short_name_by_id is None:
397
+ self._lazy_short_name_by_id = {
398
+ compute_resolver_id(short): short
399
+ for short in (*self._stream_protos, *self._expression_feature_by_resolver_name)
400
+ }
401
+ short_name = self._lazy_short_name_by_id.get(resolver_id)
402
+ return None if short_name is None else self.resolver_by_name(short_name)
403
+
404
+ # --- ResolvedGraph -------------------------------------------------------------------------
405
+
406
+ @property
407
+ def resolvers(self):
408
+ self._materialize_all("resolvers")
409
+ return super().resolvers
410
+
411
+ @property
412
+ def resolver_id_to_resolver(self) -> Mapping[int, ResolverParsed]:
413
+ if self._fully_materialized:
414
+ return super().resolver_id_to_resolver
415
+ return self._resolver_id_to_resolver
416
+
417
+ @property
418
+ def stream_resolvers(self):
419
+ self._hold_streams("stream_resolvers")
420
+ return super().stream_resolvers
421
+
422
+ def resolver_by_name(self, short_name_or_fqn: str) -> ResolverParsed | None:
423
+ held = super().resolver_by_name(short_name_or_fqn)
424
+ if held is not None or self._fully_materialized:
425
+ return held
426
+ short_name = short_name_or_fqn.rsplit(".", maxsplit=1)[-1]
427
+ if short_name in self._stream_protos:
428
+ return self._stream_resolver(short_name)
429
+ feature = self._expression_feature_by_resolver_name.get(short_name)
430
+ if feature is None:
431
+ return None
432
+ return next((r for r in self._expression_resolvers(feature) if r.short_name == short_name), None)
433
+
434
+ @cached_property
435
+ def composite_pkey_resolvers(self) -> Mapping[ScalarFeatureType, list[ResolverParsed]]:
436
+ # A cached_property like the base's, so the override keeps its attribute kind. The mapping is
437
+ # built once in __init__ and indexes lazily itself, so caching it changes nothing; it is exact
438
+ # before and after every resolver is converted, and keeps the keys indexed so far.
439
+ return self._composite_pkey_resolvers
440
+
441
+
442
+ def lazy_resolver_index(graph: object) -> LazyResolverIndex | None:
443
+ """``graph``'s lazy resolver index when it is a ``LazyResolvedGraph`` that has not converted every
444
+ resolver yet; otherwise ``None``, and the caller walks ``graph.resolvers``."""
445
+ if isinstance(graph, LazyResolvedGraph):
446
+ return graph.lazy_resolver_index()
447
+ return None
448
+
449
+
450
+ def held_resolvers_or_all(graph: ResolvedGraph) -> Collection[ResolverParsed]:
451
+ """The held resolvers of a lazy graph that has not converted every resolver; otherwise
452
+ ``graph.resolvers``. For callers that look only at resolvers a graph declares, which a stream or
453
+ expression resolver never is (a static or ChalkSQL resolver); each caller states why that holds."""
454
+ index = lazy_resolver_index(graph)
455
+ if index is None:
456
+ return graph.resolvers
457
+ return index.held_resolvers
458
+
459
+
460
+ def dataframe_output_candidates(graph: ResolvedGraph) -> Collection[ResolverParsed]:
461
+ """Every resolver of ``graph`` that can have ``output_is_df``, in ``resolvers`` order, for callers
462
+ that keep only those. On a lazy graph that has not converted every resolver, these are the held
463
+ resolvers plus the stream resolvers whose proto declares a DataFrame output: ``output_is_df`` is
464
+ set exactly by a ``df`` output, and an expression resolver outputs only its own scalar feature
465
+ (``LazyResolvedGraph._expression_resolvers`` checks it). The held resolvers precede the remaining
466
+ stream resolvers in ``resolvers``, so the order is kept."""
467
+ index = lazy_resolver_index(graph)
468
+ if index is None:
469
+ return graph.resolvers
470
+ if index.streams_held:
471
+ return index.held_resolvers
472
+ assert isinstance(graph, LazyResolvedGraph)
473
+ streams = graph.stream_resolvers_where(lambda proto: any(o.HasField("df") for o in proto.outputs))
474
+ return (*index.held_resolvers, *streams)