chalkruntime 3.43.82__tar.gz → 3.43.83__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/PKG-INFO +1 -1
  2. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/vectorized_hasmany_sampler.py +237 -20
  3. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime.egg-info/PKG-INFO +1 -1
  4. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/pyproject.toml +2 -2
  5. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/README.md +0 -0
  6. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/__init__.py +0 -0
  7. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/constants.py +0 -0
  8. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/dataframe/__init__.py +0 -0
  9. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/dataframe/dataframe.py +0 -0
  10. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/dataframe/lazyframe.py +0 -0
  11. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/exc/__init__.py +0 -0
  12. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/exc/failed_argument.py +0 -0
  13. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/exc/resolver_errors.py +0 -0
  14. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/exc/wrapped_resolver_exception.py +0 -0
  15. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/__init__.py +0 -0
  16. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/_sql_source_pickle_patch.py +0 -0
  17. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/chalk_overload.py +0 -0
  18. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/convert_chalkpy_underscore.py +0 -0
  19. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/dataframe_resolver_correlation.py +0 -0
  20. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/feature.py +0 -0
  21. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/feature_alias.py +0 -0
  22. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/filter_conversion.py +0 -0
  23. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/global_graph.py +0 -0
  24. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/graph.py +0 -0
  25. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/graph_impl.py +0 -0
  26. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/graph_proxy.py +0 -0
  27. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/graph_state.py +0 -0
  28. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/jinja_parser.py +0 -0
  29. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/mat_agg_filter_expression.py +0 -0
  30. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/materializations.py +0 -0
  31. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/materialized_feature_view.py +0 -0
  32. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/maybe_named_collection.py +0 -0
  33. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/named_query.py +0 -0
  34. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/nearest_neighbor.py +0 -0
  35. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/overlay_graph.py +0 -0
  36. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/protograph_deserializer.py +0 -0
  37. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/protograph_serializer.py +0 -0
  38. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/pseudofeatures.py +0 -0
  39. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/resolver.py +0 -0
  40. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/singletons.py +0 -0
  41. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/sklearn_model_parser.py +0 -0
  42. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/source_secrets.py +0 -0
  43. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/stream_resolver.py +0 -0
  44. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/underscore.py +0 -0
  45. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/underscore_codec_info.py +0 -0
  46. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/underscore_operation_registry.py +0 -0
  47. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/graph/variables.py +0 -0
  48. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/heaptrack_launcher.py +0 -0
  49. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/incrementalization/__init__.py +0 -0
  50. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/incrementalization/group_incrementalizer.py +0 -0
  51. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/incrementalization/incrementalizer.py +0 -0
  52. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/__init__.py +0 -0
  53. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/batch_result_collector.py +0 -0
  54. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/bound_invoker.py +0 -0
  55. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/bound_invoker_cache.py +0 -0
  56. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/general_bound_invoker.py +0 -0
  57. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/no_arg_scalar_invoker.py +0 -0
  58. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/one_to_one_invoker.py +0 -0
  59. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/overlay_features.py +0 -0
  60. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/parse_external_resolver.py +0 -0
  61. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/partition_batch.py +0 -0
  62. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/query_execution_parameters.py +0 -0
  63. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_args_builder.py +0 -0
  64. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_input.py +0 -0
  65. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_input_upload.py +0 -0
  66. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_output_metadata.py +0 -0
  67. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_raw_output_parsing.py +0 -0
  68. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_result.py +0 -0
  69. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/resolver_runner.py +0 -0
  70. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/sample.py +0 -0
  71. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/invoker/validator.py +0 -0
  72. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/loader/__init__.py +0 -0
  73. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/loader/converter.py +0 -0
  74. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/loader/importer.py +0 -0
  75. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/memray_launcher.py +0 -0
  76. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/metadata.py +0 -0
  77. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/py.typed +0 -0
  78. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/__init__.py +0 -0
  79. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/branch_model_endpoint_config.py +0 -0
  80. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/config.py +0 -0
  81. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/entrypoint.py +0 -0
  82. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/env_helper.py +0 -0
  83. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/remote_python_function_registry_client.py +0 -0
  84. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/server_context_snapshot.py +0 -0
  85. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/service.py +0 -0
  86. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/server/subprocess_hook_service.py +0 -0
  87. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/__init__.py +0 -0
  88. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/composed_rewriter.py +0 -0
  89. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/contextual_query_rewriter.py +0 -0
  90. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/filter_injection_site.py +0 -0
  91. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/filter_query_rewriter.py +0 -0
  92. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/identity_rewriter.py +0 -0
  93. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/query_rewriter.py +0 -0
  94. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/query_rewriter_helper.py +0 -0
  95. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/sqlglot_import_lock.py +0 -0
  96. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sql_rewriter/sqlglot_parsing.py +0 -0
  97. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/sqlalchemy_mysql_keepalive.py +0 -0
  98. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/__init__.py +0 -0
  99. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/converter_utils.py +0 -0
  100. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/exc.py +0 -0
  101. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/message_parsing.py +0 -0
  102. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/resolver_utils.py +0 -0
  103. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/streaming_invoker_kernel/__init__.py +0 -0
  104. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/streaming_invoker_kernel/local_streaming_invoker_kernel.py +0 -0
  105. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/types.py +0 -0
  106. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/streaming/window_keys.py +0 -0
  107. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/__init__.py +0 -0
  108. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/async_helpers.py +0 -0
  109. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/contextvars.py +0 -0
  110. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/datadog.py +0 -0
  111. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/internal_pl_utils.py +0 -0
  112. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/posix_file_lock.py +0 -0
  113. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/proto_bytes_cache.py +0 -0
  114. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/pyarrow_utils.py +0 -0
  115. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/tracing.py +0 -0
  116. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/utils/viztracer_profiling.py +0 -0
  117. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime/valgrind_launcher.py +0 -0
  118. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime.egg-info/SOURCES.txt +0 -0
  119. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime.egg-info/dependency_links.txt +0 -0
  120. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime.egg-info/requires.txt +0 -0
  121. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/chalkruntime.egg-info/top_level.txt +0 -0
  122. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/setup.cfg +0 -0
  123. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/setup.py +0 -0
  124. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_branch_model_endpoint_config.py +0 -0
  125. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_captured_global_function_refs.py +0 -0
  126. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_datadog.py +0 -0
  127. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_feature_aliases.py +0 -0
  128. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_graph_cache.py +0 -0
  129. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_internal_pl_utils.py +0 -0
  130. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_mat_agg_filter_expressions.py +0 -0
  131. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_mat_agg_projection_expressions.py +0 -0
  132. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_materialized_feature_view.py +0 -0
  133. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_posix_file_lock.py +0 -0
  134. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_pyarrow_utils.py +0 -0
  135. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_server_context_snapshot.py +0 -0
  136. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_sql_max_row_version_lookback_deserialization.py +0 -0
  137. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_sql_retry_policy_deserialization.py +0 -0
  138. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_sqlalchemy_mysql_keepalive.py +0 -0
  139. {chalkruntime-3.43.82 → chalkruntime-3.43.83}/tests/test_subprocess_hook_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chalkruntime
3
- Version: 3.43.82
3
+ Version: 3.43.83
4
4
  Summary: Runtime support library for Chalk AI
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -11,6 +11,7 @@ import polars as pl
11
11
  import pyarrow as pa
12
12
  import pyarrow.compute as pc
13
13
  from chalk.features import DataFrame
14
+ from chalk.features.underscore import _
14
15
  from chalk.features.dataframe._filters import PolarsFilterConverter
15
16
  from chalk.features.dataframe._validation import (
16
17
  _generate_empty_series_for_dtype, # pyright: ignore[reportPrivateUsage]
@@ -18,8 +19,10 @@ from chalk.features.dataframe._validation import (
18
19
  )
19
20
  from chalk.utils.collections import FrozenOrderedSet, OrderedSet, unwrap_optional
20
21
  from chalk.utils.df_utils import pa_cast, pa_table_to_pl_df
22
+ from chalk.utils.environment_parsing import env_var_bool
21
23
  from chalk.utils.pl_helpers import is_new_polars, pl_is_uniquable_on, str_json_decode_compat
22
24
  from chalk.utils.tracing import safe_trace
25
+ from libchalk.chalktable import Aggregation, WindowFrame, WindowFrameBoundaryType
23
26
 
24
27
  from chalkruntime.constants import (
25
28
  HAS_MANY_INPUT_TS_COL_NAME,
@@ -67,6 +70,214 @@ PKey: TypeAlias = "int | str"
67
70
  RESOLVER_INPUT_IDX_COL_NAME = "___CHALK_RESOLVER_INPUT_IDX_COL_NAME___"
68
71
  GROUP_TEMP_COL_NAME = "___CHALK_LAZY_GROUP_INDICES___"
69
72
 
73
+ _ASOF_TIME_COL_NAME = "___CHALK_HM_ASOF_TIME___"
74
+ _ASOF_IS_INPUT_COL_NAME = "___CHALK_HM_ASOF_IS_INPUT___"
75
+ _ASOF_VERSION_OAT_COL_NAME = "___CHALK_HM_ASOF_VERSION_OAT___"
76
+ _ASOF_MATCHED_OAT_COL_NAME = "___CHALK_HM_ASOF_MATCHED_OAT___"
77
+ _HOT_KEY_INPUTS_COL_NAME = "___CHALK_HM_HOT_KEY_INPUTS___"
78
+ _HOT_KEY_VERSIONS_COL_NAME = "___CHALK_HM_HOT_KEY_VERSIONS___"
79
+ # A join key takes the as-of path once its direct probes (inputs x versions) exceed this many per row
80
+ # (inputs + versions). Measured on the as-of path costing ~5-10x a direct probe per row on uniform data.
81
+ _AS_OF_MIN_PROBES_PER_ROW = 32
82
+
83
+
84
+ def join_inputs_to_versions_and_filter_validity(
85
+ *,
86
+ inputs: chalkdf.DataFrame,
87
+ versions: chalkdf.DataFrame,
88
+ on: dict[str, str],
89
+ input_ts_col: str,
90
+ ) -> chalkdf.DataFrame:
91
+ """
92
+ Joins each input row to every version with its join key and keeps the versions valid at the input's time:
93
+ `__oat__ <= ts < __rat__`, where a null `__oat__` or `__rat__` leaves that side unbounded.
94
+ Probe work is inputs(key) x versions(key), so a key with many versions and many inputs is quadratic.
95
+ """
96
+ joined = inputs.join(
97
+ versions,
98
+ on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
99
+ how="inner",
100
+ probe_with_right_side=True,
101
+ )
102
+ oat = joined.col(OBSERVED_AT_FEATURE_COL_NAME)
103
+ rat = joined.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
104
+ ts = joined.col(input_ts_col)
105
+ return joined.filter((oat.is_null() | (oat <= ts)) & (rat.is_null() | (ts < rat)))
106
+
107
+
108
+ def join_inputs_to_valid_versions(
109
+ *,
110
+ inputs: chalkdf.DataFrame,
111
+ versions: chalkdf.DataFrame,
112
+ on: dict[str, str],
113
+ version_partition_cols: Sequence[str],
114
+ input_ts_col: str,
115
+ ) -> chalkdf.DataFrame:
116
+ """
117
+ Returns the rows of `join_inputs_to_versions_and_filter_validity` for versions whose `__rat__` is the next
118
+ `__oat__` within `version_partition_cols`, ordered by `__oat__` ascending with nulls last.
119
+
120
+ A join key whose inputs x versions is large relative to inputs + versions goes through
121
+ `_join_inputs_to_valid_versions_as_of`, whose work is linear in both; the rest keep the direct join, which is
122
+ cheaper per row. A match needs equal keys, so each input's matches all come from its key's half; the as-of half
123
+ sees only the versions under hot keys, which it handles because `__rat__` still comes from the full history.
124
+ """
125
+ left_keys = list(on.keys())
126
+ right_keys = list(on.values())
127
+ input_counts = inputs.agg(left_keys, _.count().alias(_HOT_KEY_INPUTS_COL_NAME))
128
+ version_counts = versions.agg(right_keys, _.count().alias(_HOT_KEY_VERSIONS_COL_NAME))
129
+ key_counts = input_counts.join(
130
+ version_counts,
131
+ on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
132
+ how="inner",
133
+ )
134
+ n_inputs = key_counts.col(_HOT_KEY_INPUTS_COL_NAME)
135
+ n_versions = key_counts.col(_HOT_KEY_VERSIONS_COL_NAME)
136
+ hot_keys = key_counts.filter(n_inputs * n_versions > (n_inputs + n_versions) * _AS_OF_MIN_PROBES_PER_ROW).select(
137
+ *left_keys
138
+ )
139
+ to_hot_keys = {right_key: left_key for left_key, right_key in on.items()}
140
+
141
+ # Removing the hot keys from the inputs, the build side, is enough: their versions then miss in the probe.
142
+ cold_matches = join_inputs_to_versions_and_filter_validity(
143
+ inputs=inputs.join(hot_keys, on=left_keys, how="anti"),
144
+ versions=versions,
145
+ on=on,
146
+ input_ts_col=input_ts_col,
147
+ )
148
+ hot_matches = _join_inputs_to_valid_versions_as_of(
149
+ inputs=inputs.join(hot_keys, on=left_keys, how="semi"),
150
+ versions=versions.join(hot_keys, on=to_hot_keys, how="semi"), # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
151
+ on=on,
152
+ version_partition_cols=version_partition_cols,
153
+ input_ts_col=input_ts_col,
154
+ )
155
+ return cold_matches.union_all(hot_matches)
156
+
157
+
158
+ def _join_inputs_to_valid_versions_as_of(
159
+ *,
160
+ inputs: chalkdf.DataFrame,
161
+ versions: chalkdf.DataFrame,
162
+ on: dict[str, str],
163
+ version_partition_cols: Sequence[str],
164
+ input_ts_col: str,
165
+ ) -> chalkdf.DataFrame:
166
+ """
167
+ `join_inputs_to_valid_versions` for any subset of the versions that keeps their `__rat__` from the full history.
168
+
169
+ The non-empty `[__oat__, __rat__)` intervals of a partition are disjoint, so at most one timed version of a
170
+ partition is valid at any time. Each input is paired once with each partition that has a version under its join
171
+ key, and a running max over the partition, with inputs sorted after the versions at their time, finds the latest
172
+ version with `__oat__ <= ts`. When the subset lacks the version valid at `ts`, the one found was replaced by it
173
+ at or before `ts`, so the `ts < __rat__` check rejects it. Work is linear in versions plus (input, partition)
174
+ pairs instead of inputs(key) x versions(key), but a window over the union costs several times a hash probe.
175
+ """
176
+ oat = versions.col(OBSERVED_AT_FEATURE_COL_NAME)
177
+ rat = versions.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
178
+ # A version tied on `__oat__` with the next one in its partition has an empty interval and is never valid. The
179
+ # `ts < __rat__` check below would reject it after the lookup; dropping it here keeps it out of the sort.
180
+ versions = versions.filter(oat.is_null() | rat.is_null() | (oat < rat))
181
+
182
+ # Null `__oat__` sorts last, so such a version's `__rat__` is null and it is valid at every time; a null partition
183
+ # value cannot be matched back to its version by an equi-join. Both are rare and keep the direct filter.
184
+ untimed_or_null_partition = functools.reduce(
185
+ lambda acc, col: acc | versions.col(col).is_null(),
186
+ version_partition_cols,
187
+ versions.col(OBSERVED_AT_FEATURE_COL_NAME).is_null(),
188
+ )
189
+ direct_matches = join_inputs_to_versions_and_filter_validity(
190
+ inputs=inputs,
191
+ versions=versions.filter(untimed_or_null_partition),
192
+ on=on,
193
+ input_ts_col=input_ts_col,
194
+ )
195
+ timed = versions.filter(~untimed_or_null_partition)
196
+
197
+ # Partition columns may also be join keys, which a join drops from its right side, so carry them under aliases.
198
+ partition_aliases = {col: f"___CHALK_HM_ASOF_PARTITION_{i}___" for i, col in enumerate(version_partition_cols)}
199
+ pairs = timed.project(
200
+ {right_key: timed.col(right_key) for right_key in on.values()}
201
+ | {alias: timed.col(col) for col, alias in partition_aliases.items()}
202
+ )
203
+ pairs = pairs.distinct_on(*pairs.column_names)
204
+ candidates = inputs.join(
205
+ pairs,
206
+ on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
207
+ how="inner",
208
+ probe_with_right_side=True,
209
+ )
210
+ # A null input time is never at or after a timed version's `__oat__`.
211
+ candidates = candidates.filter(~candidates.col(input_ts_col).is_null())
212
+
213
+ input_ts_type = candidates.schema[input_ts_col]
214
+ oat_type = timed.schema[OBSERVED_AT_FEATURE_COL_NAME]
215
+ timed_oat = timed.col(OBSERVED_AT_FEATURE_COL_NAME)
216
+ input_rows = candidates.with_columns(
217
+ {
218
+ _ASOF_TIME_COL_NAME: candidates.col(input_ts_col),
219
+ _ASOF_IS_INPUT_COL_NAME: chalkdf.Expr.lit(pa.scalar(True)),
220
+ _ASOF_VERSION_OAT_COL_NAME: chalkdf.Expr.lit(pa.scalar(None, oat_type)),
221
+ }
222
+ )
223
+ version_rows = timed.project(
224
+ {
225
+ col: chalkdf.Expr.lit(pa.scalar(None, candidates.schema[col]))
226
+ for col in candidates.column_names
227
+ if col not in partition_aliases.values()
228
+ }
229
+ | {alias: timed.col(col) for col, alias in partition_aliases.items()}
230
+ | {
231
+ _ASOF_TIME_COL_NAME: timed_oat if oat_type == input_ts_type else timed_oat.cast(input_ts_type),
232
+ _ASOF_IS_INPUT_COL_NAME: chalkdf.Expr.lit(pa.scalar(False)),
233
+ _ASOF_VERSION_OAT_COL_NAME: timed_oat,
234
+ }
235
+ )
236
+ # Versions sort before inputs at the same time (False < True), so a version observed exactly at the input's time
237
+ # is visible to it, matching `__oat__ <= ts`.
238
+ sequenced = input_rows.union_all(version_rows).window(
239
+ list(partition_aliases.values()),
240
+ [_ASOF_TIME_COL_NAME, _ASOF_IS_INPUT_COL_NAME],
241
+ chalkdf.WindowExpr.from_aggregation(
242
+ Aggregation(
243
+ aggregation_fn="max",
244
+ input_column_names=[_ASOF_VERSION_OAT_COL_NAME],
245
+ mask_column_name=None,
246
+ output_column_name=_ASOF_MATCHED_OAT_COL_NAME,
247
+ ),
248
+ WindowFrame(
249
+ WindowFrameBoundaryType.UNBOUNDED_PRECEDING,
250
+ None,
251
+ WindowFrameBoundaryType.CURRENT_ROW,
252
+ None,
253
+ ),
254
+ ),
255
+ )
256
+ matched = sequenced.filter(
257
+ sequenced.col(_ASOF_IS_INPUT_COL_NAME) & ~sequenced.col(_ASOF_MATCHED_OAT_COL_NAME).is_null()
258
+ ).drop(_ASOF_TIME_COL_NAME, _ASOF_IS_INPUT_COL_NAME, _ASOF_VERSION_OAT_COL_NAME)
259
+
260
+ timed_matches = matched.join(
261
+ timed,
262
+ on={alias: col for col, alias in partition_aliases.items()}
263
+ | {_ASOF_MATCHED_OAT_COL_NAME: OBSERVED_AT_FEATURE_COL_NAME}, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
264
+ how="inner",
265
+ ).rename(
266
+ {alias: col for col, alias in partition_aliases.items()}
267
+ | {_ASOF_MATCHED_OAT_COL_NAME: OBSERVED_AT_FEATURE_COL_NAME}
268
+ )
269
+ # The pairing only says the partition has some version under the input's key; the valid version may have a
270
+ # different key, in which case the direct join would not have produced it either.
271
+ matched_rat = timed_matches.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
272
+ timed_matches = timed_matches.filter(
273
+ functools.reduce(
274
+ lambda acc, expr: acc & expr,
275
+ [timed_matches.col(left_key) == timed_matches.col(right_key) for left_key, right_key in on.items()],
276
+ matched_rat.is_null() | (timed_matches.col(input_ts_col) < matched_rat),
277
+ )
278
+ ).drop(*on.values())
279
+ return direct_matches.union_all(timed_matches)
280
+
70
281
 
71
282
  class VectorizedHasManySampler(Protocol):
72
283
  def yield_groups_per_row(self) -> Iterable[DataFrame]:
@@ -346,6 +557,9 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
346
557
  # to what is done in offline query, if it was not already done in the offline or inject operator result nodes
347
558
  # The observed at is the TS feature
348
559
  # The replaced observed at should be the TS of the next row for the given pkey
560
+ # Set when `__rat__` is derived below as the next `__oat__` within this partition, which is what lets the join
561
+ # use `join_inputs_to_valid_versions` instead of filtering every version of a key against every input row.
562
+ version_partition_cols: list[str] | None = None
349
563
  if needs_oat_rat_calculations and OBSERVED_AT_FEATURE_COL_NAME not in hm_lf.column_names:
350
564
  over = [foreign_pkey_feature.fqn]
351
565
  if INDEX_COL_NAME in hm_lf.column_names:
@@ -366,6 +580,7 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
366
580
  [foreign_ts_feature.fqn],
367
581
  chalkdf.WindowExpr.shift(foreign_ts_feature.fqn, REPLACED_OBSERVED_AT_FEATURE_COL_NAME, -1), # pyright: ignore[reportCallIssue] # ty:ignore[missing-argument, too-many-positional-arguments]
368
582
  )
583
+ version_partition_cols = over
369
584
  # hm_lf = hm_lf.with_columns(
370
585
  # [
371
586
  # pl.col(foreign_ts_feature.fqn).alias(OBSERVED_AT_FEATURE_COL_NAME),
@@ -438,28 +653,30 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
438
653
 
439
654
  resolver_inputs_lf_renamed = resolver_inputs_lf.rename(inputs_rename_dict) # pyright: ignore[reportArgumentType]
440
655
 
441
- final_resolver_inputs_for_join = resolver_inputs_lf_renamed
442
- final_hm_lf_for_join = hm_lf
443
-
444
- joined_df = final_resolver_inputs_for_join.join(
445
- final_hm_lf_for_join,
446
- on=on_dict, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
447
- how="inner",
448
- probe_with_right_side=True,
449
- )
450
-
451
- if needs_oat_rat_calculations:
452
- hm_oat_before_input_ts_expr = joined_df.col(OBSERVED_AT_FEATURE_COL_NAME).is_null() | (
453
- joined_df.col(OBSERVED_AT_FEATURE_COL_NAME) <= joined_df.col(RESOLVER_INPUT_PREFIX + TS_COL_NAME)
656
+ # Off by default: routing hot keys costs a few extra passes over the inputs and versions even when no join key
657
+ # is hot.
658
+ if version_partition_cols is not None and env_var_bool("CHALK_HAS_MANY_HOT_KEY_AS_OF"):
659
+ joined_df = join_inputs_to_valid_versions(
660
+ inputs=resolver_inputs_lf_renamed,
661
+ versions=hm_lf,
662
+ on=on_dict,
663
+ version_partition_cols=version_partition_cols,
664
+ input_ts_col=RESOLVER_INPUT_PREFIX + TS_COL_NAME,
454
665
  )
455
- hm_rat_after_input_ts = joined_df.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME).is_null() | (
456
- joined_df.col(RESOLVER_INPUT_PREFIX + TS_COL_NAME)
457
- < joined_df.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
666
+ elif needs_oat_rat_calculations:
667
+ joined_df = join_inputs_to_versions_and_filter_validity(
668
+ inputs=resolver_inputs_lf_renamed,
669
+ versions=hm_lf,
670
+ on=on_dict,
671
+ input_ts_col=RESOLVER_INPUT_PREFIX + TS_COL_NAME,
672
+ )
673
+ else:
674
+ joined_df = resolver_inputs_lf_renamed.join(
675
+ hm_lf,
676
+ on=on_dict, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
677
+ how="inner",
678
+ probe_with_right_side=True,
458
679
  )
459
-
460
- ts_consistency_expr = hm_oat_before_input_ts_expr & hm_rat_after_input_ts
461
-
462
- joined_df = joined_df.filter(ts_consistency_expr)
463
680
 
464
681
  # Get the columns we want
465
682
  # for left_join_col, right_join_col in zip(left_join_cols, right_join_cols):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: chalkruntime
3
- Version: 3.43.82
3
+ Version: 3.43.83
4
4
  Summary: Runtime support library for Chalk AI
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -45,7 +45,7 @@ name = "chalkruntime"
45
45
  description = "Runtime support library for Chalk AI"
46
46
  readme = "README.md"
47
47
  requires-python = ">=3.11"
48
- version = "3.43.82"
48
+ version = "3.43.83"
49
49
 
50
50
 
51
51
  [tool.deptry]
@@ -57,7 +57,7 @@ chalk-viztracer = ["viztracer"]
57
57
  DEP001 = [
58
58
  # Sideloaded deps or conditionally imported
59
59
  # Note: chalkpy dep should match shared/pyproject.toml
60
- "chalk==2.166.1",
60
+ "chalk==2.166.2",
61
61
  "dotenv",
62
62
  "langchain>=0.2.0,<0.3",
63
63
  "tenacity>=8.2.1",
File without changes
File without changes
File without changes