chalkruntime 3.43.82__tar.gz → 3.43.84__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/PKG-INFO +1 -1
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/vectorized_hasmany_sampler.py +237 -20
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime.egg-info/PKG-INFO +1 -1
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/pyproject.toml +2 -2
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/README.md +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/constants.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/dataframe/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/dataframe/dataframe.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/dataframe/lazyframe.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/exc/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/exc/failed_argument.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/exc/resolver_errors.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/exc/wrapped_resolver_exception.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/_sql_source_pickle_patch.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/chalk_overload.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/convert_chalkpy_underscore.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/dataframe_resolver_correlation.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/feature.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/feature_alias.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/filter_conversion.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/global_graph.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/graph.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/graph_impl.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/graph_proxy.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/graph_state.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/jinja_parser.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/mat_agg_filter_expression.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/materializations.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/materialized_feature_view.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/maybe_named_collection.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/named_query.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/nearest_neighbor.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/overlay_graph.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/protograph_deserializer.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/protograph_serializer.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/pseudofeatures.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/resolver.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/singletons.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/sklearn_model_parser.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/source_secrets.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/stream_resolver.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/underscore.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/underscore_codec_info.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/underscore_operation_registry.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/variables.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/heaptrack_launcher.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/incrementalization/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/incrementalization/group_incrementalizer.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/incrementalization/incrementalizer.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/batch_result_collector.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/bound_invoker.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/bound_invoker_cache.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/general_bound_invoker.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/no_arg_scalar_invoker.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/one_to_one_invoker.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/overlay_features.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/parse_external_resolver.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/partition_batch.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/query_execution_parameters.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_args_builder.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_input.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_input_upload.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_output_metadata.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_raw_output_parsing.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_result.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_runner.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/sample.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/validator.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/loader/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/loader/converter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/loader/importer.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/memray_launcher.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/metadata.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/py.typed +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/branch_model_endpoint_config.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/config.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/entrypoint.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/env_helper.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/remote_python_function_registry_client.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/server_context_snapshot.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/service.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/subprocess_hook_service.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/composed_rewriter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/contextual_query_rewriter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/filter_injection_site.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/filter_query_rewriter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/identity_rewriter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/query_rewriter.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/query_rewriter_helper.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/sqlglot_import_lock.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/sqlglot_parsing.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sqlalchemy_mysql_keepalive.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/converter_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/exc.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/message_parsing.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/resolver_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/streaming_invoker_kernel/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/streaming_invoker_kernel/local_streaming_invoker_kernel.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/types.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/streaming/window_keys.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/__init__.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/async_helpers.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/contextvars.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/datadog.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/internal_pl_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/posix_file_lock.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/proto_bytes_cache.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/pyarrow_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/tracing.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/utils/viztracer_profiling.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/valgrind_launcher.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime.egg-info/SOURCES.txt +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime.egg-info/dependency_links.txt +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime.egg-info/requires.txt +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime.egg-info/top_level.txt +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/setup.cfg +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/setup.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_branch_model_endpoint_config.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_captured_global_function_refs.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_datadog.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_feature_aliases.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_graph_cache.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_internal_pl_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_mat_agg_filter_expressions.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_mat_agg_projection_expressions.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_materialized_feature_view.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_posix_file_lock.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_pyarrow_utils.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_server_context_snapshot.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_sql_max_row_version_lookback_deserialization.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_sql_retry_policy_deserialization.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_sqlalchemy_mysql_keepalive.py +0 -0
- {chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_subprocess_hook_service.py +0 -0
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/vectorized_hasmany_sampler.py
RENAMED
|
@@ -11,6 +11,7 @@ import polars as pl
|
|
|
11
11
|
import pyarrow as pa
|
|
12
12
|
import pyarrow.compute as pc
|
|
13
13
|
from chalk.features import DataFrame
|
|
14
|
+
from chalk.features.underscore import _
|
|
14
15
|
from chalk.features.dataframe._filters import PolarsFilterConverter
|
|
15
16
|
from chalk.features.dataframe._validation import (
|
|
16
17
|
_generate_empty_series_for_dtype, # pyright: ignore[reportPrivateUsage]
|
|
@@ -18,8 +19,10 @@ from chalk.features.dataframe._validation import (
|
|
|
18
19
|
)
|
|
19
20
|
from chalk.utils.collections import FrozenOrderedSet, OrderedSet, unwrap_optional
|
|
20
21
|
from chalk.utils.df_utils import pa_cast, pa_table_to_pl_df
|
|
22
|
+
from chalk.utils.environment_parsing import env_var_bool
|
|
21
23
|
from chalk.utils.pl_helpers import is_new_polars, pl_is_uniquable_on, str_json_decode_compat
|
|
22
24
|
from chalk.utils.tracing import safe_trace
|
|
25
|
+
from libchalk.chalktable import Aggregation, WindowFrame, WindowFrameBoundaryType
|
|
23
26
|
|
|
24
27
|
from chalkruntime.constants import (
|
|
25
28
|
HAS_MANY_INPUT_TS_COL_NAME,
|
|
@@ -67,6 +70,214 @@ PKey: TypeAlias = "int | str"
|
|
|
67
70
|
RESOLVER_INPUT_IDX_COL_NAME = "___CHALK_RESOLVER_INPUT_IDX_COL_NAME___"
|
|
68
71
|
GROUP_TEMP_COL_NAME = "___CHALK_LAZY_GROUP_INDICES___"
|
|
69
72
|
|
|
73
|
+
_ASOF_TIME_COL_NAME = "___CHALK_HM_ASOF_TIME___"
|
|
74
|
+
_ASOF_IS_INPUT_COL_NAME = "___CHALK_HM_ASOF_IS_INPUT___"
|
|
75
|
+
_ASOF_VERSION_OAT_COL_NAME = "___CHALK_HM_ASOF_VERSION_OAT___"
|
|
76
|
+
_ASOF_MATCHED_OAT_COL_NAME = "___CHALK_HM_ASOF_MATCHED_OAT___"
|
|
77
|
+
_HOT_KEY_INPUTS_COL_NAME = "___CHALK_HM_HOT_KEY_INPUTS___"
|
|
78
|
+
_HOT_KEY_VERSIONS_COL_NAME = "___CHALK_HM_HOT_KEY_VERSIONS___"
|
|
79
|
+
# A join key takes the as-of path once its direct probes (inputs x versions) exceed this many per row
|
|
80
|
+
# (inputs + versions). Measured on the as-of path costing ~5-10x a direct probe per row on uniform data.
|
|
81
|
+
_AS_OF_MIN_PROBES_PER_ROW = 32
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def join_inputs_to_versions_and_filter_validity(
|
|
85
|
+
*,
|
|
86
|
+
inputs: chalkdf.DataFrame,
|
|
87
|
+
versions: chalkdf.DataFrame,
|
|
88
|
+
on: dict[str, str],
|
|
89
|
+
input_ts_col: str,
|
|
90
|
+
) -> chalkdf.DataFrame:
|
|
91
|
+
"""
|
|
92
|
+
Joins each input row to every version with its join key and keeps the versions valid at the input's time:
|
|
93
|
+
`__oat__ <= ts < __rat__`, where a null `__oat__` or `__rat__` leaves that side unbounded.
|
|
94
|
+
Probe work is inputs(key) x versions(key), so a key with many versions and many inputs is quadratic.
|
|
95
|
+
"""
|
|
96
|
+
joined = inputs.join(
|
|
97
|
+
versions,
|
|
98
|
+
on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
99
|
+
how="inner",
|
|
100
|
+
probe_with_right_side=True,
|
|
101
|
+
)
|
|
102
|
+
oat = joined.col(OBSERVED_AT_FEATURE_COL_NAME)
|
|
103
|
+
rat = joined.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
|
|
104
|
+
ts = joined.col(input_ts_col)
|
|
105
|
+
return joined.filter((oat.is_null() | (oat <= ts)) & (rat.is_null() | (ts < rat)))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def join_inputs_to_valid_versions(
|
|
109
|
+
*,
|
|
110
|
+
inputs: chalkdf.DataFrame,
|
|
111
|
+
versions: chalkdf.DataFrame,
|
|
112
|
+
on: dict[str, str],
|
|
113
|
+
version_partition_cols: Sequence[str],
|
|
114
|
+
input_ts_col: str,
|
|
115
|
+
) -> chalkdf.DataFrame:
|
|
116
|
+
"""
|
|
117
|
+
Returns the rows of `join_inputs_to_versions_and_filter_validity` for versions whose `__rat__` is the next
|
|
118
|
+
`__oat__` within `version_partition_cols`, ordered by `__oat__` ascending with nulls last.
|
|
119
|
+
|
|
120
|
+
A join key whose inputs x versions is large relative to inputs + versions goes through
|
|
121
|
+
`_join_inputs_to_valid_versions_as_of`, whose work is linear in both; the rest keep the direct join, which is
|
|
122
|
+
cheaper per row. A match needs equal keys, so each input's matches all come from its key's half; the as-of half
|
|
123
|
+
sees only the versions under hot keys, which it handles because `__rat__` still comes from the full history.
|
|
124
|
+
"""
|
|
125
|
+
left_keys = list(on.keys())
|
|
126
|
+
right_keys = list(on.values())
|
|
127
|
+
input_counts = inputs.agg(left_keys, _.count().alias(_HOT_KEY_INPUTS_COL_NAME))
|
|
128
|
+
version_counts = versions.agg(right_keys, _.count().alias(_HOT_KEY_VERSIONS_COL_NAME))
|
|
129
|
+
key_counts = input_counts.join(
|
|
130
|
+
version_counts,
|
|
131
|
+
on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
132
|
+
how="inner",
|
|
133
|
+
)
|
|
134
|
+
n_inputs = key_counts.col(_HOT_KEY_INPUTS_COL_NAME)
|
|
135
|
+
n_versions = key_counts.col(_HOT_KEY_VERSIONS_COL_NAME)
|
|
136
|
+
hot_keys = key_counts.filter(n_inputs * n_versions > (n_inputs + n_versions) * _AS_OF_MIN_PROBES_PER_ROW).select(
|
|
137
|
+
*left_keys
|
|
138
|
+
)
|
|
139
|
+
to_hot_keys = {right_key: left_key for left_key, right_key in on.items()}
|
|
140
|
+
|
|
141
|
+
# Removing the hot keys from the inputs, the build side, is enough: their versions then miss in the probe.
|
|
142
|
+
cold_matches = join_inputs_to_versions_and_filter_validity(
|
|
143
|
+
inputs=inputs.join(hot_keys, on=left_keys, how="anti"),
|
|
144
|
+
versions=versions,
|
|
145
|
+
on=on,
|
|
146
|
+
input_ts_col=input_ts_col,
|
|
147
|
+
)
|
|
148
|
+
hot_matches = _join_inputs_to_valid_versions_as_of(
|
|
149
|
+
inputs=inputs.join(hot_keys, on=left_keys, how="semi"),
|
|
150
|
+
versions=versions.join(hot_keys, on=to_hot_keys, how="semi"), # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
151
|
+
on=on,
|
|
152
|
+
version_partition_cols=version_partition_cols,
|
|
153
|
+
input_ts_col=input_ts_col,
|
|
154
|
+
)
|
|
155
|
+
return cold_matches.union_all(hot_matches)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _join_inputs_to_valid_versions_as_of(
|
|
159
|
+
*,
|
|
160
|
+
inputs: chalkdf.DataFrame,
|
|
161
|
+
versions: chalkdf.DataFrame,
|
|
162
|
+
on: dict[str, str],
|
|
163
|
+
version_partition_cols: Sequence[str],
|
|
164
|
+
input_ts_col: str,
|
|
165
|
+
) -> chalkdf.DataFrame:
|
|
166
|
+
"""
|
|
167
|
+
`join_inputs_to_valid_versions` for any subset of the versions that keeps their `__rat__` from the full history.
|
|
168
|
+
|
|
169
|
+
The non-empty `[__oat__, __rat__)` intervals of a partition are disjoint, so at most one timed version of a
|
|
170
|
+
partition is valid at any time. Each input is paired once with each partition that has a version under its join
|
|
171
|
+
key, and a running max over the partition, with inputs sorted after the versions at their time, finds the latest
|
|
172
|
+
version with `__oat__ <= ts`. When the subset lacks the version valid at `ts`, the one found was replaced by it
|
|
173
|
+
at or before `ts`, so the `ts < __rat__` check rejects it. Work is linear in versions plus (input, partition)
|
|
174
|
+
pairs instead of inputs(key) x versions(key), but a window over the union costs several times a hash probe.
|
|
175
|
+
"""
|
|
176
|
+
oat = versions.col(OBSERVED_AT_FEATURE_COL_NAME)
|
|
177
|
+
rat = versions.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
|
|
178
|
+
# A version tied on `__oat__` with the next one in its partition has an empty interval and is never valid. The
|
|
179
|
+
# `ts < __rat__` check below would reject it after the lookup; dropping it here keeps it out of the sort.
|
|
180
|
+
versions = versions.filter(oat.is_null() | rat.is_null() | (oat < rat))
|
|
181
|
+
|
|
182
|
+
# Null `__oat__` sorts last, so such a version's `__rat__` is null and it is valid at every time; a null partition
|
|
183
|
+
# value cannot be matched back to its version by an equi-join. Both are rare and keep the direct filter.
|
|
184
|
+
untimed_or_null_partition = functools.reduce(
|
|
185
|
+
lambda acc, col: acc | versions.col(col).is_null(),
|
|
186
|
+
version_partition_cols,
|
|
187
|
+
versions.col(OBSERVED_AT_FEATURE_COL_NAME).is_null(),
|
|
188
|
+
)
|
|
189
|
+
direct_matches = join_inputs_to_versions_and_filter_validity(
|
|
190
|
+
inputs=inputs,
|
|
191
|
+
versions=versions.filter(untimed_or_null_partition),
|
|
192
|
+
on=on,
|
|
193
|
+
input_ts_col=input_ts_col,
|
|
194
|
+
)
|
|
195
|
+
timed = versions.filter(~untimed_or_null_partition)
|
|
196
|
+
|
|
197
|
+
# Partition columns may also be join keys, which a join drops from its right side, so carry them under aliases.
|
|
198
|
+
partition_aliases = {col: f"___CHALK_HM_ASOF_PARTITION_{i}___" for i, col in enumerate(version_partition_cols)}
|
|
199
|
+
pairs = timed.project(
|
|
200
|
+
{right_key: timed.col(right_key) for right_key in on.values()}
|
|
201
|
+
| {alias: timed.col(col) for col, alias in partition_aliases.items()}
|
|
202
|
+
)
|
|
203
|
+
pairs = pairs.distinct_on(*pairs.column_names)
|
|
204
|
+
candidates = inputs.join(
|
|
205
|
+
pairs,
|
|
206
|
+
on=on, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
207
|
+
how="inner",
|
|
208
|
+
probe_with_right_side=True,
|
|
209
|
+
)
|
|
210
|
+
# A null input time is never at or after a timed version's `__oat__`.
|
|
211
|
+
candidates = candidates.filter(~candidates.col(input_ts_col).is_null())
|
|
212
|
+
|
|
213
|
+
input_ts_type = candidates.schema[input_ts_col]
|
|
214
|
+
oat_type = timed.schema[OBSERVED_AT_FEATURE_COL_NAME]
|
|
215
|
+
timed_oat = timed.col(OBSERVED_AT_FEATURE_COL_NAME)
|
|
216
|
+
input_rows = candidates.with_columns(
|
|
217
|
+
{
|
|
218
|
+
_ASOF_TIME_COL_NAME: candidates.col(input_ts_col),
|
|
219
|
+
_ASOF_IS_INPUT_COL_NAME: chalkdf.Expr.lit(pa.scalar(True)),
|
|
220
|
+
_ASOF_VERSION_OAT_COL_NAME: chalkdf.Expr.lit(pa.scalar(None, oat_type)),
|
|
221
|
+
}
|
|
222
|
+
)
|
|
223
|
+
version_rows = timed.project(
|
|
224
|
+
{
|
|
225
|
+
col: chalkdf.Expr.lit(pa.scalar(None, candidates.schema[col]))
|
|
226
|
+
for col in candidates.column_names
|
|
227
|
+
if col not in partition_aliases.values()
|
|
228
|
+
}
|
|
229
|
+
| {alias: timed.col(col) for col, alias in partition_aliases.items()}
|
|
230
|
+
| {
|
|
231
|
+
_ASOF_TIME_COL_NAME: timed_oat if oat_type == input_ts_type else timed_oat.cast(input_ts_type),
|
|
232
|
+
_ASOF_IS_INPUT_COL_NAME: chalkdf.Expr.lit(pa.scalar(False)),
|
|
233
|
+
_ASOF_VERSION_OAT_COL_NAME: timed_oat,
|
|
234
|
+
}
|
|
235
|
+
)
|
|
236
|
+
# Versions sort before inputs at the same time (False < True), so a version observed exactly at the input's time
|
|
237
|
+
# is visible to it, matching `__oat__ <= ts`.
|
|
238
|
+
sequenced = input_rows.union_all(version_rows).window(
|
|
239
|
+
list(partition_aliases.values()),
|
|
240
|
+
[_ASOF_TIME_COL_NAME, _ASOF_IS_INPUT_COL_NAME],
|
|
241
|
+
chalkdf.WindowExpr.from_aggregation(
|
|
242
|
+
Aggregation(
|
|
243
|
+
aggregation_fn="max",
|
|
244
|
+
input_column_names=[_ASOF_VERSION_OAT_COL_NAME],
|
|
245
|
+
mask_column_name=None,
|
|
246
|
+
output_column_name=_ASOF_MATCHED_OAT_COL_NAME,
|
|
247
|
+
),
|
|
248
|
+
WindowFrame(
|
|
249
|
+
WindowFrameBoundaryType.UNBOUNDED_PRECEDING,
|
|
250
|
+
None,
|
|
251
|
+
WindowFrameBoundaryType.CURRENT_ROW,
|
|
252
|
+
None,
|
|
253
|
+
),
|
|
254
|
+
),
|
|
255
|
+
)
|
|
256
|
+
matched = sequenced.filter(
|
|
257
|
+
sequenced.col(_ASOF_IS_INPUT_COL_NAME) & ~sequenced.col(_ASOF_MATCHED_OAT_COL_NAME).is_null()
|
|
258
|
+
).drop(_ASOF_TIME_COL_NAME, _ASOF_IS_INPUT_COL_NAME, _ASOF_VERSION_OAT_COL_NAME)
|
|
259
|
+
|
|
260
|
+
timed_matches = matched.join(
|
|
261
|
+
timed,
|
|
262
|
+
on={alias: col for col, alias in partition_aliases.items()}
|
|
263
|
+
| {_ASOF_MATCHED_OAT_COL_NAME: OBSERVED_AT_FEATURE_COL_NAME}, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
264
|
+
how="inner",
|
|
265
|
+
).rename(
|
|
266
|
+
{alias: col for col, alias in partition_aliases.items()}
|
|
267
|
+
| {_ASOF_MATCHED_OAT_COL_NAME: OBSERVED_AT_FEATURE_COL_NAME}
|
|
268
|
+
)
|
|
269
|
+
# The pairing only says the partition has some version under the input's key; the valid version may have a
|
|
270
|
+
# different key, in which case the direct join would not have produced it either.
|
|
271
|
+
matched_rat = timed_matches.col(REPLACED_OBSERVED_AT_FEATURE_COL_NAME)
|
|
272
|
+
timed_matches = timed_matches.filter(
|
|
273
|
+
functools.reduce(
|
|
274
|
+
lambda acc, expr: acc & expr,
|
|
275
|
+
[timed_matches.col(left_key) == timed_matches.col(right_key) for left_key, right_key in on.items()],
|
|
276
|
+
matched_rat.is_null() | (timed_matches.col(input_ts_col) < matched_rat),
|
|
277
|
+
)
|
|
278
|
+
).drop(*on.values())
|
|
279
|
+
return direct_matches.union_all(timed_matches)
|
|
280
|
+
|
|
70
281
|
|
|
71
282
|
class VectorizedHasManySampler(Protocol):
|
|
72
283
|
def yield_groups_per_row(self) -> Iterable[DataFrame]:
|
|
@@ -346,6 +557,9 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
|
|
|
346
557
|
# to what is done in offline query, if it was not already done in the offline or inject operator result nodes
|
|
347
558
|
# The observed at is the TS feature
|
|
348
559
|
# The replaced observed at should be the TS of the next row for the given pkey
|
|
560
|
+
# Set when `__rat__` is derived below as the next `__oat__` within this partition, which is what lets the join
|
|
561
|
+
# use `join_inputs_to_valid_versions` instead of filtering every version of a key against every input row.
|
|
562
|
+
version_partition_cols: list[str] | None = None
|
|
349
563
|
if needs_oat_rat_calculations and OBSERVED_AT_FEATURE_COL_NAME not in hm_lf.column_names:
|
|
350
564
|
over = [foreign_pkey_feature.fqn]
|
|
351
565
|
if INDEX_COL_NAME in hm_lf.column_names:
|
|
@@ -366,6 +580,7 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
|
|
|
366
580
|
[foreign_ts_feature.fqn],
|
|
367
581
|
chalkdf.WindowExpr.shift(foreign_ts_feature.fqn, REPLACED_OBSERVED_AT_FEATURE_COL_NAME, -1), # pyright: ignore[reportCallIssue] # ty:ignore[missing-argument, too-many-positional-arguments]
|
|
368
582
|
)
|
|
583
|
+
version_partition_cols = over
|
|
369
584
|
# hm_lf = hm_lf.with_columns(
|
|
370
585
|
# [
|
|
371
586
|
# pl.col(foreign_ts_feature.fqn).alias(OBSERVED_AT_FEATURE_COL_NAME),
|
|
@@ -438,28 +653,30 @@ class ChalkdfVectorizedHasManySampler(VectorizedHasManySampler):
|
|
|
438
653
|
|
|
439
654
|
resolver_inputs_lf_renamed = resolver_inputs_lf.rename(inputs_rename_dict) # pyright: ignore[reportArgumentType]
|
|
440
655
|
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
if needs_oat_rat_calculations:
|
|
452
|
-
hm_oat_before_input_ts_expr = joined_df.col(OBSERVED_AT_FEATURE_COL_NAME).is_null() | (
|
|
453
|
-
joined_df.col(OBSERVED_AT_FEATURE_COL_NAME) <= joined_df.col(RESOLVER_INPUT_PREFIX + TS_COL_NAME)
|
|
656
|
+
# Off by default: routing hot keys costs a few extra passes over the inputs and versions even when no join key
|
|
657
|
+
# is hot.
|
|
658
|
+
if version_partition_cols is not None and env_var_bool("CHALK_HAS_MANY_HOT_KEY_AS_OF"):
|
|
659
|
+
joined_df = join_inputs_to_valid_versions(
|
|
660
|
+
inputs=resolver_inputs_lf_renamed,
|
|
661
|
+
versions=hm_lf,
|
|
662
|
+
on=on_dict,
|
|
663
|
+
version_partition_cols=version_partition_cols,
|
|
664
|
+
input_ts_col=RESOLVER_INPUT_PREFIX + TS_COL_NAME,
|
|
454
665
|
)
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
666
|
+
elif needs_oat_rat_calculations:
|
|
667
|
+
joined_df = join_inputs_to_versions_and_filter_validity(
|
|
668
|
+
inputs=resolver_inputs_lf_renamed,
|
|
669
|
+
versions=hm_lf,
|
|
670
|
+
on=on_dict,
|
|
671
|
+
input_ts_col=RESOLVER_INPUT_PREFIX + TS_COL_NAME,
|
|
672
|
+
)
|
|
673
|
+
else:
|
|
674
|
+
joined_df = resolver_inputs_lf_renamed.join(
|
|
675
|
+
hm_lf,
|
|
676
|
+
on=on_dict, # pyright: ignore[reportArgumentType] # ty:ignore[invalid-argument-type]
|
|
677
|
+
how="inner",
|
|
678
|
+
probe_with_right_side=True,
|
|
458
679
|
)
|
|
459
|
-
|
|
460
|
-
ts_consistency_expr = hm_oat_before_input_ts_expr & hm_rat_after_input_ts
|
|
461
|
-
|
|
462
|
-
joined_df = joined_df.filter(ts_consistency_expr)
|
|
463
680
|
|
|
464
681
|
# Get the columns we want
|
|
465
682
|
# for left_join_col, right_join_col in zip(left_join_cols, right_join_cols):
|
|
@@ -45,7 +45,7 @@ name = "chalkruntime"
|
|
|
45
45
|
description = "Runtime support library for Chalk AI"
|
|
46
46
|
readme = "README.md"
|
|
47
47
|
requires-python = ">=3.11"
|
|
48
|
-
version = "3.43.
|
|
48
|
+
version = "3.43.84"
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
[tool.deptry]
|
|
@@ -57,7 +57,7 @@ chalk-viztracer = ["viztracer"]
|
|
|
57
57
|
DEP001 = [
|
|
58
58
|
# Sideloaded deps or conditionally imported
|
|
59
59
|
# Note: chalkpy dep should match shared/pyproject.toml
|
|
60
|
-
"chalk==2.166.
|
|
60
|
+
"chalk==2.166.2",
|
|
61
61
|
"dotenv",
|
|
62
62
|
"langchain>=0.2.0,<0.3",
|
|
63
63
|
"tenacity>=8.2.1",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/exc/wrapped_resolver_exception.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/_sql_source_pickle_patch.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/convert_chalkpy_underscore.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/dataframe_resolver_correlation.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/mat_agg_filter_expression.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/materialized_feature_view.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/graph/underscore_operation_registry.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/incrementalization/incrementalizer.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/batch_result_collector.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/parse_external_resolver.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/query_execution_parameters.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_output_metadata.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/invoker/resolver_raw_output_parsing.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/branch_model_endpoint_config.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/server_context_snapshot.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/server/subprocess_hook_service.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/composed_rewriter.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/contextual_query_rewriter.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/filter_injection_site.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/filter_query_rewriter.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/identity_rewriter.py
RENAMED
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/query_rewriter_helper.py
RENAMED
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/chalkruntime/sql_rewriter/sqlglot_import_lock.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{chalkruntime-3.43.82 → chalkruntime-3.43.84}/tests/test_sql_retry_policy_deserialization.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|