mloda-community-offset 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mloda/community/feature_groups/data_operations/row_preserving/offset/__init__.py +0 -0
- mloda/community/feature_groups/data_operations/row_preserving/offset/base.py +216 -0
- mloda/community/feature_groups/data_operations/row_preserving/offset/duckdb_offset.py +99 -0
- mloda/community/feature_groups/data_operations/row_preserving/offset/pandas_offset.py +66 -0
- mloda/community/feature_groups/data_operations/row_preserving/offset/polars_lazy_offset.py +70 -0
- mloda/community/feature_groups/data_operations/row_preserving/offset/sqlite_offset.py +104 -0
- mloda_community_offset-0.3.1.dist-info/METADATA +24 -0
- mloda_community_offset-0.3.1.dist-info/RECORD +10 -0
- mloda_community_offset-0.3.1.dist-info/WHEEL +5 -0
- mloda_community_offset-0.3.1.dist-info/top_level.txt +1 -0
|
File without changes
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Base class for offset feature groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from mloda.core.abstract_plugins.components.feature import Feature
|
|
8
|
+
from mloda.core.abstract_plugins.components.feature_chainer.feature_chain_parser import FeatureChainParser
|
|
9
|
+
from mloda.core.abstract_plugins.components.feature_chainer.feature_chain_parser_mixin import FeatureChainParserMixin
|
|
10
|
+
from mloda.core.abstract_plugins.components.feature_set import FeatureSet
|
|
11
|
+
from mloda.provider import DefaultOptionKeys, FeatureGroup
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class OffsetFeatureGroup(FeatureChainParserMixin, FeatureGroup):
|
|
15
|
+
"""Base class for offset operations that preserve row count.
|
|
16
|
+
|
|
17
|
+
Offset operations access values at a fixed offset from the current row
|
|
18
|
+
within an ordered partition. The output always has the same number of
|
|
19
|
+
rows as the input.
|
|
20
|
+
|
|
21
|
+
## Supported Offset Types
|
|
22
|
+
|
|
23
|
+
- ``lag_N``: Value N rows before the current row
|
|
24
|
+
- ``lead_N``: Value N rows after the current row
|
|
25
|
+
- ``diff_N``: Difference between current value and value N rows before
|
|
26
|
+
- ``pct_change_N``: Percentage change from value N rows before
|
|
27
|
+
- ``first_value``: First value in the partition
|
|
28
|
+
- ``last_value``: Last value in the partition
|
|
29
|
+
|
|
30
|
+
## Feature Creation Methods
|
|
31
|
+
|
|
32
|
+
### 1. String-Based Creation
|
|
33
|
+
|
|
34
|
+
Features follow the naming pattern: ``{source_column}__{offset_type}_offset``
|
|
35
|
+
|
|
36
|
+
Examples::
|
|
37
|
+
|
|
38
|
+
features = [
|
|
39
|
+
Feature("sales__lag_1_offset", options=Options(context={
|
|
40
|
+
"partition_by": ["region"], "order_by": "timestamp",
|
|
41
|
+
})),
|
|
42
|
+
Feature("price__diff_1_offset", options=Options(context={
|
|
43
|
+
"partition_by": ["category"], "order_by": "date",
|
|
44
|
+
})),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
### 2. Configuration-Based Creation
|
|
48
|
+
|
|
49
|
+
Uses Options with proper context parameter separation::
|
|
50
|
+
|
|
51
|
+
feature = Feature(
|
|
52
|
+
name="my_lag",
|
|
53
|
+
options=Options(
|
|
54
|
+
context={
|
|
55
|
+
"offset_type": "lag_1",
|
|
56
|
+
"in_features": "sales",
|
|
57
|
+
"partition_by": ["region"],
|
|
58
|
+
"order_by": "timestamp",
|
|
59
|
+
}
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
## Null Behavior
|
|
64
|
+
|
|
65
|
+
Offset positions that fall outside the partition boundary produce null.
|
|
66
|
+
For example, ``lag_1`` on the first row of a partition returns null.
|
|
67
|
+
``first_value`` and ``last_value`` ignore null source values; if all
|
|
68
|
+
values in a partition are null, the result is null.
|
|
69
|
+
|
|
70
|
+
## Parameter Classification
|
|
71
|
+
|
|
72
|
+
### Context Parameters
|
|
73
|
+
- ``offset_type``: The type of offset to perform
|
|
74
|
+
- ``in_features``: The source feature
|
|
75
|
+
- ``partition_by``: List of columns to partition by
|
|
76
|
+
- ``order_by``: Column to order by within each partition
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
PREFIX_PATTERN = r".*__([\w]+)_offset$"
|
|
80
|
+
|
|
81
|
+
MIN_IN_FEATURES = 1
|
|
82
|
+
MAX_IN_FEATURES = 1
|
|
83
|
+
|
|
84
|
+
OFFSET_TYPE = "offset_type"
|
|
85
|
+
PARTITION_BY = "partition_by"
|
|
86
|
+
ORDER_BY = "order_by"
|
|
87
|
+
|
|
88
|
+
OFFSET_TYPES = {
|
|
89
|
+
"first_value": "First value in partition",
|
|
90
|
+
"last_value": "Last value in partition",
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
PROPERTY_MAPPING = {
|
|
94
|
+
OFFSET_TYPE: {
|
|
95
|
+
**OFFSET_TYPES,
|
|
96
|
+
DefaultOptionKeys.context: True,
|
|
97
|
+
DefaultOptionKeys.strict_validation: False,
|
|
98
|
+
DefaultOptionKeys.type_validator: lambda v: isinstance(v, str),
|
|
99
|
+
},
|
|
100
|
+
DefaultOptionKeys.in_features: {
|
|
101
|
+
"explanation": "Source feature for offset operation",
|
|
102
|
+
DefaultOptionKeys.context: True,
|
|
103
|
+
DefaultOptionKeys.strict_validation: False,
|
|
104
|
+
},
|
|
105
|
+
PARTITION_BY: {
|
|
106
|
+
"explanation": "List of columns to partition by",
|
|
107
|
+
DefaultOptionKeys.context: True,
|
|
108
|
+
DefaultOptionKeys.strict_validation: False,
|
|
109
|
+
},
|
|
110
|
+
ORDER_BY: {
|
|
111
|
+
"explanation": "Column to order by within each partition",
|
|
112
|
+
DefaultOptionKeys.context: True,
|
|
113
|
+
DefaultOptionKeys.strict_validation: False,
|
|
114
|
+
},
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
@classmethod
|
|
118
|
+
def _supports_offset_type(cls, offset_type: str) -> bool:
|
|
119
|
+
"""Check if the given offset type is supported."""
|
|
120
|
+
if offset_type in cls.OFFSET_TYPES:
|
|
121
|
+
return True
|
|
122
|
+
for prefix in ("lag_", "lead_", "diff_", "pct_change_"):
|
|
123
|
+
if offset_type.startswith(prefix):
|
|
124
|
+
suffix = offset_type[len(prefix) :]
|
|
125
|
+
if suffix.isdigit() and int(suffix) >= 1:
|
|
126
|
+
return True
|
|
127
|
+
return False
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def _validate_string_match(cls, feature_name: str, operation_config: str, source_feature: str) -> bool:
|
|
131
|
+
"""Validate that the offset type is supported."""
|
|
132
|
+
return cls._supports_offset_type(operation_config)
|
|
133
|
+
|
|
134
|
+
@classmethod
|
|
135
|
+
def match_feature_group_criteria(
|
|
136
|
+
cls,
|
|
137
|
+
feature_name: Any,
|
|
138
|
+
options: Any,
|
|
139
|
+
_data_access_collection: Any = None,
|
|
140
|
+
) -> bool:
|
|
141
|
+
"""Extend mixin matching with partition_by, order_by, and in_features validation."""
|
|
142
|
+
if not super().match_feature_group_criteria(feature_name, options, _data_access_collection):
|
|
143
|
+
return False
|
|
144
|
+
|
|
145
|
+
partition_by = options.get(cls.PARTITION_BY)
|
|
146
|
+
if not isinstance(partition_by, (list, tuple)):
|
|
147
|
+
return False
|
|
148
|
+
if not all(isinstance(item, str) for item in partition_by):
|
|
149
|
+
return False
|
|
150
|
+
|
|
151
|
+
order_by = options.get(cls.ORDER_BY)
|
|
152
|
+
if order_by is None:
|
|
153
|
+
return False
|
|
154
|
+
if not isinstance(order_by, str):
|
|
155
|
+
return False
|
|
156
|
+
|
|
157
|
+
in_features_raw = options.get(DefaultOptionKeys.in_features)
|
|
158
|
+
if in_features_raw is not None:
|
|
159
|
+
in_features = options.get_in_features()
|
|
160
|
+
if len(in_features) > cls.MAX_IN_FEATURES:
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
return True
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def get_offset_type(cls, feature_name: str) -> str:
|
|
167
|
+
"""Extract the offset type from a feature name string."""
|
|
168
|
+
prefix_patterns = cls._get_prefix_patterns()
|
|
169
|
+
operation_config, _ = FeatureChainParser.parse_feature_name(feature_name, prefix_patterns)
|
|
170
|
+
if operation_config is not None:
|
|
171
|
+
return operation_config
|
|
172
|
+
raise ValueError(f"Could not extract offset type from feature name: {feature_name}")
|
|
173
|
+
|
|
174
|
+
@classmethod
|
|
175
|
+
def _extract_offset_type(cls, feature: Feature) -> str:
|
|
176
|
+
"""Extract offset type from feature (string-based or config-based)."""
|
|
177
|
+
feature_name = feature.name
|
|
178
|
+
prefix_patterns = cls._get_prefix_patterns()
|
|
179
|
+
operation_config, _ = FeatureChainParser.parse_feature_name(feature_name, prefix_patterns)
|
|
180
|
+
if operation_config is not None:
|
|
181
|
+
return operation_config
|
|
182
|
+
offset_type = feature.options.get(cls.OFFSET_TYPE)
|
|
183
|
+
if offset_type is None:
|
|
184
|
+
raise ValueError(f"Could not extract offset type for {feature_name}")
|
|
185
|
+
return str(offset_type)
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def calculate_feature(cls, data: Any, features: FeatureSet) -> Any:
|
|
189
|
+
"""Shared loop: extract params from each feature, delegate to _compute_offset."""
|
|
190
|
+
table = data
|
|
191
|
+
|
|
192
|
+
for feature in features.features:
|
|
193
|
+
feature_name = feature.name
|
|
194
|
+
|
|
195
|
+
source_features = cls._extract_source_features(feature)
|
|
196
|
+
source_col = source_features[0]
|
|
197
|
+
offset_type = cls._extract_offset_type(feature)
|
|
198
|
+
partition_by = feature.options.get(cls.PARTITION_BY)
|
|
199
|
+
order_by = feature.options.get(cls.ORDER_BY)
|
|
200
|
+
|
|
201
|
+
table = cls._compute_offset(table, feature_name, source_col, partition_by, order_by, offset_type)
|
|
202
|
+
|
|
203
|
+
return table
|
|
204
|
+
|
|
205
|
+
@classmethod
|
|
206
|
+
def _compute_offset(
|
|
207
|
+
cls,
|
|
208
|
+
data: Any,
|
|
209
|
+
feature_name: str,
|
|
210
|
+
source_col: str,
|
|
211
|
+
partition_by: list[str],
|
|
212
|
+
order_by: str,
|
|
213
|
+
offset_type: str,
|
|
214
|
+
) -> Any:
|
|
215
|
+
"""Subclasses must implement the actual offset computation."""
|
|
216
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""DuckDB implementation for offset feature groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from mloda.provider import ComputeFramework
|
|
6
|
+
from mloda_plugins.compute_framework.base_implementations.duckdb.duckdb_framework import DuckDBFramework
|
|
7
|
+
from mloda_plugins.compute_framework.base_implementations.duckdb.duckdb_relation import DuckdbRelation
|
|
8
|
+
from mloda_plugins.compute_framework.base_implementations.sql.sql_utils import quote_ident
|
|
9
|
+
|
|
10
|
+
from mloda.community.feature_groups.data_operations.reserved_columns import assert_no_reserved_columns
|
|
11
|
+
from mloda.community.feature_groups.data_operations.row_preserving.offset.base import (
|
|
12
|
+
OffsetFeatureGroup,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
_RN_COL = "__mloda_orig_rn__"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class DuckdbOffset(OffsetFeatureGroup):
|
|
19
|
+
@classmethod
|
|
20
|
+
def compute_framework_rule(cls) -> set[type[ComputeFramework]] | None:
|
|
21
|
+
return {DuckDBFramework}
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def _compute_offset(
|
|
25
|
+
cls,
|
|
26
|
+
data: DuckdbRelation,
|
|
27
|
+
feature_name: str,
|
|
28
|
+
source_col: str,
|
|
29
|
+
partition_by: list[str],
|
|
30
|
+
order_by: str,
|
|
31
|
+
offset_type: str,
|
|
32
|
+
) -> DuckdbRelation:
|
|
33
|
+
assert_no_reserved_columns(data.columns, framework="DuckDB", operation="offset")
|
|
34
|
+
|
|
35
|
+
quoted_source = quote_ident(source_col)
|
|
36
|
+
quoted_order = quote_ident(order_by)
|
|
37
|
+
quoted_feature = quote_ident(feature_name)
|
|
38
|
+
partition_clause = ", ".join(quote_ident(col) for col in partition_by)
|
|
39
|
+
|
|
40
|
+
window_clause = f"PARTITION BY {partition_clause} ORDER BY {quoted_order} ASC NULLS LAST"
|
|
41
|
+
|
|
42
|
+
if offset_type.startswith("lag_"):
|
|
43
|
+
offset_n = int(offset_type[len("lag_") :])
|
|
44
|
+
offset_expr = f"LAG({quoted_source}, {offset_n})"
|
|
45
|
+
elif offset_type.startswith("lead_"):
|
|
46
|
+
offset_n = int(offset_type[len("lead_") :])
|
|
47
|
+
offset_expr = f"LEAD({quoted_source}, {offset_n})"
|
|
48
|
+
elif offset_type.startswith("diff_"):
|
|
49
|
+
offset_n = int(offset_type[len("diff_") :])
|
|
50
|
+
offset_expr = f"{quoted_source} - LAG({quoted_source}, {offset_n})"
|
|
51
|
+
elif offset_type.startswith("pct_change_"):
|
|
52
|
+
offset_n = int(offset_type[len("pct_change_") :])
|
|
53
|
+
prev = f"LAG({quoted_source}, {offset_n}) OVER ({window_clause})"
|
|
54
|
+
offset_expr = (
|
|
55
|
+
f"CASE WHEN {prev} IS NOT NULL AND {prev} != 0 "
|
|
56
|
+
f"THEN ({quoted_source} - {prev}) / CAST({prev} AS DOUBLE) END"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
qrn = quote_ident(_RN_COL)
|
|
60
|
+
sql = (
|
|
61
|
+
f"SELECT *, " # nosec
|
|
62
|
+
f"{offset_expr} AS {quoted_feature}, "
|
|
63
|
+
f"ROW_NUMBER() OVER () AS {qrn} "
|
|
64
|
+
f"FROM __t ORDER BY {qrn}"
|
|
65
|
+
)
|
|
66
|
+
new_rel = data._relation.query("__t", sql)
|
|
67
|
+
result_rel = new_rel.project(
|
|
68
|
+
", ".join(quote_ident(c) for c in [col for col in new_rel.columns if col != _RN_COL])
|
|
69
|
+
)
|
|
70
|
+
return DuckdbRelation(data.connection, result_rel)
|
|
71
|
+
elif offset_type == "first_value":
|
|
72
|
+
offset_expr = f"FIRST_VALUE({quoted_source} IGNORE NULLS)"
|
|
73
|
+
# PyArrow parity: the reference scans the entire partition for
|
|
74
|
+
# first/last non-null. DuckDB default ordered-window frame
|
|
75
|
+
# (UNBOUNDED PRECEDING to CURRENT ROW) would make LAST_VALUE
|
|
76
|
+
# return the current row. Explicit UNBOUNDED FOLLOWING gives
|
|
77
|
+
# partition-wide visibility.
|
|
78
|
+
window_clause += " ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
|
|
79
|
+
elif offset_type == "last_value":
|
|
80
|
+
offset_expr = f"LAST_VALUE({quoted_source} IGNORE NULLS)"
|
|
81
|
+
window_clause += " ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING"
|
|
82
|
+
else:
|
|
83
|
+
raise ValueError(f"Unsupported offset type for DuckDB: {offset_type}")
|
|
84
|
+
|
|
85
|
+
# PyArrow parity: the reference returns results in original row
|
|
86
|
+
# order. DuckDB window functions with ORDER BY reorder rows; tag
|
|
87
|
+
# positions with ROW_NUMBER() and ORDER BY qrn to restore.
|
|
88
|
+
qrn = quote_ident(_RN_COL)
|
|
89
|
+
sql = (
|
|
90
|
+
f"SELECT *, " # nosec
|
|
91
|
+
f"{offset_expr} OVER ({window_clause}) AS {quoted_feature}, "
|
|
92
|
+
f"ROW_NUMBER() OVER () AS {qrn} "
|
|
93
|
+
f"FROM __t ORDER BY {qrn}"
|
|
94
|
+
)
|
|
95
|
+
new_rel = data._relation.query("__t", sql)
|
|
96
|
+
result_rel = new_rel.project(
|
|
97
|
+
", ".join(quote_ident(c) for c in [col for col in new_rel.columns if col != _RN_COL])
|
|
98
|
+
)
|
|
99
|
+
return DuckdbRelation(data.connection, result_rel)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Pandas implementation for offset feature groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from mloda.provider import ComputeFramework
|
|
8
|
+
from mloda_plugins.compute_framework.base_implementations.pandas.dataframe import PandasDataFrame
|
|
9
|
+
|
|
10
|
+
from mloda.community.feature_groups.data_operations.reserved_columns import assert_no_reserved_columns
|
|
11
|
+
from mloda.community.feature_groups.data_operations.row_preserving.offset.base import (
|
|
12
|
+
OffsetFeatureGroup,
|
|
13
|
+
)
|
|
14
|
+
from mloda.community.feature_groups.data_operations.pandas_helpers import null_safe_groupby
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PandasOffset(OffsetFeatureGroup):
|
|
18
|
+
@classmethod
|
|
19
|
+
def compute_framework_rule(cls) -> set[type[ComputeFramework]] | None:
|
|
20
|
+
return {PandasDataFrame}
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def _compute_offset(
|
|
24
|
+
cls,
|
|
25
|
+
data: pd.DataFrame,
|
|
26
|
+
feature_name: str,
|
|
27
|
+
source_col: str,
|
|
28
|
+
partition_by: list[str],
|
|
29
|
+
order_by: str,
|
|
30
|
+
offset_type: str,
|
|
31
|
+
) -> pd.DataFrame:
|
|
32
|
+
assert_no_reserved_columns(data.columns, framework="Pandas", operation="offset")
|
|
33
|
+
|
|
34
|
+
data = data.copy()
|
|
35
|
+
|
|
36
|
+
# Sort by partition + order_by (nulls last) to ensure correct offset
|
|
37
|
+
null_sort = data[order_by].isna().astype(int)
|
|
38
|
+
data["__mloda_null_sort"] = null_sort
|
|
39
|
+
data = data.sort_values(partition_by + ["__mloda_null_sort", order_by])
|
|
40
|
+
|
|
41
|
+
grouped = null_safe_groupby(data, partition_by, source_col)
|
|
42
|
+
|
|
43
|
+
if offset_type.startswith("lag_"):
|
|
44
|
+
offset_n = int(offset_type[len("lag_") :])
|
|
45
|
+
data[feature_name] = grouped.shift(offset_n)
|
|
46
|
+
elif offset_type.startswith("lead_"):
|
|
47
|
+
offset_n = int(offset_type[len("lead_") :])
|
|
48
|
+
data[feature_name] = grouped.shift(-offset_n)
|
|
49
|
+
elif offset_type.startswith("diff_"):
|
|
50
|
+
offset_n = int(offset_type[len("diff_") :])
|
|
51
|
+
data[feature_name] = data[source_col] - grouped.shift(offset_n)
|
|
52
|
+
elif offset_type.startswith("pct_change_"):
|
|
53
|
+
offset_n = int(offset_type[len("pct_change_") :])
|
|
54
|
+
prev = grouped.shift(offset_n)
|
|
55
|
+
data[feature_name] = (data[source_col] - prev) / prev.replace(0, float("nan"))
|
|
56
|
+
elif offset_type == "first_value":
|
|
57
|
+
data[feature_name] = grouped.transform(lambda x: x.dropna().iloc[0] if x.dropna().any() else None)
|
|
58
|
+
elif offset_type == "last_value":
|
|
59
|
+
data[feature_name] = grouped.transform(lambda x: x.dropna().iloc[-1] if x.dropna().any() else None)
|
|
60
|
+
else:
|
|
61
|
+
raise ValueError(f"Unsupported offset type: {offset_type}")
|
|
62
|
+
|
|
63
|
+
data = data.drop(columns=["__mloda_null_sort"])
|
|
64
|
+
# Restore original row order
|
|
65
|
+
data = data.sort_index()
|
|
66
|
+
return data
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Polars Lazy implementation for offset feature groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
import polars as pl
|
|
7
|
+
|
|
8
|
+
from mloda.provider import ComputeFramework
|
|
9
|
+
from mloda_plugins.compute_framework.base_implementations.polars.lazy_dataframe import PolarsLazyDataFrame
|
|
10
|
+
|
|
11
|
+
from mloda.community.feature_groups.data_operations.reserved_columns import assert_no_reserved_columns
|
|
12
|
+
from mloda.community.feature_groups.data_operations.row_preserving.offset.base import (
|
|
13
|
+
OffsetFeatureGroup,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class PolarsLazyOffset(OffsetFeatureGroup):
|
|
18
|
+
@classmethod
|
|
19
|
+
def compute_framework_rule(cls) -> set[type[ComputeFramework]] | None:
|
|
20
|
+
return {PolarsLazyDataFrame}
|
|
21
|
+
|
|
22
|
+
@classmethod
|
|
23
|
+
def _compute_offset(
|
|
24
|
+
cls,
|
|
25
|
+
data: pl.LazyFrame,
|
|
26
|
+
feature_name: str,
|
|
27
|
+
source_col: str,
|
|
28
|
+
partition_by: list[str],
|
|
29
|
+
order_by: str,
|
|
30
|
+
offset_type: str,
|
|
31
|
+
) -> pl.LazyFrame:
|
|
32
|
+
"""PyArrow parity: offset requires sorting by order_by, which reorders rows.
|
|
33
|
+
Tag rows with an index before sorting and restore input order afterward."""
|
|
34
|
+
assert_no_reserved_columns(data.collect_schema().names(), framework="Polars", operation="offset")
|
|
35
|
+
|
|
36
|
+
# Track original row order
|
|
37
|
+
data = data.with_row_index("__mloda_orig_idx")
|
|
38
|
+
|
|
39
|
+
# Sort by partition_by + order_by (nulls last) for correct offset
|
|
40
|
+
data = data.sort(partition_by + [order_by], nulls_last=True)
|
|
41
|
+
|
|
42
|
+
if offset_type.startswith("lag_"):
|
|
43
|
+
offset_n = int(offset_type[len("lag_") :])
|
|
44
|
+
expr = pl.col(source_col).shift(offset_n).over(partition_by).alias(feature_name)
|
|
45
|
+
elif offset_type.startswith("lead_"):
|
|
46
|
+
offset_n = int(offset_type[len("lead_") :])
|
|
47
|
+
expr = pl.col(source_col).shift(-offset_n).over(partition_by).alias(feature_name)
|
|
48
|
+
elif offset_type.startswith("diff_"):
|
|
49
|
+
offset_n = int(offset_type[len("diff_") :])
|
|
50
|
+
expr = (pl.col(source_col) - pl.col(source_col).shift(offset_n)).over(partition_by).alias(feature_name)
|
|
51
|
+
elif offset_type.startswith("pct_change_"):
|
|
52
|
+
offset_n = int(offset_type[len("pct_change_") :])
|
|
53
|
+
prev = pl.col(source_col).shift(offset_n)
|
|
54
|
+
expr = (
|
|
55
|
+
pl.when(prev.is_not_null() & (prev != 0))
|
|
56
|
+
.then((pl.col(source_col) - prev) / prev)
|
|
57
|
+
.otherwise(pl.lit(None))
|
|
58
|
+
.over(partition_by)
|
|
59
|
+
.alias(feature_name)
|
|
60
|
+
)
|
|
61
|
+
elif offset_type == "first_value":
|
|
62
|
+
expr = pl.col(source_col).drop_nulls().first().over(partition_by).alias(feature_name)
|
|
63
|
+
elif offset_type == "last_value":
|
|
64
|
+
expr = pl.col(source_col).drop_nulls().last().over(partition_by).alias(feature_name)
|
|
65
|
+
else:
|
|
66
|
+
raise ValueError(f"Unsupported offset type: {offset_type}")
|
|
67
|
+
|
|
68
|
+
result = data.with_columns(expr)
|
|
69
|
+
# Restore original row order and drop helper column
|
|
70
|
+
return result.sort("__mloda_orig_idx").drop("__mloda_orig_idx")
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""SQLite implementation for offset feature groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from mloda.provider import ComputeFramework
|
|
6
|
+
from mloda_plugins.compute_framework.base_implementations.sql.sql_utils import quote_ident
|
|
7
|
+
from mloda_plugins.compute_framework.base_implementations.sqlite.sqlite_framework import SqliteFramework
|
|
8
|
+
from mloda_plugins.compute_framework.base_implementations.sqlite.sqlite_relation import SqliteRelation
|
|
9
|
+
|
|
10
|
+
from mloda.community.feature_groups.data_operations.reserved_columns import assert_no_reserved_columns
|
|
11
|
+
from mloda.community.feature_groups.data_operations.row_preserving.offset.base import (
|
|
12
|
+
OffsetFeatureGroup,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class SqliteOffset(OffsetFeatureGroup):
|
|
17
|
+
@classmethod
|
|
18
|
+
def compute_framework_rule(cls) -> set[type[ComputeFramework]] | None:
|
|
19
|
+
return {SqliteFramework}
|
|
20
|
+
|
|
21
|
+
@classmethod
|
|
22
|
+
def _compute_offset(
|
|
23
|
+
cls,
|
|
24
|
+
data: SqliteRelation,
|
|
25
|
+
feature_name: str,
|
|
26
|
+
source_col: str,
|
|
27
|
+
partition_by: list[str],
|
|
28
|
+
order_by: str,
|
|
29
|
+
offset_type: str,
|
|
30
|
+
) -> SqliteRelation:
|
|
31
|
+
assert_no_reserved_columns(data.columns, framework="SQLite", operation="offset")
|
|
32
|
+
|
|
33
|
+
quoted_source = quote_ident(source_col)
|
|
34
|
+
quoted_order = quote_ident(order_by)
|
|
35
|
+
quoted_feature = quote_ident(feature_name)
|
|
36
|
+
partition_clause = ", ".join(quote_ident(col) for col in partition_by)
|
|
37
|
+
qrn = quote_ident("__mloda_rn__")
|
|
38
|
+
|
|
39
|
+
null_sort = f"CASE WHEN {quoted_order} IS NULL THEN 1 ELSE 0 END"
|
|
40
|
+
order_clause = f"{null_sort}, {quoted_order}"
|
|
41
|
+
window_clause = f"PARTITION BY {partition_clause} ORDER BY {order_clause}" # nosec
|
|
42
|
+
|
|
43
|
+
if offset_type.startswith("lag_"):
|
|
44
|
+
offset_n = int(offset_type[len("lag_") :])
|
|
45
|
+
offset_expr = f"LAG({quoted_source}, {offset_n})"
|
|
46
|
+
elif offset_type.startswith("lead_"):
|
|
47
|
+
offset_n = int(offset_type[len("lead_") :])
|
|
48
|
+
offset_expr = f"LEAD({quoted_source}, {offset_n})"
|
|
49
|
+
elif offset_type in ("first_value", "last_value"):
|
|
50
|
+
return cls._compute_first_last(data, feature_name, source_col, partition_by, order_by, offset_type)
|
|
51
|
+
else:
|
|
52
|
+
supported = "lag, lead, first_value, last_value"
|
|
53
|
+
raise ValueError(f"Unsupported offset type for SQLite: {offset_type}. Supported types: {supported}")
|
|
54
|
+
|
|
55
|
+
sql = (
|
|
56
|
+
f"SELECT {offset_expr} OVER ({window_clause}) AS {quoted_feature}, " # nosec
|
|
57
|
+
f"ROW_NUMBER() OVER (ORDER BY rowid) AS {qrn} "
|
|
58
|
+
f"FROM {quote_ident(data.table_name)} ORDER BY {qrn}"
|
|
59
|
+
)
|
|
60
|
+
cursor = data.connection.execute(sql)
|
|
61
|
+
rows = cursor.fetchall()
|
|
62
|
+
|
|
63
|
+
result_values = [row[0] for row in rows]
|
|
64
|
+
return data.append_column(feature_name, result_values)
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def _compute_first_last(
|
|
68
|
+
cls,
|
|
69
|
+
data: SqliteRelation,
|
|
70
|
+
feature_name: str,
|
|
71
|
+
source_col: str,
|
|
72
|
+
partition_by: list[str],
|
|
73
|
+
order_by: str,
|
|
74
|
+
offset_type: str,
|
|
75
|
+
) -> SqliteRelation:
|
|
76
|
+
"""SQLite lacks IGNORE NULLS in window functions, so select the first
|
|
77
|
+
(or last) non-null value per partition via a correlated subquery."""
|
|
78
|
+
quoted_source = quote_ident(source_col)
|
|
79
|
+
quoted_order = quote_ident(order_by)
|
|
80
|
+
quoted_feature = quote_ident(feature_name)
|
|
81
|
+
qrn = quote_ident("__mloda_rn__")
|
|
82
|
+
|
|
83
|
+
partition_match = " AND ".join(f"t2.{quote_ident(col)} IS t1.{quote_ident(col)}" for col in partition_by)
|
|
84
|
+
null_sort = f"CASE WHEN t2.{quoted_order} IS NULL THEN 1 ELSE 0 END"
|
|
85
|
+
|
|
86
|
+
if offset_type == "first_value":
|
|
87
|
+
sort_clause = f"{null_sort}, t2.{quoted_order}"
|
|
88
|
+
else:
|
|
89
|
+
sort_clause = f"{null_sort} DESC, t2.{quoted_order} DESC"
|
|
90
|
+
|
|
91
|
+
subquery = (
|
|
92
|
+
f"(SELECT t2.{quoted_source} FROM {quote_ident(data.table_name)} t2 " # nosec
|
|
93
|
+
f"WHERE {partition_match} AND t2.{quoted_source} IS NOT NULL "
|
|
94
|
+
f"ORDER BY {sort_clause} LIMIT 1)"
|
|
95
|
+
)
|
|
96
|
+
sql = (
|
|
97
|
+
f"SELECT {subquery} AS {quoted_feature}, " # nosec
|
|
98
|
+
f"ROW_NUMBER() OVER (ORDER BY rowid) AS {qrn} "
|
|
99
|
+
f"FROM {quote_ident(data.table_name)} t1 ORDER BY {qrn}"
|
|
100
|
+
)
|
|
101
|
+
cursor = data.connection.execute(sql)
|
|
102
|
+
rows = cursor.fetchall()
|
|
103
|
+
result_values = [row[0] for row in rows]
|
|
104
|
+
return data.append_column(feature_name, result_values)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mloda-community-offset
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Offset feature group (lag, lead, diff, pct_change, first_value, last_value, row-preserving)
|
|
5
|
+
Author-email: Tom Kaltofen <info@mloda.ai>
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://mloda.ai
|
|
8
|
+
Project-URL: Repository, https://github.com/mloda-ai/mloda-registry
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Requires-Dist: mloda-community-data-operations>=0.2.12
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: mloda-testing; extra == "dev"
|
|
13
|
+
Requires-Dist: pytest>=9.0.3; extra == "dev"
|
|
14
|
+
Provides-Extra: sqlite
|
|
15
|
+
Provides-Extra: duckdb
|
|
16
|
+
Requires-Dist: duckdb; extra == "duckdb"
|
|
17
|
+
Provides-Extra: polars
|
|
18
|
+
Requires-Dist: polars; extra == "polars"
|
|
19
|
+
Provides-Extra: pandas
|
|
20
|
+
Requires-Dist: pandas; extra == "pandas"
|
|
21
|
+
Provides-Extra: all
|
|
22
|
+
Requires-Dist: polars; extra == "all"
|
|
23
|
+
Requires-Dist: pandas; extra == "all"
|
|
24
|
+
Requires-Dist: duckdb; extra == "all"
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/base.py,sha256=tnbvAQkLegYkfgdLRX5c8XuExgP92U7o7aAZ6a-vsX8,7860
|
|
3
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/duckdb_offset.py,sha256=6IkCwUWE5JU07so7YqQ1nnGcKbbg_cfDLhJWhpNsDbI,4490
|
|
4
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/pandas_offset.py,sha256=jOFf_-hLXu5TYjXfrVnR97kTLtiDMOjHmthDz2_k0OQ,2700
|
|
5
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/polars_lazy_offset.py,sha256=AkzAnI6MyYnrqe2CXkhY4RUDX45_xPr35BzTg-YTc5s,2993
|
|
6
|
+
mloda/community/feature_groups/data_operations/row_preserving/offset/sqlite_offset.py,sha256=n3hDmGBtr-6Fjehb_ELzJmu3jYMStn4DOFrX7yt2rEA,4384
|
|
7
|
+
mloda_community_offset-0.3.1.dist-info/METADATA,sha256=iCgKmErAiH6HS9M5T65dh8xynHU82_lAYIPxxaNRCwg,887
|
|
8
|
+
mloda_community_offset-0.3.1.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
|
|
9
|
+
mloda_community_offset-0.3.1.dist-info/top_level.txt,sha256=srwNjqzXpP1WfLRhbrVc9JLt8TdfsaQe-Th7cGcAvYY,6
|
|
10
|
+
mloda_community_offset-0.3.1.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
mloda
|