gitlabds 2.4.4__tar.gz → 2.4.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gitlabds-2.4.4 → gitlabds-2.4.5}/PKG-INFO +2 -2
- {gitlabds-2.4.4 → gitlabds-2.4.5}/README.md +1 -1
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/serving_features.py +48 -22
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds.egg-info/PKG-INFO +2 -2
- {gitlabds-2.4.4 → gitlabds-2.4.5}/tests/test_serving_features.py +22 -6
- {gitlabds-2.4.4 → gitlabds-2.4.5}/LICENSE +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/__init__.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/baselines.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/config_generator.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/dummy.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/feature_descriptions.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/feature_reduction.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/insights.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/memory_optimization.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/missing.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/model_evaluator.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/monitoring_metrics.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/outliers.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/record_benchmarking.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/snowflake_connectors.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/split_data.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds/trends.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds.egg-info/SOURCES.txt +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds.egg-info/dependency_links.txt +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds.egg-info/requires.txt +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/gitlabds.egg-info/top_level.txt +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/pyproject.toml +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/setup.cfg +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/setup.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/tests/test_feature_reduction.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/tests/test_record_benchmarking.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/tests/test_snowflake_connectors.py +0 -0
- {gitlabds-2.4.4 → gitlabds-2.4.5}/tests/test_split_data.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlabds
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.5
|
|
4
4
|
Summary: GitLab Data Science Tools
|
|
5
5
|
Author-email: Kevin Dietz <kdietz@gitlab.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -1402,7 +1402,7 @@ Serve features from Snowflake Feature Store with per-call UDF parameters and ret
|
|
|
1402
1402
|
- **Schema:** view and UDF names are qualified with the feature view's database and schema, so a `USE SCHEMA` in your session doesn't change what is served.
|
|
1403
1403
|
- **Redeploying in the same session:** the temporary views last until the session closes, so a session that serves and then redeploys the permanent `_DYNAMIC` view keeps reading its temporary view until the next `serve_features` call. Use a fresh session to read the redeployed defaults.
|
|
1404
1404
|
- **Concurrency:** calls sharing one session object run one at a time. Open one session per thread for real parallelism.
|
|
1405
|
-
- **Feature-date check:** when `feature_date` is set, the served date is
|
|
1405
|
+
- **Feature-date check:** when `feature_date` is set, the served date is checked and a mismatch raises a warning, not an error: some feature views use a timestamp other than the snapshot date (e.g. a refresh week), so a mismatch isn't always wrong. With `spine_timestamp_col` and `include_feature_view_timestamp_col=True` the returned `{FV}_{version}_FEATURE_DATE` column is checked; otherwise the session's temporary view is queried before retrieval, which may run the whole UDF.
|
|
1406
1406
|
- **Spine timestamps earlier than `feature_date`** match no feature rows in the point-in-time join and return all-null features. A warning is raised when this happens.
|
|
1407
1407
|
|
|
1408
1408
|
`gitlabds.serve_features(session, feature_store, feature_views_dict, spine_df=None, feature_date=None, spine_timestamp_col=None, include_feature_view_timestamp_col=False, lookback_window_value=None, lookback_window_unit=None, lock_timeout_seconds=None, max_fetch_retries=3, fetch_retry_backoff_seconds=30):`
|
|
@@ -1366,7 +1366,7 @@ Serve features from Snowflake Feature Store with per-call UDF parameters and ret
|
|
|
1366
1366
|
- **Schema:** view and UDF names are qualified with the feature view's database and schema, so a `USE SCHEMA` in your session doesn't change what is served.
|
|
1367
1367
|
- **Redeploying in the same session:** the temporary views last until the session closes, so a session that serves and then redeploys the permanent `_DYNAMIC` view keeps reading its temporary view until the next `serve_features` call. Use a fresh session to read the redeployed defaults.
|
|
1368
1368
|
- **Concurrency:** calls sharing one session object run one at a time. Open one session per thread for real parallelism.
|
|
1369
|
-
- **Feature-date check:** when `feature_date` is set, the served date is
|
|
1369
|
+
- **Feature-date check:** when `feature_date` is set, the served date is checked and a mismatch raises a warning, not an error: some feature views use a timestamp other than the snapshot date (e.g. a refresh week), so a mismatch isn't always wrong. With `spine_timestamp_col` and `include_feature_view_timestamp_col=True` the returned `{FV}_{version}_FEATURE_DATE` column is checked; otherwise the session's temporary view is queried before retrieval, which may run the whole UDF.
|
|
1370
1370
|
- **Spine timestamps earlier than `feature_date`** match no feature rows in the point-in-time join and return all-null features. A warning is raised when this happens.
|
|
1371
1371
|
|
|
1372
1372
|
`gitlabds.serve_features(session, feature_store, feature_views_dict, spine_df=None, feature_date=None, spine_timestamp_col=None, include_feature_view_timestamp_col=False, lookback_window_value=None, lookback_window_unit=None, lock_timeout_seconds=None, max_fetch_retries=3, fetch_retry_backoff_seconds=30):`
|
|
@@ -74,28 +74,29 @@ def _check_dynamic_view_feature_date(
|
|
|
74
74
|
dynamic_view: str,
|
|
75
75
|
timestamp_col: str,
|
|
76
76
|
feature_date: str,
|
|
77
|
-
|
|
78
|
-
|
|
77
|
+
mismatches: List[str],
|
|
78
|
+
) -> bool:
|
|
79
|
+
"""Check the session's dynamic view serves feature_date; append to mismatches if not."""
|
|
79
80
|
rows = session.sql(f"SELECT {timestamp_col} FROM {dynamic_view} LIMIT 1").collect()
|
|
80
81
|
if not rows:
|
|
81
82
|
warnings.warn(
|
|
82
83
|
f"{dynamic_view} returned no rows; could not verify feature_date {feature_date}"
|
|
83
84
|
)
|
|
84
|
-
return
|
|
85
|
+
return False
|
|
85
86
|
actual = rows[0][0]
|
|
86
87
|
if actual is None or _to_date(actual) != _to_date(feature_date):
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
)
|
|
88
|
+
mismatches.append(f"{dynamic_view} returned {timestamp_col} {actual}, expected {feature_date}")
|
|
89
|
+
return False
|
|
90
|
+
return True
|
|
91
91
|
|
|
92
92
|
|
|
93
93
|
def _check_retrieved_feature_date(
|
|
94
94
|
features_df: pd.DataFrame,
|
|
95
95
|
feature_date_col: str,
|
|
96
96
|
feature_date: str,
|
|
97
|
-
|
|
98
|
-
|
|
97
|
+
mismatches: List[str],
|
|
98
|
+
) -> bool:
|
|
99
|
+
"""Check a retrieved feature-date column holds only feature_date; append to mismatches if not."""
|
|
99
100
|
values = features_df[feature_date_col].dropna()
|
|
100
101
|
if values.empty:
|
|
101
102
|
warnings.warn(
|
|
@@ -103,14 +104,26 @@ def _check_retrieved_feature_date(
|
|
|
103
104
|
f"null. A spine timestamp earlier than feature_date {feature_date} matches no "
|
|
104
105
|
f"feature rows in the point-in-time join."
|
|
105
106
|
)
|
|
106
|
-
return
|
|
107
|
+
return False
|
|
107
108
|
expected = _to_date(feature_date)
|
|
108
109
|
mismatched = sorted({d for d in values.map(_to_date) if d != expected})
|
|
109
110
|
if mismatched:
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
111
|
+
mismatches.append(f"{feature_date_col} returned {mismatched}, expected {feature_date}")
|
|
112
|
+
return False
|
|
113
|
+
return True
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _warn_feature_date_mismatches(mismatches: List[str]) -> None:
|
|
117
|
+
"""Raise one warning listing every feature_date mismatch, with the explanation once."""
|
|
118
|
+
if not mismatches:
|
|
119
|
+
return
|
|
120
|
+
listed = "\n".join(f" - {m}" for m in mismatches)
|
|
121
|
+
warnings.warn(
|
|
122
|
+
f"feature_date mismatch in {len(mismatches)} feature view(s):\n{listed}\n"
|
|
123
|
+
f"Some feature views use a timestamp other than the snapshot date (e.g. a refresh "
|
|
124
|
+
f"week), in which case this is expected; otherwise the features may not be for the "
|
|
125
|
+
f"requested date."
|
|
126
|
+
)
|
|
114
127
|
|
|
115
128
|
|
|
116
129
|
def _warn_spine_before_feature_date(
|
|
@@ -265,7 +278,10 @@ def serve_features(
|
|
|
265
278
|
Calls sharing one session object run one at a time: a process-local lock per session
|
|
266
279
|
is held for the whole call. Open one session per thread for real parallelism.
|
|
267
280
|
|
|
268
|
-
When ``feature_date`` is set, the served feature date is
|
|
281
|
+
When ``feature_date`` is set, the served feature date is checked, and a mismatch warns:
|
|
282
|
+
some feature views use a timestamp other than the snapshot date
|
|
283
|
+
(e.g. a refresh week), so a mismatch is not always an error. All mismatches are
|
|
284
|
+
reported together in one warning after retrieval. With
|
|
269
285
|
``spine_timestamp_col`` and ``include_feature_view_timestamp_col=True`` the returned
|
|
270
286
|
``{FV}_{version}_{timestamp_col}`` column is checked; otherwise the session's temporary view is
|
|
271
287
|
queried before retrieval (``SELECT {timestamp_col} ... LIMIT 1``), which may run the
|
|
@@ -319,8 +335,8 @@ def serve_features(
|
|
|
319
335
|
TypeError
|
|
320
336
|
When spine_df or feature_views_dict has the wrong type.
|
|
321
337
|
ValueError
|
|
322
|
-
When spine_df is None, feature_views_dict is empty, session is not the feature
|
|
323
|
-
store's session
|
|
338
|
+
When spine_df is None, feature_views_dict is empty, or session is not the feature
|
|
339
|
+
store's session.
|
|
324
340
|
|
|
325
341
|
Examples
|
|
326
342
|
--------
|
|
@@ -376,6 +392,7 @@ def serve_features(
|
|
|
376
392
|
try:
|
|
377
393
|
feature_views_list = []
|
|
378
394
|
feature_date_cols = {}
|
|
395
|
+
date_mismatches: List[str] = []
|
|
379
396
|
|
|
380
397
|
for fv_name, version in feature_views_dict.items():
|
|
381
398
|
fv = feature_store.get_feature_view(fv_name, version)
|
|
@@ -416,8 +433,10 @@ def serve_features(
|
|
|
416
433
|
timestamp_col,
|
|
417
434
|
)
|
|
418
435
|
else:
|
|
419
|
-
_check_dynamic_view_feature_date(
|
|
420
|
-
|
|
436
|
+
if _check_dynamic_view_feature_date(
|
|
437
|
+
session, dynamic_view, timestamp_col, feature_date, date_mismatches
|
|
438
|
+
):
|
|
439
|
+
print(f" [{call_id}] Verified {dynamic_view} serves {timestamp_col} {feature_date}")
|
|
421
440
|
|
|
422
441
|
if isinstance(spine_df, str):
|
|
423
442
|
snowpark_spine_df = session.sql(spine_df)
|
|
@@ -467,10 +486,17 @@ def serve_features(
|
|
|
467
486
|
returned_cols = {c.upper(): c for c in combined_features.columns}
|
|
468
487
|
for col, (dynamic_view, timestamp_col) in feature_date_cols.items():
|
|
469
488
|
if col in returned_cols:
|
|
470
|
-
_check_retrieved_feature_date(
|
|
489
|
+
matched = _check_retrieved_feature_date(
|
|
490
|
+
combined_features, returned_cols[col], feature_date, date_mismatches
|
|
491
|
+
)
|
|
471
492
|
else:
|
|
472
|
-
_check_dynamic_view_feature_date(
|
|
473
|
-
|
|
493
|
+
matched = _check_dynamic_view_feature_date(
|
|
494
|
+
session, dynamic_view, timestamp_col, feature_date, date_mismatches
|
|
495
|
+
)
|
|
496
|
+
if matched:
|
|
497
|
+
print(f"[{call_id}] Verified {col} matches feature_date {feature_date}")
|
|
498
|
+
|
|
499
|
+
_warn_feature_date_mismatches(date_mismatches)
|
|
474
500
|
|
|
475
501
|
if spine_timestamp_col is not None and feature_date:
|
|
476
502
|
_warn_spine_before_feature_date(combined_features, spine_timestamp_col, feature_date)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gitlabds
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.5
|
|
4
4
|
Summary: GitLab Data Science Tools
|
|
5
5
|
Author-email: Kevin Dietz <kdietz@gitlab.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -1402,7 +1402,7 @@ Serve features from Snowflake Feature Store with per-call UDF parameters and ret
|
|
|
1402
1402
|
- **Schema:** view and UDF names are qualified with the feature view's database and schema, so a `USE SCHEMA` in your session doesn't change what is served.
|
|
1403
1403
|
- **Redeploying in the same session:** the temporary views last until the session closes, so a session that serves and then redeploys the permanent `_DYNAMIC` view keeps reading its temporary view until the next `serve_features` call. Use a fresh session to read the redeployed defaults.
|
|
1404
1404
|
- **Concurrency:** calls sharing one session object run one at a time. Open one session per thread for real parallelism.
|
|
1405
|
-
- **Feature-date check:** when `feature_date` is set, the served date is
|
|
1405
|
+
- **Feature-date check:** when `feature_date` is set, the served date is checked and a mismatch raises a warning, not an error: some feature views use a timestamp other than the snapshot date (e.g. a refresh week), so a mismatch isn't always wrong. With `spine_timestamp_col` and `include_feature_view_timestamp_col=True` the returned `{FV}_{version}_FEATURE_DATE` column is checked; otherwise the session's temporary view is queried before retrieval, which may run the whole UDF.
|
|
1406
1406
|
- **Spine timestamps earlier than `feature_date`** match no feature rows in the point-in-time join and return all-null features. A warning is raised when this happens.
|
|
1407
1407
|
|
|
1408
1408
|
`gitlabds.serve_features(session, feature_store, feature_views_dict, spine_df=None, feature_date=None, spine_timestamp_col=None, include_feature_view_timestamp_col=False, lookback_window_value=None, lookback_window_unit=None, lock_timeout_seconds=None, max_fetch_retries=3, fetch_retry_backoff_seconds=30):`
|
|
@@ -128,30 +128,46 @@ def test_same_session_calls_are_serialized():
|
|
|
128
128
|
assert active["max"] == 1
|
|
129
129
|
|
|
130
130
|
|
|
131
|
-
def
|
|
131
|
+
def test_temp_view_feature_date_mismatch_warns_and_still_serves():
|
|
132
132
|
session = FakeSession(served_date="2024-10-05")
|
|
133
133
|
store = FakeFeatureStore(session)
|
|
134
|
-
with pytest.
|
|
134
|
+
with pytest.warns(UserWarning, match="expected 2025-01-01"):
|
|
135
135
|
sf.serve_features(
|
|
136
136
|
session, store, {"a": "1.0"}, spine_df=SPINE,
|
|
137
137
|
feature_date="2025-01-01",
|
|
138
138
|
)
|
|
139
|
-
assert
|
|
139
|
+
assert store.retrieved
|
|
140
140
|
|
|
141
141
|
|
|
142
|
-
def
|
|
142
|
+
def test_mismatches_across_views_warn_once():
|
|
143
|
+
session = FakeSession(served_date="2024-10-05")
|
|
144
|
+
with pytest.warns(UserWarning) as record:
|
|
145
|
+
sf.serve_features(
|
|
146
|
+
session, FakeFeatureStore(session), {"a": "1.0", "b": "1.0"}, spine_df=SPINE,
|
|
147
|
+
feature_date="2025-01-01",
|
|
148
|
+
)
|
|
149
|
+
mismatch_warnings = [str(w.message) for w in record if "feature_date mismatch" in str(w.message)]
|
|
150
|
+
assert len(mismatch_warnings) == 1
|
|
151
|
+
message = mismatch_warnings[0]
|
|
152
|
+
assert "FS_DB.FS_SCHEMA.A_DYNAMIC returned" in message
|
|
153
|
+
assert "FS_DB.FS_SCHEMA.B_DYNAMIC returned" in message
|
|
154
|
+
assert message.count("Some feature views use a timestamp") == 1
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def test_returned_feature_date_mismatch_warns():
|
|
143
158
|
result = pd.DataFrame({
|
|
144
159
|
"ACCOUNT_ID": [1, 2],
|
|
145
160
|
"SNAPSHOT_DATE": pd.to_datetime(["2025-01-15"] * 2),
|
|
146
161
|
"A_1.0_FEATURE_DATE": pd.to_datetime(["2024-10-05"] * 2),
|
|
147
162
|
})
|
|
148
163
|
session = FakeSession(served_date="2025-01-01")
|
|
149
|
-
with pytest.
|
|
150
|
-
sf.serve_features(
|
|
164
|
+
with pytest.warns(UserWarning, match="A_1.0_FEATURE_DATE returned"):
|
|
165
|
+
out = sf.serve_features(
|
|
151
166
|
session, FakeFeatureStore(session, result_df=result), {"a": "1.0"}, spine_df=SPINE,
|
|
152
167
|
feature_date="2025-01-01", spine_timestamp_col="SNAPSHOT_DATE",
|
|
153
168
|
include_feature_view_timestamp_col=True,
|
|
154
169
|
)
|
|
170
|
+
assert len(out) == 2
|
|
155
171
|
|
|
156
172
|
|
|
157
173
|
def test_session_other_than_feature_store_session_raises():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|