fastdfs 1.2__tar.gz → 1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fastdfs-1.2 → fastdfs-1.3}/PKG-INFO +27 -8
- {fastdfs-1.2 → fastdfs-1.3}/README.md +26 -7
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/__init__.py +7 -2
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/api.py +15 -44
- fastdfs-1.3/fastdfs/dfs/__init__.py +24 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dfs/base_engine.py +193 -33
- fastdfs-1.3/fastdfs/dfs/dfs2sql_engine.py +535 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dfs/duckdb_database.py +10 -4
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dfs/gen_sqls.py +10 -4
- fastdfs-1.3/fastdfs/dfs/session.py +268 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs.egg-info/PKG-INFO +27 -8
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs.egg-info/SOURCES.txt +4 -0
- {fastdfs-1.2 → fastdfs-1.3}/pyproject.toml +1 -1
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_api.py +34 -0
- fastdfs-1.3/tests/test_dfs2sql_types.py +390 -0
- fastdfs-1.3/tests/test_dfs2sql_validation.py +191 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_dfs_engines.py +66 -1
- fastdfs-1.3/tests/test_dfs_session.py +494 -0
- fastdfs-1.2/fastdfs/dfs/__init__.py +0 -11
- fastdfs-1.2/fastdfs/dfs/dfs2sql_engine.py +0 -366
- {fastdfs-1.2 → fastdfs-1.3}/LICENSE +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/__init__.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/__init__.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/dataset_meta.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/download.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/download_config.yaml +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/rdb_dataset.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/table_loader.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/table_writer.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/version.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/dbinfer_bench/yaml_utils.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/duckdb.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/mysql.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/postgres.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/relbench.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/sql_base.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/adapter/sqlite.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dataset/__init__.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dataset/loader.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dataset/meta.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dataset/rdb.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dataset/writer.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dfs/featuretools_engine.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/dfs/merge_sql.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/__init__.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/base.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/datetime_transform.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/dummy_table_transform.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/encode_categorical.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/fill_missing_pk.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/filter_transform.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/infer_schema.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/type_transform.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/transform/utils.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/__init__.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/datetime_utils.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/device.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/logging_config.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/type_utils.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs/utils/yaml_utils.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs.egg-info/dependency_links.txt +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs.egg-info/requires.txt +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/fastdfs.egg-info/top_level.txt +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/setup.cfg +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_adapter_hints.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_complex_target_columns.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_create_rdb.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_dbinfer_adapter.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_dfs2sql_assembly.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_dfs2sql_parallel.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_duckdb_adapter.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_dummy_table_transform.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_encode_categorical.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_fill_missing_primary_key.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_infer_schema.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_merge_sql.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_rdb.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_rdb_save.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_relbench_adapter.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_sqlite_adapter.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_target_history.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_transforms.py +0 -0
- {fastdfs-1.2 → fastdfs-1.3}/tests/test_type_transform.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fastdfs
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3
|
|
4
4
|
Summary: Fast Deep Feature Synthesis for tabular data
|
|
5
5
|
Author: HKU Shanghai X-Lab
|
|
6
6
|
Maintainer: HKU Shanghai X-Lab
|
|
@@ -95,13 +95,13 @@ features = fastdfs.compute_dfs_features(
|
|
|
95
95
|
target_dataframe=target_df,
|
|
96
96
|
key_mappings=key_mappings,
|
|
97
97
|
cutoff_time_column="interaction_time",
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
98
|
+
config=fastdfs.DFSConfig(
|
|
99
|
+
engine="dfs2sql",
|
|
100
|
+
engine_path=":memory:",
|
|
101
|
+
dfs2sql_sql_workers=2,
|
|
102
|
+
dfs2sql_threads=8,
|
|
103
|
+
dfs2sql_memory_limit="4GB",
|
|
104
|
+
),
|
|
105
105
|
)
|
|
106
106
|
```
|
|
107
107
|
|
|
@@ -126,6 +126,25 @@ Parallel floating-point aggregation and tied MODE values can differ between runs
|
|
|
126
126
|
|
|
127
127
|
Full tables and setup notes: [dfs2sql parallel SQL benchmarks](docs/dfs2sql_parallel_sql_benchmarks.md).
|
|
128
128
|
|
|
129
|
+
## Multi-split sessions (train / val / test)
|
|
130
|
+
|
|
131
|
+
When the RDB is fixed and only the target rows change, reuse planning and DuckDB ingest with `DFSSession` (`engine="dfs2sql"`):
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
config = fastdfs.DFSConfig(engine="dfs2sql", engine_path=":memory:")
|
|
135
|
+
with fastdfs.DFSSession(
|
|
136
|
+
rdb,
|
|
137
|
+
key_mappings={"user_id": "user.user_id", "item_id": "item.item_id"},
|
|
138
|
+
cutoff_time_column="interaction_time",
|
|
139
|
+
config=config,
|
|
140
|
+
) as session:
|
|
141
|
+
train_features = session.compute(train_df)
|
|
142
|
+
val_features = session.compute(val_df)
|
|
143
|
+
test_features = session.compute(test_df)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
For a single target matrix, keep using `compute_dfs_features`. See `examples/session_train_val_example.py`.
|
|
147
|
+
|
|
129
148
|
## Engine Comparison
|
|
130
149
|
|
|
131
150
|
| Feature | Featuretools | DFS2SQL |
|
|
@@ -37,13 +37,13 @@ features = fastdfs.compute_dfs_features(
|
|
|
37
37
|
target_dataframe=target_df,
|
|
38
38
|
key_mappings=key_mappings,
|
|
39
39
|
cutoff_time_column="interaction_time",
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
40
|
+
config=fastdfs.DFSConfig(
|
|
41
|
+
engine="dfs2sql",
|
|
42
|
+
engine_path=":memory:",
|
|
43
|
+
dfs2sql_sql_workers=2,
|
|
44
|
+
dfs2sql_threads=8,
|
|
45
|
+
dfs2sql_memory_limit="4GB",
|
|
46
|
+
),
|
|
47
47
|
)
|
|
48
48
|
```
|
|
49
49
|
|
|
@@ -68,6 +68,25 @@ Parallel floating-point aggregation and tied MODE values can differ between runs
|
|
|
68
68
|
|
|
69
69
|
Full tables and setup notes: [dfs2sql parallel SQL benchmarks](docs/dfs2sql_parallel_sql_benchmarks.md).
|
|
70
70
|
|
|
71
|
+
## Multi-split sessions (train / val / test)
|
|
72
|
+
|
|
73
|
+
When the RDB is fixed and only the target rows change, reuse planning and DuckDB ingest with `DFSSession` (`engine="dfs2sql"`):
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
config = fastdfs.DFSConfig(engine="dfs2sql", engine_path=":memory:")
|
|
77
|
+
with fastdfs.DFSSession(
|
|
78
|
+
rdb,
|
|
79
|
+
key_mappings={"user_id": "user.user_id", "item_id": "item.item_id"},
|
|
80
|
+
cutoff_time_column="interaction_time",
|
|
81
|
+
config=config,
|
|
82
|
+
) as session:
|
|
83
|
+
train_features = session.compute(train_df)
|
|
84
|
+
val_features = session.compute(val_df)
|
|
85
|
+
test_features = session.compute(test_df)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
For a single target matrix, keep using `compute_dfs_features`. See `examples/session_train_val_example.py`.
|
|
89
|
+
|
|
71
90
|
## Engine Comparison
|
|
72
91
|
|
|
73
92
|
| Feature | Featuretools | DFS2SQL |
|
|
@@ -15,11 +15,13 @@ from .api import (
|
|
|
15
15
|
create_rdb,
|
|
16
16
|
load_rdb,
|
|
17
17
|
compute_dfs_features,
|
|
18
|
-
DFSPipeline
|
|
18
|
+
DFSPipeline,
|
|
19
|
+
DFSSession,
|
|
20
|
+
DFSSessionError,
|
|
19
21
|
)
|
|
20
22
|
|
|
21
23
|
# Core components
|
|
22
|
-
from .dfs import DFSConfig
|
|
24
|
+
from .dfs import DFSConfig, InvalidTimeColumnError
|
|
23
25
|
from .dataset.rdb import RDB, RDBDataset
|
|
24
26
|
from .transform import (
|
|
25
27
|
RDBTransform,
|
|
@@ -36,6 +38,9 @@ __all__ = [
|
|
|
36
38
|
"load_rdb",
|
|
37
39
|
"compute_dfs_features",
|
|
38
40
|
"DFSPipeline",
|
|
41
|
+
"DFSSession",
|
|
42
|
+
"DFSSessionError",
|
|
43
|
+
"InvalidTimeColumnError",
|
|
39
44
|
|
|
40
45
|
# Core components
|
|
41
46
|
"DFSConfig",
|
|
@@ -8,14 +8,22 @@ the RDB interface.
|
|
|
8
8
|
from typing import Dict, Optional, Any, List, Tuple
|
|
9
9
|
import pandas as pd
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from loguru import logger
|
|
12
11
|
|
|
13
12
|
from .dfs import DFSConfig, get_dfs_engine
|
|
13
|
+
from .dfs.base_engine import coerce_target_key_types
|
|
14
|
+
from .dfs.session import DFSSession, DFSSessionError
|
|
14
15
|
from .dataset.rdb import RDB
|
|
15
|
-
from .dataset.meta import RDBMeta, RDBTableSchema, RDBTableDataFormat
|
|
16
|
+
from .dataset.meta import RDBMeta, RDBTableSchema, RDBTableDataFormat
|
|
16
17
|
from .transform.infer_schema import InferSchemaTransform
|
|
17
18
|
|
|
18
|
-
__all__ = [
|
|
19
|
+
__all__ = [
|
|
20
|
+
'load_rdb',
|
|
21
|
+
'create_rdb',
|
|
22
|
+
'compute_dfs_features',
|
|
23
|
+
'DFSPipeline',
|
|
24
|
+
'DFSSession',
|
|
25
|
+
'DFSSessionError',
|
|
26
|
+
]
|
|
19
27
|
|
|
20
28
|
|
|
21
29
|
def load_rdb(path: str) -> RDB:
|
|
@@ -125,54 +133,17 @@ def compute_dfs_features(
|
|
|
125
133
|
... target_dataframe=target_df,
|
|
126
134
|
... key_mappings={"user_id": "user.user_id", "item_id": "item.item_id"},
|
|
127
135
|
... cutoff_time_column="interaction_time",
|
|
128
|
-
...
|
|
136
|
+
... config=DFSConfig(engine="dfs2sql"),
|
|
137
|
+
... config_overrides={"max_depth": 2}
|
|
129
138
|
... )
|
|
130
139
|
"""
|
|
131
140
|
# Use default config if not provided
|
|
132
141
|
if config is None:
|
|
133
142
|
config = DFSConfig()
|
|
134
143
|
|
|
135
|
-
# Validate key
|
|
136
|
-
|
|
137
|
-
if target_col not in target_dataframe.columns:
|
|
138
|
-
raise ValueError(f"Key column '{target_col}' not found in target dataframe.")
|
|
139
|
-
|
|
140
|
-
table_name, col_name = rdb_key.split('.')
|
|
141
|
-
try:
|
|
142
|
-
table_meta = rdb.get_table_metadata(table_name)
|
|
143
|
-
except ValueError:
|
|
144
|
-
raise ValueError(f"Table '{table_name}' not found in RDB.")
|
|
145
|
-
|
|
146
|
-
if col_name not in table_meta.column_dict:
|
|
147
|
-
raise ValueError(f"Column '{col_name}' not found in table '{table_name}'.")
|
|
148
|
-
|
|
149
|
-
col_meta = table_meta.column_dict[col_name]
|
|
144
|
+
# Validate key mappings and convert target key columns to string
|
|
145
|
+
coerce_target_key_types(rdb, target_dataframe, key_mappings)
|
|
150
146
|
|
|
151
|
-
if col_meta.dtype != RDBColumnDType.primary_key:
|
|
152
|
-
raise ValueError(f"RDB column '{rdb_key}' is not a primary key. Key mappings must point to primary keys.")
|
|
153
|
-
|
|
154
|
-
if col_meta.dtype in (RDBColumnDType.primary_key, RDBColumnDType.foreign_key):
|
|
155
|
-
# Convert target column to string to match RDB key columns (which should always be string)
|
|
156
|
-
target_col_dtype = target_dataframe[target_col].dtype
|
|
157
|
-
|
|
158
|
-
# Convert to string if not already string (categorical is converted for consistency)
|
|
159
|
-
if not pd.api.types.is_string_dtype(target_col_dtype):
|
|
160
|
-
try:
|
|
161
|
-
target_dataframe[target_col] = target_dataframe[target_col].astype(str)
|
|
162
|
-
|
|
163
|
-
# Log the conversion for transparency
|
|
164
|
-
logger.debug(
|
|
165
|
-
f"Converted target column '{target_col}' from {target_col_dtype} to string "
|
|
166
|
-
f"to match RDB key '{rdb_key}'"
|
|
167
|
-
)
|
|
168
|
-
except (ValueError, TypeError) as e:
|
|
169
|
-
# If conversion fails, raise a helpful error
|
|
170
|
-
raise TypeError(
|
|
171
|
-
f"Failed to convert column '{target_col}' from {target_col_dtype} to string "
|
|
172
|
-
f"to match RDB key '{rdb_key}'. Original error: {e}. "
|
|
173
|
-
f"Please ensure the values are compatible or manually convert: "
|
|
174
|
-
f"target_df['{target_col}'] = target_df['{target_col}'].astype(str)"
|
|
175
|
-
)
|
|
176
147
|
# Canonicalize key types in RDB to ensure consistency (e.g., all keys as strings)
|
|
177
148
|
rdb=rdb.canonicalize_key_types()
|
|
178
149
|
# Validate key consistency across all tables in the RDB
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
New DFS Engine Interface Package.
|
|
3
|
+
|
|
4
|
+
This package provides the new table-centric DFS engine interface.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .base_engine import DFSEngine, DFSConfig, InvalidTimeColumnError, get_dfs_engine, dfs_engine, dfs_feature_column_name
|
|
8
|
+
from .featuretools_engine import FeaturetoolsEngine
|
|
9
|
+
from .dfs2sql_engine import DFS2SQLEngine, assemble_dfs2sql_feature_frames
|
|
10
|
+
from .session import DFSSession, DFSSessionError
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
'DFSEngine',
|
|
14
|
+
'DFSConfig',
|
|
15
|
+
'dfs_feature_column_name',
|
|
16
|
+
'get_dfs_engine',
|
|
17
|
+
'dfs_engine',
|
|
18
|
+
'FeaturetoolsEngine',
|
|
19
|
+
'DFS2SQLEngine',
|
|
20
|
+
'assemble_dfs2sql_feature_frames',
|
|
21
|
+
'DFSSession',
|
|
22
|
+
'DFSSessionError',
|
|
23
|
+
'InvalidTimeColumnError',
|
|
24
|
+
]
|
|
@@ -6,13 +6,17 @@ target dataframes and simplified RDB datasets, removing the dependency on tasks.
|
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
import abc
|
|
9
|
+
import difflib
|
|
10
|
+
import warnings
|
|
9
11
|
from typing import Dict, List, Optional, Any, Tuple
|
|
10
12
|
import pandas as pd
|
|
11
13
|
import featuretools as ft
|
|
12
14
|
import numpy as np
|
|
13
15
|
from loguru import logger
|
|
14
16
|
import pydantic
|
|
17
|
+
import woodwork as ww
|
|
15
18
|
from woodwork.column_schema import ColumnSchema
|
|
19
|
+
from woodwork.type_sys.utils import col_is_datetime
|
|
16
20
|
|
|
17
21
|
from ..dataset.rdb import RDB
|
|
18
22
|
from ..dataset.meta import RDBColumnDType, RDBColumnSchema
|
|
@@ -44,6 +48,98 @@ def dfs_feature_column_name(feature: ft.FeatureBase) -> str:
|
|
|
44
48
|
return f"d{int(feature.get_depth())}__{feature.get_name()}"
|
|
45
49
|
|
|
46
50
|
|
|
51
|
+
def dfs_feature_renames(features: List[ft.FeatureBase]) -> Dict[str, str]:
|
|
52
|
+
"""Map Featuretools feature names to ``dfs_feature_column_name`` where they differ."""
|
|
53
|
+
renames = {}
|
|
54
|
+
for f in features:
|
|
55
|
+
old = f.get_name()
|
|
56
|
+
new = dfs_feature_column_name(f)
|
|
57
|
+
if old != new:
|
|
58
|
+
renames[old] = new
|
|
59
|
+
return renames
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _warn_unknown_config_keys(keys, known, where: str) -> None:
|
|
63
|
+
"""Warn once per key that is not a DFSConfig field (such keys are ignored)."""
|
|
64
|
+
for key in keys:
|
|
65
|
+
if key in known:
|
|
66
|
+
continue
|
|
67
|
+
close = difflib.get_close_matches(str(key), list(known), n=1)
|
|
68
|
+
hint = f" Did you mean '{close[0]}'?" if close else ""
|
|
69
|
+
warnings.warn(f"Unknown DFSConfig key '{key}' in {where} is ignored.{hint}", UserWarning)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class InvalidTimeColumnError(TypeError, ValueError):
|
|
73
|
+
"""A time column or cutoff column whose values are not datetimes or numbers.
|
|
74
|
+
|
|
75
|
+
Subclasses TypeError, which woodwork raised for these inputs before dfs2sql
|
|
76
|
+
planned on 0-row frames, and ValueError, since the values are what is wrong.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def to_naive_datetime(series: pd.Series) -> pd.Series:
|
|
81
|
+
"""Convert to naive datetimes with woodwork's ``Datetime`` logical type.
|
|
82
|
+
|
|
83
|
+
This is the conversion full-frame planning applied, so values match it: the
|
|
84
|
+
format is inferred from a sample, two-digit years use woodwork's pivot,
|
|
85
|
+
values that don't fit become NaT (with a warning), and timezones are dropped
|
|
86
|
+
keeping the wall time (mixed UTC offsets are converted to UTC first).
|
|
87
|
+
"""
|
|
88
|
+
return ww.logical_types.Datetime().transform(series)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def is_valid_time_column(series: pd.Series) -> bool:
|
|
92
|
+
"""Woodwork's rule for a ``Datetime`` time index: numeric, or values that all
|
|
93
|
+
parse as datetimes (missing values aside)."""
|
|
94
|
+
return pd.api.types.is_numeric_dtype(series) or col_is_datetime(series)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def coerce_target_key_types(
|
|
98
|
+
rdb: RDB, target_dataframe: pd.DataFrame, key_mappings: Dict[str, str]
|
|
99
|
+
) -> None:
|
|
100
|
+
"""Validate key mappings and convert target key columns to string, in place."""
|
|
101
|
+
for target_col, rdb_key in key_mappings.items():
|
|
102
|
+
if target_col not in target_dataframe.columns:
|
|
103
|
+
raise ValueError(f"Key column '{target_col}' not found in target dataframe.")
|
|
104
|
+
|
|
105
|
+
table_name, col_name = rdb_key.split('.')
|
|
106
|
+
try:
|
|
107
|
+
table_meta = rdb.get_table_metadata(table_name)
|
|
108
|
+
except ValueError:
|
|
109
|
+
raise ValueError(f"Table '{table_name}' not found in RDB.")
|
|
110
|
+
|
|
111
|
+
if col_name not in table_meta.column_dict:
|
|
112
|
+
raise ValueError(f"Column '{col_name}' not found in table '{table_name}'.")
|
|
113
|
+
|
|
114
|
+
col_meta = table_meta.column_dict[col_name]
|
|
115
|
+
|
|
116
|
+
if col_meta.dtype != RDBColumnDType.primary_key:
|
|
117
|
+
raise ValueError(f"RDB column '{rdb_key}' is not a primary key. Key mappings must point to primary keys.")
|
|
118
|
+
|
|
119
|
+
if col_meta.dtype in (RDBColumnDType.primary_key, RDBColumnDType.foreign_key):
|
|
120
|
+
# Convert target column to string to match RDB key columns (which should always be string)
|
|
121
|
+
target_col_dtype = target_dataframe[target_col].dtype
|
|
122
|
+
|
|
123
|
+
# Convert to string if not already string (categorical is converted for consistency)
|
|
124
|
+
if not pd.api.types.is_string_dtype(target_col_dtype):
|
|
125
|
+
try:
|
|
126
|
+
target_dataframe[target_col] = target_dataframe[target_col].astype(str)
|
|
127
|
+
|
|
128
|
+
# Log the conversion for transparency
|
|
129
|
+
logger.debug(
|
|
130
|
+
f"Converted target column '{target_col}' from {target_col_dtype} to string "
|
|
131
|
+
f"to match RDB key '{rdb_key}'"
|
|
132
|
+
)
|
|
133
|
+
except (ValueError, TypeError) as e:
|
|
134
|
+
# If conversion fails, raise a helpful error
|
|
135
|
+
raise TypeError(
|
|
136
|
+
f"Failed to convert column '{target_col}' from {target_col_dtype} to string "
|
|
137
|
+
f"to match RDB key '{rdb_key}'. Original error: {e}. "
|
|
138
|
+
f"Please ensure the values are compatible or manually convert: "
|
|
139
|
+
f"target_df['{target_col}'] = target_df['{target_col}'].astype(str)"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
47
143
|
# Custom quantile aggregation primitives
|
|
48
144
|
class Quantile25(ft.primitives.AggregationPrimitive):
|
|
49
145
|
"""Calculates the 25th percentile (Q1) of a numeric column."""
|
|
@@ -155,10 +251,22 @@ class DFSConfig(pydantic.BaseModel):
|
|
|
155
251
|
dfs2sql_temp_directory: Optional[str] = None
|
|
156
252
|
include_cutoff_time: bool = False
|
|
157
253
|
|
|
254
|
+
@pydantic.model_validator(mode="before")
|
|
255
|
+
@classmethod
|
|
256
|
+
def _warn_unknown_keys(cls, data: Any) -> Any:
|
|
257
|
+
# Unknown keys are ignored (pydantic's default); warn so typos are noticed.
|
|
258
|
+
if isinstance(data, dict):
|
|
259
|
+
_warn_unknown_config_keys(data, cls.model_fields, "DFSConfig")
|
|
260
|
+
return data
|
|
261
|
+
|
|
158
262
|
|
|
159
263
|
class DFSEngine:
|
|
160
264
|
"""Base class for DFS computation engines."""
|
|
161
265
|
|
|
266
|
+
# Whether planning registers 0-row RDB frames (columns and types only) in the
|
|
267
|
+
# Featuretools EntitySet. Only valid for engines that compute values outside it.
|
|
268
|
+
plans_on_empty_frames = False
|
|
269
|
+
|
|
162
270
|
def __init__(self, config: DFSConfig):
|
|
163
271
|
self.config = config
|
|
164
272
|
|
|
@@ -185,10 +293,12 @@ class DFSEngine:
|
|
|
185
293
|
DataFrame with original target_dataframe data plus generated features
|
|
186
294
|
"""
|
|
187
295
|
# Merge config overrides
|
|
188
|
-
effective_config = self.config.
|
|
296
|
+
effective_config = self.config.model_copy(deep=True)
|
|
189
297
|
if config_overrides:
|
|
298
|
+
_warn_unknown_config_keys(config_overrides, DFSConfig.model_fields, "config_overrides")
|
|
190
299
|
for key, value in config_overrides.items():
|
|
191
|
-
|
|
300
|
+
# Only declared fields: hasattr() would also accept methods like copy.
|
|
301
|
+
if key in DFSConfig.model_fields:
|
|
192
302
|
setattr(effective_config, key, value)
|
|
193
303
|
|
|
194
304
|
return self._compute_features_impl(
|
|
@@ -209,26 +319,10 @@ class DFSEngine:
|
|
|
209
319
|
if len(target_dataframe) == 0:
|
|
210
320
|
return target_dataframe
|
|
211
321
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
target_index = "__target_index__"
|
|
215
|
-
if target_index in target_dataframe.columns:
|
|
216
|
-
logger.error(f"Target dataframe cannot contain reserved column name '{target_index}'.")
|
|
217
|
-
# Assign creates a new dataframe but reuses underlying blocks where possible.
|
|
218
|
-
target_df_with_index = target_dataframe.assign(
|
|
219
|
-
**{target_index: np.arange(len(target_dataframe))}
|
|
322
|
+
target_df_with_index, target_df_for_engine = self._prepare_target_for_engine(
|
|
323
|
+
target_dataframe, key_mappings, cutoff_time_column
|
|
220
324
|
)
|
|
221
325
|
|
|
222
|
-
# Optimization: Filter target dataframe to only include necessary columns
|
|
223
|
-
# This defines the contract for the engines: they only see keys, time, and index.
|
|
224
|
-
columns_to_keep = {target_index}
|
|
225
|
-
columns_to_keep.update(key_mappings.keys())
|
|
226
|
-
if cutoff_time_column:
|
|
227
|
-
columns_to_keep.add(cutoff_time_column)
|
|
228
|
-
|
|
229
|
-
# Work on a defensive copy that engines are free to mutate.
|
|
230
|
-
target_df_for_engine = target_df_with_index[list(columns_to_keep)].copy()
|
|
231
|
-
|
|
232
326
|
# Phase 1: Feature preparation (common logic in base class)
|
|
233
327
|
features = self.prepare_features(rdb, target_df_for_engine, key_mappings, cutoff_time_column, config)
|
|
234
328
|
|
|
@@ -241,27 +335,79 @@ class DFSEngine:
|
|
|
241
335
|
rdb, target_df_for_engine, key_mappings, cutoff_time_column, features, config
|
|
242
336
|
)
|
|
243
337
|
|
|
244
|
-
renames =
|
|
245
|
-
for f in features:
|
|
246
|
-
old = f.get_name()
|
|
247
|
-
new = dfs_feature_column_name(f)
|
|
248
|
-
if old != new and old in feature_matrix.columns:
|
|
249
|
-
renames[old] = new
|
|
338
|
+
renames = dfs_feature_renames(features)
|
|
250
339
|
if renames:
|
|
251
340
|
feature_matrix = feature_matrix.rename(columns=renames, copy=False)
|
|
252
341
|
|
|
342
|
+
return self._join_features_to_target(
|
|
343
|
+
target_df_with_index, feature_matrix, target_dataframe.index
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
def _prepare_target_for_engine(
|
|
347
|
+
self,
|
|
348
|
+
target_dataframe: pd.DataFrame,
|
|
349
|
+
key_mappings: Dict[str, str],
|
|
350
|
+
cutoff_time_column: Optional[str],
|
|
351
|
+
) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
|
352
|
+
"""Add ``__target_index__`` and keep only the columns engines may see.
|
|
353
|
+
|
|
354
|
+
Returns ``(target_df_with_index, target_df_for_engine)``: the full target
|
|
355
|
+
with the index column, and a defensive copy holding only the index, key
|
|
356
|
+
and cutoff columns, which engines are free to mutate.
|
|
357
|
+
"""
|
|
358
|
+
target_index = "__target_index__"
|
|
359
|
+
if target_index in target_dataframe.columns:
|
|
360
|
+
raise ValueError(f"Target dataframe cannot contain reserved column name '{target_index}'.")
|
|
361
|
+
# Assign creates a new dataframe but reuses underlying blocks where possible.
|
|
362
|
+
target_df_with_index = target_dataframe.assign(
|
|
363
|
+
**{target_index: np.arange(len(target_dataframe))}
|
|
364
|
+
)
|
|
365
|
+
|
|
366
|
+
# This defines the contract for the engines: they only see keys, time, and index.
|
|
367
|
+
columns_to_keep = {target_index}
|
|
368
|
+
columns_to_keep.update(key_mappings.keys())
|
|
369
|
+
if cutoff_time_column:
|
|
370
|
+
columns_to_keep.add(cutoff_time_column)
|
|
371
|
+
|
|
372
|
+
target_df_for_engine = target_df_with_index[list(columns_to_keep)].copy()
|
|
373
|
+
if cutoff_time_column:
|
|
374
|
+
cutoff = target_df_for_engine[cutoff_time_column]
|
|
375
|
+
# Reject values woodwork would reject for a time index: a NaT cutoff
|
|
376
|
+
# would silently match no history. Check all-missing explicitly too,
|
|
377
|
+
# because col_is_datetime accepts an all-NaT datetime64 column by dtype
|
|
378
|
+
# (one-shot planning still rejects it, but a session only plans once).
|
|
379
|
+
# Bool counts as numeric for pandas, but woodwork rejects it as a cutoff.
|
|
380
|
+
# An empty split has nothing to check.
|
|
381
|
+
if len(cutoff) and (
|
|
382
|
+
pd.api.types.is_bool_dtype(cutoff)
|
|
383
|
+
or cutoff.isna().all()
|
|
384
|
+
or not is_valid_time_column(cutoff)
|
|
385
|
+
):
|
|
386
|
+
raise InvalidTimeColumnError(
|
|
387
|
+
f"Cutoff time column '{cutoff_time_column}' has values that cannot be "
|
|
388
|
+
f"parsed as datetimes (or has no values)."
|
|
389
|
+
)
|
|
390
|
+
# Fix the cutoff type here, before planning and ingest, so every split
|
|
391
|
+
# (and every DFSSession compute) reaches DuckDB as datetime64[ns].
|
|
392
|
+
if not pd.api.types.is_numeric_dtype(cutoff):
|
|
393
|
+
target_df_for_engine[cutoff_time_column] = to_naive_datetime(cutoff)
|
|
394
|
+
return target_df_with_index, target_df_for_engine
|
|
395
|
+
|
|
396
|
+
def _join_features_to_target(
|
|
397
|
+
self,
|
|
398
|
+
target_df_with_index: pd.DataFrame,
|
|
399
|
+
feature_matrix: pd.DataFrame,
|
|
400
|
+
original_index: pd.Index,
|
|
401
|
+
) -> pd.DataFrame:
|
|
402
|
+
"""Join features onto the target rows by ``__target_index__`` and restore the original index."""
|
|
403
|
+
target_index = "__target_index__"
|
|
253
404
|
if target_index not in feature_matrix.columns:
|
|
254
|
-
|
|
405
|
+
raise ValueError(f"Feature matrix is missing the target index column '{target_index}'.")
|
|
255
406
|
|
|
256
|
-
# Align features with original target rows using the target index
|
|
257
407
|
target_with_idx = target_df_with_index.set_index(target_index)
|
|
258
408
|
features_with_idx = feature_matrix.set_index(target_index)
|
|
259
|
-
|
|
260
409
|
merged = target_with_idx.join(features_with_idx, how="left")
|
|
261
|
-
|
|
262
|
-
# Restore the original index ordering
|
|
263
410
|
merged.index = original_index
|
|
264
|
-
|
|
265
411
|
return merged
|
|
266
412
|
|
|
267
413
|
def prepare_features(
|
|
@@ -320,9 +466,11 @@ class DFSEngine:
|
|
|
320
466
|
target_entity_name = "__target__"
|
|
321
467
|
target_index = "__target_index__"
|
|
322
468
|
|
|
469
|
+
# Woodwork converts registered frames in place; plan on a copy so the frame
|
|
470
|
+
# engines later compute from never depends on planning.
|
|
323
471
|
entity_set = entity_set.add_dataframe(
|
|
324
472
|
dataframe_name=target_entity_name,
|
|
325
|
-
dataframe=target_dataframe,
|
|
473
|
+
dataframe=target_dataframe.copy(),
|
|
326
474
|
index=target_index,
|
|
327
475
|
time_index=cutoff_time_column
|
|
328
476
|
)
|
|
@@ -364,6 +512,18 @@ class DFSEngine:
|
|
|
364
512
|
for table_name in rdb.table_names:
|
|
365
513
|
df = rdb.get_table(table_name)
|
|
366
514
|
table_meta = rdb.get_table_metadata(table_name)
|
|
515
|
+
if self.plans_on_empty_frames:
|
|
516
|
+
# Featuretools only needs columns and logical types (set below from
|
|
517
|
+
# metadata) to plan; values come from the engine, not this EntitySet.
|
|
518
|
+
df = df.iloc[:0].copy()
|
|
519
|
+
# Woodwork cannot convert an empty column of strings to a time index,
|
|
520
|
+
# so give a datetime time column its real type up front.
|
|
521
|
+
time_col = table_meta.time_column
|
|
522
|
+
if (
|
|
523
|
+
time_col in df.columns
|
|
524
|
+
and table_meta.column_dict[time_col].dtype == RDBColumnDType.datetime_t
|
|
525
|
+
):
|
|
526
|
+
df[time_col] = to_naive_datetime(df[time_col])
|
|
367
527
|
|
|
368
528
|
# Parse columns and build logical types/semantic tags (reuse existing logic)
|
|
369
529
|
logical_types = {}
|