featkit 0.1.0__tar.gz → 0.2.0__tar.gz

{featkit-0.1.0 → featkit-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: featkit
-Version: 0.1.0
+Version: 0.2.0
 Summary: featkit — automated feature store generation from relational facts tables
 Project-URL: Repository, https://github.com/Mirkiux/featkit
 Project-URL: Documentation, https://mirkiux.github.io/featkit
@@ -49,6 +49,7 @@ Provides-Extra: dev
 Requires-Dist: build>=1.0; extra == 'dev'
 Requires-Dist: hatch>=1.9; extra == 'dev'
 Requires-Dist: mypy>=1.0; extra == 'dev'
+Requires-Dist: pandas>=1.5; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0; extra == 'dev'
 Requires-Dist: pytest>=7.0; extra == 'dev'
 Requires-Dist: ruff>=0.4; extra == 'dev'
@@ -57,6 +58,8 @@ Provides-Extra: docs
 Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
 Requires-Dist: mkdocs>=1.6; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.25; extra == 'docs'
+Provides-Extra: execution
+Requires-Dist: pandas>=1.5; extra == 'execution'
 Provides-Extra: ibis
 Requires-Dist: ibis-framework>=9.0; extra == 'ibis'
 Provides-Extra: spark

featkit-0.2.0/docs/example_databricks_notebook.md

@@ -0,0 +1,141 @@
+# Example — Dynamic domain resolution in a Databricks notebook
+
+This example shows how to let featkit resolve the `allowed_values` domain of a
+`CategoricalField` at runtime by querying the facts table directly from a
+Databricks notebook.
+
+`DatabricksNotebookAdapter` discovers the pre-injected `spark` session
+automatically — no constructor arguments are needed.
+
+## Notebook cells
+
+### Cell 1 — imports
+
+```python
+from featkit.config import FeatureStoreConfig
+from featkit.dataset.base import SimpleDataset
+from featkit.enums import CategoricalTreatment, MeasurementType, TimeGranularity
+from featkit.execution.adapters import DatabricksNotebookAdapter
+from featkit.fields.categorical_field import CategoricalField
+from featkit.fields.id_field import IDField
+from featkit.fields.measurement_field import MeasurementField
+from featkit.fields.time_field import TimeField
+from featkit.generators.sql.databricks import DatabricksSQLCodeGenerator
+from featkit.pipeline import FeatureStorePipeline
+```
+
+### Cell 2 — define the dataset (no `allowed_values` on the categorical)
+
+```python
+ds = SimpleDataset(
+    "mydb.myschema.silver_transactions",
+    [
+        IDField("client_id"),
+        TimeField("period", TimeGranularity.MONTHLY, TimeGranularity.MONTHLY),
+        MeasurementField("amount", MeasurementType.MONTO),
+        MeasurementField("txn_count", MeasurementType.CANTIDAD),
+        # No allowed_values — the adapter will resolve the domain at build() time
+        CategoricalField("segment", CategoricalTreatment.PIVOT),
+        CategoricalField("product_type", CategoricalTreatment.PIVOT),
+    ],
+)
+```
+
+### Cell 3 — configure with the notebook adapter
+
+```python
+adapter = DatabricksNotebookAdapter()
+
+cfg = FeatureStoreConfig(
+    dataset=ds,
+    output_schema="analytics",
+    output_table_prefix="feat_",
+    time_windows=[3, 6, 12],
+    include_marginals=True,
+    adapter=adapter,           # triggers SELECT DISTINCT resolution at build()
+)
+```
+
+### Cell 4 — build and generate
+
+```python
+# build() fires one SELECT DISTINCT per unresolved CategoricalField
+pipeline = FeatureStorePipeline(config=cfg).build()
+
+print(f"Layer 2A columns : {len(pipeline.layer2a)}")
+print(f"Layer 3  features: {len(pipeline.layer3)}")
+
+result = DatabricksSQLCodeGenerator().generate(pipeline)
+print(result.code.sql[:500])
+```
+
+### Cell 5 — save the artefacts to DBFS
+
+```python
+result.save("/dbfs/mnt/output/features/")
+# Writes:
+#   /dbfs/mnt/output/features/script.sql
+#   /dbfs/mnt/output/features/dag.json
+#   /dbfs/mnt/output/features/diagram.md
+```
+
+## How it works
+
+When `FeatureStorePipeline.build()` is called with an `adapter` set on the
+config, it constructs an `AdapterDomainResolver` and passes it to
+`PivotSpaceBuilder` as the `domain_resolver` callable.  For each
+`CategoricalField` that has no `allowed_values`, the builder calls the resolver,
+which executes:
+
+```sql
+SELECT DISTINCT segment
+FROM mydb.myschema.silver_transactions
+WHERE segment IS NOT NULL
+ORDER BY 1
+```
+
+The returned values become the column domain exactly as if they had been listed
+in `allowed_values` at configuration time.
+
+## Mixing static and dynamic domains
+
+Static and dynamic fields can coexist in the same dataset.  Fields that have
+`allowed_values` set are used as-is; only fields without it trigger a query:
+
+```python
+ds = SimpleDataset(
+    "mydb.myschema.silver_transactions",
+    [
+        IDField("client_id"),
+        TimeField("period", TimeGranularity.MONTHLY, TimeGranularity.MONTHLY),
+        MeasurementField("amount", MeasurementType.MONTO),
+        # Static domain — no query fired
+        CategoricalField(
+            "channel",
+            CategoricalTreatment.PIVOT,
+            allowed_values=["branch", "online", "mobile"],
+        ),
+        # Dynamic domain — one SELECT DISTINCT executed at build()
+        CategoricalField("segment", CategoricalTreatment.PIVOT),
+    ],
+)
+```
+
+## Using a different adapter
+
+Swap `DatabricksNotebookAdapter` for any other adapter without changing the
+rest of the code:
+
+```python
+from featkit.execution.adapters import DatabricksAdapter
+
+adapter = DatabricksAdapter(
+    host="<workspace>.azuredatabricks.net",
+    token="<pat>",
+    http_path="/sql/1.0/warehouses/<warehouse-id>",
+    catalog="mydb",
+    schema="myschema",
+)
+
+cfg = FeatureStoreConfig(..., adapter=adapter)
+```

{featkit-0.1.0 → featkit-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "featkit"
-version = "0.1.0"
+version = "0.2.0"
 description = "featkit — automated feature store generation from relational facts tables"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -35,6 +35,7 @@ dependencies = [
 ibis = ["ibis-framework>=9.0"]
 spark = ["pyspark>=3.4"]
 databricks = ["databricks-sql-connector>=3.0"]
+execution = ["pandas>=1.5"]
 docs = [
     "mkdocs>=1.6",
     "mkdocs-material>=9.5",
@@ -48,6 +49,7 @@ dev = [
     "hatch>=1.9",
     "build>=1.0",
     "twine>=5.0",
+    "pandas>=1.5",
 ]
 
 [project.urls]

{featkit-0.1.0 → featkit-0.2.0}/src/featkit/config.py

@@ -3,10 +3,14 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
 
 from featkit.dataset.base import AbstractDataset
 from featkit.enums import Layer2Aggregator, Layer2OutputType, MeasurementType, TemporalOperator
 
+if TYPE_CHECKING:
+    from featkit.execution.adapters.base import DataSourceAdapter
+
 
 @dataclass
 class FeatureStoreConfig:
@@ -26,6 +30,10 @@ class FeatureStoreConfig:
             aggregators. Only contract-valid aggregators are used.
         operators_override: Per-output-type override for temporal operators.
             Only contract-valid operators are used.
+        adapter: Optional execution adapter.  When provided, categorical fields
+            with no ``allowed_values`` have their domain resolved at
+            ``FeatureStorePipeline.build()`` time via a ``SELECT DISTINCT``
+            query against the facts table.
     """
 
     dataset: AbstractDataset
@@ -36,3 +44,4 @@ class FeatureStoreConfig:
     include_marginals: bool = True
     aggregators_override: dict[MeasurementType, list[Layer2Aggregator]] | None = None
     operators_override: dict[Layer2OutputType, list[TemporalOperator]] | None = field(default=None)
+    adapter: DataSourceAdapter | None = None

featkit-0.2.0/src/featkit/execution/__init__.py

@@ -0,0 +1 @@
+"""featkit.execution — adapters and domain resolution for live query execution."""

featkit-0.2.0/src/featkit/execution/adapters/__init__.py

@@ -0,0 +1,18 @@
+"""featkit.execution.adapters — data source adapters for domain resolution."""
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+from featkit.execution.adapters.databricks_adapter import DatabricksAdapter
+from featkit.execution.adapters.databricks_notebook_adapter import DatabricksNotebookAdapter
+from featkit.execution.adapters.mock_adapter import MockAdapter
+from featkit.execution.adapters.spark_adapter import SparkAdapter
+from featkit.execution.adapters.sqlalchemy_adapter import SQLAlchemyAdapter
+
+__all__ = [
+    "DataSourceAdapter",
+    "DatabricksAdapter",
+    "DatabricksNotebookAdapter",
+    "EngineType",
+    "MockAdapter",
+    "SparkAdapter",
+    "SQLAlchemyAdapter",
+]

featkit-0.2.0/src/featkit/execution/adapters/base.py

@@ -0,0 +1,63 @@
+"""Base adapter ABC and EngineType enum for featkit execution layer."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from enum import Enum
+
+import pandas as pd
+
+_log = logging.getLogger(__name__)
+
+
+class EngineType(Enum):
+    """Identifies the execution engine behind a :class:`DataSourceAdapter`."""
+
+    SQLALCHEMY = "sqlalchemy"
+    DATABRICKS = "databricks"
+    SPARK = "spark"
+    MOCK = "mock"
+
+
+class DataSourceAdapter(ABC):
+    """Abstract base for all data source adapters.
+
+    Subclasses implement :meth:`engine_execute` with the engine-specific query
+    logic.  The public :meth:`execute` method wraps ``engine_execute`` with
+    error handling: if the query fails the offending SQL is logged at ``ERROR``
+    level before re-raising, making debugging significantly easier in
+    production pipelines.
+
+    This follows the *Template Method* pattern — the base class owns the
+    algorithm skeleton; subclasses supply only the engine-specific step.
+    """
+
+    def execute(self, sql: str) -> pd.DataFrame:
+        """Execute *sql* and return a DataFrame.
+
+        Delegates to :meth:`engine_execute`.  On failure, logs the SQL that
+        caused the error and re-raises the original exception unchanged.
+        """
+        try:
+            return self.engine_execute(sql)
+        except Exception:
+            _log.exception(
+                "SQL execution failed on %s.\nFailed query:\n%s",
+                self.__class__.__name__,
+                sql,
+            )
+            raise
+
+    @abstractmethod
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        """Execute *sql* against the underlying engine and return a DataFrame.
+
+        Implement this method in each adapter subclass.  Do not call it
+        directly — use :meth:`execute` instead so that error handling and any
+        future cross-cutting behaviour (retries, metrics) are applied.
+        """
+
+    @abstractmethod
+    def engine_type(self) -> EngineType:
+        """Return the :class:`EngineType` for this adapter."""

featkit-0.2.0/src/featkit/execution/adapters/databricks_adapter.py

@@ -0,0 +1,61 @@
+"""Adapter for Databricks SQL warehouses."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+
+
+class DatabricksAdapter(DataSourceAdapter):
+    """Adapter for Databricks SQL warehouses.
+
+    Requires the optional ``databricks-sql-connector`` package::
+
+        pip install databricks-sql-connector
+
+    The connector is imported lazily inside :meth:`engine_execute` so that the
+    class can be referenced in code that runs on environments without the
+    package installed — the ``ImportError`` is only raised when a query is
+    actually attempted.
+    """
+
+    def __init__(
+        self,
+        host: str,
+        token: str,
+        http_path: str,
+        catalog: str,
+        schema: str,
+    ) -> None:
+        self._host = host
+        self._token = token
+        self._http_path = http_path
+        self._catalog = catalog
+        self._schema = schema
+
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        try:
+            from databricks import sql as dbsql
+        except ImportError as exc:
+            raise ImportError(
+                "databricks-sql-connector is required for DatabricksAdapter. "
+                "Install with: pip install databricks-sql-connector"
+            ) from exc
+
+        with (
+            dbsql.connect(
+                server_hostname=self._host,
+                http_path=self._http_path,
+                access_token=self._token,
+                catalog=self._catalog,
+                schema=self._schema,
+            ) as conn,
+            conn.cursor() as cursor,
+        ):
+            cursor.execute(sql)
+            result: pd.DataFrame = cursor.fetchall_arrow().to_pandas()
+            return result
+
+    def engine_type(self) -> EngineType:
+        return EngineType.DATABRICKS

featkit-0.2.0/src/featkit/execution/adapters/databricks_notebook_adapter.py

@@ -0,0 +1,117 @@
+"""Adapter for Databricks notebook environments.
+
+In Databricks notebooks the Spark session is pre-instantiated and injected
+into the notebook's global namespace as ``spark``.  This adapter discovers
+that session automatically — no constructor arguments required.
+
+Usage inside a Databricks notebook::
+
+    from featkit.execution.adapters import DatabricksNotebookAdapter
+
+    adapter = DatabricksNotebookAdapter()
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+import pandas as pd
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+
+
+def _resolve_spark() -> Any:
+    """Locate the ``spark`` session injected by the Databricks notebook runtime.
+
+    Databricks injects ``spark`` into the ``__main__`` module namespace before
+    the first notebook cell executes.  This function retrieves it without
+    requiring the caller to hold a reference or import PySpark explicitly.
+
+    Raises
+    ------
+    RuntimeError
+        When no ``spark`` object can be found.  This most commonly means the
+        adapter is being used outside a Databricks notebook environment.
+    """
+    main = sys.modules.get("__main__", None)
+    spark = getattr(main, "spark", None) if main is not None else None
+
+    if spark is None:
+        raise RuntimeError(
+            "Could not locate 'spark' in the notebook runtime namespace. "
+            "DatabricksNotebookAdapter is intended for use inside Databricks "
+            "notebooks where 'spark' is pre-injected by the runtime. "
+            "Outside that environment, use SparkAdapter(spark_session) instead."
+        )
+    return spark
+
+
+class DatabricksNotebookAdapter(DataSourceAdapter):
+    """Adapter for Databricks notebook environments.
+
+    Wraps the ``spark`` session that the Databricks runtime pre-injects into
+    every notebook's global namespace.  No constructor arguments are needed —
+    the session is resolved lazily on the first :meth:`execute` call.
+
+    Examples
+    --------
+    Inside a Databricks notebook::
+
+        from featkit.execution.adapters import DatabricksNotebookAdapter
+        from featkit.config import FeatureStoreConfig
+        from featkit.dataset.base import SimpleDataset
+        from featkit.enums import CategoricalTreatment, MeasurementType, TimeGranularity
+        from featkit.fields.categorical_field import CategoricalField
+        from featkit.fields.id_field import IDField
+        from featkit.fields.measurement_field import MeasurementField
+        from featkit.fields.time_field import TimeField
+        from featkit.generators.sql.databricks import DatabricksSQLCodeGenerator
+        from featkit.pipeline import FeatureStorePipeline
+
+        adapter = DatabricksNotebookAdapter()
+
+        ds = SimpleDataset(
+            "mydb.silver_transactions",
+            [
+                IDField("client_id"),
+                TimeField("period", TimeGranularity.MONTHLY, TimeGranularity.MONTHLY),
+                MeasurementField("amount", MeasurementType.MONTO),
+                CategoricalField("segment", CategoricalTreatment.PIVOT),  # no allowed_values
+            ],
+        )
+
+        cfg = FeatureStoreConfig(
+            dataset=ds,
+            output_schema="analytics",
+            output_table_prefix="feat_",
+            time_windows=[3, 6, 12],
+            adapter=adapter,
+        )
+
+        pipeline = FeatureStorePipeline(config=cfg).build()
+        result = DatabricksSQLCodeGenerator().generate(pipeline)
+        result.save("/dbfs/mnt/output/features/")
+
+    Outside a notebook (e.g. in a standalone script or test), use
+    :class:`SparkAdapter` instead and pass the session explicitly::
+
+        adapter = SparkAdapter(spark_session)
+    """
+
+    def __init__(self) -> None:
+        self._spark: Any = None
+
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        """Execute *sql* via the notebook's pre-injected ``spark`` session.
+
+        The session is resolved once and cached for the lifetime of this
+        adapter instance.
+        """
+        if self._spark is None:
+            self._spark = _resolve_spark()
+        result: pd.DataFrame = self._spark.sql(sql).toPandas()
+        return result
+
+    def engine_type(self) -> EngineType:
+        return EngineType.SPARK

featkit-0.2.0/src/featkit/execution/adapters/mock_adapter.py

@@ -0,0 +1,35 @@
+"""In-memory adapter for tests and examples."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+
+
+class MockAdapter(DataSourceAdapter):
+    """In-memory adapter for tests and examples.
+
+    Accepts a dict mapping SQL strings to pre-built DataFrames.
+    Raises ``KeyError`` when an unregistered SQL string is executed.
+    Has no external *engine* dependencies (no database connector required);
+    ``pandas`` is still needed as it is the shared return type of all adapters.
+
+    Each call to :meth:`execute` increments an internal counter so tests can
+    verify how many times a query was executed.
+    """
+
+    def __init__(self, results: dict[str, pd.DataFrame]) -> None:
+        self._results = results
+        self._call_counts: dict[str, int] = {}
+
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        self._call_counts[sql] = self._call_counts.get(sql, 0) + 1
+        return self._results[sql]
+
+    def call_count(self, sql: str) -> int:
+        """Return the number of times *sql* has been executed."""
+        return self._call_counts.get(sql, 0)
+
+    def engine_type(self) -> EngineType:
+        return EngineType.MOCK

featkit-0.2.0/src/featkit/execution/adapters/spark_adapter.py

@@ -0,0 +1,28 @@
+"""Adapter that wraps a live PySpark SparkSession."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+
+
+class SparkAdapter(DataSourceAdapter):
+    """Adapter that wraps a live PySpark ``SparkSession``.
+
+    The ``spark_session`` parameter is typed as ``Any`` so that pyspark does
+    not need to be installed in environments that use other adapters.  Pass a
+    real ``SparkSession`` at runtime; pass a duck-typed fake in tests.
+    """
+
+    def __init__(self, spark_session: Any) -> None:
+        self._spark = spark_session
+
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        result: pd.DataFrame = self._spark.sql(sql).toPandas()
+        return result
+
+    def engine_type(self) -> EngineType:
+        return EngineType.SPARK

featkit-0.2.0/src/featkit/execution/adapters/sqlalchemy_adapter.py

@@ -0,0 +1,44 @@
+"""Adapter backed by a SQLAlchemy-compatible connection string."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from featkit.execution.adapters.base import DataSourceAdapter, EngineType
+
+
+class SQLAlchemyAdapter(DataSourceAdapter):
+    """Adapter backed by a SQLAlchemy-compatible connection string.
+
+    Supports any engine that SQLAlchemy supports: PostgreSQL, Oracle,
+    Snowflake, MySQL, SQLite, and any JDBC-compatible backend via the
+    appropriate dialect package.
+
+    The SQLAlchemy engine is created lazily on the first :meth:`execute` call.
+    """
+
+    def __init__(self, connection_string: str) -> None:
+        self._connection_string = connection_string
+        self._engine: Any = None
+
+    def _get_engine(self) -> Any:
+        if self._engine is None:
+            try:
+                from sqlalchemy import create_engine
+            except ImportError as exc:
+                raise ImportError(
+                    "sqlalchemy is required for SQLAlchemyAdapter. "
+                    "Install with: pip install sqlalchemy"
+                ) from exc
+            self._engine = create_engine(self._connection_string)
+        return self._engine
+
+    def engine_execute(self, sql: str) -> pd.DataFrame:
+        engine = self._get_engine()
+        result: pd.DataFrame = pd.read_sql_query(sql, engine)
+        return result
+
+    def engine_type(self) -> EngineType:
+        return EngineType.SQLALCHEMY