feast 0.67.0.dev122__tar.gz → 0.67.0.dev123__tar.gz

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/.secrets.baseline

@@ -957,7 +957,7 @@
         "filename": "infra/feast-operator/api/v1/featurestore_types.go",
         "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c",
         "is_verified": false,
-        "line_number": 1113
+        "line_number": 1118
       }
     ],
     "infra/feast-operator/api/v1/zz_generated.deepcopy.go": [
@@ -980,7 +980,7 @@
         "filename": "infra/feast-operator/api/v1/zz_generated.deepcopy.go",
         "hashed_secret": "c2028031c154bbe86fd69bef740855c74b927dcf",
         "is_verified": false,
-        "line_number": 1773
+        "line_number": 1778
       }
     ],
     "infra/feast-operator/api/v1alpha1/featurestore_types.go": [

{feast-0.67.0.dev122/sdk/python/feast.egg-info → feast-0.67.0.dev123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feast
-Version: 0.67.0.dev122
+Version: 0.67.0.dev123
 Summary: Python SDK for Feast
 Author: Feast
 License-Expression: Apache-2.0
@@ -130,6 +130,9 @@ Provides-Extra: spark
 Requires-Dist: pyspark>=4.0.0; extra == "spark"
 Provides-Extra: iceberg
 Requires-Dist: pyiceberg>=0.7.0; extra == "iceberg"
+Provides-Extra: lance
+Requires-Dist: pylance>=12.0.0; extra == "lance"
+Requires-Dist: lance-namespace>=0.11.1; extra == "lance"
 Provides-Extra: trino
 Requires-Dist: trino<0.400.0,>=0.305.0; extra == "trino"
 Requires-Dist: regex; extra == "trino"
@@ -189,7 +192,7 @@ Requires-Dist: minio==7.2.11; extra == "test"
 Requires-Dist: python-keycloak<5,>=4.7.3; extra == "test"
 Requires-Dist: cryptography>=43.0; extra == "test"
 Provides-Extra: ci
-Requires-Dist: feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,glide,go,grpcio,hazelcast,hbase,ibis,iceberg,image,k8s,mcp,mcp-server,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,scylladb,singlestore,snowflake,spark,sqlite_vec,test,trino]; extra == "ci"
+Requires-Dist: feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,glide,go,grpcio,hazelcast,hbase,ibis,iceberg,image,k8s,lance,mcp,mcp-server,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,scylladb,singlestore,snowflake,spark,sqlite_vec,test,trino]; extra == "ci"
 Requires-Dist: build; extra == "ci"
 Requires-Dist: virtualenv==21.14.1; extra == "ci"
 Requires-Dist: dbt-artifacts-parser; extra == "ci"
@@ -377,7 +380,7 @@ feast materialize 2021-04-12T00:00:00 $CURRENT_TIME
 feast materialize --disable-event-timestamp
 ```
 
-The `--disable-event-timestamp` flag allows you to materialize all available feature data using the current datetime as the event timestamp, without needing to specify start and end timestamps. This is useful when your source data lacks proper event timestamp columns.
+The `--disable-event-timestamp` flag allows you to materialize all available data (from 1970-01-01 up to the current UTC time) without specifying start and end timestamps. Rows keep the event timestamps from the source.
 
 ```commandline
 Materializing feature view driver_hourly_stats from 2021-04-14 to 2021-04-15 done!

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/README.md

@@ -127,7 +127,7 @@ feast materialize 2021-04-12T00:00:00 $CURRENT_TIME
 feast materialize --disable-event-timestamp
 ```
 
-The `--disable-event-timestamp` flag allows you to materialize all available feature data using the current datetime as the event timestamp, without needing to specify start and end timestamps. This is useful when your source data lacks proper event timestamp columns.
+The `--disable-event-timestamp` flag allows you to materialize all available data (from 1970-01-01 up to the current UTC time) without specifying start and end timestamps. Rows keep the event timestamps from the source.
 
 ```commandline
 Materializing feature view driver_hourly_stats from 2021-04-14 to 2021-04-15 done!

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/pixi.lock

@@ -2635,6 +2635,8 @@ packages:
   - psutil ; extra == 'opentelemetry'
   - pyspark>=4.0.0 ; extra == 'spark'
   - pyiceberg>=0.7.0 ; extra == 'iceberg'
+  - pylance>=12.0.0 ; extra == 'lance'
+  - lance-namespace>=0.11.1 ; extra == 'lance'
   - trino>=0.305.0,<0.400.0 ; extra == 'trino'
   - regex ; extra == 'trino'
   - psycopg[binary,pool]>=3.2.5 ; extra == 'postgres'
@@ -2676,7 +2678,7 @@ packages:
   - minio==7.2.11 ; extra == 'test'
   - python-keycloak>=4.7.3,<5 ; extra == 'test'
   - cryptography>=43.0 ; extra == 'test'
-  - feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,glide,go,grpcio,hazelcast,hbase,ibis,iceberg,image,k8s,mcp,mcp-server,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,scylladb,singlestore,snowflake,spark,sqlite-vec,test,trino] ; extra == 'ci'
+  - feast[aws,azure,cassandra,clickhouse,couchbase,delta,docling,duckdb,elasticsearch,faiss,gcp,ge,glide,go,grpcio,hazelcast,hbase,ibis,iceberg,image,k8s,lance,mcp,mcp-server,milvus,mlflow,mongodb,mssql,mysql,openlineage,opentelemetry,oracle,postgres,pytorch,qdrant,rag,ray,redis,scylladb,singlestore,snowflake,spark,sqlite-vec,test,trino] ; extra == 'ci'
   - build ; extra == 'ci'
   - virtualenv==21.14.1 ; extra == 'ci'
   - dbt-artifacts-parser ; extra == 'ci'

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/pyproject.toml

@@ -116,6 +116,10 @@ openlineage = ["openlineage-python>=1.40.0"]
 opentelemetry = ["prometheus_client", "psutil"]
 spark = ["pyspark>=4.0.0"]
 iceberg = ["pyiceberg>=0.7.0"]
+lance = [
+    "pylance>=12.0.0",
+    "lance-namespace>=0.11.1",
+]
 trino = ["trino>=0.305.0,<0.400.0", "regex"]
 postgres = ["psycopg[binary,pool]>=3.2.5"]
 # psycopg[c] install requires a system with a C compiler, python dev headers, & postgresql client dev headers
@@ -178,7 +182,7 @@ test = [
 ]
 
 ci = [
-    "feast[test, aws, azure, cassandra, clickhouse, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, glide, go, grpcio, hazelcast, hbase, ibis, iceberg, image, k8s, mcp, mcp-server, milvus, mlflow, mongodb, mssql, mysql, openlineage, opentelemetry, oracle, scylladb, spark, trino, postgres, pytorch, qdrant, rag, ray, redis, singlestore, snowflake, sqlite_vec]",
+    "feast[test, aws, azure, cassandra, clickhouse, couchbase, delta, docling, duckdb, elasticsearch, faiss, gcp, ge, glide, go, grpcio, hazelcast, hbase, ibis, iceberg, image, k8s, lance, mcp, mcp-server, milvus, mlflow, mongodb, mssql, mysql, openlineage, opentelemetry, oracle, scylladb, spark, trino, postgres, pytorch, qdrant, rag, ray, redis, singlestore, snowflake, sqlite_vec]",
     "build",
     "virtualenv==21.14.1",
     "dbt-artifacts-parser",

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/sdk/python/feast/cli/cli.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 import json
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from importlib.metadata import version as importlib_version
 from pathlib import Path
 from typing import List, Optional
@@ -363,7 +363,7 @@ def registry_dump_command(ctx: click.Context):
 @click.option(
     "--disable-event-timestamp",
     is_flag=True,
-    help="Materialize all available data using current datetime as event timestamp (useful when source data lacks event timestamps)",
+    help="Materialize all available data, from 1970-01-01 up to the current UTC time, without specifying START_TS and END_TS (rows keep their source event timestamps)",
 )
 @click.option(
     "--version",
@@ -388,7 +388,7 @@ def materialize_command(
 
     START_TS and END_TS should be in ISO 8601 format, e.g. '2021-07-16T19:20:01'
 
-    If --disable-event-timestamp is used, timestamps are not required and all available data will be materialized using the current datetime as the event timestamp.
+    If --disable-event-timestamp is used, timestamps are not required and all available data (from 1970-01-01 up to the current UTC time) will be materialized. Rows keep their source event timestamps.
     """
     store = create_feature_store(ctx)
 
@@ -397,12 +397,9 @@ def materialize_command(
             raise click.UsageError(
                 "Cannot specify START_TS or END_TS when --disable-event-timestamp is used"
             )
-        now = datetime.now()
-        # Query all available data and use current datetime as event timestamp
-        start_date = datetime(
-            1970, 1, 1
-        )  # Beginning of time to capture all historical data
-        end_date = now
+        # Query all available data, from the beginning of time up to now (UTC)
+        start_date = datetime(1970, 1, 1, tzinfo=timezone.utc)
+        end_date = utils._utc_now()
     else:
         if not start_ts or not end_ts:
             raise click.UsageError(

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/sdk/python/feast/feature_server.py

@@ -22,7 +22,7 @@ import traceback
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import asynccontextmanager
-from datetime import datetime
+from datetime import datetime, timezone
 from importlib import resources as importlib_resources
 from types import SimpleNamespace
 from typing import Any, DefaultDict, Dict, List, NamedTuple, Optional, Set, Union
@@ -478,8 +478,7 @@ def _parse_materialize_timestamps(
 ) -> tuple:
     """Parse and validate start/end timestamps from a MaterializeRequest."""
     if request.disable_event_timestamp:
-        now = datetime.now()
-        return datetime(1970, 1, 1), now
+        return datetime(1970, 1, 1, tzinfo=timezone.utc), utils._utc_now()
 
     if not request.start_ts or not request.end_ts:
         raise ValueError(

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/sdk/python/feast/feature_store.py

@@ -2855,7 +2855,7 @@ class FeatureStore:
             end_date (datetime): End date for time range of data to materialize into the online store
             feature_views (List[str]): Optional list of feature view names. If selected, will only run
                 materialization for the specified feature views.
-            disable_event_timestamp (bool): If True, materializes all available data using current datetime as event timestamp instead of source event timestamps
+            disable_event_timestamp (bool): If True, materializes all available data (from 1970-01-01 up to the current UTC time). Rows keep their source event timestamps.
             full_feature_names (bool): If True, feature names will be prefixed with the corresponding
                 feature view name.
             version (str): Optional version to materialize (e.g., 'v2'). Requires feature_views

{feast-0.67.0.dev122 → feast-0.67.0.dev123}/sdk/python/feast/infra/compute_engines/dag/model.py

@@ -7,3 +7,4 @@ class DAGFormat(str, Enum):
     ARROW = "arrow"
     RAY = "ray"
     FLINK = "flink"
+    TRINO = "trino"

feast-0.67.0.dev123/sdk/python/feast/infra/compute_engines/trino/__init__.py

@@ -0,0 +1,6 @@
+from feast.infra.compute_engines.trino.compute import (
+    TrinoComputeEngine,
+    TrinoComputeEngineConfig,
+)
+
+__all__ = ["TrinoComputeEngine", "TrinoComputeEngineConfig"]

feast-0.67.0.dev123/sdk/python/feast/infra/compute_engines/trino/compute.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, Literal, Optional, Sequence, Union
+
+from pydantic import ConfigDict, Field, StrictStr
+from trino.exceptions import TrinoConnectionError
+
+from feast import (
+    BatchFeatureView,
+    Entity,
+    FeatureView,
+    OnDemandFeatureView,
+    StreamFeatureView,
+)
+from feast.infra.common.materialization_job import (
+    MaterializationJob,
+    MaterializationJobStatus,
+    MaterializationTask,
+)
+from feast.infra.common.retrieval_task import HistoricalRetrievalTask
+from feast.infra.compute_engines.base import ComputeEngine
+from feast.infra.compute_engines.trino.feature_builder import TrinoFeatureBuilder
+from feast.infra.compute_engines.trino.job import (
+    TrinoDAGRetrievalJob,
+    TrinoMaterializationJob,
+)
+from feast.infra.compute_engines.trino.utils import (
+    get_or_create_trino_client,
+    stream_trino_arrow_batches,
+    write_arrow_batches_to_online_store,
+)
+from feast.infra.offline_stores.contrib.trino_offline_store.trino import (
+    AuthConfig,
+    TrinoOfflineStoreConfig,
+)
+from feast.infra.offline_stores.contrib.trino_offline_store.trino_queries import Trino
+from feast.infra.offline_stores.offline_store import OfflineStore, RetrievalJob
+from feast.infra.online_stores.online_store import OnlineStore
+from feast.infra.registry.base_registry import BaseRegistry
+from feast.repo_config import FeastConfigBaseModel, RepoConfig
+from feast.utils import _get_column_names
+
+logger = logging.getLogger(__name__)
+
+
+class TrinoComputeEngineConfig(FeastConfigBaseModel):
+    """Configuration for Trino Compute Engine."""
+
+    type: Literal["trino.engine", "trino"] = "trino.engine"
+    """ Compute engine type selector """
+
+    host: Optional[StrictStr] = None
+    """ Host of the Trino coordinator """
+
+    port: Optional[int] = 8080
+    """ Port of the Trino coordinator """
+
+    catalog: Optional[StrictStr] = None
+    """ Target Trino catalog (e.g. iceberg, hive, delta, memory) """
+
+    dataset: Optional[StrictStr] = "feast"
+    """ Schema/dataset name for temporary staging tables """
+
+    user: Optional[StrictStr] = None
+    """ Username for connecting to Trino """
+
+    source: Optional[StrictStr] = "trino-python-client"
+    """ Application source identifier for Trino query tracking """
+
+    http_scheme: Literal["http", "https"] = Field(default="http", alias="http-scheme")
+    """ Protocol scheme (http or https) """
+
+    verify: bool = True
+    """ Whether to verify SSL certificates """
+
+    extra_credential: Optional[StrictStr] = None
+    """ Optional extra credential header passed to Trino """
+
+    connector: Optional[Dict[str, str]] = None
+    """ Connector arguments for Trino offline store table operations """
+
+    auth: Optional[AuthConfig] = None
+    """ Authentication mechanism (kerberos, basic, oauth2) """
+
+    batch_size: int = 10000
+    """ Streaming batch chunk size for PyArrow record batches """
+
+    write_concurrency: int = 4
+    """ Number of concurrent threads used when writing batches to online store """
+
+    offline_write_mode: Literal["append", "overwrite", "merge"] = "append"
+    """ Write mode for offline feature view materialization ('append', 'overwrite', or 'merge') """
+
+    model_config = ConfigDict(populate_by_name=True, extra="allow")
+
+
+class TrinoComputeEngine(ComputeEngine):
+    """Distributed Trino SQL compute engine for Feast materialization and retrieval."""
+
+    def __init__(
+        self,
+        *,
+        repo_config: RepoConfig,
+        offline_store: OfflineStore,
+        online_store: OnlineStore,
+        **kwargs: Any,
+    ):
+        super().__init__(
+            repo_config=repo_config,
+            offline_store=offline_store,
+            online_store=online_store,
+            **kwargs,
+        )
+
+        batch_engine_config = repo_config.batch_engine
+        if isinstance(batch_engine_config, TrinoComputeEngineConfig):
+            self.config = batch_engine_config
+        elif isinstance(batch_engine_config, dict):
+            self.config = TrinoComputeEngineConfig(**batch_engine_config)
+        elif isinstance(batch_engine_config, str):
+            self.config = TrinoComputeEngineConfig()
+        else:
+            self.config = TrinoComputeEngineConfig()
+
+        # Automatic fallback: Inherit connection details from Trino offline store if unspecified
+        if (
+            isinstance(repo_config.offline_store, TrinoOfflineStoreConfig)
+            and self.config.host is None
+        ):
+            offline_conf = repo_config.offline_store
+            self.config.host = offline_conf.host
+            self.config.port = offline_conf.port
+            self.config.catalog = offline_conf.catalog
+            self.config.user = offline_conf.user
+            self.config.source = offline_conf.source
+            self.config.http_scheme = offline_conf.http_scheme
+            self.config.verify = offline_conf.verify
+            self.config.extra_credential = offline_conf.extra_credential
+            self.config.connector = offline_conf.connector
+            self.config.dataset = offline_conf.dataset
+            self.config.auth = offline_conf.auth
+
+        self.client: Optional[Trino] = None
+        if self.config.host and self.config.catalog and self.config.user:
+            self.client = get_or_create_trino_client(self.config)
+
+    def update(
+        self,
+        project: str,
+        views_to_delete: Sequence[
+            Union[BatchFeatureView, StreamFeatureView, FeatureView]
+        ],
+        views_to_keep: Sequence[
+            Union[BatchFeatureView, StreamFeatureView, FeatureView, OnDemandFeatureView]
+        ],
+        entities_to_delete: Sequence[Entity],
+        entities_to_keep: Sequence[Entity],
+    ) -> None:
+        """Trino compute engine does not require persistent infrastructure updates."""
+        logger.debug("Trino compute engine does not require updating")
+        pass
+
+    def teardown_infra(
+        self,
+        project: str,
+        fvs: Sequence[Union[BatchFeatureView, StreamFeatureView, FeatureView]],
+        entities: Sequence[Entity],
+    ) -> None:
+        """Trino compute engine does not require infrastructure teardown."""
+        logger.debug("Trino compute engine does not require teardown")
+        pass
+
+    def _materialize_one(
+        self,
+        registry: BaseRegistry,
+        task: MaterializationTask,
+        from_offline_store: bool = False,
+        **kwargs: Any,
+    ) -> MaterializationJob:
+        if from_offline_store:
+            return self._materialize_from_offline_store(
+                registry=registry,
+                feature_view=task.feature_view,
+                start_date=task.start_time,
+                end_date=task.end_time,
+                project=task.project,
+            )
+
+        job_id = f"{task.feature_view.name}-{task.start_time}-{task.end_time}"
+        context = self.get_execution_context(registry, task)
+
+        try:
+            if self.client is None:
+                raise RuntimeError(
+                    "Trino client is not configured. Set host, catalog, and user "
+                    "in batch_engine config or use a TrinoOfflineStoreConfig."
+                )
+            builder = TrinoFeatureBuilder(
+                registry=registry,
+                client=self.client,
+                task=task,
+            )
+            plan = builder.build()
+            plan.execute(context)
+
+            return TrinoMaterializationJob(
+                job_id=job_id, status=MaterializationJobStatus.SUCCEEDED
+            )
+        except TrinoConnectionError as ce:
+            logger.error("Connection to Trino failed during materialization: %s", ce)
+            return TrinoMaterializationJob(
+                job_id=job_id, status=MaterializationJobStatus.ERROR, error=ce
+            )
+        except Exception as e:
+            logger.debug("Materialization failed for %s: %s", task.feature_view.name, e)
+            return TrinoMaterializationJob(
+                job_id=job_id, status=MaterializationJobStatus.ERROR, error=e
+            )
+
+    def _materialize_from_offline_store(
+        self,
+        registry: BaseRegistry,
+        feature_view: Union[BatchFeatureView, StreamFeatureView, FeatureView],
+        start_date: datetime,
+        end_date: datetime,
+        project: str,
+    ) -> MaterializationJob:
+        """Legacy materialization fallback method."""
+        logger.warning(
+            "Materializing from offline store will be deprecated in the future. "
+            "Please use the unified DAG materialization engine."
+        )
+        entities = []
+        for entity_name in feature_view.entities:
+            entities.append(registry.get_entity(entity_name, project))
+
+        (
+            join_key_columns,
+            feature_name_columns,
+            timestamp_field,
+            created_timestamp_column,
+        ) = _get_column_names(feature_view, entities)
+
+        job_id = f"{feature_view.name}-{start_date}-{end_date}"
+
+        try:
+            offline_job = self.offline_store.pull_latest_from_table_or_query(
+                config=self.repo_config,
+                data_source=feature_view.batch_source,  # type: ignore[arg-type]
+                join_key_columns=join_key_columns,
+                feature_name_columns=feature_name_columns,
+                timestamp_field=timestamp_field,
+                created_timestamp_column=created_timestamp_column,
+                start_date=start_date,
+                end_date=end_date,
+            )
+
+            if not self.client:
+                raise RuntimeError("Trino client required for materialization.")
+
+            batches = stream_trino_arrow_batches(
+                client=self.client,
+                query_text=offline_job.to_sql(),
+                batch_size=self.config.batch_size,
+            )
+            write_arrow_batches_to_online_store(
+                batches=batches,
+                feature_view=feature_view,  # type: ignore[arg-type]
+                online_store=self.online_store,
+                repo_config=self.repo_config,
+                concurrency=self.config.write_concurrency,
+            )
+
+            return TrinoMaterializationJob(
+                job_id=job_id, status=MaterializationJobStatus.SUCCEEDED
+            )
+        except Exception as e:
+            return TrinoMaterializationJob(
+                job_id=job_id, status=MaterializationJobStatus.ERROR, error=e
+            )
+
+    def get_historical_features(
+        self, registry: BaseRegistry, task: HistoricalRetrievalTask
+    ) -> RetrievalJob:
+        """Compile and return lazy Trino historical retrieval job."""
+        context = self.get_execution_context(registry, task)
+
+        try:
+            if self.client is None:
+                raise RuntimeError(
+                    "Trino client is not configured. Set host, catalog, and user "
+                    "in batch_engine config or use a TrinoOfflineStoreConfig."
+                )
+            builder = TrinoFeatureBuilder(
+                registry=registry,
+                client=self.client,
+                task=task,
+            )
+            plan = builder.build()
+
+            return TrinoDAGRetrievalJob(
+                client=self.client,
+                plan=plan,
+                context=context,
+                config=self.repo_config,
+                full_feature_names=task.full_feature_name,
+                on_demand_feature_views=getattr(task, "on_demand_feature_views", None),
+                metadata=getattr(task, "metadata", None),
+            )
+        except Exception as e:
+            logger.error("Failed to build historical feature retrieval DAG: %s", e)
+            return TrinoDAGRetrievalJob(
+                client=self.client,
+                plan=None,
+                context=context,
+                config=self.repo_config,
+                full_feature_names=task.full_feature_name,
+                on_demand_feature_views=getattr(task, "on_demand_feature_views", None),
+                metadata=getattr(task, "metadata", None),
+                error=e,
+            )