qwak-core 0.5.4__py3-none-any.whl → 0.5.12__py3-none-any.whl

_qwak_proto/qwak/model_group/model_group_repository_details_pb2.py

@@ -13,7 +13,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5qwak/model_group/model_group_repository_details.proto\x12\x1cqwak.model_groups.management\"r\n\x11RepositoryDetails\x12\x45\n\x0frepository_type\x18\x01 \x01(\x0b\x32,.qwak.model_groups.management.RepositoryType\x12\x16\n\x0erepository_key\x18\x02 \x01(\t\"\xf0\x01\n\x17RemoteRepositoryDetails\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x02 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x1d\n\x15repository_remote_url\x18\x03 \x01(\tB\x11\n\x0frepository_type\"\x12\n\x10\x44ockerRepository\"\x13\n\x11\x44\x61tasetRepository\"\x14\n\x12\x41rtifactRepository\"\x17\n\x15HuggingFaceRepository\"\xdd\x02\n\x0eRepositoryType\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12M\n\x12\x64\x61taset_repository\x18\x02 \x01(\x0b\x32/.qwak.model_groups.management.DatasetRepositoryH\x00\x12O\n\x13\x61rtifact_repository\x18\x03 \x01(\x0b\x32\x30.qwak.model_groups.management.ArtifactRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x04 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x42\x06\n\x04typeBN\n&com.qwak.ai.management.model.group.apiB\"ModelGroupRepositoriesDetailsProtoP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5qwak/model_group/model_group_repository_details.proto\x12\x1cqwak.model_groups.management\"r\n\x11RepositoryDetails\x12\x45\n\x0frepository_type\x18\x01 \x01(\x0b\x32,.qwak.model_groups.management.RepositoryType\x12\x16\n\x0erepository_key\x18\x02 \x01(\t\"\x80\x03\n\x17RemoteRepositoryDetails\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x02 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x45\n\x0enpm_repository\x18\x04 \x01(\x0b\x32+.qwak.model_groups.management.NpmRepositoryH\x00\x12G\n\x0fpypi_repository\x18\x05 \x01(\x0b\x32,.qwak.model_groups.management.PypiRepositoryH\x00\x12\x1d\n\x15repository_remote_url\x18\x03 \x01(\tB\x11\n\x0frepository_type\"\x12\n\x10\x44ockerRepository\"\x13\n\x11\x44\x61tasetRepository\"\x14\n\x12\x41rtifactRepository\"\x17\n\x15HuggingFaceRepository\"\x0f\n\rNpmRepository\"\x10\n\x0ePypiRepository\"\xed\x03\n\x0eRepositoryType\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12M\n\x12\x64\x61taset_repository\x18\x02 \x01(\x0b\x32/.qwak.model_groups.management.DatasetRepositoryH\x00\x12O\n\x13\x61rtifact_repository\x18\x03 \x01(\x0b\x32\x30.qwak.model_groups.management.ArtifactRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x04 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x45\n\x0enpm_repository\x18\x05 \x01(\x0b\x32+.qwak.model_groups.management.NpmRepositoryH\x00\x12G\n\x0fpypi_repository\x18\x06 \x01(\x0b\x32,.qwak.model_groups.management.PypiRepositoryH\x00\x42\x06\n\x04typeBN\n&com.qwak.ai.management.model.group.apiB\"ModelGroupRepositoriesDetailsProtoP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -25,15 +25,19 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_REPOSITORYDETAILS']._serialized_start=87
   _globals['_REPOSITORYDETAILS']._serialized_end=201
   _globals['_REMOTEREPOSITORYDETAILS']._serialized_start=204
-  _globals['_REMOTEREPOSITORYDETAILS']._serialized_end=444
-  _globals['_DOCKERREPOSITORY']._serialized_start=446
-  _globals['_DOCKERREPOSITORY']._serialized_end=464
-  _globals['_DATASETREPOSITORY']._serialized_start=466
-  _globals['_DATASETREPOSITORY']._serialized_end=485
-  _globals['_ARTIFACTREPOSITORY']._serialized_start=487
-  _globals['_ARTIFACTREPOSITORY']._serialized_end=507
-  _globals['_HUGGINGFACEREPOSITORY']._serialized_start=509
-  _globals['_HUGGINGFACEREPOSITORY']._serialized_end=532
-  _globals['_REPOSITORYTYPE']._serialized_start=535
-  _globals['_REPOSITORYTYPE']._serialized_end=884
+  _globals['_REMOTEREPOSITORYDETAILS']._serialized_end=588
+  _globals['_DOCKERREPOSITORY']._serialized_start=590
+  _globals['_DOCKERREPOSITORY']._serialized_end=608
+  _globals['_DATASETREPOSITORY']._serialized_start=610
+  _globals['_DATASETREPOSITORY']._serialized_end=629
+  _globals['_ARTIFACTREPOSITORY']._serialized_start=631
+  _globals['_ARTIFACTREPOSITORY']._serialized_end=651
+  _globals['_HUGGINGFACEREPOSITORY']._serialized_start=653
+  _globals['_HUGGINGFACEREPOSITORY']._serialized_end=676
+  _globals['_NPMREPOSITORY']._serialized_start=678
+  _globals['_NPMREPOSITORY']._serialized_end=693
+  _globals['_PYPIREPOSITORY']._serialized_start=695
+  _globals['_PYPIREPOSITORY']._serialized_end=711
+  _globals['_REPOSITORYTYPE']._serialized_start=714
+  _globals['_REPOSITORYTYPE']._serialized_end=1207
 # @@protoc_insertion_point(module_scope)

_qwak_proto/qwak/model_group/model_group_repository_details_pb2.pyi

@@ -40,6 +40,8 @@ class RemoteRepositoryDetails(google.protobuf.message.Message):
 
     DOCKER_REPOSITORY_FIELD_NUMBER: builtins.int
     HUGGING_FACE_REPOSITORY_FIELD_NUMBER: builtins.int
+    NPM_REPOSITORY_FIELD_NUMBER: builtins.int
+    PYPI_REPOSITORY_FIELD_NUMBER: builtins.int
     REPOSITORY_REMOTE_URL_FIELD_NUMBER: builtins.int
     @property
     def docker_repository(self) -> global___DockerRepository:
@@ -47,6 +49,12 @@ class RemoteRepositoryDetails(google.protobuf.message.Message):
     @property
     def hugging_face_repository(self) -> global___HuggingFaceRepository:
         """HuggingFace repository"""
+    @property
+    def npm_repository(self) -> global___NpmRepository:
+        """NPM repository"""
+    @property
+    def pypi_repository(self) -> global___PypiRepository:
+        """PyPI repository"""
     repository_remote_url: builtins.str
     """The remote repository URL"""
     def __init__(
@@ -54,11 +62,13 @@ class RemoteRepositoryDetails(google.protobuf.message.Message):
         *,
         docker_repository: global___DockerRepository | None = ...,
         hugging_face_repository: global___HuggingFaceRepository | None = ...,
+        npm_repository: global___NpmRepository | None = ...,
+        pypi_repository: global___PypiRepository | None = ...,
         repository_remote_url: builtins.str = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "repository_type", b"repository_type"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "repository_remote_url", b"repository_remote_url", "repository_type", b"repository_type"]) -> None: ...
-    def WhichOneof(self, oneof_group: typing_extensions.Literal["repository_type", b"repository_type"]) -> typing_extensions.Literal["docker_repository", "hugging_face_repository"] | None: ...
+    def HasField(self, field_name: typing_extensions.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "repository_type", b"repository_type"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "repository_remote_url", b"repository_remote_url", "repository_type", b"repository_type"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["repository_type", b"repository_type"]) -> typing_extensions.Literal["docker_repository", "hugging_face_repository", "npm_repository", "pypi_repository"] | None: ...
 
 global___RemoteRepositoryDetails = RemoteRepositoryDetails
 
@@ -98,6 +108,24 @@ class HuggingFaceRepository(google.protobuf.message.Message):
 
 global___HuggingFaceRepository = HuggingFaceRepository
 
+class NpmRepository(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    def __init__(
+        self,
+    ) -> None: ...
+
+global___NpmRepository = NpmRepository
+
+class PypiRepository(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    def __init__(
+        self,
+    ) -> None: ...
+
+global___PypiRepository = PypiRepository
+
 class RepositoryType(google.protobuf.message.Message):
     """The repository type"""
 
@@ -107,6 +135,8 @@ class RepositoryType(google.protobuf.message.Message):
     DATASET_REPOSITORY_FIELD_NUMBER: builtins.int
     ARTIFACT_REPOSITORY_FIELD_NUMBER: builtins.int
     HUGGING_FACE_REPOSITORY_FIELD_NUMBER: builtins.int
+    NPM_REPOSITORY_FIELD_NUMBER: builtins.int
+    PYPI_REPOSITORY_FIELD_NUMBER: builtins.int
     @property
     def docker_repository(self) -> global___DockerRepository:
         """Docker repository"""
@@ -119,6 +149,12 @@ class RepositoryType(google.protobuf.message.Message):
     @property
     def hugging_face_repository(self) -> global___HuggingFaceRepository:
         """HuggingFace repository"""
+    @property
+    def npm_repository(self) -> global___NpmRepository:
+        """NPM repository"""
+    @property
+    def pypi_repository(self) -> global___PypiRepository:
+        """PyPI repository"""
     def __init__(
         self,
         *,
@@ -126,9 +162,11 @@ class RepositoryType(google.protobuf.message.Message):
         dataset_repository: global___DatasetRepository | None = ...,
         artifact_repository: global___ArtifactRepository | None = ...,
         hugging_face_repository: global___HuggingFaceRepository | None = ...,
+        npm_repository: global___NpmRepository | None = ...,
+        pypi_repository: global___PypiRepository | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing_extensions.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "type", b"type"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing_extensions.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "type", b"type"]) -> None: ...
-    def WhichOneof(self, oneof_group: typing_extensions.Literal["type", b"type"]) -> typing_extensions.Literal["docker_repository", "dataset_repository", "artifact_repository", "hugging_face_repository"] | None: ...
+    def HasField(self, field_name: typing_extensions.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "type", b"type"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "type", b"type"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing_extensions.Literal["type", b"type"]) -> typing_extensions.Literal["docker_repository", "dataset_repository", "artifact_repository", "hugging_face_repository", "npm_repository", "pypi_repository"] | None: ...
 
 global___RepositoryType = RepositoryType

qwak/__init__.py

@@ -1,7 +1,7 @@
 """Top-level package for qwak-core."""
 
 __author__ = "Qwak.ai"
-__version__ = "0.5.4"
+__version__ = "0.5.12"
 
 from qwak.inner.di_configuration import wire_dependencies
 from qwak.model.experiment_tracking import log_metric, log_param

qwak/clients/feature_store/execution_management_client.py

@@ -1,4 +1,7 @@
 from _qwak_proto.qwak.execution.v1.backfill_pb2 import BackfillSpec
+from _qwak_proto.qwak.execution.v1.streaming_aggregation_pb2 import (
+    StreamingAggregationBackfillIngestion,
+)
 from _qwak_proto.qwak.execution.v1.batch_pb2 import BatchIngestion
 from _qwak_proto.qwak.execution.v1.execution_service_pb2 import (
     GetExecutionEntryRequest,
@@ -9,6 +12,8 @@ from _qwak_proto.qwak.execution.v1.execution_service_pb2 import (
     TriggerBackfillResponse,
     TriggerBatchFeaturesetRequest,
     TriggerBatchFeaturesetResponse,
+    TriggerStreamingAggregationBackfillRequest,
+    TriggerStreamingAggregationBackfillResponse,
 )
 from _qwak_proto.qwak.execution.v1.execution_service_pb2_grpc import (
     FeatureStoreExecutionServiceStub,
@@ -29,6 +34,29 @@ class ExecutionManagementClient:
             grpc_channel
         )
 
+    def trigger_streaming_aggregation_backfill(
+        self, backfill_ingestion: StreamingAggregationBackfillIngestion
+    ) -> TriggerStreamingAggregationBackfillResponse:
+        """
+        Receives a configured streaming aggregation backfill proto and triggers a streaming aggregation
+        backfill against the execution manager
+
+        Args:
+            backfill_ingestion (StreamingAggregationBackfillIngestion): A protobuf message
+                containing the backfill specification details
+
+        Returns:
+            TriggerStreamingAggregationBackfillResponse: response object from the execution manager
+        """
+        try:
+            return self._feature_store_execution_service.TriggerStreamingAggregationBackfill(
+                TriggerStreamingAggregationBackfillRequest(backfill=backfill_ingestion)
+            )
+        except RpcError as e:
+            raise QwakException(
+                f"Failed to trigger streaming aggregation backfill job, error encountered {e}"
+            )
+
     def trigger_batch_backfill(
         self, batch_backfill_spec: BackfillSpec
     ) -> TriggerBackfillResponse:

qwak/feature_store/execution/streaming_backfill.py

@@ -0,0 +1,48 @@
+import pathlib
+
+from qwak.clients.feature_store.execution_management_client import (
+    ExecutionManagementClient,
+)
+from qwak.feature_store.feature_sets.streaming_backfill import StreamingBackfill
+
+
+class StreamingAggregationBackfill:
+    def __init__(
+        self,
+        streaming_backfill: StreamingBackfill,
+        source_definition_path: pathlib.Path,
+    ):
+        """
+        Initialize the streaming aggregation backfill executor.
+
+        Args:
+            streaming_backfill (StreamingBackfill): Specification containing the
+                              featureset name, time range, data sources, and transformation
+            source_definition_path (Path): Path to the Python file containing the backfill
+                                   definition. Required for locating UDF artifacts.
+        """
+        self._streaming_backfill = streaming_backfill
+        self._source_definition_path = source_definition_path
+
+    def trigger(self) -> str:
+        """
+        Triggers the streaming aggregation backfill execution.
+
+        Converts the backfill specification to proto format and sends it to
+        the execution manager to start the backfill job.
+
+        Returns:
+            str: The execution ID for tracking the backfill job status
+
+        Raises:
+            QwakException: If the execution manager request fails
+        """
+        backfill_proto = self._streaming_backfill._to_proto(
+            str(self._source_definition_path)
+        )
+
+        execution_client = ExecutionManagementClient()
+        response = execution_client.trigger_streaming_aggregation_backfill(
+            backfill_proto
+        )
+        return response.execution_id

qwak/feature_store/feature_sets/streaming.py

@@ -2,7 +2,7 @@ import collections
 import functools
 import inspect
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 from _qwak_proto.qwak.feature_store.features.execution_pb2 import (
@@ -21,6 +21,7 @@ from _qwak_proto.qwak.feature_store.sources.streaming_pb2 import (
     StreamingSource,
     StreamingSource as ProtoStreamingSource,
 )
+from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
 from qwak.clients.feature_store import FeatureRegistryClient
 from qwak.exceptions import QwakException
 from qwak.feature_store._common.artifact_utils import ArtifactSpec, ArtifactsUploader
@@ -34,7 +35,7 @@ from qwak.feature_store.feature_sets.metadata import (
     set_metadata_on_function,
 )
 from qwak.feature_store.feature_sets.streaming_backfill import (
-    BackfillBatchDataSourceSpec,
+    BackfillDataSource,
     StreamingBackfill,
 )
 from qwak.feature_store.feature_sets.transformations import (
@@ -75,6 +76,7 @@ def feature_set(
     key: Optional[str] = None,
     auxiliary_sinks: List[BaseSink] = [],
     repository: Optional[str] = None,
+    backfill_max_timestamp: Optional[datetime] = None,
 ):
     """
     Creates a streaming feature set for the specified entity using the given streaming data sources.
@@ -110,6 +112,11 @@ def feature_set(
     """
 
     def decorator(function):
+        if isinstance(function, StreamingBackfill):
+            raise QwakException(
+                "Backfill can no longer be defined as a decorator on the feature set, it must be triggered after feature set creation."
+            )
+
         user_transformation = function()
         FeaturesetUtils.validate_base_featureset_decorator(
             user_transformation=user_transformation, entity=entity, key=key
@@ -120,10 +127,6 @@ def feature_set(
             offline_scheduling_policy=offline_scheduling_policy,
         )
 
-        streaming_backfill: Optional[StreamingBackfill] = (
-            StreamingBackfill.get_streaming_backfill_from_function(function=function)
-        )
-
         fs_name = name or function.__name__
         streaming_feature_set = StreamingFeatureSet(
             name=fs_name,
@@ -150,7 +153,7 @@ def feature_set(
             online_cluster_template=getattr(
                 function, _ONLINE_CLUSTER_SPEC, ClusterTemplate.SMALL
             ),
-            backfill=streaming_backfill,
+            backfill_max_timestamp=backfill_max_timestamp,
             __instance_module_path__=inspect.stack()[1].filename,
             auxiliary_sinks=auxiliary_sinks,
         )
@@ -197,55 +200,68 @@ def execution_specification(
 @typechecked
 def backfill(
     *,
-    start_date: datetime,
-    end_date: datetime,
-    data_sources: Union[List[str], List[BackfillBatchDataSourceSpec]],
-    backfill_transformation: SparkSqlTransformation,
+    feature_set_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+    data_sources: Union[List[str], List[BackfillDataSource]],
     backfill_cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL,
 ):
     """
-    Configures optional backfill specification for a streaming featureset. Currently available for streaming
+    Triggers a backfill execution for an existing streaming featureset. Currently available for streaming
     aggregation featuresets only.
 
-    :param start_date: backfill start date
-    :param end_date: backfill end date
-    :param data_sources: a list of dict representations connecting each batch source name to its requested time range
-    :param backfill_transformation: a backfill SQL transformation. Only SQL transformations are supported for backfill
-    :param backfill_cluster_template: an optional cluster specification for the backfill job. Defaults to SMALL.
-
-    Example:
-
-    ... code-block:: python
-
-        @streaming.feature_set(
-            entity="users",
-            data_sources=["users_registration_stream"],
-            timestamp_column_name="reg_date"
-        )
+    Args:
+        feature_set_name (str): Name of the FeatureSet to trigger a backfill for.
+        start_date (datetime): Backfill start date, on Streaming Aggregation Feature Sets,
+            needs to align with the FeatureSet tiles.
+        end_date (datetime): Backfill end date, on Streaming Aggregation Feature Sets,
+            needs to align with the FeatureSet tiles and be smaller than the Feature Set's backfill_max_timestamp.
+        data_sources (list[BackfillDataSource] | list[str]): A list of BackfillDataSource objects containing
+            batch source name and optional time range, or a list of batch source names (with no time range limits).
+        backfill_cluster_template (ClusterTemplate, optional): An optional cluster specification for the backfill job.
+            Defaults to SMALL.
+
+    Examples:
         @streaming.backfill(
-            start_date=start_date=(2022,01,01,0,0,0),
-            end_date=start_date=(2023,09,01,0,0,0),
-            data_sources=[{"batch_backfill_source_name": BackfillSourceRange(start_date=(2023,01,01,0,0,0),
-                                                                             end_date=(2023,08,01,0,0,0))}],
+            feature_set_name="user_streaming_agg_features",
+            start_date=datetime(2022,1,1,0,0,0),
+            end_date=datetime(2023,9,1,0,0,0),
+            data_sources=[BackfillDataSource(data_source_name="backfill_data_source",
+                                             start_datetime=datetime(2023,1,1,0,0,0),
+                                             end_datetime=datetime(2023,8,1,0,0,0))],
             backfill_cluster_template=ClusterTemplate.SMALL
-            backfill_transformation=SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM backfill_data_source")
         )
-        def user_streaming_features():
-            return SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM data_source")
+        def backfill_transformation():
+            return SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM backfill_data_source")
     """
 
     def decorator(function):
-        _validate_decorator_ordering(function)
-        StreamingBackfill.set_streaming_backfill_on_function(
-            function=function,
-            start_date=start_date,
-            end_date=end_date,
-            data_sources=data_sources,
-            backfill_transformation=backfill_transformation,
-            backfill_cluster_template=backfill_cluster_template,
+        if isinstance(function, StreamingFeatureSet):
+            raise QwakException(
+                "Backfill can no longer be defined as a decorator on the feature set, it must be triggered after feature set creation."
+            )
+
+        backfill_transformation: SparkSqlTransformation = function()
+
+        if not isinstance(backfill_transformation, SparkSqlTransformation):
+            raise QwakException(
+                "Backfill must defined on a method returning a SparkSqlTransformation"
+            )
+
+        streaming_backfill = StreamingBackfill(
+            featureset_name=feature_set_name,
+            start_datetime=start_date,
+            end_datetime=end_date,
+            data_sources=StreamingBackfill._get_normalized_backfill_sources_spec(
+                data_sources
+            ),
+            transform=backfill_transformation,
+            cluster_template=backfill_cluster_template,
         )
 
-        return function
+        functools.update_wrapper(streaming_backfill, backfill_transformation)
+
+        return streaming_backfill
 
     return decorator
 
@@ -316,7 +332,7 @@ class StreamingFeatureSet(BaseFeatureSet):
     offline_cluster_template: Optional[ClusterTemplate] = None
     online_cluster_template: Optional[ClusterTemplate] = None
     metadata: Optional[Metadata] = None
-    backfill: Optional[StreamingBackfill] = None
+    backfill_max_timestamp: Optional[datetime] = None
     auxiliary_sinks: List[BaseSink] = field(default_factory=lambda: [])
 
     def __post_init__(self):
@@ -399,7 +415,6 @@ class StreamingFeatureSet(BaseFeatureSet):
             proto_featureset_type = self._get_streaming_aggregation_featureset_proto(
                 artifact_url=artifact_url,
                 streaming_sources=data_sources,
-                feature_registry=feature_registry,
                 initial_tile_size=maybe_initial_tile_size,
             )
 
@@ -453,10 +468,9 @@ class StreamingFeatureSet(BaseFeatureSet):
                 "Auxiliary Sinks Are not supported in Streaming Aggregation Feature Sets"
             )
 
-        # streaming backfill not yet supported for row-level streaming
-        if self.backfill and not is_streaming_agg:
+        if self.backfill_max_timestamp and not is_streaming_agg:
             raise QwakException(
-                "Streaming backfill is only supported for streaming aggregation feature sets at the moment"
+                "backfill_max_timestamp can only be set for Streaming Aggregation FeatureSet."
             )
 
         # Validate transformation is PySpark when multiple data sources are used
@@ -515,18 +529,29 @@ class StreamingFeatureSet(BaseFeatureSet):
                 )
             raise QwakException(error_message_str)
 
-        # Validate the backfill, if defined
-        if self.backfill:
-            self.backfill._validate_tile_size(initial_tile_size)
+        if not self.backfill_max_timestamp:
+            raise QwakException(
+                """
+                backfill_max_timestamp must be set for Streaming Aggregation FeatureSet.
+                 Events earlier than this timestamp can only be processed by triggering backfill,
+                 the Streaming job will not process events that are earlier than this timestamp.
+            """
+            )
+
+        self._validate_streaming_aggregation_backfill_max_timestamp()
 
         return initial_tile_size
 
-    def _validate_streaming_aggregation_backfill(self):
+    def _validate_streaming_aggregation_backfill_max_timestamp(self):
         initial_tile_size, _ = StreamingFeatureSet._get_default_slide_period(
             self.transformation.windows
         )
 
-        self.backfill._validate_tile_size(initial_tile_size)
+        if self.backfill_max_timestamp.timestamp() % initial_tile_size != 0:
+            raise QwakException(
+                f"Chosen backfill max timestamp is invalid,"
+                f" it has to be exactly dividable by slice size of {initial_tile_size} seconds."
+            )
 
     @staticmethod
     def _get_default_slide_period(
@@ -596,9 +621,12 @@ class StreamingFeatureSet(BaseFeatureSet):
         self,
         artifact_url: Optional[str],
         streaming_sources: List[StreamingSource],
-        feature_registry: FeatureRegistryClient,
         initial_tile_size: int,
     ) -> ProtoFeatureSetType:
+        backfill_max_timestamp = ProtoTimestamp()
+        backfill_max_timestamp.FromDatetime(
+            self.backfill_max_timestamp.astimezone(timezone.utc)
+        )
         return ProtoFeatureSetType(
             streaming_aggregation_feature_set=ProtoStreamingAggregationFeatureSet(
                 transformation=self.transformation._to_proto(
@@ -621,14 +649,7 @@ class StreamingFeatureSet(BaseFeatureSet):
                     allowed_late_arrival_seconds=60 * 10,
                     aggregations=self.transformation._get_aggregations_proto(),
                 ),
-                backfill_spec=(
-                    self.backfill._to_proto(
-                        feature_registry=feature_registry,
-                        original_instance_module_path=self.__instance_module_path__,
-                        featureset_name=self.name,
-                    )
-                    if self.backfill
-                    else None
-                ),
+                backfill_spec=None,
+                backfill_max_timestamp=backfill_max_timestamp,
             )
         )

qwak/feature_store/feature_sets/streaming_backfill.py

@@ -1,24 +1,18 @@
-from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import List, Optional, Set, Union
 
 from _qwak_proto.qwak.feature_store.features.execution_pb2 import (
-    BackfillExecutionSpec as ProtoBackfillExecutionSpec,
+    ExecutionSpec as ProtoExecutionSpec,
 )
-from _qwak_proto.qwak.feature_store.features.feature_set_types_pb2 import (
-    BackfillBatchDataSourceSpec as ProtoBackfillBatchDataSourceSpec,
-    BackfillDataSourceSpec as ProtoBackfillDataSourceSpec,
-    BackfillSpec as ProtoBackfillSpec,
-)
-from _qwak_proto.qwak.feature_store.sources.batch_pb2 import (
-    BatchSource as ProtoBatchSource,
+from _qwak_proto.qwak.execution.v1.streaming_aggregation_pb2 import (
+    StreamingAggregationBackfillIngestion as ProtoStreamingAggregationBackfillIngestion,
+    BackfillDataSource as ProtoBackfillDataSource,
+    TimeRange as ProtoTimeRange,
 )
 from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
-from qwak.clients.feature_store import FeatureRegistryClient
 from qwak.exceptions import QwakException
 from qwak.feature_store._common.artifact_utils import ArtifactSpec, ArtifactsUploader
-from qwak.feature_store._common.feature_set_utils import get_batch_source_for_featureset
 from qwak.feature_store.feature_sets.execution_spec import ClusterTemplate
 from qwak.feature_store.feature_sets.transformations import SparkSqlTransformation
 
@@ -26,36 +20,15 @@ _BACKFILL_ = "_qwak_backfill_specification"
 
 
 @dataclass
-class DataSourceBackfillSpec(ABC):
+class BackfillDataSource:
     data_source_name: str
-
-    @abstractmethod
-    def _to_proto(self, feature_registry: FeatureRegistryClient):
-        pass
-
-    @classmethod
-    def _from_proto(cls, proto: ProtoBackfillDataSourceSpec):
-        function_mapping = {"batch_data_source_spec": BackfillBatchDataSourceSpec}
-
-        backfill_source_type: str = proto.WhichOneof("type")
-
-        if backfill_source_type in function_mapping:
-            function_class = function_mapping.get(backfill_source_type)
-            return function_class._from_proto(proto)
-
-        raise QwakException(
-            f"Got unsupported backfill source type {backfill_source_type} for streaming backfill"
-        )
-
-
-@dataclass
-class BackfillBatchDataSourceSpec(DataSourceBackfillSpec):
     start_datetime: Optional[datetime] = None
     end_datetime: Optional[datetime] = None
 
-    def _to_proto(
-        self, feature_registry: FeatureRegistryClient
-    ) -> ProtoBackfillBatchDataSourceSpec:
+    def __post_init__(self):
+        self._validate()
+
+    def _to_proto(self) -> ProtoBackfillDataSource:
         start_timestamp: Optional[ProtoTimestamp] = None
         end_timestamp: Optional[ProtoTimestamp] = None
 
@@ -67,63 +40,94 @@ class BackfillBatchDataSourceSpec(DataSourceBackfillSpec):
             start_timestamp = ProtoTimestamp()
             start_timestamp.FromDatetime(self.start_datetime.astimezone(timezone.utc))
 
-        proto_data_source: ProtoBatchSource = get_batch_source_for_featureset(
-            batch_ds_name=self.data_source_name, feature_registry=feature_registry
-        )
-
-        return ProtoBackfillBatchDataSourceSpec(
-            data_source=proto_data_source,
+        time_range = ProtoTimeRange(
             start_timestamp=start_timestamp,
             end_timestamp=end_timestamp,
         )
 
+        return ProtoBackfillDataSource(
+            data_source_name=self.data_source_name,
+            time_range=time_range,
+        )
+
     @classmethod
-    def _from_proto(
-        cls, proto: ProtoBackfillDataSourceSpec
-    ) -> "BackfillBatchDataSourceSpec":
+    def _from_proto(cls, proto: ProtoBackfillDataSource) -> "BackfillDataSource":
         start_datetime: Optional[datetime] = None
         end_datetime: Optional[datetime] = None
 
-        batch_backfill_spec: ProtoBackfillBatchDataSourceSpec = (
-            proto.batch_data_source_spec
-        )
+        time_range: ProtoTimeRange = proto.time_range
 
-        proto_start_timestamp: ProtoTimestamp = batch_backfill_spec.start_timestamp
-        proto_end_timestamp: ProtoTimestamp = batch_backfill_spec.end_timestamp
+        proto_start_timestamp: Optional[ProtoTimestamp] = (
+            time_range.start_timestamp if time_range.start_timestamp else None
+        )
+        proto_end_timestamp: Optional[ProtoTimestamp] = (
+            time_range.end_timestamp if time_range.end_timestamp else None
+        )
 
-        start_datetime = datetime.fromtimestamp(
-            proto_start_timestamp.seconds + proto_start_timestamp.nanos / 1e9
+        start_datetime = (
+            datetime.fromtimestamp(
+                proto_start_timestamp.seconds + proto_start_timestamp.nanos / 1e9
+            )
+            if proto_start_timestamp
+            else None
         )
 
-        end_datetime = datetime.fromtimestamp(
-            proto_end_timestamp.seconds + proto_end_timestamp.nanos / 1e9
+        end_datetime = (
+            datetime.fromtimestamp(
+                proto_end_timestamp.seconds + proto_end_timestamp.nanos / 1e9
+            )
+            if proto_end_timestamp
+            else None
         )
 
         return cls(
-            data_source_name=batch_backfill_spec.data_source.name,
+            data_source_name=proto.data_source_name,
             start_datetime=start_datetime,
             end_datetime=end_datetime,
         )
 
+    def _validate(self):
+        if self.start_datetime and self.end_datetime:
+            if self.start_datetime >= self.end_datetime:
+                raise QwakException(
+                    f"Backfill data source {self.data_source_name} has invalid time range: "
+                    f"start_datetime {self.start_datetime} is after or equal end_datetime {self.end_datetime}."
+                )
+
+        if not self.data_source_name:
+            raise QwakException(
+                "Backfill data source must have a valid data source name."
+            )
+
 
 @dataclass
 class StreamingBackfill:
+    featureset_name: str
     start_datetime: datetime
     end_datetime: datetime
-    data_sources_specs: List[DataSourceBackfillSpec]
+    data_sources: List[BackfillDataSource]
     transform: "SparkSqlTransformation"
     cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL
 
     def __post_init__(self):
-        if not self.data_sources_specs:
+        if not self.featureset_name:
+            raise QwakException("featureset_name must be provided for backfill.")
+
+        if not self.start_datetime or not self.end_datetime:
             raise QwakException(
-                "Trying to create a streaming backfill with no data sources. "
-                "At least one data source has to be provided when trying to create a streaming backfill."
+                "For Streaming backfill, start_datetime and end_datetime are mandatory fields."
             )
 
-        if not self.start_datetime or not self.end_datetime:
+        if self.start_datetime >= self.end_datetime:
             raise QwakException(
-                "For backfill, start_datetime and end_datetime are mandatory fields."
+                f"Backfill has invalid time range: "
+                f"start_datetime {self.start_datetime} is after or equal end_datetime {self.end_datetime}."
+            )
+
+        if not self.data_sources:
+            raise QwakException(
+                "Trying to create a streaming backfill with no data sources. "
+                "At least one data source has to be provided when trying to create a streaming backfill."
             )
 
         if type(self.transform) is not SparkSqlTransformation:
@@ -135,7 +139,7 @@ class StreamingBackfill:
 
     def _validate_unique_sources(self):
         source_names: List[str] = [
-            data_source.data_source_name for data_source in self.data_sources_specs
+            data_source.data_source_name for data_source in self.data_sources
         ]
         duplicates: Set[str] = {
             item for item in source_names if source_names.count(item) > 1
@@ -146,23 +150,14 @@ class StreamingBackfill:
                 f"Found these duplicates: {', '.join(set(duplicates))}"
             )
 
-    def _validate_tile_size(self, initial_tile_size: int):
-        if self.end_datetime.timestamp() % initial_tile_size != 0:
-            raise QwakException(
-                f"Chosen backfill end datetime is invalid,"
-                f" it has to be exactly dividable by slice size of {initial_tile_size} seconds."
-            )
-
     def _to_proto(
         self,
-        feature_registry: FeatureRegistryClient,
-        featureset_name: str,
         original_instance_module_path: str,
-    ) -> ProtoBackfillSpec:
+    ) -> ProtoStreamingAggregationBackfillIngestion:
         artifact_url: Optional[str] = None
         artifact_spec: Optional[ArtifactSpec] = ArtifactsUploader.get_artifact_spec(
             transformation=self.transform,
-            featureset_name=f"{featureset_name}-backfill",
+            featureset_name=f"{self.featureset_name}-backfill",
             __instance_module_path__=original_instance_module_path,
         )
 
@@ -175,85 +170,54 @@ class StreamingBackfill:
         start_timestamp = ProtoTimestamp()
         start_timestamp.FromDatetime(self.start_datetime.astimezone(timezone.utc))
 
-        return ProtoBackfillSpec(
+        return ProtoStreamingAggregationBackfillIngestion(
+            featureset_name=self.featureset_name,
             start_timestamp=start_timestamp,
             end_timestamp=end_timestamp,
-            execution_spec=ProtoBackfillExecutionSpec(
-                **{"cluster_template": ClusterTemplate.to_proto(self.cluster_template)}
+            execution_spec=ProtoExecutionSpec(
+                cluster_template=ClusterTemplate.to_proto(self.cluster_template)
             ),
             transformation=self.transform._to_proto(artifact_path=artifact_url),
             data_source_specs=[
-                ProtoBackfillDataSourceSpec(
-                    batch_data_source_spec=data_source_spec._to_proto(
-                        feature_registry=feature_registry
-                    )
-                )
-                for data_source_spec in self.data_sources_specs
+                data_source._to_proto() for data_source in self.data_sources
             ],
         )
 
     @classmethod
-    def _from_proto(cls, proto: ProtoBackfillSpec):
-        datetime.fromtimestamp(
-            proto.start_timestamp.seconds + proto.start_timestamp.nanos / 1e9
-        )
-
-        data_sources_specs = [
-            BackfillBatchDataSourceSpec._from_proto(ds)
-            for ds in proto.data_source_specs
+    def _from_proto(cls, proto: ProtoStreamingAggregationBackfillIngestion):
+        backfill_data_sources = [
+            BackfillDataSource._from_proto(ds) for ds in proto.data_source_specs
         ]
 
         return cls(
+            featureset_name=proto.featureset_name,
             start_datetime=datetime.fromtimestamp(
                 proto.start_timestamp.seconds + proto.start_timestamp.nanos / 1e9
             ),
             end_datetime=datetime.fromtimestamp(
                 proto.end_timestamp.seconds + proto.end_timestamp.nanos / 1e9
             ),
-            data_sources_specs=data_sources_specs,
+            data_sources=backfill_data_sources,
             transform=SparkSqlTransformation._from_proto(
                 proto.transformation.sql_transformation
             ),
+            cluster_template=(
+                ClusterTemplate.from_proto(proto.execution_spec.cluster_template)
+                if proto.execution_spec.cluster_template
+                else None
+            ),
         )
 
     @staticmethod
     def _get_normalized_backfill_sources_spec(
-        data_sources: Union[List[str], List[DataSourceBackfillSpec]],
-    ) -> List[DataSourceBackfillSpec]:
-        # reformat all data source specs to 'DataSourceBackfillSpec'
+        data_sources: Union[List[str], List[BackfillDataSource]],
+    ) -> List[BackfillDataSource]:
+        # reformat all data source names to 'BackfillDataSource'
         return [
             (
-                BackfillBatchDataSourceSpec(data_source_name=data_source)
+                BackfillDataSource(data_source_name=data_source)
                 if isinstance(data_source, str)
                 else data_source
             )
             for data_source in data_sources
         ]
-
-    @classmethod
-    def set_streaming_backfill_on_function(
-        cls,
-        function,
-        start_date: datetime,
-        end_date: datetime,
-        data_sources: Union[List[str], List[DataSourceBackfillSpec]],
-        backfill_transformation: SparkSqlTransformation,
-        backfill_cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL,
-    ):
-        setattr(
-            function,
-            _BACKFILL_,
-            cls(
-                start_datetime=start_date,
-                end_datetime=end_date,
-                data_sources_specs=StreamingBackfill._get_normalized_backfill_sources_spec(
-                    data_sources
-                ),
-                transform=backfill_transformation,
-                cluster_template=backfill_cluster_template,
-            ),
-        )
-
-    @staticmethod
-    def get_streaming_backfill_from_function(function):
-        return getattr(function, _BACKFILL_, None)

{qwak_core-0.5.4.dist-info → qwak_core-0.5.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: qwak-core
-Version: 0.5.4
+Version: 0.5.12
 Summary: Qwak Core contains the necessary objects and communication tools for using the Qwak Platform
 License: Apache-2.0
 Keywords: mlops,ml,deployment,serving,model

{qwak_core-0.5.4.dist-info → qwak_core-0.5.12.dist-info}/RECORD

@@ -511,8 +511,8 @@ _qwak_proto/qwak/model_descriptor/open_ai_descriptor_pb2_grpc.py,sha256=1oboBPFx
 _qwak_proto/qwak/model_group/model_group_pb2.py,sha256=oM2O7f-CGykrJ2HCePU1lONl-hJoJtpSLBlvdqrBNO4,5244
 _qwak_proto/qwak/model_group/model_group_pb2.pyi,sha256=YDCW1Oj16lzWku0xJ5YYPPmzPmrqOfMwG8vqeYxiyZg,9245
 _qwak_proto/qwak/model_group/model_group_pb2_grpc.py,sha256=OnYtTijqKbxC13VpOUKJavMHae7BMWRusAUp-nz8h_g,9256
-_qwak_proto/qwak/model_group/model_group_repository_details_pb2.py,sha256=4rV7Xy1yrQNrO-AfZvekccyAyRr5_ymvuBa2tawMxcc,3089
-_qwak_proto/qwak/model_group/model_group_repository_details_pb2.pyi,sha256=GgYhrSXu8L2RdEV0clooKgB9Gye3_no4q0YrMy8QrZw,5701
+_qwak_proto/qwak/model_group/model_group_repository_details_pb2.py,sha256=_MMAM40q5l9Luubv40LEa-Uv4CCkbNmRt3wQ-916Zz0,3735
+_qwak_proto/qwak/model_group/model_group_repository_details_pb2.pyi,sha256=qOjP2x_4qC0gZayL6bkACeZU0GSfkH_6wIuWYHsNOe8,7320
 _qwak_proto/qwak/model_group/model_group_repository_details_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 _qwak_proto/qwak/models/models_pb2.py,sha256=w2jAVJioQN5Ck1XW9jelfsOqmFRryVjmFukD49I_zUQ,18771
 _qwak_proto/qwak/models/models_pb2.pyi,sha256=T08V38RgCNy2x6ZW4_ZZdK09dZs7TzwXEHaGtNMW0y8,42643
@@ -622,7 +622,7 @@ _qwak_proto/qwak/workspace/workspace_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXH
 _qwak_proto/qwak/workspace/workspace_service_pb2.py,sha256=93uNm83VVrkLG_XVsuBOBTPs4UJF91YD1xJTOEbRyig,9239
 _qwak_proto/qwak/workspace/workspace_service_pb2.pyi,sha256=nKKCHwnovZhsy8TSVmdz-Vtl0nviOOoX56HD-41Xo08,13726
 _qwak_proto/qwak/workspace/workspace_service_pb2_grpc.py,sha256=yKGuexxTBza99Ihe0DSTniV2ZSd_AG47inHenqfi890,27193
-qwak/__init__.py,sha256=eS6NWShAiOsKkqTwZM6VUxq-d8TXXV3EZwVwJDhnZsQ,585
+qwak/__init__.py,sha256=eBEVeYrxbXaW9CHSWF1DsMseWHkw5gUHrSzHYBxiMmo,586
 qwak/automations/__init__.py,sha256=qFZRvCxUUn8gcxkJR0v19ulHW2oJ0x6-Rif7HiheDP4,1522
 qwak/automations/automation_executions.py,sha256=5MeH_epYYWb8NKXgAozwT_jPyyUDednBHG7izloi7RY,3228
 qwak/automations/automations.py,sha256=2Wyrgj2mW5VmJzTKAXj3vQM_svYCuq_Bsu3_Vu9SdR4,12732
@@ -671,7 +671,7 @@ qwak/clients/data_versioning/data_tag_filter.py,sha256=vJaZ-Zot4eYxrzonbkdEWWGXZ
 qwak/clients/deployment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak/clients/deployment/client.py,sha256=v8w-xEA7HeKBcsSobVkPdfQq_SLA9IbHo4qMJIAA5dM,6806
 qwak/clients/feature_store/__init__.py,sha256=mMCPBHDga6Y7dtJfNoHvfOvCyjNUHrVDX5uVsL2JkGk,53
-qwak/clients/feature_store/execution_management_client.py,sha256=tKLIML8wsvgsl1hBfx74izFL0rHYdNM69sEstomDfYM,4051
+qwak/clients/feature_store/execution_management_client.py,sha256=P1nZnttIC3QNVUI_XD7K3ywRSIq6jtifa1FnPMvUtFg,5280
 qwak/clients/feature_store/job_registry_client.py,sha256=7VMtEj7aofKcABrYpldgdxyv8Vkdq_mocjns1O0uCqA,2635
 qwak/clients/feature_store/management_client.py,sha256=NTfuwV1SQdaVxxyCE-htz1zXOpJHAKEyiB3bnGUyCW4,18220
 qwak/clients/feature_store/offline_serving_client.py,sha256=gz8hqaboPA1Und8leOf1O0dXa9xorHDTU3b7-Ne9YSE,9344
@@ -775,6 +775,7 @@ qwak/feature_store/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 qwak/feature_store/execution/backfill.py,sha256=bzF_-ZrXEfS14-SfJ5ldI6CKjWVQQnlSsI-X4jI7VrU,6470
 qwak/feature_store/execution/execution.py,sha256=msFzcZamWTlb5pE8IslP6MmJMAqnmBXt94uSH70koIc,21243
 qwak/feature_store/execution/execution_query.py,sha256=B5KRU1jvI042kEQoKj7A3G8Hu80nd8orWf8cOD9hSm8,3564
+qwak/feature_store/execution/streaming_backfill.py,sha256=wkQzK1SzZ4AAP35BSDMIiHGZk2mugGa5znRcQqXHIO8,1683
 qwak/feature_store/feature_sets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak/feature_store/feature_sets/_utils/_featureset_utils.py,sha256=AWnQBS8TrpRb_J7h4wQLOzq4D--GMfc2xkRElioOKBE,1636
 qwak/feature_store/feature_sets/backfill.py,sha256=70yEP3fLTt8ecNuxeBnFp-qp0OUJ53KkOQRsYv2QEzc,1979
@@ -784,8 +785,8 @@ qwak/feature_store/feature_sets/context.py,sha256=zV6r0O70cfM4pmxlfC6xxAtro-wBhe
 qwak/feature_store/feature_sets/execution_spec.py,sha256=SXlhgNJeFDIKvrH35ZOyjBqGUesdoxjWkPgHmm518Y0,1877
 qwak/feature_store/feature_sets/metadata.py,sha256=ckr-zr0hZgsK-tRWucMomF7Tj8XIYcp_cnNGwtMV4mA,1814
 qwak/feature_store/feature_sets/read_policies.py,sha256=ZVMRiducxfb5YbU0NQQDPb78BH5fRttsE5y2TL7Jaj4,6820
-qwak/feature_store/feature_sets/streaming.py,sha256=G0Prg6hyjrzimL0u7tA71G-Bq3mnn57ZZCneMfudL0Y,25161
-qwak/feature_store/feature_sets/streaming_backfill.py,sha256=NInPNnc0WYx2FDsjaOt5woTO7EtzgrZCbMB_Ull0ux0,9585
+qwak/feature_store/feature_sets/streaming.py,sha256=4_fOOwagmOIRDruZZzNlpMDb6ElsXC3wYj3rlNP9zT0,26462
+qwak/feature_store/feature_sets/streaming_backfill.py,sha256=dAkxZjmMdk1Oj0EFqFm-adwzw2XpURQ12I7bjbpV_2I,8280
 qwak/feature_store/feature_sets/transformations/__init__.py,sha256=TtzeMNhqxuYu8NIxqBCcGHJoA-B8389RTIHrDMvpMbw,857
 qwak/feature_store/feature_sets/transformations/aggregations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak/feature_store/feature_sets/transformations/aggregations/aggregations.py,sha256=MENQyuDEG5G4TQs7fd1L1FEfP4kWd0LaHbG61Kw4rbY,14117
@@ -1064,7 +1065,7 @@ qwak_services_mock/mocks/build_orchestrator_service_api.py,sha256=IlM6VzovSSV6o1
 qwak_services_mock/mocks/data_versioning_service.py,sha256=dUUrKwE3xc2bZtHa8YMg6wYn8zFa2BSZ5DaRsb1zT_Q,2453
 qwak_services_mock/mocks/deployment_management_service.py,sha256=T4fmFC1sUgEh5sXH6ilwyPmfRfEBFqyYrGGljl_Abg8,20609
 qwak_services_mock/mocks/ecosystem_service_api.py,sha256=f0-IeZMprFKvi1MvdW9aJZgO7BBdnJHQB8spDiVWy-A,1607
-qwak_services_mock/mocks/execution_management_service.py,sha256=D7bmYfCFKnxo6_LAO2ogy5gNqFB-V84KtpZxaCXfczk,886
+qwak_services_mock/mocks/execution_management_service.py,sha256=L6uSHniFNaJDZUwRuMnt_-j6NmpJoOsD0RwpoI6w9ag,1190
 qwak_services_mock/mocks/feature_store_data_sources_manager_api.py,sha256=zfiq5x7xxg4IvBS1OndqSnFHDOckMW_-4K_Ajlko6no,5241
 qwak_services_mock/mocks/feature_store_entities_manager_api.py,sha256=I9wsVEsb0qVcD8jgekeC-EG-Ke8SziRLUND8uF4OJbc,3349
 qwak_services_mock/mocks/feature_store_feature_set_manager_api.py,sha256=TZIGeKGwOP45MbcnuSAoB2Gc28j_wzU9Vuu6Q757x-s,10918
@@ -1097,6 +1098,6 @@ qwak_services_mock/mocks/workspace_manager_service_mock.py,sha256=O9ZSwln4T4kHVk
 qwak_services_mock/services_mock.py,sha256=0BWwS2re9ryO3JrJJgSNyEQ0lDOjyrpV36oa8t7Pd7A,19163
 qwak_services_mock/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak_services_mock/utils/service_utils.py,sha256=ZlB0CnB1J6oBn6_m7fQO2U8tKoboHdUa6ljjkRMYNXU,265
-qwak_core-0.5.4.dist-info/METADATA,sha256=g7VkaPj5hngSVkBOSPOGivSYzCwiaapUnAqtWL6oHZQ,1986
-qwak_core-0.5.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-qwak_core-0.5.4.dist-info/RECORD,,
+qwak_core-0.5.12.dist-info/METADATA,sha256=Idr0TViACsw1-5_03xRXvOcr9a8mg8njLW252T28YHQ,1987
+qwak_core-0.5.12.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+qwak_core-0.5.12.dist-info/RECORD,,

qwak_services_mock/mocks/execution_management_service.py

@@ -1,7 +1,8 @@
-from _qwak_proto.qwak.execution.v1.execution_service_pb2 import TriggerBackfillResponse
+from _qwak_proto.qwak.execution.v1.execution_service_pb2 import (TriggerBackfillResponse, TriggerStreamingAggregationBackfillResponse)
 from _qwak_proto.qwak.execution.v1.execution_service_pb2_grpc import (
     FeatureStoreExecutionServiceServicer,
 )
+
 from grpc import RpcError
 
 
@@ -23,3 +24,10 @@ class ExecutionManagementServiceMock(FeatureStoreExecutionServiceServicer):
         if self._raise_exception_on_request:
             raise RpcError
         return TriggerBackfillResponse(execution_id=self._execution_id)
+
+    def TriggerStreamingAggregationBackfill(self, request, context):
+        if self._raise_exception_on_request:
+            raise RpcError
+        return TriggerStreamingAggregationBackfillResponse(
+            execution_id=self._execution_id
+        )