frogml 1.2.21__py3-none-any.whl → 1.2.28__py3-none-any.whl

frogml/__init__.py

@@ -1,5 +1,5 @@
 __author__ = "jfrog"
-__version__ = "1.2.21"
+__version__ = "1.2.28"
 
 from frogml.sdk.model.decorators.api import api_decorator as api
 from frogml.sdk.model_loggers.model_logger import load_model, log_model

frogml/_proto/jfml/model_version/v1/model_repository_spec_pb2.pyi

@@ -19,7 +19,10 @@ class ModelRepositorySpec(google.protobuf.message.Message):
     MODEL_ID_FIELD_NUMBER: builtins.int
     MODEL_NAME_FIELD_NUMBER: builtins.int
     project_key: builtins.str
-    """JFrog project key"""
+    """JFrog project key
+    This is not used in the request of `CreateModelVersion`. The project key used there is
+    derived from the repository. The request value is being ignored in this endpoint.
+    """
     repository_key: builtins.str
     model_id: builtins.str
     model_name: builtins.str

frogml/_proto/qwak/model_group/model_group_repository_details_pb2.py

@@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default()
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5qwak/model_group/model_group_repository_details.proto\x12\x1cqwak.model_groups.management\"r\n\x11RepositoryDetails\x12\x45\n\x0frepository_type\x18\x01 \x01(\x0b\x32,.qwak.model_groups.management.RepositoryType\x12\x16\n\x0erepository_key\x18\x02 \x01(\t\"\xf0\x01\n\x17RemoteRepositoryDetails\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x02 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x1d\n\x15repository_remote_url\x18\x03 \x01(\tB\x11\n\x0frepository_type\"\x12\n\x10\x44ockerRepository\"\x13\n\x11\x44\x61tasetRepository\"\x14\n\x12\x41rtifactRepository\"\x17\n\x15HuggingFaceRepository\"\xdd\x02\n\x0eRepositoryType\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12M\n\x12\x64\x61taset_repository\x18\x02 \x01(\x0b\x32/.qwak.model_groups.management.DatasetRepositoryH\x00\x12O\n\x13\x61rtifact_repository\x18\x03 \x01(\x0b\x32\x30.qwak.model_groups.management.ArtifactRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x04 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x42\x06\n\x04typeBN\n&com.qwak.ai.management.model.group.apiB\"ModelGroupRepositoriesDetailsProtoP\x01\x62\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5qwak/model_group/model_group_repository_details.proto\x12\x1cqwak.model_groups.management\"r\n\x11RepositoryDetails\x12\x45\n\x0frepository_type\x18\x01 \x01(\x0b\x32,.qwak.model_groups.management.RepositoryType\x12\x16\n\x0erepository_key\x18\x02 \x01(\t\"\x80\x03\n\x17RemoteRepositoryDetails\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x02 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x45\n\x0enpm_repository\x18\x04 \x01(\x0b\x32+.qwak.model_groups.management.NpmRepositoryH\x00\x12G\n\x0fpypi_repository\x18\x05 \x01(\x0b\x32,.qwak.model_groups.management.PypiRepositoryH\x00\x12\x1d\n\x15repository_remote_url\x18\x03 \x01(\tB\x11\n\x0frepository_type\"\x12\n\x10\x44ockerRepository\"\x13\n\x11\x44\x61tasetRepository\"\x14\n\x12\x41rtifactRepository\"\x17\n\x15HuggingFaceRepository\"\x0f\n\rNpmRepository\"\x10\n\x0ePypiRepository\"\xed\x03\n\x0eRepositoryType\x12K\n\x11\x64ocker_repository\x18\x01 \x01(\x0b\x32..qwak.model_groups.management.DockerRepositoryH\x00\x12M\n\x12\x64\x61taset_repository\x18\x02 \x01(\x0b\x32/.qwak.model_groups.management.DatasetRepositoryH\x00\x12O\n\x13\x61rtifact_repository\x18\x03 \x01(\x0b\x32\x30.qwak.model_groups.management.ArtifactRepositoryH\x00\x12V\n\x17hugging_face_repository\x18\x04 \x01(\x0b\x32\x33.qwak.model_groups.management.HuggingFaceRepositoryH\x00\x12\x45\n\x0enpm_repository\x18\x05 \x01(\x0b\x32+.qwak.model_groups.management.NpmRepositoryH\x00\x12G\n\x0fpypi_repository\x18\x06 \x01(\x0b\x32,.qwak.model_groups.management.PypiRepositoryH\x00\x42\x06\n\x04typeBN\n&com.qwak.ai.management.model.group.apiB\"ModelGroupRepositoriesDetailsProtoP\x01\x62\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -25,15 +25,19 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_REPOSITORYDETAILS']._serialized_start=87
   _globals['_REPOSITORYDETAILS']._serialized_end=201
   _globals['_REMOTEREPOSITORYDETAILS']._serialized_start=204
-  _globals['_REMOTEREPOSITORYDETAILS']._serialized_end=444
-  _globals['_DOCKERREPOSITORY']._serialized_start=446
-  _globals['_DOCKERREPOSITORY']._serialized_end=464
-  _globals['_DATASETREPOSITORY']._serialized_start=466
-  _globals['_DATASETREPOSITORY']._serialized_end=485
-  _globals['_ARTIFACTREPOSITORY']._serialized_start=487
-  _globals['_ARTIFACTREPOSITORY']._serialized_end=507
-  _globals['_HUGGINGFACEREPOSITORY']._serialized_start=509
-  _globals['_HUGGINGFACEREPOSITORY']._serialized_end=532
-  _globals['_REPOSITORYTYPE']._serialized_start=535
-  _globals['_REPOSITORYTYPE']._serialized_end=884
+  _globals['_REMOTEREPOSITORYDETAILS']._serialized_end=588
+  _globals['_DOCKERREPOSITORY']._serialized_start=590
+  _globals['_DOCKERREPOSITORY']._serialized_end=608
+  _globals['_DATASETREPOSITORY']._serialized_start=610
+  _globals['_DATASETREPOSITORY']._serialized_end=629
+  _globals['_ARTIFACTREPOSITORY']._serialized_start=631
+  _globals['_ARTIFACTREPOSITORY']._serialized_end=651
+  _globals['_HUGGINGFACEREPOSITORY']._serialized_start=653
+  _globals['_HUGGINGFACEREPOSITORY']._serialized_end=676
+  _globals['_NPMREPOSITORY']._serialized_start=678
+  _globals['_NPMREPOSITORY']._serialized_end=693
+  _globals['_PYPIREPOSITORY']._serialized_start=695
+  _globals['_PYPIREPOSITORY']._serialized_end=711
+  _globals['_REPOSITORYTYPE']._serialized_start=714
+  _globals['_REPOSITORYTYPE']._serialized_end=1207
 # @@protoc_insertion_point(module_scope)

frogml/_proto/qwak/model_group/model_group_repository_details_pb2.pyi

@@ -39,6 +39,8 @@ class RemoteRepositoryDetails(google.protobuf.message.Message):
 
     DOCKER_REPOSITORY_FIELD_NUMBER: builtins.int
     HUGGING_FACE_REPOSITORY_FIELD_NUMBER: builtins.int
+    NPM_REPOSITORY_FIELD_NUMBER: builtins.int
+    PYPI_REPOSITORY_FIELD_NUMBER: builtins.int
     REPOSITORY_REMOTE_URL_FIELD_NUMBER: builtins.int
     repository_remote_url: builtins.str
     """The remote repository URL"""
@@ -50,16 +52,26 @@ class RemoteRepositoryDetails(google.protobuf.message.Message):
     def hugging_face_repository(self) -> global___HuggingFaceRepository:
         """HuggingFace repository"""
 
+    @property
+    def npm_repository(self) -> global___NpmRepository:
+        """NPM repository"""
+
+    @property
+    def pypi_repository(self) -> global___PypiRepository:
+        """PyPI repository"""
+
     def __init__(
         self,
         *,
         docker_repository: global___DockerRepository | None = ...,
         hugging_face_repository: global___HuggingFaceRepository | None = ...,
+        npm_repository: global___NpmRepository | None = ...,
+        pypi_repository: global___PypiRepository | None = ...,
         repository_remote_url: builtins.str = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "repository_type", b"repository_type"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "repository_remote_url", b"repository_remote_url", "repository_type", b"repository_type"]) -> None: ...
-    def WhichOneof(self, oneof_group: typing.Literal["repository_type", b"repository_type"]) -> typing.Literal["docker_repository", "hugging_face_repository"] | None: ...
+    def HasField(self, field_name: typing.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "repository_type", b"repository_type"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "repository_remote_url", b"repository_remote_url", "repository_type", b"repository_type"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing.Literal["repository_type", b"repository_type"]) -> typing.Literal["docker_repository", "hugging_face_repository", "npm_repository", "pypi_repository"] | None: ...
 
 global___RemoteRepositoryDetails = RemoteRepositoryDetails
 
@@ -103,6 +115,26 @@ class HuggingFaceRepository(google.protobuf.message.Message):
 
 global___HuggingFaceRepository = HuggingFaceRepository
 
+@typing.final
+class NpmRepository(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    def __init__(
+        self,
+    ) -> None: ...
+
+global___NpmRepository = NpmRepository
+
+@typing.final
+class PypiRepository(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    def __init__(
+        self,
+    ) -> None: ...
+
+global___PypiRepository = PypiRepository
+
 @typing.final
 class RepositoryType(google.protobuf.message.Message):
     """The repository type"""
@@ -113,6 +145,8 @@ class RepositoryType(google.protobuf.message.Message):
     DATASET_REPOSITORY_FIELD_NUMBER: builtins.int
     ARTIFACT_REPOSITORY_FIELD_NUMBER: builtins.int
     HUGGING_FACE_REPOSITORY_FIELD_NUMBER: builtins.int
+    NPM_REPOSITORY_FIELD_NUMBER: builtins.int
+    PYPI_REPOSITORY_FIELD_NUMBER: builtins.int
     @property
     def docker_repository(self) -> global___DockerRepository:
         """Docker repository"""
@@ -129,6 +163,14 @@ class RepositoryType(google.protobuf.message.Message):
     def hugging_face_repository(self) -> global___HuggingFaceRepository:
         """HuggingFace repository"""
 
+    @property
+    def npm_repository(self) -> global___NpmRepository:
+        """NPM repository"""
+
+    @property
+    def pypi_repository(self) -> global___PypiRepository:
+        """PyPI repository"""
+
     def __init__(
         self,
         *,
@@ -136,9 +178,11 @@ class RepositoryType(google.protobuf.message.Message):
         dataset_repository: global___DatasetRepository | None = ...,
         artifact_repository: global___ArtifactRepository | None = ...,
         hugging_face_repository: global___HuggingFaceRepository | None = ...,
+        npm_repository: global___NpmRepository | None = ...,
+        pypi_repository: global___PypiRepository | None = ...,
     ) -> None: ...
-    def HasField(self, field_name: typing.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "type", b"type"]) -> builtins.bool: ...
-    def ClearField(self, field_name: typing.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "type", b"type"]) -> None: ...
-    def WhichOneof(self, oneof_group: typing.Literal["type", b"type"]) -> typing.Literal["docker_repository", "dataset_repository", "artifact_repository", "hugging_face_repository"] | None: ...
+    def HasField(self, field_name: typing.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "type", b"type"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["artifact_repository", b"artifact_repository", "dataset_repository", b"dataset_repository", "docker_repository", b"docker_repository", "hugging_face_repository", b"hugging_face_repository", "npm_repository", b"npm_repository", "pypi_repository", b"pypi_repository", "type", b"type"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing.Literal["type", b"type"]) -> typing.Literal["docker_repository", "dataset_repository", "artifact_repository", "hugging_face_repository", "npm_repository", "pypi_repository"] | None: ...
 
 global___RepositoryType = RepositoryType

frogml/core/clients/feature_store/execution_management_client.py

@@ -3,6 +3,9 @@ from grpc import RpcError
 
 from frogml._proto.qwak.execution.v1.backfill_pb2 import BackfillSpec
 from frogml._proto.qwak.execution.v1.batch_pb2 import BatchIngestion
+from frogml._proto.qwak.execution.v1.streaming_aggregation_pb2 import (
+    StreamingAggregationBackfillIngestion,
+)
 from frogml._proto.qwak.execution.v1.execution_service_pb2 import (
     GetExecutionEntryRequest,
     GetExecutionEntryResponse,
@@ -12,6 +15,8 @@ from frogml._proto.qwak.execution.v1.execution_service_pb2 import (
     TriggerBackfillResponse,
     TriggerBatchFeaturesetRequest,
     TriggerBatchFeaturesetResponse,
+    TriggerStreamingAggregationBackfillRequest,
+    TriggerStreamingAggregationBackfillResponse,
 )
 from frogml._proto.qwak.execution.v1.execution_service_pb2_grpc import (
     FeatureStoreExecutionServiceStub,
@@ -100,3 +105,25 @@ class ExecutionManagementClient:
             raise FrogmlException(
                 f"Failed to get execution entry, error encountered {e}"
             )
+
+    def trigger_streaming_aggregation_backfill(
+        self, backfill_ingestion: StreamingAggregationBackfillIngestion
+    ) -> TriggerStreamingAggregationBackfillResponse:
+        """
+        Receives a configured streaming aggregation backfill proto and triggers a streaming aggregation backfill against the execution manager
+
+        Args:
+            backfill_ingestion (StreamingAggregationBackfillIngestion): A protobuf message
+                containing the backfill specification details
+
+        Returns:
+            TriggerStreamingAggregationBackfillResponse: response object from the execution manager
+        """
+        try:
+            return self._feature_store_execution_service.TriggerStreamingAggregationBackfill(
+                TriggerStreamingAggregationBackfillRequest(backfill=backfill_ingestion)
+            )
+        except RpcError as e:
+            raise FrogmlException(
+                f"Failed to trigger streaming aggregation backfill job, error encountered {e}"
+            )

frogml/core/clients/model_version_manager/client.py

@@ -6,18 +6,8 @@ from dependency_injector.wiring import Provide
 from grpc import RpcError
 from typing_extensions import Self
 
-from frogml.core.clients.model_version_manager.build_model_version_dto import (
-    BuildConfigDTO,
-)
-from frogml.core.clients.model_version_manager.build_model_version_request_mapper import (
-    map_build_conf_to_build_spec,
-)
-from frogml.core.exceptions import FrogmlException
-from frogml.core.inner.di_configuration import FrogmlContainer
-from frogml.core.inner.tool.grpc.grpc_try_wrapping import grpc_try_catch_wrapper
-from frogml.core.utils.model_utils import get_model_id_from_model_name
-from frogml.core.utils.proto_utils import ProtoUtils
 from frogml._proto.jfml.model_version.v1.artifact_pb2 import Artifact as ArtifactProto
+from frogml._proto.jfml.model_version.v1.build_spec_pb2 import BuildSpec
 from frogml._proto.jfml.model_version.v1.model_repository_spec_pb2 import (
     ModelRepositorySpec,
 )
@@ -39,10 +29,19 @@ from frogml._proto.jfml.model_version.v1.model_version_pb2 import (
     ParameterValue,
     MetricValue,
 )
-from frogml._proto.jfml.model_version.v1.build_spec_pb2 import BuildSpec
+from frogml.core.clients.model_version_manager.build_model_version_dto import (
+    BuildConfigDTO,
+)
+from frogml.core.clients.model_version_manager.build_model_version_request_mapper import (
+    map_build_conf_to_build_spec,
+)
+from frogml.core.exceptions import FrogmlException
+from frogml.core.inner.di_configuration import FrogmlContainer
+from frogml.core.inner.tool.grpc.grpc_try_wrapping import grpc_try_catch_wrapper
+from frogml.core.utils.model_utils import get_model_id_from_model_name
+from frogml.core.utils.proto_utils import ProtoUtils
 from frogml.storage.models.entity_manifest import Artifact
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -87,7 +86,6 @@ class ModelVersionManagerClient:
 
     def __build_create_model_version_request(
         self,
-        project_key: str,
         repository_key: str,
         model_name: str,
         model_version_name: str,
@@ -107,7 +105,6 @@ class ModelVersionManagerClient:
             dry_run=dry_run,
             model_version=ModelVersionSpec(
                 repository_spec=ModelRepositorySpec(
-                    project_key=project_key,
                     repository_key=repository_key,
                     model_id=get_model_id_from_model_name(model_name),
                     model_name=model_name,
@@ -121,7 +118,6 @@ class ModelVersionManagerClient:
 
     def validate_create_model_version(
         self,
-        project_key: str,
         repository_key: str,
         model_name: str,
         model_version_name: str,
@@ -142,7 +138,6 @@ class ModelVersionManagerClient:
             )
 
             create_model_request = self.__build_create_model_version_request(
-                project_key=project_key,
                 repository_key=repository_key,
                 model_name=model_name,
                 model_version_name=model_version_name,
@@ -165,7 +160,6 @@ class ModelVersionManagerClient:
 
     def create_model_version(
         self,
-        project_key: str,
         repository_key: str,
         model_name: str,
         model_version_name: str,
@@ -186,7 +180,6 @@ class ModelVersionManagerClient:
             )
 
             create_model_request = self.__build_create_model_version_request(
-                project_key=project_key,
                 repository_key=repository_key,
                 model_name=model_name,
                 model_version_name=model_version_name,

frogml/core/feature_store/execution/streaming_backfill.py

@@ -0,0 +1,49 @@
+import pathlib
+
+from frogml.core.clients.feature_store.execution_management_client import (
+    ExecutionManagementClient,
+)
+from frogml.feature_store.feature_sets.streaming_backfill import StreamingBackfill
+
+
+class StreamingAggregationBackfill:
+
+    def __init__(
+        self,
+        streaming_backfill: StreamingBackfill,
+        source_definition_path: pathlib.Path,
+    ):
+        """
+        Initialize the streaming aggregation backfill executor.
+
+        Args:
+            streaming_backfill (StreamingBackfill): Specification containing the
+                              featureset name, time range, data sources, and transformation
+            source_definition_path (Path): Path to the Python file containing the backfill
+                                   definition. Required for locating UDF artifacts.
+        """
+        self._streaming_backfill = streaming_backfill
+        self._source_definition_path = source_definition_path
+
+    def trigger(self) -> str:
+        """
+        Triggers the streaming aggregation backfill execution.
+
+        Converts the backfill specification to proto format and sends it to
+        the execution manager to start the backfill job.
+
+        Returns:
+            str: The execution ID for tracking the backfill job status
+
+        Raises:
+            FrogmlException: If the execution manager request fails
+        """
+        backfill_proto = self._streaming_backfill._to_proto(
+            str(self._source_definition_path)
+        )
+
+        execution_client = ExecutionManagementClient()
+        response = execution_client.trigger_streaming_aggregation_backfill(
+            backfill_proto
+        )
+        return response.execution_id

frogml/feature_store/feature_sets/streaming.py

@@ -2,7 +2,7 @@ import collections
 import functools
 import inspect
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 from typeguard import typechecked
@@ -29,6 +29,7 @@ from frogml._proto.qwak.feature_store.sources.streaming_pb2 import StreamingSour
 from frogml._proto.qwak.feature_store.sources.streaming_pb2 import (
     StreamingSource as ProtoStreamingSource,
 )
+from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
 from frogml.core.clients.feature_store import FeatureRegistryClient
 from frogml.core.exceptions import FrogmlException
 from frogml.feature_store._common.artifact_utils import (
@@ -47,7 +48,7 @@ from frogml.core.feature_store.feature_sets.metadata import (
     set_metadata_on_function,
 )
 from frogml.feature_store.feature_sets.streaming_backfill import (
-    BackfillBatchDataSourceSpec,
+    BackfillDataSource,
     StreamingBackfill,
 )
 from frogml.core.feature_store.feature_sets.transformations import (
@@ -87,6 +88,7 @@ def feature_set(
     key: Optional[str] = None,
     auxiliary_sinks: List[BaseSink] = [],
     repository: Optional[str] = None,
+    backfill_max_timestamp: Optional[datetime] = None,
 ):
     """
     Creates a streaming feature set for the specified entity using the given streaming data sources.
@@ -122,6 +124,11 @@ def feature_set(
     """
 
     def decorator(function):
+        if isinstance(function, StreamingBackfill):
+            raise FrogmlException(
+                "Backfill can no longer be defined as a decorator on the feature set, it must be triggered after feature set creation."
+            )
+
         user_transformation = function()
         FeaturesetUtils.validate_base_featureset_decorator(
             user_transformation=user_transformation, entity=entity, key=key
@@ -132,10 +139,6 @@ def feature_set(
             offline_scheduling_policy=offline_scheduling_policy,
         )
 
-        streaming_backfill: Optional[StreamingBackfill] = (
-            StreamingBackfill.get_streaming_backfill_from_function(function=function)
-        )
-
         fs_name = name or function.__name__
         streaming_feature_set = StreamingFeatureSet(
             name=fs_name,
@@ -162,7 +165,7 @@ def feature_set(
             online_cluster_template=getattr(
                 function, _ONLINE_CLUSTER_SPEC, ClusterTemplate.SMALL
             ),
-            backfill=streaming_backfill,
+            backfill_max_timestamp=backfill_max_timestamp,
             __instance_module_path__=inspect.stack()[1].filename,
             auxiliary_sinks=auxiliary_sinks,
         )
@@ -209,55 +212,68 @@ def execution_specification(
 @typechecked
 def backfill(
     *,
-    start_date: datetime,
-    end_date: datetime,
-    data_sources: Union[List[str], List[BackfillBatchDataSourceSpec]],
-    backfill_transformation: SparkSqlTransformation,
+    feature_set_name: str,
+    start_date: Optional[datetime],
+    end_date: Optional[datetime],
+    data_sources: Union[List[str], List[BackfillDataSource]],
     backfill_cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL,
 ):
     """
-    Configures optional backfill specification for a streaming featureset. Currently available for streaming
+    Triggers a backfill execution for an existing streaming featureset. Currently available for streaming
     aggregation featuresets only.
 
-    :param start_date: backfill start date
-    :param end_date: backfill end date
-    :param data_sources: a list of dict representations connecting each batch source name to its requested time range
-    :param backfill_transformation: a backfill SQL transformation. Only SQL transformations are supported for backfill
-    :param backfill_cluster_template: an optional cluster specification for the backfill job. Defaults to SMALL.
-
-    Example:
-
-    ... code-block:: python
-
-        @streaming.feature_set(
-            entity="users",
-            data_sources=["users_registration_stream"],
-            timestamp_column_name="reg_date"
-        )
+    Args:
+        feature_set_name (str): Name of the FeatureSet to trigger a backfill for.
+        start_date (datetime): Backfill start date, on Streaming Aggregation Feature Sets,
+            needs to align with the FeatureSet tiles.
+        end_date (datetime): Backfill end date, on Streaming Aggregation Feature Sets,
+            needs to align with the FeatureSet tiles and be smaller than the Feature Set's backfill_max_timestamp.
+        data_sources (list[BackfillDataSource] | list[str]): A list of BackfillDataSource objects containing
+            batch source name and optional time range, or a list of batch source names (with no time range limits).
+        backfill_cluster_template (ClusterTemplate, optional): An optional cluster specification for the backfill job.
+            Defaults to SMALL.
+
+    Examples:
         @streaming.backfill(
-            start_date=start_date=(2022,01,01,0,0,0),
-            end_date=start_date=(2023,09,01,0,0,0),
-            data_sources=[{"batch_backfill_source_name": BackfillSourceRange(start_date=(2023,01,01,0,0,0),
-                                                                             end_date=(2023,08,01,0,0,0))}],
+            feature_set_name="user_streaming_agg_features",
+            start_date=datetime(2022,1,1,0,0,0),
+            end_date=datetime(2023,9,1,0,0,0),
+            data_sources=[BackfillDataSource(data_source_name="backfill_data_source",
+                                             start_datetime=datetime(2023,1,1,0,0,0),
+                                             end_datetime=datetime(2023,8,1,0,0,0))],
             backfill_cluster_template=ClusterTemplate.SMALL
-            backfill_transformation=SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM backfill_data_source")
         )
-        def user_streaming_features():
-            return SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM data_source")
+        def backfill_transformation():
+            return SparkSqlTransformation("SELECT user_id, reg_country, reg_date FROM backfill_data_source")
     """
 
     def decorator(function):
-        _validate_decorator_ordering(function)
-        StreamingBackfill.set_streaming_backfill_on_function(
-            function=function,
-            start_date=start_date,
-            end_date=end_date,
-            data_sources=data_sources,
-            backfill_transformation=backfill_transformation,
-            backfill_cluster_template=backfill_cluster_template,
+        if isinstance(function, StreamingFeatureSet):
+            raise FrogmlException(
+                "Backfill can no longer be defined as a decorator on the feature set, it must be triggered after feature set creation."
+            )
+
+        backfill_transformation: SparkSqlTransformation = function()
+
+        if not isinstance(backfill_transformation, SparkSqlTransformation):
+            raise FrogmlException(
+                "Backfill must defined on a method returning a SparkSqlTransformation"
+            )
+
+        streaming_backfill = StreamingBackfill(
+            featureset_name=feature_set_name,
+            start_datetime=start_date,
+            end_datetime=end_date,
+            data_sources=StreamingBackfill._get_normalized_backfill_sources_spec(
+                data_sources
+            ),
+            transform=backfill_transformation,
+            cluster_template=backfill_cluster_template,
         )
 
-        return function
+        functools.update_wrapper(streaming_backfill, backfill_transformation)
+
+        return streaming_backfill
 
     return decorator
 
@@ -328,7 +344,7 @@ class StreamingFeatureSet(BaseFeatureSet):
     offline_cluster_template: Optional[ClusterTemplate] = None
     online_cluster_template: Optional[ClusterTemplate] = None
     metadata: Optional[Metadata] = None
-    backfill: Optional[StreamingBackfill] = None
+    backfill_max_timestamp: Optional[StreamingBackfill] = None
     auxiliary_sinks: List[BaseSink] = field(default_factory=lambda: [])
 
     def __post_init__(self):
@@ -411,7 +427,6 @@ class StreamingFeatureSet(BaseFeatureSet):
             proto_featureset_type = self._get_streaming_aggregation_featureset_proto(
                 artifact_url=artifact_url,
                 streaming_sources=data_sources,
-                feature_registry=feature_registry,
                 initial_tile_size=maybe_initial_tile_size,
             )
 
@@ -465,10 +480,9 @@ class StreamingFeatureSet(BaseFeatureSet):
                 "Auxiliary Sinks Are not supported in Streaming Aggregation Feature Sets"
             )
 
-        # streaming backfill not yet supported for row-level streaming
-        if self.backfill and not is_streaming_agg:
+        if self.backfill_max_timestamp and not is_streaming_agg:
             raise FrogmlException(
-                "Streaming backfill is only supported for streaming aggregation feature sets at the moment"
+                "backfill_max_timestamp can only be set for Streaming Aggregation FeatureSet."
             )
 
         # Validate transformation is PySpark when multiple data sources are used
@@ -527,18 +541,29 @@ class StreamingFeatureSet(BaseFeatureSet):
                 )
             raise FrogmlException(error_message_str)
 
-        # Validate the backfill, if defined
-        if self.backfill:
-            self.backfill._validate_tile_size(initial_tile_size)
+        if not self.backfill_max_timestamp:
+            raise FrogmlException(
+                """
+                backfill_max_timestamp must be set for Streaming Aggregation FeatureSet.
+                 Events earlier than this timestamp can only be processed by triggering backfill,
+                 the Streaming job will not process events that are earlier than this timestamp.
+            """
+            )
+
+        self._validate_streaming_aggregation_backfill_max_timestamp()
 
         return initial_tile_size
 
-    def _validate_streaming_aggregation_backfill(self):
+    def _validate_streaming_aggregation_backfill_max_timestamp(self):
         initial_tile_size, _ = StreamingFeatureSet._get_default_slide_period(
             self.transformation.windows
         )
 
-        self.backfill._validate_tile_size(initial_tile_size)
+        if self.backfill_max_timestamp.timestamp() % initial_tile_size != 0:
+            raise FrogmlException(
+                f"Chosen backfill max timestamp is invalid,"
+                f" it has to be exactly dividable by slice size of {initial_tile_size} seconds."
+            )
 
     @staticmethod
     def _get_default_slide_period(
@@ -608,9 +633,12 @@ class StreamingFeatureSet(BaseFeatureSet):
         self,
         artifact_url: Optional[str],
         streaming_sources: List[StreamingSource],
-        feature_registry: FeatureRegistryClient,
         initial_tile_size: int,
     ) -> ProtoFeatureSetType:
+        backfill_max_timestamp = ProtoTimestamp()
+        backfill_max_timestamp.FromDatetime(
+            self.backfill_max_timestamp.astimezone(timezone.utc)
+        )
         return ProtoFeatureSetType(
             streaming_aggregation_feature_set=ProtoStreamingAggregationFeatureSet(
                 transformation=self.transformation._to_proto(
@@ -633,14 +661,7 @@ class StreamingFeatureSet(BaseFeatureSet):
                     allowed_late_arrival_seconds=60 * 10,
                     aggregations=self.transformation._get_aggregations_proto(),
                 ),
-                backfill_spec=(
-                    self.backfill._to_proto(
-                        feature_registry=feature_registry,
-                        original_instance_module_path=self.__instance_module_path__,
-                        featureset_name=self.name,
-                    )
-                    if self.backfill
-                    else None
-                ),
+                backfill_spec=None,
+                backfill_max_timestamp=backfill_max_timestamp,
             )
         )

frogml/feature_store/feature_sets/streaming_backfill.py

@@ -1,34 +1,18 @@
-from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import List, Optional, Set, Union
 
-from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
-
 from frogml._proto.qwak.feature_store.features.execution_pb2 import (
-    BackfillExecutionSpec as ProtoBackfillExecutionSpec,
-)
-from frogml._proto.qwak.feature_store.features.feature_set_types_pb2 import (
-    BackfillBatchDataSourceSpec as ProtoBackfillBatchDataSourceSpec,
-)
-from frogml._proto.qwak.feature_store.features.feature_set_types_pb2 import (
-    BackfillDataSourceSpec as ProtoBackfillDataSourceSpec,
+    ExecutionSpec as ProtoExecutionSpec,
 )
-from frogml._proto.qwak.feature_store.features.feature_set_types_pb2 import (
-    BackfillSpec as ProtoBackfillSpec,
+from frogml._proto.qwak.execution.v1.streaming_aggregation_pb2 import (
+    StreamingAggregationBackfillIngestion as ProtoStreamingAggregationBackfillIngestion,
+    BackfillDataSource as ProtoBackfillDataSource,
+    TimeRange as ProtoTimeRange,
 )
-from frogml._proto.qwak.feature_store.sources.batch_pb2 import (
-    BatchSource as ProtoBatchSource,
-)
-from frogml.core.clients.feature_store import FeatureRegistryClient
+from google.protobuf.timestamp_pb2 import Timestamp as ProtoTimestamp
 from frogml.core.exceptions import FrogmlException
-from frogml.feature_store._common.artifact_utils import (
-    ArtifactSpec,
-    ArtifactsUploader,
-)
-from frogml.feature_store._common.feature_set_utils import (
-    get_batch_source_for_featureset,
-)
+from frogml.feature_store._common.artifact_utils import ArtifactSpec, ArtifactsUploader
 from frogml.core.feature_store.feature_sets.execution_spec import ClusterTemplate
 from frogml.core.feature_store.feature_sets.transformations import (
     SparkSqlTransformation,
@@ -38,36 +22,15 @@ _BACKFILL_ = "_qwak_backfill_specification"
 
 
 @dataclass
-class DataSourceBackfillSpec(ABC):
+class BackfillDataSource:
     data_source_name: str
-
-    @abstractmethod
-    def _to_proto(self, feature_registry: FeatureRegistryClient):
-        pass
-
-    @classmethod
-    def _from_proto(cls, proto: ProtoBackfillDataSourceSpec):
-        function_mapping = {"batch_data_source_spec": BackfillBatchDataSourceSpec}
-
-        backfill_source_type: str = proto.WhichOneof("type")
-
-        if backfill_source_type in function_mapping:
-            function_class = function_mapping.get(backfill_source_type)
-            return function_class._from_proto(proto)
-
-        raise FrogmlException(
-            f"Got unsupported backfill source type {backfill_source_type} for streaming backfill"
-        )
-
-
-@dataclass
-class BackfillBatchDataSourceSpec(DataSourceBackfillSpec):
     start_datetime: Optional[datetime] = None
     end_datetime: Optional[datetime] = None
 
-    def _to_proto(
-        self, feature_registry: FeatureRegistryClient
-    ) -> ProtoBackfillBatchDataSourceSpec:
+    def __post_init__(self):
+        self._validate()
+
+    def _to_proto(self) -> ProtoBackfillDataSource:
         start_timestamp: Optional[ProtoTimestamp] = None
         end_timestamp: Optional[ProtoTimestamp] = None
 
@@ -79,63 +42,94 @@ class BackfillBatchDataSourceSpec(DataSourceBackfillSpec):
             start_timestamp = ProtoTimestamp()
             start_timestamp.FromDatetime(self.start_datetime.astimezone(timezone.utc))
 
-        proto_data_source: ProtoBatchSource = get_batch_source_for_featureset(
-            batch_ds_name=self.data_source_name, feature_registry=feature_registry
-        )
-
-        return ProtoBackfillBatchDataSourceSpec(
-            data_source=proto_data_source,
+        time_range = ProtoTimeRange(
             start_timestamp=start_timestamp,
             end_timestamp=end_timestamp,
         )
 
+        return ProtoBackfillDataSource(
+            data_source_name=self.data_source_name,
+            time_range=time_range,
+        )
+
     @classmethod
-    def _from_proto(
-        cls, proto: ProtoBackfillDataSourceSpec
-    ) -> "BackfillBatchDataSourceSpec":
+    def _from_proto(cls, proto: ProtoBackfillDataSource) -> "BackfillDataSource":
         start_datetime: Optional[datetime] = None
         end_datetime: Optional[datetime] = None
 
-        batch_backfill_spec: ProtoBackfillBatchDataSourceSpec = (
-            proto.batch_data_source_spec
-        )
+        time_range: ProtoTimeRange = proto.time_range
 
-        proto_start_timestamp: ProtoTimestamp = batch_backfill_spec.start_timestamp
-        proto_end_timestamp: ProtoTimestamp = batch_backfill_spec.end_timestamp
+        proto_start_timestamp: Optional[ProtoTimestamp] = (
+            time_range.start_timestamp if time_range.start_timestamp else None
+        )
+        proto_end_timestamp: Optional[ProtoTimestamp] = (
+            time_range.end_timestamp if time_range.end_timestamp else None
+        )
 
-        start_datetime = datetime.fromtimestamp(
-            proto_start_timestamp.seconds + proto_start_timestamp.nanos / 1e9
+        start_datetime = (
+            datetime.fromtimestamp(
+                proto_start_timestamp.seconds + proto_start_timestamp.nanos / 1e9
+            )
+            if proto_start_timestamp
+            else None
         )
 
-        end_datetime = datetime.fromtimestamp(
-            proto_end_timestamp.seconds + proto_end_timestamp.nanos / 1e9
+        end_datetime = (
+            datetime.fromtimestamp(
+                proto_end_timestamp.seconds + proto_end_timestamp.nanos / 1e9
+            )
+            if proto_end_timestamp
+            else None
         )
 
         return cls(
-            data_source_name=batch_backfill_spec.data_source.name,
+            data_source_name=proto.data_source_name,
             start_datetime=start_datetime,
             end_datetime=end_datetime,
         )
 
+    def _validate(self):
+        if self.start_datetime and self.end_datetime:
+            if self.start_datetime >= self.end_datetime:
+                raise FrogmlException(
+                    f"Backfill data source {self.data_source_name} has invalid time range: "
+                    f"start_datetime {self.start_datetime} is after or equal end_datetime {self.end_datetime}."
+                )
+
+        if not self.data_source_name:
+            raise FrogmlException(
+                "Backfill data source must have a valid data source name."
+            )
+
 
 @dataclass
 class StreamingBackfill:
+    featureset_name: str
     start_datetime: datetime
     end_datetime: datetime
-    data_sources_specs: List[DataSourceBackfillSpec]
+    data_sources: List[BackfillDataSource]
     transform: "SparkSqlTransformation"
     cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL
 
     def __post_init__(self):
-        if not self.data_sources_specs:
+        if not self.featureset_name:
+            raise FrogmlException("featureset_name must be provided for backfill.")
+
+        if not self.start_datetime or not self.end_datetime:
             raise FrogmlException(
-                "Trying to create a streaming backfill with no data sources. "
-                "At least one data source has to be provided when trying to create a streaming backfill."
+                "For streaming aggregation backfill, both start_datetime and end_datetime are mandatory fields."
             )
 
-        if not self.start_datetime or not self.end_datetime:
+        if self.start_datetime >= self.end_datetime:
             raise FrogmlException(
-                "For backfill, start_datetime and end_datetime are mandatory fields."
+                f"Backfill has invalid time range: "
+                f"start_datetime {self.start_datetime} is after or equal end_datetime {self.end_datetime}."
+            )
+
+        if not self.data_sources:
+            raise FrogmlException(
+                "Trying to create a streaming backfill with no data sources. "
+                "At least one data source has to be provided when trying to create a streaming backfill."
             )
 
         if type(self.transform) is not SparkSqlTransformation:
@@ -147,7 +141,7 @@ class StreamingBackfill:
 
     def _validate_unique_sources(self):
         source_names: List[str] = [
-            data_source.data_source_name for data_source in self.data_sources_specs
+            data_source.data_source_name for data_source in self.data_sources
         ]
         duplicates: Set[str] = {
             item for item in source_names if source_names.count(item) > 1
@@ -158,23 +152,14 @@ class StreamingBackfill:
                 f"Found these duplicates: {', '.join(set(duplicates))}"
             )
 
-    def _validate_tile_size(self, initial_tile_size: int):
-        if self.end_datetime.timestamp() % initial_tile_size != 0:
-            raise FrogmlException(
-                f"Chosen backfill end datetime is invalid,"
-                f" it has to be exactly dividable by slice size of {initial_tile_size} seconds."
-            )
-
     def _to_proto(
         self,
-        feature_registry: FeatureRegistryClient,
-        featureset_name: str,
         original_instance_module_path: str,
-    ) -> ProtoBackfillSpec:
+    ) -> ProtoStreamingAggregationBackfillIngestion:
         artifact_url: Optional[str] = None
         artifact_spec: Optional[ArtifactSpec] = ArtifactsUploader.get_artifact_spec(
             transformation=self.transform,
-            featureset_name=f"{featureset_name}-backfill",
+            featureset_name=f"{self.featureset_name}-backfill",
             __instance_module_path__=original_instance_module_path,
         )
 
@@ -187,85 +172,54 @@ class StreamingBackfill:
         start_timestamp = ProtoTimestamp()
         start_timestamp.FromDatetime(self.start_datetime.astimezone(timezone.utc))
 
-        return ProtoBackfillSpec(
+        return ProtoStreamingAggregationBackfillIngestion(
+            featureset_name=self.featureset_name,
             start_timestamp=start_timestamp,
             end_timestamp=end_timestamp,
-            execution_spec=ProtoBackfillExecutionSpec(
-                **{"cluster_template": ClusterTemplate.to_proto(self.cluster_template)}
+            execution_spec=ProtoExecutionSpec(
+                cluster_template=ClusterTemplate.to_proto(self.cluster_template)
             ),
             transformation=self.transform._to_proto(artifact_path=artifact_url),
             data_source_specs=[
-                ProtoBackfillDataSourceSpec(
-                    batch_data_source_spec=data_source_spec._to_proto(
-                        feature_registry=feature_registry
-                    )
-                )
-                for data_source_spec in self.data_sources_specs
+                data_source._to_proto() for data_source in self.data_sources
             ],
         )
 
     @classmethod
-    def _from_proto(cls, proto: ProtoBackfillSpec):
-        datetime.fromtimestamp(
-            proto.start_timestamp.seconds + proto.start_timestamp.nanos / 1e9
-        )
-
-        data_sources_specs = [
-            BackfillBatchDataSourceSpec._from_proto(ds)
-            for ds in proto.data_source_specs
+    def _from_proto(cls, proto: ProtoStreamingAggregationBackfillIngestion):
+        backfill_data_sources = [
+            BackfillDataSource._from_proto(ds) for ds in proto.data_source_specs
         ]
 
         return cls(
+            featureset_name=proto.featureset_name,
             start_datetime=datetime.fromtimestamp(
                 proto.start_timestamp.seconds + proto.start_timestamp.nanos / 1e9
             ),
             end_datetime=datetime.fromtimestamp(
                 proto.end_timestamp.seconds + proto.end_timestamp.nanos / 1e9
             ),
-            data_sources_specs=data_sources_specs,
+            data_sources=backfill_data_sources,
             transform=SparkSqlTransformation._from_proto(
                 proto.transformation.sql_transformation
             ),
+            cluster_template=(
+                ClusterTemplate.from_proto(proto.execution_spec.cluster_template)
+                if proto.execution_spec.cluster_template
+                else None
+            ),
         )
 
     @staticmethod
     def _get_normalized_backfill_sources_spec(
-        data_sources: Union[List[str], List[DataSourceBackfillSpec]],
-    ) -> List[DataSourceBackfillSpec]:
-        # reformat all data source specs to 'DataSourceBackfillSpec'
+        data_sources: Union[List[str], List[BackfillDataSource]],
+    ) -> List[BackfillDataSource]:
+        # reformat all data source names to 'BackfillDataSource'
         return [
             (
-                BackfillBatchDataSourceSpec(data_source_name=data_source)
+                BackfillDataSource(data_source_name=data_source)
                 if isinstance(data_source, str)
                 else data_source
             )
             for data_source in data_sources
         ]
-
-    @classmethod
-    def set_streaming_backfill_on_function(
-        cls,
-        function,
-        start_date: datetime,
-        end_date: datetime,
-        data_sources: Union[List[str], List[DataSourceBackfillSpec]],
-        backfill_transformation: SparkSqlTransformation,
-        backfill_cluster_template: Optional[ClusterTemplate] = ClusterTemplate.SMALL,
-    ):
-        setattr(
-            function,
-            _BACKFILL_,
-            cls(
-                start_datetime=start_date,
-                end_datetime=end_date,
-                data_sources_specs=StreamingBackfill._get_normalized_backfill_sources_spec(
-                    data_sources
-                ),
-                transform=backfill_transformation,
-                cluster_template=backfill_cluster_template,
-            ),
-        )
-
-    @staticmethod
-    def get_streaming_backfill_from_function(function):
-        return getattr(function, _BACKFILL_, None)

frogml/sdk/model_version/utils/jml/customer_client.py

@@ -3,23 +3,19 @@ import os
 import sys
 from typing import Optional, Dict, Any, TextIO, Union
 
-import requests
-from requests import Response
-
-from frogml.core.clients.administration.eco_system.client import EcosystemClient
-from frogml.storage.frog_ml import FrogMLStorage
-from frogml.storage.models.serialization_metadata import SerializationMetadata
-from frogml.storage.models.frogml_model_version import FrogMLModelVersion
-
 from typing_extensions import Self
 
 from frogml._proto.jfml.model_version.v1.model_version_framework_pb2 import (
     ModelVersionFramework,
 )
+from frogml.core.clients.administration.eco_system.client import EcosystemClient
 from frogml.core.clients.model_version_manager import ModelVersionManagerClient
 from frogml.core.exceptions import FrogmlException
 from frogml.sdk.model_version.constants import FROGML_LOG_LEVEL_ENVAR_NAME
+from frogml.storage.frog_ml import FrogMLStorage
+from frogml.storage.models.frogml_model_version import FrogMLModelVersion
 from frogml.storage.models.model_manifest import ModelManifest
+from frogml.storage.models.serialization_metadata import SerializationMetadata
 
 
 class JmlCustomerClient:
@@ -66,7 +62,6 @@ class JmlCustomerClient:
 
     def validate_model_version(
         self: Self,
-        project_key: str,
         repository: str,
         model_name: str,
         model_version: str,
@@ -74,7 +69,6 @@ class JmlCustomerClient:
     ):
         try:
             self.__model_version_manager_client.validate_create_model_version(
-                project_key=project_key,
                 repository_key=repository,
                 model_name=model_name,
                 model_version_name=model_version,
@@ -84,29 +78,6 @@ class JmlCustomerClient:
         except Exception as e:
             raise ValueError(str(e)) from e
 
-    def get_project_key(self: Self, repository_key: str) -> str:
-        self.__logger.info("Getting project key for repository %s", repository_key)
-        repo_configuration_response: Response = (
-            self.__ml_storage.get_repository_configuration(
-                repository_key=repository_key
-            )
-        )
-        try:
-            repo_configuration_response.raise_for_status()
-        except requests.exceptions.HTTPError as err:
-            # Attach the response body for better error logging
-            raise FrogmlException(f"{err}\nResponse body: {err.response.text}") from err
-
-        configuration_json: dict[str, object] = repo_configuration_response.json()
-        project_key: Optional[str] = configuration_json.get("projectKey")
-
-        if project_key is None:
-            raise ValueError(
-                f"Repository '{repository_key}' does not belong to any project"
-            )
-
-        return project_key
-
     def log_model_to_artifactory(
         self: Self,
         dependencies: Optional[list[str]],
@@ -131,7 +102,6 @@ class JmlCustomerClient:
 
     def register_model_version_in_jml(
         self: Self,
-        project_key: str,
         repository: str,
         model_name: str,
         model_version_name: str,
@@ -143,7 +113,6 @@ class JmlCustomerClient:
         """Register a model version in JFrog ML."""
         try:
             self.__model_version_manager_client.create_model_version(
-                project_key=project_key,
                 repository_key=repository,
                 model_name=model_name,
                 model_version_name=model_version_name,

frogml/sdk/model_version/utils/storage.py

@@ -3,8 +3,23 @@ import os
 from platform import python_version
 from typing import Dict, Optional, Callable, cast, Any
 
+from frogml._proto.jfml.model_version.v1.model_version_framework_pb2 import (
+    ModelVersionFramework,
+    CatboostFramework,
+    HuggingFaceFramework,
+    OnnxFramework,
+    PythonPickleFramework,
+    PytorchFramework,
+    ScikitLearnFramework,
+)
 from frogml.core.exceptions import FrogmlException
 from frogml.sdk.model_version.constants import (
+    CATBOOST_SERIALIZED_TYPE,
+    HUGGINGFACE_FRAMEWORK_FORMAT,
+    ONNX_FRAMEWORK_FORMAT,
+    PYTHON_FRAMEWORK_FORMAT,
+    PYTORCH_FRAMEWORK_FORMAT,
+    SCIKIT_LEARN_FRAMEWORK_FORMAT,
     FROGML_LOG_LEVEL_ENVAR_NAME,
     ModelFramework,
     STORAGE_MODEL_ENTITY_TYPE,
@@ -28,23 +43,6 @@ from frogml.sdk.model_version.utils.validations import (
 from frogml.storage.frog_ml import SerializationMetadata, FrogMLStorage
 from frogml.storage.models.frogml_model_version import FrogMLModelVersion
 from frogml.storage.models.model_manifest import ModelManifest
-from frogml._proto.jfml.model_version.v1.model_version_framework_pb2 import (
-    ModelVersionFramework,
-    CatboostFramework,
-    HuggingFaceFramework,
-    OnnxFramework,
-    PythonPickleFramework,
-    PytorchFramework,
-    ScikitLearnFramework,
-)
-from frogml.sdk.model_version.constants import (
-    CATBOOST_SERIALIZED_TYPE,
-    HUGGINGFACE_FRAMEWORK_FORMAT,
-    ONNX_FRAMEWORK_FORMAT,
-    PYTHON_FRAMEWORK_FORMAT,
-    PYTORCH_FRAMEWORK_FORMAT,
-    SCIKIT_LEARN_FRAMEWORK_FORMAT,
-)
 
 _PYTHON_RUNTIME = "python"
 
@@ -128,9 +126,7 @@ def _log_model(config: ModelLogConfig) -> None:
     )
 
     if is_customer_exists_in_jml:
-        project_key: str = jml_customer_client.get_project_key(config.repository)
         jml_customer_client.validate_model_version(
-            project_key,
             config.repository,
             config.model_name,
             config.version,
@@ -170,7 +166,6 @@ def _log_model(config: ModelLogConfig) -> None:
 
     if is_customer_exists_in_jml:
         jml_customer_client.register_model_version_in_jml(
-            project_key,
             config.repository,
             config.model_name,
             config.version,

frogml/storage/artifactory/_artifactory_api.py

@@ -280,12 +280,6 @@ class ArtifactoryApi:
             url=join_url(self.uri, "/api/security/encryptedPassword")
         )
 
-    def get_repository_configuration(self: Self, repository_key: str) -> Response:
-        url: str = modify_url(
-            self.uri, f"artifactory/api/v2/repositories/{repository_key}"
-        )
-        return self.http_client.get(url=url)
-
     def generate_access_token(self: Self) -> Response:
         """
         Generates an access token for authenticating with the JFrog Platform.

frogml/storage/frog_ml.py

@@ -42,7 +42,6 @@ from frogml.storage.utils import (
     validate_not_folder_paths,
     validate_path_exists,
 )
-from typing_extensions import Self
 
 
 class FileType(Enum):
@@ -74,11 +73,6 @@ class FrogMLStorage(BaseStorage):
             max_workers=self.http_threads_count
         )
 
-    def get_repository_configuration(self: Self, repository_key: str):
-        return self.artifactory_api.get_repository_configuration(
-            repository_key=repository_key
-        )
-
     def upload_dataset_version(
         self,
         repository: str,

{frogml-1.2.21.dist-info → frogml-1.2.28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: frogml
-Version: 1.2.21
+Version: 1.2.28
 Summary: frogml contains the necessary objects and communication tools for using the JFrog ml Platform
 License: Apache-2.0
 Keywords: mlops,ml,deployment,serving,model

{frogml-1.2.21.dist-info → frogml-1.2.28.dist-info}/RECORD

@@ -1,4 +1,4 @@
-frogml/__init__.py,sha256=s0nXtzrdJYbEJh8aECBl2rR5DzeGnr3f1eWUqvmOB1g,741
+frogml/__init__.py,sha256=82ta2BQl2e3AbEOMtmrDec89eOL9uy9NwlWqaiqf6q0,741
 frogml/_proto/jfml/hosting_gateway/v1/build_upload_url_pb2.py,sha256=pY7-QarLJWbL9uNmZ69RfE7IPVzwpVIhtd3A0wztSNY,1942
 frogml/_proto/jfml/hosting_gateway/v1/build_upload_url_pb2.pyi,sha256=505I4uUrCmv2hsUw2H5eT-nlkNOCQOWodpvEkH2fHIo,3073
 frogml/_proto/jfml/hosting_gateway/v1/build_upload_url_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
@@ -12,7 +12,7 @@ frogml/_proto/jfml/model_version/v1/build_spec_pb2.py,sha256=S_pdnyFvTLnXNe_iRZ0
 frogml/_proto/jfml/model_version/v1/build_spec_pb2.pyi,sha256=P7We8OZGBi-_0g-s1p4hPwvqYE3ygaiZsqjYiBmJA2E,4821
 frogml/_proto/jfml/model_version/v1/build_spec_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 frogml/_proto/jfml/model_version/v1/model_repository_spec_pb2.py,sha256=C41E30HmYwfYtAwPUM9lJW_oWg6JH7u9NAQWN3DnMWI,1503
-frogml/_proto/jfml/model_version/v1/model_repository_spec_pb2.pyi,sha256=Wtd9aR04T4JrU6CJDlyn8WCo8N8MtiHv87RvpzA4gRk,1478
+frogml/_proto/jfml/model_version/v1/model_repository_spec_pb2.pyi,sha256=_Rr7sdDxsVjl0L4GReKMAaeUC_yZJ_OgvkNg81IrmWs,1660
 frogml/_proto/jfml/model_version/v1/model_repository_spec_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 frogml/_proto/jfml/model_version/v1/model_version_framework_pb2.py,sha256=ACFTNdlRZ1yHF1mV0ZxjE_rR-gWZxS8HB2maPUw59J0,2765
 frogml/_proto/jfml/model_version/v1/model_version_framework_pb2.pyi,sha256=Ov8O3lWPtc-KmGjpm3-It15IINzCVt2QXAC6JXNrupY,3725
@@ -542,8 +542,8 @@ frogml/_proto/qwak/model_descriptor/open_ai_descriptor_pb2_grpc.py,sha256=1oboBP
 frogml/_proto/qwak/model_group/model_group_pb2.py,sha256=vKhDaG_NnEMNxqGSemEpHATsSWUS9nhl5Xaz9MhPIfg,5317
 frogml/_proto/qwak/model_group/model_group_pb2.pyi,sha256=OxpEeUGJXQACiiCbAAWDSkBuTSsT7qnKcH-3A-oA7wQ,9144
 frogml/_proto/qwak/model_group/model_group_pb2_grpc.py,sha256=YgDt3DtagEdZ7UtxcP5iuoJeFDcVwQOrOU9o_tk4vLs,9258
-frogml/_proto/qwak/model_group/model_group_repository_details_pb2.py,sha256=yIUxJ45ipdeLKrGbM0j8zypQtyDs0-PPFEjDq48t4oA,3160
-frogml/_proto/qwak/model_group/model_group_repository_details_pb2.pyi,sha256=ugutJEvcu-lFaMiVWlmXvFN-ngsWP_LXeiTzxun68Zc,5594
+frogml/_proto/qwak/model_group/model_group_repository_details_pb2.py,sha256=BxaVjDayd9h5Va2ZId3pNPeJFAqVXb8TJHIWmYsURN4,3806
+frogml/_proto/qwak/model_group/model_group_repository_details_pb2.pyi,sha256=3oCsYmmfvBjGkA1nRcx2cg11WNr40uuR3gAWXC7Wz2I,7245
 frogml/_proto/qwak/model_group/model_group_repository_details_pb2_grpc.py,sha256=1oboBPFxaTEXt9Aw7EAj8gXHDCNMhZD2VXqocC9l_gk,159
 frogml/_proto/qwak/models/models_pb2.py,sha256=XF3-e8xW62ymbg7xGRCwc4fNsgXV2UZJ1lPgv47Ylkk,19046
 frogml/_proto/qwak/models/models_pb2.pyi,sha256=m2NGA7K3Jz3YFp8nOW4l_AB5kgjJUEDmr3xMUPYu3Dc,42638
@@ -672,7 +672,7 @@ frogml/core/clients/data_versioning/data_tag_filter.py,sha256=5YK031WGNrvkydrruW
 frogml/core/clients/deployment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/core/clients/deployment/client.py,sha256=mMwguoxysULkue-ABYQwVdl-RtUE6VtD-ulq2IpwD-A,6883
 frogml/core/clients/feature_store/__init__.py,sha256=mMCPBHDga6Y7dtJfNoHvfOvCyjNUHrVDX5uVsL2JkGk,53
-frogml/core/clients/feature_store/execution_management_client.py,sha256=cJf1qUaghI8zfTdJQ2cVxYqoOz3SXfnuBx6LKCQFwEE,4118
+frogml/core/clients/feature_store/execution_management_client.py,sha256=xftjUoCfgmQs1IuXprxgiYDyW_RCrOxdOWVFCfVEQQk,5343
 frogml/core/clients/feature_store/job_registry_client.py,sha256=Rhb5YLvu9JmPpgwsK2piZYxi4OosSaFfFymUnrp67as,2670
 frogml/core/clients/feature_store/management_client.py,sha256=o0VtaxAq0SFq2s1sEW2LmoHoTI8e5xS63ZRGjtoMiIw,18284
 frogml/core/clients/feature_store/offline_serving_client.py,sha256=yRLAmdOscqGOo4w_DoeAR6GaakOCSxw_378tggkum9I,9377
@@ -702,7 +702,7 @@ frogml/core/clients/model_management/client.py,sha256=L3rUPfxRNA2s4KV5294QXkPCmi
 frogml/core/clients/model_version_manager/__init__.py,sha256=4Pnfzj4Egps48__dProdbSKQl5pNip0hGJd75w67BfY,46
 frogml/core/clients/model_version_manager/build_model_version_dto.py,sha256=N-W84WyKDiwr-lmcpLss0gOV3QK4EAcpv2lqv1TROos,1984
 frogml/core/clients/model_version_manager/build_model_version_request_mapper.py,sha256=BYK_nmfLF0o9QWTKfcOfsP5j1hmdu5RQkmsQ0ypO6rI,4760
-frogml/core/clients/model_version_manager/client.py,sha256=VMgOoZW8EoSwx3BZCtknbxlhwOob1Y3GH9XSj7bBg6Y,11314
+frogml/core/clients/model_version_manager/client.py,sha256=T3pJmjgkL8E0SPSGWZHXDVmEAs3Ow36p-nHAg-tjgxo,11108
 frogml/core/clients/model_version_manager/const.py,sha256=SBEi06KU-ntF6oxjTpSXbd9R72x_mm9cD9JhYQLXsMY,684
 frogml/core/clients/secret_service/__init__.py,sha256=TdQl1lgplXCKVHYSN4feRIAoonZ7XDz50zALjwVDcM4,40
 frogml/core/clients/secret_service/client.py,sha256=RDicLLGWIV1Ecm5A4F4TcQec5SHudTkdXG8eRilHJ8M,3657
@@ -743,6 +743,7 @@ frogml/core/feature_store/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQ
 frogml/core/feature_store/execution/backfill.py,sha256=35h8Q82AlPSN09kgIR5iyHEg-0yK2AvUOcxthpWF7qo,6716
 frogml/core/feature_store/execution/execution.py,sha256=CgPJLPFRhBI8pQiAOkDSO0OtKIAQat3kbaJ_NsFB-iE,21324
 frogml/core/feature_store/execution/execution_query.py,sha256=eTMxx-SmyjDPoBZwqJTJn-qs1TfkRgs3akvN_kMId1Y,3646
+frogml/core/feature_store/execution/streaming_backfill.py,sha256=xzba3rjHh-RolA7nw70dgC6BgOUTRcwjyWYUQakLVmA,1695
 frogml/core/feature_store/feature_sets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/core/feature_store/feature_sets/_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/core/feature_store/feature_sets/_utils/_featureset_utils.py,sha256=xWy-7_ai9OA7tCkQIrGwdqjZHKXrtQgmqHxHq1l3afo,1658
@@ -903,8 +904,8 @@ frogml/feature_store/data_sources/streaming/kafka/kafka.py,sha256=Bo_yxjNcP-IJ4T
 frogml/feature_store/feature_sets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/feature_store/feature_sets/base_feature_set.py,sha256=Rk8ZWjQigPnLBERePXjdJT_Pf1VzF9js7bPLnPCCYLM,5454
 frogml/feature_store/feature_sets/batch.py,sha256=JHEybhYZuVLFeJ7MiXGYHCITfldUpw9El6BcGCIEPzg,17626
-frogml/feature_store/feature_sets/streaming.py,sha256=_UkXWbMdJdD0asfPRw1lDKCTLcqsEfPcJ3nDkBd9ndk,25595
-frogml/feature_store/feature_sets/streaming_backfill.py,sha256=Y_hdKEO2wCJj0pcljUBBGIS4IdgMNjjmp9TWsoXxrJs,9829
+frogml/feature_store/feature_sets/streaming.py,sha256=1f63h_YP-qDDYwNCTZiG5M0Gh1iHRLvo0QoFdPyA2fc,26915
+frogml/feature_store/feature_sets/streaming_backfill.py,sha256=vjgFGjYNnER7a7awG56dtFQ5U4zKhTNSt805SRNEtaQ,8351
 frogml/feature_store/offline/__init__.py,sha256=TUu3VuxdbVrYpWD9B2LDno1HfoGbI2X4fHw7vjSJ2uk,98
 frogml/feature_store/offline/client_v2.py,sha256=33CPNWMmZGw8nVet5TrJa5G0hBr93SVsgVKbdiTHK9I,14841
 frogml/feature_store/online/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1056,16 +1057,16 @@ frogml/sdk/model_version/transformers/initialize_model_transformer.py,sha256=7AT
 frogml/sdk/model_version/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/sdk/model_version/utils/dependencies_tools.py,sha256=Sdmbf3Nb_xlc8uY4fK8bvPOGwh7OtejdZd1m5POfsDU,6403
 frogml/sdk/model_version/utils/files_tools.py,sha256=9995FuZ5xMbN5Bm77BzDYU0cQ4nwCYYAl6goynJVvdI,11595
-frogml/sdk/model_version/utils/jml/customer_client.py,sha256=YNSPpAGH3VIv-Ese-ru31dksbLvbLbGV_6IrM4z7mOE,6167
+frogml/sdk/model_version/utils/jml/customer_client.py,sha256=a5S2CaRFiFSXmFsuFCmd7OO_bpweha-KcAHzcyjz1fA,5034
 frogml/sdk/model_version/utils/log_model_helper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/sdk/model_version/utils/model_log_config.py,sha256=6Geo68s02zJc9pqqmEoOe8za5DU2ttvWBdKf8eXhvVY,12267
-frogml/sdk/model_version/utils/storage.py,sha256=BTAv5BrTLwWSIsEZTotA6TlS-opUgVICNma01kkfkJA,10438
+frogml/sdk/model_version/utils/storage.py,sha256=aVyixU-BygUTVFrxV4nMfA5AiSPCyuia8GL38YoJmiM,10255
 frogml/sdk/model_version/utils/storage_helper.py,sha256=fZLIjEAHVxhFXVk08z0pfFWFzOeJDZjLSVVnp3ZY9kw,374
 frogml/sdk/model_version/utils/validations.py,sha256=pGrhd5f5WeL5qit9CCG0Jlx21WY5M3V_psTD9tlcs7U,1203
 frogml/storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml/storage/_environment.py,sha256=zuzOJBtBwFaguwn_JkKjfhXStZoustgP30KzOP3mYv8,707
 frogml/storage/artifactory/__init__.py,sha256=C02rcm7kqsZBVA6c6Gztxamj96hn8Aj6BuzYWFRmWbQ,71
-frogml/storage/artifactory/_artifactory_api.py,sha256=26ktPpiMoVFPvVF_jzc3Y_3Q7X6jdcEQzjD63Mbu38E,12766
+frogml/storage/artifactory/_artifactory_api.py,sha256=MkK6ZZ5tJcj7xTk6OqMcfY2MEnFv7zO40Vdndo4ocUQ,12522
 frogml/storage/authentication/login/__init__.py,sha256=ch8UhQwh3o5ddyoJykQ0Bnog5-8qKHmeDpwqny8xMzM,46
 frogml/storage/authentication/login/_login_cli.py,sha256=ZSTIGdoiH2vsOJCqaN0DVwgMgKnV--7SnMkySIWMiks,7798
 frogml/storage/authentication/login/_login_command.py,sha256=4ib9UVMC-CmsMuJATd_l3luvUhRvzbIs35fZw2bA0Ms,2845
@@ -1080,7 +1081,7 @@ frogml/storage/base_storage.py,sha256=k9CAj9wlnlydxuVZKcC-IcW9GP0KdVoDNKUevemeaY
 frogml/storage/constants.py,sha256=9MxTEogYeHYn-6jE5AGBxVcHVX8N8a5eOwfRDGHEvGY,2268
 frogml/storage/exceptions/checksum_verification_error.py,sha256=t1muLRYvqza7Q4jo1mYHck9ZoDwlGE7WxcheAFAnIQs,159
 frogml/storage/exceptions/validation_error.py,sha256=_twu_xcL-O7D26qkskuuR-_NcnPPTADIUO9HGY4g0x0,156
-frogml/storage/frog_ml.py,sha256=hU5v9XrlDNNq_A203Pe6QLaiEP1srlSP_udQbWgXoE4,23750
+frogml/storage/frog_ml.py,sha256=s9rfedc3e2RYTflU2Vt59MnEIZvaPW4FwmD5R2PTHxs,23525
 frogml/storage/http/__init__.py,sha256=BOXGzjq1gJXlihldaZvMQNGJIdCy-UHWsk3u-bXEZnQ,36
 frogml/storage/http/http_client.py,sha256=tXegbonhMiyN0FzjTSqOeg5OBsFYBozkbMtRfZibTUM,2435
 frogml/storage/logging/__init__.py,sha256=JaBPWQQt3pjoLhUcZtgqPydGlKAWYpSvfPkt2z1Wi90,32
@@ -1116,7 +1117,7 @@ frogml_services_mock/mocks/build_orchestrator_service_api.py,sha256=7mZzOahph29Z
 frogml_services_mock/mocks/data_versioning_service.py,sha256=8DzV5oxH7DXZAu_ZBiPEwW9m1AwbOlYOO1PFPjkq_Dk,2470
 frogml_services_mock/mocks/deployment_management_service.py,sha256=wofBAw-2NKToTiFYxHqjR3QDrvplpV8NWNZMiIX6U_Q,20583
 frogml_services_mock/mocks/ecosystem_service_api.py,sha256=saJYdT43nGVNyozWeDVc6HIXAsNvCdU5J1i-NNrnOr4,2089
-frogml_services_mock/mocks/execution_management_service.py,sha256=jkGMflwC7D34eyV8mM-8gwiH5S--Yiyhe8iHaX3WMcM,891
+frogml_services_mock/mocks/execution_management_service.py,sha256=pRX_DWzayMhwbw7zQVRx2blVpiubqVWpZng2Xn8ZgI0,1205
 frogml_services_mock/mocks/feature_store_data_sources_manager_api.py,sha256=YNd-0abshQH6bfrIEr1z0awIde_SYabfuZHV83LmcJ4,5249
 frogml_services_mock/mocks/feature_store_entities_manager_api.py,sha256=fF3hhJvbd7PDPyghvqtiJudISOR1GasP6MDs5X44Tic,3355
 frogml_services_mock/mocks/feature_store_feature_set_manager_api.py,sha256=ghTbNTVj-FjSV63AYdha-Z7GCX0Bm6PDT6xYzUa5DVA,10935
@@ -1149,6 +1150,6 @@ frogml_services_mock/mocks/utils/exception_handlers.py,sha256=k_8mez3cwjNjKE9yGQ
 frogml_services_mock/services_mock.py,sha256=y1_h189Ldu5MdVUW4nj-WMBtvUfkM_aKse7UfAb6Rxk,19419
 frogml_services_mock/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 frogml_services_mock/utils/service_utils.py,sha256=ZlB0CnB1J6oBn6_m7fQO2U8tKoboHdUa6ljjkRMYNXU,265
-frogml-1.2.21.dist-info/METADATA,sha256=90MYtAI3jbVGuBmuOf_N3cz8i0J6ZBDRJY4i8w61reM,5599
-frogml-1.2.21.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-frogml-1.2.21.dist-info/RECORD,,
+frogml-1.2.28.dist-info/METADATA,sha256=WWtBNiKfWF8wRe_m94_O3KZhbM99DzKZiThZ5vQpt7w,5599
+frogml-1.2.28.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+frogml-1.2.28.dist-info/RECORD,,

frogml_services_mock/mocks/execution_management_service.py

@@ -1,6 +1,9 @@
 from grpc import RpcError
 
-from frogml._proto.qwak.execution.v1.execution_service_pb2 import TriggerBackfillResponse
+from frogml._proto.qwak.execution.v1.execution_service_pb2 import (
+    TriggerBackfillResponse,
+    TriggerStreamingAggregationBackfillResponse,
+)
 from frogml._proto.qwak.execution.v1.execution_service_pb2_grpc import (
     FeatureStoreExecutionServiceServicer,
 )
@@ -24,3 +27,10 @@ class ExecutionManagementServiceMock(FeatureStoreExecutionServiceServicer):
         if self._raise_exception_on_request:
             raise RpcError
         return TriggerBackfillResponse(execution_id=self._execution_id)
+
+    def TriggerStreamingAggregationBackfill(self, request, context):
+        if self._raise_exception_on_request:
+            raise RpcError
+        return TriggerStreamingAggregationBackfillResponse(
+            execution_id=self._execution_id
+        )