qwak-sdk 0.5.36__py3-none-any.whl → 0.5.38__py3-none-any.whl

qwak_sdk/__init__.py

@@ -1,6 +1,6 @@
 # fmt: off
 __author__ = '''Qwak.ai'''
-__version__ = '0.5.36'
+__version__ = '0.5.38'
 
 from qwak.inner import wire_dependencies
 

qwak_sdk/commands/feature_store/register/_logic.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, cast
 
 from _qwak_proto.qwak.feature_store.entities.entity_pb2 import (
     EntityDefinition as ProtoEntityDefinition,
@@ -10,17 +10,19 @@ from _qwak_proto.qwak.feature_store.features.feature_set_service_pb2 import (
     GetFeatureSetByNameResponse as ProtoGetFeatureSetByNameResponse,
 )
 from _qwak_proto.qwak.features_operator.v3.features_operator_pb2 import (
-    Outputs,
     SparkColumnDescription,
     ValidationSuccessResponse,
 )
 from qwak.clients.feature_store import FeatureRegistryClient
 from qwak.clients.feature_store.operator_client import FeaturesOperatorClient
 from qwak.exceptions import QwakException
-from qwak.feature_store.data_sources.batch._batch import BaseBatchSource
+from qwak.feature_store.data_sources.base import BaseSource
 from qwak.feature_store.entities.entity import Entity
 from qwak.feature_store.feature_sets.base_feature_set import BaseFeatureSet
-from qwak.feature_store.feature_sets.batch import BatchFeatureSet
+from qwak.feature_store.validations.validation_response import (
+    SuccessValidationResponse,
+    ValidationResponse,
+)
 from tabulate import tabulate
 
 from qwak_sdk.inner.file_registry import extract_class_objects
@@ -31,7 +33,9 @@ DELIMITER = "----------------------------------------"
 
 
 def _register_entities(
-    qwak_python_files: List[str], registry: FeatureRegistryClient, force: bool
+    qwak_python_files: List[Tuple[str, str]],
+    registry: FeatureRegistryClient,
+    force: bool,
 ):
     """
     Register Feature Store Entity Objects
@@ -68,7 +72,7 @@ def _register_entities(
 
 
 def _register_data_sources(
-    qwak_python_files: List[str],
+    qwak_python_files: List[Tuple[str, str]],
     registry: FeatureRegistryClient,
     operator_client: FeaturesOperatorClient,
     force: bool,
@@ -89,7 +93,7 @@ def _register_data_sources(
     with qwak_spinner(
         begin_text="Finding Data Sources to register", print_callback=print
     ):
-        qwak_sources = extract_class_objects(qwak_python_files, BaseBatchSource)
+        qwak_sources = extract_class_objects(qwak_python_files, BaseSource)
 
     print(f"👀 Found {len(qwak_sources)} Data Sources")
     for data_source, source_file_path in qwak_sources:
@@ -129,7 +133,7 @@ def _register_data_sources(
 
 def _handle_data_source_validation(
     operator_client: FeaturesOperatorClient,
-    data_source: BaseBatchSource,
+    data_source: BaseSource,
 ):
     print(f"Validating '{data_source.name}' data source")
     with qwak_spinner(begin_text="", print_callback=print):
@@ -139,7 +143,7 @@ def _handle_data_source_validation(
 
     response = getattr(result, result.WhichOneof("type"))
     if isinstance(response, ValidationSuccessResponse):
-        print_validation_outputs(response)
+        print_validation_outputs(response.outputs.stdout, response.outputs.stderr)
         print("✅ Validation completed successfully, got data source columns:")
 
         table = [
@@ -147,13 +151,12 @@ def _handle_data_source_validation(
         ]
         print(tabulate(table, headers=["column name", "type"]))
     else:
-        raise QwakException(f"🧨 Validation failed:\n{response}")
+        raise QwakException(f"🧨 Validation failed: \n{response}")
 
 
 def _register_features_sets(
-    qwak_python_files: List[str],
+    qwak_python_files: List[Tuple[str, str]],
     registry: FeatureRegistryClient,
-    operator_client: FeaturesOperatorClient,
     force: bool,
     git_commit: str,
     no_validation: bool,
@@ -165,7 +168,6 @@ def _register_features_sets(
     Args:
         qwak_python_files: a list of python files containing qwak package imports
         registry: FeatureRegistryClient
-        operator_client: features operator grpc client
         force: boolean determining if to force register all encountered Feature Set objects
         git_commit: the git commit of the parent folder
         no_validation: whether to validate entities
@@ -174,43 +176,48 @@ def _register_features_sets(
     with qwak_spinner(
         begin_text="Finding Feature Sets to register", print_callback=print
     ):
-        qwak_feature_sets = extract_class_objects(qwak_python_files, BatchFeatureSet)
+        qwak_feature_sets = extract_class_objects(qwak_python_files, BaseFeatureSet)
 
     print(f"👀 Found {len(qwak_feature_sets)} Feature Set(s)")
 
-    for feature_set, source_file_path in qwak_feature_sets:
+    for featureset, source_file_path in qwak_feature_sets:
+        featureset = cast(BaseFeatureSet, featureset)
         existing_feature_set: ProtoGetFeatureSetByNameResponse = (
-            registry.get_feature_set_by_name(feature_set.name)
+            registry.get_feature_set_by_name(featureset.name)
         )
 
         registration: bool = False
         if existing_feature_set:
             # Provide entity information of registered feature set before any other operation
-            if feature_set.key:
-                feature_set.entity = (
+            if featureset.key:
+                featureset.entity = (
                     existing_feature_set.feature_set.feature_set_definition.feature_set_spec.entity.entity_spec.name
                 )
 
             registration = ask_yesno(
-                f"Update existing feature set '{feature_set.name}' from source file '{source_file_path}'?",  # nosec B608
+                f"Update existing feature set '{featureset.name}' from source file '{source_file_path}'?",  # nosec B608
                 force,
             )
         else:
             registration = ask_yesno(
-                f"Create new feature set '{feature_set.name}' from source file '{source_file_path}'?",
+                f"Create new feature set '{featureset.name}' from source file '{source_file_path}'?",
                 force,
             )
 
         if registration:
-            spark_columns_features = _batch_v1_feature_set_validation(
-                feature_set=feature_set,
+            features: Optional[List[ProtoFeature]] = []
+            artifact_url: Optional[str] = None
+
+            features, artifact_url = _validate_featureset(
+                featureset=featureset,
                 no_validation=no_validation,
-                operator=operator_client,
-                registry=registry,
             )
 
-            proto_feature_set = feature_set._to_proto(
-                git_commit, spark_columns_features, registry
+            proto_feature_set, _ = featureset._to_proto(
+                git_commit=git_commit,
+                features=features,
+                feature_registry=registry,
+                artifact_url=artifact_url,
             )
 
             if existing_feature_set:
@@ -220,11 +227,12 @@ def _register_features_sets(
                 )
             else:
                 _register_new_feature_set(
-                    new_feature_set=feature_set,
+                    new_feature_set=featureset,
                     entity_key=proto_feature_set.entity,
                     registry=registry,
                     git_commit=git_commit,
-                    spark_columns_features=spark_columns_features,
+                    features=features,
+                    artifact_url=artifact_url,
                 )
 
 
@@ -233,7 +241,8 @@ def _register_new_feature_set(
     entity_key: ProtoEntityDefinition,
     registry: FeatureRegistryClient,
     git_commit: str,
-    spark_columns_features: Optional[List[ProtoFeature]],
+    features: Optional[List[ProtoFeature]],
+    artifact_url: Optional[str] = None,
 ):
     # Create entity for the defined key and set it as the fs's entity
     if new_feature_set.key:
@@ -248,8 +257,8 @@ def _register_new_feature_set(
         new_feature_set.entity = new_entity.name
 
     # this is done to retrieve the entities metadata
-    proto_feature_set = new_feature_set._to_proto(
-        git_commit, spark_columns_features, registry
+    proto_feature_set, _ = new_feature_set._to_proto(
+        git_commit, features, registry, artifact_url=artifact_url
     )
 
     try:
@@ -264,32 +273,26 @@ def _register_new_feature_set(
         raise e1
 
 
-def _handle_batch_v1_feature_set_validation(
-    operator_client: FeaturesOperatorClient,
-    registry_client: FeatureRegistryClient,
-    feature_set: BatchFeatureSet,
-) -> List[SparkColumnDescription]:
-    print(f"Validating '{feature_set.name}' feature set")
+def _handle_featureset_validation(
+    featureset: BaseFeatureSet,
+) -> Tuple[List[ProtoFeature], Optional[str]]:
+    print(f"Validating '{featureset.name}' feature set")
     with qwak_spinner(begin_text="", print_callback=print):
-        feature_set_spec = feature_set._to_proto(
-            feature_registry=registry_client, features=None, git_commit=None
-        )
-        result = operator_client.validate_featureset_blocking(
-            feature_set_spec, resource_path=None, num_samples=10
-        )
+        from qwak.feature_store.validations.validator import FeaturesOperatorValidator
 
-    response = getattr(result, result.WhichOneof("type"))
-    if isinstance(response, ValidationSuccessResponse):
-        print_validation_outputs(response)
+        v = FeaturesOperatorValidator()
+        response: ValidationResponse
+        artifact_url: Optional[str]
+
+        response, artifact_url = v.validate_featureset(featureset=featureset)
+    if isinstance(response, SuccessValidationResponse):
+        print_validation_outputs(response.stdout, response.stderr)
         print("✅ Validation completed successfully, got data source columns:")
-        table = [
-            (x.column_name, x.spark_type) for x in response.spark_column_description
-        ]
+        table = [(x.feature_name, x.feature_type) for x in response.features]
         print(tabulate(table, headers=["column name", "type"]))
-        return response.spark_column_description
-
+        return response.features, artifact_url
     else:
-        raise QwakException(f"🧨 Validation failed:\n{response}")
+        raise QwakException(f"🧨 Validation failed: \n{response}")
 
 
 def _get_features_from_spark_columns_description(
@@ -315,55 +318,47 @@ def _get_features_from_spark_columns_description(
         return None
 
 
-def _batch_v1_feature_set_validation(
-    feature_set: BatchFeatureSet,
+def _validate_featureset(
+    featureset: BaseFeatureSet,
     no_validation: bool,
-    operator: FeaturesOperatorClient,
-    registry: FeatureRegistryClient,
-) -> Optional[List[ProtoFeature]]:
+) -> Tuple[List[ProtoFeature], Optional[str]]:
     """
     Validates featureset transformation
     Args:
-        feature_set: BatchFeatureSet featureset
+        featureset: BaseFeatureSet featureset
         no_validation: skip validation
         operator: Operator client
         registry: Registry client
     Returns:
         Optional list of features returned from validation
     """
-    spark_columns_features: Optional[List[ProtoFeature]] = []
-
+    features: List[ProtoFeature] = []
+    artifact_url: Optional[str] = None
     if not no_validation:
         try:
-            spark_columns_description = _handle_batch_v1_feature_set_validation(
-                operator_client=operator,
-                registry_client=registry,
-                feature_set=feature_set,
-            )
-            spark_columns_features = _get_features_from_spark_columns_description(
-                spark_columns_description
+            features, artifact_url = _handle_featureset_validation(
+                featureset=featureset
             )
         except Exception as e:
             print(str(e))
             print("Not continuing to registration due to failure in validation")
             exit(1)
     else:
-        print(f"Skipping validation for '{feature_set.name}' feature set")
-    return spark_columns_features
+        print(f"Skipping validation for '{featureset.name}' feature set")
+    return features, artifact_url
 
 
-def print_validation_outputs(response: ValidationSuccessResponse) -> bool:
+def print_validation_outputs(stdout: str, stderr: str) -> bool:
     did_print = False
-    outputs = response.outputs
 
-    if isinstance(outputs, Outputs) and (outputs.stdout or outputs.stderr):
+    if stdout or stderr:
         message = "Validation outputs: "
 
-        if outputs.stdout:
-            message += f"stdout: {outputs.stdout}\n "
+        if stdout:
+            message += f"stdout: {stdout}\n "
 
-        if outputs.stderr:
-            message += f"stderr: {outputs.stderr}"
+        if stderr:
+            message += f"stderr: {stderr}"
 
         print(message)
         did_print = True

qwak_sdk/commands/feature_store/register/ui.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from typing import List, Tuple
 
 import click
 from qwak.clients.feature_store import FeatureRegistryClient
@@ -59,6 +60,8 @@ def register_fs_objects(
     ignore_validation_errors: bool,
     **kwargs,
 ):
+    qwak_python_files: List[Tuple[str, str]]
+
     if not path:
         path = Path.cwd()
     else:
@@ -97,7 +100,6 @@ def register_fs_objects(
     _register_features_sets(
         qwak_python_files,
         registry_client,
-        operator_client,
         force,
         git_commit,
         no_validation,

qwak_sdk/inner/file_registry.py

@@ -3,11 +3,12 @@ import os
 import sys
 from contextlib import contextmanager
 from pathlib import Path
+from typing import List, Tuple
 
 from yaspin.core import Yaspin
 
 
-def list_qwak_python_files(path: Path, sp: Yaspin):
+def list_qwak_python_files(path: Path, sp: Yaspin) -> List[Tuple[str, str]]:
     """
     Helper function which finds python files with qwak imports in a given module
 

{qwak_sdk-0.5.36.dist-info → qwak_sdk-0.5.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qwak-sdk
-Version: 0.5.36
+Version: 0.5.38
 Summary: Qwak SDK and CLI for qwak models
 License: Apache-2.0
 Keywords: mlops,ml,deployment,serving,model
@@ -28,7 +28,7 @@ Requires-Dist: pandas (<1.4) ; (python_full_version >= "3.7.1" and python_versio
 Requires-Dist: pandas (>=1.4.3,<2.0.0) ; (python_version >= "3.8" and python_version < "3.10") and (extra == "batch" or extra == "feedback")
 Requires-Dist: pyarrow (>=6.0.0,<11.0.0) ; extra == "batch"
 Requires-Dist: python-json-logger (>=2.0.2)
-Requires-Dist: qwak-core (==0.1.86)
+Requires-Dist: qwak-core (==0.2.2)
 Requires-Dist: qwak-inference (==0.1.8)
 Requires-Dist: tabulate (>=0.8.0)
 Requires-Dist: yaspin (>=2.0.0)

{qwak_sdk-0.5.36.dist-info → qwak_sdk-0.5.38.dist-info}/RECORD

@@ -1,4 +1,4 @@
-qwak_sdk/__init__.py,sha256=AJQ_L-p5V4be66ohMVFo1sxqdwbVQ5mnp_ygDAVqXoc,135
+qwak_sdk/__init__.py,sha256=oaDII2u9M60aLGucJRmp7wpB06VbI3k_SXEQ-9qmRcE,135
 qwak_sdk/cli.py,sha256=FIK1dUNxR57ypb-CeD7fKSJnPJ02lrjR9G4aj2qMLPU,2458
 qwak_sdk/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak_sdk/commands/_logic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -92,8 +92,8 @@ qwak_sdk/commands/feature_store/list/ui.py,sha256=NuXnQ3cdXSDCFAcC2jmgx4sAdjNuyI
 qwak_sdk/commands/feature_store/pause/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak_sdk/commands/feature_store/pause/ui.py,sha256=C89yl4ly-RpyZub30q5xV5g1UDnI_GmHSndvXAkXkw4,695
 qwak_sdk/commands/feature_store/register/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-qwak_sdk/commands/feature_store/register/_logic.py,sha256=Gqf2RBXCtCm7o3or9s-0clmAcjmcvgimWN_9Gtan08E,13658
-qwak_sdk/commands/feature_store/register/ui.py,sha256=WDtQ1V0WoPDir14mZ_N8_RDFZfe68QrtqLDlf64eNwY,2822
+qwak_sdk/commands/feature_store/register/_logic.py,sha256=pK2vd28go45KmLM4D1KdSPynQW_FVsUxh4UVGEOqpkw,13304
+qwak_sdk/commands/feature_store/register/ui.py,sha256=fOnUj970AKLGpWFGzpImAsAKQOO-LGQx0oBlWfB8xNA,2874
 qwak_sdk/commands/feature_store/resume/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak_sdk/commands/feature_store/resume/ui.py,sha256=nI87xvA30qNQVJnT67lYJgwKGBtvZAurC6XX9ttpCZA,700
 qwak_sdk/commands/feature_store/trigger/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -311,7 +311,7 @@ qwak_sdk/exceptions/qwak_command_exception.py,sha256=R2gZ12PA-_32UcxbC19k6a0JJ1Q
 qwak_sdk/exceptions/qwak_deploy_new_build_failed.py,sha256=W7-csnigTM2vo7k-j4-MFTFRiPOqa2uEgw--LEnKaCc,129
 qwak_sdk/exceptions/qwak_resource_not_found.py,sha256=EmNKlezjQ_tl5agt1IJOkjPKTzR8mw_S7ys7CCjlPo0,48
 qwak_sdk/inner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-qwak_sdk/inner/file_registry.py,sha256=CViXsi9ipYpLuPfIPY0R4DftSGjY7BN-5mxjjkQcFIY,3164
+qwak_sdk/inner/file_registry.py,sha256=uRw0gM5j6OVYb0M5-4itExwk2bi1PP6NN-Yt8eMfk_k,3220
 qwak_sdk/inner/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 qwak_sdk/inner/tools/cli_tools.py,sha256=Kz3cOno2sNEO9foJDNhx8G8Os71FvZcoTTnDfWlwYcQ,6170
 qwak_sdk/inner/tools/config_handler.py,sha256=BtmyIztAjMxccepPfv5-aqdFp1DlZtoG-m0ZIgEA5GQ,874
@@ -326,7 +326,7 @@ qwak_sdk/tools/colors.py,sha256=7pui_GGjC4uZKYFsIyXaJjYsjLxJVHb4OrfTgr93hqo,287
 qwak_sdk/tools/files.py,sha256=AyKJTOy7NhvP3SrqwIw_lxYNCOy1CvLgMmSJpWZ0OKM,2257
 qwak_sdk/tools/log_handling.py,sha256=NIQ-5uCb2NHzB7KwIbqbR9EhmhL1bawXLD_oamB6Gok,5711
 qwak_sdk/tools/utils.py,sha256=SHmU4r_m2ABZyFYMC03P17GvltPbYbmB39hvalIZEtI,1168
-qwak_sdk-0.5.36.dist-info/entry_points.txt,sha256=vSl0ELYDyj640oMM57u0AjBP87wtLYxCcGOendhEx80,47
-qwak_sdk-0.5.36.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
-qwak_sdk-0.5.36.dist-info/METADATA,sha256=qUkeFBWszp6D8CthFJX147irT4TXUX5AxjGwz3oeZIs,1885
-qwak_sdk-0.5.36.dist-info/RECORD,,
+qwak_sdk-0.5.38.dist-info/entry_points.txt,sha256=vSl0ELYDyj640oMM57u0AjBP87wtLYxCcGOendhEx80,47
+qwak_sdk-0.5.38.dist-info/WHEEL,sha256=vVCvjcmxuUltf8cYhJ0sJMRDLr1XsPuxEId8YDzbyCY,88
+qwak_sdk-0.5.38.dist-info/METADATA,sha256=dk39NLWIDLRL0E9MHljpzI-RwfxGU-GgeadiPeIT7Zo,1884
+qwak_sdk-0.5.38.dist-info/RECORD,,