qwak-sdk 0.5.37__tar.gz → 0.5.39__tar.gz

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qwak-sdk
-Version: 0.5.37
+Version: 0.5.39
 Summary: Qwak SDK and CLI for qwak models
 License: Apache-2.0
 Keywords: mlops,ml,deployment,serving,model
@@ -28,7 +28,7 @@ Requires-Dist: pandas (<1.4) ; (python_full_version >= "3.7.1" and python_versio
 Requires-Dist: pandas (>=1.4.3,<2.0.0) ; (python_version >= "3.8" and python_version < "3.10") and (extra == "batch" or extra == "feedback")
 Requires-Dist: pyarrow (>=6.0.0,<11.0.0) ; extra == "batch"
 Requires-Dist: python-json-logger (>=2.0.2)
-Requires-Dist: qwak-core (==0.1.102)
+Requires-Dist: qwak-core (==0.2.8)
 Requires-Dist: qwak-inference (==0.1.8)
 Requires-Dist: tabulate (>=0.8.0)
 Requires-Dist: yaspin (>=2.0.0)

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qwak-sdk"
-version = "0.5.37"
+version = "0.5.39"
 description = "Qwak SDK and CLI for qwak models"
 authors = ["Qwak <info@qwak.com>"]
 readme = "README.md"
@@ -27,7 +27,7 @@ qwak = "qwak_sdk.main:qwak_cli"
 
 [tool.poetry.dependencies]
 python = ">=3.7.1,<3.10"
-qwak-core = "==0.1.102"
+qwak-core = "==0.2.8"
 qwak-inference = "==0.1.8"
 croniter = "==1.4.1"
 tabulate = ">=0.8.0"

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/qwak_sdk/__init__.py

@@ -1,6 +1,6 @@
 # fmt: off
 __author__ = '''Qwak.ai'''
-__version__ = '0.5.37'
+__version__ = '0.5.39'
 
 from qwak.inner import wire_dependencies
 

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/qwak_sdk/commands/feature_store/register/_logic.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, cast
 
 from _qwak_proto.qwak.feature_store.entities.entity_pb2 import (
     EntityDefinition as ProtoEntityDefinition,
@@ -10,7 +10,6 @@ from _qwak_proto.qwak.feature_store.features.feature_set_service_pb2 import (
     GetFeatureSetByNameResponse as ProtoGetFeatureSetByNameResponse,
 )
 from _qwak_proto.qwak.features_operator.v3.features_operator_pb2 import (
-    Outputs,
     SparkColumnDescription,
     ValidationSuccessResponse,
 )
@@ -20,7 +19,10 @@ from qwak.exceptions import QwakException
 from qwak.feature_store.data_sources.base import BaseSource
 from qwak.feature_store.entities.entity import Entity
 from qwak.feature_store.feature_sets.base_feature_set import BaseFeatureSet
-from qwak.feature_store.feature_sets.batch import BatchFeatureSet
+from qwak.feature_store.validations.validation_response import (
+    SuccessValidationResponse,
+    ValidationResponse,
+)
 from tabulate import tabulate
 
 from qwak_sdk.inner.file_registry import extract_class_objects
@@ -31,7 +33,9 @@ DELIMITER = "----------------------------------------"
 
 
 def _register_entities(
-    qwak_python_files: List[str], registry: FeatureRegistryClient, force: bool
+    qwak_python_files: List[Tuple[str, str]],
+    registry: FeatureRegistryClient,
+    force: bool,
 ):
     """
     Register Feature Store Entity Objects
@@ -68,7 +72,7 @@ def _register_entities(
 
 
 def _register_data_sources(
-    qwak_python_files: List[str],
+    qwak_python_files: List[Tuple[str, str]],
     registry: FeatureRegistryClient,
     operator_client: FeaturesOperatorClient,
     force: bool,
@@ -139,7 +143,7 @@ def _handle_data_source_validation(
 
     response = getattr(result, result.WhichOneof("type"))
     if isinstance(response, ValidationSuccessResponse):
-        print_validation_outputs(response)
+        print_validation_outputs(response.outputs.stdout, response.outputs.stderr)
         print("✅ Validation completed successfully, got data source columns:")
 
         table = [
@@ -147,13 +151,12 @@ def _handle_data_source_validation(
         ]
         print(tabulate(table, headers=["column name", "type"]))
     else:
-        raise QwakException(f"🧨 Validation failed:\n{response}")
+        raise QwakException(f"🧨 Validation failed: \n{response}")
 
 
 def _register_features_sets(
-    qwak_python_files: List[str],
+    qwak_python_files: List[Tuple[str, str]],
     registry: FeatureRegistryClient,
-    operator_client: FeaturesOperatorClient,
     force: bool,
     git_commit: str,
     no_validation: bool,
@@ -165,7 +168,6 @@ def _register_features_sets(
     Args:
         qwak_python_files: a list of python files containing qwak package imports
         registry: FeatureRegistryClient
-        operator_client: features operator grpc client
         force: boolean determining if to force register all encountered Feature Set objects
         git_commit: the git commit of the parent folder
         no_validation: whether to validate entities
@@ -174,43 +176,48 @@ def _register_features_sets(
     with qwak_spinner(
         begin_text="Finding Feature Sets to register", print_callback=print
     ):
-        qwak_feature_sets = extract_class_objects(qwak_python_files, BatchFeatureSet)
+        qwak_feature_sets = extract_class_objects(qwak_python_files, BaseFeatureSet)
 
     print(f"👀 Found {len(qwak_feature_sets)} Feature Set(s)")
 
-    for feature_set, source_file_path in qwak_feature_sets:
+    for featureset, source_file_path in qwak_feature_sets:
+        featureset = cast(BaseFeatureSet, featureset)
         existing_feature_set: ProtoGetFeatureSetByNameResponse = (
-            registry.get_feature_set_by_name(feature_set.name)
+            registry.get_feature_set_by_name(featureset.name)
         )
 
         registration: bool = False
         if existing_feature_set:
             # Provide entity information of registered feature set before any other operation
-            if feature_set.key:
-                feature_set.entity = (
+            if featureset.key:
+                featureset.entity = (
                     existing_feature_set.feature_set.feature_set_definition.feature_set_spec.entity.entity_spec.name
                 )
 
             registration = ask_yesno(
-                f"Update existing feature set '{feature_set.name}' from source file '{source_file_path}'?",  # nosec B608
+                f"Update existing feature set '{featureset.name}' from source file '{source_file_path}'?",  # nosec B608
                 force,
             )
         else:
             registration = ask_yesno(
-                f"Create new feature set '{feature_set.name}' from source file '{source_file_path}'?",
+                f"Create new feature set '{featureset.name}' from source file '{source_file_path}'?",
                 force,
             )
 
         if registration:
-            spark_columns_features = _batch_v1_feature_set_validation(
-                feature_set=feature_set,
+            features: Optional[List[ProtoFeature]] = []
+            artifact_url: Optional[str] = None
+
+            features, artifact_url = _validate_featureset(
+                featureset=featureset,
                 no_validation=no_validation,
-                operator=operator_client,
-                registry=registry,
             )
 
-            proto_feature_set = feature_set._to_proto(
-                git_commit, spark_columns_features, registry
+            proto_feature_set, _ = featureset._to_proto(
+                git_commit=git_commit,
+                features=features,
+                feature_registry=registry,
+                artifact_url=artifact_url,
             )
 
             if existing_feature_set:
@@ -220,11 +227,12 @@ def _register_features_sets(
                 )
             else:
                 _register_new_feature_set(
-                    new_feature_set=feature_set,
+                    new_feature_set=featureset,
                     entity_key=proto_feature_set.entity,
                     registry=registry,
                     git_commit=git_commit,
-                    spark_columns_features=spark_columns_features,
+                    features=features,
+                    artifact_url=artifact_url,
                 )
 
 
@@ -233,7 +241,8 @@ def _register_new_feature_set(
     entity_key: ProtoEntityDefinition,
     registry: FeatureRegistryClient,
     git_commit: str,
-    spark_columns_features: Optional[List[ProtoFeature]],
+    features: Optional[List[ProtoFeature]],
+    artifact_url: Optional[str] = None,
 ):
     # Create entity for the defined key and set it as the fs's entity
     if new_feature_set.key:
@@ -248,8 +257,8 @@ def _register_new_feature_set(
         new_feature_set.entity = new_entity.name
 
     # this is done to retrieve the entities metadata
-    proto_feature_set = new_feature_set._to_proto(
-        git_commit, spark_columns_features, registry
+    proto_feature_set, _ = new_feature_set._to_proto(
+        git_commit, features, registry, artifact_url=artifact_url
     )
 
     try:
@@ -264,32 +273,26 @@ def _register_new_feature_set(
         raise e1
 
 
-def _handle_batch_v1_feature_set_validation(
-    operator_client: FeaturesOperatorClient,
-    registry_client: FeatureRegistryClient,
-    feature_set: BatchFeatureSet,
-) -> List[SparkColumnDescription]:
-    print(f"Validating '{feature_set.name}' feature set")
+def _handle_featureset_validation(
+    featureset: BaseFeatureSet,
+) -> Tuple[List[ProtoFeature], Optional[str]]:
+    print(f"Validating '{featureset.name}' feature set")
     with qwak_spinner(begin_text="", print_callback=print):
-        feature_set_spec = feature_set._to_proto(
-            feature_registry=registry_client, features=None, git_commit=None
-        )
-        result = operator_client.validate_featureset_blocking(
-            feature_set_spec, resource_path=None, num_samples=10
-        )
+        from qwak.feature_store.validations.validator import FeaturesOperatorValidator
 
-    response = getattr(result, result.WhichOneof("type"))
-    if isinstance(response, ValidationSuccessResponse):
-        print_validation_outputs(response)
+        v = FeaturesOperatorValidator()
+        response: ValidationResponse
+        artifact_url: Optional[str]
+
+        response, artifact_url = v.validate_featureset(featureset=featureset)
+    if isinstance(response, SuccessValidationResponse):
+        print_validation_outputs(response.stdout, response.stderr)
         print("✅ Validation completed successfully, got data source columns:")
-        table = [
-            (x.column_name, x.spark_type) for x in response.spark_column_description
-        ]
+        table = [(x.feature_name, x.feature_type) for x in response.features]
         print(tabulate(table, headers=["column name", "type"]))
-        return response.spark_column_description
-
+        return response.features, artifact_url
     else:
-        raise QwakException(f"🧨 Validation failed:\n{response}")
+        raise QwakException(f"🧨 Validation failed: \n{response}")
 
 
 def _get_features_from_spark_columns_description(
@@ -315,55 +318,47 @@ def _get_features_from_spark_columns_description(
         return None
 
 
-def _batch_v1_feature_set_validation(
-    feature_set: BatchFeatureSet,
+def _validate_featureset(
+    featureset: BaseFeatureSet,
     no_validation: bool,
-    operator: FeaturesOperatorClient,
-    registry: FeatureRegistryClient,
-) -> Optional[List[ProtoFeature]]:
+) -> Tuple[List[ProtoFeature], Optional[str]]:
     """
     Validates featureset transformation
     Args:
-        feature_set: BatchFeatureSet featureset
+        featureset: BaseFeatureSet featureset
         no_validation: skip validation
         operator: Operator client
         registry: Registry client
     Returns:
         Optional list of features returned from validation
     """
-    spark_columns_features: Optional[List[ProtoFeature]] = []
-
+    features: List[ProtoFeature] = []
+    artifact_url: Optional[str] = None
     if not no_validation:
         try:
-            spark_columns_description = _handle_batch_v1_feature_set_validation(
-                operator_client=operator,
-                registry_client=registry,
-                feature_set=feature_set,
-            )
-            spark_columns_features = _get_features_from_spark_columns_description(
-                spark_columns_description
+            features, artifact_url = _handle_featureset_validation(
+                featureset=featureset
             )
         except Exception as e:
             print(str(e))
             print("Not continuing to registration due to failure in validation")
             exit(1)
     else:
-        print(f"Skipping validation for '{feature_set.name}' feature set")
-    return spark_columns_features
+        print(f"Skipping validation for '{featureset.name}' feature set")
+    return features, artifact_url
 
 
-def print_validation_outputs(response: ValidationSuccessResponse) -> bool:
+def print_validation_outputs(stdout: str, stderr: str) -> bool:
     did_print = False
-    outputs = response.outputs
 
-    if isinstance(outputs, Outputs) and (outputs.stdout or outputs.stderr):
+    if stdout or stderr:
         message = "Validation outputs: "
 
-        if outputs.stdout:
-            message += f"stdout: {outputs.stdout}\n "
+        if stdout:
+            message += f"stdout: {stdout}\n "
 
-        if outputs.stderr:
-            message += f"stderr: {outputs.stderr}"
+        if stderr:
+            message += f"stderr: {stderr}"
 
         print(message)
         did_print = True

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/qwak_sdk/commands/feature_store/register/ui.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+from typing import List, Tuple
 
 import click
 from qwak.clients.feature_store import FeatureRegistryClient
@@ -59,6 +60,8 @@ def register_fs_objects(
     ignore_validation_errors: bool,
     **kwargs,
 ):
+    qwak_python_files: List[Tuple[str, str]]
+
     if not path:
         path = Path.cwd()
     else:
@@ -97,7 +100,6 @@ def register_fs_objects(
     _register_features_sets(
         qwak_python_files,
         registry_client,
-        operator_client,
         force,
         git_commit,
         no_validation,

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/qwak_sdk/inner/file_registry.py

@@ -3,11 +3,12 @@ import os
 import sys
 from contextlib import contextmanager
 from pathlib import Path
+from typing import List, Tuple
 
 from yaspin.core import Yaspin
 
 
-def list_qwak_python_files(path: Path, sp: Yaspin):
+def list_qwak_python_files(path: Path, sp: Yaspin) -> List[Tuple[str, str]]:
     """
     Helper function which finds python files with qwak imports in a given module
 

{qwak_sdk-0.5.37 → qwak_sdk-0.5.39}/setup.py

@@ -137,7 +137,7 @@ install_requires = \
  'croniter==1.4.1',
  'gitpython>=2.1.0',
  'python-json-logger>=2.0.2',
- 'qwak-core==0.1.102',
+ 'qwak-core==0.2.8',
  'qwak-inference==0.1.8',
  'tabulate>=0.8.0',
  'yaspin>=2.0.0']
@@ -161,7 +161,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'qwak-sdk',
-    'version': '0.5.37',
+    'version': '0.5.39',
     'description': 'Qwak SDK and CLI for qwak models',
     'long_description': '# Qwak SDK\n\nQwak is an end-to-end production ML platform designed to allow data scientists to build, deploy, and monitor their models in production with minimal engineering friction.\n',
     'author': 'Qwak',