qwak-core 0.1.37__py3-none-any.whl → 0.5.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of qwak-core might be problematic. Click here for more details.
- _qwak_proto/jfml/hosting_gateway/v1/build_upload_url_pb2.py +29 -0
- _qwak_proto/jfml/hosting_gateway/v1/build_upload_url_pb2.pyi +70 -0
- _qwak_proto/jfml/hosting_gateway/v1/build_upload_url_pb2_grpc.py +4 -0
- _qwak_proto/jfml/hosting_gateway/v1/hosting_gateway_service_pb2.py +32 -0
- _qwak_proto/jfml/hosting_gateway/v1/hosting_gateway_service_pb2.pyi +48 -0
- _qwak_proto/jfml/hosting_gateway/v1/hosting_gateway_service_pb2_grpc.py +67 -0
- _qwak_proto/qwak/administration/account/v1/account_pb2.py +29 -60
- _qwak_proto/qwak/administration/account/v1/account_pb2.pyi +120 -2
- _qwak_proto/qwak/administration/account/v1/account_service_pb2.py +67 -0
- _qwak_proto/qwak/administration/account/v1/account_service_pb2.pyi +345 -0
- _qwak_proto/qwak/administration/account/v1/account_service_pb2_grpc.py +338 -0
- _qwak_proto/qwak/administration/account/v1/jfrog_tenant_details_pb2.py +49 -0
- _qwak_proto/qwak/administration/account/v1/jfrog_tenant_details_pb2.pyi +209 -0
- _qwak_proto/qwak/administration/account/v1/jfrog_tenant_details_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/account/v1/personalization_pb2.py +8 -24
- _qwak_proto/qwak/administration/account/v1/preferences_pb2.py +6 -14
- _qwak_proto/qwak/administration/account/v1/terms_pb2.py +35 -0
- _qwak_proto/qwak/administration/account/v1/terms_pb2.pyi +94 -0
- _qwak_proto/qwak/administration/account/v1/terms_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/authenticated_user/v1/authenticated_user_service_pb2.py +18 -67
- _qwak_proto/qwak/administration/authenticated_user/v1/credentials_pb2.py +8 -24
- _qwak_proto/qwak/administration/authenticated_user/v1/credentials_pb2.pyi +1 -1
- _qwak_proto/qwak/administration/authenticated_user/v1/details_pb2.py +14 -54
- _qwak_proto/qwak/administration/cluster/v2/cluster_pb2.py +62 -0
- _qwak_proto/qwak/administration/cluster/v2/cluster_pb2.pyi +354 -0
- _qwak_proto/qwak/administration/cluster/v2/cluster_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/container_registry_config_pb2.py +32 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/container_registry_config_pb2.pyi +65 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/container_registry_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/auth_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/auth_pb2.pyi +36 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/auth_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/secret_pb2.py +29 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/secret_pb2.pyi +49 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/creds/secret_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/data_catalog_config_pb2.py +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/data_catalog_config_pb2.pyi +89 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/data_catalog_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/databricks/auth_pb2.py +34 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/databricks/auth_pb2.pyi +97 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/databricks/auth_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/elasticsearch_config_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/elasticsearch_config_pb2.pyi +39 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/elasticsearch_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/kafka_config_pb2.py +34 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/kafka_config_pb2.pyi +79 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/kafka_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/prometheus_config_pb2.py +29 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/prometheus_config_pb2.pyi +46 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/prometheus_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/redis_config_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/redis_config_pb2.pyi +39 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/redis_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/victoriametrics_config_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/victoriametrics_config_pb2.pyi +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/external/victoriametrics_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/feature_store_config_pb2.py +32 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/feature_store_config_pb2.pyi +66 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/feature_store_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/aws/auth_pb2.py +42 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/aws/auth_pb2.pyi +159 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/aws/auth_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/azure/auth_pb2.py +39 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/azure/auth_pb2.pyi +126 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting/azure/auth_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting_config_pb2.py +40 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting_config_pb2.pyi +146 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/hosting_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/logs_storage_config_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/logs_storage_config_pb2.pyi +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/logs_storage_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/model_analytics_storage_config_pb2.py +28 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/model_analytics_storage_config_pb2.pyi +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/model_analytics_storage_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/network_config_pb2.py +36 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/network_config_pb2.pyi +107 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/network_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/object_storage_config_pb2.py +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/object_storage_config_pb2.pyi +93 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/object_storage_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/observability_config_pb2.py +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/observability_config_pb2.pyi +79 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/observability_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/runtime_config_pb2.py +36 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/runtime_config_pb2.pyi +72 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/runtime_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/sql_engine_config_pb2.py +33 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/sql_engine_config_pb2.pyi +83 -0
- _qwak_proto/qwak/administration/runtime_configuration/v0/sql_engine_config_pb2_grpc.py +4 -0
- _qwak_proto/qwak/administration/v0/authentication/authentication_service_pb2.py +14 -47
- _qwak_proto/qwak/administration/v0/environments/configuration_pb2.py +23 -55
- _qwak_proto/qwak/administration/v0/environments/configuration_pb2.pyi +103 -7
- _qwak_proto/qwak/administration/v0/environments/environment_pb2.py +15 -33
- _qwak_proto/qwak/administration/v0/environments/environment_pb2.pyi +50 -2
- _qwak_proto/qwak/administration/v0/environments/environment_service_pb2.py +36 -158
- _qwak_proto/qwak/administration/v0/environments/personalization_pb2.py +8 -24
- _qwak_proto/qwak/administration/v0/users/user_pb2.py +18 -71
- _qwak_proto/qwak/admiral/secret/v0/secret_pb2.py +53 -0
- _qwak_proto/qwak/admiral/secret/v0/secret_pb2.pyi +256 -0
- _qwak_proto/qwak/admiral/secret/v0/secret_pb2_grpc.py +4 -0
- _qwak_proto/qwak/admiral/secret/v0/secret_service_pb2.py +40 -0
- _qwak_proto/qwak/admiral/secret/v0/secret_service_pb2.pyi +93 -0
- _qwak_proto/qwak/admiral/secret/v0/secret_service_pb2_grpc.py +135 -0
- _qwak_proto/qwak/admiral/secret/v0/system_secret_service_pb2.py +40 -0
- _qwak_proto/qwak/admiral/secret/v0/system_secret_service_pb2.pyi +101 -0
- _qwak_proto/qwak/admiral/secret/v0/system_secret_service_pb2_grpc.py +135 -0
- _qwak_proto/qwak/admiral/user_application_instance/v0/user_application_instance_pb2.py +39 -105
- _qwak_proto/qwak/admiral/user_application_instance/v0/user_application_instance_pb2.pyi +131 -3
- _qwak_proto/qwak/admiral/user_application_instance/v0/user_application_instance_service_pb2.py +22 -87
- _qwak_proto/qwak/analytics/analytics_pb2.py +28 -123
- _qwak_proto/qwak/analytics/analytics_pb2.pyi +5 -1
- _qwak_proto/qwak/analytics/analytics_service_pb2.py +48 -149
- _qwak_proto/qwak/analytics/analytics_service_pb2.pyi +87 -0
- _qwak_proto/qwak/analytics/analytics_service_pb2_grpc.py +101 -0
- _qwak_proto/qwak/artifactory_settings/artifactory_settings_pb2.py +29 -0
- _qwak_proto/qwak/artifactory_settings/artifactory_settings_pb2.pyi +57 -0
- _qwak_proto/qwak/artifactory_settings/artifactory_settings_pb2_grpc.py +4 -0
- _qwak_proto/qwak/audience/v1/audience_api_pb2.py +30 -127
- _qwak_proto/qwak/audience/v1/audience_pb2.py +27 -114
- _qwak_proto/qwak/audience/v1/audience_pb2.pyi +5 -1
- _qwak_proto/qwak/auto_scaling/v1/auto_scaling_pb2.py +18 -79
- _qwak_proto/qwak/auto_scaling/v1/auto_scaling_service_pb2.py +10 -27
- _qwak_proto/qwak/automation/v1/action_pb2.py +42 -164
- _qwak_proto/qwak/automation/v1/action_pb2.pyi +82 -7
- _qwak_proto/qwak/automation/v1/auto_scaling_pb2.py +19 -77
- _qwak_proto/qwak/automation/v1/auto_scaling_pb2.pyi +6 -0
- _qwak_proto/qwak/automation/v1/automation_execution_pb2.py +12 -43
- _qwak_proto/qwak/automation/v1/automation_management_service_pb2.py +56 -258
- _qwak_proto/qwak/automation/v1/automation_pb2.py +11 -35
- _qwak_proto/qwak/automation/v1/automation_pb2.pyi +8 -1
- _qwak_proto/qwak/automation/v1/common_pb2.py +11 -33
- _qwak_proto/qwak/automation/v1/common_pb2.pyi +6 -0
- _qwak_proto/qwak/automation/v1/notification_pb2.py +16 -68
- _qwak_proto/qwak/automation/v1/trigger_pb2.py +14 -54
- _qwak_proto/qwak/batch_job/v1/batch_job_events_pb2.py +27 -116
- _qwak_proto/qwak/batch_job/v1/batch_job_events_pb2.pyi +6 -1
- _qwak_proto/qwak/batch_job/v1/batch_job_resources_pb2.py +9 -26
- _qwak_proto/qwak/batch_job/v1/batch_job_resources_pb2.pyi +2 -0
- _qwak_proto/qwak/batch_job/v1/batch_job_service_pb2.py +121 -502
- _qwak_proto/qwak/batch_job/v1/batch_job_service_pb2.pyi +111 -4
- _qwak_proto/qwak/batch_job/v1/batch_job_service_pb2_grpc.py +67 -0
- _qwak_proto/qwak/build/v1/build_api_pb2.py +98 -278
- _qwak_proto/qwak/build/v1/build_api_pb2.pyi +220 -0
- _qwak_proto/qwak/build/v1/build_api_pb2_grpc.py +198 -0
- _qwak_proto/qwak/build/v1/build_pb2.py +115 -417
- _qwak_proto/qwak/build/v1/build_pb2.pyi +294 -5
- _qwak_proto/qwak/build_settings/build_settings_api_pb2.py +40 -0
- _qwak_proto/qwak/build_settings/build_settings_api_pb2.pyi +106 -0
- _qwak_proto/qwak/build_settings/build_settings_api_pb2_grpc.py +132 -0
- _qwak_proto/qwak/build_settings/build_settings_pb2.py +39 -0
- _qwak_proto/qwak/build_settings/build_settings_pb2.pyi +148 -0
- _qwak_proto/qwak/build_settings/build_settings_pb2_grpc.py +4 -0
- _qwak_proto/qwak/builds/build_pb2.py +46 -200
- _qwak_proto/qwak/builds/build_pb2.pyi +111 -5
- _qwak_proto/qwak/builds/build_url_pb2.py +27 -54
- _qwak_proto/qwak/builds/build_url_pb2.pyi +131 -2
- _qwak_proto/qwak/builds/build_values_pb2.py +82 -0
- _qwak_proto/qwak/builds/build_values_pb2.pyi +559 -0
- _qwak_proto/qwak/builds/build_values_pb2_grpc.py +4 -0
- _qwak_proto/qwak/builds/builds_orchestrator_service_pb2.py +93 -168
- _qwak_proto/qwak/builds/builds_orchestrator_service_pb2.pyi +342 -3
- _qwak_proto/qwak/builds/builds_orchestrator_service_pb2_grpc.py +166 -0
- _qwak_proto/qwak/builds/builds_pb2.py +87 -528
- _qwak_proto/qwak/builds/builds_pb2.pyi +5 -192
- _qwak_proto/qwak/builds/builds_pb2_grpc.py +0 -233
- _qwak_proto/qwak/builds/internal_builds_orchestrator_service_pb2.py +12 -37
- _qwak_proto/qwak/data_versioning/data_versioning_pb2.py +9 -15
- _qwak_proto/qwak/data_versioning/data_versioning_pb2.pyi +21 -0
- _qwak_proto/qwak/data_versioning/data_versioning_service_pb2.py +15 -48
- _qwak_proto/qwak/data_versioning/data_versioning_service_pb2.pyi +12 -1
- _qwak_proto/qwak/deployment/alert_pb2.py +25 -108
- _qwak_proto/qwak/deployment/alert_pb2.pyi +2 -0
- _qwak_proto/qwak/deployment/alert_service_pb2.py +30 -127
- _qwak_proto/qwak/deployment/deployment_messages_pb2.py +6 -14
- _qwak_proto/qwak/deployment/deployment_pb2.py +127 -578
- _qwak_proto/qwak/deployment/deployment_pb2.pyi +51 -8
- _qwak_proto/qwak/deployment/deployment_service_pb2.py +91 -400
- _qwak_proto/qwak/deployment/deployment_service_pb2.pyi +30 -0
- _qwak_proto/qwak/deployment/deployment_service_pb2_grpc.py +34 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_pb2.py +72 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_pb2.pyi +489 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_pb2_grpc.py +4 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_service_pb2.py +44 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_service_pb2.pyi +144 -0
- _qwak_proto/qwak/ecosystem/jfrog/v0/token_service_pb2_grpc.py +169 -0
- _qwak_proto/qwak/ecosystem/v0/auth_pb2.py +28 -0
- _qwak_proto/qwak/ecosystem/v0/auth_pb2.pyi +37 -0
- _qwak_proto/qwak/ecosystem/v0/auth_pb2_grpc.py +4 -0
- _qwak_proto/qwak/ecosystem/v0/azure_credentials_pb2.py +37 -0
- _qwak_proto/qwak/ecosystem/v0/azure_credentials_pb2.pyi +117 -0
- _qwak_proto/qwak/ecosystem/v0/azure_credentials_pb2_grpc.py +4 -0
- _qwak_proto/qwak/ecosystem/v0/credentials_pb2.py +13 -26
- _qwak_proto/qwak/ecosystem/v0/credentials_pb2.pyi +33 -3
- _qwak_proto/qwak/ecosystem/v0/ecosystem_pb2.py +34 -140
- _qwak_proto/qwak/ecosystem/v0/ecosystem_pb2.pyi +45 -7
- _qwak_proto/qwak/ecosystem/v0/ecosystem_runtime_service_pb2.py +54 -229
- _qwak_proto/qwak/ecosystem/v0/ecosystem_runtime_service_pb2.pyi +22 -3
- _qwak_proto/qwak/execution/v1/backfill_pb2.py +18 -74
- _qwak_proto/qwak/execution/v1/batch_pb2.py +10 -34
- _qwak_proto/qwak/execution/v1/deletion_pb2.py +33 -0
- _qwak_proto/qwak/execution/v1/deletion_pb2.pyi +67 -0
- _qwak_proto/qwak/execution/v1/deletion_pb2_grpc.py +4 -0
- _qwak_proto/qwak/execution/v1/execution_pb2.py +20 -60
- _qwak_proto/qwak/execution/v1/execution_pb2.pyi +61 -4
- _qwak_proto/qwak/execution/v1/execution_service_pb2.py +70 -108
- _qwak_proto/qwak/execution/v1/execution_service_pb2.pyi +306 -0
- _qwak_proto/qwak/execution/v1/execution_service_pb2_grpc.py +304 -2
- _qwak_proto/qwak/execution/v1/internal/deployment/platform_details_pb2.py +27 -0
- _qwak_proto/qwak/execution/v1/internal/deployment/platform_details_pb2.pyi +29 -0
- _qwak_proto/qwak/execution/v1/internal/deployment/platform_details_pb2_grpc.py +4 -0
- _qwak_proto/qwak/execution/v1/jobs/job_pb2.py +19 -67
- _qwak_proto/qwak/execution/v1/jobs/job_pb2.pyi +33 -2
- _qwak_proto/qwak/execution/v1/jobs/job_service_pb2.py +56 -88
- _qwak_proto/qwak/execution/v1/jobs/job_service_pb2.pyi +311 -0
- _qwak_proto/qwak/execution/v1/jobs/job_service_pb2_grpc.py +231 -0
- _qwak_proto/qwak/execution/v1/jobs/reports/report_pb2.py +21 -75
- _qwak_proto/qwak/execution/v1/jobs/reports/report_pb2.pyi +29 -2
- _qwak_proto/qwak/execution/v1/state/execution_state_pb2.py +6 -14
- _qwak_proto/qwak/execution/v1/state/execution_state_service_pb2.py +31 -108
- _qwak_proto/qwak/execution/v1/state/execution_state_service_pb2.pyi +30 -0
- _qwak_proto/qwak/execution/v1/state/execution_state_service_pb2_grpc.py +34 -0
- _qwak_proto/qwak/execution/v1/state/featureset_state_pb2.py +6 -14
- _qwak_proto/qwak/execution/v1/state/spark_execution_state_pb2.py +14 -68
- _qwak_proto/qwak/execution/v1/streaming_aggregation_pb2.py +42 -0
- _qwak_proto/qwak/execution/v1/streaming_aggregation_pb2.pyi +165 -0
- _qwak_proto/qwak/execution/v1/streaming_aggregation_pb2_grpc.py +4 -0
- _qwak_proto/qwak/execution/v1/streaming_pb2.py +33 -0
- _qwak_proto/qwak/execution/v1/streaming_pb2.pyi +79 -0
- _qwak_proto/qwak/execution/v1/streaming_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/entities/entity_pb2.py +16 -57
- _qwak_proto/qwak/feature_store/entities/entity_service_pb2.py +30 -127
- _qwak_proto/qwak/feature_store/features/aggregation_pb2.py +38 -175
- _qwak_proto/qwak/feature_store/features/deployment_pb2.py +36 -167
- _qwak_proto/qwak/feature_store/features/deployment_service_pb2.py +10 -27
- _qwak_proto/qwak/feature_store/features/execution_pb2.py +14 -56
- _qwak_proto/qwak/feature_store/features/feature_set_attribute_pb2.py +28 -0
- _qwak_proto/qwak/feature_store/features/feature_set_attribute_pb2.pyi +33 -0
- _qwak_proto/qwak/feature_store/features/feature_set_attribute_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/features/feature_set_pb2.py +49 -106
- _qwak_proto/qwak/feature_store/features/feature_set_pb2.pyi +154 -8
- _qwak_proto/qwak/feature_store/features/feature_set_service_pb2.py +101 -349
- _qwak_proto/qwak/feature_store/features/feature_set_service_pb2.pyi +160 -0
- _qwak_proto/qwak/feature_store/features/feature_set_service_pb2_grpc.py +170 -0
- _qwak_proto/qwak/feature_store/features/feature_set_state_pb2.py +38 -181
- _qwak_proto/qwak/feature_store/features/feature_set_state_service_pb2.py +26 -107
- _qwak_proto/qwak/feature_store/features/feature_set_types_pb2.py +84 -298
- _qwak_proto/qwak/feature_store/features/feature_set_types_pb2.pyi +123 -7
- _qwak_proto/qwak/feature_store/features/monitoring_pb2.py +10 -34
- _qwak_proto/qwak/feature_store/features/real_time_feature_extractor_pb2.py +32 -143
- _qwak_proto/qwak/feature_store/jobs/job_pb2.py +12 -46
- _qwak_proto/qwak/feature_store/jobs/job_service_pb2.py +30 -127
- _qwak_proto/qwak/feature_store/jobs/v1/job_pb2.py +13 -36
- _qwak_proto/qwak/feature_store/jobs/v1/job_pb2.pyi +19 -0
- _qwak_proto/qwak/feature_store/jobs/v1/job_service_pb2.py +44 -148
- _qwak_proto/qwak/feature_store/jobs/v1/job_service_pb2.pyi +64 -0
- _qwak_proto/qwak/feature_store/jobs/v1/job_service_pb2_grpc.py +67 -0
- _qwak_proto/qwak/feature_store/platform/platform_details_pb2.py +30 -0
- _qwak_proto/qwak/feature_store/platform/platform_details_pb2.pyi +54 -0
- _qwak_proto/qwak/feature_store/platform/platform_details_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/reports/report_pb2.py +14 -53
- _qwak_proto/qwak/feature_store/repository/common/platform_pb2.py +31 -0
- _qwak_proto/qwak/feature_store/repository/common/platform_pb2.pyi +70 -0
- _qwak_proto/qwak/feature_store/repository/common/platform_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/serving/management_pb2.py +36 -0
- _qwak_proto/qwak/feature_store/serving/management_pb2.pyi +72 -0
- _qwak_proto/qwak/feature_store/serving/management_pb2_grpc.py +101 -0
- _qwak_proto/qwak/feature_store/serving/metadata_pb2.py +13 -35
- _qwak_proto/qwak/feature_store/serving/metadata_pb2.pyi +27 -0
- _qwak_proto/qwak/feature_store/serving/serving_pb2.py +50 -222
- _qwak_proto/qwak/feature_store/serving/v1/value_pb2.py +8 -24
- _qwak_proto/qwak/feature_store/sinks/sink_pb2.py +36 -0
- _qwak_proto/qwak/feature_store/sinks/sink_pb2.pyi +137 -0
- _qwak_proto/qwak/feature_store/sinks/sink_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/sources/batch_pb2.py +80 -175
- _qwak_proto/qwak/feature_store/sources/batch_pb2.pyi +461 -13
- _qwak_proto/qwak/feature_store/sources/data_source_attribute_pb2.py +28 -0
- _qwak_proto/qwak/feature_store/sources/data_source_attribute_pb2.pyi +33 -0
- _qwak_proto/qwak/feature_store/sources/data_source_attribute_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/sources/data_source_pb2.py +21 -55
- _qwak_proto/qwak/feature_store/sources/data_source_pb2.pyi +39 -4
- _qwak_proto/qwak/feature_store/sources/data_source_service_pb2.py +41 -148
- _qwak_proto/qwak/feature_store/sources/data_source_service_pb2.pyi +25 -0
- _qwak_proto/qwak/feature_store/sources/data_source_service_pb2_grpc.py +34 -0
- _qwak_proto/qwak/feature_store/sources/streaming_pb2.py +34 -146
- _qwak_proto/qwak/feature_store/v1/common/jfrog_artifact/jfrog_artifact_pb2.py +27 -0
- _qwak_proto/qwak/feature_store/v1/common/jfrog_artifact/jfrog_artifact_pb2.pyi +30 -0
- _qwak_proto/qwak/feature_store/v1/common/jfrog_artifact/jfrog_artifact_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/v1/common/source_code/source_code_pb2.py +32 -0
- _qwak_proto/qwak/feature_store/v1/common/source_code/source_code_pb2.pyi +77 -0
- _qwak_proto/qwak/feature_store/v1/common/source_code/source_code_pb2_grpc.py +4 -0
- _qwak_proto/qwak/feature_store/v1/internal/data_source/data_source_service_pb2.py +39 -0
- _qwak_proto/qwak/feature_store/v1/internal/data_source/data_source_service_pb2.pyi +99 -0
- _qwak_proto/qwak/feature_store/v1/internal/data_source/data_source_service_pb2_grpc.py +99 -0
- _qwak_proto/qwak/feature_store/v1/internal/featureset/featureset_token_service_pb2.py +31 -0
- _qwak_proto/qwak/feature_store/v1/internal/featureset/featureset_token_service_pb2.pyi +46 -0
- _qwak_proto/qwak/feature_store/v1/internal/featureset/featureset_token_service_pb2_grpc.py +66 -0
- _qwak_proto/qwak/features_operator/v1/features_operator_pb2.py +14 -54
- _qwak_proto/qwak/features_operator/v1/features_operator_service_pb2.py +14 -47
- _qwak_proto/qwak/features_operator/v2/features_operator_pb2.py +16 -64
- _qwak_proto/qwak/features_operator/v2/features_operator_service_pb2.py +12 -37
- _qwak_proto/qwak/features_operator/v3/features_operator_async_service_pb2.py +26 -107
- _qwak_proto/qwak/features_operator/v3/features_operator_pb2.py +16 -64
- _qwak_proto/qwak/file_versioning/file_versioning_pb2.py +11 -24
- _qwak_proto/qwak/file_versioning/file_versioning_pb2.pyi +21 -0
- _qwak_proto/qwak/file_versioning/file_versioning_service_pb2.py +15 -48
- _qwak_proto/qwak/file_versioning/file_versioning_service_pb2.pyi +12 -1
- _qwak_proto/qwak/fitness_service/constructs_pb2.py +23 -93
- _qwak_proto/qwak/fitness_service/constructs_pb2.pyi +59 -1
- _qwak_proto/qwak/fitness_service/fitness_pb2.py +7 -34
- _qwak_proto/qwak/fitness_service/fitness_pb2.pyi +0 -77
- _qwak_proto/qwak/fitness_service/status_pb2.py +17 -107
- _qwak_proto/qwak/fitness_service/status_pb2.pyi +1 -117
- _qwak_proto/qwak/inference/feedback/feedback_pb2.py +28 -96
- _qwak_proto/qwak/instance_template/instance_template_pb2.py +13 -41
- _qwak_proto/qwak/instance_template/instance_template_pb2.pyi +13 -1
- _qwak_proto/qwak/instance_template/instance_template_service_pb2.py +14 -47
- _qwak_proto/qwak/integration/hugging_face_integration_pb2.py +37 -0
- _qwak_proto/qwak/integration/hugging_face_integration_pb2.pyi +106 -0
- _qwak_proto/qwak/integration/hugging_face_integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/integration/integration_pb2.py +55 -0
- _qwak_proto/qwak/integration/integration_pb2.pyi +357 -0
- _qwak_proto/qwak/integration/integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/integration/integration_service_pb2.py +56 -0
- _qwak_proto/qwak/integration/integration_service_pb2.pyi +234 -0
- _qwak_proto/qwak/integration/integration_service_pb2_grpc.py +271 -0
- _qwak_proto/qwak/integration/open_a_i_integration_pb2.py +37 -0
- _qwak_proto/qwak/integration/open_a_i_integration_pb2.pyi +109 -0
- _qwak_proto/qwak/integration/open_a_i_integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/integration/opsgenie_integration_pb2.py +43 -0
- _qwak_proto/qwak/integration/opsgenie_integration_pb2.pyi +168 -0
- _qwak_proto/qwak/integration/opsgenie_integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/integration/pagerduty_integration_pb2.py +47 -0
- _qwak_proto/qwak/integration/pagerduty_integration_pb2.pyi +195 -0
- _qwak_proto/qwak/integration/pagerduty_integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/integration/slack_app_integration_pb2.py +47 -0
- _qwak_proto/qwak/integration/slack_app_integration_pb2.pyi +198 -0
- _qwak_proto/qwak/integration/slack_app_integration_pb2_grpc.py +4 -0
- _qwak_proto/qwak/kube_deployment_captain/alert_pb2.py +22 -97
- _qwak_proto/qwak/kube_deployment_captain/alerting_pb2.py +12 -44
- _qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.py +76 -249
- _qwak_proto/qwak/kube_deployment_captain/batch_job_pb2.pyi +283 -6
- _qwak_proto/qwak/kube_deployment_captain/deployment_pb2.py +45 -161
- _qwak_proto/qwak/kube_deployment_captain/deployment_pb2.pyi +52 -1
- _qwak_proto/qwak/kube_deployment_captain/feature_set_deployment_pb2.py +33 -145
- _qwak_proto/qwak/kube_deployment_captain/feature_set_deployment_pb2.pyi +6 -2
- _qwak_proto/qwak/kube_deployment_captain/kube_deployment_captain_service_pb2.py +111 -510
- _qwak_proto/qwak/kube_deployment_captain/kube_deployment_captain_service_pb2.pyi +24 -1
- _qwak_proto/qwak/kube_deployment_captain/kube_deployment_captain_service_pb2_grpc.py +34 -0
- _qwak_proto/qwak/kube_deployment_captain/traffic_mapping_pb2.py +8 -24
- _qwak_proto/qwak/logging/log_filter_pb2.py +12 -44
- _qwak_proto/qwak/logging/log_line_pb2.py +8 -25
- _qwak_proto/qwak/logging/log_reader_service_pb2.py +10 -27
- _qwak_proto/qwak/logging/log_source_pb2.py +30 -134
- _qwak_proto/qwak/model_descriptor/open_ai_descriptor_pb2.py +40 -0
- _qwak_proto/qwak/model_descriptor/open_ai_descriptor_pb2.pyi +215 -0
- _qwak_proto/qwak/model_descriptor/open_ai_descriptor_pb2_grpc.py +4 -0
- _qwak_proto/qwak/model_group/model_group_pb2.py +47 -0
- _qwak_proto/qwak/model_group/model_group_pb2.pyi +192 -0
- _qwak_proto/qwak/model_group/model_group_pb2_grpc.py +169 -0
- _qwak_proto/qwak/model_group/model_group_repository_details_pb2.py +43 -0
- _qwak_proto/qwak/model_group/model_group_repository_details_pb2.pyi +172 -0
- _qwak_proto/qwak/model_group/model_group_repository_details_pb2_grpc.py +4 -0
- _qwak_proto/qwak/models/models_pb2.py +101 -356
- _qwak_proto/qwak/models/models_pb2.pyi +221 -6
- _qwak_proto/qwak/models/models_pb2_grpc.py +68 -0
- _qwak_proto/qwak/models/models_query_pb2.py +37 -0
- _qwak_proto/qwak/models/models_query_pb2.pyi +157 -0
- _qwak_proto/qwak/models/models_query_pb2_grpc.py +4 -0
- _qwak_proto/qwak/monitoring/v0/alerting_channel_management_service_pb2.py +30 -127
- _qwak_proto/qwak/monitoring/v0/alerting_channel_pb2.py +34 -152
- _qwak_proto/qwak/monitoring/v0/alerting_channel_sync_service_pb2.py +14 -47
- _qwak_proto/qwak/offline/serving/v1/feature_values_pb2.py +27 -0
- _qwak_proto/qwak/offline/serving/v1/feature_values_pb2.pyi +37 -0
- _qwak_proto/qwak/offline/serving/v1/feature_values_pb2_grpc.py +4 -0
- _qwak_proto/qwak/offline/serving/v1/offline_serving_async_service_pb2.py +55 -0
- _qwak_proto/qwak/offline/serving/v1/offline_serving_async_service_pb2.pyi +279 -0
- _qwak_proto/qwak/offline/serving/v1/offline_serving_async_service_pb2_grpc.py +165 -0
- _qwak_proto/qwak/offline/serving/v1/options_pb2.py +27 -0
- _qwak_proto/qwak/offline/serving/v1/options_pb2.pyi +30 -0
- _qwak_proto/qwak/offline/serving/v1/options_pb2_grpc.py +4 -0
- _qwak_proto/qwak/offline/serving/v1/population_pb2.py +35 -0
- _qwak_proto/qwak/offline/serving/v1/population_pb2.pyi +99 -0
- _qwak_proto/qwak/offline/serving/v1/population_pb2_grpc.py +4 -0
- _qwak_proto/qwak/projects/jfrog_project_spec_pb2.py +27 -0
- _qwak_proto/qwak/projects/jfrog_project_spec_pb2.pyi +32 -0
- _qwak_proto/qwak/projects/jfrog_project_spec_pb2_grpc.py +4 -0
- _qwak_proto/qwak/projects/projects_pb2.py +51 -142
- _qwak_proto/qwak/projects/projects_pb2.pyi +149 -3
- _qwak_proto/qwak/prompt/v1/prompt/prompt_manager_service_pb2.py +77 -0
- _qwak_proto/qwak/prompt/v1/prompt/prompt_manager_service_pb2.pyi +417 -0
- _qwak_proto/qwak/prompt/v1/prompt/prompt_manager_service_pb2_grpc.py +441 -0
- _qwak_proto/qwak/prompt/v1/prompt/prompt_pb2.py +69 -0
- _qwak_proto/qwak/prompt/v1/prompt/prompt_pb2.pyi +415 -0
- _qwak_proto/qwak/prompt/v1/prompt/prompt_pb2_grpc.py +4 -0
- _qwak_proto/qwak/secret_service/secret_service_pb2.py +18 -67
- _qwak_proto/qwak/secret_service/secret_service_pb2.pyi +1 -1
- _qwak_proto/qwak/self_service/account/v0/account_membership_pb2.py +30 -65
- _qwak_proto/qwak/self_service/account/v0/account_membership_pb2.pyi +99 -2
- _qwak_proto/qwak/self_service/account/v0/account_membership_service_pb2.py +51 -208
- _qwak_proto/qwak/self_service/account/v0/account_membership_service_pb2.pyi +28 -0
- _qwak_proto/qwak/self_service/account/v0/account_membership_service_pb2_grpc.py +34 -0
- _qwak_proto/qwak/self_service/account/v0/account_status_pb2.py +44 -0
- _qwak_proto/qwak/self_service/account/v0/account_status_pb2.pyi +172 -0
- _qwak_proto/qwak/self_service/account/v0/account_status_pb2_grpc.py +4 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_pb2.py +60 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_pb2.pyi +317 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_pb2_grpc.py +4 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_service_pb2.py +45 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_service_pb2.pyi +136 -0
- _qwak_proto/qwak/self_service/account/v0/managing_account_service_pb2_grpc.py +169 -0
- _qwak_proto/qwak/self_service/user/v1/api_key_pb2.py +6 -14
- _qwak_proto/qwak/self_service/user/v1/user_pb2.py +6 -14
- _qwak_proto/qwak/self_service/user/v1/user_service_pb2.py +30 -121
- _qwak_proto/qwak/service_discovery/service_discovery_location_pb2.py +33 -0
- _qwak_proto/qwak/service_discovery/service_discovery_location_pb2.pyi +73 -0
- _qwak_proto/qwak/service_discovery/service_discovery_location_pb2_grpc.py +4 -0
- _qwak_proto/qwak/service_discovery/service_discovery_location_service_pb2.py +32 -0
- _qwak_proto/qwak/service_discovery/service_discovery_location_service_pb2.pyi +41 -0
- _qwak_proto/qwak/service_discovery/service_discovery_location_service_pb2_grpc.py +231 -0
- _qwak_proto/qwak/traffic/v1/traffic_api_pb2.py +23 -88
- _qwak_proto/qwak/traffic/v1/traffic_api_pb2.pyi +13 -2
- _qwak_proto/qwak/traffic/v1/traffic_pb2.py +27 -114
- _qwak_proto/qwak/traffic/v1/traffic_pb2.pyi +5 -1
- _qwak_proto/qwak/user_application/common/v0/resources_pb2.py +31 -132
- _qwak_proto/qwak/user_application/common/v0/resources_pb2.pyi +54 -0
- _qwak_proto/qwak/user_application/v0/user_application_pb2.py +15 -39
- _qwak_proto/qwak/user_application/v0/user_application_pb2.pyi +16 -0
- _qwak_proto/qwak/vectors/v1/collection/collection_pb2.py +25 -104
- _qwak_proto/qwak/vectors/v1/collection/collection_pb2.pyi +17 -1
- _qwak_proto/qwak/vectors/v1/collection/collection_service_pb2.py +38 -167
- _qwak_proto/qwak/vectors/v1/collection/event/collection_event_pb2.py +7 -15
- _qwak_proto/qwak/vectors/v1/collection/event/collection_event_pb2.pyi +5 -1
- _qwak_proto/qwak/vectors/v1/filters_pb2.py +30 -134
- _qwak_proto/qwak/vectors/v1/vector_pb2.py +17 -45
- _qwak_proto/qwak/vectors/v1/vector_pb2.pyi +25 -2
- _qwak_proto/qwak/vectors/v1/vector_service_pb2.py +31 -108
- _qwak_proto/qwak/vectors/v1/vector_service_pb2.pyi +30 -7
- _qwak_proto/qwak/workspace/workspace_pb2.py +23 -108
- _qwak_proto/qwak/workspace/workspace_pb2.pyi +10 -2
- _qwak_proto/qwak/workspace/workspace_service_pb2.py +63 -248
- _qwak_proto/qwak/workspace/workspace_service_pb2.pyi +56 -0
- _qwak_proto/qwak/workspace/workspace_service_pb2_grpc.py +66 -0
- qwak/__init__.py +2 -3
- qwak/automations/automation_executions.py +14 -10
- qwak/automations/automations.py +33 -23
- qwak/automations/build_and_deploy_action.py +147 -28
- qwak/automations/common.py +6 -0
- qwak/clients/administration/eco_system/client.py +2 -2
- qwak/clients/administration/eco_system/eco_system_utils.py +15 -0
- qwak/clients/administration/environment/client.py +16 -0
- qwak/clients/analytics/client.py +28 -3
- qwak/clients/batch_job_management/client.py +94 -3
- qwak/clients/batch_job_management/executions_config.py +18 -0
- qwak/clients/build_orchestrator/build_model_request_getter.py +3 -0
- qwak/clients/build_orchestrator/client.py +155 -12
- qwak/clients/data_versioning/client.py +19 -1
- qwak/clients/data_versioning/data_tag_filter.py +32 -0
- qwak/clients/deployment/client.py +14 -0
- qwak/clients/feature_store/execution_management_client.py +28 -0
- qwak/clients/feature_store/management_client.py +203 -146
- qwak/clients/feature_store/offline_serving_client.py +251 -0
- qwak/clients/feature_store/operator_client.py +21 -4
- qwak/clients/file_versioning/client.py +19 -1
- qwak/clients/file_versioning/file_tag_filter.py +32 -0
- qwak/clients/instance_template/client.py +11 -10
- qwak/clients/integration_management/integration_manager_client.py +34 -0
- qwak/clients/integration_management/integration_utils.py +27 -0
- qwak/clients/integration_management/openai/openai_system_secret.py +56 -0
- qwak/clients/kube_deployment_captain/client.py +3 -1
- qwak/clients/location_discovery/__init__.py +1 -0
- qwak/clients/location_discovery/client.py +73 -0
- qwak/clients/logging_client/client.py +13 -18
- qwak/clients/model_management/client.py +7 -2
- qwak/clients/project/client.py +7 -2
- qwak/clients/prompt_manager/__init__.py +0 -0
- qwak/clients/prompt_manager/model_descriptor_mapper.py +183 -0
- qwak/clients/prompt_manager/prompt_manager_client.py +167 -0
- qwak/clients/prompt_manager/prompt_proto_mapper.py +236 -0
- qwak/clients/secret_service/client.py +8 -3
- qwak/clients/system_secret/__init__.py +0 -0
- qwak/clients/system_secret/system_secret_client.py +30 -0
- qwak/clients/vector_store/management_client.py +2 -0
- qwak/clients/vector_store/serving_client.py +64 -18
- qwak/clients/workspace_manager/client.py +15 -0
- qwak/exceptions/__init__.py +7 -0
- qwak/exceptions/qwak_decode_exception.py +7 -0
- qwak/exceptions/qwak_exception.py +2 -1
- qwak/exceptions/qwak_external_exception.py +11 -0
- qwak/exceptions/qwak_general_build_exception.py +13 -0
- qwak/exceptions/qwak_grpc_address_exception.py +9 -0
- qwak/exceptions/qwak_load_configuration_exception.py +2 -0
- qwak/exceptions/qwak_remote_build_failed.py +5 -0
- qwak/exceptions/qwak_suggestion_exception.py +27 -0
- qwak/feature_store/_common/artifact_utils.py +59 -0
- qwak/feature_store/_common/feature_set_utils.py +142 -19
- qwak/feature_store/_common/functions.py +0 -43
- qwak/feature_store/_common/packaging.py +262 -0
- qwak/feature_store/_common/source_code_spec.py +108 -0
- qwak/feature_store/_common/source_code_spec_factory.py +104 -0
- qwak/feature_store/_common/value.py +10 -0
- qwak/feature_store/data_sources/__init__.py +53 -17
- qwak/feature_store/data_sources/attributes.py +23 -0
- qwak/feature_store/data_sources/base.py +125 -0
- qwak/feature_store/data_sources/batch/__init__.py +0 -0
- qwak/feature_store/data_sources/batch/_batch.py +9 -0
- qwak/feature_store/data_sources/{batch_sources → batch}/_jdbc.py +2 -1
- qwak/feature_store/data_sources/batch/athena.py +260 -0
- qwak/feature_store/data_sources/{batch_sources → batch}/big_query.py +5 -5
- qwak/feature_store/data_sources/batch/clickhouse.py +60 -0
- qwak/feature_store/data_sources/{batch_sources → batch}/csv.py +9 -5
- qwak/feature_store/data_sources/{batch_sources → batch}/elastic_search.py +5 -5
- qwak/feature_store/data_sources/batch/filesystem/__init__.py +0 -0
- qwak/feature_store/data_sources/{batch_sources/filesystem_config.py → batch/filesystem/aws.py} +30 -38
- qwak/feature_store/data_sources/batch/filesystem/base_config.py +13 -0
- qwak/feature_store/data_sources/batch/filesystem/gcp.py +50 -0
- qwak/feature_store/data_sources/batch/filesystem/utils.py +51 -0
- qwak/feature_store/data_sources/{batch_sources → batch}/mongodb.py +6 -5
- qwak/feature_store/data_sources/{batch_sources → batch}/mysql.py +5 -7
- qwak/feature_store/data_sources/{batch_sources → batch}/parquet.py +10 -7
- qwak/feature_store/data_sources/{batch_sources → batch}/postgres.py +5 -7
- qwak/feature_store/data_sources/{batch_sources → batch}/redshift.py +6 -7
- qwak/feature_store/data_sources/{batch_sources → batch}/snowflake.py +39 -13
- qwak/feature_store/data_sources/batch/unity_catalog.py +107 -0
- qwak/feature_store/data_sources/{batch_sources → batch}/vertica.py +6 -5
- qwak/feature_store/data_sources/source_authentication.py +33 -0
- qwak/feature_store/data_sources/streaming/__init__.py +0 -0
- qwak/feature_store/data_sources/streaming/_streaming.py +9 -0
- qwak/feature_store/data_sources/streaming/kafka/__init__.py +25 -0
- qwak/feature_store/data_sources/streaming/kafka/authentication.py +120 -0
- qwak/feature_store/data_sources/streaming/kafka/deserialization.py +109 -0
- qwak/feature_store/data_sources/streaming/kafka/kafka.py +120 -0
- qwak/feature_store/data_sources/time_partition_columns.py +166 -0
- qwak/feature_store/entities/entity.py +17 -4
- qwak/feature_store/execution/backfill.py +8 -2
- qwak/feature_store/execution/streaming_backfill.py +48 -0
- qwak/feature_store/feature_sets/_utils/_featureset_utils.py +45 -0
- qwak/feature_store/feature_sets/backfill.py +13 -0
- qwak/feature_store/feature_sets/base_feature_set.py +148 -0
- qwak/feature_store/feature_sets/batch.py +142 -151
- qwak/feature_store/feature_sets/execution_spec.py +6 -0
- qwak/feature_store/feature_sets/metadata.py +51 -1
- qwak/feature_store/feature_sets/streaming.py +655 -0
- qwak/feature_store/feature_sets/streaming_backfill.py +223 -0
- qwak/feature_store/feature_sets/transformations/__init__.py +33 -0
- qwak/feature_store/feature_sets/transformations/aggregations/__init__.py +0 -0
- qwak/feature_store/feature_sets/transformations/aggregations/aggregations.py +425 -0
- qwak/feature_store/feature_sets/transformations/aggregations/windows.py +70 -0
- qwak/feature_store/feature_sets/transformations/functions/__init__.py +10 -0
- qwak/feature_store/feature_sets/transformations/functions/qwak_pandas.py +64 -0
- qwak/feature_store/feature_sets/transformations/functions/schema.py +57 -0
- qwak/feature_store/feature_sets/transformations/transformations.py +451 -0
- qwak/feature_store/feature_sets/transformations/validations/validations_util.py +78 -0
- qwak/feature_store/offline/__init__.py +2 -2
- qwak/feature_store/offline/_offline_serving_validations.py +31 -0
- qwak/feature_store/offline/client_v2.py +327 -0
- qwak/feature_store/offline/feature_set_features.py +25 -0
- qwak/feature_store/online/client.py +152 -46
- qwak/feature_store/online/endpoint_utils.py +63 -0
- qwak/feature_store/sinks/__init__.py +0 -0
- qwak/feature_store/sinks/base.py +18 -0
- qwak/feature_store/sinks/kafka.py +59 -0
- qwak/feature_store/sinks/streaming/__init__.py +0 -0
- qwak/feature_store/sinks/streaming/attachment.py +30 -0
- qwak/feature_store/sinks/streaming/factory.py +60 -0
- qwak/feature_store/validations/__init__.py +0 -0
- qwak/feature_store/validations/validation_decorators.py +70 -0
- qwak/feature_store/validations/validation_options.py +77 -0
- qwak/feature_store/validations/validation_response.py +119 -0
- qwak/feature_store/validations/validator.py +114 -0
- qwak/inner/build_config/__init__.py +0 -0
- qwak/inner/build_config/build_config_v1.py +311 -0
- qwak/inner/build_logic/__init__.py +0 -0
- qwak/inner/build_logic/build_loggers/__init__.py +0 -0
- qwak/inner/build_logic/build_loggers/trigger_build_logger.py +51 -0
- qwak/inner/build_logic/constants/__init__.py +0 -0
- qwak/inner/build_logic/constants/dependencies.py +5 -0
- qwak/inner/build_logic/constants/host_resource.py +4 -0
- qwak/inner/build_logic/constants/messages.py +3 -0
- qwak/inner/build_logic/constants/temp_dir.py +1 -0
- qwak/inner/build_logic/constants/upload_tag.py +7 -0
- qwak/inner/build_logic/dependency_manager_type.py +8 -0
- qwak/inner/build_logic/execute_build_pipeline.py +62 -0
- qwak/inner/build_logic/interface/__init__.py +0 -0
- qwak/inner/build_logic/interface/build_logger_interface.py +28 -0
- qwak/inner/build_logic/interface/build_phase.py +24 -0
- qwak/inner/build_logic/interface/context_interface.py +66 -0
- qwak/inner/build_logic/interface/phase_run_handler.py +71 -0
- qwak/inner/build_logic/interface/step_inteface.py +26 -0
- qwak/inner/build_logic/interface/time_source.py +23 -0
- qwak/inner/build_logic/phases/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_model_step.py +43 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/common.py +32 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/fetch_strategy_manager.py +62 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/folder/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/folder/folder_strategy.py +131 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/git/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/git/git_strategy.py +186 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/strategy.py +48 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/zip/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/fetch_strategy_manager/strategy/zip/zip_strategy.py +64 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/post_fetch_validation_step.py +140 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/pre_fetch_validation_step.py +266 -0
- qwak/inner/build_logic/phases/phase_010_fetch_model/set_version_step.py +33 -0
- qwak/inner/build_logic/phases/phase_020_remote_register_qwak_build/__init__.py +0 -0
- qwak/inner/build_logic/phases/phase_020_remote_register_qwak_build/cleanup_step.py +20 -0
- qwak/inner/build_logic/phases/phase_020_remote_register_qwak_build/start_remote_build_step.py +43 -0
- qwak/inner/build_logic/phases/phase_020_remote_register_qwak_build/upload_step.py +258 -0
- qwak/inner/build_logic/phases/phases_pipeline.py +44 -0
- qwak/inner/build_logic/run_handlers/__init__.py +0 -0
- qwak/inner/build_logic/run_handlers/programmatic_phase_run_handler.py +99 -0
- qwak/inner/build_logic/tools/__init__.py +0 -0
- qwak/inner/build_logic/tools/dependencies_tools.py +68 -0
- qwak/inner/build_logic/tools/files.py +237 -0
- qwak/inner/build_logic/tools/ignore_files.py +20 -0
- qwak/inner/build_logic/tools/text.py +9 -0
- qwak/inner/build_logic/trigger_build_context.py +10 -0
- qwak/inner/const.py +2 -4
- qwak/inner/di_configuration/__init__.py +1 -63
- qwak/inner/di_configuration/account.py +32 -15
- qwak/inner/di_configuration/config.yml +2 -0
- qwak/inner/di_configuration/containers.py +10 -2
- qwak/inner/di_configuration/dependency_wiring.py +98 -0
- qwak/inner/di_configuration/session.py +5 -0
- qwak/inner/instance_template/__init__.py +0 -0
- qwak/inner/instance_template/verify_template_id.py +48 -0
- qwak/inner/model_loggers_utils.py +10 -7
- qwak/inner/provider.py +6 -0
- qwak/inner/tool/auth.py +42 -7
- qwak/inner/tool/grpc/grpc_auth.py +27 -8
- qwak/inner/tool/grpc/grpc_tools.py +124 -4
- qwak/inner/tool/grpc/grpc_try_wrapping.py +139 -4
- qwak/inner/tool/protobuf_factory.py +45 -0
- qwak/inner/tool/run_config/__init__.py +2 -0
- qwak/inner/tool/run_config/base.py +32 -7
- qwak/inner/tool/run_config/utils.py +66 -1
- qwak/llmops/__init__.py +0 -0
- qwak/llmops/generation/__init__.py +0 -0
- qwak/llmops/generation/_steaming.py +77 -0
- qwak/llmops/generation/base.py +5 -0
- qwak/llmops/generation/chat/__init__.py +0 -0
- qwak/llmops/generation/chat/openai/LICENSE.txt +201 -0
- qwak/llmops/generation/chat/openai/types/__init__.py +0 -0
- qwak/llmops/generation/chat/openai/types/chat/__init__.py +0 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion.py +105 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_assistant_message_param.py +65 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_chunk.py +188 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_content_part_text_param.py +28 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_function_call_option_param.py +25 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_function_message_param.py +33 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_message.py +59 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_message_param.py +34 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_message_tool_call.py +46 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_message_tool_call_param.py +44 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_named_tool_choice_param.py +32 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_role.py +20 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_system_message_param.py +35 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_token_logprob.py +71 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_tool_choice_option_param.py +28 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_tool_message_param.py +31 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_tool_param.py +29 -0
- qwak/llmops/generation/chat/openai/types/chat/chat_completion_user_message_param.py +35 -0
- qwak/llmops/generation/chat/openai/types/chat/completion_create_params.py +279 -0
- qwak/llmops/generation/chat/openai/types/completion_choice.py +47 -0
- qwak/llmops/generation/chat/openai/types/completion_create_params.py +209 -0
- qwak/llmops/generation/chat/openai/types/completion_usage.py +30 -0
- qwak/llmops/generation/chat/openai/types/model.py +35 -0
- qwak/llmops/generation/chat/openai/types/shared/__init__.py +3 -0
- qwak/llmops/generation/chat/openai/types/shared/error_object.py +27 -0
- qwak/llmops/generation/chat/openai/types/shared/function_definition.py +49 -0
- qwak/llmops/generation/chat/openai/types/shared/function_parameters.py +20 -0
- qwak/llmops/generation/chat/openai/types/shared_params/__init__.py +2 -0
- qwak/llmops/generation/chat/openai/types/shared_params/function_definition.py +49 -0
- qwak/llmops/generation/chat/openai/types/shared_params/function_parameters.py +22 -0
- qwak/llmops/generation/streaming.py +26 -0
- qwak/llmops/model/__init__.py +0 -0
- qwak/llmops/model/descriptor.py +39 -0
- qwak/llmops/prompt/__init__.py +0 -0
- qwak/llmops/prompt/base.py +133 -0
- qwak/llmops/prompt/chat/__init__.py +0 -0
- qwak/llmops/prompt/chat/message.py +24 -0
- qwak/llmops/prompt/chat/template.py +113 -0
- qwak/llmops/prompt/chat/value.py +10 -0
- qwak/llmops/prompt/manager.py +134 -0
- qwak/llmops/prompt/template.py +24 -0
- qwak/llmops/prompt/value.py +14 -0
- qwak/llmops/provider/__init__.py +0 -0
- qwak/llmops/provider/chat.py +42 -0
- qwak/llmops/provider/openai/__init__.py +0 -0
- qwak/llmops/provider/openai/client.py +123 -0
- qwak/llmops/provider/openai/provider.py +89 -0
- qwak/model/_entity_extraction.py +40 -46
- qwak/model/adapters/input_adapters/multi_input_adapter.py +17 -1
- qwak/model/base.py +1 -1
- qwak/model/decorators/api.py +3 -1
- qwak/model/decorators/impl/api_implementation.py +1 -0
- qwak/model/fs_info_mapping_retriever.py +58 -0
- qwak/model/schema.py +1 -1
- qwak/model/schema_entities.py +1 -1
- qwak/model/tools/__init__.py +1 -1
- qwak/model/tools/adapters/input_adapters/dataframe_input.py +1 -1
- qwak/model/tools/adapters/output.py +1 -1
- qwak/model/tools/run_model_locally.py +15 -3
- qwak/model/utils/feature_utils.py +69 -0
- qwak/model_loggers/artifact_logger.py +46 -9
- qwak/model_loggers/data_logger.py +16 -2
- qwak/qwak_client/batch_jobs/execution.py +49 -0
- qwak/qwak_client/batch_jobs/task.py +43 -0
- qwak/qwak_client/build_api_helpers/__init__.py +0 -0
- qwak/qwak_client/build_api_helpers/build_api_steps.py +53 -0
- qwak/qwak_client/build_api_helpers/messages.py +6 -0
- qwak/qwak_client/build_api_helpers/trigger_build_api.py +65 -0
- qwak/qwak_client/client.py +276 -24
- qwak/qwak_client/data_versioning/__init__.py +0 -0
- qwak/qwak_client/data_versioning/data_tag.py +24 -0
- qwak/qwak_client/deployments/deployment.py +11 -9
- qwak/qwak_client/file_versioning/__init__.py +0 -0
- qwak/qwak_client/file_versioning/file_tag.py +24 -0
- qwak/qwak_client/projects/project.py +4 -2
- qwak/tools/logger/logger.py +1 -1
- qwak/utils/__init__.py +0 -0
- qwak/utils/datetime_utils.py +18 -0
- qwak/utils/dict_utils.py +12 -0
- qwak/vector_store/client.py +8 -0
- qwak/vector_store/collection.py +173 -73
- qwak/vector_store/inference_client.py +30 -2
- qwak/vector_store/rest_helpers.py +13 -21
- qwak/vector_store/utils/filter_utils.py +4 -17
- qwak/vector_store/utils/upsert_utils.py +217 -0
- {qwak_core-0.1.37.dist-info → qwak_core-0.5.19.dist-info}/METADATA +20 -15
- qwak_core-0.5.19.dist-info/RECORD +1100 -0
- {qwak_core-0.1.37.dist-info → qwak_core-0.5.19.dist-info}/WHEEL +1 -1
- qwak_services_mock/mocks/authentication_service.py +5 -4
- qwak_services_mock/mocks/batch_job_manager_service.py +53 -6
- qwak_services_mock/mocks/build_orchestrator_build_api.py +18 -1
- qwak_services_mock/mocks/build_orchestrator_build_settings_api.py +35 -0
- qwak_services_mock/mocks/build_orchestrator_service_api.py +4 -0
- qwak_services_mock/mocks/data_versioning_service.py +26 -2
- qwak_services_mock/mocks/deployment_management_service.py +46 -18
- qwak_services_mock/mocks/ecosystem_service_api.py +0 -1
- qwak_services_mock/mocks/execution_management_service.py +9 -1
- qwak_services_mock/mocks/feature_store_data_sources_manager_api.py +98 -2
- qwak_services_mock/mocks/feature_store_entities_manager_api.py +46 -3
- qwak_services_mock/mocks/feature_store_feature_set_manager_api.py +197 -21
- qwak_services_mock/mocks/features_online_serving_api.py +9 -0
- qwak_services_mock/mocks/features_operator_v3_service.py +20 -0
- qwak_services_mock/mocks/features_set_state_service_api.py +0 -1
- qwak_services_mock/mocks/file_versioning_service.py +29 -2
- qwak_services_mock/mocks/fs_offline_serving_service.py +53 -0
- qwak_services_mock/mocks/instance_template_management_service.py +35 -1
- qwak_services_mock/mocks/integration_management_service.py +72 -0
- qwak_services_mock/mocks/location_discovery_service_api.py +104 -0
- qwak_services_mock/mocks/model_management_service.py +11 -5
- qwak_services_mock/mocks/prompt_manager_service.py +244 -0
- qwak_services_mock/mocks/qwak_mocks.py +20 -2
- qwak_services_mock/mocks/system_secret_service.py +54 -0
- qwak_services_mock/mocks/vector_serving_api.py +44 -8
- qwak_services_mock/mocks/workspace_manager_service_mock.py +24 -8
- qwak_services_mock/services_mock.py +78 -12
- _qwak_proto/qwak/fitness_service/fitness_service_pb2.py +0 -110
- _qwak_proto/qwak/fitness_service/fitness_service_pb2.pyi +0 -121
- _qwak_proto/qwak/fitness_service/fitness_service_pb2_grpc.py +0 -169
- qwak/clients/build_management/__init__.py +0 -1
- qwak/clients/build_management/client.py +0 -114
- qwak/feature_store/_common/featureset_asterisk_handler.py +0 -115
- qwak/feature_store/data_sources/batch_sources/__init__.py +0 -6
- qwak/feature_store/data_sources/batch_sources/_batch.py +0 -86
- qwak/feature_store/feature_sets/transformations.py +0 -48
- qwak/feature_store/offline/_query_engine.py +0 -32
- qwak/feature_store/offline/athena/athena_query_engine.py +0 -153
- qwak/feature_store/offline/client.py +0 -699
- qwak/feature_store/validation_options.py +0 -69
- qwak_core-0.1.37.dist-info/RECORD +0 -682
- qwak_services_mock/mocks/build_management.py +0 -100
- {_qwak_proto → qwak/clients/integration_management}/__init__.py +0 -0
- /qwak/{feature_store/offline/athena → clients/integration_management/openai}/__init__.py +0 -0
|
@@ -1,699 +0,0 @@
|
|
|
1
|
-
from collections import defaultdict
|
|
2
|
-
from datetime import date, datetime
|
|
3
|
-
from functools import reduce
|
|
4
|
-
from typing import DefaultDict, Dict, List, Tuple, Union
|
|
5
|
-
|
|
6
|
-
from qwak.clients.administration.eco_system.client import EcosystemClient
|
|
7
|
-
|
|
8
|
-
try:
|
|
9
|
-
import pandas as pd
|
|
10
|
-
except ImportError:
|
|
11
|
-
pass
|
|
12
|
-
|
|
13
|
-
from dateutil.parser import ParserError
|
|
14
|
-
from qwak.exceptions import QwakException
|
|
15
|
-
from qwak.feature_store._common.functions import normalize_cols
|
|
16
|
-
from qwak.feature_store.offline.athena.athena_query_engine import AthenaQueryEngine
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class OfflineClient:
|
|
20
|
-
"""
|
|
21
|
-
A class used to retrieve data from the offline store - mainly used to get train data for models.
|
|
22
|
-
It requires qwak configure and aws access.
|
|
23
|
-
"""
|
|
24
|
-
|
|
25
|
-
# Default SCD parameters of the feature store etl
|
|
26
|
-
ANALYTICS_DB_PREFIX = "qwak_analytics_feature_store"
|
|
27
|
-
FS_DB_PREFIX = "qwak_feature_store"
|
|
28
|
-
FS_TABLE_NAME_PREFIX = "offline_feature_store"
|
|
29
|
-
FS_START_TIME_COLUMN = "start_timestamp"
|
|
30
|
-
FS_END_TIME_COLUMN = "end_timestamp"
|
|
31
|
-
FEATURE_STORE_PREFIX = "feature_store"
|
|
32
|
-
|
|
33
|
-
DEFAULT_NUMBER_OF_SAMPLE_DATA_ROWS = "100"
|
|
34
|
-
|
|
35
|
-
def __init__(
|
|
36
|
-
self,
|
|
37
|
-
query_engine=None,
|
|
38
|
-
environment_id=None,
|
|
39
|
-
):
|
|
40
|
-
self.query_engine = (
|
|
41
|
-
query_engine if query_engine is not None else AthenaQueryEngine()
|
|
42
|
-
)
|
|
43
|
-
self.quotes = self.query_engine.get_quotes()
|
|
44
|
-
if environment_id is None:
|
|
45
|
-
user_context = EcosystemClient().get_authenticated_user_context().user
|
|
46
|
-
environment_id = (
|
|
47
|
-
user_context.account_details.default_environment_id.replace("-", "_")
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
self.environment_id = environment_id.replace("-", "_")
|
|
51
|
-
self.FS_DB_NAME = self.FS_DB_PREFIX + "_" + self.environment_id
|
|
52
|
-
self.FS_ANALYTICS_DB_NAME = self.ANALYTICS_DB_PREFIX + "_" + self.environment_id
|
|
53
|
-
|
|
54
|
-
def get_feature_range_values(
|
|
55
|
-
self,
|
|
56
|
-
entity_key_to_features: dict,
|
|
57
|
-
start_date: Union[datetime, date],
|
|
58
|
-
end_date: Union[datetime, date],
|
|
59
|
-
):
|
|
60
|
-
"""
|
|
61
|
-
:param entity_key_to_features: a dictionary { entity_key(s) -> features list }.
|
|
62
|
-
:param start_date: the column name of the point in time column (default - timestamp)
|
|
63
|
-
:param end_date: the column name of the point in time column (default - timestamp)
|
|
64
|
-
:return: a pandas dataframe or a list of dataframes (a dataframe for every entity_key) - all feature values for
|
|
65
|
-
all entites under the given date range
|
|
66
|
-
|
|
67
|
-
each row in the returned data-frame is constructed by retrieving the requested features of the entity
|
|
68
|
-
key(s) for all entity values in within the defined date tange.
|
|
69
|
-
|
|
70
|
-
Feature sets should be named [Feature Set Name].[Feature Name],
|
|
71
|
-
i.e: user_purchases.number_of_purchases.
|
|
72
|
-
|
|
73
|
-
Examples:
|
|
74
|
-
>>> from datetime import datetime
|
|
75
|
-
>>> from qwak.feature_store.offline import OfflineClient
|
|
76
|
-
>>>
|
|
77
|
-
>>> start_date = datetime(year=2021, month=1, day=1)
|
|
78
|
-
>>> end_date = datetime(year=2021, month=1, day=3)
|
|
79
|
-
>>>
|
|
80
|
-
>>> key_to_features = {'uuid': ['user_purchases.number_of_purchases',
|
|
81
|
-
>>> 'user_purchases.avg_purchase_amount']}
|
|
82
|
-
>>>
|
|
83
|
-
>>> offline_feature_store = OfflineClient()
|
|
84
|
-
>>>
|
|
85
|
-
>>> train_df = offline_feature_store.get_feature_range_values(
|
|
86
|
-
>>> entity_key_to_features=key_to_features,
|
|
87
|
-
>>> start_date=start_date,
|
|
88
|
-
>>> end_date=end_date)
|
|
89
|
-
>>>
|
|
90
|
-
>>> print(train_df.head())
|
|
91
|
-
>>> # uuid timestamp user_purchases.number_of_purchases user_purchases.avg_purchase_amount
|
|
92
|
-
>>> # 0 1 2021-01-02 17:00:00 76 4.796842
|
|
93
|
-
>>> # 1 1 2021-01-01 12:00:00 5 1.548000
|
|
94
|
-
>>> # 2 2 2021-01-02 12:00:00 5 5.548000
|
|
95
|
-
>>> # 3 2 2021-01-01 18:00:00 5 2.788000
|
|
96
|
-
"""
|
|
97
|
-
try:
|
|
98
|
-
from qwak.feature_store._common.featureset_asterisk_handler import (
|
|
99
|
-
unpack_asterisk_features_from_key_mapping,
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
entity_key_to_features = unpack_asterisk_features_from_key_mapping(
|
|
103
|
-
entity_key_to_features, lambda: self
|
|
104
|
-
)
|
|
105
|
-
self._validate_range_query_inputs(
|
|
106
|
-
entity_key_to_features, start_date, end_date
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
feature_set_name_to_feature_list = (
|
|
110
|
-
self._partition_feature_set_by_entity_feature(entity_key_to_features)
|
|
111
|
-
)
|
|
112
|
-
feature_set_to_dtypes = self._validate_database_and_get_feature_set_dtypes(
|
|
113
|
-
feature_set_name_to_feature_list
|
|
114
|
-
)
|
|
115
|
-
self._validate_features_exist(
|
|
116
|
-
feature_set_name_to_feature_list, feature_set_to_dtypes
|
|
117
|
-
)
|
|
118
|
-
|
|
119
|
-
df = self._run_ranges_query(
|
|
120
|
-
feature_set_name_to_feature_list, start_date, end_date
|
|
121
|
-
)
|
|
122
|
-
|
|
123
|
-
return self._normalize_df(df)
|
|
124
|
-
|
|
125
|
-
except QwakException as qwak_exception:
|
|
126
|
-
raise QwakException(
|
|
127
|
-
f"Got the following Qwak generated exception: {qwak_exception}"
|
|
128
|
-
)
|
|
129
|
-
except Exception as e:
|
|
130
|
-
raise QwakException(f"Got the following run-time exception: {e}")
|
|
131
|
-
finally:
|
|
132
|
-
try:
|
|
133
|
-
self.query_engine.cleanup()
|
|
134
|
-
except Exception as e:
|
|
135
|
-
print(f"Got the following run-time exception during cleanup: {e}")
|
|
136
|
-
|
|
137
|
-
def get_feature_values(
|
|
138
|
-
self,
|
|
139
|
-
entity_key_to_features: dict,
|
|
140
|
-
population: "pd.DataFrame",
|
|
141
|
-
point_in_time_column_name: str = "timestamp",
|
|
142
|
-
):
|
|
143
|
-
"""
|
|
144
|
-
:param entity_key_to_features: a dictionary { entity_key(s) -> features list }.
|
|
145
|
-
:param population: a pandas data-frame with a point in time column
|
|
146
|
-
and a column for each entity key defined at entity_key_to_features.
|
|
147
|
-
:param point_in_time_column_name: the column name of the point in time column (default - timestamp)
|
|
148
|
-
:return: a pandas data-frame - the population joined with the feature values for all
|
|
149
|
-
the requested entities and features.
|
|
150
|
-
|
|
151
|
-
each row in the returned data-frame is constructed by retrieving the requested features of the entity key(s) for
|
|
152
|
-
the specific entity value(s) in the population and on the specific point in time defined.
|
|
153
|
-
|
|
154
|
-
Feature sets should be named [Feature Set Name].[Feature Name],
|
|
155
|
-
i.e: user_purchases.number_of_purchases.
|
|
156
|
-
|
|
157
|
-
Examples:
|
|
158
|
-
>>> import pandas as pd
|
|
159
|
-
>>> from qwak.feature_store.offline import OfflineClient
|
|
160
|
-
>>>
|
|
161
|
-
>>> population_df = pd.DataFrame(
|
|
162
|
-
>>> columns= ['uuid', 'timestamp' ],
|
|
163
|
-
>>> data =[[ '1' , '2021-01-02 17:00:00' ],
|
|
164
|
-
>>> [ '2' , '2021-01-01 12:00:00' ]])
|
|
165
|
-
>>>
|
|
166
|
-
>>> key_to_features = {'uuid': ['user_purchases.number_of_purchases',
|
|
167
|
-
>>> 'user_purchases.avg_purchase_amount']}
|
|
168
|
-
>>>
|
|
169
|
-
>>> offline_feature_store = OfflineClient()
|
|
170
|
-
>>>
|
|
171
|
-
>>> train_df = offline_feature_store.get_feature_values(
|
|
172
|
-
>>> entity_key_to_features=key_to_features,
|
|
173
|
-
>>> population=population_df,
|
|
174
|
-
>>> point_in_time_column_name='timestamp')
|
|
175
|
-
>>>
|
|
176
|
-
>>> print(train_df.head())
|
|
177
|
-
>>> # uuid timestamp user_purchases.number_of_purchases user_purchases.avg_purchase_amount
|
|
178
|
-
>>> # 0 1 2021-04-24 17:00:00 76 4.796842
|
|
179
|
-
>>> # 1 2 2021-04-24 12:00:00 5 1.548000
|
|
180
|
-
"""
|
|
181
|
-
|
|
182
|
-
import pandas as pd
|
|
183
|
-
|
|
184
|
-
try:
|
|
185
|
-
from qwak.feature_store._common.featureset_asterisk_handler import (
|
|
186
|
-
unpack_asterisk_features_from_key_mapping,
|
|
187
|
-
)
|
|
188
|
-
|
|
189
|
-
population = population.copy()
|
|
190
|
-
|
|
191
|
-
entity_key_to_features = unpack_asterisk_features_from_key_mapping(
|
|
192
|
-
entity_key_to_features, lambda: self
|
|
193
|
-
)
|
|
194
|
-
|
|
195
|
-
self._validate_point_in_time_query_inputs(
|
|
196
|
-
entity_key_to_features, population, point_in_time_column_name
|
|
197
|
-
)
|
|
198
|
-
|
|
199
|
-
feature_set_name_to_feature_dict = (
|
|
200
|
-
self._partition_feature_set_by_entity_feature(entity_key_to_features)
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
feature_set_to_dtypes = self._validate_database_and_get_feature_set_dtypes(
|
|
204
|
-
feature_set_name_to_feature_dict
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
self._validate_features_exist(
|
|
208
|
-
feature_set_name_to_feature_dict,
|
|
209
|
-
feature_set_to_dtypes,
|
|
210
|
-
)
|
|
211
|
-
|
|
212
|
-
population = self._align_entity_key_dtype(
|
|
213
|
-
feature_set_to_dtypes, entity_key_to_features, population
|
|
214
|
-
)
|
|
215
|
-
|
|
216
|
-
uploaded_population_path = self.query_engine.upload_table(population)
|
|
217
|
-
|
|
218
|
-
df = pd.DataFrame()
|
|
219
|
-
|
|
220
|
-
if feature_set_name_to_feature_dict:
|
|
221
|
-
df = self._run_point_in_time_query(
|
|
222
|
-
feature_set_name_to_feature_dict,
|
|
223
|
-
uploaded_population_path,
|
|
224
|
-
point_in_time_column_name,
|
|
225
|
-
[column.lower() for column in population.columns],
|
|
226
|
-
)
|
|
227
|
-
|
|
228
|
-
return self._normalize_df(df)
|
|
229
|
-
|
|
230
|
-
except QwakException as qwak_exception:
|
|
231
|
-
raise QwakException(
|
|
232
|
-
f"Got the following Qwak generated exception: {qwak_exception}"
|
|
233
|
-
)
|
|
234
|
-
except Exception as e:
|
|
235
|
-
raise QwakException(f"Got the following run-time exception: {e}")
|
|
236
|
-
finally:
|
|
237
|
-
try:
|
|
238
|
-
self.query_engine.cleanup()
|
|
239
|
-
except Exception as e:
|
|
240
|
-
print(f"Got the following run-time exception during cleanup: {e}")
|
|
241
|
-
|
|
242
|
-
@staticmethod
|
|
243
|
-
def _normalize_df(df: "pd.DataFrame") -> "pd.DataFrame":
|
|
244
|
-
columns = df.columns.values.tolist()
|
|
245
|
-
new_columns = normalize_cols(columns)
|
|
246
|
-
df.columns = new_columns
|
|
247
|
-
return df
|
|
248
|
-
|
|
249
|
-
@staticmethod
|
|
250
|
-
def _validate_range_query_inputs(
|
|
251
|
-
entity_key_to_features: dict, start_date: datetime, end_date: datetime
|
|
252
|
-
):
|
|
253
|
-
missing_features_entity_keys = [
|
|
254
|
-
entity_key
|
|
255
|
-
for entity_key, features in entity_key_to_features.items()
|
|
256
|
-
if not features
|
|
257
|
-
]
|
|
258
|
-
|
|
259
|
-
if missing_features_entity_keys:
|
|
260
|
-
raise QwakException(
|
|
261
|
-
f"Features of an entity key must exist, missing features for: [{missing_features_entity_keys}]"
|
|
262
|
-
)
|
|
263
|
-
|
|
264
|
-
if (end_date - start_date).total_seconds() < 0:
|
|
265
|
-
raise QwakException("Invalid date range - end date is before start date")
|
|
266
|
-
|
|
267
|
-
@staticmethod
|
|
268
|
-
def _validate_point_in_time_query_inputs(
|
|
269
|
-
entity_key_to_features: dict,
|
|
270
|
-
population: "pd.DataFrame",
|
|
271
|
-
point_in_time_column_name: str,
|
|
272
|
-
):
|
|
273
|
-
"""
|
|
274
|
-
Validates that the entity keys, timestamp cols and features exist in DB
|
|
275
|
-
"""
|
|
276
|
-
missing_keys = [
|
|
277
|
-
entity_key
|
|
278
|
-
for entity_key in entity_key_to_features.keys()
|
|
279
|
-
if entity_key not in population
|
|
280
|
-
]
|
|
281
|
-
if missing_keys:
|
|
282
|
-
raise QwakException(
|
|
283
|
-
f"The entity keys must be in population_df columns, missing: [{missing_keys}]"
|
|
284
|
-
)
|
|
285
|
-
|
|
286
|
-
missing_features_entity_keys = [
|
|
287
|
-
entity_key
|
|
288
|
-
for entity_key, features in entity_key_to_features.items()
|
|
289
|
-
if not features
|
|
290
|
-
]
|
|
291
|
-
|
|
292
|
-
if missing_features_entity_keys:
|
|
293
|
-
raise QwakException(
|
|
294
|
-
f"Features of an entity key must exist, missing features for: [{missing_features_entity_keys}]"
|
|
295
|
-
)
|
|
296
|
-
|
|
297
|
-
if point_in_time_column_name not in population:
|
|
298
|
-
raise QwakException(
|
|
299
|
-
"The point in time column must be part of the population dataframe"
|
|
300
|
-
)
|
|
301
|
-
|
|
302
|
-
from pandas.api.types import is_datetime64_any_dtype
|
|
303
|
-
|
|
304
|
-
if not is_datetime64_any_dtype(population[point_in_time_column_name]):
|
|
305
|
-
try:
|
|
306
|
-
population[point_in_time_column_name] = pd.to_datetime(
|
|
307
|
-
population[point_in_time_column_name]
|
|
308
|
-
)
|
|
309
|
-
except ParserError as e:
|
|
310
|
-
raise QwakException(
|
|
311
|
-
f"It was not possible to cast provided point in time column to datetime"
|
|
312
|
-
f"\nError message: {e}"
|
|
313
|
-
)
|
|
314
|
-
|
|
315
|
-
@staticmethod
|
|
316
|
-
def _partition_feature_set_by_entity_feature(
|
|
317
|
-
entity_key_to_features,
|
|
318
|
-
) -> DefaultDict[Tuple[str, str], List[str]]:
|
|
319
|
-
"""
|
|
320
|
-
Partition feature by entity key and featureset name
|
|
321
|
-
Args:
|
|
322
|
-
entity_key_to_features: dict of entity_key -> full feature name
|
|
323
|
-
Returns:
|
|
324
|
-
dict of (entity_key,featureset_name) -> list of feature names
|
|
325
|
-
"""
|
|
326
|
-
feature_name_to_feature_list = defaultdict(list)
|
|
327
|
-
|
|
328
|
-
for entity_key, feature_list in entity_key_to_features.items():
|
|
329
|
-
for feature in feature_list:
|
|
330
|
-
split_feature_set_and_feature = feature.lower().split(".")
|
|
331
|
-
if len(split_feature_set_and_feature) != 2:
|
|
332
|
-
raise QwakException(
|
|
333
|
-
f"Failed to verify features. Name should be: <feature set name>.<feature name>. "
|
|
334
|
-
f"Current name is: {feature}"
|
|
335
|
-
)
|
|
336
|
-
feature_set_name = split_feature_set_and_feature[0]
|
|
337
|
-
feature_name_to_feature_list[(entity_key, feature_set_name)].append(
|
|
338
|
-
feature
|
|
339
|
-
)
|
|
340
|
-
|
|
341
|
-
return feature_name_to_feature_list
|
|
342
|
-
|
|
343
|
-
def _validate_database_and_get_feature_set_dtypes(
|
|
344
|
-
self, feature_name_to_feature_list: DefaultDict[Tuple[str, str], List[str]]
|
|
345
|
-
) -> Dict[Tuple[str, str], List[Tuple[str, str]]]:
|
|
346
|
-
"""
|
|
347
|
-
Args:
|
|
348
|
-
feature_name_to_feature_list: dictionary from feature set name to its' features
|
|
349
|
-
|
|
350
|
-
Returns
|
|
351
|
-
dictionary from feature set name and entity key to a list of feature name, feature type
|
|
352
|
-
"""
|
|
353
|
-
if self.FS_DB_NAME not in self._fs_db_names():
|
|
354
|
-
raise QwakException("Offline feature store does not contain any data")
|
|
355
|
-
|
|
356
|
-
feature_set_to_dtypes = {}
|
|
357
|
-
for (
|
|
358
|
-
entity_key,
|
|
359
|
-
feature_set_name,
|
|
360
|
-
), feature_list in feature_name_to_feature_list.items():
|
|
361
|
-
table_name = self._get_offline_feature_store_full_name(feature_set_name)
|
|
362
|
-
if table_name not in self._fs_tables_names():
|
|
363
|
-
raise QwakException(
|
|
364
|
-
f"[{feature_set_name}] feature set does not contain any data"
|
|
365
|
-
)
|
|
366
|
-
|
|
367
|
-
columns_query_result = self.query_engine.run_query(
|
|
368
|
-
f"SELECT * FROM INFORMATION_SCHEMA.COLUMNS " # nosec B608
|
|
369
|
-
f"WHERE TABLE_SCHEMA = '{self.FS_DB_NAME}' "
|
|
370
|
-
f"AND TABLE_NAME = '{table_name}'"
|
|
371
|
-
)
|
|
372
|
-
|
|
373
|
-
feature_set_to_dtypes[(entity_key, feature_set_name)] = [
|
|
374
|
-
(column_tup[3], column_tup[7]) for column_tup in columns_query_result
|
|
375
|
-
]
|
|
376
|
-
return feature_set_to_dtypes
|
|
377
|
-
|
|
378
|
-
@staticmethod
|
|
379
|
-
def _validate_features_exist(
|
|
380
|
-
feature_name_to_feature_list: defaultdict,
|
|
381
|
-
feature_set_to_dtypes: Dict[Tuple[str, str], List[Tuple[str, str]]],
|
|
382
|
-
):
|
|
383
|
-
"""
|
|
384
|
-
Args:
|
|
385
|
-
feature_name_to_feature_list: dictionary from feature set name to its' features
|
|
386
|
-
feature_set_to_dtypes: dictionary from feature set name and entity key
|
|
387
|
-
to a list of feature name, feature type
|
|
388
|
-
"""
|
|
389
|
-
for (
|
|
390
|
-
entity_key,
|
|
391
|
-
feature_set_name,
|
|
392
|
-
), feature_list in feature_name_to_feature_list.items():
|
|
393
|
-
columns = [
|
|
394
|
-
column_tuple[0].lower()
|
|
395
|
-
for column_tuple in feature_set_to_dtypes[
|
|
396
|
-
(entity_key, feature_set_name)
|
|
397
|
-
]
|
|
398
|
-
]
|
|
399
|
-
absent_features = [
|
|
400
|
-
feature for feature in feature_list if feature not in columns
|
|
401
|
-
]
|
|
402
|
-
if absent_features:
|
|
403
|
-
raise QwakException(
|
|
404
|
-
f"Missing the following features for the feature set "
|
|
405
|
-
f"[{feature_set_name}]:"
|
|
406
|
-
f"\n{absent_features}"
|
|
407
|
-
)
|
|
408
|
-
|
|
409
|
-
def _align_entity_key_dtype(
|
|
410
|
-
self,
|
|
411
|
-
feature_set_to_dtypes: Dict[Tuple[str, str], List[Tuple[str, str]]],
|
|
412
|
-
entity_key_to_features: Dict[str, List[str]],
|
|
413
|
-
population: "pd.DataFrame",
|
|
414
|
-
) -> "pd.DataFrame":
|
|
415
|
-
"""
|
|
416
|
-
Args:
|
|
417
|
-
feature_set_to_dtypes: dictionary from feature set name and entity key
|
|
418
|
-
to a list of feature name, feature type
|
|
419
|
-
entity_key_to_features: a dictionary { entity_key(s) -> features list }.
|
|
420
|
-
population: a pandas data-frame with a point in time column
|
|
421
|
-
and a column for each entity key defined at entity_key_to_features.
|
|
422
|
-
Returns:
|
|
423
|
-
entity type aligned population df
|
|
424
|
-
"""
|
|
425
|
-
sql_type_to_pandas = {
|
|
426
|
-
"string": "string",
|
|
427
|
-
"integer": "int",
|
|
428
|
-
"varchar": "string",
|
|
429
|
-
"text": "string",
|
|
430
|
-
"bigint": "int",
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
entity_key_to_dtype = self._validate_and_get_entity_keys_dtypes(
|
|
434
|
-
entity_key_to_features, population
|
|
435
|
-
)
|
|
436
|
-
|
|
437
|
-
for (
|
|
438
|
-
entity_key,
|
|
439
|
-
feature_set_name,
|
|
440
|
-
), feature_dtypes_list in feature_set_to_dtypes.items():
|
|
441
|
-
given_entity_key_dtype = entity_key_to_dtype[entity_key]
|
|
442
|
-
entity_column_tuple = [
|
|
443
|
-
column_tuple
|
|
444
|
-
for column_tuple in feature_dtypes_list
|
|
445
|
-
if column_tuple[0] == entity_key
|
|
446
|
-
]
|
|
447
|
-
if not entity_column_tuple:
|
|
448
|
-
raise QwakException(
|
|
449
|
-
f"Did not find entity key [{entity_key}] in the table of [{feature_set_name}] "
|
|
450
|
-
f"- existing columns are: "
|
|
451
|
-
f"{[column_tuple[0] for column_tuple in feature_dtypes_list]}"
|
|
452
|
-
)
|
|
453
|
-
actual_entity_type = entity_column_tuple[0][1]
|
|
454
|
-
if actual_entity_type == given_entity_key_dtype:
|
|
455
|
-
continue
|
|
456
|
-
else:
|
|
457
|
-
try:
|
|
458
|
-
population[entity_key] = population[entity_key].astype(
|
|
459
|
-
sql_type_to_pandas[actual_entity_type]
|
|
460
|
-
)
|
|
461
|
-
print(
|
|
462
|
-
f"Entity [{entity_key}] given type [{given_entity_key_dtype}] "
|
|
463
|
-
f"was not aligned with actual type [{actual_entity_type}] - casted to correct type"
|
|
464
|
-
)
|
|
465
|
-
except ValueError as e:
|
|
466
|
-
raise QwakException(
|
|
467
|
-
f"Mismatched entity type for [{entity_key}] - [{given_entity_key_dtype}] "
|
|
468
|
-
f"- failed to cast to actual type [{actual_entity_type}], Error: {e}"
|
|
469
|
-
)
|
|
470
|
-
|
|
471
|
-
return population
|
|
472
|
-
|
|
473
|
-
def _validate_and_get_entity_keys_dtypes(
|
|
474
|
-
self,
|
|
475
|
-
entity_key_to_features: Dict[str, List[str]],
|
|
476
|
-
population_df: "pd.DataFrame",
|
|
477
|
-
) -> Dict[str, str]:
|
|
478
|
-
"""
|
|
479
|
-
Args:
|
|
480
|
-
entity_key_to_features: a dictionary { entity_key(s) -> features list }.
|
|
481
|
-
population_df: a pandas data-frame with a point in time column
|
|
482
|
-
and a column for each entity key defined at entity_key_to_features.
|
|
483
|
-
|
|
484
|
-
Returns:
|
|
485
|
-
dictionary of entity key to it's dtype
|
|
486
|
-
"""
|
|
487
|
-
supported_dtypes_to_actual_type = {
|
|
488
|
-
"object": "string",
|
|
489
|
-
"int32": "integer",
|
|
490
|
-
"int64": "integer",
|
|
491
|
-
}
|
|
492
|
-
entity_key_to_dtype = {}
|
|
493
|
-
for entity_key in entity_key_to_features.keys():
|
|
494
|
-
entity_pandas_dtype = population_df.dtypes[entity_key].name
|
|
495
|
-
if entity_pandas_dtype not in supported_dtypes_to_actual_type:
|
|
496
|
-
raise QwakException(
|
|
497
|
-
f"Got an unsupported dtype for the entity key "
|
|
498
|
-
f"[{entity_key}] - [{entity_pandas_dtype}]"
|
|
499
|
-
)
|
|
500
|
-
entity_key_to_dtype[entity_key] = supported_dtypes_to_actual_type[
|
|
501
|
-
entity_pandas_dtype
|
|
502
|
-
]
|
|
503
|
-
return entity_key_to_dtype
|
|
504
|
-
|
|
505
|
-
def _run_ranges_query(
|
|
506
|
-
self,
|
|
507
|
-
feature_name_to_feature_list: defaultdict,
|
|
508
|
-
start_date: datetime,
|
|
509
|
-
end_date: datetime,
|
|
510
|
-
):
|
|
511
|
-
result_dfs = []
|
|
512
|
-
features_set_by_entity = defaultdict(lambda: defaultdict(set))
|
|
513
|
-
for (
|
|
514
|
-
(entity_key, feature_set_name),
|
|
515
|
-
feature_list,
|
|
516
|
-
) in feature_name_to_feature_list.items():
|
|
517
|
-
for feature in feature_list:
|
|
518
|
-
feature_set_name = feature.split(".")[0]
|
|
519
|
-
features_set_by_entity[entity_key][feature_set_name].add(feature)
|
|
520
|
-
|
|
521
|
-
for entity_key, features_dict in features_set_by_entity.items():
|
|
522
|
-
entity_dfs = []
|
|
523
|
-
for feature_set_name, feature_list in features_dict.items():
|
|
524
|
-
offline_feature_store_full_path, table_path = self.get_fs_full_path(
|
|
525
|
-
feature_set_name
|
|
526
|
-
)
|
|
527
|
-
|
|
528
|
-
features = ", ".join(
|
|
529
|
-
[
|
|
530
|
-
f"{offline_feature_store_full_path}.{self.quotes}{feature}{self.quotes}"
|
|
531
|
-
for feature in feature_list
|
|
532
|
-
]
|
|
533
|
-
)
|
|
534
|
-
|
|
535
|
-
where_part = (
|
|
536
|
-
"WHERE "
|
|
537
|
-
f"{table_path}.{self.FS_START_TIME_COLUMN} >= timestamp '{start_date}' "
|
|
538
|
-
f"AND ({self.FS_END_TIME_COLUMN} <= "
|
|
539
|
-
f"timestamp '{end_date}' OR {table_path}.{self.FS_END_TIME_COLUMN} IS NULL) "
|
|
540
|
-
f"AND {table_path}.{self.FS_START_TIME_COLUMN} < timestamp '{end_date}'"
|
|
541
|
-
)
|
|
542
|
-
|
|
543
|
-
full_sql = (
|
|
544
|
-
f"SELECT {offline_feature_store_full_path}.{self.quotes}{entity_key}{self.quotes}, " # nosec B608
|
|
545
|
-
f"{offline_feature_store_full_path}.{self.quotes}{self.FS_START_TIME_COLUMN}{self.quotes}, "
|
|
546
|
-
f"{features} "
|
|
547
|
-
f"FROM {offline_feature_store_full_path} "
|
|
548
|
-
f"{where_part}"
|
|
549
|
-
)
|
|
550
|
-
|
|
551
|
-
entity_dfs.append(self.query_engine.read_pandas_from_query(full_sql))
|
|
552
|
-
|
|
553
|
-
entity_final_df = reduce(
|
|
554
|
-
lambda left, right: pd.merge(
|
|
555
|
-
left, right, on=[entity_key, self.FS_START_TIME_COLUMN], how="outer"
|
|
556
|
-
),
|
|
557
|
-
entity_dfs,
|
|
558
|
-
)
|
|
559
|
-
result_dfs.append(entity_final_df.reset_index(drop=True))
|
|
560
|
-
|
|
561
|
-
return result_dfs[0] if len(result_dfs) == 1 else result_dfs
|
|
562
|
-
|
|
563
|
-
def _run_point_in_time_query(
|
|
564
|
-
self,
|
|
565
|
-
feature_name_to_feature_list: defaultdict,
|
|
566
|
-
uploaded_population_path: str,
|
|
567
|
-
point_in_time_column_name: str,
|
|
568
|
-
population_list: list,
|
|
569
|
-
) -> "pd.DataFrame":
|
|
570
|
-
"""
|
|
571
|
-
creates SQL query for pint in time feature fetching based on population and requested features
|
|
572
|
-
"""
|
|
573
|
-
dfs = []
|
|
574
|
-
|
|
575
|
-
for index, ((entity_key, feature_set_name), feature_list) in enumerate(
|
|
576
|
-
feature_name_to_feature_list.items()
|
|
577
|
-
):
|
|
578
|
-
offline_feature_store_full_path, table_path = self.get_fs_full_path(
|
|
579
|
-
feature_set_name
|
|
580
|
-
)
|
|
581
|
-
|
|
582
|
-
join_part = self._get_join_population_sql(
|
|
583
|
-
entity_key, offline_feature_store_full_path, uploaded_population_path
|
|
584
|
-
)
|
|
585
|
-
|
|
586
|
-
point_in_time_column_full_path = (
|
|
587
|
-
f"{uploaded_population_path}.{point_in_time_column_name}"
|
|
588
|
-
)
|
|
589
|
-
|
|
590
|
-
where_part = (
|
|
591
|
-
"WHERE "
|
|
592
|
-
f"{point_in_time_column_full_path} >= "
|
|
593
|
-
f"{table_path}.{self.FS_START_TIME_COLUMN} "
|
|
594
|
-
f"AND ({point_in_time_column_full_path} < "
|
|
595
|
-
f"{table_path}.{self.FS_END_TIME_COLUMN} OR "
|
|
596
|
-
f"{table_path}.{self.FS_END_TIME_COLUMN} IS NULL)"
|
|
597
|
-
)
|
|
598
|
-
|
|
599
|
-
features = ", ".join(
|
|
600
|
-
[
|
|
601
|
-
f"{offline_feature_store_full_path}.{self.quotes}{feature}{self.quotes} as {self.quotes}{feature}{self.quotes}"
|
|
602
|
-
for feature in feature_list
|
|
603
|
-
]
|
|
604
|
-
)
|
|
605
|
-
|
|
606
|
-
final_query_features = ", ".join(
|
|
607
|
-
[
|
|
608
|
-
f"filtered_features.{self.quotes}{feature}{self.quotes} as {self.quotes}{feature}{self.quotes}"
|
|
609
|
-
for feature in feature_list
|
|
610
|
-
]
|
|
611
|
-
)
|
|
612
|
-
|
|
613
|
-
full_sql = (
|
|
614
|
-
f"WITH " # nosec B608
|
|
615
|
-
"filtered_features AS ( "
|
|
616
|
-
f"SELECT {uploaded_population_path}.*, {features} "
|
|
617
|
-
f"FROM {uploaded_population_path} "
|
|
618
|
-
f"{join_part} "
|
|
619
|
-
f"{where_part} "
|
|
620
|
-
") "
|
|
621
|
-
f"SELECT population.*, {final_query_features} "
|
|
622
|
-
f"FROM {uploaded_population_path} population "
|
|
623
|
-
f"LEFT JOIN filtered_features "
|
|
624
|
-
f"ON population.{entity_key} = filtered_features.{entity_key} "
|
|
625
|
-
f"AND population.{point_in_time_column_name} = filtered_features.{point_in_time_column_name}"
|
|
626
|
-
)
|
|
627
|
-
|
|
628
|
-
dfs.append(
|
|
629
|
-
self.query_engine.read_pandas_from_query(
|
|
630
|
-
full_sql, [point_in_time_column_name]
|
|
631
|
-
)
|
|
632
|
-
)
|
|
633
|
-
return self._merge_query_dataframes_results(dfs, population_list)
|
|
634
|
-
|
|
635
|
-
@staticmethod
|
|
636
|
-
def _merge_query_dataframes_results(
|
|
637
|
-
dfs: List["pd.DataFrame"], population_list
|
|
638
|
-
) -> "pd.DataFrame":
|
|
639
|
-
"""
|
|
640
|
-
merges query result dataframes according to population list cols
|
|
641
|
-
"""
|
|
642
|
-
if dfs:
|
|
643
|
-
df_final = reduce(
|
|
644
|
-
lambda left, right: pd.merge(
|
|
645
|
-
left, right, on=population_list, how="outer"
|
|
646
|
-
),
|
|
647
|
-
dfs,
|
|
648
|
-
)
|
|
649
|
-
|
|
650
|
-
ordered_cols = population_list + (
|
|
651
|
-
df_final.columns.drop(population_list).tolist()
|
|
652
|
-
)
|
|
653
|
-
|
|
654
|
-
return df_final[ordered_cols].reset_index(drop=True)
|
|
655
|
-
else:
|
|
656
|
-
return pd.DataFrame()
|
|
657
|
-
|
|
658
|
-
def _get_join_population_sql(
|
|
659
|
-
self, entity_key, offline_feature_store_full_path, uploaded_population_path
|
|
660
|
-
):
|
|
661
|
-
"""
|
|
662
|
-
return a join sql query for uploaded population table
|
|
663
|
-
"""
|
|
664
|
-
join_part = (
|
|
665
|
-
f"LEFT JOIN {offline_feature_store_full_path} ON "
|
|
666
|
-
f"{offline_feature_store_full_path}.{entity_key} = "
|
|
667
|
-
f"{uploaded_population_path}.{entity_key} "
|
|
668
|
-
)
|
|
669
|
-
return join_part
|
|
670
|
-
|
|
671
|
-
def get_fs_full_path(self, feature_set_name: str) -> Tuple[str, str]:
|
|
672
|
-
offline_feature_store_table_name = self._get_offline_feature_store_full_name(
|
|
673
|
-
feature_set_name
|
|
674
|
-
)
|
|
675
|
-
table_path = f"{self.quotes}{offline_feature_store_table_name}{self.quotes}"
|
|
676
|
-
offline_feature_store_full_path = (
|
|
677
|
-
f"{self.quotes}{self.FS_DB_NAME}{self.quotes}."
|
|
678
|
-
f"{self.quotes}{offline_feature_store_table_name}{self.quotes}"
|
|
679
|
-
)
|
|
680
|
-
return offline_feature_store_full_path, table_path
|
|
681
|
-
|
|
682
|
-
def _fs_db_names(self) -> List[str]:
|
|
683
|
-
return [
|
|
684
|
-
database_tuple[0]
|
|
685
|
-
for database_tuple in self.query_engine.run_query("SHOW SCHEMAS")
|
|
686
|
-
]
|
|
687
|
-
|
|
688
|
-
def _fs_tables_names(self) -> List[str]:
|
|
689
|
-
return [
|
|
690
|
-
table_tuple[0]
|
|
691
|
-
for table_tuple in self.query_engine.run_query(
|
|
692
|
-
f"SHOW TABLES IN {self.FS_DB_NAME}"
|
|
693
|
-
)
|
|
694
|
-
]
|
|
695
|
-
|
|
696
|
-
def _get_offline_feature_store_full_name(self, feature_set_name: str) -> str:
|
|
697
|
-
return f"{self.FS_TABLE_NAME_PREFIX}_{feature_set_name}".lower().replace(
|
|
698
|
-
"-", "_"
|
|
699
|
-
)
|