snowflake-ml-python 1.8.2__py3-none-any.whl → 1.8.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake/cortex/_classify_text.py +3 -3
- snowflake/cortex/_complete.py +23 -24
- snowflake/cortex/_embed_text_1024.py +4 -4
- snowflake/cortex/_embed_text_768.py +4 -4
- snowflake/cortex/_finetune.py +8 -8
- snowflake/cortex/_util.py +8 -12
- snowflake/ml/_internal/env.py +4 -3
- snowflake/ml/_internal/env_utils.py +63 -34
- snowflake/ml/_internal/file_utils.py +10 -21
- snowflake/ml/_internal/human_readable_id/hrid_generator_base.py +5 -7
- snowflake/ml/_internal/init_utils.py +2 -3
- snowflake/ml/_internal/lineage/lineage_utils.py +6 -6
- snowflake/ml/_internal/platform_capabilities.py +6 -6
- snowflake/ml/_internal/telemetry.py +39 -52
- snowflake/ml/_internal/type_utils.py +3 -3
- snowflake/ml/_internal/utils/db_utils.py +2 -2
- snowflake/ml/_internal/utils/identifier.py +8 -8
- snowflake/ml/_internal/utils/import_utils.py +2 -2
- snowflake/ml/_internal/utils/parallelize.py +7 -7
- snowflake/ml/_internal/utils/pkg_version_utils.py +11 -11
- snowflake/ml/_internal/utils/query_result_checker.py +4 -4
- snowflake/ml/_internal/utils/snowflake_env.py +28 -6
- snowflake/ml/_internal/utils/snowpark_dataframe_utils.py +2 -2
- snowflake/ml/_internal/utils/sql_identifier.py +3 -3
- snowflake/ml/_internal/utils/table_manager.py +9 -9
- snowflake/ml/data/_internal/arrow_ingestor.py +7 -7
- snowflake/ml/data/data_connector.py +15 -36
- snowflake/ml/data/data_ingestor.py +4 -15
- snowflake/ml/data/data_source.py +2 -2
- snowflake/ml/data/ingestor_utils.py +3 -3
- snowflake/ml/data/torch_utils.py +5 -5
- snowflake/ml/dataset/dataset.py +11 -11
- snowflake/ml/dataset/dataset_metadata.py +8 -8
- snowflake/ml/dataset/dataset_reader.py +7 -7
- snowflake/ml/feature_store/__init__.py +1 -1
- snowflake/ml/feature_store/access_manager.py +7 -7
- snowflake/ml/feature_store/entity.py +6 -6
- snowflake/ml/feature_store/examples/airline_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/airline_features/features/plane_features.py +1 -3
- snowflake/ml/feature_store/examples/airline_features/features/weather_features.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/features/station_feature.py +1 -3
- snowflake/ml/feature_store/examples/citibike_trip_features/features/trip_feature.py +1 -3
- snowflake/ml/feature_store/examples/example_helper.py +16 -16
- snowflake/ml/feature_store/examples/new_york_taxi_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/location_features.py +1 -3
- snowflake/ml/feature_store/examples/new_york_taxi_features/features/trip_features.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/entities.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/features/managed_wine_features.py +1 -3
- snowflake/ml/feature_store/examples/wine_quality_features/features/static_wine_features.py +1 -3
- snowflake/ml/feature_store/feature_store.py +52 -64
- snowflake/ml/feature_store/feature_view.py +24 -24
- snowflake/ml/fileset/embedded_stage_fs.py +5 -5
- snowflake/ml/fileset/fileset.py +5 -5
- snowflake/ml/fileset/sfcfs.py +13 -13
- snowflake/ml/fileset/stage_fs.py +15 -15
- snowflake/ml/jobs/_utils/interop_utils.py +10 -10
- snowflake/ml/jobs/_utils/payload_utils.py +6 -16
- snowflake/ml/jobs/_utils/scripts/mljob_launcher.py +7 -4
- snowflake/ml/jobs/_utils/scripts/signal_workers.py +8 -8
- snowflake/ml/jobs/_utils/spec_utils.py +17 -28
- snowflake/ml/jobs/_utils/types.py +2 -2
- snowflake/ml/jobs/decorators.py +4 -5
- snowflake/ml/jobs/job.py +24 -14
- snowflake/ml/jobs/manager.py +37 -41
- snowflake/ml/lineage/lineage_node.py +5 -5
- snowflake/ml/model/_client/model/model_impl.py +3 -3
- snowflake/ml/model/_client/model/model_version_impl.py +103 -35
- snowflake/ml/model/_client/ops/metadata_ops.py +7 -7
- snowflake/ml/model/_client/ops/model_ops.py +41 -41
- snowflake/ml/model/_client/ops/service_ops.py +199 -26
- snowflake/ml/model/_client/service/model_deployment_spec.py +171 -47
- snowflake/ml/model/_client/service/model_deployment_spec_schema.py +44 -24
- snowflake/ml/model/_client/sql/model.py +8 -8
- snowflake/ml/model/_client/sql/model_version.py +26 -26
- snowflake/ml/model/_client/sql/service.py +13 -13
- snowflake/ml/model/_client/sql/stage.py +2 -2
- snowflake/ml/model/_client/sql/tag.py +6 -6
- snowflake/ml/model/_model_composer/model_composer.py +17 -14
- snowflake/ml/model/_model_composer/model_manifest/model_manifest.py +20 -16
- snowflake/ml/model/_model_composer/model_manifest/model_manifest_schema.py +14 -13
- snowflake/ml/model/_model_composer/model_method/model_method.py +3 -3
- snowflake/ml/model/_packager/model_env/model_env.py +28 -25
- snowflake/ml/model/_packager/model_handler.py +4 -4
- snowflake/ml/model/_packager/model_handlers/_base.py +2 -2
- snowflake/ml/model/_packager/model_handlers/_utils.py +15 -3
- snowflake/ml/model/_packager/model_handlers/catboost.py +5 -5
- snowflake/ml/model/_packager/model_handlers/custom.py +8 -4
- snowflake/ml/model/_packager/model_handlers/huggingface_pipeline.py +7 -21
- snowflake/ml/model/_packager/model_handlers/keras.py +4 -4
- snowflake/ml/model/_packager/model_handlers/lightgbm.py +4 -14
- snowflake/ml/model/_packager/model_handlers/mlflow.py +3 -3
- snowflake/ml/model/_packager/model_handlers/pytorch.py +4 -4
- snowflake/ml/model/_packager/model_handlers/sentence_transformers.py +5 -5
- snowflake/ml/model/_packager/model_handlers/sklearn.py +5 -6
- snowflake/ml/model/_packager/model_handlers/snowmlmodel.py +3 -3
- snowflake/ml/model/_packager/model_handlers/tensorflow.py +4 -4
- snowflake/ml/model/_packager/model_handlers/torchscript.py +4 -4
- snowflake/ml/model/_packager/model_handlers/xgboost.py +5 -15
- snowflake/ml/model/_packager/model_meta/model_blob_meta.py +2 -2
- snowflake/ml/model/_packager/model_meta/model_meta.py +37 -37
- snowflake/ml/model/_packager/model_meta/model_meta_schema.py +13 -11
- snowflake/ml/model/_packager/model_meta_migrator/base_migrator.py +3 -3
- snowflake/ml/model/_packager/model_meta_migrator/migrator_plans.py +3 -3
- snowflake/ml/model/_packager/model_meta_migrator/migrator_v1.py +4 -4
- snowflake/ml/model/_packager/model_packager.py +11 -9
- snowflake/ml/model/_packager/model_runtime/_snowml_inference_alternative_requirements.py +32 -1
- snowflake/ml/model/_packager/model_runtime/model_runtime.py +4 -2
- snowflake/ml/model/_signatures/core.py +16 -24
- snowflake/ml/model/_signatures/dmatrix_handler.py +2 -2
- snowflake/ml/model/_signatures/utils.py +6 -6
- snowflake/ml/model/custom_model.py +8 -8
- snowflake/ml/model/model_signature.py +9 -20
- snowflake/ml/model/models/huggingface_pipeline.py +7 -4
- snowflake/ml/model/type_hints.py +3 -3
- snowflake/ml/modeling/_internal/estimator_utils.py +7 -7
- snowflake/ml/modeling/_internal/local_implementations/pandas_handlers.py +6 -6
- snowflake/ml/modeling/_internal/local_implementations/pandas_trainer.py +7 -7
- snowflake/ml/modeling/_internal/model_specifications.py +8 -10
- snowflake/ml/modeling/_internal/model_trainer.py +5 -5
- snowflake/ml/modeling/_internal/model_trainer_builder.py +6 -6
- snowflake/ml/modeling/_internal/snowpark_implementations/distributed_hpo_trainer.py +30 -30
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_handlers.py +13 -13
- snowflake/ml/modeling/_internal/snowpark_implementations/snowpark_trainer.py +31 -31
- snowflake/ml/modeling/_internal/snowpark_implementations/xgboost_external_memory_trainer.py +19 -19
- snowflake/ml/modeling/_internal/transformer_protocols.py +17 -17
- snowflake/ml/modeling/framework/_utils.py +10 -10
- snowflake/ml/modeling/framework/base.py +32 -32
- snowflake/ml/modeling/impute/__init__.py +1 -1
- snowflake/ml/modeling/impute/simple_imputer.py +5 -5
- snowflake/ml/modeling/metrics/__init__.py +1 -1
- snowflake/ml/modeling/metrics/classification.py +39 -39
- snowflake/ml/modeling/metrics/metrics_utils.py +12 -12
- snowflake/ml/modeling/metrics/ranking.py +7 -7
- snowflake/ml/modeling/metrics/regression.py +13 -13
- snowflake/ml/modeling/model_selection/__init__.py +1 -1
- snowflake/ml/modeling/model_selection/grid_search_cv.py +7 -7
- snowflake/ml/modeling/model_selection/randomized_search_cv.py +7 -7
- snowflake/ml/modeling/pipeline/__init__.py +1 -1
- snowflake/ml/modeling/pipeline/pipeline.py +18 -18
- snowflake/ml/modeling/preprocessing/__init__.py +1 -1
- snowflake/ml/modeling/preprocessing/k_bins_discretizer.py +13 -13
- snowflake/ml/modeling/preprocessing/max_abs_scaler.py +4 -4
- snowflake/ml/modeling/preprocessing/min_max_scaler.py +8 -8
- snowflake/ml/modeling/preprocessing/normalizer.py +0 -1
- snowflake/ml/modeling/preprocessing/one_hot_encoder.py +28 -28
- snowflake/ml/modeling/preprocessing/ordinal_encoder.py +9 -9
- snowflake/ml/modeling/preprocessing/robust_scaler.py +7 -7
- snowflake/ml/modeling/preprocessing/standard_scaler.py +5 -5
- snowflake/ml/monitoring/_client/model_monitor_sql_client.py +26 -26
- snowflake/ml/monitoring/_manager/model_monitor_manager.py +5 -5
- snowflake/ml/monitoring/entities/model_monitor_config.py +6 -6
- snowflake/ml/registry/_manager/model_manager.py +33 -31
- snowflake/ml/registry/registry.py +29 -22
- snowflake/ml/utils/authentication.py +2 -2
- snowflake/ml/utils/connection_params.py +5 -5
- snowflake/ml/utils/sparse.py +5 -4
- snowflake/ml/utils/sql_client.py +1 -2
- snowflake/ml/version.py +2 -1
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.3.dist-info}/METADATA +16 -7
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.3.dist-info}/RECORD +164 -166
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.3.dist-info}/WHEEL +1 -1
- snowflake/ml/model/_packager/model_meta/_packaging_requirements.py +0 -1
- snowflake/ml/modeling/_internal/constants.py +0 -2
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.3.dist-info}/licenses/LICENSE.txt +0 -0
- {snowflake_ml_python-1.8.2.dist-info → snowflake_ml_python-1.8.3.dist-info}/top_level.txt +0 -0
@@ -6,31 +6,26 @@ import zipfile
|
|
6
6
|
from contextlib import contextmanager
|
7
7
|
from datetime import datetime
|
8
8
|
from types import ModuleType
|
9
|
-
from typing import Any,
|
9
|
+
from typing import Any, Generator, Optional, TypedDict
|
10
10
|
|
11
11
|
import cloudpickle
|
12
12
|
import yaml
|
13
13
|
from packaging import requirements, version
|
14
14
|
from typing_extensions import Required
|
15
15
|
|
16
|
-
from snowflake.ml
|
16
|
+
from snowflake.ml import version as snowml_version
|
17
|
+
from snowflake.ml._internal import env_utils, file_utils
|
17
18
|
from snowflake.ml.model import model_signature, type_hints as model_types
|
18
19
|
from snowflake.ml.model._packager.model_env import model_env
|
19
|
-
from snowflake.ml.model._packager.model_meta import
|
20
|
-
_packaging_requirements,
|
21
|
-
model_blob_meta,
|
22
|
-
model_meta_schema,
|
23
|
-
)
|
20
|
+
from snowflake.ml.model._packager.model_meta import model_blob_meta, model_meta_schema
|
24
21
|
from snowflake.ml.model._packager.model_meta_migrator import migrator_plans
|
25
22
|
from snowflake.ml.model._packager.model_runtime import model_runtime
|
26
23
|
|
27
24
|
MODEL_METADATA_FILE = "model.yaml"
|
28
25
|
MODEL_CODE_DIR = "code"
|
29
26
|
|
30
|
-
_PACKAGING_REQUIREMENTS = [
|
31
|
-
|
32
|
-
for r in _packaging_requirements.REQUIREMENTS
|
33
|
-
]
|
27
|
+
_PACKAGING_REQUIREMENTS = ["cloudpickle"]
|
28
|
+
|
34
29
|
_SNOWFLAKE_PKG_NAME = "snowflake"
|
35
30
|
_SNOWFLAKE_ML_PKG_NAME = f"{_SNOWFLAKE_PKG_NAME}.ml"
|
36
31
|
|
@@ -41,15 +36,16 @@ def create_model_metadata(
|
|
41
36
|
model_dir_path: str,
|
42
37
|
name: str,
|
43
38
|
model_type: model_types.SupportedModelHandlerType,
|
44
|
-
signatures: Optional[
|
45
|
-
function_properties: Optional[
|
46
|
-
metadata: Optional[
|
47
|
-
code_paths: Optional[
|
48
|
-
ext_modules: Optional[
|
49
|
-
conda_dependencies: Optional[
|
50
|
-
pip_requirements: Optional[
|
51
|
-
artifact_repository_map: Optional[
|
52
|
-
|
39
|
+
signatures: Optional[dict[str, model_signature.ModelSignature]] = None,
|
40
|
+
function_properties: Optional[dict[str, dict[str, Any]]] = None,
|
41
|
+
metadata: Optional[dict[str, str]] = None,
|
42
|
+
code_paths: Optional[list[str]] = None,
|
43
|
+
ext_modules: Optional[list[ModuleType]] = None,
|
44
|
+
conda_dependencies: Optional[list[str]] = None,
|
45
|
+
pip_requirements: Optional[list[str]] = None,
|
46
|
+
artifact_repository_map: Optional[dict[str, str]] = None,
|
47
|
+
resource_constraint: Optional[dict[str, str]] = None,
|
48
|
+
target_platforms: Optional[list[model_types.TargetPlatform]] = None,
|
53
49
|
python_version: Optional[str] = None,
|
54
50
|
task: model_types.Task = model_types.Task.UNKNOWN,
|
55
51
|
**kwargs: Any,
|
@@ -70,6 +66,7 @@ def create_model_metadata(
|
|
70
66
|
conda_dependencies: List of conda requirements for running the model. Defaults to None.
|
71
67
|
pip_requirements: List of pip Python packages requirements for running the model. Defaults to None.
|
72
68
|
artifact_repository_map: A dict mapping from package channel to artifact repository name.
|
69
|
+
resource_constraint: Mapping of resource constraint keys and values, e.g. {"architecture": "x86"}.
|
73
70
|
target_platforms: List of target platforms to run the model.
|
74
71
|
python_version: A string of python version where model is run. Used for user override. If specified as None,
|
75
72
|
current version would be captured. Defaults to None.
|
@@ -108,13 +105,14 @@ def create_model_metadata(
|
|
108
105
|
conda_dependencies=conda_dependencies,
|
109
106
|
pip_requirements=pip_requirements,
|
110
107
|
artifact_repository_map=artifact_repository_map,
|
108
|
+
resource_constraint=resource_constraint,
|
111
109
|
python_version=python_version,
|
112
110
|
embed_local_ml_library=embed_local_ml_library,
|
113
111
|
prefer_pip=prefer_pip,
|
114
112
|
)
|
115
113
|
|
116
114
|
if embed_local_ml_library:
|
117
|
-
env.snowpark_ml_version = f"{
|
115
|
+
env.snowpark_ml_version = f"{snowml_version.VERSION}+{file_utils.hash_directory(path_to_copy)}"
|
118
116
|
|
119
117
|
model_meta = ModelMetadata(
|
120
118
|
name=name,
|
@@ -156,9 +154,10 @@ def create_model_metadata(
|
|
156
154
|
|
157
155
|
def _create_env_for_model_metadata(
|
158
156
|
*,
|
159
|
-
conda_dependencies: Optional[
|
160
|
-
pip_requirements: Optional[
|
161
|
-
artifact_repository_map: Optional[
|
157
|
+
conda_dependencies: Optional[list[str]] = None,
|
158
|
+
pip_requirements: Optional[list[str]] = None,
|
159
|
+
artifact_repository_map: Optional[dict[str, str]] = None,
|
160
|
+
resource_constraint: Optional[dict[str, str]] = None,
|
162
161
|
python_version: Optional[str] = None,
|
163
162
|
embed_local_ml_library: bool = False,
|
164
163
|
prefer_pip: bool = False,
|
@@ -169,8 +168,9 @@ def _create_env_for_model_metadata(
|
|
169
168
|
env.conda_dependencies = conda_dependencies # type: ignore[assignment]
|
170
169
|
env.pip_requirements = pip_requirements # type: ignore[assignment]
|
171
170
|
env.artifact_repository_map = artifact_repository_map
|
171
|
+
env.resource_constraint = resource_constraint
|
172
172
|
env.python_version = python_version # type: ignore[assignment]
|
173
|
-
env.snowpark_ml_version =
|
173
|
+
env.snowpark_ml_version = snowml_version.VERSION
|
174
174
|
|
175
175
|
requirements_to_add = _PACKAGING_REQUIREMENTS
|
176
176
|
|
@@ -242,20 +242,20 @@ class ModelMetadata:
|
|
242
242
|
name: str,
|
243
243
|
env: model_env.ModelEnv,
|
244
244
|
model_type: model_types.SupportedModelHandlerType,
|
245
|
-
runtimes: Optional[
|
246
|
-
signatures: Optional[
|
247
|
-
function_properties: Optional[
|
248
|
-
user_files: Optional[
|
249
|
-
metadata: Optional[
|
245
|
+
runtimes: Optional[dict[str, model_runtime.ModelRuntime]] = None,
|
246
|
+
signatures: Optional[dict[str, model_signature.ModelSignature]] = None,
|
247
|
+
function_properties: Optional[dict[str, dict[str, Any]]] = None,
|
248
|
+
user_files: Optional[dict[str, list[str]]] = None,
|
249
|
+
metadata: Optional[dict[str, str]] = None,
|
250
250
|
creation_timestamp: Optional[str] = None,
|
251
251
|
min_snowpark_ml_version: Optional[str] = None,
|
252
|
-
models: Optional[
|
252
|
+
models: Optional[dict[str, model_blob_meta.ModelBlobMeta]] = None,
|
253
253
|
original_metadata_version: Optional[str] = model_meta_schema.MODEL_METADATA_VERSION,
|
254
254
|
task: model_types.Task = model_types.Task.UNKNOWN,
|
255
255
|
explain_algorithm: Optional[model_meta_schema.ModelExplainAlgorithm] = None,
|
256
256
|
) -> None:
|
257
257
|
self.name = name
|
258
|
-
self.signatures:
|
258
|
+
self.signatures: dict[str, model_signature.ModelSignature] = dict()
|
259
259
|
if signatures:
|
260
260
|
self.signatures = signatures
|
261
261
|
self.function_properties = function_properties or {}
|
@@ -270,7 +270,7 @@ class ModelMetadata:
|
|
270
270
|
else model_meta_schema.MODEL_METADATA_MIN_SNOWPARK_ML_VERSION
|
271
271
|
)
|
272
272
|
|
273
|
-
self.models:
|
273
|
+
self.models: dict[str, model_blob_meta.ModelBlobMeta] = dict()
|
274
274
|
if models:
|
275
275
|
self.models = models
|
276
276
|
|
@@ -291,7 +291,7 @@ class ModelMetadata:
|
|
291
291
|
self._min_snowpark_ml_version = max(self._min_snowpark_ml_version, parsed_min_snowpark_ml_version)
|
292
292
|
|
293
293
|
@property
|
294
|
-
def runtimes(self) ->
|
294
|
+
def runtimes(self) -> dict[str, model_runtime.ModelRuntime]:
|
295
295
|
if self._runtimes and "cpu" in self._runtimes:
|
296
296
|
return self._runtimes
|
297
297
|
runtimes = {
|
@@ -358,11 +358,11 @@ class ModelMetadata:
|
|
358
358
|
|
359
359
|
loaded_meta_min_snowpark_ml_version = loaded_meta.get("min_snowpark_ml_version", None)
|
360
360
|
if not loaded_meta_min_snowpark_ml_version or (
|
361
|
-
version.parse(loaded_meta_min_snowpark_ml_version) > version.parse(
|
361
|
+
version.parse(loaded_meta_min_snowpark_ml_version) > version.parse(snowml_version.VERSION)
|
362
362
|
):
|
363
363
|
raise RuntimeError(
|
364
364
|
f"The minimal version required to load the model is {loaded_meta_min_snowpark_ml_version}, "
|
365
|
-
f"while current version of Snowpark ML library is {
|
365
|
+
f"while current version of Snowpark ML library is {snowml_version.VERSION}."
|
366
366
|
)
|
367
367
|
return model_meta_schema.ModelMetadataDict(
|
368
368
|
creation_timestamp=loaded_meta["creation_timestamp"],
|
@@ -405,7 +405,7 @@ class ModelMetadata:
|
|
405
405
|
env = model_env.ModelEnv()
|
406
406
|
env.load_from_dict(pathlib.Path(model_dir_path), model_dict["env"])
|
407
407
|
|
408
|
-
runtimes: Optional[
|
408
|
+
runtimes: Optional[dict[str, model_runtime.ModelRuntime]]
|
409
409
|
if model_dict.get("runtimes", None):
|
410
410
|
runtimes = {
|
411
411
|
name: model_runtime.ModelRuntime.load(pathlib.Path(model_dir_path), name, env, runtime_dict)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# This files contains schema definition of what will be written into model.yml
|
2
2
|
# Changing this file should lead to a change of the schema version.
|
3
3
|
from enum import Enum
|
4
|
-
from typing import Any,
|
4
|
+
from typing import Any, Optional, TypedDict, Union
|
5
5
|
|
6
6
|
from typing_extensions import NotRequired, Required
|
7
7
|
|
@@ -18,18 +18,20 @@ class FunctionProperties(Enum):
|
|
18
18
|
class ModelRuntimeDependenciesDict(TypedDict):
|
19
19
|
conda: Required[str]
|
20
20
|
pip: Required[str]
|
21
|
-
artifact_repository_map: NotRequired[Optional[
|
21
|
+
artifact_repository_map: NotRequired[Optional[dict[str, str]]]
|
22
22
|
|
23
23
|
|
24
24
|
class ModelRuntimeDict(TypedDict):
|
25
|
-
imports: Required[
|
25
|
+
imports: Required[list[str]]
|
26
26
|
dependencies: Required[ModelRuntimeDependenciesDict]
|
27
|
+
resource_constraint: NotRequired[Optional[dict[str, str]]]
|
27
28
|
|
28
29
|
|
29
30
|
class ModelEnvDict(TypedDict):
|
30
31
|
conda: Required[str]
|
31
32
|
pip: Required[str]
|
32
|
-
artifact_repository_map: NotRequired[Optional[
|
33
|
+
artifact_repository_map: NotRequired[Optional[dict[str, str]]]
|
34
|
+
resource_constraint: NotRequired[Optional[dict[str, str]]]
|
33
35
|
python_version: Required[str]
|
34
36
|
cuda_version: NotRequired[Optional[str]]
|
35
37
|
snowpark_ml_version: Required[str]
|
@@ -102,25 +104,25 @@ class ModelBlobMetadataDict(TypedDict):
|
|
102
104
|
model_type: Required[type_hints.SupportedModelHandlerType]
|
103
105
|
path: Required[str]
|
104
106
|
handler_version: Required[str]
|
105
|
-
function_properties: NotRequired[
|
106
|
-
artifacts: NotRequired[
|
107
|
+
function_properties: NotRequired[dict[str, dict[str, Any]]]
|
108
|
+
artifacts: NotRequired[dict[str, str]]
|
107
109
|
options: NotRequired[ModelBlobOptions]
|
108
110
|
|
109
111
|
|
110
112
|
class ModelMetadataDict(TypedDict):
|
111
113
|
creation_timestamp: Required[str]
|
112
114
|
env: Required[ModelEnvDict]
|
113
|
-
runtimes: NotRequired[
|
114
|
-
metadata: NotRequired[Optional[
|
115
|
+
runtimes: NotRequired[dict[str, ModelRuntimeDict]]
|
116
|
+
metadata: NotRequired[Optional[dict[str, str]]]
|
115
117
|
model_type: Required[type_hints.SupportedModelHandlerType]
|
116
|
-
models: Required[
|
118
|
+
models: Required[dict[str, ModelBlobMetadataDict]]
|
117
119
|
name: Required[str]
|
118
|
-
signatures: Required[
|
120
|
+
signatures: Required[dict[str, dict[str, Any]]]
|
119
121
|
version: Required[str]
|
120
122
|
min_snowpark_ml_version: Required[str]
|
121
123
|
task: Required[str]
|
122
124
|
explainability: NotRequired[Optional[ExplainabilityMetadataDict]]
|
123
|
-
function_properties: NotRequired[
|
125
|
+
function_properties: NotRequired[dict[str, dict[str, Any]]]
|
124
126
|
|
125
127
|
|
126
128
|
class ModelExplainAlgorithm(Enum):
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import copy
|
2
2
|
from abc import abstractmethod
|
3
|
-
from typing import Any,
|
3
|
+
from typing import Any, Protocol, final
|
4
4
|
|
5
5
|
from snowflake.ml._internal import migrator_utils
|
6
6
|
|
@@ -11,13 +11,13 @@ class _BaseModelMetaMigratorProtocol(Protocol):
|
|
11
11
|
|
12
12
|
@staticmethod
|
13
13
|
@abstractmethod
|
14
|
-
def upgrade(original_meta_dict:
|
14
|
+
def upgrade(original_meta_dict: dict[str, Any]) -> dict[str, Any]:
|
15
15
|
raise NotImplementedError
|
16
16
|
|
17
17
|
|
18
18
|
class BaseModelMetaMigrator(_BaseModelMetaMigratorProtocol):
|
19
19
|
@final
|
20
|
-
def try_upgrade(self, original_meta_dict:
|
20
|
+
def try_upgrade(self, original_meta_dict: dict[str, Any]) -> dict[str, Any]:
|
21
21
|
loaded_meta_version = original_meta_dict.get("version", None)
|
22
22
|
if not loaded_meta_version or str(loaded_meta_version) != self.source_version:
|
23
23
|
raise NotImplementedError(
|
@@ -1,14 +1,14 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Any
|
2
2
|
|
3
3
|
from snowflake.ml.model._packager.model_meta import model_meta_schema
|
4
4
|
from snowflake.ml.model._packager.model_meta_migrator import base_migrator, migrator_v1
|
5
5
|
|
6
|
-
MODEL_META_MIGRATOR_PLANS:
|
6
|
+
MODEL_META_MIGRATOR_PLANS: dict[str, type[base_migrator.BaseModelMetaMigrator]] = {
|
7
7
|
"1": migrator_v1.MetaMigrator_v1,
|
8
8
|
}
|
9
9
|
|
10
10
|
|
11
|
-
def migrate_metadata(loaded_meta:
|
11
|
+
def migrate_metadata(loaded_meta: dict[str, Any]) -> dict[str, Any]:
|
12
12
|
loaded_meta_version = str(loaded_meta.get("version", None))
|
13
13
|
while loaded_meta_version != model_meta_schema.MODEL_METADATA_VERSION:
|
14
14
|
if loaded_meta_version not in MODEL_META_MIGRATOR_PLANS.keys():
|
@@ -1,8 +1,8 @@
|
|
1
|
-
from typing import Any
|
1
|
+
from typing import Any
|
2
2
|
|
3
3
|
from packaging import requirements, version
|
4
4
|
|
5
|
-
from snowflake.ml
|
5
|
+
from snowflake.ml import version as snowml_version
|
6
6
|
from snowflake.ml.model._packager.model_meta_migrator import base_migrator
|
7
7
|
|
8
8
|
|
@@ -11,7 +11,7 @@ class MetaMigrator_v1(base_migrator.BaseModelMetaMigrator):
|
|
11
11
|
target_version = "2023-12-01"
|
12
12
|
|
13
13
|
@staticmethod
|
14
|
-
def upgrade(original_meta_dict:
|
14
|
+
def upgrade(original_meta_dict: dict[str, Any]) -> dict[str, Any]:
|
15
15
|
loaded_python_version = version.parse(original_meta_dict["python_version"])
|
16
16
|
if original_meta_dict.get("local_ml_library_version", None):
|
17
17
|
loaded_lib_version = str(version.parse(original_meta_dict["local_ml_library_version"]))
|
@@ -24,7 +24,7 @@ class MetaMigrator_v1(base_migrator.BaseModelMetaMigrator):
|
|
24
24
|
None,
|
25
25
|
)
|
26
26
|
if lib_spec_str is None:
|
27
|
-
loaded_lib_version =
|
27
|
+
loaded_lib_version = snowml_version.VERSION
|
28
28
|
loaded_lib_version = list(requirements.Requirement(str(lib_spec_str)).specifier)[0].version
|
29
29
|
|
30
30
|
return dict(
|
@@ -1,6 +1,6 @@
|
|
1
1
|
import os
|
2
2
|
from types import ModuleType
|
3
|
-
from typing import
|
3
|
+
from typing import Optional
|
4
4
|
|
5
5
|
from absl import logging
|
6
6
|
|
@@ -38,16 +38,17 @@ class ModelPackager:
|
|
38
38
|
*,
|
39
39
|
name: str,
|
40
40
|
model: model_types.SupportedModelType,
|
41
|
-
signatures: Optional[
|
41
|
+
signatures: Optional[dict[str, model_signature.ModelSignature]] = None,
|
42
42
|
sample_input_data: Optional[model_types.SupportedDataType] = None,
|
43
|
-
metadata: Optional[
|
44
|
-
conda_dependencies: Optional[
|
45
|
-
pip_requirements: Optional[
|
46
|
-
artifact_repository_map: Optional[
|
47
|
-
|
43
|
+
metadata: Optional[dict[str, str]] = None,
|
44
|
+
conda_dependencies: Optional[list[str]] = None,
|
45
|
+
pip_requirements: Optional[list[str]] = None,
|
46
|
+
artifact_repository_map: Optional[dict[str, str]] = None,
|
47
|
+
resource_constraint: Optional[dict[str, str]] = None,
|
48
|
+
target_platforms: Optional[list[model_types.TargetPlatform]] = None,
|
48
49
|
python_version: Optional[str] = None,
|
49
|
-
ext_modules: Optional[
|
50
|
-
code_paths: Optional[
|
50
|
+
ext_modules: Optional[list[ModuleType]] = None,
|
51
|
+
code_paths: Optional[list[str]] = None,
|
51
52
|
options: model_types.ModelSaveOption,
|
52
53
|
task: model_types.Task = model_types.Task.UNKNOWN,
|
53
54
|
) -> model_meta.ModelMetadata:
|
@@ -76,6 +77,7 @@ class ModelPackager:
|
|
76
77
|
conda_dependencies=conda_dependencies,
|
77
78
|
pip_requirements=pip_requirements,
|
78
79
|
artifact_repository_map=artifact_repository_map,
|
80
|
+
resource_constraint=resource_constraint,
|
79
81
|
python_version=python_version,
|
80
82
|
task=task,
|
81
83
|
target_platforms=target_platforms,
|
@@ -1 +1,32 @@
|
|
1
|
-
|
1
|
+
# DO NOT EDIT!
|
2
|
+
# Generate by running 'bazel run --config=pre_build //bazel/requirements:sync_requirements'
|
3
|
+
|
4
|
+
REQUIREMENTS = [
|
5
|
+
"absl-py>=0.15,<2",
|
6
|
+
"aiohttp!=4.0.0a0, !=4.0.0a1",
|
7
|
+
"anyio>=3.5.0,<5",
|
8
|
+
"cachetools>=3.1.1,<6",
|
9
|
+
"cloudpickle>=2.0.0,<3",
|
10
|
+
"cryptography",
|
11
|
+
"fsspec>=2024.6.1,<2026",
|
12
|
+
"importlib_resources>=6.1.1, <7",
|
13
|
+
"numpy>=1.23,<2",
|
14
|
+
"packaging>=20.9,<25",
|
15
|
+
"pandas>=1.0.0,<3",
|
16
|
+
"pyarrow",
|
17
|
+
"pydantic>=2.8.2, <3",
|
18
|
+
"pyjwt>=2.0.0, <3",
|
19
|
+
"pytimeparse>=1.1.8,<2",
|
20
|
+
"pyyaml>=6.0,<7",
|
21
|
+
"requests",
|
22
|
+
"retrying>=1.3.3,<2",
|
23
|
+
"s3fs>=2024.6.1,<2026",
|
24
|
+
"scikit-learn>=1.4,<1.6",
|
25
|
+
"scipy>=1.9,<2",
|
26
|
+
"snowflake-connector-python>=3.12.0,<4",
|
27
|
+
"snowflake-snowpark-python>=1.17.0,<2,!=1.26.0",
|
28
|
+
"snowflake.core>=1.0.2,<2",
|
29
|
+
"sqlparse>=0.4,<1",
|
30
|
+
"typing-extensions>=4.1.0,<5",
|
31
|
+
"xgboost>=1.7.3,<3",
|
32
|
+
]
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import copy
|
2
2
|
import pathlib
|
3
3
|
import warnings
|
4
|
-
from typing import
|
4
|
+
from typing import Optional
|
5
5
|
|
6
6
|
from packaging import requirements
|
7
7
|
|
@@ -37,7 +37,7 @@ class ModelRuntime:
|
|
37
37
|
self,
|
38
38
|
name: str,
|
39
39
|
env: model_env.ModelEnv,
|
40
|
-
imports: Optional[
|
40
|
+
imports: Optional[list[str]] = None,
|
41
41
|
is_warehouse: bool = False,
|
42
42
|
is_gpu: bool = False,
|
43
43
|
loading_from_file: bool = False,
|
@@ -102,6 +102,7 @@ class ModelRuntime:
|
|
102
102
|
if env_dict.get("artifact_repository_map") is not None
|
103
103
|
else {},
|
104
104
|
),
|
105
|
+
resource_constraint=env_dict["resource_constraint"],
|
105
106
|
)
|
106
107
|
|
107
108
|
@staticmethod
|
@@ -116,6 +117,7 @@ class ModelRuntime:
|
|
116
117
|
env.cuda_version = meta_env.cuda_version
|
117
118
|
env.snowpark_ml_version = meta_env.snowpark_ml_version
|
118
119
|
env.artifact_repository_map = meta_env.artifact_repository_map
|
120
|
+
env.resource_constraint = meta_env.resource_constraint
|
119
121
|
|
120
122
|
conda_env_rel_path = pathlib.PurePosixPath(loaded_dict["dependencies"]["conda"])
|
121
123
|
pip_requirements_rel_path = pathlib.PurePosixPath(loaded_dict["dependencies"]["pip"])
|
@@ -6,12 +6,8 @@ from typing import (
|
|
6
6
|
TYPE_CHECKING,
|
7
7
|
Any,
|
8
8
|
Callable,
|
9
|
-
Dict,
|
10
|
-
List,
|
11
9
|
Optional,
|
12
10
|
Sequence,
|
13
|
-
Tuple,
|
14
|
-
Type,
|
15
11
|
Union,
|
16
12
|
final,
|
17
13
|
get_args,
|
@@ -48,7 +44,7 @@ PandasExtensionTypes = Union[
|
|
48
44
|
|
49
45
|
|
50
46
|
class DataType(Enum):
|
51
|
-
def __init__(self, value: str, snowpark_type:
|
47
|
+
def __init__(self, value: str, snowpark_type: type[spt.DataType], numpy_type: npt.DTypeLike) -> None:
|
52
48
|
self._value = value
|
53
49
|
self._snowpark_type = snowpark_type
|
54
50
|
self._numpy_type = numpy_type
|
@@ -159,7 +155,7 @@ class DataType(Enum):
|
|
159
155
|
else:
|
160
156
|
actual_sp_type = snowpark_type
|
161
157
|
|
162
|
-
snowpark_to_snowml_type_mapping:
|
158
|
+
snowpark_to_snowml_type_mapping: dict[type[spt.DataType], DataType] = {
|
163
159
|
i._snowpark_type: i
|
164
160
|
for i in DataType
|
165
161
|
# We by default infer as signed integer.
|
@@ -199,7 +195,7 @@ class DataType(Enum):
|
|
199
195
|
class BaseFeatureSpec(ABC):
|
200
196
|
"""Abstract Class for specification of a feature."""
|
201
197
|
|
202
|
-
def __init__(self, name: str, shape: Optional[
|
198
|
+
def __init__(self, name: str, shape: Optional[tuple[int, ...]]) -> None:
|
203
199
|
self._name = name
|
204
200
|
|
205
201
|
if shape and not isinstance(shape, tuple):
|
@@ -218,23 +214,19 @@ class BaseFeatureSpec(ABC):
|
|
218
214
|
@abstractmethod
|
219
215
|
def as_snowpark_type(self) -> spt.DataType:
|
220
216
|
"""Convert to corresponding Snowpark Type."""
|
221
|
-
pass
|
222
217
|
|
223
218
|
@abstractmethod
|
224
219
|
def as_dtype(self, force_numpy_dtype: bool = False) -> Union[npt.DTypeLike, str, PandasExtensionTypes]:
|
225
220
|
"""Convert to corresponding local Type."""
|
226
|
-
pass
|
227
221
|
|
228
222
|
@abstractmethod
|
229
|
-
def to_dict(self) ->
|
223
|
+
def to_dict(self) -> dict[str, Any]:
|
230
224
|
"""Serialization"""
|
231
|
-
pass
|
232
225
|
|
233
226
|
@classmethod
|
234
227
|
@abstractmethod
|
235
|
-
def from_dict(self, input_dict:
|
228
|
+
def from_dict(self, input_dict: dict[str, Any]) -> "BaseFeatureSpec":
|
236
229
|
"""Deserialization"""
|
237
|
-
pass
|
238
230
|
|
239
231
|
|
240
232
|
class FeatureSpec(BaseFeatureSpec):
|
@@ -244,7 +236,7 @@ class FeatureSpec(BaseFeatureSpec):
|
|
244
236
|
self,
|
245
237
|
name: str,
|
246
238
|
dtype: DataType,
|
247
|
-
shape: Optional[
|
239
|
+
shape: Optional[tuple[int, ...]] = None,
|
248
240
|
nullable: bool = True,
|
249
241
|
) -> None:
|
250
242
|
"""
|
@@ -330,19 +322,19 @@ class FeatureSpec(BaseFeatureSpec):
|
|
330
322
|
f"name={repr(self._name)}{shape_str}, nullable={repr(self._nullable)})"
|
331
323
|
)
|
332
324
|
|
333
|
-
def to_dict(self) ->
|
325
|
+
def to_dict(self) -> dict[str, Any]:
|
334
326
|
"""Serialize the feature group into a dict.
|
335
327
|
|
336
328
|
Returns:
|
337
329
|
A dict that serializes the feature group.
|
338
330
|
"""
|
339
|
-
base_dict:
|
331
|
+
base_dict: dict[str, Any] = {"type": self._dtype.name, "name": self._name, "nullable": self._nullable}
|
340
332
|
if self._shape is not None:
|
341
333
|
base_dict["shape"] = self._shape
|
342
334
|
return base_dict
|
343
335
|
|
344
336
|
@classmethod
|
345
|
-
def from_dict(cls, input_dict:
|
337
|
+
def from_dict(cls, input_dict: dict[str, Any]) -> "FeatureSpec":
|
346
338
|
"""Deserialize the feature specification from a dict.
|
347
339
|
|
348
340
|
Args:
|
@@ -391,7 +383,7 @@ class FeatureSpec(BaseFeatureSpec):
|
|
391
383
|
class FeatureGroupSpec(BaseFeatureSpec):
|
392
384
|
"""Specification of a group of features in Snowflake native model packaging."""
|
393
385
|
|
394
|
-
def __init__(self, name: str, specs:
|
386
|
+
def __init__(self, name: str, specs: list[BaseFeatureSpec], shape: Optional[tuple[int, ...]] = None) -> None:
|
395
387
|
"""Initialize a feature group.
|
396
388
|
|
397
389
|
Args:
|
@@ -458,19 +450,19 @@ class FeatureGroupSpec(BaseFeatureSpec):
|
|
458
450
|
def as_dtype(self, force_numpy_dtype: bool = False) -> Union[npt.DTypeLike, str, PandasExtensionTypes]:
|
459
451
|
return np.object_
|
460
452
|
|
461
|
-
def to_dict(self) ->
|
453
|
+
def to_dict(self) -> dict[str, Any]:
|
462
454
|
"""Serialize the feature group into a dict.
|
463
455
|
|
464
456
|
Returns:
|
465
457
|
A dict that serializes the feature group.
|
466
458
|
"""
|
467
|
-
base_dict:
|
459
|
+
base_dict: dict[str, Any] = {"name": self._name, "specs": [s.to_dict() for s in self._specs]}
|
468
460
|
if self._shape is not None:
|
469
461
|
base_dict["shape"] = self._shape
|
470
462
|
return base_dict
|
471
463
|
|
472
464
|
@classmethod
|
473
|
-
def from_dict(cls, input_dict:
|
465
|
+
def from_dict(cls, input_dict: dict[str, Any]) -> "FeatureGroupSpec":
|
474
466
|
"""Deserialize the feature group from a dict.
|
475
467
|
|
476
468
|
Args:
|
@@ -520,7 +512,7 @@ class ModelSignature:
|
|
520
512
|
else:
|
521
513
|
return False
|
522
514
|
|
523
|
-
def to_dict(self) ->
|
515
|
+
def to_dict(self) -> dict[str, Any]:
|
524
516
|
"""Generate a dict to represent the whole signature.
|
525
517
|
|
526
518
|
Returns:
|
@@ -533,7 +525,7 @@ class ModelSignature:
|
|
533
525
|
}
|
534
526
|
|
535
527
|
@classmethod
|
536
|
-
def from_dict(cls, loaded:
|
528
|
+
def from_dict(cls, loaded: dict[str, Any]) -> "ModelSignature":
|
537
529
|
"""Create a signature given the dict containing specifications of children features and feature groups.
|
538
530
|
|
539
531
|
Args:
|
@@ -545,7 +537,7 @@ class ModelSignature:
|
|
545
537
|
sig_outs = loaded["outputs"]
|
546
538
|
sig_inputs = loaded["inputs"]
|
547
539
|
|
548
|
-
deserialize_spec: Callable[[
|
540
|
+
deserialize_spec: Callable[[dict[str, Any]], BaseFeatureSpec] = lambda sig_spec: (
|
549
541
|
FeatureGroupSpec.from_dict(sig_spec) if "specs" in sig_spec else FeatureSpec.from_dict(sig_spec)
|
550
542
|
)
|
551
543
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import TYPE_CHECKING,
|
1
|
+
from typing import TYPE_CHECKING, Literal, Optional, Sequence
|
2
2
|
|
3
3
|
import numpy as np
|
4
4
|
import pandas as pd
|
@@ -45,7 +45,7 @@ class XGBoostDMatrixHandler(base_handler.BaseDataHandler["xgboost.DMatrix"]):
|
|
45
45
|
@staticmethod
|
46
46
|
def infer_signature(data: "xgboost.DMatrix", role: Literal["input", "output"]) -> Sequence[core.BaseFeatureSpec]:
|
47
47
|
feature_prefix = f"{XGBoostDMatrixHandler.FEATURE_PREFIX}_"
|
48
|
-
features:
|
48
|
+
features: list[core.BaseFeatureSpec] = []
|
49
49
|
role_prefix = (
|
50
50
|
XGBoostDMatrixHandler.INPUT_PREFIX if role == "input" else XGBoostDMatrixHandler.OUTPUT_PREFIX
|
51
51
|
) + "_"
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import warnings
|
2
|
-
from typing import Any,
|
2
|
+
from typing import Any, Optional, Sequence
|
3
3
|
|
4
4
|
import numpy as np
|
5
5
|
import numpy.typing as npt
|
@@ -12,7 +12,7 @@ from snowflake.ml._internal.exceptions import (
|
|
12
12
|
from snowflake.ml.model._signatures import core
|
13
13
|
|
14
14
|
|
15
|
-
def convert_list_to_ndarray(data:
|
15
|
+
def convert_list_to_ndarray(data: list[Any]) -> npt.NDArray[Any]:
|
16
16
|
"""Create a numpy array from list or nested list. Avoid ragged list and unaligned types.
|
17
17
|
|
18
18
|
Args:
|
@@ -49,7 +49,7 @@ def convert_list_to_ndarray(data: List[Any]) -> npt.NDArray[Any]:
|
|
49
49
|
|
50
50
|
|
51
51
|
def rename_features(
|
52
|
-
features: Sequence[core.BaseFeatureSpec], feature_names: Optional[
|
52
|
+
features: Sequence[core.BaseFeatureSpec], feature_names: Optional[list[str]] = None
|
53
53
|
) -> Sequence[core.BaseFeatureSpec]:
|
54
54
|
"""It renames the feature in features provided optional feature names.
|
55
55
|
|
@@ -104,7 +104,7 @@ def rename_pandas_df(data: pd.DataFrame, features: Sequence[core.BaseFeatureSpec
|
|
104
104
|
return data
|
105
105
|
|
106
106
|
|
107
|
-
def huggingface_pipeline_signature_auto_infer(task: str, params:
|
107
|
+
def huggingface_pipeline_signature_auto_infer(task: str, params: dict[str, Any]) -> Optional[core.ModelSignature]:
|
108
108
|
# Text
|
109
109
|
|
110
110
|
# https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.ConversationalPipeline
|
@@ -351,7 +351,7 @@ def series_dropna(series: pd.Series) -> pd.Series:
|
|
351
351
|
return series.dropna(inplace=False).reset_index(drop=True).convert_dtypes()
|
352
352
|
|
353
353
|
|
354
|
-
def infer_list(name: str, data:
|
354
|
+
def infer_list(name: str, data: list[Any]) -> core.BaseFeatureSpec:
|
355
355
|
"""Infer the feature specification from a list.
|
356
356
|
|
357
357
|
Args:
|
@@ -382,7 +382,7 @@ def infer_list(name: str, data: List[Any]) -> core.BaseFeatureSpec:
|
|
382
382
|
return core.FeatureSpec(name=name, dtype=arr_dtype, shape=arr.shape)
|
383
383
|
|
384
384
|
|
385
|
-
def infer_dict(name: str, data:
|
385
|
+
def infer_dict(name: str, data: dict[str, Any]) -> core.FeatureGroupSpec:
|
386
386
|
"""Infer the feature specification from a dictionary.
|
387
387
|
|
388
388
|
Args:
|