genesis-flow 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- genesis_flow-1.0.0.dist-info/METADATA +822 -0
- genesis_flow-1.0.0.dist-info/RECORD +645 -0
- genesis_flow-1.0.0.dist-info/WHEEL +5 -0
- genesis_flow-1.0.0.dist-info/entry_points.txt +19 -0
- genesis_flow-1.0.0.dist-info/licenses/LICENSE.txt +202 -0
- genesis_flow-1.0.0.dist-info/top_level.txt +1 -0
- mlflow/__init__.py +367 -0
- mlflow/__main__.py +3 -0
- mlflow/ag2/__init__.py +56 -0
- mlflow/ag2/ag2_logger.py +294 -0
- mlflow/anthropic/__init__.py +40 -0
- mlflow/anthropic/autolog.py +129 -0
- mlflow/anthropic/chat.py +144 -0
- mlflow/artifacts/__init__.py +268 -0
- mlflow/autogen/__init__.py +144 -0
- mlflow/autogen/chat.py +142 -0
- mlflow/azure/__init__.py +26 -0
- mlflow/azure/auth_handler.py +257 -0
- mlflow/azure/client.py +319 -0
- mlflow/azure/config.py +120 -0
- mlflow/azure/connection_factory.py +340 -0
- mlflow/azure/exceptions.py +27 -0
- mlflow/azure/stores.py +327 -0
- mlflow/azure/utils.py +183 -0
- mlflow/bedrock/__init__.py +45 -0
- mlflow/bedrock/_autolog.py +202 -0
- mlflow/bedrock/chat.py +122 -0
- mlflow/bedrock/stream.py +160 -0
- mlflow/bedrock/utils.py +43 -0
- mlflow/cli.py +707 -0
- mlflow/client.py +12 -0
- mlflow/config/__init__.py +56 -0
- mlflow/crewai/__init__.py +79 -0
- mlflow/crewai/autolog.py +253 -0
- mlflow/crewai/chat.py +29 -0
- mlflow/data/__init__.py +75 -0
- mlflow/data/artifact_dataset_sources.py +170 -0
- mlflow/data/code_dataset_source.py +40 -0
- mlflow/data/dataset.py +123 -0
- mlflow/data/dataset_registry.py +168 -0
- mlflow/data/dataset_source.py +110 -0
- mlflow/data/dataset_source_registry.py +219 -0
- mlflow/data/delta_dataset_source.py +167 -0
- mlflow/data/digest_utils.py +108 -0
- mlflow/data/evaluation_dataset.py +562 -0
- mlflow/data/filesystem_dataset_source.py +81 -0
- mlflow/data/http_dataset_source.py +145 -0
- mlflow/data/huggingface_dataset.py +258 -0
- mlflow/data/huggingface_dataset_source.py +118 -0
- mlflow/data/meta_dataset.py +104 -0
- mlflow/data/numpy_dataset.py +223 -0
- mlflow/data/pandas_dataset.py +231 -0
- mlflow/data/polars_dataset.py +352 -0
- mlflow/data/pyfunc_dataset_mixin.py +31 -0
- mlflow/data/schema.py +76 -0
- mlflow/data/sources.py +1 -0
- mlflow/data/spark_dataset.py +406 -0
- mlflow/data/spark_dataset_source.py +74 -0
- mlflow/data/spark_delta_utils.py +118 -0
- mlflow/data/tensorflow_dataset.py +350 -0
- mlflow/data/uc_volume_dataset_source.py +81 -0
- mlflow/db.py +27 -0
- mlflow/dspy/__init__.py +17 -0
- mlflow/dspy/autolog.py +197 -0
- mlflow/dspy/callback.py +398 -0
- mlflow/dspy/constant.py +1 -0
- mlflow/dspy/load.py +93 -0
- mlflow/dspy/save.py +393 -0
- mlflow/dspy/util.py +109 -0
- mlflow/dspy/wrapper.py +226 -0
- mlflow/entities/__init__.py +104 -0
- mlflow/entities/_mlflow_object.py +52 -0
- mlflow/entities/assessment.py +545 -0
- mlflow/entities/assessment_error.py +80 -0
- mlflow/entities/assessment_source.py +141 -0
- mlflow/entities/dataset.py +92 -0
- mlflow/entities/dataset_input.py +51 -0
- mlflow/entities/dataset_summary.py +62 -0
- mlflow/entities/document.py +48 -0
- mlflow/entities/experiment.py +109 -0
- mlflow/entities/experiment_tag.py +35 -0
- mlflow/entities/file_info.py +45 -0
- mlflow/entities/input_tag.py +35 -0
- mlflow/entities/lifecycle_stage.py +35 -0
- mlflow/entities/logged_model.py +228 -0
- mlflow/entities/logged_model_input.py +26 -0
- mlflow/entities/logged_model_output.py +32 -0
- mlflow/entities/logged_model_parameter.py +46 -0
- mlflow/entities/logged_model_status.py +74 -0
- mlflow/entities/logged_model_tag.py +33 -0
- mlflow/entities/metric.py +200 -0
- mlflow/entities/model_registry/__init__.py +29 -0
- mlflow/entities/model_registry/_model_registry_entity.py +13 -0
- mlflow/entities/model_registry/model_version.py +243 -0
- mlflow/entities/model_registry/model_version_deployment_job_run_state.py +44 -0
- mlflow/entities/model_registry/model_version_deployment_job_state.py +70 -0
- mlflow/entities/model_registry/model_version_search.py +25 -0
- mlflow/entities/model_registry/model_version_stages.py +25 -0
- mlflow/entities/model_registry/model_version_status.py +35 -0
- mlflow/entities/model_registry/model_version_tag.py +35 -0
- mlflow/entities/model_registry/prompt.py +73 -0
- mlflow/entities/model_registry/prompt_version.py +244 -0
- mlflow/entities/model_registry/registered_model.py +175 -0
- mlflow/entities/model_registry/registered_model_alias.py +35 -0
- mlflow/entities/model_registry/registered_model_deployment_job_state.py +39 -0
- mlflow/entities/model_registry/registered_model_search.py +25 -0
- mlflow/entities/model_registry/registered_model_tag.py +35 -0
- mlflow/entities/multipart_upload.py +74 -0
- mlflow/entities/param.py +49 -0
- mlflow/entities/run.py +97 -0
- mlflow/entities/run_data.py +84 -0
- mlflow/entities/run_info.py +188 -0
- mlflow/entities/run_inputs.py +59 -0
- mlflow/entities/run_outputs.py +43 -0
- mlflow/entities/run_status.py +41 -0
- mlflow/entities/run_tag.py +36 -0
- mlflow/entities/source_type.py +31 -0
- mlflow/entities/span.py +774 -0
- mlflow/entities/span_event.py +96 -0
- mlflow/entities/span_status.py +102 -0
- mlflow/entities/trace.py +317 -0
- mlflow/entities/trace_data.py +71 -0
- mlflow/entities/trace_info.py +220 -0
- mlflow/entities/trace_info_v2.py +162 -0
- mlflow/entities/trace_location.py +173 -0
- mlflow/entities/trace_state.py +39 -0
- mlflow/entities/trace_status.py +68 -0
- mlflow/entities/view_type.py +51 -0
- mlflow/environment_variables.py +866 -0
- mlflow/evaluation/__init__.py +16 -0
- mlflow/evaluation/assessment.py +369 -0
- mlflow/evaluation/evaluation.py +411 -0
- mlflow/evaluation/evaluation_tag.py +61 -0
- mlflow/evaluation/fluent.py +48 -0
- mlflow/evaluation/utils.py +201 -0
- mlflow/exceptions.py +213 -0
- mlflow/experiments.py +140 -0
- mlflow/gemini/__init__.py +81 -0
- mlflow/gemini/autolog.py +186 -0
- mlflow/gemini/chat.py +261 -0
- mlflow/genai/__init__.py +71 -0
- mlflow/genai/datasets/__init__.py +67 -0
- mlflow/genai/datasets/evaluation_dataset.py +131 -0
- mlflow/genai/evaluation/__init__.py +3 -0
- mlflow/genai/evaluation/base.py +411 -0
- mlflow/genai/evaluation/constant.py +23 -0
- mlflow/genai/evaluation/utils.py +244 -0
- mlflow/genai/judges/__init__.py +21 -0
- mlflow/genai/judges/databricks.py +404 -0
- mlflow/genai/label_schemas/__init__.py +153 -0
- mlflow/genai/label_schemas/label_schemas.py +209 -0
- mlflow/genai/labeling/__init__.py +159 -0
- mlflow/genai/labeling/labeling.py +250 -0
- mlflow/genai/optimize/__init__.py +13 -0
- mlflow/genai/optimize/base.py +198 -0
- mlflow/genai/optimize/optimizers/__init__.py +4 -0
- mlflow/genai/optimize/optimizers/base_optimizer.py +38 -0
- mlflow/genai/optimize/optimizers/dspy_mipro_optimizer.py +221 -0
- mlflow/genai/optimize/optimizers/dspy_optimizer.py +91 -0
- mlflow/genai/optimize/optimizers/utils/dspy_mipro_callback.py +76 -0
- mlflow/genai/optimize/optimizers/utils/dspy_mipro_utils.py +18 -0
- mlflow/genai/optimize/types.py +75 -0
- mlflow/genai/optimize/util.py +30 -0
- mlflow/genai/prompts/__init__.py +206 -0
- mlflow/genai/scheduled_scorers.py +431 -0
- mlflow/genai/scorers/__init__.py +26 -0
- mlflow/genai/scorers/base.py +492 -0
- mlflow/genai/scorers/builtin_scorers.py +765 -0
- mlflow/genai/scorers/scorer_utils.py +138 -0
- mlflow/genai/scorers/validation.py +165 -0
- mlflow/genai/utils/data_validation.py +146 -0
- mlflow/genai/utils/enum_utils.py +23 -0
- mlflow/genai/utils/trace_utils.py +211 -0
- mlflow/groq/__init__.py +42 -0
- mlflow/groq/_groq_autolog.py +74 -0
- mlflow/johnsnowlabs/__init__.py +888 -0
- mlflow/langchain/__init__.py +24 -0
- mlflow/langchain/api_request_parallel_processor.py +330 -0
- mlflow/langchain/autolog.py +147 -0
- mlflow/langchain/chat_agent_langgraph.py +340 -0
- mlflow/langchain/constant.py +1 -0
- mlflow/langchain/constants.py +1 -0
- mlflow/langchain/databricks_dependencies.py +444 -0
- mlflow/langchain/langchain_tracer.py +597 -0
- mlflow/langchain/model.py +919 -0
- mlflow/langchain/output_parsers.py +142 -0
- mlflow/langchain/retriever_chain.py +153 -0
- mlflow/langchain/runnables.py +527 -0
- mlflow/langchain/utils/chat.py +402 -0
- mlflow/langchain/utils/logging.py +671 -0
- mlflow/langchain/utils/serialization.py +36 -0
- mlflow/legacy_databricks_cli/__init__.py +0 -0
- mlflow/legacy_databricks_cli/configure/__init__.py +0 -0
- mlflow/legacy_databricks_cli/configure/provider.py +482 -0
- mlflow/litellm/__init__.py +175 -0
- mlflow/llama_index/__init__.py +22 -0
- mlflow/llama_index/autolog.py +55 -0
- mlflow/llama_index/chat.py +43 -0
- mlflow/llama_index/constant.py +1 -0
- mlflow/llama_index/model.py +577 -0
- mlflow/llama_index/pyfunc_wrapper.py +332 -0
- mlflow/llama_index/serialize_objects.py +188 -0
- mlflow/llama_index/tracer.py +561 -0
- mlflow/metrics/__init__.py +479 -0
- mlflow/metrics/base.py +39 -0
- mlflow/metrics/genai/__init__.py +25 -0
- mlflow/metrics/genai/base.py +101 -0
- mlflow/metrics/genai/genai_metric.py +771 -0
- mlflow/metrics/genai/metric_definitions.py +450 -0
- mlflow/metrics/genai/model_utils.py +371 -0
- mlflow/metrics/genai/prompt_template.py +68 -0
- mlflow/metrics/genai/prompts/__init__.py +0 -0
- mlflow/metrics/genai/prompts/v1.py +422 -0
- mlflow/metrics/genai/utils.py +6 -0
- mlflow/metrics/metric_definitions.py +619 -0
- mlflow/mismatch.py +34 -0
- mlflow/mistral/__init__.py +34 -0
- mlflow/mistral/autolog.py +71 -0
- mlflow/mistral/chat.py +135 -0
- mlflow/ml_package_versions.py +452 -0
- mlflow/models/__init__.py +97 -0
- mlflow/models/auth_policy.py +83 -0
- mlflow/models/cli.py +354 -0
- mlflow/models/container/__init__.py +294 -0
- mlflow/models/container/scoring_server/__init__.py +0 -0
- mlflow/models/container/scoring_server/nginx.conf +39 -0
- mlflow/models/dependencies_schemas.py +287 -0
- mlflow/models/display_utils.py +158 -0
- mlflow/models/docker_utils.py +211 -0
- mlflow/models/evaluation/__init__.py +23 -0
- mlflow/models/evaluation/_shap_patch.py +64 -0
- mlflow/models/evaluation/artifacts.py +194 -0
- mlflow/models/evaluation/base.py +1811 -0
- mlflow/models/evaluation/calibration_curve.py +109 -0
- mlflow/models/evaluation/default_evaluator.py +996 -0
- mlflow/models/evaluation/deprecated.py +23 -0
- mlflow/models/evaluation/evaluator_registry.py +80 -0
- mlflow/models/evaluation/evaluators/classifier.py +704 -0
- mlflow/models/evaluation/evaluators/default.py +233 -0
- mlflow/models/evaluation/evaluators/regressor.py +96 -0
- mlflow/models/evaluation/evaluators/shap.py +296 -0
- mlflow/models/evaluation/lift_curve.py +178 -0
- mlflow/models/evaluation/utils/metric.py +123 -0
- mlflow/models/evaluation/utils/trace.py +179 -0
- mlflow/models/evaluation/validation.py +434 -0
- mlflow/models/flavor_backend.py +93 -0
- mlflow/models/flavor_backend_registry.py +53 -0
- mlflow/models/model.py +1639 -0
- mlflow/models/model_config.py +150 -0
- mlflow/models/notebook_resources/agent_evaluation_template.html +235 -0
- mlflow/models/notebook_resources/eval_with_dataset_example.py +22 -0
- mlflow/models/notebook_resources/eval_with_synthetic_example.py +22 -0
- mlflow/models/python_api.py +369 -0
- mlflow/models/rag_signatures.py +128 -0
- mlflow/models/resources.py +321 -0
- mlflow/models/signature.py +662 -0
- mlflow/models/utils.py +2054 -0
- mlflow/models/wheeled_model.py +280 -0
- mlflow/openai/__init__.py +57 -0
- mlflow/openai/_agent_tracer.py +364 -0
- mlflow/openai/api_request_parallel_processor.py +131 -0
- mlflow/openai/autolog.py +509 -0
- mlflow/openai/constant.py +1 -0
- mlflow/openai/model.py +824 -0
- mlflow/openai/utils/chat_schema.py +367 -0
- mlflow/optuna/__init__.py +3 -0
- mlflow/optuna/storage.py +646 -0
- mlflow/plugins/__init__.py +72 -0
- mlflow/plugins/base.py +358 -0
- mlflow/plugins/builtin/__init__.py +24 -0
- mlflow/plugins/builtin/pytorch_plugin.py +150 -0
- mlflow/plugins/builtin/sklearn_plugin.py +158 -0
- mlflow/plugins/builtin/transformers_plugin.py +187 -0
- mlflow/plugins/cli.py +321 -0
- mlflow/plugins/discovery.py +340 -0
- mlflow/plugins/manager.py +465 -0
- mlflow/plugins/registry.py +316 -0
- mlflow/plugins/templates/framework_plugin_template.py +329 -0
- mlflow/prompt/constants.py +20 -0
- mlflow/prompt/promptlab_model.py +197 -0
- mlflow/prompt/registry_utils.py +248 -0
- mlflow/promptflow/__init__.py +495 -0
- mlflow/protos/__init__.py +0 -0
- mlflow/protos/assessments_pb2.py +174 -0
- mlflow/protos/databricks_artifacts_pb2.py +489 -0
- mlflow/protos/databricks_filesystem_service_pb2.py +196 -0
- mlflow/protos/databricks_managed_catalog_messages_pb2.py +95 -0
- mlflow/protos/databricks_managed_catalog_service_pb2.py +86 -0
- mlflow/protos/databricks_pb2.py +267 -0
- mlflow/protos/databricks_trace_server_pb2.py +374 -0
- mlflow/protos/databricks_uc_registry_messages_pb2.py +1249 -0
- mlflow/protos/databricks_uc_registry_service_pb2.py +170 -0
- mlflow/protos/facet_feature_statistics_pb2.py +296 -0
- mlflow/protos/internal_pb2.py +77 -0
- mlflow/protos/mlflow_artifacts_pb2.py +336 -0
- mlflow/protos/model_registry_pb2.py +1073 -0
- mlflow/protos/scalapb/__init__.py +0 -0
- mlflow/protos/scalapb/scalapb_pb2.py +104 -0
- mlflow/protos/service_pb2.py +2600 -0
- mlflow/protos/unity_catalog_oss_messages_pb2.py +457 -0
- mlflow/protos/unity_catalog_oss_service_pb2.py +130 -0
- mlflow/protos/unity_catalog_prompt_messages_pb2.py +447 -0
- mlflow/protos/unity_catalog_prompt_messages_pb2_grpc.py +24 -0
- mlflow/protos/unity_catalog_prompt_service_pb2.py +164 -0
- mlflow/protos/unity_catalog_prompt_service_pb2_grpc.py +785 -0
- mlflow/py.typed +0 -0
- mlflow/pydantic_ai/__init__.py +57 -0
- mlflow/pydantic_ai/autolog.py +173 -0
- mlflow/pyfunc/__init__.py +3844 -0
- mlflow/pyfunc/_mlflow_pyfunc_backend_predict.py +61 -0
- mlflow/pyfunc/backend.py +523 -0
- mlflow/pyfunc/context.py +78 -0
- mlflow/pyfunc/dbconnect_artifact_cache.py +144 -0
- mlflow/pyfunc/loaders/__init__.py +7 -0
- mlflow/pyfunc/loaders/chat_agent.py +117 -0
- mlflow/pyfunc/loaders/chat_model.py +125 -0
- mlflow/pyfunc/loaders/code_model.py +31 -0
- mlflow/pyfunc/loaders/responses_agent.py +112 -0
- mlflow/pyfunc/mlserver.py +46 -0
- mlflow/pyfunc/model.py +1473 -0
- mlflow/pyfunc/scoring_server/__init__.py +604 -0
- mlflow/pyfunc/scoring_server/app.py +7 -0
- mlflow/pyfunc/scoring_server/client.py +146 -0
- mlflow/pyfunc/spark_model_cache.py +48 -0
- mlflow/pyfunc/stdin_server.py +44 -0
- mlflow/pyfunc/utils/__init__.py +3 -0
- mlflow/pyfunc/utils/data_validation.py +224 -0
- mlflow/pyfunc/utils/environment.py +22 -0
- mlflow/pyfunc/utils/input_converter.py +47 -0
- mlflow/pyfunc/utils/serving_data_parser.py +11 -0
- mlflow/pytorch/__init__.py +1171 -0
- mlflow/pytorch/_lightning_autolog.py +580 -0
- mlflow/pytorch/_pytorch_autolog.py +50 -0
- mlflow/pytorch/pickle_module.py +35 -0
- mlflow/rfunc/__init__.py +42 -0
- mlflow/rfunc/backend.py +134 -0
- mlflow/runs.py +89 -0
- mlflow/server/__init__.py +302 -0
- mlflow/server/auth/__init__.py +1224 -0
- mlflow/server/auth/__main__.py +4 -0
- mlflow/server/auth/basic_auth.ini +6 -0
- mlflow/server/auth/cli.py +11 -0
- mlflow/server/auth/client.py +537 -0
- mlflow/server/auth/config.py +34 -0
- mlflow/server/auth/db/__init__.py +0 -0
- mlflow/server/auth/db/cli.py +18 -0
- mlflow/server/auth/db/migrations/__init__.py +0 -0
- mlflow/server/auth/db/migrations/alembic.ini +110 -0
- mlflow/server/auth/db/migrations/env.py +76 -0
- mlflow/server/auth/db/migrations/versions/8606fa83a998_initial_migration.py +51 -0
- mlflow/server/auth/db/migrations/versions/__init__.py +0 -0
- mlflow/server/auth/db/models.py +67 -0
- mlflow/server/auth/db/utils.py +37 -0
- mlflow/server/auth/entities.py +165 -0
- mlflow/server/auth/logo.py +14 -0
- mlflow/server/auth/permissions.py +65 -0
- mlflow/server/auth/routes.py +18 -0
- mlflow/server/auth/sqlalchemy_store.py +263 -0
- mlflow/server/graphql/__init__.py +0 -0
- mlflow/server/graphql/autogenerated_graphql_schema.py +353 -0
- mlflow/server/graphql/graphql_custom_scalars.py +24 -0
- mlflow/server/graphql/graphql_errors.py +15 -0
- mlflow/server/graphql/graphql_no_batching.py +89 -0
- mlflow/server/graphql/graphql_schema_extensions.py +74 -0
- mlflow/server/handlers.py +3217 -0
- mlflow/server/prometheus_exporter.py +17 -0
- mlflow/server/validation.py +30 -0
- mlflow/shap/__init__.py +691 -0
- mlflow/sklearn/__init__.py +1994 -0
- mlflow/sklearn/utils.py +1041 -0
- mlflow/smolagents/__init__.py +66 -0
- mlflow/smolagents/autolog.py +139 -0
- mlflow/smolagents/chat.py +29 -0
- mlflow/store/__init__.py +10 -0
- mlflow/store/_unity_catalog/__init__.py +1 -0
- mlflow/store/_unity_catalog/lineage/__init__.py +1 -0
- mlflow/store/_unity_catalog/lineage/constants.py +2 -0
- mlflow/store/_unity_catalog/registry/__init__.py +6 -0
- mlflow/store/_unity_catalog/registry/prompt_info.py +75 -0
- mlflow/store/_unity_catalog/registry/rest_store.py +1740 -0
- mlflow/store/_unity_catalog/registry/uc_oss_rest_store.py +507 -0
- mlflow/store/_unity_catalog/registry/utils.py +121 -0
- mlflow/store/artifact/__init__.py +0 -0
- mlflow/store/artifact/artifact_repo.py +472 -0
- mlflow/store/artifact/artifact_repository_registry.py +154 -0
- mlflow/store/artifact/azure_blob_artifact_repo.py +275 -0
- mlflow/store/artifact/azure_data_lake_artifact_repo.py +295 -0
- mlflow/store/artifact/cli.py +141 -0
- mlflow/store/artifact/cloud_artifact_repo.py +332 -0
- mlflow/store/artifact/databricks_artifact_repo.py +729 -0
- mlflow/store/artifact/databricks_artifact_repo_resources.py +301 -0
- mlflow/store/artifact/databricks_logged_model_artifact_repo.py +93 -0
- mlflow/store/artifact/databricks_models_artifact_repo.py +216 -0
- mlflow/store/artifact/databricks_sdk_artifact_repo.py +134 -0
- mlflow/store/artifact/databricks_sdk_models_artifact_repo.py +97 -0
- mlflow/store/artifact/dbfs_artifact_repo.py +240 -0
- mlflow/store/artifact/ftp_artifact_repo.py +132 -0
- mlflow/store/artifact/gcs_artifact_repo.py +296 -0
- mlflow/store/artifact/hdfs_artifact_repo.py +209 -0
- mlflow/store/artifact/http_artifact_repo.py +218 -0
- mlflow/store/artifact/local_artifact_repo.py +142 -0
- mlflow/store/artifact/mlflow_artifacts_repo.py +94 -0
- mlflow/store/artifact/models_artifact_repo.py +259 -0
- mlflow/store/artifact/optimized_s3_artifact_repo.py +356 -0
- mlflow/store/artifact/presigned_url_artifact_repo.py +173 -0
- mlflow/store/artifact/r2_artifact_repo.py +70 -0
- mlflow/store/artifact/runs_artifact_repo.py +265 -0
- mlflow/store/artifact/s3_artifact_repo.py +330 -0
- mlflow/store/artifact/sftp_artifact_repo.py +141 -0
- mlflow/store/artifact/uc_volume_artifact_repo.py +76 -0
- mlflow/store/artifact/unity_catalog_models_artifact_repo.py +168 -0
- mlflow/store/artifact/unity_catalog_oss_models_artifact_repo.py +168 -0
- mlflow/store/artifact/utils/__init__.py +0 -0
- mlflow/store/artifact/utils/models.py +148 -0
- mlflow/store/db/__init__.py +0 -0
- mlflow/store/db/base_sql_model.py +3 -0
- mlflow/store/db/db_types.py +10 -0
- mlflow/store/db/utils.py +314 -0
- mlflow/store/db_migrations/__init__.py +0 -0
- mlflow/store/db_migrations/alembic.ini +74 -0
- mlflow/store/db_migrations/env.py +84 -0
- mlflow/store/db_migrations/versions/0584bdc529eb_add_cascading_deletion_to_datasets_from_experiments.py +88 -0
- mlflow/store/db_migrations/versions/0a8213491aaa_drop_duplicate_killed_constraint.py +49 -0
- mlflow/store/db_migrations/versions/0c779009ac13_add_deleted_time_field_to_runs_table.py +24 -0
- mlflow/store/db_migrations/versions/181f10493468_allow_nulls_for_metric_values.py +35 -0
- mlflow/store/db_migrations/versions/27a6a02d2cf1_add_model_version_tags_table.py +38 -0
- mlflow/store/db_migrations/versions/2b4d017a5e9b_add_model_registry_tables_to_db.py +77 -0
- mlflow/store/db_migrations/versions/2d6e25af4d3e_increase_max_param_val_length.py +33 -0
- mlflow/store/db_migrations/versions/3500859a5d39_add_model_aliases_table.py +50 -0
- mlflow/store/db_migrations/versions/39d1c3be5f05_add_is_nan_constraint_for_metrics_tables_if_necessary.py +41 -0
- mlflow/store/db_migrations/versions/400f98739977_add_logged_model_tables.py +123 -0
- mlflow/store/db_migrations/versions/4465047574b1_increase_max_dataset_schema_size.py +38 -0
- mlflow/store/db_migrations/versions/451aebb31d03_add_metric_step.py +35 -0
- mlflow/store/db_migrations/versions/5b0e9adcef9c_add_cascade_deletion_to_trace_tables_fk.py +40 -0
- mlflow/store/db_migrations/versions/6953534de441_add_step_to_inputs_table.py +25 -0
- mlflow/store/db_migrations/versions/728d730b5ebd_add_registered_model_tags_table.py +38 -0
- mlflow/store/db_migrations/versions/7ac759974ad8_update_run_tags_with_larger_limit.py +36 -0
- mlflow/store/db_migrations/versions/7f2a7d5fae7d_add_datasets_inputs_input_tags_tables.py +82 -0
- mlflow/store/db_migrations/versions/84291f40a231_add_run_link_to_model_version.py +26 -0
- mlflow/store/db_migrations/versions/867495a8f9d4_add_trace_tables.py +90 -0
- mlflow/store/db_migrations/versions/89d4b8295536_create_latest_metrics_table.py +169 -0
- mlflow/store/db_migrations/versions/90e64c465722_migrate_user_column_to_tags.py +64 -0
- mlflow/store/db_migrations/versions/97727af70f4d_creation_time_last_update_time_experiments.py +25 -0
- mlflow/store/db_migrations/versions/__init__.py +0 -0
- mlflow/store/db_migrations/versions/a8c4a736bde6_allow_nulls_for_run_id.py +27 -0
- mlflow/store/db_migrations/versions/acf3f17fdcc7_add_storage_location_field_to_model_.py +29 -0
- mlflow/store/db_migrations/versions/bd07f7e963c5_create_index_on_run_uuid.py +26 -0
- mlflow/store/db_migrations/versions/bda7b8c39065_increase_model_version_tag_value_limit.py +38 -0
- mlflow/store/db_migrations/versions/c48cb773bb87_reset_default_value_for_is_nan_in_metrics_table_for_mysql.py +41 -0
- mlflow/store/db_migrations/versions/cbc13b556ace_add_v3_trace_schema_columns.py +31 -0
- mlflow/store/db_migrations/versions/cc1f77228345_change_param_value_length_to_500.py +34 -0
- mlflow/store/db_migrations/versions/cfd24bdc0731_update_run_status_constraint_with_killed.py +78 -0
- mlflow/store/db_migrations/versions/df50e92ffc5e_add_experiment_tags_table.py +38 -0
- mlflow/store/db_migrations/versions/f5a4f2784254_increase_run_tag_value_limit.py +36 -0
- mlflow/store/entities/__init__.py +3 -0
- mlflow/store/entities/paged_list.py +18 -0
- mlflow/store/model_registry/__init__.py +10 -0
- mlflow/store/model_registry/abstract_store.py +1081 -0
- mlflow/store/model_registry/base_rest_store.py +44 -0
- mlflow/store/model_registry/databricks_workspace_model_registry_rest_store.py +37 -0
- mlflow/store/model_registry/dbmodels/__init__.py +0 -0
- mlflow/store/model_registry/dbmodels/models.py +206 -0
- mlflow/store/model_registry/file_store.py +1091 -0
- mlflow/store/model_registry/rest_store.py +481 -0
- mlflow/store/model_registry/sqlalchemy_store.py +1286 -0
- mlflow/store/tracking/__init__.py +23 -0
- mlflow/store/tracking/abstract_store.py +816 -0
- mlflow/store/tracking/dbmodels/__init__.py +0 -0
- mlflow/store/tracking/dbmodels/initial_models.py +243 -0
- mlflow/store/tracking/dbmodels/models.py +1073 -0
- mlflow/store/tracking/file_store.py +2438 -0
- mlflow/store/tracking/postgres_managed_identity.py +146 -0
- mlflow/store/tracking/rest_store.py +1131 -0
- mlflow/store/tracking/sqlalchemy_store.py +2785 -0
- mlflow/system_metrics/__init__.py +61 -0
- mlflow/system_metrics/metrics/__init__.py +0 -0
- mlflow/system_metrics/metrics/base_metrics_monitor.py +32 -0
- mlflow/system_metrics/metrics/cpu_monitor.py +23 -0
- mlflow/system_metrics/metrics/disk_monitor.py +21 -0
- mlflow/system_metrics/metrics/gpu_monitor.py +71 -0
- mlflow/system_metrics/metrics/network_monitor.py +34 -0
- mlflow/system_metrics/metrics/rocm_monitor.py +123 -0
- mlflow/system_metrics/system_metrics_monitor.py +198 -0
- mlflow/tracing/__init__.py +16 -0
- mlflow/tracing/assessment.py +356 -0
- mlflow/tracing/client.py +531 -0
- mlflow/tracing/config.py +125 -0
- mlflow/tracing/constant.py +105 -0
- mlflow/tracing/destination.py +81 -0
- mlflow/tracing/display/__init__.py +40 -0
- mlflow/tracing/display/display_handler.py +196 -0
- mlflow/tracing/export/async_export_queue.py +186 -0
- mlflow/tracing/export/inference_table.py +138 -0
- mlflow/tracing/export/mlflow_v3.py +137 -0
- mlflow/tracing/export/utils.py +70 -0
- mlflow/tracing/fluent.py +1417 -0
- mlflow/tracing/processor/base_mlflow.py +199 -0
- mlflow/tracing/processor/inference_table.py +175 -0
- mlflow/tracing/processor/mlflow_v3.py +47 -0
- mlflow/tracing/processor/otel.py +73 -0
- mlflow/tracing/provider.py +487 -0
- mlflow/tracing/trace_manager.py +200 -0
- mlflow/tracing/utils/__init__.py +616 -0
- mlflow/tracing/utils/artifact_utils.py +28 -0
- mlflow/tracing/utils/copy.py +55 -0
- mlflow/tracing/utils/environment.py +55 -0
- mlflow/tracing/utils/exception.py +21 -0
- mlflow/tracing/utils/once.py +35 -0
- mlflow/tracing/utils/otlp.py +63 -0
- mlflow/tracing/utils/processor.py +54 -0
- mlflow/tracing/utils/search.py +292 -0
- mlflow/tracing/utils/timeout.py +250 -0
- mlflow/tracing/utils/token.py +19 -0
- mlflow/tracing/utils/truncation.py +124 -0
- mlflow/tracing/utils/warning.py +76 -0
- mlflow/tracking/__init__.py +39 -0
- mlflow/tracking/_model_registry/__init__.py +1 -0
- mlflow/tracking/_model_registry/client.py +764 -0
- mlflow/tracking/_model_registry/fluent.py +853 -0
- mlflow/tracking/_model_registry/registry.py +67 -0
- mlflow/tracking/_model_registry/utils.py +251 -0
- mlflow/tracking/_tracking_service/__init__.py +0 -0
- mlflow/tracking/_tracking_service/client.py +883 -0
- mlflow/tracking/_tracking_service/registry.py +56 -0
- mlflow/tracking/_tracking_service/utils.py +275 -0
- mlflow/tracking/artifact_utils.py +179 -0
- mlflow/tracking/client.py +5900 -0
- mlflow/tracking/context/__init__.py +0 -0
- mlflow/tracking/context/abstract_context.py +35 -0
- mlflow/tracking/context/databricks_cluster_context.py +15 -0
- mlflow/tracking/context/databricks_command_context.py +15 -0
- mlflow/tracking/context/databricks_job_context.py +49 -0
- mlflow/tracking/context/databricks_notebook_context.py +41 -0
- mlflow/tracking/context/databricks_repo_context.py +43 -0
- mlflow/tracking/context/default_context.py +51 -0
- mlflow/tracking/context/git_context.py +32 -0
- mlflow/tracking/context/registry.py +98 -0
- mlflow/tracking/context/system_environment_context.py +15 -0
- mlflow/tracking/default_experiment/__init__.py +1 -0
- mlflow/tracking/default_experiment/abstract_context.py +43 -0
- mlflow/tracking/default_experiment/databricks_notebook_experiment_provider.py +44 -0
- mlflow/tracking/default_experiment/registry.py +75 -0
- mlflow/tracking/fluent.py +3595 -0
- mlflow/tracking/metric_value_conversion_utils.py +93 -0
- mlflow/tracking/multimedia.py +206 -0
- mlflow/tracking/registry.py +86 -0
- mlflow/tracking/request_auth/__init__.py +0 -0
- mlflow/tracking/request_auth/abstract_request_auth_provider.py +34 -0
- mlflow/tracking/request_auth/registry.py +60 -0
- mlflow/tracking/request_header/__init__.py +0 -0
- mlflow/tracking/request_header/abstract_request_header_provider.py +36 -0
- mlflow/tracking/request_header/databricks_request_header_provider.py +38 -0
- mlflow/tracking/request_header/default_request_header_provider.py +17 -0
- mlflow/tracking/request_header/registry.py +79 -0
- mlflow/transformers/__init__.py +2982 -0
- mlflow/transformers/flavor_config.py +258 -0
- mlflow/transformers/hub_utils.py +83 -0
- mlflow/transformers/llm_inference_utils.py +468 -0
- mlflow/transformers/model_io.py +301 -0
- mlflow/transformers/peft.py +51 -0
- mlflow/transformers/signature.py +183 -0
- mlflow/transformers/torch_utils.py +55 -0
- mlflow/types/__init__.py +21 -0
- mlflow/types/agent.py +270 -0
- mlflow/types/chat.py +240 -0
- mlflow/types/llm.py +935 -0
- mlflow/types/responses.py +139 -0
- mlflow/types/responses_helpers.py +416 -0
- mlflow/types/schema.py +1505 -0
- mlflow/types/type_hints.py +647 -0
- mlflow/types/utils.py +753 -0
- mlflow/utils/__init__.py +283 -0
- mlflow/utils/_capture_modules.py +256 -0
- mlflow/utils/_capture_transformers_modules.py +75 -0
- mlflow/utils/_spark_utils.py +201 -0
- mlflow/utils/_unity_catalog_oss_utils.py +97 -0
- mlflow/utils/_unity_catalog_utils.py +479 -0
- mlflow/utils/annotations.py +218 -0
- mlflow/utils/arguments_utils.py +16 -0
- mlflow/utils/async_logging/__init__.py +1 -0
- mlflow/utils/async_logging/async_artifacts_logging_queue.py +258 -0
- mlflow/utils/async_logging/async_logging_queue.py +366 -0
- mlflow/utils/async_logging/run_artifact.py +38 -0
- mlflow/utils/async_logging/run_batch.py +58 -0
- mlflow/utils/async_logging/run_operations.py +49 -0
- mlflow/utils/autologging_utils/__init__.py +737 -0
- mlflow/utils/autologging_utils/client.py +432 -0
- mlflow/utils/autologging_utils/config.py +33 -0
- mlflow/utils/autologging_utils/events.py +294 -0
- mlflow/utils/autologging_utils/logging_and_warnings.py +328 -0
- mlflow/utils/autologging_utils/metrics_queue.py +71 -0
- mlflow/utils/autologging_utils/safety.py +1104 -0
- mlflow/utils/autologging_utils/versioning.py +95 -0
- mlflow/utils/checkpoint_utils.py +206 -0
- mlflow/utils/class_utils.py +6 -0
- mlflow/utils/cli_args.py +257 -0
- mlflow/utils/conda.py +354 -0
- mlflow/utils/credentials.py +231 -0
- mlflow/utils/data_utils.py +17 -0
- mlflow/utils/databricks_utils.py +1436 -0
- mlflow/utils/docstring_utils.py +477 -0
- mlflow/utils/doctor.py +133 -0
- mlflow/utils/download_cloud_file_chunk.py +43 -0
- mlflow/utils/env_manager.py +16 -0
- mlflow/utils/env_pack.py +131 -0
- mlflow/utils/environment.py +1009 -0
- mlflow/utils/exception_utils.py +14 -0
- mlflow/utils/file_utils.py +978 -0
- mlflow/utils/git_utils.py +77 -0
- mlflow/utils/gorilla.py +797 -0
- mlflow/utils/import_hooks/__init__.py +363 -0
- mlflow/utils/lazy_load.py +51 -0
- mlflow/utils/logging_utils.py +168 -0
- mlflow/utils/mime_type_utils.py +58 -0
- mlflow/utils/mlflow_tags.py +103 -0
- mlflow/utils/model_utils.py +486 -0
- mlflow/utils/name_utils.py +346 -0
- mlflow/utils/nfs_on_spark.py +62 -0
- mlflow/utils/openai_utils.py +164 -0
- mlflow/utils/os.py +12 -0
- mlflow/utils/oss_registry_utils.py +29 -0
- mlflow/utils/plugins.py +17 -0
- mlflow/utils/process.py +182 -0
- mlflow/utils/promptlab_utils.py +146 -0
- mlflow/utils/proto_json_utils.py +743 -0
- mlflow/utils/pydantic_utils.py +54 -0
- mlflow/utils/request_utils.py +279 -0
- mlflow/utils/requirements_utils.py +704 -0
- mlflow/utils/rest_utils.py +673 -0
- mlflow/utils/search_logged_model_utils.py +127 -0
- mlflow/utils/search_utils.py +2111 -0
- mlflow/utils/secure_loading.py +221 -0
- mlflow/utils/security_validation.py +384 -0
- mlflow/utils/server_cli_utils.py +61 -0
- mlflow/utils/spark_utils.py +15 -0
- mlflow/utils/string_utils.py +138 -0
- mlflow/utils/thread_utils.py +63 -0
- mlflow/utils/time.py +54 -0
- mlflow/utils/timeout.py +42 -0
- mlflow/utils/uri.py +572 -0
- mlflow/utils/validation.py +662 -0
- mlflow/utils/virtualenv.py +458 -0
- mlflow/utils/warnings_utils.py +25 -0
- mlflow/utils/yaml_utils.py +179 -0
- mlflow/version.py +24 -0
@@ -0,0 +1,1091 @@
|
|
1
|
+
import logging
|
2
|
+
import os
|
3
|
+
import shutil
|
4
|
+
import sys
|
5
|
+
import time
|
6
|
+
import urllib
|
7
|
+
from os.path import join
|
8
|
+
from typing import Optional
|
9
|
+
|
10
|
+
from mlflow.entities.model_registry import (
|
11
|
+
ModelVersion,
|
12
|
+
ModelVersionTag,
|
13
|
+
RegisteredModel,
|
14
|
+
RegisteredModelAlias,
|
15
|
+
RegisteredModelTag,
|
16
|
+
)
|
17
|
+
from mlflow.entities.model_registry.model_version_stages import (
|
18
|
+
ALL_STAGES,
|
19
|
+
DEFAULT_STAGES_FOR_GET_LATEST_VERSIONS,
|
20
|
+
STAGE_ARCHIVED,
|
21
|
+
STAGE_DELETED_INTERNAL,
|
22
|
+
STAGE_NONE,
|
23
|
+
get_canonical_stage,
|
24
|
+
)
|
25
|
+
from mlflow.environment_variables import MLFLOW_REGISTRY_DIR
|
26
|
+
from mlflow.exceptions import MlflowException
|
27
|
+
from mlflow.prompt.registry_utils import (
|
28
|
+
add_prompt_filter_string,
|
29
|
+
handle_resource_already_exist_error,
|
30
|
+
has_prompt_tag,
|
31
|
+
)
|
32
|
+
from mlflow.protos.databricks_pb2 import (
|
33
|
+
INVALID_PARAMETER_VALUE,
|
34
|
+
RESOURCE_ALREADY_EXISTS,
|
35
|
+
RESOURCE_DOES_NOT_EXIST,
|
36
|
+
ErrorCode,
|
37
|
+
)
|
38
|
+
from mlflow.store.artifact.utils.models import _parse_model_uri
|
39
|
+
from mlflow.store.entities.paged_list import PagedList
|
40
|
+
from mlflow.store.model_registry import (
|
41
|
+
DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH,
|
42
|
+
SEARCH_MODEL_VERSION_MAX_RESULTS_THRESHOLD,
|
43
|
+
SEARCH_REGISTERED_MODEL_MAX_RESULTS_THRESHOLD,
|
44
|
+
)
|
45
|
+
from mlflow.store.model_registry.abstract_store import AbstractStore
|
46
|
+
from mlflow.utils.file_utils import (
|
47
|
+
contains_path_separator,
|
48
|
+
contains_percent,
|
49
|
+
exists,
|
50
|
+
find,
|
51
|
+
is_directory,
|
52
|
+
list_all,
|
53
|
+
list_subdirs,
|
54
|
+
local_file_uri_to_path,
|
55
|
+
make_containing_dirs,
|
56
|
+
mkdir,
|
57
|
+
read_file,
|
58
|
+
write_to,
|
59
|
+
)
|
60
|
+
from mlflow.utils.search_utils import SearchModelUtils, SearchModelVersionUtils, SearchUtils
|
61
|
+
from mlflow.utils.string_utils import is_string_type
|
62
|
+
from mlflow.utils.time import get_current_time_millis
|
63
|
+
from mlflow.utils.validation import (
|
64
|
+
_validate_model_alias_name,
|
65
|
+
_validate_model_version,
|
66
|
+
_validate_model_version_tag,
|
67
|
+
_validate_registered_model_tag,
|
68
|
+
_validate_tag_name,
|
69
|
+
)
|
70
|
+
from mlflow.utils.validation import (
|
71
|
+
_validate_model_name as _original_validate_model_name,
|
72
|
+
)
|
73
|
+
from mlflow.utils.yaml_utils import overwrite_yaml, read_yaml, write_yaml
|
74
|
+
|
75
|
+
|
76
|
+
def _default_root_dir():
|
77
|
+
return MLFLOW_REGISTRY_DIR.get() or os.path.abspath(DEFAULT_LOCAL_FILE_AND_ARTIFACT_PATH)
|
78
|
+
|
79
|
+
|
80
|
+
def _validate_model_name(name):
|
81
|
+
_original_validate_model_name(name)
|
82
|
+
if contains_path_separator(name):
|
83
|
+
raise MlflowException(
|
84
|
+
f"Invalid name: '{name}'. Registered model name cannot contain path separator",
|
85
|
+
INVALID_PARAMETER_VALUE,
|
86
|
+
)
|
87
|
+
if contains_percent(name):
|
88
|
+
raise MlflowException(
|
89
|
+
f"Invalid name: '{name}'. Registered model name cannot contain '%' character",
|
90
|
+
INVALID_PARAMETER_VALUE,
|
91
|
+
)
|
92
|
+
|
93
|
+
|
94
|
+
class FileModelVersion(ModelVersion):
|
95
|
+
def __init__(self, storage_location=None, **kwargs):
|
96
|
+
super().__init__(**kwargs)
|
97
|
+
self._storage_location = storage_location
|
98
|
+
|
99
|
+
@property
|
100
|
+
def storage_location(self):
|
101
|
+
"""String. The storage location of the model version."""
|
102
|
+
return self._storage_location
|
103
|
+
|
104
|
+
@storage_location.setter
|
105
|
+
def storage_location(self, location):
|
106
|
+
self._storage_location = location
|
107
|
+
|
108
|
+
@classmethod
|
109
|
+
def _properties(cls):
|
110
|
+
# aggregate with parent class with subclass properties
|
111
|
+
return sorted(ModelVersion._properties() + cls._get_properties_helper())
|
112
|
+
|
113
|
+
def to_mlflow_entity(self):
|
114
|
+
meta = dict(self)
|
115
|
+
return ModelVersion.from_dictionary(
|
116
|
+
{**meta, "tags": [ModelVersionTag(k, v) for k, v in meta["tags"].items()]}
|
117
|
+
)
|
118
|
+
|
119
|
+
|
120
|
+
class FileStore(AbstractStore):
|
121
|
+
MODELS_FOLDER_NAME = "models"
|
122
|
+
META_DATA_FILE_NAME = "meta.yaml"
|
123
|
+
TAGS_FOLDER_NAME = "tags"
|
124
|
+
MODEL_VERSION_TAGS_FOLDER_NAME = "tags"
|
125
|
+
CREATE_MODEL_VERSION_RETRIES = 3
|
126
|
+
REGISTERED_MODELS_ALIASES_FOLDER_NAME = "aliases"
|
127
|
+
|
128
|
+
def __init__(self, root_directory=None):
|
129
|
+
"""
|
130
|
+
Create a new FileStore with the given root directory.
|
131
|
+
"""
|
132
|
+
|
133
|
+
super().__init__()
|
134
|
+
self.root_directory = local_file_uri_to_path(root_directory or _default_root_dir())
|
135
|
+
# Create models directory if needed
|
136
|
+
if not exists(self.models_directory):
|
137
|
+
mkdir(self.models_directory)
|
138
|
+
|
139
|
+
@property
|
140
|
+
def models_directory(self):
|
141
|
+
return os.path.join(self.root_directory, FileStore.MODELS_FOLDER_NAME)
|
142
|
+
|
143
|
+
def _check_root_dir(self):
|
144
|
+
"""
|
145
|
+
Run checks before running directory operations.
|
146
|
+
"""
|
147
|
+
if not exists(self.root_directory):
|
148
|
+
raise Exception(f"'{self.root_directory}' does not exist.")
|
149
|
+
if not is_directory(self.root_directory):
|
150
|
+
raise Exception(f"'{self.root_directory}' is not a directory.")
|
151
|
+
|
152
|
+
def _validate_registered_model_does_not_exist(self, name):
|
153
|
+
model_path = self._get_registered_model_path(name)
|
154
|
+
if exists(model_path):
|
155
|
+
raise MlflowException(
|
156
|
+
f"Registered Model (name={name}) already exists.",
|
157
|
+
RESOURCE_ALREADY_EXISTS,
|
158
|
+
)
|
159
|
+
|
160
|
+
def _save_registered_model_as_meta_file(self, registered_model, meta_dir=None, overwrite=True):
|
161
|
+
registered_model_dict = dict(registered_model)
|
162
|
+
# tags are stored under TAGS_FOLDER_NAME so remove them in meta file.
|
163
|
+
del registered_model_dict["tags"]
|
164
|
+
del registered_model_dict["latest_versions"]
|
165
|
+
meta_dir = meta_dir or self._get_registered_model_path(registered_model.name)
|
166
|
+
if overwrite:
|
167
|
+
overwrite_yaml(
|
168
|
+
meta_dir,
|
169
|
+
FileStore.META_DATA_FILE_NAME,
|
170
|
+
registered_model_dict,
|
171
|
+
)
|
172
|
+
else:
|
173
|
+
write_yaml(
|
174
|
+
meta_dir,
|
175
|
+
FileStore.META_DATA_FILE_NAME,
|
176
|
+
registered_model_dict,
|
177
|
+
)
|
178
|
+
|
179
|
+
def _update_registered_model_last_updated_time(self, name, updated_time):
|
180
|
+
registered_model = self.get_registered_model(name)
|
181
|
+
registered_model.last_updated_timestamp = updated_time
|
182
|
+
self._save_registered_model_as_meta_file(registered_model)
|
183
|
+
|
184
|
+
def create_registered_model(self, name, tags=None, description=None, deployment_job_id=None):
|
185
|
+
"""
|
186
|
+
Create a new registered model in backend store.
|
187
|
+
|
188
|
+
Args:
|
189
|
+
name: Name of the new model. This is expected to be unique in the backend store.
|
190
|
+
tags: A list of :py:class:`mlflow.entities.model_registry.RegisteredModelTag`
|
191
|
+
instances associated with this registered model.
|
192
|
+
description: Description of the model.
|
193
|
+
deployment_job_id: Optional deployment job ID.
|
194
|
+
|
195
|
+
Returns:
|
196
|
+
A single object of :py:class:`mlflow.entities.model_registry.RegisteredModel`
|
197
|
+
created in the backend.
|
198
|
+
|
199
|
+
"""
|
200
|
+
|
201
|
+
self._check_root_dir()
|
202
|
+
|
203
|
+
_validate_model_name(name)
|
204
|
+
try:
|
205
|
+
self._validate_registered_model_does_not_exist(name)
|
206
|
+
except MlflowException as e:
|
207
|
+
if e.error_code == ErrorCode.Name(RESOURCE_ALREADY_EXISTS):
|
208
|
+
existing_model = self.get_registered_model(name)
|
209
|
+
handle_resource_already_exist_error(
|
210
|
+
name, has_prompt_tag(existing_model._tags), has_prompt_tag(tags)
|
211
|
+
)
|
212
|
+
else:
|
213
|
+
raise
|
214
|
+
|
215
|
+
for tag in tags or []:
|
216
|
+
_validate_registered_model_tag(tag.key, tag.value)
|
217
|
+
meta_dir = self._get_registered_model_path(name)
|
218
|
+
mkdir(meta_dir)
|
219
|
+
creation_time = get_current_time_millis()
|
220
|
+
latest_versions = []
|
221
|
+
registered_model = RegisteredModel(
|
222
|
+
name=name,
|
223
|
+
creation_timestamp=creation_time,
|
224
|
+
last_updated_timestamp=creation_time,
|
225
|
+
description=description,
|
226
|
+
latest_versions=latest_versions,
|
227
|
+
tags=tags,
|
228
|
+
)
|
229
|
+
self._save_registered_model_as_meta_file(
|
230
|
+
registered_model, meta_dir=meta_dir, overwrite=False
|
231
|
+
)
|
232
|
+
if tags is not None:
|
233
|
+
for tag in tags:
|
234
|
+
self.set_registered_model_tag(name, tag)
|
235
|
+
return registered_model
|
236
|
+
|
237
|
+
def _get_registered_model_path(self, name):
|
238
|
+
self._check_root_dir()
|
239
|
+
_validate_model_name(name)
|
240
|
+
return join(self.root_directory, FileStore.MODELS_FOLDER_NAME, name)
|
241
|
+
|
242
|
+
def _get_registered_model_from_path(self, model_path):
|
243
|
+
meta = FileStore._read_yaml(model_path, FileStore.META_DATA_FILE_NAME)
|
244
|
+
meta["tags"] = self.get_all_registered_model_tags_from_path(model_path)
|
245
|
+
meta["aliases"] = self.get_all_registered_model_aliases_from_path(model_path)
|
246
|
+
registered_model = RegisteredModel.from_dictionary(meta)
|
247
|
+
registered_model.latest_versions = self.get_latest_versions(os.path.basename(model_path))
|
248
|
+
return registered_model
|
249
|
+
|
250
|
+
def update_registered_model(self, name, description, deployment_job_id=None):
|
251
|
+
"""
|
252
|
+
Update description of the registered model.
|
253
|
+
|
254
|
+
Args:
|
255
|
+
name: Registered model name.
|
256
|
+
description: New description.
|
257
|
+
deployment_job_id: Optional deployment job ID.
|
258
|
+
|
259
|
+
Returns:
|
260
|
+
A single updated :py:class:`mlflow.entities.model_registry.RegisteredModel` object.
|
261
|
+
|
262
|
+
"""
|
263
|
+
registered_model = self.get_registered_model(name)
|
264
|
+
updated_time = get_current_time_millis()
|
265
|
+
registered_model.description = description
|
266
|
+
registered_model.last_updated_timestamp = updated_time
|
267
|
+
self._save_registered_model_as_meta_file(registered_model)
|
268
|
+
return registered_model
|
269
|
+
|
270
|
+
def rename_registered_model(self, name, new_name):
|
271
|
+
"""
|
272
|
+
Rename the registered model.
|
273
|
+
|
274
|
+
Args:
|
275
|
+
name: Registered model name.
|
276
|
+
new_name: New proposed name.
|
277
|
+
|
278
|
+
Returns:
|
279
|
+
A single updated :py:class:`mlflow.entities.model_registry.RegisteredModel` object.
|
280
|
+
|
281
|
+
"""
|
282
|
+
model_path = self._get_registered_model_path(name)
|
283
|
+
if not exists(model_path):
|
284
|
+
raise MlflowException(
|
285
|
+
f"Registered Model with name={name} not found",
|
286
|
+
RESOURCE_DOES_NOT_EXIST,
|
287
|
+
)
|
288
|
+
registered_model = self._get_registered_model_from_path(model_path)
|
289
|
+
|
290
|
+
new_meta_dir = self._get_registered_model_path(new_name)
|
291
|
+
if not exists(new_meta_dir):
|
292
|
+
mkdir(new_meta_dir)
|
293
|
+
updated_time = get_current_time_millis()
|
294
|
+
registered_model.name = new_name
|
295
|
+
registered_model.last_updated_timestamp = updated_time
|
296
|
+
self._save_registered_model_as_meta_file(
|
297
|
+
registered_model, meta_dir=new_meta_dir, overwrite=False
|
298
|
+
)
|
299
|
+
model_versions = self._list_file_model_versions_under_path(model_path)
|
300
|
+
for mv in model_versions:
|
301
|
+
mv.name = new_name
|
302
|
+
mv.last_updated_timestamp = updated_time
|
303
|
+
new_model_version_dir = join(new_meta_dir, f"version-{mv.version}")
|
304
|
+
mkdir(new_model_version_dir)
|
305
|
+
self._save_model_version_as_meta_file(
|
306
|
+
mv, meta_dir=new_model_version_dir, overwrite=False
|
307
|
+
)
|
308
|
+
if mv.tags is not None:
|
309
|
+
for tag in mv.tags:
|
310
|
+
self.set_model_version_tag(new_name, mv.version, tag)
|
311
|
+
shutil.rmtree(model_path)
|
312
|
+
else:
|
313
|
+
raise MlflowException(
|
314
|
+
f"Registered Model (name={new_name}) already exists.",
|
315
|
+
RESOURCE_ALREADY_EXISTS,
|
316
|
+
)
|
317
|
+
|
318
|
+
return registered_model
|
319
|
+
|
320
|
+
def delete_registered_model(self, name):
|
321
|
+
"""
|
322
|
+
Delete the registered model.
|
323
|
+
Backend raises exception if a registered model with given name does not exist.
|
324
|
+
|
325
|
+
Args:
|
326
|
+
name: Registered model name.
|
327
|
+
|
328
|
+
Returns:
|
329
|
+
None
|
330
|
+
"""
|
331
|
+
meta_dir = self._get_registered_model_path(name)
|
332
|
+
if not exists(meta_dir):
|
333
|
+
raise MlflowException(
|
334
|
+
f"Registered Model with name={name} not found",
|
335
|
+
RESOURCE_DOES_NOT_EXIST,
|
336
|
+
)
|
337
|
+
shutil.rmtree(meta_dir)
|
338
|
+
|
339
|
+
def list_registered_models(self, max_results, page_token):
|
340
|
+
"""
|
341
|
+
List of all registered models.
|
342
|
+
|
343
|
+
Args:
|
344
|
+
max_results: Maximum number of registered models desired.
|
345
|
+
page_token: Token specifying the next page of results. It should be obtained from
|
346
|
+
a ``list_registered_models`` call.
|
347
|
+
|
348
|
+
Returns:
|
349
|
+
A PagedList of :py:class:`mlflow.entities.model_registry.RegisteredModel` objects
|
350
|
+
that satisfy the search expressions. The pagination token for the next page can be
|
351
|
+
obtained via the ``token`` attribute of the object.
|
352
|
+
|
353
|
+
"""
|
354
|
+
return self.search_registered_models(max_results=max_results, page_token=page_token)
|
355
|
+
|
356
|
+
def _list_all_registered_models(self):
|
357
|
+
registered_model_paths = self._get_all_registered_model_paths()
|
358
|
+
registered_models = []
|
359
|
+
for path in registered_model_paths:
|
360
|
+
registered_models.append(self._get_registered_model_from_path(path))
|
361
|
+
return registered_models
|
362
|
+
|
363
|
+
def search_registered_models(
|
364
|
+
self, filter_string=None, max_results=None, order_by=None, page_token=None
|
365
|
+
):
|
366
|
+
"""
|
367
|
+
Search for registered models in backend that satisfy the filter criteria.
|
368
|
+
|
369
|
+
Args:
|
370
|
+
filter_string: Filter query string, defaults to searching all registered models.
|
371
|
+
max_results: Maximum number of registered models desired.
|
372
|
+
order_by: List of column names with ASC|DESC annotation, to be used for ordering
|
373
|
+
matching search results.
|
374
|
+
page_token: Token specifying the next page of results. It should be obtained from
|
375
|
+
a ``search_registered_models`` call.
|
376
|
+
|
377
|
+
Returns:
|
378
|
+
A PagedList of :py:class:`mlflow.entities.model_registry.RegisteredModel` objects
|
379
|
+
that satisfy the search expressions. The pagination token for the next page can be
|
380
|
+
obtained via the ``token`` attribute of the object.
|
381
|
+
"""
|
382
|
+
if not isinstance(max_results, int) or max_results < 1:
|
383
|
+
raise MlflowException(
|
384
|
+
"Invalid value for max_results. It must be a positive integer,"
|
385
|
+
f" but got {max_results}",
|
386
|
+
INVALID_PARAMETER_VALUE,
|
387
|
+
)
|
388
|
+
|
389
|
+
if max_results > SEARCH_REGISTERED_MODEL_MAX_RESULTS_THRESHOLD:
|
390
|
+
raise MlflowException(
|
391
|
+
"Invalid value for request parameter max_results. It must be at most "
|
392
|
+
f"{SEARCH_REGISTERED_MODEL_MAX_RESULTS_THRESHOLD}, but got value {max_results}",
|
393
|
+
INVALID_PARAMETER_VALUE,
|
394
|
+
)
|
395
|
+
|
396
|
+
filter_string = add_prompt_filter_string(filter_string, is_prompt=False)
|
397
|
+
|
398
|
+
registered_models = self._list_all_registered_models()
|
399
|
+
filtered_rms = SearchModelUtils.filter(registered_models, filter_string)
|
400
|
+
sorted_rms = SearchModelUtils.sort(filtered_rms, order_by)
|
401
|
+
start_offset = SearchUtils.parse_start_offset_from_page_token(page_token)
|
402
|
+
final_offset = start_offset + max_results
|
403
|
+
|
404
|
+
paginated_rms = sorted_rms[start_offset:final_offset]
|
405
|
+
next_page_token = None
|
406
|
+
if final_offset < len(sorted_rms):
|
407
|
+
next_page_token = SearchUtils.create_page_token(final_offset)
|
408
|
+
return PagedList(paginated_rms, next_page_token)
|
409
|
+
|
410
|
+
def get_registered_model(self, name):
|
411
|
+
"""
|
412
|
+
Get registered model instance by name.
|
413
|
+
|
414
|
+
Args:
|
415
|
+
name: Registered model name.
|
416
|
+
|
417
|
+
Returns:
|
418
|
+
A single :py:class:`mlflow.entities.model_registry.RegisteredModel` object.
|
419
|
+
"""
|
420
|
+
_validate_model_name(name)
|
421
|
+
model_path = self._get_registered_model_path(name)
|
422
|
+
if not exists(model_path):
|
423
|
+
raise MlflowException(
|
424
|
+
f"Registered Model with name={name} not found",
|
425
|
+
RESOURCE_DOES_NOT_EXIST,
|
426
|
+
)
|
427
|
+
return self._get_registered_model_from_path(model_path)
|
428
|
+
|
429
|
+
def get_latest_versions(self, name, stages=None) -> list[ModelVersion]:
|
430
|
+
"""
|
431
|
+
Latest version models for each requested stage. If no ``stages`` argument is provided,
|
432
|
+
returns the latest version for each stage.
|
433
|
+
|
434
|
+
Args:
|
435
|
+
name: Registered model name.
|
436
|
+
stages: List of desired stages. If input list is None, return latest versions for
|
437
|
+
each stage.
|
438
|
+
|
439
|
+
Returns:
|
440
|
+
List of :py:class:`mlflow.entities.model_registry.ModelVersion` objects.
|
441
|
+
"""
|
442
|
+
registered_model_path = self._get_registered_model_path(name)
|
443
|
+
if not exists(registered_model_path):
|
444
|
+
raise MlflowException(
|
445
|
+
f"Registered Model with name={name} not found",
|
446
|
+
RESOURCE_DOES_NOT_EXIST,
|
447
|
+
)
|
448
|
+
model_versions = self._list_file_model_versions_under_path(registered_model_path)
|
449
|
+
if stages is None or len(stages) == 0:
|
450
|
+
expected_stages = {get_canonical_stage(stage) for stage in ALL_STAGES}
|
451
|
+
else:
|
452
|
+
expected_stages = {get_canonical_stage(stage) for stage in stages}
|
453
|
+
latest_versions = {}
|
454
|
+
for mv in model_versions:
|
455
|
+
if mv.current_stage in expected_stages:
|
456
|
+
if (
|
457
|
+
mv.current_stage not in latest_versions
|
458
|
+
or latest_versions[mv.current_stage].version < mv.version
|
459
|
+
):
|
460
|
+
latest_versions[mv.current_stage] = mv.to_mlflow_entity()
|
461
|
+
|
462
|
+
return [latest_versions[stage] for stage in expected_stages if stage in latest_versions]
|
463
|
+
|
464
|
+
def _get_registered_model_tag_path(self, name, tag_name):
|
465
|
+
_validate_model_name(name)
|
466
|
+
_validate_tag_name(tag_name)
|
467
|
+
registered_model_path = self._get_registered_model_path(name)
|
468
|
+
if not exists(registered_model_path):
|
469
|
+
raise MlflowException(
|
470
|
+
f"Registered Model with name={name} not found",
|
471
|
+
RESOURCE_DOES_NOT_EXIST,
|
472
|
+
)
|
473
|
+
return os.path.join(registered_model_path, FileStore.TAGS_FOLDER_NAME, tag_name)
|
474
|
+
|
475
|
+
def _get_registered_model_tag_from_file(self, parent_path, tag_name):
|
476
|
+
_validate_tag_name(tag_name)
|
477
|
+
tag_data = read_file(parent_path, tag_name)
|
478
|
+
return RegisteredModelTag(tag_name, tag_data)
|
479
|
+
|
480
|
+
def _get_registered_model_alias_from_file(self, parent_path, alias_name):
|
481
|
+
alias_data = read_file(parent_path, alias_name)
|
482
|
+
return RegisteredModelAlias(alias_name, alias_data)
|
483
|
+
|
484
|
+
def _get_resource_files(self, root_dir, subfolder_name):
|
485
|
+
source_dirs = find(root_dir, subfolder_name, full_path=True)
|
486
|
+
if len(source_dirs) == 0:
|
487
|
+
return root_dir, []
|
488
|
+
file_names = []
|
489
|
+
for root, _, files in os.walk(source_dirs[0]):
|
490
|
+
for name in files:
|
491
|
+
abspath = join(root, name)
|
492
|
+
file_names.append(os.path.relpath(abspath, source_dirs[0]))
|
493
|
+
if sys.platform == "win32":
|
494
|
+
# Turn registered models / model versions relative path into metric name.
|
495
|
+
# Registered models and model versions can have '/' in the name.
|
496
|
+
# On windows, '/' is interpreted as a separator.
|
497
|
+
# When the model / model version is read back the path will use '\' for separator.
|
498
|
+
# We need to translate the path into posix path.
|
499
|
+
from mlflow.utils.file_utils import relative_path_to_artifact_path
|
500
|
+
|
501
|
+
file_names = [relative_path_to_artifact_path(x) for x in file_names]
|
502
|
+
return source_dirs[0], file_names
|
503
|
+
|
504
|
+
def get_all_registered_model_tags_from_path(self, model_path):
|
505
|
+
parent_path, tag_files = self._get_resource_files(model_path, FileStore.TAGS_FOLDER_NAME)
|
506
|
+
tags = []
|
507
|
+
for tag_file in tag_files:
|
508
|
+
tags.append(self._get_registered_model_tag_from_file(parent_path, tag_file))
|
509
|
+
return tags
|
510
|
+
|
511
|
+
def get_all_registered_model_aliases_from_path(self, model_path):
|
512
|
+
parent_path, alias_files = self._get_resource_files(
|
513
|
+
model_path, FileStore.REGISTERED_MODELS_ALIASES_FOLDER_NAME
|
514
|
+
)
|
515
|
+
aliases = []
|
516
|
+
for alias_file in alias_files:
|
517
|
+
aliases.append(self._get_registered_model_alias_from_file(parent_path, alias_file))
|
518
|
+
return aliases
|
519
|
+
|
520
|
+
def _writeable_value(self, tag_value):
|
521
|
+
if tag_value is None:
|
522
|
+
return ""
|
523
|
+
elif is_string_type(tag_value):
|
524
|
+
return tag_value
|
525
|
+
else:
|
526
|
+
return f"{tag_value}"
|
527
|
+
|
528
|
+
def set_registered_model_tag(self, name, tag):
|
529
|
+
"""
|
530
|
+
Set a tag for the registered model.
|
531
|
+
|
532
|
+
Args:
|
533
|
+
name: Registered model name.
|
534
|
+
tag: :py:class:`mlflow.entities.model_registry.RegisteredModelTag` instance to log.
|
535
|
+
|
536
|
+
Returns:
|
537
|
+
None
|
538
|
+
"""
|
539
|
+
_validate_registered_model_tag(tag.key, tag.value)
|
540
|
+
tag_path = self._get_registered_model_tag_path(name, tag.key)
|
541
|
+
make_containing_dirs(tag_path)
|
542
|
+
write_to(tag_path, self._writeable_value(tag.value))
|
543
|
+
updated_time = get_current_time_millis()
|
544
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
545
|
+
|
546
|
+
def delete_registered_model_tag(self, name, key):
|
547
|
+
"""
|
548
|
+
Delete a tag associated with the registered model.
|
549
|
+
|
550
|
+
Args:
|
551
|
+
name: Registered model name.
|
552
|
+
key: Registered model tag key.
|
553
|
+
|
554
|
+
Returns:
|
555
|
+
None
|
556
|
+
"""
|
557
|
+
tag_path = self._get_registered_model_tag_path(name, key)
|
558
|
+
if exists(tag_path):
|
559
|
+
os.remove(tag_path)
|
560
|
+
updated_time = get_current_time_millis()
|
561
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
562
|
+
|
563
|
+
# CRUD API for ModelVersion objects
|
564
|
+
|
565
|
+
def _get_registered_model_version_tag_from_file(self, parent_path, tag_name) -> ModelVersionTag:
|
566
|
+
_validate_tag_name(tag_name)
|
567
|
+
tag_data = read_file(parent_path, tag_name)
|
568
|
+
return ModelVersionTag(tag_name, tag_data)
|
569
|
+
|
570
|
+
def _get_model_version_tags_from_dir(self, directory) -> list[ModelVersionTag]:
|
571
|
+
parent_path, tag_files = self._get_resource_files(directory, FileStore.TAGS_FOLDER_NAME)
|
572
|
+
tags = []
|
573
|
+
for tag_file in tag_files:
|
574
|
+
tags.append(self._get_registered_model_version_tag_from_file(parent_path, tag_file))
|
575
|
+
return tags
|
576
|
+
|
577
|
+
def _get_model_version_dir(self, name, version):
|
578
|
+
registered_model_path = self._get_registered_model_path(name)
|
579
|
+
if not exists(registered_model_path):
|
580
|
+
raise MlflowException(
|
581
|
+
f"Registered Model with name={name} not found",
|
582
|
+
RESOURCE_DOES_NOT_EXIST,
|
583
|
+
)
|
584
|
+
return join(registered_model_path, f"version-{version}")
|
585
|
+
|
586
|
+
def _get_model_version_aliases(self, directory):
|
587
|
+
aliases = self.get_all_registered_model_aliases_from_path(os.path.dirname(directory))
|
588
|
+
version = os.path.basename(directory).replace("version-", "")
|
589
|
+
return [alias.alias for alias in aliases if alias.version == version]
|
590
|
+
|
591
|
+
def _get_file_model_version_from_dir(self, directory) -> FileModelVersion:
|
592
|
+
from mlflow.tracking.client import MlflowClient
|
593
|
+
|
594
|
+
meta = FileStore._read_yaml(directory, FileStore.META_DATA_FILE_NAME)
|
595
|
+
meta["tags"] = self._get_model_version_tags_from_dir(directory)
|
596
|
+
meta["aliases"] = self._get_model_version_aliases(directory)
|
597
|
+
# Fetch metrics and params from model ID
|
598
|
+
#
|
599
|
+
# TODO: Propagate tracking URI to file store directly, rather than relying on global
|
600
|
+
# URI (individual MlflowClient instances may have different tracking URIs)
|
601
|
+
if "model_id" in meta:
|
602
|
+
try:
|
603
|
+
model = MlflowClient().get_logged_model(meta["model_id"])
|
604
|
+
meta["metrics"] = model.metrics
|
605
|
+
meta["params"] = model.params
|
606
|
+
except Exception:
|
607
|
+
# TODO: Make this exception handling more specific
|
608
|
+
pass
|
609
|
+
return FileModelVersion.from_dictionary(meta)
|
610
|
+
|
611
|
+
def _save_model_version_as_meta_file(
|
612
|
+
self, model_version: FileModelVersion, meta_dir=None, overwrite=True
|
613
|
+
):
|
614
|
+
model_version_dict = dict(model_version)
|
615
|
+
del model_version_dict["tags"]
|
616
|
+
meta_dir = meta_dir or self._get_model_version_dir(
|
617
|
+
model_version.name, model_version.version
|
618
|
+
)
|
619
|
+
if overwrite:
|
620
|
+
overwrite_yaml(
|
621
|
+
meta_dir,
|
622
|
+
FileStore.META_DATA_FILE_NAME,
|
623
|
+
model_version_dict,
|
624
|
+
)
|
625
|
+
else:
|
626
|
+
write_yaml(
|
627
|
+
meta_dir,
|
628
|
+
FileStore.META_DATA_FILE_NAME,
|
629
|
+
model_version_dict,
|
630
|
+
)
|
631
|
+
|
632
|
+
def create_model_version(
|
633
|
+
self,
|
634
|
+
name,
|
635
|
+
source,
|
636
|
+
run_id=None,
|
637
|
+
tags=None,
|
638
|
+
run_link=None,
|
639
|
+
description=None,
|
640
|
+
local_model_path=None,
|
641
|
+
model_id: Optional[str] = None,
|
642
|
+
) -> ModelVersion:
|
643
|
+
"""
|
644
|
+
Create a new model version from given source and run ID.
|
645
|
+
|
646
|
+
Args:
|
647
|
+
name: Registered model name.
|
648
|
+
source: URI indicating the location of the model artifacts.
|
649
|
+
run_id: Run ID from MLflow tracking server that generated the model.
|
650
|
+
tags: A list of :py:class:`mlflow.entities.model_registry.ModelVersionTag`
|
651
|
+
instances associated with this model version.
|
652
|
+
run_link: Link to the run from an MLflow tracking server that generated this model.
|
653
|
+
description: Description of the version.
|
654
|
+
local_model_path: Unused.
|
655
|
+
model_id: The ID of the model (from an Experiment) that is being promoted to a
|
656
|
+
registered model version, if applicable.
|
657
|
+
|
658
|
+
Returns:
|
659
|
+
A single object of :py:class:`mlflow.entities.model_registry.ModelVersion`
|
660
|
+
created in the backend.
|
661
|
+
|
662
|
+
"""
|
663
|
+
|
664
|
+
def next_version(registered_model_name):
|
665
|
+
path = self._get_registered_model_path(registered_model_name)
|
666
|
+
model_versions = self._list_file_model_versions_under_path(path)
|
667
|
+
if model_versions:
|
668
|
+
return max(mv.version for mv in model_versions) + 1
|
669
|
+
else:
|
670
|
+
return 1
|
671
|
+
|
672
|
+
_validate_model_name(name)
|
673
|
+
for tag in tags or []:
|
674
|
+
_validate_model_version_tag(tag.key, tag.value)
|
675
|
+
storage_location = source
|
676
|
+
if urllib.parse.urlparse(source).scheme == "models":
|
677
|
+
parsed_model_uri = _parse_model_uri(source)
|
678
|
+
try:
|
679
|
+
from mlflow.tracking.client import MlflowClient
|
680
|
+
|
681
|
+
if parsed_model_uri.model_id is not None:
|
682
|
+
# TODO: Propagate tracking URI to file store directly, rather than relying on
|
683
|
+
# global URI (individual MlflowClient instances may have different tracking
|
684
|
+
# URIs)
|
685
|
+
model = MlflowClient().get_logged_model(parsed_model_uri.model_id)
|
686
|
+
storage_location = model.artifact_location
|
687
|
+
run_id = run_id or model.source_run_id
|
688
|
+
else:
|
689
|
+
storage_location = self.get_model_version_download_uri(
|
690
|
+
parsed_model_uri.name, parsed_model_uri.version
|
691
|
+
)
|
692
|
+
except Exception as e:
|
693
|
+
raise MlflowException(
|
694
|
+
f"Unable to fetch model from model URI source artifact location '{source}'."
|
695
|
+
f"Error: {e}"
|
696
|
+
) from e
|
697
|
+
for attempt in range(self.CREATE_MODEL_VERSION_RETRIES):
|
698
|
+
try:
|
699
|
+
creation_time = get_current_time_millis()
|
700
|
+
registered_model = self.get_registered_model(name)
|
701
|
+
registered_model.last_updated_timestamp = creation_time
|
702
|
+
self._save_registered_model_as_meta_file(registered_model)
|
703
|
+
version = next_version(name)
|
704
|
+
model_version = FileModelVersion(
|
705
|
+
name=name,
|
706
|
+
version=version,
|
707
|
+
creation_timestamp=creation_time,
|
708
|
+
last_updated_timestamp=creation_time,
|
709
|
+
description=description,
|
710
|
+
current_stage=STAGE_NONE,
|
711
|
+
source=source,
|
712
|
+
run_id=run_id,
|
713
|
+
run_link=run_link,
|
714
|
+
tags=tags,
|
715
|
+
aliases=[],
|
716
|
+
storage_location=storage_location,
|
717
|
+
model_id=model_id,
|
718
|
+
)
|
719
|
+
model_version_dir = self._get_model_version_dir(name, version)
|
720
|
+
mkdir(model_version_dir)
|
721
|
+
self._save_model_version_as_meta_file(
|
722
|
+
model_version, meta_dir=model_version_dir, overwrite=False
|
723
|
+
)
|
724
|
+
self._save_registered_model_as_meta_file(registered_model)
|
725
|
+
if tags is not None:
|
726
|
+
for tag in tags:
|
727
|
+
self.set_model_version_tag(name, version, tag)
|
728
|
+
return self.get_model_version(name, version)
|
729
|
+
except Exception as e:
|
730
|
+
more_retries = self.CREATE_MODEL_VERSION_RETRIES - attempt - 1
|
731
|
+
logging.warning(
|
732
|
+
"Model Version creation error (name=%s) Retrying %s more time%s.",
|
733
|
+
name,
|
734
|
+
str(more_retries),
|
735
|
+
"s" if more_retries > 1 else "",
|
736
|
+
)
|
737
|
+
if more_retries == 0:
|
738
|
+
raise MlflowException(
|
739
|
+
f"Model Version creation error (name={name}). Error: {e}. Giving up after "
|
740
|
+
f"{self.CREATE_MODEL_VERSION_RETRIES} attempts."
|
741
|
+
)
|
742
|
+
|
743
|
+
def update_model_version(self, name, version, description) -> ModelVersion:
|
744
|
+
"""
|
745
|
+
Update metadata associated with a model version in backend.
|
746
|
+
|
747
|
+
Args:
|
748
|
+
name: Registered model name.
|
749
|
+
version: Registered model version.
|
750
|
+
description: New model description.
|
751
|
+
|
752
|
+
Returns:
|
753
|
+
A single :py:class:`mlflow.entities.model_registry.ModelVersion` object.
|
754
|
+
|
755
|
+
"""
|
756
|
+
updated_time = get_current_time_millis()
|
757
|
+
model_version = self._fetch_file_model_version_if_exists(name=name, version=version)
|
758
|
+
model_version.description = description
|
759
|
+
model_version.last_updated_timestamp = updated_time
|
760
|
+
self._save_model_version_as_meta_file(model_version)
|
761
|
+
return model_version.to_mlflow_entity()
|
762
|
+
|
763
|
+
def transition_model_version_stage(
|
764
|
+
self, name, version, stage, archive_existing_versions
|
765
|
+
) -> ModelVersion:
|
766
|
+
"""
|
767
|
+
Update model version stage.
|
768
|
+
|
769
|
+
Args:
|
770
|
+
name: Registered model name.
|
771
|
+
version: Registered model version.
|
772
|
+
stage: New desired stage for this model version.
|
773
|
+
archive_existing_versions: If this flag is set to ``True``, all existing model
|
774
|
+
versions in the stage will be automatically moved to the "archived" stage. Only
|
775
|
+
valid when ``stage`` is ``"staging"`` or ``"production"`` otherwise an error will be
|
776
|
+
raised.
|
777
|
+
|
778
|
+
Returns:
|
779
|
+
A single :py:class:`mlflow.entities.model_registry.ModelVersion` object.
|
780
|
+
|
781
|
+
"""
|
782
|
+
is_active_stage = get_canonical_stage(stage) in DEFAULT_STAGES_FOR_GET_LATEST_VERSIONS
|
783
|
+
if archive_existing_versions and not is_active_stage:
|
784
|
+
msg_tpl = (
|
785
|
+
"Model version transition cannot archive existing model versions "
|
786
|
+
"because '{}' is not an Active stage. Valid stages are {}"
|
787
|
+
)
|
788
|
+
raise MlflowException(msg_tpl.format(stage, DEFAULT_STAGES_FOR_GET_LATEST_VERSIONS))
|
789
|
+
|
790
|
+
last_updated_time = get_current_time_millis()
|
791
|
+
model_versions = []
|
792
|
+
if archive_existing_versions:
|
793
|
+
registered_model_path = self._get_registered_model_path(name)
|
794
|
+
model_versions = self._list_file_model_versions_under_path(registered_model_path)
|
795
|
+
for mv in model_versions:
|
796
|
+
if mv.version != version and mv.current_stage == get_canonical_stage(stage):
|
797
|
+
mv.current_stage = STAGE_ARCHIVED
|
798
|
+
mv.last_updated_timestamp = last_updated_time
|
799
|
+
self._save_model_version_as_meta_file(mv)
|
800
|
+
|
801
|
+
model_version = self._fetch_file_model_version_if_exists(name, version)
|
802
|
+
model_version.current_stage = get_canonical_stage(stage)
|
803
|
+
model_version.last_updated_timestamp = last_updated_time
|
804
|
+
self._save_model_version_as_meta_file(model_version)
|
805
|
+
self._update_registered_model_last_updated_time(name, last_updated_time)
|
806
|
+
return model_version.to_mlflow_entity()
|
807
|
+
|
808
|
+
def delete_model_version(self, name, version):
|
809
|
+
"""
|
810
|
+
Delete model version in backend.
|
811
|
+
|
812
|
+
Args:
|
813
|
+
name: Registered model name.
|
814
|
+
version: Registered model version.
|
815
|
+
|
816
|
+
Returns:
|
817
|
+
None
|
818
|
+
"""
|
819
|
+
model_version = self._fetch_file_model_version_if_exists(name=name, version=version)
|
820
|
+
model_version.current_stage = STAGE_DELETED_INTERNAL
|
821
|
+
updated_time = get_current_time_millis()
|
822
|
+
model_version.last_updated_timestamp = updated_time
|
823
|
+
self._save_model_version_as_meta_file(model_version)
|
824
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
825
|
+
for alias in model_version.aliases:
|
826
|
+
self.delete_registered_model_alias(name, alias)
|
827
|
+
|
828
|
+
def _fetch_file_model_version_if_exists(self, name, version) -> FileModelVersion:
|
829
|
+
_validate_model_name(name)
|
830
|
+
_validate_model_version(version)
|
831
|
+
registered_model_version_dir = self._get_model_version_dir(name, version)
|
832
|
+
if not exists(registered_model_version_dir):
|
833
|
+
raise MlflowException(
|
834
|
+
f"Model Version (name={name}, version={version}) not found",
|
835
|
+
RESOURCE_DOES_NOT_EXIST,
|
836
|
+
)
|
837
|
+
model_version = self._get_file_model_version_from_dir(registered_model_version_dir)
|
838
|
+
if model_version.current_stage == STAGE_DELETED_INTERNAL:
|
839
|
+
raise MlflowException(
|
840
|
+
f"Model Version (name={name}, version={version}) not found",
|
841
|
+
RESOURCE_DOES_NOT_EXIST,
|
842
|
+
)
|
843
|
+
return model_version
|
844
|
+
|
845
|
+
def get_model_version(self, name, version) -> ModelVersion:
|
846
|
+
"""
|
847
|
+
Get the model version instance by name and version.
|
848
|
+
|
849
|
+
Args:
|
850
|
+
name: Registered model name.
|
851
|
+
version: Registered model version.
|
852
|
+
|
853
|
+
Returns:
|
854
|
+
A single :py:class:`mlflow.entities.model_registry.ModelVersion` object.
|
855
|
+
"""
|
856
|
+
return self._fetch_file_model_version_if_exists(name, version).to_mlflow_entity()
|
857
|
+
|
858
|
+
def get_model_version_download_uri(self, name, version) -> str:
|
859
|
+
"""
|
860
|
+
Get the download location in Model Registry for this model version.
|
861
|
+
NOTE: For first version of Model Registry, since the models are not copied over to another
|
862
|
+
location, download URI points to input source path.
|
863
|
+
|
864
|
+
Args:
|
865
|
+
name: Registered model name.
|
866
|
+
version: Registered model version.
|
867
|
+
|
868
|
+
Returns:
|
869
|
+
A single URI location that allows reads for downloading.
|
870
|
+
"""
|
871
|
+
model_version = self._fetch_file_model_version_if_exists(name, version)
|
872
|
+
return model_version.storage_location or model_version.source
|
873
|
+
|
874
|
+
def _get_all_registered_model_paths(self):
|
875
|
+
self._check_root_dir()
|
876
|
+
return list_subdirs(join(self.root_directory, FileStore.MODELS_FOLDER_NAME), full_path=True)
|
877
|
+
|
878
|
+
def _list_file_model_versions_under_path(self, path) -> list[FileModelVersion]:
|
879
|
+
model_versions = []
|
880
|
+
model_version_dirs = list_all(
|
881
|
+
path,
|
882
|
+
filter_func=lambda x: os.path.isdir(x)
|
883
|
+
and os.path.basename(os.path.normpath(x)).startswith("version-"),
|
884
|
+
full_path=True,
|
885
|
+
)
|
886
|
+
for directory in model_version_dirs:
|
887
|
+
model_versions.append(self._get_file_model_version_from_dir(directory))
|
888
|
+
return model_versions
|
889
|
+
|
890
|
+
def search_model_versions(
|
891
|
+
self, filter_string=None, max_results=None, order_by=None, page_token=None
|
892
|
+
) -> list[ModelVersion]:
|
893
|
+
"""
|
894
|
+
Search for model versions in backend that satisfy the filter criteria.
|
895
|
+
|
896
|
+
Args:
|
897
|
+
filter_string: A filter string expression. Currently supports a single filter
|
898
|
+
condition either name of model like ``name = 'model_name'`` or
|
899
|
+
``run_id = '...'``.
|
900
|
+
max_results: Maximum number of model versions desired.
|
901
|
+
order_by: List of column names with ASC|DESC annotation, to be used for ordering
|
902
|
+
matching search results.
|
903
|
+
page_token: Token specifying the next page of results. It should be obtained from
|
904
|
+
a ``search_model_versions`` call.
|
905
|
+
|
906
|
+
Returns:
|
907
|
+
A PagedList of :py:class:`mlflow.entities.model_registry.ModelVersion`
|
908
|
+
objects that satisfy the search expressions. The pagination token for the next
|
909
|
+
page can be obtained via the ``token`` attribute of the object.
|
910
|
+
|
911
|
+
"""
|
912
|
+
if not isinstance(max_results, int) or max_results < 1:
|
913
|
+
raise MlflowException(
|
914
|
+
"Invalid value for max_results. It must be a positive integer,"
|
915
|
+
f" but got {max_results}",
|
916
|
+
INVALID_PARAMETER_VALUE,
|
917
|
+
)
|
918
|
+
|
919
|
+
if max_results > SEARCH_MODEL_VERSION_MAX_RESULTS_THRESHOLD:
|
920
|
+
raise MlflowException(
|
921
|
+
"Invalid value for request parameter max_results. It must be at most "
|
922
|
+
f"{SEARCH_MODEL_VERSION_MAX_RESULTS_THRESHOLD}, but got value {max_results}",
|
923
|
+
INVALID_PARAMETER_VALUE,
|
924
|
+
)
|
925
|
+
|
926
|
+
registered_model_paths = self._get_all_registered_model_paths()
|
927
|
+
model_versions = []
|
928
|
+
for path in registered_model_paths:
|
929
|
+
model_versions.extend(
|
930
|
+
file_mv.to_mlflow_entity()
|
931
|
+
for file_mv in self._list_file_model_versions_under_path(path)
|
932
|
+
)
|
933
|
+
filter_string = add_prompt_filter_string(filter_string, is_prompt=False)
|
934
|
+
filtered_mvs = SearchModelVersionUtils.filter(model_versions, filter_string)
|
935
|
+
|
936
|
+
sorted_mvs = SearchModelVersionUtils.sort(
|
937
|
+
filtered_mvs,
|
938
|
+
order_by or ["last_updated_timestamp DESC", "name ASC", "version_number DESC"],
|
939
|
+
)
|
940
|
+
start_offset = SearchUtils.parse_start_offset_from_page_token(page_token)
|
941
|
+
final_offset = start_offset + max_results
|
942
|
+
|
943
|
+
paginated_mvs = sorted_mvs[start_offset:final_offset]
|
944
|
+
next_page_token = None
|
945
|
+
if final_offset < len(sorted_mvs):
|
946
|
+
next_page_token = SearchUtils.create_page_token(final_offset)
|
947
|
+
return PagedList(paginated_mvs, next_page_token)
|
948
|
+
|
949
|
+
def _get_registered_model_version_tag_path(self, name, version, tag_name):
|
950
|
+
_validate_tag_name(tag_name)
|
951
|
+
self._fetch_file_model_version_if_exists(name, version)
|
952
|
+
registered_model_version_path = self._get_model_version_dir(name, version)
|
953
|
+
return os.path.join(registered_model_version_path, FileStore.TAGS_FOLDER_NAME, tag_name)
|
954
|
+
|
955
|
+
def set_model_version_tag(self, name, version, tag):
|
956
|
+
"""
|
957
|
+
Set a tag for the model version.
|
958
|
+
|
959
|
+
Args:
|
960
|
+
name: Registered model name.
|
961
|
+
version: Registered model version.
|
962
|
+
tag: :py:class:`mlflow.entities.model_registry.ModelVersionTag` instance to log.
|
963
|
+
|
964
|
+
Returns:
|
965
|
+
None
|
966
|
+
"""
|
967
|
+
_validate_model_version_tag(tag.key, tag.value)
|
968
|
+
tag_path = self._get_registered_model_version_tag_path(name, version, tag.key)
|
969
|
+
make_containing_dirs(tag_path)
|
970
|
+
write_to(tag_path, self._writeable_value(tag.value))
|
971
|
+
updated_time = get_current_time_millis()
|
972
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
973
|
+
|
974
|
+
def delete_model_version_tag(self, name, version, key):
|
975
|
+
"""
|
976
|
+
Delete a tag associated with the model version.
|
977
|
+
|
978
|
+
Args:
|
979
|
+
name: Registered model name.
|
980
|
+
version: Registered model version.
|
981
|
+
key: Tag key.
|
982
|
+
|
983
|
+
Returns:
|
984
|
+
None
|
985
|
+
"""
|
986
|
+
tag_path = self._get_registered_model_version_tag_path(name, version, key)
|
987
|
+
if exists(tag_path):
|
988
|
+
os.remove(tag_path)
|
989
|
+
updated_time = get_current_time_millis()
|
990
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
991
|
+
|
992
|
+
def _get_registered_model_alias_path(self, name, alias):
|
993
|
+
_validate_model_name(name)
|
994
|
+
_validate_model_alias_name(alias)
|
995
|
+
registered_model_path = self._get_registered_model_path(name)
|
996
|
+
if not exists(registered_model_path):
|
997
|
+
raise MlflowException(
|
998
|
+
f"Registered Model with name={name} not found",
|
999
|
+
RESOURCE_DOES_NOT_EXIST,
|
1000
|
+
)
|
1001
|
+
return os.path.join(
|
1002
|
+
registered_model_path, FileStore.REGISTERED_MODELS_ALIASES_FOLDER_NAME, alias
|
1003
|
+
)
|
1004
|
+
|
1005
|
+
def set_registered_model_alias(self, name, alias, version):
|
1006
|
+
"""
|
1007
|
+
Set a registered model alias pointing to a model version.
|
1008
|
+
|
1009
|
+
Args:
|
1010
|
+
name: Registered model name.
|
1011
|
+
alias: Name of the alias.
|
1012
|
+
version: Registered model version number.
|
1013
|
+
|
1014
|
+
Returns:
|
1015
|
+
None
|
1016
|
+
"""
|
1017
|
+
alias_path = self._get_registered_model_alias_path(name, alias)
|
1018
|
+
self._fetch_file_model_version_if_exists(name, version)
|
1019
|
+
make_containing_dirs(alias_path)
|
1020
|
+
write_to(alias_path, self._writeable_value(version))
|
1021
|
+
updated_time = get_current_time_millis()
|
1022
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
1023
|
+
|
1024
|
+
def delete_registered_model_alias(self, name, alias):
|
1025
|
+
"""
|
1026
|
+
Delete an alias associated with a registered model.
|
1027
|
+
|
1028
|
+
Args:
|
1029
|
+
name: Registered model name.
|
1030
|
+
alias: Name of the alias.
|
1031
|
+
|
1032
|
+
Returns:
|
1033
|
+
None
|
1034
|
+
"""
|
1035
|
+
alias_path = self._get_registered_model_alias_path(name, alias)
|
1036
|
+
if exists(alias_path):
|
1037
|
+
os.remove(alias_path)
|
1038
|
+
updated_time = get_current_time_millis()
|
1039
|
+
self._update_registered_model_last_updated_time(name, updated_time)
|
1040
|
+
|
1041
|
+
def get_model_version_by_alias(self, name, alias) -> ModelVersion:
|
1042
|
+
"""
|
1043
|
+
Get the model version instance by name and alias.
|
1044
|
+
|
1045
|
+
Args:
|
1046
|
+
name: Registered model name.
|
1047
|
+
alias: Name of the alias.
|
1048
|
+
|
1049
|
+
Returns:
|
1050
|
+
A single :py:class:`mlflow.entities.model_registry.ModelVersion` object.
|
1051
|
+
"""
|
1052
|
+
alias_path = self._get_registered_model_alias_path(name, alias)
|
1053
|
+
if exists(alias_path):
|
1054
|
+
version = read_file(os.path.dirname(alias_path), os.path.basename(alias_path))
|
1055
|
+
return self.get_model_version(name, version)
|
1056
|
+
else:
|
1057
|
+
raise MlflowException(
|
1058
|
+
f"Registered model alias {alias} not found.", INVALID_PARAMETER_VALUE
|
1059
|
+
)
|
1060
|
+
|
1061
|
+
@staticmethod
|
1062
|
+
def _read_yaml(root, file_name, retries=2):
|
1063
|
+
"""
|
1064
|
+
Read data from yaml file and return as dictionary, retrying up to
|
1065
|
+
a specified number of times if the file contents are unexpectedly
|
1066
|
+
empty due to a concurrent write.
|
1067
|
+
|
1068
|
+
Args:
|
1069
|
+
root: Directory name.
|
1070
|
+
file_name: File name. Expects to have '.yaml' extension.
|
1071
|
+
retries: The number of times to retry for unexpected empty content.
|
1072
|
+
|
1073
|
+
Returns:
|
1074
|
+
Data in yaml file as dictionary.
|
1075
|
+
"""
|
1076
|
+
|
1077
|
+
def _read_helper(root, file_name, attempts_remaining=2):
|
1078
|
+
result = read_yaml(root, file_name)
|
1079
|
+
if result is not None or attempts_remaining == 0:
|
1080
|
+
return result
|
1081
|
+
else:
|
1082
|
+
time.sleep(0.1 * (3 - attempts_remaining))
|
1083
|
+
return _read_helper(root, file_name, attempts_remaining - 1)
|
1084
|
+
|
1085
|
+
return _read_helper(root, file_name, attempts_remaining=retries)
|
1086
|
+
|
1087
|
+
def _await_model_version_creation(self, mv, await_creation_for):
|
1088
|
+
"""
|
1089
|
+
Does not wait for the model version to become READY as a successful creation will
|
1090
|
+
immediately place the model version in a READY state.
|
1091
|
+
"""
|