truefoundry 0.5.3rc4__py3-none-any.whl → 0.5.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/__init__.py +10 -1
- truefoundry/autodeploy/cli.py +2 -2
- truefoundry/cli/__main__.py +0 -4
- truefoundry/cli/util.py +12 -3
- truefoundry/common/auth_service_client.py +7 -4
- truefoundry/common/constants.py +3 -1
- truefoundry/common/credential_provider.py +7 -8
- truefoundry/common/exceptions.py +11 -7
- truefoundry/common/request_utils.py +96 -14
- truefoundry/common/servicefoundry_client.py +31 -29
- truefoundry/common/session.py +93 -0
- truefoundry/common/storage_provider_utils.py +331 -0
- truefoundry/common/utils.py +9 -9
- truefoundry/common/warnings.py +21 -0
- truefoundry/deploy/builder/builders/tfy_python_buildpack/dockerfile_template.py +8 -20
- truefoundry/deploy/cli/commands/deploy_command.py +4 -4
- truefoundry/deploy/lib/clients/servicefoundry_client.py +14 -38
- truefoundry/deploy/lib/dao/application.py +2 -2
- truefoundry/deploy/lib/dao/workspace.py +1 -1
- truefoundry/deploy/lib/session.py +8 -1
- truefoundry/deploy/v2/lib/deploy.py +2 -2
- truefoundry/deploy/v2/lib/deploy_workflow.py +1 -1
- truefoundry/deploy/v2/lib/patched_models.py +70 -4
- truefoundry/deploy/v2/lib/source.py +2 -1
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +33 -297
- truefoundry/ml/autogen/client/__init__.py +2 -2
- truefoundry/ml/autogen/client/api/mlfoundry_artifacts_api.py +18 -16
- truefoundry/ml/autogen/client/models/__init__.py +2 -2
- truefoundry/ml/autogen/client/models/artifact_version_manifest.py +23 -5
- truefoundry/ml/autogen/client/models/{get_artifact_tags_response_dto.py → get_artifact_version_aliases_response_dto.py} +12 -10
- truefoundry/ml/autogen/client/models/model_version_manifest.py +16 -5
- truefoundry/ml/autogen/client_README.md +2 -2
- truefoundry/ml/autogen/entities/artifacts.py +4 -9
- truefoundry/ml/clients/servicefoundry_client.py +36 -15
- truefoundry/ml/exceptions.py +2 -1
- truefoundry/ml/log_types/artifacts/artifact.py +16 -15
- truefoundry/ml/log_types/artifacts/model.py +20 -19
- truefoundry/ml/log_types/artifacts/utils.py +2 -2
- truefoundry/ml/mlfoundry_api.py +6 -38
- truefoundry/ml/mlfoundry_run.py +6 -15
- truefoundry/ml/model_framework.py +2 -1
- truefoundry/ml/session.py +69 -97
- truefoundry/workflow/remote_filesystem/tfy_signed_url_client.py +42 -9
- truefoundry/workflow/remote_filesystem/tfy_signed_url_fs.py +126 -7
- {truefoundry-0.5.3rc4.dist-info → truefoundry-0.5.4.dist-info}/METADATA +2 -2
- {truefoundry-0.5.3rc4.dist-info → truefoundry-0.5.4.dist-info}/RECORD +48 -54
- {truefoundry-0.5.3rc4.dist-info → truefoundry-0.5.4.dist-info}/WHEEL +1 -1
- truefoundry/cli/commands/pat.py +0 -24
- truefoundry/deploy/lib/auth/servicefoundry_session.py +0 -61
- truefoundry/gateway/__init__.py +0 -1
- truefoundry/gateway/cli/cli.py +0 -51
- truefoundry/gateway/lib/client.py +0 -51
- truefoundry/gateway/lib/entities.py +0 -33
- truefoundry/gateway/lib/models.py +0 -67
- truefoundry/ml/clients/entities.py +0 -8
- truefoundry/ml/clients/utils.py +0 -122
- {truefoundry-0.5.3rc4.dist-info → truefoundry-0.5.4.dist-info}/entry_points.txt +0 -0
|
@@ -155,8 +155,8 @@ from truefoundry.ml.autogen.client.models.finalize_artifact_version_request_dto
|
|
|
155
155
|
FinalizeArtifactVersionRequestDto,
|
|
156
156
|
)
|
|
157
157
|
from truefoundry.ml.autogen.client.models.framework import Framework
|
|
158
|
-
from truefoundry.ml.autogen.client.models.
|
|
159
|
-
|
|
158
|
+
from truefoundry.ml.autogen.client.models.get_artifact_version_aliases_response_dto import (
|
|
159
|
+
GetArtifactVersionAliasesResponseDto,
|
|
160
160
|
)
|
|
161
161
|
from truefoundry.ml.autogen.client.models.get_experiment_response_dto import (
|
|
162
162
|
GetExperimentResponseDto,
|
|
@@ -24,7 +24,6 @@ from truefoundry.pydantic_v1 import (
|
|
|
24
24
|
Field,
|
|
25
25
|
StrictStr,
|
|
26
26
|
conint,
|
|
27
|
-
conlist,
|
|
28
27
|
constr,
|
|
29
28
|
validator,
|
|
30
29
|
)
|
|
@@ -43,16 +42,35 @@ class ArtifactVersionManifest(BaseModel):
|
|
|
43
42
|
default=...,
|
|
44
43
|
description="+label=Metadata +docs=Metadata for the artifact or model version +usage=Metadata for the artifact or model version +uiType=JsonInput",
|
|
45
44
|
)
|
|
46
|
-
|
|
45
|
+
version_alias: Optional[constr(strict=True, max_length=128)] = Field(
|
|
47
46
|
default=None,
|
|
48
|
-
description="+label=
|
|
47
|
+
description="+label=Version Alias +usage=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc) +docs=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc) +message=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc)",
|
|
49
48
|
)
|
|
50
49
|
type: Optional[StrictStr] = "artifact-version"
|
|
51
50
|
source: Source = Field(...)
|
|
52
51
|
step: Optional[conint(strict=True, ge=0)] = Field(
|
|
53
52
|
default=0, description="+label=Step"
|
|
54
53
|
)
|
|
55
|
-
__properties = [
|
|
54
|
+
__properties = [
|
|
55
|
+
"description",
|
|
56
|
+
"metadata",
|
|
57
|
+
"version_alias",
|
|
58
|
+
"type",
|
|
59
|
+
"source",
|
|
60
|
+
"step",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
@validator("version_alias")
|
|
64
|
+
def version_alias_validate_regular_expression(cls, value):
|
|
65
|
+
"""Validates the regular expression"""
|
|
66
|
+
if value is None:
|
|
67
|
+
return value
|
|
68
|
+
|
|
69
|
+
if not re.match(r"^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$", value):
|
|
70
|
+
raise ValueError(
|
|
71
|
+
r"must validate the regular expression /^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$/"
|
|
72
|
+
)
|
|
73
|
+
return value
|
|
56
74
|
|
|
57
75
|
@validator("type")
|
|
58
76
|
def type_validate_enum(cls, value):
|
|
@@ -104,7 +122,7 @@ class ArtifactVersionManifest(BaseModel):
|
|
|
104
122
|
{
|
|
105
123
|
"description": obj.get("description"),
|
|
106
124
|
"metadata": obj.get("metadata"),
|
|
107
|
-
"
|
|
125
|
+
"version_alias": obj.get("version_alias"),
|
|
108
126
|
"type": obj.get("type")
|
|
109
127
|
if obj.get("type") is not None
|
|
110
128
|
else "artifact-version",
|
|
@@ -20,13 +20,13 @@ import re # noqa: F401
|
|
|
20
20
|
from truefoundry.pydantic_v1 import BaseModel, Field, StrictStr, conlist
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
class
|
|
23
|
+
class GetArtifactVersionAliasesResponseDto(BaseModel):
|
|
24
24
|
"""
|
|
25
|
-
|
|
25
|
+
GetArtifactVersionAliasesResponseDto
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
__properties = ["
|
|
28
|
+
version_aliases: conlist(StrictStr) = Field(...)
|
|
29
|
+
__properties = ["version_aliases"]
|
|
30
30
|
|
|
31
31
|
class Config:
|
|
32
32
|
"""Pydantic configuration"""
|
|
@@ -43,8 +43,8 @@ class GetArtifactTagsResponseDto(BaseModel):
|
|
|
43
43
|
return json.dumps(self.to_dict())
|
|
44
44
|
|
|
45
45
|
@classmethod
|
|
46
|
-
def from_json(cls, json_str: str) ->
|
|
47
|
-
"""Create an instance of
|
|
46
|
+
def from_json(cls, json_str: str) -> GetArtifactVersionAliasesResponseDto:
|
|
47
|
+
"""Create an instance of GetArtifactVersionAliasesResponseDto from a JSON string"""
|
|
48
48
|
return cls.from_dict(json.loads(json_str))
|
|
49
49
|
|
|
50
50
|
def to_dict(self):
|
|
@@ -53,13 +53,15 @@ class GetArtifactTagsResponseDto(BaseModel):
|
|
|
53
53
|
return _dict
|
|
54
54
|
|
|
55
55
|
@classmethod
|
|
56
|
-
def from_dict(cls, obj: dict) ->
|
|
57
|
-
"""Create an instance of
|
|
56
|
+
def from_dict(cls, obj: dict) -> GetArtifactVersionAliasesResponseDto:
|
|
57
|
+
"""Create an instance of GetArtifactVersionAliasesResponseDto from a dict"""
|
|
58
58
|
if obj is None:
|
|
59
59
|
return None
|
|
60
60
|
|
|
61
61
|
if not isinstance(obj, dict):
|
|
62
|
-
return
|
|
62
|
+
return GetArtifactVersionAliasesResponseDto.parse_obj(obj)
|
|
63
63
|
|
|
64
|
-
_obj =
|
|
64
|
+
_obj = GetArtifactVersionAliasesResponseDto.parse_obj(
|
|
65
|
+
{"version_aliases": obj.get("version_aliases")}
|
|
66
|
+
)
|
|
65
67
|
return _obj
|
|
@@ -28,7 +28,6 @@ from truefoundry.pydantic_v1 import (
|
|
|
28
28
|
Field,
|
|
29
29
|
StrictStr,
|
|
30
30
|
conint,
|
|
31
|
-
conlist,
|
|
32
31
|
constr,
|
|
33
32
|
validator,
|
|
34
33
|
)
|
|
@@ -47,9 +46,9 @@ class ModelVersionManifest(BaseModel):
|
|
|
47
46
|
default=...,
|
|
48
47
|
description="+label=Metadata +docs=Metadata for the artifact or model version +usage=Metadata for the artifact or model version +uiType=JsonInput",
|
|
49
48
|
)
|
|
50
|
-
|
|
49
|
+
version_alias: Optional[constr(strict=True, max_length=128)] = Field(
|
|
51
50
|
default=None,
|
|
52
|
-
description="+label=
|
|
51
|
+
description="+label=Version Alias +usage=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc) +docs=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc) +message=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc)",
|
|
53
52
|
)
|
|
54
53
|
type: Optional[StrictStr] = "model-version"
|
|
55
54
|
source: Source1 = Field(...)
|
|
@@ -61,7 +60,7 @@ class ModelVersionManifest(BaseModel):
|
|
|
61
60
|
__properties = [
|
|
62
61
|
"description",
|
|
63
62
|
"metadata",
|
|
64
|
-
"
|
|
63
|
+
"version_alias",
|
|
65
64
|
"type",
|
|
66
65
|
"source",
|
|
67
66
|
"framework",
|
|
@@ -69,6 +68,18 @@ class ModelVersionManifest(BaseModel):
|
|
|
69
68
|
"step",
|
|
70
69
|
]
|
|
71
70
|
|
|
71
|
+
@validator("version_alias")
|
|
72
|
+
def version_alias_validate_regular_expression(cls, value):
|
|
73
|
+
"""Validates the regular expression"""
|
|
74
|
+
if value is None:
|
|
75
|
+
return value
|
|
76
|
+
|
|
77
|
+
if not re.match(r"^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$", value):
|
|
78
|
+
raise ValueError(
|
|
79
|
+
r"must validate the regular expression /^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$/"
|
|
80
|
+
)
|
|
81
|
+
return value
|
|
82
|
+
|
|
72
83
|
@validator("type")
|
|
73
84
|
def type_validate_enum(cls, value):
|
|
74
85
|
"""Validates the enum"""
|
|
@@ -125,7 +136,7 @@ class ModelVersionManifest(BaseModel):
|
|
|
125
136
|
{
|
|
126
137
|
"description": obj.get("description"),
|
|
127
138
|
"metadata": obj.get("metadata"),
|
|
128
|
-
"
|
|
139
|
+
"version_alias": obj.get("version_alias"),
|
|
129
140
|
"type": obj.get("type")
|
|
130
141
|
if obj.get("type") is not None
|
|
131
142
|
else "model-version",
|
|
@@ -121,7 +121,7 @@ Class | Method | HTTP request | Description
|
|
|
121
121
|
*MlfoundryArtifactsApi* | [**get_signed_urls_for_dataset_write_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#get_signed_urls_for_dataset_write_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/datasets/get-signed-urls-for-write | Get Signed Urls For Dataset Write
|
|
122
122
|
*MlfoundryArtifactsApi* | [**get_signed_urls_for_read_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#get_signed_urls_for_read_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/artifact-versions/get-signed-urls-for-read | Get Signed Urls For Read
|
|
123
123
|
*MlfoundryArtifactsApi* | [**get_signed_urls_for_write_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#get_signed_urls_for_write_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/artifact-versions/get-signed-urls-for-write | Get Signed Urls For Write
|
|
124
|
-
*MlfoundryArtifactsApi* | [**
|
|
124
|
+
*MlfoundryArtifactsApi* | [**get_version_aliases_for_artifact_get**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#get_version_aliases_for_artifact_get) | **GET** /api/2.0/mlflow/mlfoundry-artifacts/artifacts/get-version-aliases | Get Version Aliases For Artifact
|
|
125
125
|
*MlfoundryArtifactsApi* | [**list_artifact_versions_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#list_artifact_versions_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/artifact-versions/list | List Artifact Versions
|
|
126
126
|
*MlfoundryArtifactsApi* | [**list_artifacts_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#list_artifacts_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/artifacts/list | List Artifacts
|
|
127
127
|
*MlfoundryArtifactsApi* | [**list_datasets_post**](truefoundry/ml/autogen/client/docs/MlfoundryArtifactsApi.md#list_datasets_post) | **POST** /api/2.0/mlflow/mlfoundry-artifacts/datasets/list | List Datasets
|
|
@@ -220,7 +220,7 @@ Class | Method | HTTP request | Description
|
|
|
220
220
|
- [FileInfoDto](truefoundry/ml/autogen/client/docs/FileInfoDto.md)
|
|
221
221
|
- [FinalizeArtifactVersionRequestDto](truefoundry/ml/autogen/client/docs/FinalizeArtifactVersionRequestDto.md)
|
|
222
222
|
- [Framework](truefoundry/ml/autogen/client/docs/Framework.md)
|
|
223
|
-
- [
|
|
223
|
+
- [GetArtifactVersionAliasesResponseDto](truefoundry/ml/autogen/client/docs/GetArtifactVersionAliasesResponseDto.md)
|
|
224
224
|
- [GetExperimentResponseDto](truefoundry/ml/autogen/client/docs/GetExperimentResponseDto.md)
|
|
225
225
|
- [GetLatestRunLogResponseDto](truefoundry/ml/autogen/client/docs/GetLatestRunLogResponseDto.md)
|
|
226
226
|
- [GetMetricHistoryResponse](truefoundry/ml/autogen/client/docs/GetMetricHistoryResponse.md)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# generated by datamodel-codegen:
|
|
2
2
|
# filename: artifacts.json
|
|
3
|
-
# timestamp:
|
|
3
|
+
# timestamp: 2025-01-03T09:11:54+00:00
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
@@ -58,16 +58,11 @@ class BaseArtifactVersion(BaseModel):
|
|
|
58
58
|
...,
|
|
59
59
|
description="+label=Metadata\n+docs=Metadata for the artifact or model version\n+usage=Metadata for the artifact or model version\n+uiType=JsonInput",
|
|
60
60
|
)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
constr(
|
|
64
|
-
regex=r"^(?!^\d+$)(?!.*[._-]{2,})[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*$",
|
|
65
|
-
max_length=128,
|
|
66
|
-
)
|
|
67
|
-
]
|
|
61
|
+
version_alias: Optional[
|
|
62
|
+
constr(regex=r"^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$", max_length=128)
|
|
68
63
|
] = Field(
|
|
69
64
|
None,
|
|
70
|
-
description="+label=
|
|
65
|
+
description="+label=Version Alias\n+usage=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc)\n+docs=The version alias for artifact or model version which should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc)\n+message=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v-prod, v-dev, etc)",
|
|
71
66
|
)
|
|
72
67
|
|
|
73
68
|
|
|
@@ -1,36 +1,57 @@
|
|
|
1
|
-
|
|
1
|
+
import functools
|
|
2
2
|
|
|
3
3
|
from truefoundry.common.constants import (
|
|
4
4
|
SERVICEFOUNDRY_CLIENT_MAX_RETRIES,
|
|
5
5
|
VERSION_PREFIX,
|
|
6
6
|
)
|
|
7
|
+
from truefoundry.common.exceptions import HttpRequestException
|
|
8
|
+
from truefoundry.common.request_utils import (
|
|
9
|
+
http_request,
|
|
10
|
+
request_handling,
|
|
11
|
+
requests_retry_session,
|
|
12
|
+
)
|
|
7
13
|
from truefoundry.common.servicefoundry_client import (
|
|
8
14
|
ServiceFoundryServiceClient as BaseServiceFoundryServiceClient,
|
|
9
15
|
)
|
|
10
|
-
from truefoundry.ml.clients.entities import (
|
|
11
|
-
HostCreds,
|
|
12
|
-
)
|
|
13
|
-
from truefoundry.ml.clients.utils import http_request_safe
|
|
14
16
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
def __init__(self, tfy_host: str, token: str):
|
|
21
|
+
super().__init__(tfy_host=tfy_host)
|
|
22
|
+
self._token = token
|
|
23
|
+
|
|
24
|
+
@functools.cached_property
|
|
25
|
+
def _min_cli_version_required(self) -> str:
|
|
26
|
+
# TODO (chiragjn): read the mlfoundry min cli version from the config?
|
|
27
|
+
return self.python_sdk_config.truefoundry_cli_min_version
|
|
22
28
|
|
|
23
29
|
def get_integration_from_id(self, integration_id: str):
|
|
24
30
|
integration_id = integration_id or ""
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
endpoint=f"{VERSION_PREFIX}/provider-accounts/provider-integrations",
|
|
28
|
-
params={"id": integration_id, "type": "blob-storage"},
|
|
31
|
+
session = requests_retry_session(retries=SERVICEFOUNDRY_CLIENT_MAX_RETRIES)
|
|
32
|
+
response = http_request(
|
|
29
33
|
method="get",
|
|
34
|
+
url=f"{self._api_server_url}/{VERSION_PREFIX}/provider-accounts/provider-integrations",
|
|
35
|
+
token=self._token,
|
|
30
36
|
timeout=3,
|
|
31
|
-
|
|
37
|
+
params={"id": integration_id, "type": "blob-storage"},
|
|
38
|
+
session=session,
|
|
32
39
|
)
|
|
33
|
-
|
|
40
|
+
|
|
41
|
+
try:
|
|
42
|
+
data = request_handling(response)
|
|
43
|
+
assert isinstance(data, dict)
|
|
44
|
+
except HttpRequestException as he:
|
|
45
|
+
raise MlFoundryException(
|
|
46
|
+
f"Failed to get storage integration from id: {integration_id}. Error: {he.message}",
|
|
47
|
+
status_code=he.status_code,
|
|
48
|
+
) from None
|
|
49
|
+
except Exception as e:
|
|
50
|
+
raise MlFoundryException(
|
|
51
|
+
f"Failed to get storage integration from id: {integration_id}. Error: {str(e)}"
|
|
52
|
+
) from None
|
|
53
|
+
|
|
54
|
+
# TODO (chiragjn): Parse this using Pydantic
|
|
34
55
|
if (
|
|
35
56
|
data.get("providerIntegrations")
|
|
36
57
|
and len(data["providerIntegrations"]) > 0
|
truefoundry/ml/exceptions.py
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
3
|
|
|
4
|
+
# TODO (chiragjn): We need to establish uniform exception handling across codebase
|
|
4
5
|
class MlFoundryException(Exception):
|
|
5
|
-
def __init__(self, message, status_code: Optional[int] = None):
|
|
6
|
+
def __init__(self, message: str, status_code: Optional[int] = None):
|
|
6
7
|
self.message = str(message)
|
|
7
8
|
self.status_code = status_code
|
|
8
9
|
super().__init__(message)
|
|
@@ -9,6 +9,7 @@ import warnings
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Union
|
|
11
11
|
|
|
12
|
+
from truefoundry.common.warnings import TrueFoundryDeprecationWarning
|
|
12
13
|
from truefoundry.ml.artifact.truefoundry_artifact_repo import (
|
|
13
14
|
ArtifactIdentifier,
|
|
14
15
|
MlFoundryArtifactsRepository,
|
|
@@ -83,7 +84,7 @@ class ArtifactVersion:
|
|
|
83
84
|
self._deleted = False
|
|
84
85
|
self._description: str = ""
|
|
85
86
|
self._metadata: Dict[str, Any] = {}
|
|
86
|
-
self.
|
|
87
|
+
self._version_alias: Optional[str] = None
|
|
87
88
|
self._set_mutable_attrs()
|
|
88
89
|
|
|
89
90
|
@classmethod
|
|
@@ -131,13 +132,13 @@ class ArtifactVersion:
|
|
|
131
132
|
manifest = self._artifact_version.manifest.actual_instance
|
|
132
133
|
self._description = manifest.description or ""
|
|
133
134
|
self._metadata = copy.deepcopy(manifest.metadata)
|
|
134
|
-
self.
|
|
135
|
+
self._version_alias = manifest.version_alias or None
|
|
135
136
|
else:
|
|
136
137
|
self._description = self._artifact_version.description or ""
|
|
137
138
|
self._metadata = copy.deepcopy(
|
|
138
139
|
self._artifact_version.artifact_metadata or {}
|
|
139
140
|
)
|
|
140
|
-
self.
|
|
141
|
+
self._version_alias = None
|
|
141
142
|
|
|
142
143
|
def _refetch_artifact_version(self, reset_mutable_attrs: bool = True):
|
|
143
144
|
_artifact_version = (
|
|
@@ -210,31 +211,31 @@ class ArtifactVersion:
|
|
|
210
211
|
self._metadata = copy.deepcopy(value)
|
|
211
212
|
|
|
212
213
|
@property
|
|
213
|
-
def
|
|
214
|
+
def version_alias(self) -> Optional[str]:
|
|
214
215
|
"""
|
|
215
|
-
Get
|
|
216
|
+
Get version alias for the current artifact version
|
|
216
217
|
"""
|
|
217
218
|
if not self._artifact_version.manifest:
|
|
218
219
|
warnings.warn(
|
|
219
|
-
message="This
|
|
220
|
-
category=
|
|
220
|
+
message="This artifact version was created using an older serialization format. version alias does not exist",
|
|
221
|
+
category=TrueFoundryDeprecationWarning,
|
|
221
222
|
stacklevel=2,
|
|
222
223
|
)
|
|
223
|
-
return self.
|
|
224
|
+
return self._version_alias
|
|
224
225
|
|
|
225
|
-
@
|
|
226
|
-
def
|
|
226
|
+
@version_alias.setter
|
|
227
|
+
def version_alias(self, value: Optional[str]):
|
|
227
228
|
"""
|
|
228
|
-
Set the
|
|
229
|
+
Set the version alias for current artifact version
|
|
229
230
|
"""
|
|
230
231
|
if not self._artifact_version.manifest:
|
|
231
232
|
warnings.warn(
|
|
232
|
-
message="This
|
|
233
|
-
category=
|
|
233
|
+
message="This artifact version was created using an older serialization format. version alias will not be updated",
|
|
234
|
+
category=TrueFoundryDeprecationWarning,
|
|
234
235
|
stacklevel=2,
|
|
235
236
|
)
|
|
236
237
|
return
|
|
237
|
-
self.
|
|
238
|
+
self._version_alias = value
|
|
238
239
|
|
|
239
240
|
@property
|
|
240
241
|
def created_at(self) -> Optional[datetime.datetime]:
|
|
@@ -398,7 +399,7 @@ class ArtifactVersion:
|
|
|
398
399
|
assert isinstance(manifest, ArtifactVersionManifest)
|
|
399
400
|
manifest.description = self.description
|
|
400
401
|
manifest.metadata = self.metadata
|
|
401
|
-
manifest.
|
|
402
|
+
manifest.version_alias = self.version_alias
|
|
402
403
|
else:
|
|
403
404
|
manifest = None
|
|
404
405
|
try:
|
|
@@ -8,8 +8,9 @@ import typing
|
|
|
8
8
|
import uuid
|
|
9
9
|
import warnings
|
|
10
10
|
from pathlib import Path
|
|
11
|
-
from typing import TYPE_CHECKING, Any, Dict,
|
|
11
|
+
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
|
12
12
|
|
|
13
|
+
from truefoundry.common.warnings import TrueFoundryDeprecationWarning
|
|
13
14
|
from truefoundry.ml.artifact.truefoundry_artifact_repo import (
|
|
14
15
|
ArtifactIdentifier,
|
|
15
16
|
MlFoundryArtifactsRepository,
|
|
@@ -108,7 +109,7 @@ class ModelVersion:
|
|
|
108
109
|
self._metadata: Dict[str, Any] = {}
|
|
109
110
|
self._environment: Optional[ModelVersionEnvironment] = None
|
|
110
111
|
self._framework: Optional[ModelFrameworkType] = None
|
|
111
|
-
self.
|
|
112
|
+
self._version_alias: Optional[str] = None
|
|
112
113
|
self._set_mutable_attrs()
|
|
113
114
|
|
|
114
115
|
@classmethod
|
|
@@ -159,7 +160,7 @@ class ModelVersion:
|
|
|
159
160
|
if manifest.framework
|
|
160
161
|
else None
|
|
161
162
|
)
|
|
162
|
-
self.
|
|
163
|
+
self._version_alias = self._model_version.manifest.version_alias or None
|
|
163
164
|
else:
|
|
164
165
|
self._description = self._model_version.description or ""
|
|
165
166
|
self._metadata = copy.deepcopy(self._model_version.artifact_metadata or {})
|
|
@@ -167,7 +168,7 @@ class ModelVersion:
|
|
|
167
168
|
self._framework = _ModelFramework.to_model_framework_type(
|
|
168
169
|
self._model_version.model_framework
|
|
169
170
|
)
|
|
170
|
-
self.
|
|
171
|
+
self._version_alias = None
|
|
171
172
|
|
|
172
173
|
def _refetch_model_version(self, reset_mutable_attrs: bool = True):
|
|
173
174
|
_model_version = self._mlfoundry_artifacts_api.get_model_version_get(
|
|
@@ -238,31 +239,31 @@ class ModelVersion:
|
|
|
238
239
|
self._metadata = copy.deepcopy(value)
|
|
239
240
|
|
|
240
241
|
@property
|
|
241
|
-
def
|
|
242
|
+
def version_alias(self) -> Optional[str]:
|
|
242
243
|
"""
|
|
243
|
-
Get
|
|
244
|
+
Get version alias for the current model version
|
|
244
245
|
"""
|
|
245
246
|
if not self._model_version.manifest:
|
|
246
247
|
warnings.warn(
|
|
247
|
-
message="This model version was created using an older serialization format.
|
|
248
|
-
category=
|
|
248
|
+
message="This model version was created using an older serialization format. version alias does not exist",
|
|
249
|
+
category=TrueFoundryDeprecationWarning,
|
|
249
250
|
stacklevel=2,
|
|
250
251
|
)
|
|
251
|
-
return self.
|
|
252
|
+
return self._version_alias
|
|
252
253
|
|
|
253
|
-
@
|
|
254
|
-
def
|
|
254
|
+
@version_alias.setter
|
|
255
|
+
def version_alias(self, value: Optional[str]):
|
|
255
256
|
"""
|
|
256
|
-
Set the
|
|
257
|
+
Set the version alias for current artifact version
|
|
257
258
|
"""
|
|
258
259
|
if not self._model_version.manifest:
|
|
259
260
|
warnings.warn(
|
|
260
|
-
message="This model version was created using an older serialization format.
|
|
261
|
-
category=
|
|
261
|
+
message="This model version was created using an older serialization format. version alias will not be updated",
|
|
262
|
+
category=TrueFoundryDeprecationWarning,
|
|
262
263
|
stacklevel=2,
|
|
263
264
|
)
|
|
264
265
|
return
|
|
265
|
-
self.
|
|
266
|
+
self._version_alias = value
|
|
266
267
|
|
|
267
268
|
@property
|
|
268
269
|
def environment(self) -> Optional[ModelVersionEnvironment]:
|
|
@@ -270,7 +271,7 @@ class ModelVersion:
|
|
|
270
271
|
if not self._model_version.manifest:
|
|
271
272
|
warnings.warn(
|
|
272
273
|
message="This model version was created using an older serialization format. environment does not exist, returning None",
|
|
273
|
-
category=
|
|
274
|
+
category=TrueFoundryDeprecationWarning,
|
|
274
275
|
stacklevel=2,
|
|
275
276
|
)
|
|
276
277
|
return self._environment
|
|
@@ -281,7 +282,7 @@ class ModelVersion:
|
|
|
281
282
|
if not self._model_version.manifest:
|
|
282
283
|
warnings.warn(
|
|
283
284
|
message="This model version was created using an older serialization format. Environment will not be updated",
|
|
284
|
-
category=
|
|
285
|
+
category=TrueFoundryDeprecationWarning,
|
|
285
286
|
stacklevel=2,
|
|
286
287
|
)
|
|
287
288
|
return
|
|
@@ -300,7 +301,7 @@ class ModelVersion:
|
|
|
300
301
|
if not self._model_version.manifest:
|
|
301
302
|
warnings.warn(
|
|
302
303
|
message="This model version was created using an older serialization format. Framework will not be updated",
|
|
303
|
-
category=
|
|
304
|
+
category=TrueFoundryDeprecationWarning,
|
|
304
305
|
stacklevel=2,
|
|
305
306
|
)
|
|
306
307
|
return
|
|
@@ -488,7 +489,7 @@ class ModelVersion:
|
|
|
488
489
|
self._model_version.manifest.framework = (
|
|
489
490
|
Framework.from_dict(self.framework.dict()) if self.framework else None
|
|
490
491
|
)
|
|
491
|
-
self._model_version.manifest.
|
|
492
|
+
self._model_version.manifest.version_alias = self.version_alias
|
|
492
493
|
try:
|
|
493
494
|
_model_version = self._mlfoundry_artifacts_api.update_model_version_post(
|
|
494
495
|
update_model_version_request_dto=UpdateModelVersionRequestDto(
|
|
@@ -3,7 +3,7 @@ import logging
|
|
|
3
3
|
import os
|
|
4
4
|
import posixpath
|
|
5
5
|
from pathlib import Path, PureWindowsPath
|
|
6
|
-
from typing import Any, Dict, Optional, Sequence, Tuple, Union
|
|
6
|
+
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
|
|
7
7
|
|
|
8
8
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
9
9
|
from truefoundry.ml.log_types.artifacts.constants import DESCRIPTION_MAX_LENGTH
|
|
@@ -63,7 +63,7 @@ def get_single_file_path_if_only_one_in_directory(path: str) -> Optional[str]:
|
|
|
63
63
|
|
|
64
64
|
# If it's a directory, check if it contains a single file
|
|
65
65
|
if is_destination_path_dirlike(path):
|
|
66
|
-
all_files = []
|
|
66
|
+
all_files: List[str] = []
|
|
67
67
|
for root, _, files in os.walk(path):
|
|
68
68
|
# Collect all files found in any subdirectory
|
|
69
69
|
all_files.extend(os.path.join(root, f) for f in files)
|
truefoundry/ml/mlfoundry_api.py
CHANGED
|
@@ -17,7 +17,7 @@ from typing import (
|
|
|
17
17
|
|
|
18
18
|
import coolname
|
|
19
19
|
|
|
20
|
-
from truefoundry.common.utils import ContextualDirectoryManager
|
|
20
|
+
from truefoundry.common.utils import ContextualDirectoryManager
|
|
21
21
|
from truefoundry.ml import constants
|
|
22
22
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
23
23
|
ArtifactDto,
|
|
@@ -63,10 +63,8 @@ from truefoundry.ml.log_types.artifacts.model import (
|
|
|
63
63
|
from truefoundry.ml.logger import logger
|
|
64
64
|
from truefoundry.ml.mlfoundry_run import MlFoundryRun
|
|
65
65
|
from truefoundry.ml.session import (
|
|
66
|
-
|
|
66
|
+
MLFoundrySession,
|
|
67
67
|
_get_api_client,
|
|
68
|
-
get_active_session,
|
|
69
|
-
init_session,
|
|
70
68
|
)
|
|
71
69
|
from truefoundry.ml.validation_utils import (
|
|
72
70
|
_validate_ml_repo_description,
|
|
@@ -112,13 +110,12 @@ class MlFoundry:
|
|
|
112
110
|
"""MlFoundry."""
|
|
113
111
|
|
|
114
112
|
# TODO (chiragjn): Don't allow session as None here!
|
|
115
|
-
def __init__(self, session:
|
|
113
|
+
def __init__(self, session: MLFoundrySession):
|
|
116
114
|
"""__init__
|
|
117
115
|
|
|
118
116
|
Args:
|
|
119
117
|
session (Optional[Session], optional): Session instance to get auth credentials from
|
|
120
118
|
"""
|
|
121
|
-
self._tracking_uri: str = session.tracking_uri
|
|
122
119
|
self._api_client = _get_api_client(session=session)
|
|
123
120
|
self._experiments_api = ExperimentsApi(api_client=self._api_client)
|
|
124
121
|
self._runs_api = RunsApi(api_client=self._api_client)
|
|
@@ -236,16 +233,9 @@ class MlFoundry:
|
|
|
236
233
|
raise MlFoundryException(err_msg) from e
|
|
237
234
|
return
|
|
238
235
|
|
|
239
|
-
session = get_active_session()
|
|
240
|
-
if session is None:
|
|
241
|
-
raise MlFoundryException(
|
|
242
|
-
relogin_error_message(
|
|
243
|
-
"No active session found. Perhaps you are not logged in?",
|
|
244
|
-
)
|
|
245
|
-
)
|
|
246
236
|
servicefoundry_client = ServiceFoundryServiceClient(
|
|
247
|
-
|
|
248
|
-
token=
|
|
237
|
+
tfy_host=self._api_client.tfy_host,
|
|
238
|
+
token=self._api_client.access_token,
|
|
249
239
|
)
|
|
250
240
|
|
|
251
241
|
assert existing_ml_repo.storage_integration_id is not None
|
|
@@ -604,26 +594,6 @@ class MlFoundry:
|
|
|
604
594
|
if not runs or page_token is None:
|
|
605
595
|
done = True
|
|
606
596
|
|
|
607
|
-
def get_tracking_uri(self) -> str:
|
|
608
|
-
"""
|
|
609
|
-
Get the current tracking URI.
|
|
610
|
-
|
|
611
|
-
Returns:
|
|
612
|
-
The tracking URI.
|
|
613
|
-
|
|
614
|
-
Examples:
|
|
615
|
-
|
|
616
|
-
```python
|
|
617
|
-
import tempfile
|
|
618
|
-
from truefoundry.ml import get_client
|
|
619
|
-
|
|
620
|
-
client = get_client()
|
|
621
|
-
tracking_uri = client.get_tracking_uri()
|
|
622
|
-
print("Current tracking uri: {}".format(tracking_uri))
|
|
623
|
-
```
|
|
624
|
-
"""
|
|
625
|
-
return self._tracking_uri
|
|
626
|
-
|
|
627
597
|
def _initialize_model_server(
|
|
628
598
|
self,
|
|
629
599
|
name: str,
|
|
@@ -1239,7 +1209,6 @@ class MlFoundry:
|
|
|
1239
1209
|
raise MlFoundryException(
|
|
1240
1210
|
"artifact_paths cannot be empty, atleast one artifact_path must be passed"
|
|
1241
1211
|
)
|
|
1242
|
-
|
|
1243
1212
|
ml_repo_id = self._get_ml_repo_id(ml_repo=ml_repo)
|
|
1244
1213
|
artifact_version = _log_artifact_version(
|
|
1245
1214
|
run=None,
|
|
@@ -1377,7 +1346,6 @@ class MlFoundry:
|
|
|
1377
1346
|
|
|
1378
1347
|
"""
|
|
1379
1348
|
ml_repo_id = self._get_ml_repo_id(ml_repo=ml_repo)
|
|
1380
|
-
|
|
1381
1349
|
model_version = _log_model_version(
|
|
1382
1350
|
run=None,
|
|
1383
1351
|
mlfoundry_artifacts_api=self._mlfoundry_artifacts_api,
|
|
@@ -1648,5 +1616,5 @@ def get_client() -> MlFoundry:
|
|
|
1648
1616
|
client = get_client()
|
|
1649
1617
|
```
|
|
1650
1618
|
"""
|
|
1651
|
-
session =
|
|
1619
|
+
session = MLFoundrySession.new()
|
|
1652
1620
|
return MlFoundry(session=session)
|