truefoundry 0.4.0rc2__py3-none-any.whl → 0.4.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/__init__.py +2 -0
- truefoundry/autodeploy/agents/developer.py +1 -1
- truefoundry/autodeploy/agents/project_identifier.py +2 -2
- truefoundry/autodeploy/agents/tester.py +1 -1
- truefoundry/autodeploy/cli.py +1 -1
- truefoundry/autodeploy/tools/list_files.py +1 -1
- truefoundry/{deploy/lib/auth → common}/auth_service_client.py +50 -40
- truefoundry/common/constants.py +12 -0
- truefoundry/{deploy/lib/auth → common}/credential_file_manager.py +7 -7
- truefoundry/{deploy/lib/auth → common}/credential_provider.py +9 -12
- truefoundry/{ml/services → common}/entities.py +57 -41
- truefoundry/common/exceptions.py +12 -0
- truefoundry/common/request_utils.py +36 -8
- truefoundry/common/servicefoundry_client.py +91 -0
- truefoundry/common/utils.py +56 -0
- truefoundry/deploy/auto_gen/models.py +4 -6
- truefoundry/deploy/cli/cli.py +2 -0
- truefoundry/deploy/cli/commands/apply_command.py +1 -1
- truefoundry/deploy/cli/commands/build_command.py +1 -1
- truefoundry/deploy/cli/commands/deploy_command.py +1 -1
- truefoundry/deploy/cli/commands/login_command.py +2 -2
- truefoundry/deploy/cli/commands/patch_application_command.py +1 -1
- truefoundry/deploy/cli/commands/patch_command.py +1 -1
- truefoundry/deploy/cli/commands/terminate_comand.py +1 -1
- truefoundry/deploy/cli/util.py +1 -1
- truefoundry/deploy/function_service/remote/remote.py +1 -1
- truefoundry/deploy/lib/auth/servicefoundry_session.py +2 -2
- truefoundry/deploy/lib/clients/servicefoundry_client.py +120 -150
- truefoundry/deploy/lib/const.py +1 -35
- truefoundry/deploy/lib/exceptions.py +0 -11
- truefoundry/deploy/lib/model/entity.py +1 -112
- truefoundry/deploy/lib/session.py +13 -26
- truefoundry/deploy/lib/util.py +0 -37
- truefoundry/deploy/v2/lib/deploy.py +3 -3
- truefoundry/ml/__init__.py +0 -2
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +63 -22
- truefoundry/ml/autogen/client/__init__.py +0 -3
- truefoundry/ml/autogen/client/api/experiments_api.py +0 -165
- truefoundry/ml/autogen/client/models/__init__.py +0 -3
- truefoundry/ml/autogen/client/models/artifact_dto.py +6 -6
- truefoundry/ml/autogen/client/models/artifact_version_dto.py +8 -8
- truefoundry/ml/autogen/client/models/create_artifact_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_artifact_version_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_python_deployment_config_request_dto.py +2 -2
- truefoundry/ml/autogen/client/models/create_python_deployment_config_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_run_request_dto.py +5 -5
- truefoundry/ml/autogen/client/models/create_run_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/dataset_dto.py +10 -10
- truefoundry/ml/autogen/client/models/experiment_dto.py +18 -18
- truefoundry/ml/autogen/client/models/get_latest_run_log_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/get_tenant_id_response_dto.py +4 -5
- truefoundry/ml/autogen/client/models/model_dto.py +6 -6
- truefoundry/ml/autogen/client/models/model_version_dto.py +15 -8
- truefoundry/ml/autogen/client/models/run_info_dto.py +10 -10
- truefoundry/ml/autogen/client/models/update_run_response_dto.py +2 -3
- truefoundry/ml/autogen/client_README.md +0 -2
- truefoundry/ml/clients/entities.py +8 -0
- truefoundry/ml/{services/servicefoundry_service.py → clients/servicefoundry_client.py} +20 -10
- truefoundry/ml/{services → clients}/utils.py +2 -2
- truefoundry/ml/env_vars.py +1 -5
- truefoundry/ml/internal_namespace.py +8 -8
- truefoundry/ml/log_types/artifacts/artifact.py +7 -3
- truefoundry/ml/log_types/artifacts/dataset.py +1 -1
- truefoundry/ml/log_types/artifacts/model.py +7 -8
- truefoundry/ml/log_types/image/image.py +7 -8
- truefoundry/ml/log_types/image/image_normalizer.py +7 -6
- truefoundry/ml/mlfoundry_api.py +5 -17
- truefoundry/ml/mlfoundry_run.py +0 -5
- truefoundry/ml/run_utils.py +1 -10
- truefoundry/ml/session.py +14 -117
- truefoundry/pydantic_v1.py +1 -1
- truefoundry/workflow/__init__.py +16 -1
- {truefoundry-0.4.0rc2.dist-info → truefoundry-0.4.0rc3.dist-info}/METADATA +1 -1
- {truefoundry-0.4.0rc2.dist-info → truefoundry-0.4.0rc3.dist-info}/RECORD +77 -76
- truefoundry/deploy/lib/clients/utils.py +0 -41
- truefoundry/ml/autogen/client/models/backfill_default_storage_integration_id_request_dto.py +0 -67
- truefoundry/ml/login.py +0 -241
- truefoundry/ml/services/auth_service.py +0 -109
- /truefoundry/ml/{services → clients}/__init__.py +0 -0
- {truefoundry-0.4.0rc2.dist-info → truefoundry-0.4.0rc3.dist-info}/WHEEL +0 -0
- {truefoundry-0.4.0rc2.dist-info → truefoundry-0.4.0rc3.dist-info}/entry_points.txt +0 -0
|
@@ -24,7 +24,14 @@ from truefoundry.ml.autogen.client.models.artifact_version_status import (
|
|
|
24
24
|
)
|
|
25
25
|
from truefoundry.ml.autogen.client.models.metric_dto import MetricDto
|
|
26
26
|
from truefoundry.ml.autogen.client.models.model_schema_dto import ModelSchemaDto
|
|
27
|
-
from truefoundry.pydantic_v1 import
|
|
27
|
+
from truefoundry.pydantic_v1 import (
|
|
28
|
+
BaseModel,
|
|
29
|
+
Field,
|
|
30
|
+
StrictBool,
|
|
31
|
+
StrictInt,
|
|
32
|
+
StrictStr,
|
|
33
|
+
conlist,
|
|
34
|
+
)
|
|
28
35
|
|
|
29
36
|
|
|
30
37
|
class ModelVersionDto(BaseModel):
|
|
@@ -32,10 +39,11 @@ class ModelVersionDto(BaseModel):
|
|
|
32
39
|
ModelVersionDto
|
|
33
40
|
"""
|
|
34
41
|
|
|
35
|
-
id:
|
|
36
|
-
model_id:
|
|
37
|
-
version:
|
|
38
|
-
fqn:
|
|
42
|
+
id: StrictStr = Field(...)
|
|
43
|
+
model_id: StrictStr = Field(...)
|
|
44
|
+
version: StrictInt = Field(...)
|
|
45
|
+
fqn: StrictStr = Field(...)
|
|
46
|
+
experiment_id: StrictStr = Field(...)
|
|
39
47
|
artifact_storage_root: Optional[StrictStr] = None
|
|
40
48
|
artifact_metadata: Optional[Dict[str, Any]] = None
|
|
41
49
|
description: Optional[StrictStr] = None
|
|
@@ -50,7 +58,6 @@ class ModelVersionDto(BaseModel):
|
|
|
50
58
|
data_path: Optional[StrictStr] = None
|
|
51
59
|
monitoring_enabled: Optional[StrictBool] = False
|
|
52
60
|
usage_code_snippet: Optional[StrictStr] = None
|
|
53
|
-
experiment_id: Optional[StrictStr] = None
|
|
54
61
|
run_id: Optional[StrictStr] = None
|
|
55
62
|
metrics: Optional[conlist(MetricDto)] = None
|
|
56
63
|
model_name: Optional[StrictStr] = None
|
|
@@ -61,6 +68,7 @@ class ModelVersionDto(BaseModel):
|
|
|
61
68
|
"model_id",
|
|
62
69
|
"version",
|
|
63
70
|
"fqn",
|
|
71
|
+
"experiment_id",
|
|
64
72
|
"artifact_storage_root",
|
|
65
73
|
"artifact_metadata",
|
|
66
74
|
"description",
|
|
@@ -75,7 +83,6 @@ class ModelVersionDto(BaseModel):
|
|
|
75
83
|
"data_path",
|
|
76
84
|
"monitoring_enabled",
|
|
77
85
|
"usage_code_snippet",
|
|
78
|
-
"experiment_id",
|
|
79
86
|
"run_id",
|
|
80
87
|
"metrics",
|
|
81
88
|
"model_name",
|
|
@@ -132,6 +139,7 @@ class ModelVersionDto(BaseModel):
|
|
|
132
139
|
"model_id": obj.get("model_id"),
|
|
133
140
|
"version": obj.get("version"),
|
|
134
141
|
"fqn": obj.get("fqn"),
|
|
142
|
+
"experiment_id": obj.get("experiment_id"),
|
|
135
143
|
"artifact_storage_root": obj.get("artifact_storage_root"),
|
|
136
144
|
"artifact_metadata": obj.get("artifact_metadata"),
|
|
137
145
|
"description": obj.get("description"),
|
|
@@ -150,7 +158,6 @@ class ModelVersionDto(BaseModel):
|
|
|
150
158
|
if obj.get("monitoring_enabled") is not None
|
|
151
159
|
else False,
|
|
152
160
|
"usage_code_snippet": obj.get("usage_code_snippet"),
|
|
153
|
-
"experiment_id": obj.get("experiment_id"),
|
|
154
161
|
"run_id": obj.get("run_id"),
|
|
155
162
|
"metrics": [MetricDto.from_dict(_item) for _item in obj.get("metrics")]
|
|
156
163
|
if obj.get("metrics") is not None
|
|
@@ -18,7 +18,7 @@ import pprint
|
|
|
18
18
|
import re # noqa: F401
|
|
19
19
|
from typing import Optional
|
|
20
20
|
|
|
21
|
-
from truefoundry.pydantic_v1 import BaseModel, StrictInt, StrictStr
|
|
21
|
+
from truefoundry.pydantic_v1 import BaseModel, Field, StrictInt, StrictStr
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class RunInfoDto(BaseModel):
|
|
@@ -26,30 +26,30 @@ class RunInfoDto(BaseModel):
|
|
|
26
26
|
RunInfoDto
|
|
27
27
|
"""
|
|
28
28
|
|
|
29
|
-
run_uuid:
|
|
30
|
-
run_id:
|
|
31
|
-
experiment_id:
|
|
29
|
+
run_uuid: StrictStr = Field(...)
|
|
30
|
+
run_id: StrictStr = Field(...)
|
|
31
|
+
experiment_id: StrictStr = Field(...)
|
|
32
|
+
name: StrictStr = Field(...)
|
|
33
|
+
fqn: StrictStr = Field(...)
|
|
32
34
|
user_id: Optional[StrictStr] = None
|
|
33
35
|
status: Optional[StrictStr] = None
|
|
34
36
|
start_time: Optional[StrictInt] = None
|
|
35
37
|
end_time: Optional[StrictInt] = None
|
|
36
38
|
artifact_uri: Optional[StrictStr] = None
|
|
37
39
|
lifecycle_stage: Optional[StrictStr] = None
|
|
38
|
-
name: Optional[StrictStr] = None
|
|
39
|
-
fqn: Optional[StrictStr] = None
|
|
40
40
|
description: Optional[StrictStr] = None
|
|
41
41
|
__properties = [
|
|
42
42
|
"run_uuid",
|
|
43
43
|
"run_id",
|
|
44
44
|
"experiment_id",
|
|
45
|
+
"name",
|
|
46
|
+
"fqn",
|
|
45
47
|
"user_id",
|
|
46
48
|
"status",
|
|
47
49
|
"start_time",
|
|
48
50
|
"end_time",
|
|
49
51
|
"artifact_uri",
|
|
50
52
|
"lifecycle_stage",
|
|
51
|
-
"name",
|
|
52
|
-
"fqn",
|
|
53
53
|
"description",
|
|
54
54
|
]
|
|
55
55
|
|
|
@@ -91,14 +91,14 @@ class RunInfoDto(BaseModel):
|
|
|
91
91
|
"run_uuid": obj.get("run_uuid"),
|
|
92
92
|
"run_id": obj.get("run_id"),
|
|
93
93
|
"experiment_id": obj.get("experiment_id"),
|
|
94
|
+
"name": obj.get("name"),
|
|
95
|
+
"fqn": obj.get("fqn"),
|
|
94
96
|
"user_id": obj.get("user_id"),
|
|
95
97
|
"status": obj.get("status"),
|
|
96
98
|
"start_time": obj.get("start_time"),
|
|
97
99
|
"end_time": obj.get("end_time"),
|
|
98
100
|
"artifact_uri": obj.get("artifact_uri"),
|
|
99
101
|
"lifecycle_stage": obj.get("lifecycle_stage"),
|
|
100
|
-
"name": obj.get("name"),
|
|
101
|
-
"fqn": obj.get("fqn"),
|
|
102
102
|
"description": obj.get("description"),
|
|
103
103
|
}
|
|
104
104
|
)
|
|
@@ -16,10 +16,9 @@ from __future__ import annotations
|
|
|
16
16
|
import json
|
|
17
17
|
import pprint
|
|
18
18
|
import re # noqa: F401
|
|
19
|
-
from typing import Optional
|
|
20
19
|
|
|
21
20
|
from truefoundry.ml.autogen.client.models.run_info_dto import RunInfoDto
|
|
22
|
-
from truefoundry.pydantic_v1 import BaseModel
|
|
21
|
+
from truefoundry.pydantic_v1 import BaseModel, Field
|
|
23
22
|
|
|
24
23
|
|
|
25
24
|
class UpdateRunResponseDto(BaseModel):
|
|
@@ -27,7 +26,7 @@ class UpdateRunResponseDto(BaseModel):
|
|
|
27
26
|
UpdateRunResponseDto
|
|
28
27
|
"""
|
|
29
28
|
|
|
30
|
-
run_info:
|
|
29
|
+
run_info: RunInfoDto = Field(...)
|
|
31
30
|
__properties = ["run_info"]
|
|
32
31
|
|
|
33
32
|
class Config:
|
|
@@ -69,7 +69,6 @@ Class | Method | HTTP request | Description
|
|
|
69
69
|
*DeprecatedApi* | [**get_run_by_name_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_run_by_name_get) | **GET** /api/2.0mlflow/runs/get-by-name | Get Run By Name
|
|
70
70
|
*DeprecatedApi* | [**get_search_runs_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_search_runs_get) | **GET** /api/2.0/preview/mlflow/runs/search | Get Search Runs
|
|
71
71
|
*DeprecatedApi* | [**get_signed_urls_for_dataset_write_deprecated_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_signed_urls_for_dataset_write_deprecated_get) | **GET** /api/2.0/mlflow/mlfoundry-artifacts/datasets/get-signed-urls-for-write | Get Signed Urls For Dataset Write Deprecated
|
|
72
|
-
*ExperimentsApi* | [**backfill_storage_integration_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#backfill_storage_integration_post) | **POST** /api/2.0/mlflow/experiments/storage-integration/backfill | Backfill Storage Integration
|
|
73
72
|
*ExperimentsApi* | [**create_experiment_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#create_experiment_post) | **POST** /api/2.0/mlflow/experiments/create | Create Experiment
|
|
74
73
|
*ExperimentsApi* | [**delete_experiment_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#delete_experiment_post) | **POST** /api/2.0/mlflow/experiments/delete | Delete Experiment
|
|
75
74
|
*ExperimentsApi* | [**get_experiment_by_name_get**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#get_experiment_by_name_get) | **GET** /api/2.0/mlflow/experiments/get-by-name | Get Experiment By Name
|
|
@@ -170,7 +169,6 @@ Class | Method | HTTP request | Description
|
|
|
170
169
|
- [AssistantMessage](truefoundry/ml/autogen/client/docs/AssistantMessage.md)
|
|
171
170
|
- [AuthorizeUserForModelRequestDto](truefoundry/ml/autogen/client/docs/AuthorizeUserForModelRequestDto.md)
|
|
172
171
|
- [AuthorizeUserForModelVersionRequestDto](truefoundry/ml/autogen/client/docs/AuthorizeUserForModelVersionRequestDto.md)
|
|
173
|
-
- [BackfillDefaultStorageIntegrationIdRequestDto](truefoundry/ml/autogen/client/docs/BackfillDefaultStorageIntegrationIdRequestDto.md)
|
|
174
172
|
- [BlobStorageReference](truefoundry/ml/autogen/client/docs/BlobStorageReference.md)
|
|
175
173
|
- [BodyGetSearchRunsGet](truefoundry/ml/autogen/client/docs/BodyGetSearchRunsGet.md)
|
|
176
174
|
- [ChatPrompt](truefoundry/ml/autogen/client/docs/ChatPrompt.md)
|
|
@@ -1,28 +1,38 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from truefoundry.common.
|
|
3
|
+
from truefoundry.common.constants import (
|
|
4
|
+
SERVICEFOUNDRY_CLIENT_MAX_RETRIES,
|
|
5
|
+
VERSION_PREFIX,
|
|
6
|
+
)
|
|
7
|
+
from truefoundry.common.servicefoundry_client import (
|
|
8
|
+
ServiceFoundryServiceClient as BaseServiceFoundryServiceClient,
|
|
9
|
+
)
|
|
10
|
+
from truefoundry.ml.clients.entities import (
|
|
11
|
+
HostCreds,
|
|
12
|
+
)
|
|
13
|
+
from truefoundry.ml.clients.utils import http_request_safe
|
|
4
14
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
5
|
-
from truefoundry.ml.run_utils import
|
|
6
|
-
from truefoundry.ml.services.entities import HostCreds
|
|
7
|
-
from truefoundry.ml.services.utils import http_request_safe
|
|
15
|
+
from truefoundry.ml.run_utils import append_servicefoundry_path_to_tracking_uri
|
|
8
16
|
|
|
9
17
|
|
|
10
|
-
class
|
|
18
|
+
class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
|
|
11
19
|
def __init__(self, tracking_uri: str, token: Optional[str] = None):
|
|
20
|
+
super().__init__(base_url=tracking_uri)
|
|
12
21
|
self.host_creds = HostCreds(
|
|
13
|
-
host=
|
|
22
|
+
host=append_servicefoundry_path_to_tracking_uri(tracking_uri), token=token
|
|
14
23
|
)
|
|
15
24
|
|
|
16
25
|
def get_integration_from_id(self, integration_id: str):
|
|
17
26
|
integration_id = integration_id or ""
|
|
18
|
-
|
|
19
|
-
method="get",
|
|
27
|
+
response = http_request_safe(
|
|
20
28
|
host_creds=self.host_creds,
|
|
21
|
-
endpoint="
|
|
22
|
-
session=requests_retry_session(retries=1),
|
|
29
|
+
endpoint=f"{VERSION_PREFIX}/provider-accounts/provider-integrations",
|
|
23
30
|
params={"id": integration_id, "type": "blob-storage"},
|
|
31
|
+
method="get",
|
|
24
32
|
timeout=3,
|
|
33
|
+
max_retries=SERVICEFOUNDRY_CLIENT_MAX_RETRIES,
|
|
25
34
|
)
|
|
35
|
+
data = response.json()
|
|
26
36
|
if (
|
|
27
37
|
data.get("providerIntegrations")
|
|
28
38
|
and len(data["providerIntegrations"]) > 0
|
|
@@ -6,8 +6,8 @@ from urllib.parse import urljoin
|
|
|
6
6
|
import requests
|
|
7
7
|
|
|
8
8
|
from truefoundry.common.request_utils import requests_retry_session
|
|
9
|
+
from truefoundry.ml.clients.entities import HostCreds
|
|
9
10
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
10
|
-
from truefoundry.ml.services.entities import HostCreds
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
# TODO: This will be moved later to truefoundry.common.request_utils
|
|
@@ -77,7 +77,7 @@ def cloud_storage_http_request(
|
|
|
77
77
|
"""
|
|
78
78
|
Performs an HTTP PUT/GET request using Python's `requests` module with automatic retry.
|
|
79
79
|
"""
|
|
80
|
-
session = session or requests_retry_session(retries=5, backoff_factor=
|
|
80
|
+
session = session or requests_retry_session(retries=5, backoff_factor=0.5)
|
|
81
81
|
kwargs["headers"] = kwargs.get("headers", {}) or {}
|
|
82
82
|
if "blob.core.windows.net" in url:
|
|
83
83
|
kwargs["headers"].update({"x-ms-blob-type": "BlockBlob"})
|
truefoundry/ml/env_vars.py
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
|
-
Define all the env variable names that users
|
|
3
|
-
can use
|
|
2
|
+
Define all the env variable names that users can use
|
|
4
3
|
"""
|
|
5
4
|
|
|
6
|
-
# NOTE: These are shared between servicefoundry and mlfoundry
|
|
7
|
-
TRACKING_HOST_GLOBAL = "TFY_HOST"
|
|
8
|
-
API_KEY_GLOBAL = "TFY_API_KEY"
|
|
9
5
|
DISABLE_MULTIPART_UPLOAD = "MLF_DISABLE_MULTIPART_UPLOAD"
|
|
10
6
|
INTERNAL_ENV_VARS = [
|
|
11
7
|
"TFY_INTERNAL_APPLICATION_ID",
|
|
@@ -6,39 +6,39 @@ from truefoundry.ml.exceptions import MlFoundryException
|
|
|
6
6
|
|
|
7
7
|
class _InternalNamespace:
|
|
8
8
|
NAMESPACE = "mlf"
|
|
9
|
-
|
|
9
|
+
DELIMITER = "."
|
|
10
10
|
NAMESPACE_VIOLATION_MESSAGE = """
|
|
11
11
|
{name} cannot start with {prefix}
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
def __call__(self, name: str,
|
|
14
|
+
def __call__(self, name: str, delimiter: str = DELIMITER):
|
|
15
15
|
if not name:
|
|
16
16
|
raise MlFoundryException("name should be a non empty string")
|
|
17
|
-
return _InternalNamespace.NAMESPACE +
|
|
17
|
+
return _InternalNamespace.NAMESPACE + delimiter + name
|
|
18
18
|
|
|
19
19
|
def __truediv__(self, path: str):
|
|
20
20
|
return os.path.join(_InternalNamespace.NAMESPACE, path)
|
|
21
21
|
|
|
22
22
|
@staticmethod
|
|
23
|
-
def _validate_name_not_using_namespace(name: typing.Optional[str],
|
|
24
|
-
if name and name.startswith(_InternalNamespace.NAMESPACE +
|
|
23
|
+
def _validate_name_not_using_namespace(name: typing.Optional[str], delimiter: str):
|
|
24
|
+
if name and name.startswith(_InternalNamespace.NAMESPACE + delimiter):
|
|
25
25
|
raise MlFoundryException(
|
|
26
26
|
_InternalNamespace.NAMESPACE_VIOLATION_MESSAGE.format(
|
|
27
|
-
name=name, prefix=_InternalNamespace.NAMESPACE +
|
|
27
|
+
name=name, prefix=_InternalNamespace.NAMESPACE + delimiter
|
|
28
28
|
)
|
|
29
29
|
)
|
|
30
30
|
|
|
31
31
|
def validate_namespace_not_used(
|
|
32
32
|
self,
|
|
33
33
|
names: typing.Optional[typing.Union[str, typing.Iterable[str]]] = None,
|
|
34
|
-
|
|
34
|
+
delimiter: str = DELIMITER,
|
|
35
35
|
path: typing.Optional[str] = None,
|
|
36
36
|
):
|
|
37
37
|
if isinstance(names, str):
|
|
38
38
|
names = [names]
|
|
39
39
|
if names is not None:
|
|
40
40
|
for name_ in names:
|
|
41
|
-
self._validate_name_not_using_namespace(name_,
|
|
41
|
+
self._validate_name_not_using_namespace(name_, delimiter)
|
|
42
42
|
if path:
|
|
43
43
|
prefix = os.path.normpath(os.path.join(_InternalNamespace.NAMESPACE, ""))
|
|
44
44
|
if os.path.normpath(path).startswith(prefix):
|
|
@@ -3,6 +3,7 @@ import datetime
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
5
|
import tempfile
|
|
6
|
+
import uuid
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union
|
|
8
9
|
|
|
@@ -21,7 +22,7 @@ from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
|
21
22
|
NotifyArtifactVersionFailureDto,
|
|
22
23
|
UpdateArtifactVersionRequestDto,
|
|
23
24
|
)
|
|
24
|
-
from truefoundry.ml.autogen.client import (
|
|
25
|
+
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
25
26
|
InternalMetadata as InternalMetadataDto,
|
|
26
27
|
)
|
|
27
28
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
@@ -130,7 +131,7 @@ class ArtifactVersion:
|
|
|
130
131
|
def _get_artifacts_repo(self):
|
|
131
132
|
return MlFoundryArtifactsRepository(
|
|
132
133
|
artifact_identifier=ArtifactIdentifier(
|
|
133
|
-
artifact_version_id=self._artifact_version.id
|
|
134
|
+
artifact_version_id=uuid.UUID(self._artifact_version.id)
|
|
134
135
|
),
|
|
135
136
|
api_client=self._api_client,
|
|
136
137
|
)
|
|
@@ -374,6 +375,7 @@ def _log_artifact_version_helper(
|
|
|
374
375
|
if run:
|
|
375
376
|
mlfoundry_artifacts_api = run._mlfoundry_artifacts_api
|
|
376
377
|
|
|
378
|
+
assert mlfoundry_artifacts_api is not None
|
|
377
379
|
_create_artifact_response = mlfoundry_artifacts_api.create_artifact_version_post(
|
|
378
380
|
create_artifact_version_request_dto=CreateArtifactVersionRequestDto(
|
|
379
381
|
experiment_id=int(run._experiment_id if run else ml_repo_id),
|
|
@@ -383,7 +385,9 @@ def _log_artifact_version_helper(
|
|
|
383
385
|
)
|
|
384
386
|
version_id = _create_artifact_response.id
|
|
385
387
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
386
|
-
artifact_identifier=ArtifactIdentifier(
|
|
388
|
+
artifact_identifier=ArtifactIdentifier(
|
|
389
|
+
artifact_version_id=uuid.UUID(version_id),
|
|
390
|
+
),
|
|
387
391
|
api_client=mlfoundry_artifacts_api.api_client,
|
|
388
392
|
)
|
|
389
393
|
total_size = calculate_local_directory_size(artifact_dir)
|
|
@@ -136,7 +136,7 @@ class DataDirectory:
|
|
|
136
136
|
|
|
137
137
|
@property
|
|
138
138
|
def updated_at(self) -> datetime.datetime:
|
|
139
|
-
"""Get the information about
|
|
139
|
+
"""Get the information about when the DataDirectory was updated"""
|
|
140
140
|
return self._dataset.updated_at
|
|
141
141
|
|
|
142
142
|
def add_files(
|
|
@@ -146,11 +146,10 @@ class ModelVersion:
|
|
|
146
146
|
self._metadata = copy.deepcopy(self._model_version.artifact_metadata)
|
|
147
147
|
|
|
148
148
|
def _refetch_model_version(self):
|
|
149
|
-
|
|
150
|
-
self.
|
|
151
|
-
id=self._model_version.id
|
|
152
|
-
)
|
|
149
|
+
_model_version = self._mlfoundry_artifacts_api.get_model_version_get(
|
|
150
|
+
id=self._model_version.id
|
|
153
151
|
)
|
|
152
|
+
self._model_version = _model_version.model_version
|
|
154
153
|
self._set_metrics_attr()
|
|
155
154
|
self._set_mutable_attrs()
|
|
156
155
|
|
|
@@ -382,13 +381,11 @@ class ModelVersion:
|
|
|
382
381
|
```
|
|
383
382
|
"""
|
|
384
383
|
self._ensure_not_deleted()
|
|
385
|
-
kwargs = {}
|
|
386
384
|
_model_version = self._mlfoundry_artifacts_api.update_model_version_post(
|
|
387
385
|
update_model_version_request_dto=UpdateModelVersionRequestDto(
|
|
388
386
|
id=self._model_version.id,
|
|
389
387
|
description=self.description,
|
|
390
388
|
artifact_metadata=self.metadata,
|
|
391
|
-
**kwargs,
|
|
392
389
|
)
|
|
393
390
|
)
|
|
394
391
|
self._model_version = _model_version.model_version
|
|
@@ -536,7 +533,9 @@ def _log_model_version( # noqa: C901
|
|
|
536
533
|
)
|
|
537
534
|
version_id = _create_artifact_version_response.id
|
|
538
535
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
539
|
-
artifact_identifier=ArtifactIdentifier(
|
|
536
|
+
artifact_identifier=ArtifactIdentifier(
|
|
537
|
+
artifact_version_id=uuid.UUID(version_id)
|
|
538
|
+
),
|
|
540
539
|
api_client=mlfoundry_artifacts_api.api_client,
|
|
541
540
|
)
|
|
542
541
|
model_size = calculate_model_size(temp_dir)
|
|
@@ -586,7 +585,7 @@ def _log_model_version( # noqa: C901
|
|
|
586
585
|
model_version = _model_version.model_version
|
|
587
586
|
|
|
588
587
|
# update model schema at end
|
|
589
|
-
update_args = {
|
|
588
|
+
update_args: Dict[str, Any] = {
|
|
590
589
|
"id": version_id,
|
|
591
590
|
"model_framework": framework.value,
|
|
592
591
|
}
|
|
@@ -6,8 +6,6 @@ import tempfile
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
|
8
8
|
|
|
9
|
-
import numpy as np
|
|
10
|
-
|
|
11
9
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
12
10
|
ArtifactType,
|
|
13
11
|
)
|
|
@@ -33,17 +31,18 @@ from truefoundry.ml.run_utils import get_module
|
|
|
33
31
|
|
|
34
32
|
if TYPE_CHECKING:
|
|
35
33
|
# noinspection PyUnresolvedReferences
|
|
34
|
+
import numpy # noqa: F401
|
|
36
35
|
import PIL.Image
|
|
37
|
-
from numpy.typing import NDArray
|
|
36
|
+
from numpy.typing import NDArray # noqa: F401
|
|
38
37
|
|
|
39
38
|
from truefoundry.ml.mlfoundry_run import MlFoundryRun
|
|
40
39
|
|
|
41
40
|
ImageLikeNDArray = Union[
|
|
42
|
-
"NDArray[
|
|
43
|
-
"NDArray[
|
|
44
|
-
"NDArray[
|
|
45
|
-
"NDArray[
|
|
46
|
-
"NDArray[
|
|
41
|
+
"NDArray[numpy.int8]",
|
|
42
|
+
"NDArray[numpy.uint8]",
|
|
43
|
+
"NDArray[numpy.float32]",
|
|
44
|
+
"NDArray[numpy.float64]",
|
|
45
|
+
"NDArray[numpy.bool_]",
|
|
47
46
|
]
|
|
48
47
|
DataOrPathType = Union[str, Path, ImageLikeNDArray, "PIL.Image.Image"]
|
|
49
48
|
ClassGroupsType = Dict[str, Union[str, List[str]]]
|
|
@@ -10,15 +10,16 @@ from truefoundry.ml.logger import logger
|
|
|
10
10
|
from truefoundry.ml.run_utils import get_module
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
+
import numpy # noqa: F401
|
|
13
14
|
import PIL.Image
|
|
14
|
-
from numpy.typing import NDArray
|
|
15
|
+
from numpy.typing import NDArray # noqa: F401
|
|
15
16
|
|
|
16
17
|
ImageLikeNDArray = Union[
|
|
17
|
-
"NDArray[
|
|
18
|
-
"NDArray[
|
|
19
|
-
"NDArray[
|
|
20
|
-
"NDArray[
|
|
21
|
-
"NDArray[
|
|
18
|
+
"NDArray[numpy.int8]",
|
|
19
|
+
"NDArray[numpy.uint8]",
|
|
20
|
+
"NDArray[numpy.float32]",
|
|
21
|
+
"NDArray[numpy.float64]",
|
|
22
|
+
"NDArray[numpy.bool_]",
|
|
22
23
|
]
|
|
23
24
|
|
|
24
25
|
|
truefoundry/ml/mlfoundry_api.py
CHANGED
|
@@ -30,6 +30,7 @@ from truefoundry.ml.autogen.client.exceptions import (
|
|
|
30
30
|
ApiException,
|
|
31
31
|
NotFoundException,
|
|
32
32
|
)
|
|
33
|
+
from truefoundry.ml.clients.servicefoundry_client import ServiceFoundryServiceClient
|
|
33
34
|
from truefoundry.ml.enums import ModelFramework, ViewType
|
|
34
35
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
35
36
|
from truefoundry.ml.internal_namespace import NAMESPACE
|
|
@@ -39,7 +40,6 @@ from truefoundry.ml.log_types.artifacts.general_artifact import _log_artifact_ve
|
|
|
39
40
|
from truefoundry.ml.log_types.artifacts.model import ModelVersion, _log_model_version
|
|
40
41
|
from truefoundry.ml.logger import logger
|
|
41
42
|
from truefoundry.ml.mlfoundry_run import MlFoundryRun
|
|
42
|
-
from truefoundry.ml.services.servicefoundry_service import ServicefoundryService
|
|
43
43
|
from truefoundry.ml.session import (
|
|
44
44
|
Session,
|
|
45
45
|
_get_api_client,
|
|
@@ -123,7 +123,6 @@ class MlFoundry:
|
|
|
123
123
|
)
|
|
124
124
|
raise MlFoundryException(err_msg) from e
|
|
125
125
|
|
|
126
|
-
assert ml_repo_instance.experiment_id is not None
|
|
127
126
|
return ml_repo_instance.experiment_id
|
|
128
127
|
|
|
129
128
|
def list_ml_repos(self) -> List[str]:
|
|
@@ -213,7 +212,7 @@ class MlFoundry:
|
|
|
213
212
|
"No active session found. Perhaps you are not logged in?\n"
|
|
214
213
|
"Please log in using `tfy login [--host HOST] --relogin"
|
|
215
214
|
)
|
|
216
|
-
|
|
215
|
+
servicefoundry_client = ServiceFoundryServiceClient(
|
|
217
216
|
tracking_uri=self.get_tracking_uri(),
|
|
218
217
|
token=session.token.access_token,
|
|
219
218
|
)
|
|
@@ -221,7 +220,7 @@ class MlFoundry:
|
|
|
221
220
|
assert existing_ml_repo.storage_integration_id is not None
|
|
222
221
|
try:
|
|
223
222
|
existing_storage_integration = (
|
|
224
|
-
|
|
223
|
+
servicefoundry_client.get_integration_from_id(
|
|
225
224
|
existing_ml_repo.storage_integration_id
|
|
226
225
|
)
|
|
227
226
|
)
|
|
@@ -335,11 +334,6 @@ class MlFoundry:
|
|
|
335
334
|
)
|
|
336
335
|
)
|
|
337
336
|
run = _run.run
|
|
338
|
-
|
|
339
|
-
assert run is not None
|
|
340
|
-
assert run.info.run_id is not None
|
|
341
|
-
assert run.info.fqn is not None
|
|
342
|
-
|
|
343
337
|
mlf_run_id = run.info.run_id
|
|
344
338
|
kwargs.setdefault("auto_end", True)
|
|
345
339
|
mlf_run = MlFoundryRun(experiment_id=ml_repo_id, run_id=mlf_run_id, **kwargs)
|
|
@@ -378,9 +372,6 @@ class MlFoundry:
|
|
|
378
372
|
return self.get_run_by_fqn(run_id)
|
|
379
373
|
_run = self._runs_api.get_run_get(run_id=run_id)
|
|
380
374
|
run = _run.run
|
|
381
|
-
|
|
382
|
-
assert run is not None
|
|
383
|
-
|
|
384
375
|
mlfoundry_run = MlFoundryRun._from_dto(run)
|
|
385
376
|
logger.info(
|
|
386
377
|
f"Link to the dashboard for the run: {mlfoundry_run.dashboard_link}"
|
|
@@ -722,7 +713,6 @@ class MlFoundry:
|
|
|
722
713
|
version=resolved_version,
|
|
723
714
|
)
|
|
724
715
|
model_version = _model_version.model_version
|
|
725
|
-
assert model_version.model_id is not None
|
|
726
716
|
_model = self._mlfoundry_artifacts_api.get_model_get(id=model_version.model_id)
|
|
727
717
|
model = _model.model
|
|
728
718
|
|
|
@@ -865,7 +855,6 @@ class MlFoundry:
|
|
|
865
855
|
model: Optional[ModelDto] = None,
|
|
866
856
|
) -> Iterator[ModelVersion]:
|
|
867
857
|
if model and not model_id:
|
|
868
|
-
assert model.id is not None
|
|
869
858
|
model_id = model.id
|
|
870
859
|
elif not model and model_id:
|
|
871
860
|
_model = self._mlfoundry_artifacts_api.get_model_get(id=str(model_id))
|
|
@@ -940,7 +929,6 @@ class MlFoundry:
|
|
|
940
929
|
)
|
|
941
930
|
)
|
|
942
931
|
artifact_version = _artifact_version.artifact_version
|
|
943
|
-
assert artifact_version.artifact_id is not None
|
|
944
932
|
_artifact = self._mlfoundry_artifacts_api.get_artifact_by_id_get(
|
|
945
933
|
id=artifact_version.artifact_id
|
|
946
934
|
)
|
|
@@ -1069,7 +1057,6 @@ class MlFoundry:
|
|
|
1069
1057
|
artifact: Optional[ArtifactDto] = None,
|
|
1070
1058
|
) -> Iterator[ArtifactVersion]:
|
|
1071
1059
|
if artifact and not artifact_id:
|
|
1072
|
-
assert artifact.id is not None
|
|
1073
1060
|
artifact_id = artifact.id
|
|
1074
1061
|
elif not artifact and artifact_id:
|
|
1075
1062
|
_artifact = self._mlfoundry_artifacts_api.get_artifact_by_id_get(
|
|
@@ -1273,6 +1260,7 @@ class MlFoundry:
|
|
|
1273
1260
|
metadata (Optional[Dict[str, Any]], optional): arbitrary json serializable dictionary to store metadata.
|
|
1274
1261
|
For example, you can use this to store metrics, params, notes.
|
|
1275
1262
|
This field can be updated at any time after logging. Defaults to `None`
|
|
1263
|
+
progress (bool): value to show progress bar, defaults to None.
|
|
1276
1264
|
|
|
1277
1265
|
Returns:
|
|
1278
1266
|
truefoundry.ml.ModelVersion: an instance of `ModelVersion` that can be used to download the files,
|
|
@@ -1382,7 +1370,7 @@ class MlFoundry:
|
|
|
1382
1370
|
Args:
|
|
1383
1371
|
ml_repo (str): Name of the ML Repo in which you want to create data_directory
|
|
1384
1372
|
name (str): Name of the DataDirectory to be created.
|
|
1385
|
-
description (str): Description
|
|
1373
|
+
description (str): Description for the DataDirectory.
|
|
1386
1374
|
metadata (Dict <str>: Any): Metadata about the data_directory in Dictionary form.
|
|
1387
1375
|
|
|
1388
1376
|
Returns:
|
truefoundry/ml/mlfoundry_run.py
CHANGED
|
@@ -110,8 +110,6 @@ class MlFoundryRun:
|
|
|
110
110
|
@classmethod
|
|
111
111
|
def _from_dto(cls, run_dto: RunDto) -> "MlFoundryRun":
|
|
112
112
|
"""classmethod to get MLfoundry run from dto instance"""
|
|
113
|
-
assert run_dto.info.experiment_id is not None
|
|
114
|
-
assert run_dto.info.run_id is not None
|
|
115
113
|
run = cls(run_dto.info.experiment_id, run_dto.info.run_id)
|
|
116
114
|
run._run_info = run_dto.info
|
|
117
115
|
run._run_data = run_dto.data
|
|
@@ -136,7 +134,6 @@ class MlFoundryRun:
|
|
|
136
134
|
def run_name(self) -> str:
|
|
137
135
|
"""Get run_name for the current `run`"""
|
|
138
136
|
run_info = self._get_run_info()
|
|
139
|
-
assert run_info.name is not None
|
|
140
137
|
return run_info.name
|
|
141
138
|
|
|
142
139
|
@property
|
|
@@ -144,7 +141,6 @@ class MlFoundryRun:
|
|
|
144
141
|
def fqn(self) -> str:
|
|
145
142
|
"""Get fqn for the current `run`"""
|
|
146
143
|
run_info = self._get_run_info()
|
|
147
|
-
assert run_info.fqn is not None
|
|
148
144
|
return run_info.fqn
|
|
149
145
|
|
|
150
146
|
@property
|
|
@@ -162,7 +158,6 @@ class MlFoundryRun:
|
|
|
162
158
|
_experiment = self._experiments_api.get_experiment_get(
|
|
163
159
|
experiment_id=self._experiment_id
|
|
164
160
|
)
|
|
165
|
-
assert _experiment.experiment.name is not None
|
|
166
161
|
return _experiment.experiment.name
|
|
167
162
|
|
|
168
163
|
@property
|
truefoundry/ml/run_utils.py
CHANGED
|
@@ -7,7 +7,6 @@ from urllib.parse import urljoin, urlsplit
|
|
|
7
7
|
|
|
8
8
|
import numpy as np
|
|
9
9
|
|
|
10
|
-
from truefoundry.ml import env_vars
|
|
11
10
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
12
11
|
|
|
13
12
|
|
|
@@ -22,21 +21,13 @@ def get_module(
|
|
|
22
21
|
raise MlFoundryException(msg) from ex
|
|
23
22
|
|
|
24
23
|
|
|
25
|
-
def resolve_tracking_uri(tracking_uri: Optional[str]):
|
|
26
|
-
if not tracking_uri and not os.getenv(env_vars.TRACKING_HOST_GLOBAL):
|
|
27
|
-
raise ValueError(
|
|
28
|
-
f"Either `host` should be provided by --host <value>, or `{env_vars.TRACKING_HOST_GLOBAL}` env must be set"
|
|
29
|
-
)
|
|
30
|
-
return tracking_uri or os.getenv(env_vars.TRACKING_HOST_GLOBAL)
|
|
31
|
-
|
|
32
|
-
|
|
33
24
|
def append_path_to_rest_tracking_uri(tracking_uri: str):
|
|
34
25
|
if urlsplit(tracking_uri).netloc.startswith("localhost"):
|
|
35
26
|
return tracking_uri
|
|
36
27
|
return urljoin(tracking_uri, "/api/ml")
|
|
37
28
|
|
|
38
29
|
|
|
39
|
-
def
|
|
30
|
+
def append_servicefoundry_path_to_tracking_uri(tracking_uri: str):
|
|
40
31
|
if urlsplit(tracking_uri).netloc.startswith("localhost"):
|
|
41
32
|
return os.getenv("SERVICEFOUNDRY_SERVER_URL")
|
|
42
33
|
return urljoin(tracking_uri, "/api/svc")
|