truefoundry 0.4.0rc1__py3-none-any.whl → 0.4.0rc3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/__init__.py +2 -0
- truefoundry/autodeploy/agents/developer.py +1 -1
- truefoundry/autodeploy/agents/project_identifier.py +2 -2
- truefoundry/autodeploy/agents/tester.py +1 -1
- truefoundry/autodeploy/cli.py +1 -1
- truefoundry/autodeploy/tools/list_files.py +1 -1
- truefoundry/{deploy/lib/auth → common}/auth_service_client.py +50 -40
- truefoundry/common/constants.py +12 -0
- truefoundry/{deploy/lib/auth → common}/credential_file_manager.py +7 -7
- truefoundry/{deploy/lib/auth → common}/credential_provider.py +9 -12
- truefoundry/{ml/services → common}/entities.py +57 -41
- truefoundry/common/exceptions.py +12 -0
- truefoundry/common/request_utils.py +36 -8
- truefoundry/common/servicefoundry_client.py +91 -0
- truefoundry/common/utils.py +56 -0
- truefoundry/deploy/auto_gen/models.py +4 -6
- truefoundry/deploy/cli/cli.py +2 -0
- truefoundry/deploy/cli/commands/apply_command.py +1 -1
- truefoundry/deploy/cli/commands/build_command.py +1 -1
- truefoundry/deploy/cli/commands/deploy_command.py +1 -1
- truefoundry/deploy/cli/commands/login_command.py +2 -2
- truefoundry/deploy/cli/commands/patch_application_command.py +1 -1
- truefoundry/deploy/cli/commands/patch_command.py +1 -1
- truefoundry/deploy/cli/commands/terminate_comand.py +1 -1
- truefoundry/deploy/cli/util.py +1 -1
- truefoundry/deploy/function_service/remote/remote.py +1 -1
- truefoundry/deploy/lib/auth/servicefoundry_session.py +2 -2
- truefoundry/deploy/lib/clients/servicefoundry_client.py +120 -150
- truefoundry/deploy/lib/const.py +1 -35
- truefoundry/deploy/lib/exceptions.py +0 -11
- truefoundry/deploy/lib/model/entity.py +1 -112
- truefoundry/deploy/lib/session.py +13 -26
- truefoundry/deploy/lib/util.py +0 -37
- truefoundry/deploy/v2/lib/deploy.py +3 -3
- truefoundry/deploy/v2/lib/deployable_patched_models.py +1 -1
- truefoundry/ml/__init__.py +0 -9
- truefoundry/ml/artifact/truefoundry_artifact_repo.py +63 -22
- truefoundry/ml/autogen/client/__init__.py +0 -3
- truefoundry/ml/autogen/client/api/experiments_api.py +0 -165
- truefoundry/ml/autogen/client/models/__init__.py +0 -3
- truefoundry/ml/autogen/client/models/artifact_dto.py +6 -6
- truefoundry/ml/autogen/client/models/artifact_version_dto.py +8 -8
- truefoundry/ml/autogen/client/models/create_artifact_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_artifact_version_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_python_deployment_config_request_dto.py +2 -2
- truefoundry/ml/autogen/client/models/create_python_deployment_config_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/create_run_request_dto.py +5 -5
- truefoundry/ml/autogen/client/models/create_run_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/dataset_dto.py +10 -10
- truefoundry/ml/autogen/client/models/experiment_dto.py +18 -18
- truefoundry/ml/autogen/client/models/get_latest_run_log_response_dto.py +2 -3
- truefoundry/ml/autogen/client/models/get_tenant_id_response_dto.py +4 -5
- truefoundry/ml/autogen/client/models/model_dto.py +6 -6
- truefoundry/ml/autogen/client/models/model_version_dto.py +15 -8
- truefoundry/ml/autogen/client/models/run_info_dto.py +10 -10
- truefoundry/ml/autogen/client/models/update_run_response_dto.py +2 -3
- truefoundry/ml/autogen/client_README.md +0 -2
- truefoundry/ml/clients/entities.py +8 -0
- truefoundry/ml/{services/servicefoundry_service.py → clients/servicefoundry_client.py} +20 -10
- truefoundry/ml/{services → clients}/utils.py +2 -2
- truefoundry/ml/entities.py +62 -0
- truefoundry/ml/env_vars.py +1 -5
- truefoundry/ml/internal_namespace.py +8 -8
- truefoundry/ml/log_types/artifacts/artifact.py +7 -3
- truefoundry/ml/log_types/artifacts/dataset.py +5 -4
- truefoundry/ml/log_types/artifacts/model.py +8 -25
- truefoundry/ml/log_types/image/image.py +7 -8
- truefoundry/ml/log_types/image/image_normalizer.py +7 -6
- truefoundry/ml/mlfoundry_api.py +10 -55
- truefoundry/ml/mlfoundry_run.py +6 -41
- truefoundry/ml/run_utils.py +1 -10
- truefoundry/ml/session.py +14 -117
- truefoundry/pydantic_v1.py +1 -1
- truefoundry/workflow/__init__.py +16 -1
- {truefoundry-0.4.0rc1.dist-info → truefoundry-0.4.0rc3.dist-info}/METADATA +1 -1
- {truefoundry-0.4.0rc1.dist-info → truefoundry-0.4.0rc3.dist-info}/RECORD +79 -77
- truefoundry/deploy/lib/clients/utils.py +0 -41
- truefoundry/ml/autogen/client/models/backfill_default_storage_integration_id_request_dto.py +0 -67
- truefoundry/ml/login.py +0 -241
- truefoundry/ml/services/auth_service.py +0 -109
- /truefoundry/ml/{services → clients}/__init__.py +0 -0
- {truefoundry-0.4.0rc1.dist-info → truefoundry-0.4.0rc3.dist-info}/WHEEL +0 -0
- {truefoundry-0.4.0rc1.dist-info → truefoundry-0.4.0rc3.dist-info}/entry_points.txt +0 -0
|
@@ -24,7 +24,14 @@ from truefoundry.ml.autogen.client.models.artifact_version_status import (
|
|
|
24
24
|
)
|
|
25
25
|
from truefoundry.ml.autogen.client.models.metric_dto import MetricDto
|
|
26
26
|
from truefoundry.ml.autogen.client.models.model_schema_dto import ModelSchemaDto
|
|
27
|
-
from truefoundry.pydantic_v1 import
|
|
27
|
+
from truefoundry.pydantic_v1 import (
|
|
28
|
+
BaseModel,
|
|
29
|
+
Field,
|
|
30
|
+
StrictBool,
|
|
31
|
+
StrictInt,
|
|
32
|
+
StrictStr,
|
|
33
|
+
conlist,
|
|
34
|
+
)
|
|
28
35
|
|
|
29
36
|
|
|
30
37
|
class ModelVersionDto(BaseModel):
|
|
@@ -32,10 +39,11 @@ class ModelVersionDto(BaseModel):
|
|
|
32
39
|
ModelVersionDto
|
|
33
40
|
"""
|
|
34
41
|
|
|
35
|
-
id:
|
|
36
|
-
model_id:
|
|
37
|
-
version:
|
|
38
|
-
fqn:
|
|
42
|
+
id: StrictStr = Field(...)
|
|
43
|
+
model_id: StrictStr = Field(...)
|
|
44
|
+
version: StrictInt = Field(...)
|
|
45
|
+
fqn: StrictStr = Field(...)
|
|
46
|
+
experiment_id: StrictStr = Field(...)
|
|
39
47
|
artifact_storage_root: Optional[StrictStr] = None
|
|
40
48
|
artifact_metadata: Optional[Dict[str, Any]] = None
|
|
41
49
|
description: Optional[StrictStr] = None
|
|
@@ -50,7 +58,6 @@ class ModelVersionDto(BaseModel):
|
|
|
50
58
|
data_path: Optional[StrictStr] = None
|
|
51
59
|
monitoring_enabled: Optional[StrictBool] = False
|
|
52
60
|
usage_code_snippet: Optional[StrictStr] = None
|
|
53
|
-
experiment_id: Optional[StrictStr] = None
|
|
54
61
|
run_id: Optional[StrictStr] = None
|
|
55
62
|
metrics: Optional[conlist(MetricDto)] = None
|
|
56
63
|
model_name: Optional[StrictStr] = None
|
|
@@ -61,6 +68,7 @@ class ModelVersionDto(BaseModel):
|
|
|
61
68
|
"model_id",
|
|
62
69
|
"version",
|
|
63
70
|
"fqn",
|
|
71
|
+
"experiment_id",
|
|
64
72
|
"artifact_storage_root",
|
|
65
73
|
"artifact_metadata",
|
|
66
74
|
"description",
|
|
@@ -75,7 +83,6 @@ class ModelVersionDto(BaseModel):
|
|
|
75
83
|
"data_path",
|
|
76
84
|
"monitoring_enabled",
|
|
77
85
|
"usage_code_snippet",
|
|
78
|
-
"experiment_id",
|
|
79
86
|
"run_id",
|
|
80
87
|
"metrics",
|
|
81
88
|
"model_name",
|
|
@@ -132,6 +139,7 @@ class ModelVersionDto(BaseModel):
|
|
|
132
139
|
"model_id": obj.get("model_id"),
|
|
133
140
|
"version": obj.get("version"),
|
|
134
141
|
"fqn": obj.get("fqn"),
|
|
142
|
+
"experiment_id": obj.get("experiment_id"),
|
|
135
143
|
"artifact_storage_root": obj.get("artifact_storage_root"),
|
|
136
144
|
"artifact_metadata": obj.get("artifact_metadata"),
|
|
137
145
|
"description": obj.get("description"),
|
|
@@ -150,7 +158,6 @@ class ModelVersionDto(BaseModel):
|
|
|
150
158
|
if obj.get("monitoring_enabled") is not None
|
|
151
159
|
else False,
|
|
152
160
|
"usage_code_snippet": obj.get("usage_code_snippet"),
|
|
153
|
-
"experiment_id": obj.get("experiment_id"),
|
|
154
161
|
"run_id": obj.get("run_id"),
|
|
155
162
|
"metrics": [MetricDto.from_dict(_item) for _item in obj.get("metrics")]
|
|
156
163
|
if obj.get("metrics") is not None
|
|
@@ -18,7 +18,7 @@ import pprint
|
|
|
18
18
|
import re # noqa: F401
|
|
19
19
|
from typing import Optional
|
|
20
20
|
|
|
21
|
-
from truefoundry.pydantic_v1 import BaseModel, StrictInt, StrictStr
|
|
21
|
+
from truefoundry.pydantic_v1 import BaseModel, Field, StrictInt, StrictStr
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
class RunInfoDto(BaseModel):
|
|
@@ -26,30 +26,30 @@ class RunInfoDto(BaseModel):
|
|
|
26
26
|
RunInfoDto
|
|
27
27
|
"""
|
|
28
28
|
|
|
29
|
-
run_uuid:
|
|
30
|
-
run_id:
|
|
31
|
-
experiment_id:
|
|
29
|
+
run_uuid: StrictStr = Field(...)
|
|
30
|
+
run_id: StrictStr = Field(...)
|
|
31
|
+
experiment_id: StrictStr = Field(...)
|
|
32
|
+
name: StrictStr = Field(...)
|
|
33
|
+
fqn: StrictStr = Field(...)
|
|
32
34
|
user_id: Optional[StrictStr] = None
|
|
33
35
|
status: Optional[StrictStr] = None
|
|
34
36
|
start_time: Optional[StrictInt] = None
|
|
35
37
|
end_time: Optional[StrictInt] = None
|
|
36
38
|
artifact_uri: Optional[StrictStr] = None
|
|
37
39
|
lifecycle_stage: Optional[StrictStr] = None
|
|
38
|
-
name: Optional[StrictStr] = None
|
|
39
|
-
fqn: Optional[StrictStr] = None
|
|
40
40
|
description: Optional[StrictStr] = None
|
|
41
41
|
__properties = [
|
|
42
42
|
"run_uuid",
|
|
43
43
|
"run_id",
|
|
44
44
|
"experiment_id",
|
|
45
|
+
"name",
|
|
46
|
+
"fqn",
|
|
45
47
|
"user_id",
|
|
46
48
|
"status",
|
|
47
49
|
"start_time",
|
|
48
50
|
"end_time",
|
|
49
51
|
"artifact_uri",
|
|
50
52
|
"lifecycle_stage",
|
|
51
|
-
"name",
|
|
52
|
-
"fqn",
|
|
53
53
|
"description",
|
|
54
54
|
]
|
|
55
55
|
|
|
@@ -91,14 +91,14 @@ class RunInfoDto(BaseModel):
|
|
|
91
91
|
"run_uuid": obj.get("run_uuid"),
|
|
92
92
|
"run_id": obj.get("run_id"),
|
|
93
93
|
"experiment_id": obj.get("experiment_id"),
|
|
94
|
+
"name": obj.get("name"),
|
|
95
|
+
"fqn": obj.get("fqn"),
|
|
94
96
|
"user_id": obj.get("user_id"),
|
|
95
97
|
"status": obj.get("status"),
|
|
96
98
|
"start_time": obj.get("start_time"),
|
|
97
99
|
"end_time": obj.get("end_time"),
|
|
98
100
|
"artifact_uri": obj.get("artifact_uri"),
|
|
99
101
|
"lifecycle_stage": obj.get("lifecycle_stage"),
|
|
100
|
-
"name": obj.get("name"),
|
|
101
|
-
"fqn": obj.get("fqn"),
|
|
102
102
|
"description": obj.get("description"),
|
|
103
103
|
}
|
|
104
104
|
)
|
|
@@ -16,10 +16,9 @@ from __future__ import annotations
|
|
|
16
16
|
import json
|
|
17
17
|
import pprint
|
|
18
18
|
import re # noqa: F401
|
|
19
|
-
from typing import Optional
|
|
20
19
|
|
|
21
20
|
from truefoundry.ml.autogen.client.models.run_info_dto import RunInfoDto
|
|
22
|
-
from truefoundry.pydantic_v1 import BaseModel
|
|
21
|
+
from truefoundry.pydantic_v1 import BaseModel, Field
|
|
23
22
|
|
|
24
23
|
|
|
25
24
|
class UpdateRunResponseDto(BaseModel):
|
|
@@ -27,7 +26,7 @@ class UpdateRunResponseDto(BaseModel):
|
|
|
27
26
|
UpdateRunResponseDto
|
|
28
27
|
"""
|
|
29
28
|
|
|
30
|
-
run_info:
|
|
29
|
+
run_info: RunInfoDto = Field(...)
|
|
31
30
|
__properties = ["run_info"]
|
|
32
31
|
|
|
33
32
|
class Config:
|
|
@@ -69,7 +69,6 @@ Class | Method | HTTP request | Description
|
|
|
69
69
|
*DeprecatedApi* | [**get_run_by_name_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_run_by_name_get) | **GET** /api/2.0mlflow/runs/get-by-name | Get Run By Name
|
|
70
70
|
*DeprecatedApi* | [**get_search_runs_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_search_runs_get) | **GET** /api/2.0/preview/mlflow/runs/search | Get Search Runs
|
|
71
71
|
*DeprecatedApi* | [**get_signed_urls_for_dataset_write_deprecated_get**](truefoundry/ml/autogen/client/docs/DeprecatedApi.md#get_signed_urls_for_dataset_write_deprecated_get) | **GET** /api/2.0/mlflow/mlfoundry-artifacts/datasets/get-signed-urls-for-write | Get Signed Urls For Dataset Write Deprecated
|
|
72
|
-
*ExperimentsApi* | [**backfill_storage_integration_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#backfill_storage_integration_post) | **POST** /api/2.0/mlflow/experiments/storage-integration/backfill | Backfill Storage Integration
|
|
73
72
|
*ExperimentsApi* | [**create_experiment_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#create_experiment_post) | **POST** /api/2.0/mlflow/experiments/create | Create Experiment
|
|
74
73
|
*ExperimentsApi* | [**delete_experiment_post**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#delete_experiment_post) | **POST** /api/2.0/mlflow/experiments/delete | Delete Experiment
|
|
75
74
|
*ExperimentsApi* | [**get_experiment_by_name_get**](truefoundry/ml/autogen/client/docs/ExperimentsApi.md#get_experiment_by_name_get) | **GET** /api/2.0/mlflow/experiments/get-by-name | Get Experiment By Name
|
|
@@ -170,7 +169,6 @@ Class | Method | HTTP request | Description
|
|
|
170
169
|
- [AssistantMessage](truefoundry/ml/autogen/client/docs/AssistantMessage.md)
|
|
171
170
|
- [AuthorizeUserForModelRequestDto](truefoundry/ml/autogen/client/docs/AuthorizeUserForModelRequestDto.md)
|
|
172
171
|
- [AuthorizeUserForModelVersionRequestDto](truefoundry/ml/autogen/client/docs/AuthorizeUserForModelVersionRequestDto.md)
|
|
173
|
-
- [BackfillDefaultStorageIntegrationIdRequestDto](truefoundry/ml/autogen/client/docs/BackfillDefaultStorageIntegrationIdRequestDto.md)
|
|
174
172
|
- [BlobStorageReference](truefoundry/ml/autogen/client/docs/BlobStorageReference.md)
|
|
175
173
|
- [BodyGetSearchRunsGet](truefoundry/ml/autogen/client/docs/BodyGetSearchRunsGet.md)
|
|
176
174
|
- [ChatPrompt](truefoundry/ml/autogen/client/docs/ChatPrompt.md)
|
|
@@ -1,28 +1,38 @@
|
|
|
1
1
|
from typing import Optional
|
|
2
2
|
|
|
3
|
-
from truefoundry.common.
|
|
3
|
+
from truefoundry.common.constants import (
|
|
4
|
+
SERVICEFOUNDRY_CLIENT_MAX_RETRIES,
|
|
5
|
+
VERSION_PREFIX,
|
|
6
|
+
)
|
|
7
|
+
from truefoundry.common.servicefoundry_client import (
|
|
8
|
+
ServiceFoundryServiceClient as BaseServiceFoundryServiceClient,
|
|
9
|
+
)
|
|
10
|
+
from truefoundry.ml.clients.entities import (
|
|
11
|
+
HostCreds,
|
|
12
|
+
)
|
|
13
|
+
from truefoundry.ml.clients.utils import http_request_safe
|
|
4
14
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
5
|
-
from truefoundry.ml.run_utils import
|
|
6
|
-
from truefoundry.ml.services.entities import HostCreds
|
|
7
|
-
from truefoundry.ml.services.utils import http_request_safe
|
|
15
|
+
from truefoundry.ml.run_utils import append_servicefoundry_path_to_tracking_uri
|
|
8
16
|
|
|
9
17
|
|
|
10
|
-
class
|
|
18
|
+
class ServiceFoundryServiceClient(BaseServiceFoundryServiceClient):
|
|
11
19
|
def __init__(self, tracking_uri: str, token: Optional[str] = None):
|
|
20
|
+
super().__init__(base_url=tracking_uri)
|
|
12
21
|
self.host_creds = HostCreds(
|
|
13
|
-
host=
|
|
22
|
+
host=append_servicefoundry_path_to_tracking_uri(tracking_uri), token=token
|
|
14
23
|
)
|
|
15
24
|
|
|
16
25
|
def get_integration_from_id(self, integration_id: str):
|
|
17
26
|
integration_id = integration_id or ""
|
|
18
|
-
|
|
19
|
-
method="get",
|
|
27
|
+
response = http_request_safe(
|
|
20
28
|
host_creds=self.host_creds,
|
|
21
|
-
endpoint="
|
|
22
|
-
session=requests_retry_session(retries=1),
|
|
29
|
+
endpoint=f"{VERSION_PREFIX}/provider-accounts/provider-integrations",
|
|
23
30
|
params={"id": integration_id, "type": "blob-storage"},
|
|
31
|
+
method="get",
|
|
24
32
|
timeout=3,
|
|
33
|
+
max_retries=SERVICEFOUNDRY_CLIENT_MAX_RETRIES,
|
|
25
34
|
)
|
|
35
|
+
data = response.json()
|
|
26
36
|
if (
|
|
27
37
|
data.get("providerIntegrations")
|
|
28
38
|
and len(data["providerIntegrations"]) > 0
|
|
@@ -6,8 +6,8 @@ from urllib.parse import urljoin
|
|
|
6
6
|
import requests
|
|
7
7
|
|
|
8
8
|
from truefoundry.common.request_utils import requests_retry_session
|
|
9
|
+
from truefoundry.ml.clients.entities import HostCreds
|
|
9
10
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
10
|
-
from truefoundry.ml.services.entities import HostCreds
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
# TODO: This will be moved later to truefoundry.common.request_utils
|
|
@@ -77,7 +77,7 @@ def cloud_storage_http_request(
|
|
|
77
77
|
"""
|
|
78
78
|
Performs an HTTP PUT/GET request using Python's `requests` module with automatic retry.
|
|
79
79
|
"""
|
|
80
|
-
session = session or requests_retry_session(retries=5, backoff_factor=
|
|
80
|
+
session = session or requests_retry_session(retries=5, backoff_factor=0.5)
|
|
81
81
|
kwargs["headers"] = kwargs.get("headers", {}) or {}
|
|
82
82
|
if "blob.core.windows.net" in url:
|
|
83
83
|
kwargs["headers"].update({"x-ms-blob-type": "BlockBlob"})
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from typing import Optional, Union
|
|
2
|
+
|
|
3
|
+
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
4
|
+
FileInfoDto,
|
|
5
|
+
MetricDto,
|
|
6
|
+
)
|
|
7
|
+
from truefoundry.pydantic_v1 import (
|
|
8
|
+
BaseModel,
|
|
9
|
+
Field,
|
|
10
|
+
StrictBool,
|
|
11
|
+
StrictFloat,
|
|
12
|
+
StrictInt,
|
|
13
|
+
StrictStr,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Metric(BaseModel):
|
|
18
|
+
key: StrictStr = Field(...)
|
|
19
|
+
value: Optional[Union[StrictFloat, StrictInt]] = None
|
|
20
|
+
timestamp: Optional[StrictInt] = None
|
|
21
|
+
step: Optional[StrictInt] = 0
|
|
22
|
+
|
|
23
|
+
@classmethod
|
|
24
|
+
def from_dto(cls, dto: MetricDto) -> "Metric":
|
|
25
|
+
return cls(
|
|
26
|
+
key=dto.key,
|
|
27
|
+
value=dto.value,
|
|
28
|
+
timestamp=dto.timestamp,
|
|
29
|
+
step=dto.step,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def to_dto(self) -> MetricDto:
|
|
33
|
+
return MetricDto(
|
|
34
|
+
key=self.key,
|
|
35
|
+
value=self.value,
|
|
36
|
+
timestamp=self.timestamp,
|
|
37
|
+
step=self.step,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class FileInfo(BaseModel):
|
|
42
|
+
path: StrictStr
|
|
43
|
+
is_dir: StrictBool
|
|
44
|
+
file_size: Optional[StrictInt] = None
|
|
45
|
+
signed_url: Optional[StrictStr] = None
|
|
46
|
+
|
|
47
|
+
@classmethod
|
|
48
|
+
def from_dto(cls, dto: FileInfoDto) -> "FileInfo":
|
|
49
|
+
return cls(
|
|
50
|
+
path=dto.path,
|
|
51
|
+
is_dir=dto.is_dir,
|
|
52
|
+
file_size=dto.file_size,
|
|
53
|
+
signed_url=dto.signed_url,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def to_dto(self) -> FileInfoDto:
|
|
57
|
+
return FileInfoDto(
|
|
58
|
+
path=self.path,
|
|
59
|
+
is_dir=self.is_dir,
|
|
60
|
+
file_size=self.file_size,
|
|
61
|
+
signed_url=self.signed_url,
|
|
62
|
+
)
|
truefoundry/ml/env_vars.py
CHANGED
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
"""
|
|
2
|
-
Define all the env variable names that users
|
|
3
|
-
can use
|
|
2
|
+
Define all the env variable names that users can use
|
|
4
3
|
"""
|
|
5
4
|
|
|
6
|
-
# NOTE: These are shared between servicefoundry and mlfoundry
|
|
7
|
-
TRACKING_HOST_GLOBAL = "TFY_HOST"
|
|
8
|
-
API_KEY_GLOBAL = "TFY_API_KEY"
|
|
9
5
|
DISABLE_MULTIPART_UPLOAD = "MLF_DISABLE_MULTIPART_UPLOAD"
|
|
10
6
|
INTERNAL_ENV_VARS = [
|
|
11
7
|
"TFY_INTERNAL_APPLICATION_ID",
|
|
@@ -6,39 +6,39 @@ from truefoundry.ml.exceptions import MlFoundryException
|
|
|
6
6
|
|
|
7
7
|
class _InternalNamespace:
|
|
8
8
|
NAMESPACE = "mlf"
|
|
9
|
-
|
|
9
|
+
DELIMITER = "."
|
|
10
10
|
NAMESPACE_VIOLATION_MESSAGE = """
|
|
11
11
|
{name} cannot start with {prefix}
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
def __call__(self, name: str,
|
|
14
|
+
def __call__(self, name: str, delimiter: str = DELIMITER):
|
|
15
15
|
if not name:
|
|
16
16
|
raise MlFoundryException("name should be a non empty string")
|
|
17
|
-
return _InternalNamespace.NAMESPACE +
|
|
17
|
+
return _InternalNamespace.NAMESPACE + delimiter + name
|
|
18
18
|
|
|
19
19
|
def __truediv__(self, path: str):
|
|
20
20
|
return os.path.join(_InternalNamespace.NAMESPACE, path)
|
|
21
21
|
|
|
22
22
|
@staticmethod
|
|
23
|
-
def _validate_name_not_using_namespace(name: typing.Optional[str],
|
|
24
|
-
if name and name.startswith(_InternalNamespace.NAMESPACE +
|
|
23
|
+
def _validate_name_not_using_namespace(name: typing.Optional[str], delimiter: str):
|
|
24
|
+
if name and name.startswith(_InternalNamespace.NAMESPACE + delimiter):
|
|
25
25
|
raise MlFoundryException(
|
|
26
26
|
_InternalNamespace.NAMESPACE_VIOLATION_MESSAGE.format(
|
|
27
|
-
name=name, prefix=_InternalNamespace.NAMESPACE +
|
|
27
|
+
name=name, prefix=_InternalNamespace.NAMESPACE + delimiter
|
|
28
28
|
)
|
|
29
29
|
)
|
|
30
30
|
|
|
31
31
|
def validate_namespace_not_used(
|
|
32
32
|
self,
|
|
33
33
|
names: typing.Optional[typing.Union[str, typing.Iterable[str]]] = None,
|
|
34
|
-
|
|
34
|
+
delimiter: str = DELIMITER,
|
|
35
35
|
path: typing.Optional[str] = None,
|
|
36
36
|
):
|
|
37
37
|
if isinstance(names, str):
|
|
38
38
|
names = [names]
|
|
39
39
|
if names is not None:
|
|
40
40
|
for name_ in names:
|
|
41
|
-
self._validate_name_not_using_namespace(name_,
|
|
41
|
+
self._validate_name_not_using_namespace(name_, delimiter)
|
|
42
42
|
if path:
|
|
43
43
|
prefix = os.path.normpath(os.path.join(_InternalNamespace.NAMESPACE, ""))
|
|
44
44
|
if os.path.normpath(path).startswith(prefix):
|
|
@@ -3,6 +3,7 @@ import datetime
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
5
|
import tempfile
|
|
6
|
+
import uuid
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Optional, Tuple, Union
|
|
8
9
|
|
|
@@ -21,7 +22,7 @@ from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
|
21
22
|
NotifyArtifactVersionFailureDto,
|
|
22
23
|
UpdateArtifactVersionRequestDto,
|
|
23
24
|
)
|
|
24
|
-
from truefoundry.ml.autogen.client import (
|
|
25
|
+
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
25
26
|
InternalMetadata as InternalMetadataDto,
|
|
26
27
|
)
|
|
27
28
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
@@ -130,7 +131,7 @@ class ArtifactVersion:
|
|
|
130
131
|
def _get_artifacts_repo(self):
|
|
131
132
|
return MlFoundryArtifactsRepository(
|
|
132
133
|
artifact_identifier=ArtifactIdentifier(
|
|
133
|
-
artifact_version_id=self._artifact_version.id
|
|
134
|
+
artifact_version_id=uuid.UUID(self._artifact_version.id)
|
|
134
135
|
),
|
|
135
136
|
api_client=self._api_client,
|
|
136
137
|
)
|
|
@@ -374,6 +375,7 @@ def _log_artifact_version_helper(
|
|
|
374
375
|
if run:
|
|
375
376
|
mlfoundry_artifacts_api = run._mlfoundry_artifacts_api
|
|
376
377
|
|
|
378
|
+
assert mlfoundry_artifacts_api is not None
|
|
377
379
|
_create_artifact_response = mlfoundry_artifacts_api.create_artifact_version_post(
|
|
378
380
|
create_artifact_version_request_dto=CreateArtifactVersionRequestDto(
|
|
379
381
|
experiment_id=int(run._experiment_id if run else ml_repo_id),
|
|
@@ -383,7 +385,9 @@ def _log_artifact_version_helper(
|
|
|
383
385
|
)
|
|
384
386
|
version_id = _create_artifact_response.id
|
|
385
387
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
386
|
-
artifact_identifier=ArtifactIdentifier(
|
|
388
|
+
artifact_identifier=ArtifactIdentifier(
|
|
389
|
+
artifact_version_id=uuid.UUID(version_id),
|
|
390
|
+
),
|
|
387
391
|
api_client=mlfoundry_artifacts_api.api_client,
|
|
388
392
|
)
|
|
389
393
|
total_size = calculate_local_directory_size(artifact_dir)
|
|
@@ -12,10 +12,10 @@ from truefoundry.ml.artifact.truefoundry_artifact_repo import (
|
|
|
12
12
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
13
13
|
DatasetDto,
|
|
14
14
|
DeleteDatasetRequestDto,
|
|
15
|
-
FileInfoDto,
|
|
16
15
|
MlfoundryArtifactsApi,
|
|
17
16
|
UpdateDatasetRequestDto,
|
|
18
17
|
)
|
|
18
|
+
from truefoundry.ml.entities import FileInfo
|
|
19
19
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
20
20
|
from truefoundry.ml.log_types.artifacts.utils import (
|
|
21
21
|
_copy_additional_files,
|
|
@@ -136,7 +136,7 @@ class DataDirectory:
|
|
|
136
136
|
|
|
137
137
|
@property
|
|
138
138
|
def updated_at(self) -> datetime.datetime:
|
|
139
|
-
"""Get the information about
|
|
139
|
+
"""Get the information about when the DataDirectory was updated"""
|
|
140
140
|
return self._dataset.updated_at
|
|
141
141
|
|
|
142
142
|
def add_files(
|
|
@@ -239,7 +239,7 @@ class DataDirectory:
|
|
|
239
239
|
def list_files(
|
|
240
240
|
self,
|
|
241
241
|
path: Optional[str] = None,
|
|
242
|
-
) -> Iterator[
|
|
242
|
+
) -> Iterator[FileInfo]:
|
|
243
243
|
"""
|
|
244
244
|
List all the files and folders in the data_directory.
|
|
245
245
|
|
|
@@ -262,7 +262,8 @@ class DataDirectory:
|
|
|
262
262
|
```
|
|
263
263
|
"""
|
|
264
264
|
artifacts_repo = self._get_artifacts_repo()
|
|
265
|
-
|
|
265
|
+
for file_info_dto in artifacts_repo.list_artifacts(path=path):
|
|
266
|
+
yield FileInfo.from_dto(file_info_dto)
|
|
266
267
|
|
|
267
268
|
def download(
|
|
268
269
|
self,
|
|
@@ -53,7 +53,6 @@ if TYPE_CHECKING:
|
|
|
53
53
|
logger = logging.getLogger("truefoundry.ml")
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
# TODO: Add some progress indicators for upload and download
|
|
57
56
|
# TODO: Support async download and upload
|
|
58
57
|
|
|
59
58
|
|
|
@@ -98,7 +97,6 @@ class ModelVersion:
|
|
|
98
97
|
self._deleted = False
|
|
99
98
|
self._description: str = ""
|
|
100
99
|
self._metadata: Dict[str, Any] = {}
|
|
101
|
-
self._model_schema: Optional[ModelSchema] = None
|
|
102
100
|
self._metrics: List[MetricDto] = []
|
|
103
101
|
self._set_metrics_attr()
|
|
104
102
|
self._set_mutable_attrs()
|
|
@@ -146,19 +144,17 @@ class ModelVersion:
|
|
|
146
144
|
def _set_mutable_attrs(self):
|
|
147
145
|
self._description = self._model_version.description or ""
|
|
148
146
|
self._metadata = copy.deepcopy(self._model_version.artifact_metadata)
|
|
149
|
-
self._model_schema = copy.deepcopy(self._model_version.model_schema)
|
|
150
147
|
|
|
151
148
|
def _refetch_model_version(self):
|
|
152
|
-
|
|
153
|
-
self.
|
|
154
|
-
id=self._model_version.id
|
|
155
|
-
)
|
|
149
|
+
_model_version = self._mlfoundry_artifacts_api.get_model_version_get(
|
|
150
|
+
id=self._model_version.id
|
|
156
151
|
)
|
|
152
|
+
self._model_version = _model_version.model_version
|
|
157
153
|
self._set_metrics_attr()
|
|
158
154
|
self._set_mutable_attrs()
|
|
159
155
|
|
|
160
156
|
def __repr__(self):
|
|
161
|
-
return f"{self.__class__.__name__}(fqn={self.fqn!r}
|
|
157
|
+
return f"{self.__class__.__name__}(fqn={self.fqn!r})"
|
|
162
158
|
|
|
163
159
|
def _get_artifacts_repo(self):
|
|
164
160
|
return MlFoundryArtifactsRepository(
|
|
@@ -215,17 +211,6 @@ class ModelVersion:
|
|
|
215
211
|
_validate_artifact_metadata(value)
|
|
216
212
|
self._metadata = value
|
|
217
213
|
|
|
218
|
-
@property
|
|
219
|
-
def model_schema(self) -> Optional[ModelSchema]:
|
|
220
|
-
"""get the model schema for current model"""
|
|
221
|
-
return self._model_schema
|
|
222
|
-
|
|
223
|
-
@model_schema.setter
|
|
224
|
-
def model_schema(self, value: Union[Dict[str, Any], ModelSchema]):
|
|
225
|
-
if not isinstance(value, ModelSchema):
|
|
226
|
-
value = ModelSchema.parse_obj(value)
|
|
227
|
-
self._model_schema = value
|
|
228
|
-
|
|
229
214
|
@property
|
|
230
215
|
def metrics(self) -> Dict[str, Union[float, int]]:
|
|
231
216
|
"""get the metrics for the current version of the model"""
|
|
@@ -396,15 +381,11 @@ class ModelVersion:
|
|
|
396
381
|
```
|
|
397
382
|
"""
|
|
398
383
|
self._ensure_not_deleted()
|
|
399
|
-
kwargs = {}
|
|
400
|
-
if self.model_schema is not None:
|
|
401
|
-
kwargs["model_schema"] = self.model_schema
|
|
402
384
|
_model_version = self._mlfoundry_artifacts_api.update_model_version_post(
|
|
403
385
|
update_model_version_request_dto=UpdateModelVersionRequestDto(
|
|
404
386
|
id=self._model_version.id,
|
|
405
387
|
description=self.description,
|
|
406
388
|
artifact_metadata=self.metadata,
|
|
407
|
-
**kwargs,
|
|
408
389
|
)
|
|
409
390
|
)
|
|
410
391
|
self._model_version = _model_version.model_version
|
|
@@ -552,7 +533,9 @@ def _log_model_version( # noqa: C901
|
|
|
552
533
|
)
|
|
553
534
|
version_id = _create_artifact_version_response.id
|
|
554
535
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
555
|
-
artifact_identifier=ArtifactIdentifier(
|
|
536
|
+
artifact_identifier=ArtifactIdentifier(
|
|
537
|
+
artifact_version_id=uuid.UUID(version_id)
|
|
538
|
+
),
|
|
556
539
|
api_client=mlfoundry_artifacts_api.api_client,
|
|
557
540
|
)
|
|
558
541
|
model_size = calculate_model_size(temp_dir)
|
|
@@ -602,7 +585,7 @@ def _log_model_version( # noqa: C901
|
|
|
602
585
|
model_version = _model_version.model_version
|
|
603
586
|
|
|
604
587
|
# update model schema at end
|
|
605
|
-
update_args = {
|
|
588
|
+
update_args: Dict[str, Any] = {
|
|
606
589
|
"id": version_id,
|
|
607
590
|
"model_framework": framework.value,
|
|
608
591
|
}
|
|
@@ -6,8 +6,6 @@ import tempfile
|
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
|
|
8
8
|
|
|
9
|
-
import numpy as np
|
|
10
|
-
|
|
11
9
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
12
10
|
ArtifactType,
|
|
13
11
|
)
|
|
@@ -33,17 +31,18 @@ from truefoundry.ml.run_utils import get_module
|
|
|
33
31
|
|
|
34
32
|
if TYPE_CHECKING:
|
|
35
33
|
# noinspection PyUnresolvedReferences
|
|
34
|
+
import numpy # noqa: F401
|
|
36
35
|
import PIL.Image
|
|
37
|
-
from numpy.typing import NDArray
|
|
36
|
+
from numpy.typing import NDArray # noqa: F401
|
|
38
37
|
|
|
39
38
|
from truefoundry.ml.mlfoundry_run import MlFoundryRun
|
|
40
39
|
|
|
41
40
|
ImageLikeNDArray = Union[
|
|
42
|
-
"NDArray[
|
|
43
|
-
"NDArray[
|
|
44
|
-
"NDArray[
|
|
45
|
-
"NDArray[
|
|
46
|
-
"NDArray[
|
|
41
|
+
"NDArray[numpy.int8]",
|
|
42
|
+
"NDArray[numpy.uint8]",
|
|
43
|
+
"NDArray[numpy.float32]",
|
|
44
|
+
"NDArray[numpy.float64]",
|
|
45
|
+
"NDArray[numpy.bool_]",
|
|
47
46
|
]
|
|
48
47
|
DataOrPathType = Union[str, Path, ImageLikeNDArray, "PIL.Image.Image"]
|
|
49
48
|
ClassGroupsType = Dict[str, Union[str, List[str]]]
|
|
@@ -10,15 +10,16 @@ from truefoundry.ml.logger import logger
|
|
|
10
10
|
from truefoundry.ml.run_utils import get_module
|
|
11
11
|
|
|
12
12
|
if TYPE_CHECKING:
|
|
13
|
+
import numpy # noqa: F401
|
|
13
14
|
import PIL.Image
|
|
14
|
-
from numpy.typing import NDArray
|
|
15
|
+
from numpy.typing import NDArray # noqa: F401
|
|
15
16
|
|
|
16
17
|
ImageLikeNDArray = Union[
|
|
17
|
-
"NDArray[
|
|
18
|
-
"NDArray[
|
|
19
|
-
"NDArray[
|
|
20
|
-
"NDArray[
|
|
21
|
-
"NDArray[
|
|
18
|
+
"NDArray[numpy.int8]",
|
|
19
|
+
"NDArray[numpy.uint8]",
|
|
20
|
+
"NDArray[numpy.float32]",
|
|
21
|
+
"NDArray[numpy.float64]",
|
|
22
|
+
"NDArray[numpy.bool_]",
|
|
22
23
|
]
|
|
23
24
|
|
|
24
25
|
|