truefoundry 0.5.5rc2__py3-none-any.whl → 0.5.6rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of truefoundry might be problematic. Click here for more details.
- truefoundry/deploy/auto_gen/models.py +25 -25
- truefoundry/deploy/cli/commands/__init__.py +0 -1
- truefoundry/deploy/lib/clients/servicefoundry_client.py +20 -16
- truefoundry/deploy/v2/lib/deploy.py +23 -15
- truefoundry/deploy/v2/lib/patched_models.py +3 -3
- truefoundry/ml/autogen/client/__init__.py +4 -4
- truefoundry/ml/autogen/client/api/experiments_api.py +0 -156
- truefoundry/ml/autogen/client/api/mlfoundry_artifacts_api.py +49 -305
- truefoundry/ml/autogen/client/models/__init__.py +4 -4
- truefoundry/ml/autogen/client/models/agent.py +64 -13
- truefoundry/ml/autogen/client/models/agent_open_api_tool.py +63 -17
- truefoundry/ml/autogen/client/models/agent_open_api_tool_with_fqn.py +63 -17
- truefoundry/ml/autogen/client/models/agent_with_fqn.py +64 -12
- truefoundry/ml/autogen/client/models/artifact_version_manifest.py +39 -4
- truefoundry/ml/autogen/client/models/chat_prompt.py +60 -22
- truefoundry/ml/autogen/client/models/{external_artifact_source.py → external_blob_storage_source.py} +10 -11
- truefoundry/ml/autogen/client/models/model_version_manifest.py +39 -4
- truefoundry/ml/autogen/client/models/source.py +22 -22
- truefoundry/ml/autogen/client/models/source1.py +22 -22
- truefoundry/ml/autogen/client/models/{true_foundry_artifact_source.py → true_foundry_managed_source.py} +12 -11
- truefoundry/ml/autogen/client_README.md +2 -3
- truefoundry/ml/autogen/entities/artifacts.py +87 -69
- truefoundry/ml/log_types/artifacts/artifact.py +26 -18
- truefoundry/ml/log_types/artifacts/general_artifact.py +4 -13
- truefoundry/ml/log_types/artifacts/model.py +28 -21
- truefoundry/ml/mlfoundry_api.py +2 -6
- truefoundry/workflow/task.py +1 -1
- {truefoundry-0.5.5rc2.dist-info → truefoundry-0.5.6rc1.dist-info}/METADATA +2 -2
- {truefoundry-0.5.5rc2.dist-info → truefoundry-0.5.6rc1.dist-info}/RECORD +31 -32
- truefoundry/deploy/cli/commands/build_logs_command.py +0 -89
- {truefoundry-0.5.5rc2.dist-info → truefoundry-0.5.6rc1.dist-info}/WHEEL +0 -0
- {truefoundry-0.5.5rc2.dist-info → truefoundry-0.5.6rc1.dist-info}/entry_points.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# generated by datamodel-codegen:
|
|
2
2
|
# filename: artifacts.json
|
|
3
|
-
# timestamp: 2025-01-
|
|
3
|
+
# timestamp: 2025-01-28T10:54:09+00:00
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
@@ -10,27 +10,6 @@ from typing import Any, Dict, List, Literal, Optional, Union
|
|
|
10
10
|
from truefoundry.pydantic_v1 import BaseModel, Field, conint, constr
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
class Agent(BaseModel):
|
|
14
|
-
type: Literal["agent"] = Field(..., description="+value=agent")
|
|
15
|
-
name: constr(regex=r"^[a-zA-Z][a-zA-Z0-9\-]{1,30}[a-zA-Z0-9]$") = Field(
|
|
16
|
-
..., description="Names cannot repeat accross agent\n+uiType=Hidden"
|
|
17
|
-
)
|
|
18
|
-
available_tools: List[constr(min_length=1)] = Field(
|
|
19
|
-
..., description="+sort=30\n+uiType=Hidden"
|
|
20
|
-
)
|
|
21
|
-
goal: constr(min_length=1, max_length=128) = Field(
|
|
22
|
-
..., description="+sort=10\n+uiType=TextArea"
|
|
23
|
-
)
|
|
24
|
-
instruction: constr(min_length=1, max_length=2620) = Field(
|
|
25
|
-
...,
|
|
26
|
-
description='`instruction` is the system prompt for now. (2.5 * 1024)\n+sort=20\n+uiType=AgentInstructions\n+uiProps={"helpText":"Use the syntax ${Tool FQN} to reference a tool, and ${AGENT FQN} to reference another agent"}',
|
|
27
|
-
)
|
|
28
|
-
model_id: constr(min_length=1) = Field(
|
|
29
|
-
...,
|
|
30
|
-
description='+sort=40\n+uiType=EnabledModelSelector\n+uiProps={"searchable":true,"modelType":"chat","providerType":"openai"}',
|
|
31
|
-
)
|
|
32
|
-
|
|
33
|
-
|
|
34
13
|
class Method(str, Enum):
|
|
35
14
|
"""
|
|
36
15
|
+sort=50
|
|
@@ -44,25 +23,29 @@ class Method(str, Enum):
|
|
|
44
23
|
patch = "patch"
|
|
45
24
|
|
|
46
25
|
|
|
47
|
-
class AgentWithFQN(Agent):
|
|
48
|
-
id: str
|
|
49
|
-
fqn: str
|
|
50
|
-
|
|
51
|
-
|
|
52
26
|
class BaseArtifactVersion(BaseModel):
|
|
53
27
|
description: Optional[constr(max_length=512)] = Field(
|
|
54
|
-
None,
|
|
55
|
-
description="+label=Description\n+usage=Description of the artifact or model version\n+docs=Description of the artifact or model version",
|
|
28
|
+
None, description="+sort=2\n+label=Description"
|
|
56
29
|
)
|
|
57
30
|
metadata: Dict[str, Any] = Field(
|
|
58
31
|
...,
|
|
59
|
-
description="+label=Metadata\n+docs=
|
|
32
|
+
description="+label=Metadata\n+docs=Key value pairs to store additional metadata\n+usage=Key value pairs to store additional metadata\n+uiType=JsonInput",
|
|
60
33
|
)
|
|
61
34
|
version_alias: Optional[
|
|
62
35
|
constr(regex=r"^v[a-zA-Z0-9.-]*([a-zA-Z0-9]+)$", max_length=128)
|
|
63
36
|
] = Field(
|
|
64
37
|
None,
|
|
65
|
-
description="+label=Version Alias\n+usage=The version alias
|
|
38
|
+
description="+label=Version Alias\n+usage=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v1-prod, v3-dev, etc)\n+docs=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v1-prod, v3-dev, etc)\n+message=The version alias should start with 'v' followed by alphanumeric and it can include '.' and '-' in between (e.g. v1.0.0, v1-prod, v3-dev, etc)",
|
|
39
|
+
)
|
|
40
|
+
name: Optional[constr(regex=r"^[A-Za-z0-9_\-]+$", max_length=256)] = Field(
|
|
41
|
+
None,
|
|
42
|
+
description="+sort=1\n+label=Name\n+message=The name should start with lowercase alphabets and can contain alphanumeric and can include '-' in between",
|
|
43
|
+
)
|
|
44
|
+
ml_repo: Optional[constr(regex=r"^[a-zA-Z][a-zA-Z0-9\-]{1,98}[a-zA-Z0-9]$")] = (
|
|
45
|
+
Field(
|
|
46
|
+
None,
|
|
47
|
+
description="+label=ML Repo\n+usage=name of the ML Repo\n+docs=name of the ML Repo\n+uiType=Hidden",
|
|
48
|
+
)
|
|
66
49
|
)
|
|
67
50
|
|
|
68
51
|
|
|
@@ -95,16 +78,17 @@ class BlobStorageReference(BaseModel):
|
|
|
95
78
|
)
|
|
96
79
|
|
|
97
80
|
|
|
98
|
-
class
|
|
81
|
+
class ExternalBlobStorageSource(BaseModel):
|
|
99
82
|
"""
|
|
100
|
-
+label=External
|
|
83
|
+
+label=External Blob Storage
|
|
101
84
|
"""
|
|
102
85
|
|
|
103
86
|
type: Literal["external"] = Field(
|
|
104
|
-
...,
|
|
105
|
-
|
|
87
|
+
..., description="+label=Type\n+usage=Type of the source\n+value=external"
|
|
88
|
+
)
|
|
89
|
+
uri: str = Field(
|
|
90
|
+
..., description="+label=URI\n+usage=URI of the Blob Storage source"
|
|
106
91
|
)
|
|
107
|
-
uri: str = Field(..., description="+label=URI\n+usage=URI of the Artifact source")
|
|
108
92
|
|
|
109
93
|
|
|
110
94
|
class FastAIFramework(BaseModel):
|
|
@@ -415,18 +399,17 @@ class TransformersFramework(BaseModel):
|
|
|
415
399
|
)
|
|
416
400
|
|
|
417
401
|
|
|
418
|
-
class
|
|
402
|
+
class TrueFoundryManagedSource(BaseModel):
|
|
419
403
|
"""
|
|
420
|
-
+usage=Source for the
|
|
404
|
+
+usage=Source for the entity
|
|
421
405
|
+label=Upload
|
|
422
406
|
"""
|
|
423
407
|
|
|
424
408
|
type: Literal["truefoundry"] = Field(
|
|
425
|
-
...,
|
|
426
|
-
description="+usage=Type of the source\n+type=Upload\n+value=truefoundry\n+label=Upload",
|
|
409
|
+
..., description="+label=Type\n+usage=Type of the source\n+value=truefoundry"
|
|
427
410
|
)
|
|
428
|
-
uri: str = Field(
|
|
429
|
-
|
|
411
|
+
uri: Optional[str] = Field(
|
|
412
|
+
None,
|
|
430
413
|
description='+label=URI\n+uiType=ModelUploadInput\n+uiProps={"hideClear":true}',
|
|
431
414
|
)
|
|
432
415
|
|
|
@@ -480,14 +463,26 @@ class XGBoostSerializationFormat(str, Enum):
|
|
|
480
463
|
json = "json"
|
|
481
464
|
|
|
482
465
|
|
|
483
|
-
class
|
|
484
|
-
type: Literal["
|
|
485
|
-
|
|
486
|
-
..., description="
|
|
466
|
+
class Agent(BaseArtifactVersion):
|
|
467
|
+
type: Literal["agent"] = Field(..., description="+value=agent")
|
|
468
|
+
available_tools: List[constr(min_length=1)] = Field(
|
|
469
|
+
..., description="+sort=30\n+uiType=Hidden"
|
|
487
470
|
)
|
|
488
|
-
|
|
471
|
+
goal: constr(min_length=1, max_length=128) = Field(
|
|
489
472
|
..., description="+sort=10\n+uiType=TextArea"
|
|
490
473
|
)
|
|
474
|
+
instruction: constr(min_length=1, max_length=2620) = Field(
|
|
475
|
+
...,
|
|
476
|
+
description='`instruction` is the system prompt for now. (2.5 * 1024)\n+sort=20\n+uiType=AgentInstructions\n+uiProps={"helpText":"Use the syntax ${Tool FQN} to reference a tool, and ${AGENT FQN} to reference another agent"}',
|
|
477
|
+
)
|
|
478
|
+
model_id: constr(min_length=1) = Field(
|
|
479
|
+
...,
|
|
480
|
+
description='+sort=40\n+uiType=EnabledModelSelector\n+uiProps={"searchable":true,"modelType":"chat","providerType":"openai"}',
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
class AgentOpenAPITool(BaseArtifactVersion):
|
|
485
|
+
type: Literal["openapi-tool"] = Field(..., description="+value=openapi-tool")
|
|
491
486
|
openapi_spec: Union[BlobStorageReference, Dict[str, Any]] = Field(
|
|
492
487
|
..., description="+sort=20\n+uiType=OpenapiSchema"
|
|
493
488
|
)
|
|
@@ -510,12 +505,17 @@ class AgentOpenAPIToolWithFQN(AgentOpenAPITool):
|
|
|
510
505
|
fqn: str
|
|
511
506
|
|
|
512
507
|
|
|
508
|
+
class AgentWithFQN(Agent):
|
|
509
|
+
id: str
|
|
510
|
+
fqn: str
|
|
511
|
+
|
|
512
|
+
|
|
513
513
|
class ArtifactVersion(BaseArtifactVersion):
|
|
514
514
|
type: Literal["artifact-version"] = Field(
|
|
515
515
|
...,
|
|
516
516
|
description='+label=Type\n+usage=Artifact Version\n+value="artifact-version"',
|
|
517
517
|
)
|
|
518
|
-
source: Union[
|
|
518
|
+
source: Union[TrueFoundryManagedSource, ExternalBlobStorageSource] = Field(
|
|
519
519
|
...,
|
|
520
520
|
description="+label=Artifact Source\n+usage=Source for the Artifact version\n+uiType=Group",
|
|
521
521
|
)
|
|
@@ -538,38 +538,50 @@ class AssistantMessage(BaseModel):
|
|
|
538
538
|
)
|
|
539
539
|
|
|
540
540
|
|
|
541
|
-
class
|
|
541
|
+
class ChatPrompt(BaseArtifactVersion):
|
|
542
|
+
type: Literal["chat_prompt"] = Field(
|
|
543
|
+
...,
|
|
544
|
+
description='+label=Type\n+icon=fa-desktop:#326ce5\n+message=Type of the prompt\n+sort=3000\n+usage=Type of the prompt\n+docs=Type of the prompt, set to "chat_prompt"\n+default="chat_prompt"',
|
|
545
|
+
)
|
|
546
|
+
messages: List[Union[SystemMessage, AssistantMessage, UserMessage]] = Field(
|
|
547
|
+
...,
|
|
548
|
+
description="+sort=4000\n+usage=Chat completion messages\n+label=Messages in the chat conversation\n+message=Chat completion messages\n+usage=List of messages in the chat conversation, must be non-empty\n+docs=Messages that define the chat conversation, including system, assistant, and user messages.",
|
|
549
|
+
)
|
|
550
|
+
variables: Optional[Dict[str, Optional[str]]] = Field(
|
|
551
|
+
None,
|
|
552
|
+
description="+label=Variables\n+usage=Variables for the chat completion messages to be used in the prompt messages\n+sort=5000",
|
|
553
|
+
)
|
|
554
|
+
model_configuration: ModelConfiguration
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
class DataDirectory(BaseModel):
|
|
542
558
|
"""
|
|
543
|
-
|
|
559
|
+
+label=Data Directory
|
|
544
560
|
"""
|
|
545
561
|
|
|
546
|
-
|
|
562
|
+
type: Literal["data-dir"] = Field(
|
|
547
563
|
...,
|
|
548
|
-
description=
|
|
564
|
+
description='+label=Type\n+usage=Data Directory\n+docs=Data Directory\n+value="data-dir"',
|
|
549
565
|
)
|
|
550
|
-
description: Optional[constr(
|
|
566
|
+
description: Optional[constr(max_length=512)] = Field(
|
|
551
567
|
None,
|
|
552
|
-
description="+label=Description\n+
|
|
568
|
+
description="+label=Description\n+usage=Description of the data directory\n+docs=Description of the data directory",
|
|
553
569
|
)
|
|
554
|
-
|
|
555
|
-
metadata: Dict[str, str] = Field(
|
|
570
|
+
metadata: Dict[str, Any] = Field(
|
|
556
571
|
...,
|
|
557
|
-
description="+label=Metadata\n+
|
|
572
|
+
description="+label=Metadata\n+docs=Metadata for the data directory\n+usage=Metadata for the data directory\n+uiType=JsonInput",
|
|
558
573
|
)
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
class ChatPrompt(BasePrompt):
|
|
562
|
-
type: Literal["chat_prompt"] = Field(
|
|
574
|
+
name: constr(regex=r"^[A-Za-z0-9_\-]+$", max_length=256) = Field(
|
|
563
575
|
...,
|
|
564
|
-
description=
|
|
576
|
+
description="+sort=1\n+label=Name\n+usage=Name of the data directory\n+docs=Name of the data directory\n+message=The data directory name should start with lowercase alphabets and can contain alphanumeric and can include '-' in between",
|
|
565
577
|
)
|
|
566
|
-
|
|
578
|
+
ml_repo: constr(regex=r"^[a-zA-Z][a-zA-Z0-9\-]{1,98}[a-zA-Z0-9]$") = Field(
|
|
567
579
|
...,
|
|
568
|
-
description="+
|
|
580
|
+
description="+label=ML Repo\n+usage=name of the ML Repo of the data directory\n+docs=name of the ML Repo of the data directory\n+uiType=Hidden",
|
|
569
581
|
)
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
description="+label=
|
|
582
|
+
source: Union[TrueFoundryManagedSource, ExternalBlobStorageSource] = Field(
|
|
583
|
+
...,
|
|
584
|
+
description="+label=Data Directory Source\n+usage=Source for the Data Directory\n+docs=Source for the Data Directory\n+uiType=Group",
|
|
573
585
|
)
|
|
574
586
|
|
|
575
587
|
|
|
@@ -618,7 +630,7 @@ class ModelVersion(BaseArtifactVersion):
|
|
|
618
630
|
type: Literal["model-version"] = Field(
|
|
619
631
|
..., description='+label=Type\n+usage=Model Version\n+value="model-version"'
|
|
620
632
|
)
|
|
621
|
-
source: Union[
|
|
633
|
+
source: Union[TrueFoundryManagedSource, ExternalBlobStorageSource] = Field(
|
|
622
634
|
...,
|
|
623
635
|
description="+label=Model Source\n+usage=Source for the model version\n+uiType=Group",
|
|
624
636
|
)
|
|
@@ -648,5 +660,11 @@ class ModelVersion(BaseArtifactVersion):
|
|
|
648
660
|
|
|
649
661
|
class VersionedArtifactType(BaseModel):
|
|
650
662
|
__root__: Union[
|
|
651
|
-
ChatPrompt,
|
|
663
|
+
ChatPrompt,
|
|
664
|
+
AgentOpenAPITool,
|
|
665
|
+
Agent,
|
|
666
|
+
AgentApp,
|
|
667
|
+
ModelVersion,
|
|
668
|
+
ArtifactVersion,
|
|
669
|
+
DataDirectory,
|
|
652
670
|
]
|
|
@@ -21,12 +21,13 @@ from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
|
21
21
|
ArtifactVersionManifest,
|
|
22
22
|
CreateArtifactVersionRequestDto,
|
|
23
23
|
DeleteArtifactVersionsRequestDto,
|
|
24
|
-
|
|
24
|
+
ExperimentsApi,
|
|
25
|
+
ExternalBlobStorageSource,
|
|
25
26
|
FinalizeArtifactVersionRequestDto,
|
|
26
27
|
Manifest,
|
|
27
28
|
MlfoundryArtifactsApi,
|
|
28
29
|
NotifyArtifactVersionFailureDto,
|
|
29
|
-
|
|
30
|
+
TrueFoundryManagedSource,
|
|
30
31
|
UpdateArtifactVersionRequestDto,
|
|
31
32
|
)
|
|
32
33
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
@@ -488,29 +489,36 @@ def _log_artifact_version_helper(
|
|
|
488
489
|
artifact_type: ArtifactType,
|
|
489
490
|
artifact_dir: Union[tempfile.TemporaryDirectory, BlobStorageDirectory],
|
|
490
491
|
dest_to_src_map: Dict[str, str],
|
|
491
|
-
|
|
492
|
-
ml_repo_id: Optional[str] = None,
|
|
492
|
+
ml_repo: Optional[str] = None,
|
|
493
493
|
description: Optional[str] = None,
|
|
494
494
|
internal_metadata: Optional[BaseModel] = None,
|
|
495
495
|
metadata: Optional[Dict[str, Any]] = None,
|
|
496
496
|
step: int = 0,
|
|
497
497
|
progress: Optional[bool] = None,
|
|
498
498
|
) -> ArtifactVersion:
|
|
499
|
-
if
|
|
500
|
-
raise MlFoundryException(
|
|
501
|
-
|
|
502
|
-
)
|
|
503
|
-
if mlfoundry_artifacts_api and not ml_repo_id:
|
|
504
|
-
raise MlFoundryException(
|
|
505
|
-
"If mlfoundry_artifacts_api is passed, ml_repo_id must also be passed"
|
|
506
|
-
)
|
|
499
|
+
if not run and not ml_repo:
|
|
500
|
+
raise MlFoundryException("Exactly one of run, ml_repo should be passed")
|
|
501
|
+
|
|
507
502
|
if run:
|
|
508
503
|
mlfoundry_artifacts_api = run._mlfoundry_artifacts_api
|
|
504
|
+
experiments_api = run._experiments_api
|
|
505
|
+
ml_repo = experiments_api.get_experiment_get(
|
|
506
|
+
experiment_id=str(run._experiment_id)
|
|
507
|
+
).experiment.name
|
|
508
|
+
ml_repo_id = run._experiment_id
|
|
509
|
+
else:
|
|
510
|
+
assert ml_repo is not None
|
|
511
|
+
api_client = _get_api_client()
|
|
512
|
+
mlfoundry_artifacts_api = MlfoundryArtifactsApi(api_client=api_client)
|
|
513
|
+
experiments_api = ExperimentsApi(api_client=api_client)
|
|
514
|
+
ml_repo_id = experiments_api.get_experiment_by_name_get(
|
|
515
|
+
experiment_name=ml_repo
|
|
516
|
+
).experiment.experiment_id
|
|
509
517
|
|
|
510
518
|
assert mlfoundry_artifacts_api is not None
|
|
511
519
|
_create_artifact_response = mlfoundry_artifacts_api.create_artifact_version_post(
|
|
512
520
|
create_artifact_version_request_dto=CreateArtifactVersionRequestDto(
|
|
513
|
-
experiment_id=int(
|
|
521
|
+
experiment_id=int(ml_repo_id),
|
|
514
522
|
name=name,
|
|
515
523
|
artifact_type=artifact_type,
|
|
516
524
|
)
|
|
@@ -518,10 +526,8 @@ def _log_artifact_version_helper(
|
|
|
518
526
|
version_id = _create_artifact_response.id
|
|
519
527
|
artifact_storage_root = _create_artifact_response.artifact_storage_root
|
|
520
528
|
if isinstance(artifact_dir, tempfile.TemporaryDirectory):
|
|
521
|
-
# Source is of type
|
|
522
|
-
source =
|
|
523
|
-
type="truefoundry", uri=artifact_storage_root
|
|
524
|
-
)
|
|
529
|
+
# Source is of type TrueFoundryManagedSource
|
|
530
|
+
source = TrueFoundryManagedSource(type="truefoundry", uri=artifact_storage_root)
|
|
525
531
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
526
532
|
artifact_identifier=ArtifactIdentifier(
|
|
527
533
|
artifact_version_id=uuid.UUID(version_id),
|
|
@@ -558,7 +564,7 @@ def _log_artifact_version_helper(
|
|
|
558
564
|
internal_metadata.dict() if internal_metadata is not None else {}
|
|
559
565
|
)
|
|
560
566
|
elif isinstance(artifact_dir, BlobStorageDirectory):
|
|
561
|
-
source =
|
|
567
|
+
source = ExternalBlobStorageSource(type="external", uri=artifact_dir.uri)
|
|
562
568
|
else:
|
|
563
569
|
raise MlFoundryException("Invalid artifact_dir provided")
|
|
564
570
|
|
|
@@ -566,6 +572,8 @@ def _log_artifact_version_helper(
|
|
|
566
572
|
if artifact_type == ArtifactType.ARTIFACT:
|
|
567
573
|
_source_cls = typing.get_type_hints(ArtifactVersionManifest)["source"]
|
|
568
574
|
artifact_manifest = ArtifactVersionManifest(
|
|
575
|
+
name=name,
|
|
576
|
+
ml_repo=ml_repo,
|
|
569
577
|
description=description,
|
|
570
578
|
metadata=metadata or {},
|
|
571
579
|
source=_source_cls.from_dict(source.dict()),
|
|
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
|
|
4
4
|
|
|
5
5
|
from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
6
6
|
ArtifactType,
|
|
7
|
-
MlfoundryArtifactsApi,
|
|
8
7
|
)
|
|
9
8
|
from truefoundry.ml.exceptions import MlFoundryException
|
|
10
9
|
from truefoundry.ml.log_types.artifacts.artifact import (
|
|
@@ -27,21 +26,14 @@ def _log_artifact_version(
|
|
|
27
26
|
run: Optional["MlFoundryRun"],
|
|
28
27
|
name: str,
|
|
29
28
|
artifact_paths: List[Union[ArtifactPath, Tuple[str, Optional[str]], Tuple[str]]],
|
|
30
|
-
|
|
31
|
-
ml_repo_id: Optional[str] = None,
|
|
29
|
+
ml_repo: Optional[str] = None,
|
|
32
30
|
description: Optional[str] = None,
|
|
33
31
|
metadata: Optional[Dict[str, Any]] = None,
|
|
34
32
|
step: Optional[int] = 0,
|
|
35
33
|
progress: Optional[bool] = None,
|
|
36
34
|
) -> ArtifactVersion:
|
|
37
|
-
if
|
|
38
|
-
raise MlFoundryException(
|
|
39
|
-
"Exactly one of run, mlfoundry_artifacts_api should be passed"
|
|
40
|
-
)
|
|
41
|
-
if mlfoundry_artifacts_api and not ml_repo_id:
|
|
42
|
-
raise MlFoundryException(
|
|
43
|
-
"If mlfoundry_artifacts_api is passed, ml_repo_id must also be passed"
|
|
44
|
-
)
|
|
35
|
+
if not run and not ml_repo:
|
|
36
|
+
raise MlFoundryException("Exactly one of run, ml_repo should be passed")
|
|
45
37
|
for i, artifact_path in enumerate(artifact_paths):
|
|
46
38
|
if isinstance(artifact_path, ArtifactPath):
|
|
47
39
|
continue
|
|
@@ -79,12 +71,11 @@ def _log_artifact_version(
|
|
|
79
71
|
|
|
80
72
|
return _log_artifact_version_helper(
|
|
81
73
|
run=run,
|
|
82
|
-
|
|
74
|
+
ml_repo=ml_repo,
|
|
83
75
|
name=name,
|
|
84
76
|
artifact_type=ArtifactType.ARTIFACT,
|
|
85
77
|
artifact_dir=temp_dir,
|
|
86
78
|
dest_to_src_map=temp_dest_to_src_map,
|
|
87
|
-
mlfoundry_artifacts_api=mlfoundry_artifacts_api,
|
|
88
79
|
description=description,
|
|
89
80
|
internal_metadata=None,
|
|
90
81
|
metadata=metadata,
|
|
@@ -19,7 +19,8 @@ from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
|
19
19
|
ArtifactType,
|
|
20
20
|
CreateArtifactVersionRequestDto,
|
|
21
21
|
DeleteArtifactVersionsRequestDto,
|
|
22
|
-
|
|
22
|
+
ExperimentsApi,
|
|
23
|
+
ExternalBlobStorageSource,
|
|
23
24
|
FinalizeArtifactVersionRequestDto,
|
|
24
25
|
Framework,
|
|
25
26
|
Manifest,
|
|
@@ -29,7 +30,7 @@ from truefoundry.ml.autogen.client import ( # type: ignore[attr-defined]
|
|
|
29
30
|
ModelVersionEnvironment,
|
|
30
31
|
ModelVersionManifest,
|
|
31
32
|
NotifyArtifactVersionFailureDto,
|
|
32
|
-
|
|
33
|
+
TrueFoundryManagedSource,
|
|
33
34
|
UpdateModelVersionRequestDto,
|
|
34
35
|
)
|
|
35
36
|
from truefoundry.ml.autogen.models import infer_signature as _infer_signature
|
|
@@ -512,8 +513,7 @@ def _log_model_version( # noqa: C901
|
|
|
512
513
|
run: Optional["MlFoundryRun"],
|
|
513
514
|
name: str,
|
|
514
515
|
model_file_or_folder: Union[str, BlobStorageDirectory],
|
|
515
|
-
|
|
516
|
-
ml_repo_id: Optional[str] = None,
|
|
516
|
+
ml_repo: Optional[str] = None,
|
|
517
517
|
description: Optional[str] = None,
|
|
518
518
|
metadata: Optional[Dict[str, Any]] = None,
|
|
519
519
|
step: Optional[int] = 0,
|
|
@@ -521,24 +521,29 @@ def _log_model_version( # noqa: C901
|
|
|
521
521
|
framework: Optional[Union[str, ModelFramework, "ModelFrameworkType"]] = None,
|
|
522
522
|
environment: Optional[ModelVersionEnvironment] = None,
|
|
523
523
|
) -> ModelVersion:
|
|
524
|
-
if (run and mlfoundry_artifacts_api) or (not run and not mlfoundry_artifacts_api):
|
|
525
|
-
raise MlFoundryException(
|
|
526
|
-
"Exactly one of run, mlfoundry_artifacts_api should be passed"
|
|
527
|
-
)
|
|
528
|
-
|
|
529
524
|
if not isinstance(model_file_or_folder, (str, BlobStorageDirectory)):
|
|
530
525
|
raise MlFoundryException(
|
|
531
526
|
"model_file_or_folder should be of type str or BlobStorageDirectory"
|
|
532
527
|
)
|
|
533
528
|
|
|
534
|
-
if
|
|
535
|
-
raise MlFoundryException(
|
|
536
|
-
|
|
537
|
-
)
|
|
529
|
+
if not run and not ml_repo:
|
|
530
|
+
raise MlFoundryException("Exactly one of run, ml_repo should be passed")
|
|
531
|
+
|
|
538
532
|
if run:
|
|
539
533
|
mlfoundry_artifacts_api = run._mlfoundry_artifacts_api
|
|
540
|
-
|
|
541
|
-
|
|
534
|
+
experiments_api = run._experiments_api
|
|
535
|
+
ml_repo = experiments_api.get_experiment_get(
|
|
536
|
+
experiment_id=str(run._experiment_id)
|
|
537
|
+
).experiment.name
|
|
538
|
+
ml_repo_id = run._experiment_id
|
|
539
|
+
else:
|
|
540
|
+
assert ml_repo is not None
|
|
541
|
+
api_client = _get_api_client()
|
|
542
|
+
mlfoundry_artifacts_api = MlfoundryArtifactsApi(api_client=api_client)
|
|
543
|
+
experiments_api = ExperimentsApi(api_client=api_client)
|
|
544
|
+
ml_repo_id = experiments_api.get_experiment_by_name_get(
|
|
545
|
+
experiment_name=ml_repo
|
|
546
|
+
).experiment.experiment_id
|
|
542
547
|
|
|
543
548
|
step = step or 0
|
|
544
549
|
total_size = None
|
|
@@ -568,7 +573,7 @@ def _log_model_version( # noqa: C901
|
|
|
568
573
|
_create_artifact_version_response = (
|
|
569
574
|
mlfoundry_artifacts_api.create_artifact_version_post(
|
|
570
575
|
create_artifact_version_request_dto=CreateArtifactVersionRequestDto(
|
|
571
|
-
experiment_id=int(
|
|
576
|
+
experiment_id=int(ml_repo_id),
|
|
572
577
|
artifact_type=ArtifactType.MODEL,
|
|
573
578
|
name=name,
|
|
574
579
|
)
|
|
@@ -577,10 +582,8 @@ def _log_model_version( # noqa: C901
|
|
|
577
582
|
version_id = _create_artifact_version_response.id
|
|
578
583
|
artifact_storage_root = _create_artifact_version_response.artifact_storage_root
|
|
579
584
|
if isinstance(model_file_or_folder, str):
|
|
580
|
-
# Source is of type
|
|
581
|
-
source =
|
|
582
|
-
type="truefoundry", uri=artifact_storage_root
|
|
583
|
-
)
|
|
585
|
+
# Source is of type TrueFoundryManagedSource
|
|
586
|
+
source = TrueFoundryManagedSource(type="truefoundry", uri=artifact_storage_root)
|
|
584
587
|
artifacts_repo = MlFoundryArtifactsRepository(
|
|
585
588
|
artifact_identifier=ArtifactIdentifier(
|
|
586
589
|
artifact_version_id=uuid.UUID(version_id)
|
|
@@ -609,7 +612,9 @@ def _log_model_version( # noqa: C901
|
|
|
609
612
|
finally:
|
|
610
613
|
temp_dir.cleanup()
|
|
611
614
|
elif isinstance(model_file_or_folder, BlobStorageDirectory):
|
|
612
|
-
source =
|
|
615
|
+
source = ExternalBlobStorageSource(
|
|
616
|
+
type="external", uri=model_file_or_folder.uri
|
|
617
|
+
)
|
|
613
618
|
else:
|
|
614
619
|
raise MlFoundryException("Invalid model_file_or_folder provided")
|
|
615
620
|
|
|
@@ -625,6 +630,8 @@ def _log_model_version( # noqa: C901
|
|
|
625
630
|
auto_update_environment_details(environment=environment, framework=framework)
|
|
626
631
|
|
|
627
632
|
model_manifest = ModelVersionManifest(
|
|
633
|
+
name=name,
|
|
634
|
+
ml_repo=ml_repo,
|
|
628
635
|
description=description,
|
|
629
636
|
metadata=metadata,
|
|
630
637
|
source=_source_cls.from_dict(source.dict()),
|
truefoundry/ml/mlfoundry_api.py
CHANGED
|
@@ -1209,11 +1209,9 @@ class MlFoundry:
|
|
|
1209
1209
|
raise MlFoundryException(
|
|
1210
1210
|
"artifact_paths cannot be empty, atleast one artifact_path must be passed"
|
|
1211
1211
|
)
|
|
1212
|
-
ml_repo_id = self._get_ml_repo_id(ml_repo=ml_repo)
|
|
1213
1212
|
artifact_version = _log_artifact_version(
|
|
1214
1213
|
run=None,
|
|
1215
|
-
|
|
1216
|
-
ml_repo_id=ml_repo_id,
|
|
1214
|
+
ml_repo=ml_repo,
|
|
1217
1215
|
name=name,
|
|
1218
1216
|
artifact_paths=artifact_paths,
|
|
1219
1217
|
description=description,
|
|
@@ -1345,11 +1343,9 @@ class MlFoundry:
|
|
|
1345
1343
|
```
|
|
1346
1344
|
|
|
1347
1345
|
"""
|
|
1348
|
-
ml_repo_id = self._get_ml_repo_id(ml_repo=ml_repo)
|
|
1349
1346
|
model_version = _log_model_version(
|
|
1350
1347
|
run=None,
|
|
1351
|
-
|
|
1352
|
-
ml_repo_id=ml_repo_id,
|
|
1348
|
+
ml_repo=ml_repo,
|
|
1353
1349
|
name=name,
|
|
1354
1350
|
model_file_or_folder=model_file_or_folder,
|
|
1355
1351
|
description=description,
|
truefoundry/workflow/task.py
CHANGED
|
@@ -17,7 +17,7 @@ def task(
|
|
|
17
17
|
Callable[..., FuncOut],
|
|
18
18
|
]:
|
|
19
19
|
"""
|
|
20
|
-
This is the decorator used to run flyte tasks in
|
|
20
|
+
This is the decorator used to run flyte tasks in TrueFoundry.
|
|
21
21
|
|
|
22
22
|
Tasks are the building blocks of workflow. They represent users code. Tasks have the following properties
|
|
23
23
|
|