truefoundry 0.4.0.dev1__tar.gz → 0.4.0rc2__tar.gz

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: truefoundry
-Version: 0.4.0.dev1
+Version: 0.4.0rc2
 Summary: Truefoundry CLI
 Author: Abhishek Choudhary
 Author-email: abhishek@truefoundry.com

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truefoundry"
-version = "0.4.0.dev1"
+version = "0.4.0rc2"
 description = "Truefoundry CLI"
 authors = ["Abhishek Choudhary <abhishek@truefoundry.com>"]
 readme = "README.md"

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/deploy/v2/lib/deployable_patched_models.py

@@ -3,6 +3,7 @@ from typing import Literal, Union
 from truefoundry.deploy.auto_gen import models
 from truefoundry.deploy.lib.model.entity import Deployment
 from truefoundry.deploy.v2.lib.deploy import deploy_component
+from truefoundry.deploy.v2.lib.patched_models import LocalSource
 from truefoundry.pydantic_v1 import BaseModel, Field, conint
 
 
@@ -79,6 +80,9 @@ class Application(models.Application, DeployablePatchedModelBase):
 
 class Workflow(models.Workflow, DeployablePatchedModelBase):
     type: Literal["workflow"] = "workflow"
+    source: Union[models.RemoteSource, models.LocalSource] = Field(
+        default_factory=lambda: LocalSource(local_build=False)
+    )
 
     def deploy(self, workspace_fqn: str, wait: bool = True) -> Deployment:
         from truefoundry.deploy.v2.lib.deploy_workflow import deploy_workflow

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/ml/__init__.py

@@ -1,6 +1,3 @@
-from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
-    FeatureDto as Feature,
-)
 from truefoundry.ml.enums import (
     DataSlice,
     FileFormat,
@@ -15,7 +12,6 @@ from truefoundry.ml.log_types.artifacts.dataset import DataDirectory, DataDirect
 from truefoundry.ml.log_types.artifacts.model import (
     ModelVersion,
 )
-from truefoundry.ml.log_types.artifacts.model_extras import CustomMetric, ModelSchema
 from truefoundry.ml.logger import init_logger
 from truefoundry.ml.login import login
 from truefoundry.ml.mlfoundry_api import get_client
@@ -24,17 +20,14 @@ from truefoundry.ml.mlfoundry_run import MlFoundryRun
 __all__ = [
     "ArtifactPath",
     "ArtifactVersion",
-    "CustomMetric",
     "DataDirectory",
     "DataDirectoryPath",
     "DataSlice",
-    "Feature",
     "FileFormat",
     "Image",
     "MlFoundryRun",
     "MlFoundryException",
     "ModelFramework",
-    "ModelSchema",
     "ModelType",
     "ModelVersion",
     "Plot",

truefoundry-0.4.0rc2/truefoundry/ml/entities.py

@@ -0,0 +1,62 @@
+from typing import Optional, Union
+
+from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
+    FileInfoDto,
+    MetricDto,
+)
+from truefoundry.pydantic_v1 import (
+    BaseModel,
+    Field,
+    StrictBool,
+    StrictFloat,
+    StrictInt,
+    StrictStr,
+)
+
+
+class Metric(BaseModel):
+    key: StrictStr = Field(...)
+    value: Optional[Union[StrictFloat, StrictInt]] = None
+    timestamp: Optional[StrictInt] = None
+    step: Optional[StrictInt] = 0
+
+    @classmethod
+    def from_dto(cls, dto: MetricDto) -> "Metric":
+        return cls(
+            key=dto.key,
+            value=dto.value,
+            timestamp=dto.timestamp,
+            step=dto.step,
+        )
+
+    def to_dto(self) -> MetricDto:
+        return MetricDto(
+            key=self.key,
+            value=self.value,
+            timestamp=self.timestamp,
+            step=self.step,
+        )
+
+
+class FileInfo(BaseModel):
+    path: StrictStr
+    is_dir: StrictBool
+    file_size: Optional[StrictInt] = None
+    signed_url: Optional[StrictStr] = None
+
+    @classmethod
+    def from_dto(cls, dto: FileInfoDto) -> "FileInfo":
+        return cls(
+            path=dto.path,
+            is_dir=dto.is_dir,
+            file_size=dto.file_size,
+            signed_url=dto.signed_url,
+        )
+
+    def to_dto(self) -> FileInfoDto:
+        return FileInfoDto(
+            path=self.path,
+            is_dir=self.is_dir,
+            file_size=self.file_size,
+            signed_url=self.signed_url,
+        )

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/ml/log_types/artifacts/dataset.py

@@ -12,10 +12,10 @@ from truefoundry.ml.artifact.truefoundry_artifact_repo import (
 from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
     DatasetDto,
     DeleteDatasetRequestDto,
-    FileInfoDto,
     MlfoundryArtifactsApi,
     UpdateDatasetRequestDto,
 )
+from truefoundry.ml.entities import FileInfo
 from truefoundry.ml.exceptions import MlFoundryException
 from truefoundry.ml.log_types.artifacts.utils import (
     _copy_additional_files,
@@ -239,7 +239,7 @@ class DataDirectory:
     def list_files(
         self,
         path: Optional[str] = None,
-    ) -> Iterator[FileInfoDto]:
+    ) -> Iterator[FileInfo]:
         """
         List all the files and folders in the data_directory.
 
@@ -262,7 +262,8 @@ class DataDirectory:
             ```
         """
         artifacts_repo = self._get_artifacts_repo()
-        return artifacts_repo.list_artifacts(path=path)
+        for file_info_dto in artifacts_repo.list_artifacts(path=path):
+            yield FileInfo.from_dto(file_info_dto)
 
     def download(
         self,

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/ml/log_types/artifacts/model.py

@@ -53,7 +53,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger("truefoundry.ml")
 
 
-# TODO: Add some progress indicators for upload and download
 # TODO: Support async download and upload
 
 
@@ -98,7 +97,6 @@ class ModelVersion:
         self._deleted = False
         self._description: str = ""
         self._metadata: Dict[str, Any] = {}
-        self._model_schema: Optional[ModelSchema] = None
         self._metrics: List[MetricDto] = []
         self._set_metrics_attr()
         self._set_mutable_attrs()
@@ -146,7 +144,6 @@ class ModelVersion:
     def _set_mutable_attrs(self):
         self._description = self._model_version.description or ""
         self._metadata = copy.deepcopy(self._model_version.artifact_metadata)
-        self._model_schema = copy.deepcopy(self._model_version.model_schema)
 
     def _refetch_model_version(self):
         self._model_version = (
@@ -158,7 +155,7 @@ class ModelVersion:
         self._set_mutable_attrs()
 
     def __repr__(self):
-        return f"{self.__class__.__name__}(fqn={self.fqn!r}, model_schema={self._model_schema!r})"
+        return f"{self.__class__.__name__}(fqn={self.fqn!r})"
 
     def _get_artifacts_repo(self):
         return MlFoundryArtifactsRepository(
@@ -215,17 +212,6 @@ class ModelVersion:
         _validate_artifact_metadata(value)
         self._metadata = value
 
-    @property
-    def model_schema(self) -> Optional[ModelSchema]:
-        """get the model schema for current model"""
-        return self._model_schema
-
-    @model_schema.setter
-    def model_schema(self, value: Union[Dict[str, Any], ModelSchema]):
-        if not isinstance(value, ModelSchema):
-            value = ModelSchema.parse_obj(value)
-        self._model_schema = value
-
     @property
     def metrics(self) -> Dict[str, Union[float, int]]:
         """get the metrics for the current version of the model"""
@@ -397,8 +383,6 @@ class ModelVersion:
         """
         self._ensure_not_deleted()
         kwargs = {}
-        if self.model_schema is not None:
-            kwargs["model_schema"] = self.model_schema
         _model_version = self._mlfoundry_artifacts_api.update_model_version_post(
             update_model_version_request_dto=UpdateModelVersionRequestDto(
                 id=self._model_version.id,

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/ml/mlfoundry_api.py

@@ -15,19 +15,17 @@ from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
     CreateExperimentRequestDto,
     CreateRunRequestDto,
     DatasetDto,
+    ExperimentsApi,
     ListArtifactsRequestDto,
     ListArtifactVersionsRequestDto,
     ListDatasetsRequestDto,
     ListModelVersionsRequestDto,
+    MlfoundryArtifactsApi,
     ModelDto,
+    RunsApi,
     RunTagDto,
     SearchRunsRequestDto,
 )
-from truefoundry.ml.autogen.client.api import (  # type: ignore[attr-defined]
-    ExperimentsApi,
-    MlfoundryArtifactsApi,
-    RunsApi,
-)
 from truefoundry.ml.autogen.client.exceptions import (
     ApiException,
     NotFoundException,
@@ -39,7 +37,6 @@ from truefoundry.ml.log_types.artifacts.artifact import ArtifactPath, ArtifactVe
 from truefoundry.ml.log_types.artifacts.dataset import DataDirectory
 from truefoundry.ml.log_types.artifacts.general_artifact import _log_artifact_version
 from truefoundry.ml.log_types.artifacts.model import ModelVersion, _log_model_version
-from truefoundry.ml.log_types.artifacts.model_extras import CustomMetric, ModelSchema
 from truefoundry.ml.logger import logger
 from truefoundry.ml.mlfoundry_run import MlFoundryRun
 from truefoundry.ml.services.servicefoundry_service import ServicefoundryService
@@ -1220,8 +1217,7 @@ class MlFoundry:
         additional_files: Sequence[Tuple[Union[str, Path], Optional[str]]] = (),
         description: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
-        model_schema: Optional[Union[ModelSchema, Dict[str, Any]]] = None,
-        custom_metrics: Optional[List[Union[CustomMetric, Dict[str, Any]]]] = None,
+        progress: Optional[bool] = None,
     ) -> ModelVersion:
         """
         Serialize and log a versioned model under the current ml_repo. Each logged model generates a new version
@@ -1277,34 +1273,6 @@ class MlFoundry:
             metadata (Optional[Dict[str, Any]], optional): arbitrary json serializable dictionary to store metadata.
                 For example, you can use this to store metrics, params, notes.
                 This field can be updated at any time after logging. Defaults to `None`
-            model_schema (Optional[Union[Dict[str, Any], ModelSchema]], optional):
-                instance of `truefoundry.ml.ModelSchema`.
-                This schema needs to be consistent with older versions of the model under the given `name` i.e.
-                a feature's value type and model's prediction type cannot be changed in the schema of new version.
-                Features can be removed or added between versions.
-                ```
-                E.g. if there exists a v1 with
-                    schema = {"features": {"name": "feat1": "int"}, "prediction": "categorical"}, then
-
-                    schema = {"features": {"name": "feat1": "string"}, "prediction": "categorical"} or
-                    schema = {"features": {"name": "feat1": "int"}, "prediction": "numerical"}
-                    are invalid because they change the types of existing features and prediction
-
-                    while
-                    schema = {"features": {"name": "feat1": "int", "feat2": "string"}, "prediction": "categorical"} or
-                    schema = {"features": {"feat2": "string"}, "prediction": "categorical"}
-                    are valid
-
-                    This field can be updated at any time after logging. Defaults to None
-                ```
-            custom_metrics: (Optional[Union[List[Dict[str, Any]], CustomMetric]], optional): list of instances of
-                `truefoundry.ml.CustomMetric`
-                The custom metrics must be added according to the prediction type of schema.
-                custom_metrics = [{
-                    "name": "mean_square_error",
-                    "type": "metric",
-                    "value_type": "float"
-                }]
 
         Returns:
             truefoundry.ml.ModelVersion: an instance of `ModelVersion` that can be used to download the files,
@@ -1394,9 +1362,8 @@ class MlFoundry:
             additional_files=additional_files,
             description=description,
             metadata=metadata,
-            model_schema=model_schema,
-            custom_metrics=custom_metrics,
             step=None,
+            progress=progress,
         )
         logger.info(f"Logged model successfully with fqn {model_version.fqn!r}")
         return model_version

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/ml/mlfoundry_run.py

@@ -38,6 +38,7 @@ from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
     RunTagDto,
     UpdateRunRequestDto,
 )
+from truefoundry.ml.entities import Metric
 from truefoundry.ml.enums import RunStatus
 from truefoundry.ml.exceptions import MlFoundryException
 from truefoundry.ml.internal_namespace import NAMESPACE
@@ -45,7 +46,6 @@ from truefoundry.ml.log_types import Image, Plot
 from truefoundry.ml.log_types.artifacts.artifact import ArtifactPath, ArtifactVersion
 from truefoundry.ml.log_types.artifacts.general_artifact import _log_artifact_version
 from truefoundry.ml.log_types.artifacts.model import ModelVersion, _log_model_version
-from truefoundry.ml.log_types.artifacts.model_extras import CustomMetric, ModelSchema
 from truefoundry.ml.logger import logger
 from truefoundry.ml.run_utils import ParamsType, flatten_dict, process_params
 from truefoundry.ml.session import ACTIVE_RUNS, _get_api_client, get_active_session
@@ -812,7 +812,7 @@ class MlFoundryRun:
     @_ensure_not_deleted
     def get_metrics(
         self, metric_names: Optional[Iterable[str]] = None
-    ) -> Dict[str, List[MetricDto]]:
+    ) -> Dict[str, List[Metric]]:
         """Get metrics logged for the current `run` grouped by metric name.
 
         Args:
@@ -860,7 +860,7 @@ class MlFoundryRun:
         unknown_metrics = metric_names - run_metric_names
         if len(unknown_metrics) > 0:
             logger.warning(f"{unknown_metrics} metrics not present in the run")
-        metrics_dict: Dict[str, List[MetricDto]] = {
+        metrics_dict: Dict[str, List[Metric]] = {
             metric_name: [] for metric_name in unknown_metrics
         }
         valid_metrics = metric_names - unknown_metrics
@@ -868,7 +868,9 @@ class MlFoundryRun:
             _metric_history = self._metrics_api.get_metric_history_get(
                 run_id=self.run_id, metric_key=metric_name
             )
-            metrics_dict[metric_name] = _metric_history.metrics
+            metrics_dict[metric_name] = [
+                Metric.from_dto(metric) for metric in _metric_history.metrics
+            ]
         return metrics_dict
 
     @_ensure_not_deleted
@@ -911,8 +913,6 @@ class MlFoundryRun:
         additional_files: Sequence[Tuple[Union[str, Path], Optional[str]]] = (),
         description: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
-        model_schema: Optional[Union[ModelSchema, Dict[str, Any]]] = None,
-        custom_metrics: Optional[List[Union[CustomMetric, Dict[str, Any]]]] = None,
         step: int = 0,
         progress: Optional[bool] = None,
     ) -> ModelVersion:
@@ -965,34 +965,6 @@ class MlFoundryRun:
             metadata (Optional[Dict[str, Any]], optional): arbitrary json serializable dictionary to store metadata.
                 For example, you can use this to store metrics, params, notes.
                 This field can be updated at any time after logging. Defaults to `None`
-            model_schema (Optional[Union[Dict[str, Any], ModelSchema]], optional):
-                instance of `truefoundry.ml.ModelSchema`.
-                This schema needs to be consistent with older versions of the model under the given `name` i.e.
-                a feature's value type and model's prediction type cannot be changed in the schema of new version.
-                Features can be removed or added between versions.
-                ```
-                E.g. if there exists a v1 with
-                    schema = {"features": {"name": "feat1": "int"}, "prediction": "categorical"}, then
-
-                    schema = {"features": {"name": "feat1": "string"}, "prediction": "categorical"} or
-                    schema = {"features": {"name": "feat1": "int"}, "prediction": "numerical"}
-                    are invalid because they change the types of existing features and prediction
-
-                    while
-                    schema = {"features": {"name": "feat1": "int", "feat2": "string"}, "prediction": "categorical"} or
-                    schema = {"features": {"feat2": "string"}, "prediction": "categorical"}
-                    are valid
-
-                    This field can be updated at any time after logging. Defaults to None
-                ```
-            custom_metrics: (Optional[Union[List[Dict[str, Any]], CustomMetric]], optional): list of instances of
-                `truefoundry.ml.CustomMetric`
-                The custom metrics must be added according to the prediction type of schema.
-                custom_metrics = [{
-                    "name": "mean_square_error",
-                    "type": "metric",
-                    "value_type": "float"
-                }]
             step (int): step/iteration at which the model is being logged, defaults to 0.
             progress (bool): value to show progress bar, defaults to None.
 
@@ -1084,8 +1056,6 @@ class MlFoundryRun:
             additional_files=additional_files,
             description=description,
             metadata=metadata,
-            model_schema=model_schema,
-            custom_metrics=custom_metrics,
             step=step,
             progress=progress,
         )

{truefoundry-0.4.0.dev1 → truefoundry-0.4.0rc2}/truefoundry/pydantic_v1.py

@@ -1,8 +1,11 @@
 try:
+    # pydantic >1.10.18
     from pydantic.v1 import *  # noqa: F403
     from pydantic.v1 import ConstrainedStr, utils  # noqa: F401
 except ImportError:
+    # pydantic <=1.10.17
     from pydantic import *  # noqa: F403
+    from pydantic import utils  # noqa: F401
 
 
 class NonEmptyStr(ConstrainedStr):