truefoundry 0.4.0rc2__py3-none-any.whl → 0.4.0rc4__py3-none-any.whl

truefoundry/deploy/lib/model/entity.py

@@ -1,14 +1,11 @@
 from __future__ import annotations
 
 import datetime
-import time
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
-import jwt
-
 from truefoundry.deploy.lib.util import get_application_fqn_from_deployment_fqn
-from truefoundry.pydantic_v1 import BaseModel, Extra, Field, constr, validator
+from truefoundry.pydantic_v1 import BaseModel, Extra, Field
 
 # TODO: switch to Enums for str literals
 # TODO: Need a better approach to keep fields in sync with server
@@ -215,114 +212,6 @@ class Application(Entity):
         }
 
 
-class UserType(Enum):
-    user = "user"
-    serviceaccount = "serviceaccount"
-
-
-class UserInfo(BaseModel):
-    user_id: constr(min_length=1)
-    user_type: UserType = UserType.user
-    email: Optional[str] = None
-    tenant_name: constr(min_length=1) = Field(alias="tenantName")
-
-    class Config:
-        allow_population_by_field_name = True
-        allow_mutation = False
-
-
-class TenantInfo(BaseModel):
-    tenant_name: constr(min_length=1) = Field(alias="tenantName")
-    auth_server_url: str
-
-    class Config:
-        allow_population_by_field_name = True
-        allow_mutation = False
-
-
-class Token(BaseModel):
-    access_token: constr(min_length=1) = Field(alias="accessToken", repr=False)
-    refresh_token: Optional[constr(min_length=1)] = Field(
-        alias="refreshToken", repr=False
-    )
-    decoded_value: Optional[Dict] = Field(exclude=True, repr=False)
-
-    class Config:
-        allow_population_by_field_name = True
-        allow_mutation = False
-
-    @validator("decoded_value", always=True, pre=True)
-    def _decode_jwt(cls, v, values, **kwargs):
-        access_token = values["access_token"]
-        return jwt.decode(
-            access_token,
-            options={
-                "verify_signature": False,
-                "verify_aud": False,
-                "verify_exp": False,
-            },
-        )
-
-    @property
-    def tenant_name(self) -> str:
-        return self.decoded_value["tenantName"]
-
-    def is_going_to_be_expired(self, buffer_in_seconds: int = 120) -> bool:
-        exp = int(self.decoded_value["exp"])
-        return (exp - time.time()) < buffer_in_seconds
-
-    def to_user_info(self) -> UserInfo:
-        return UserInfo(
-            user_id=self.decoded_value["username"],
-            email=self.decoded_value["email"]
-            if "email" in self.decoded_value
-            else None,
-            user_type=UserType(self.decoded_value.get("userType", UserType.user.value)),
-            tenant_name=self.tenant_name,
-        )
-
-
-class CredentialsFileContent(BaseModel):
-    access_token: constr(min_length=1) = Field(repr=False)
-    refresh_token: Optional[constr(min_length=1)] = Field(repr=False)
-    host: constr(min_length=1)
-
-    class Config:
-        allow_mutation = False
-
-    def to_token(self) -> Token:
-        return Token(access_token=self.access_token, refresh_token=self.refresh_token)
-
-
-class DeviceCode(BaseModel):
-    user_code: str = Field(alias="userCode")
-    device_code: str = Field(alias="deviceCode")
-    verification_url: Optional[str] = Field(alias="verificationURI")
-    complete_verification_url: Optional[str] = Field(alias="verificationURIComplete")
-    expires_in_seconds: int = Field(alias="expiresInSeconds", default=60)
-    interval_in_seconds: int = Field(alias="intervalInSeconds", default=1)
-    message: Optional[str] = Field(alias="message")
-
-    class Config:
-        allow_population_by_field_name = True
-        allow_mutation = False
-
-    def get_user_clickable_url(self, auth_host: str) -> str:
-        return f"{auth_host}/authorize/device?userCode={self.user_code}"
-
-
-class PythonSDKConfig(BaseModel):
-    min_version: str = Field(alias="minVersion")
-    truefoundry_cli_min_version: str = Field(alias="truefoundryCliMinVersion")
-    use_sfy_server_auth_apis: Optional[bool] = Field(
-        alias="useSFYServerAuthAPIs", default=False
-    )
-
-    class Config:
-        allow_population_by_field_name = True
-        allow_mutation = False
-
-
 class JobRun(Base):
     name: str
     applicationName: str

truefoundry/deploy/lib/session.py

@@ -3,38 +3,22 @@ from typing import Optional
 
 import rich_click as click
 
+from truefoundry.common.auth_service_client import AuthServiceClient
+from truefoundry.common.constants import TFY_API_KEY_ENV_KEY, TFY_HOST_ENV_KEY
+from truefoundry.common.credential_file_manager import CredentialsFileManager
+from truefoundry.common.credential_provider import EnvCredentialProvider
+from truefoundry.common.entities import CredentialsFileContent, Token
+from truefoundry.common.utils import resolve_base_url
 from truefoundry.deploy.io.output_callback import OutputCallBack
-from truefoundry.deploy.lib.auth.auth_service_client import AuthServiceClient
-from truefoundry.deploy.lib.auth.credential_file_manager import CredentialsFileManager
-from truefoundry.deploy.lib.auth.credential_provider import EnvCredentialProvider
-from truefoundry.deploy.lib.clients.utils import resolve_base_url
 from truefoundry.deploy.lib.const import (
-    API_KEY_ENV_NAME,
-    HOST_ENV_NAME,
-    OLD_SFY_PROFILES_FILEPATH,
-    OLD_SFY_SESSIONS_FILEPATH,
     RICH_OUTPUT_CALLBACK,
 )
 from truefoundry.deploy.lib.messages import (
     PROMPT_ALREADY_LOGGED_OUT,
     PROMPT_LOGOUT_SUCCESSFUL,
 )
-from truefoundry.deploy.lib.model.entity import CredentialsFileContent, Token
 from truefoundry.logger import logger
 
-if OLD_SFY_PROFILES_FILEPATH.exists():
-    logger.warning(
-        "%s file is deprecated. You can delete this file now.",
-        OLD_SFY_PROFILES_FILEPATH,
-    )
-
-
-if OLD_SFY_SESSIONS_FILEPATH.exists():
-    logger.warning(
-        "%s file is deprecated. You can delete this file now.",
-        OLD_SFY_SESSIONS_FILEPATH,
-    )
-
 
 def login(
     api_key: Optional[str] = None,
@@ -42,13 +26,13 @@ def login(
     relogin: bool = False,
     output_hook: OutputCallBack = RICH_OUTPUT_CALLBACK,
 ) -> bool:
-    if API_KEY_ENV_NAME in os.environ and HOST_ENV_NAME in os.environ:
+    if TFY_API_KEY_ENV_KEY in os.environ and TFY_HOST_ENV_KEY in os.environ:
         logger.warning(
             "Skipping login because environment variables %s and "
             "%s are set and will be used when running truefoundry. "
             "If you want to relogin then unset these environment keys.",
-            HOST_ENV_NAME,
-            API_KEY_ENV_NAME,
+            TFY_HOST_ENV_KEY,
+            TFY_API_KEY_ENV_KEY,
         )
         return False
 
@@ -129,7 +113,10 @@ def _login_with_device_code(
         if device_code.message:
             message = device_code.message
         else:
-            message = f"Please open the following URL in a browser and enter the code {device_code.user_code} when prompted: {device_code.verification_url}"
+            message = (
+                f"Please open the following URL in a browser and enter the code {device_code.user_code} "
+                f"when prompted: {device_code.verification_url}"
+            )
     else:
         auto_open_url = device_code.get_user_clickable_url(auth_host=base_url)
     if auto_open_url:

truefoundry/deploy/lib/util.py

@@ -1,7 +1,5 @@
 import os
 import re
-from functools import lru_cache, wraps
-from time import monotonic_ns
 from typing import Union
 
 from truefoundry.deploy.lib.const import (
@@ -41,23 +39,6 @@ def get_deployment_fqn_from_application_fqn(
     return f"{application_fqn}:{version}"
 
 
-def is_notebook():
-    # https://stackoverflow.com/questions/15411967/how-can-i-check-if-code-is-executed-in-the-ipython-notebook
-    try:
-        # noinspection PyUnresolvedReferences
-        shell = get_ipython().__class__
-        if shell.__name__ == "ZMQInteractiveShell":
-            return True  # Jupyter notebook or qtconsole
-        elif shell.__name__ == "TerminalInteractiveShell":
-            return False  # Terminal running IPython
-        elif "google.colab" in str(shell):
-            return True  # google colab notebook
-        else:
-            return False  # Other type (?)
-    except NameError:
-        return False  # Probably standard Python interpreter
-
-
 def find_list_paths(data, parent_key="", sep="."):
     list_paths = []
     if isinstance(data, dict):
@@ -70,21 +51,3 @@ def find_list_paths(data, parent_key="", sep="."):
             new_key = f"{parent_key}[{i}]"
             list_paths.extend(find_list_paths(value, new_key, sep))
     return list_paths
-
-
-def timed_lru_cache(seconds: int = 300, maxsize: int = None):
-    def wrapper_cache(func):
-        func = lru_cache(maxsize=maxsize)(func)
-        func.delta = seconds * 10**9
-        func.expiration = monotonic_ns() + func.delta
-
-        @wraps(func)
-        def wrapped_func(*args, **kwargs):
-            if monotonic_ns() >= func.expiration:
-                func.cache_clear()
-                func.expiration = monotonic_ns() + func.delta
-            return func(*args, **kwargs)
-
-        return wrapped_func
-
-    return wrapper_cache

truefoundry/deploy/python_deploy_codegen.py

@@ -101,11 +101,11 @@ def add_local_source_comment(code):
     return "\n".join(new_lines)
 
 
-def convert_deployment_config_to_python(workspace_fqn: str, deployment_config: dict):
+def convert_deployment_config_to_python(workspace_fqn: str, application_spec: dict):
     """
     Convert a deployment config to a python file that can be used to deploy to a workspace
     """
-    application = Application.parse_obj(deployment_config)
+    application = Application.parse_obj(application_spec)
     application_type = application.__root__.type
 
     spec_repr = get_python_repr(application.__root__)

truefoundry/deploy/v2/lib/deploy.py

@@ -4,12 +4,12 @@ from typing import List, Optional, TypeVar
 
 from rich.status import Status
 
+from truefoundry.common.utils import poll_for_function
 from truefoundry.deploy.auto_gen import models as auto_gen_models
 from truefoundry.deploy.builder.docker_service import env_has_docker
 from truefoundry.deploy.lib.clients.servicefoundry_client import (
     ServiceFoundryServiceClient,
 )
-from truefoundry.deploy.lib.clients.utils import poll_for_function
 from truefoundry.deploy.lib.dao.workspace import get_workspace_by_fqn
 from truefoundry.deploy.lib.model.entity import Deployment, DeploymentTransitionStatus
 from truefoundry.deploy.lib.util import get_application_fqn_from_deployment_fqn
@@ -45,7 +45,7 @@ def _handle_if_local_source(component: Component, workspace_fqn: str) -> Compone
                 local_build = False
             else:
                 logger.info(
-                    "Found locally installed docker, image will be built locally and then pushed. "
+                    "Found locally installed docker, image will be built locally and then pushed.\n"
                     "If you want to always build remotely instead of locally, "
                     "please set `image.build_source.local_build` to `false` in your YAML spec or equivalently set "
                     "`image=Build(build_source=LocalSource(local_build=False, ...))` in your "
@@ -72,7 +72,7 @@ def _handle_if_local_source(component: Component, workspace_fqn: str) -> Compone
                 component_name=component.name,
             )
         else:
-            # We'll build image on Truefoundry servers, upload the source and update image.build_source
+            # We'll build image on TrueFoundry servers, upload the source and update image.build_source
             logger.info("Uploading code for %s '%s'", component.type, component.name)
             new_component.image.build_source = local_source_to_remote_source(
                 local_source=component.image.build_source,

truefoundry/ml/__init__.py

@@ -13,7 +13,6 @@ from truefoundry.ml.log_types.artifacts.model import (
     ModelVersion,
 )
 from truefoundry.ml.logger import init_logger
-from truefoundry.ml.login import login
 from truefoundry.ml.mlfoundry_api import get_client
 from truefoundry.ml.mlfoundry_run import MlFoundryRun
 
@@ -33,7 +32,6 @@ __all__ = [
     "Plot",
     "ViewType",
     "get_client",
-    "login",
 ]
 
 init_logger()

truefoundry/ml/artifact/truefoundry_artifact_repo.py

@@ -23,6 +23,7 @@ from urllib.parse import unquote
 from urllib.request import pathname2url
 
 import requests
+from rich.console import _is_jupyter
 from rich.progress import (
     BarColumn,
     DownloadColumn,
@@ -53,13 +54,13 @@ from truefoundry.ml.autogen.client import (  # type: ignore[attr-defined]
     RunArtifactsApi,
     SignedURLDto,
 )
-from truefoundry.ml.env_vars import DISABLE_MULTIPART_UPLOAD
-from truefoundry.ml.exceptions import MlFoundryException
-from truefoundry.ml.logger import logger
-from truefoundry.ml.services.utils import (
+from truefoundry.ml.clients.utils import (
     augmented_raise_for_status,
     cloud_storage_http_request,
 )
+from truefoundry.ml.env_vars import DISABLE_MULTIPART_UPLOAD
+from truefoundry.ml.exceptions import MlFoundryException
+from truefoundry.ml.logger import logger
 from truefoundry.ml.session import _get_api_client
 from truefoundry.pydantic_v1 import BaseModel, root_validator
 
@@ -91,6 +92,39 @@ _GENERATE_SIGNED_URL_BATCH_SIZE = 50
 DEFAULT_PRESIGNED_URL_EXPIRY_TIME = 3600
 
 
+def _get_relpath_if_in_tempdir(path: str) -> str:
+    tempdir = tempfile.gettempdir()
+    if path.startswith(tempdir):
+        return os.path.relpath(path, tempdir)
+    return path
+
+
+def _can_display_progress(user_choice: Optional[bool] = None) -> bool:
+    if user_choice is False:
+        return False
+
+    if sys.stdout.isatty():
+        return True
+    elif _is_jupyter():
+        try:
+            from IPython.display import display  # noqa: F401
+            from ipywidgets import Output  # noqa: F401
+
+            return True
+        except ImportError:
+            logger.warning(
+                "Detected Jupyter Environment. Install `ipywidgets` to display live progress bars.",
+            )
+    if user_choice is True:
+        logger.warning(
+            "`progress` argument is set to True but did not detect tty "
+            "or jupyter environment with ipywidgets installed. "
+            "Progress bars may not be displayed. "
+        )
+        return True
+    return False
+
+
 def relative_path_to_artifact_path(path):
     if os.path == posixpath:
         return path
@@ -206,7 +240,7 @@ def _signed_url_upload_file(
         return
 
     task_progress_bar = progress_bar.add_task(
-        f"[green]Uploading {local_file}:", start=True
+        f"[green]Uploading {_get_relpath_if_in_tempdir(local_file)}:", start=True
     )
 
     def callback(length):
@@ -304,7 +338,7 @@ def _s3_compatible_multipart_upload(
     parts = []
 
     multi_part_upload_progress = progress_bar.add_task(
-        f"[green]Uploading {local_file}:", start=True
+        f"[green]Uploading {_get_relpath_if_in_tempdir(local_file)}:", start=True
     )
 
     def upload(part_number: int, seek: int) -> None:
@@ -315,7 +349,7 @@ def _s3_compatible_multipart_upload(
             local_file,
         )
         response = _file_part_upload(
-            url=multipart_upload.part_signed_urls[part_number].url,
+            url=multipart_upload.part_signed_urls[part_number].signed_url,
             file_path=local_file,
             seek=seek,
             length=multipart_info.part_size,
@@ -374,7 +408,7 @@ def _azure_multi_part_upload(
     abort_event = abort_event or Event()
 
     multi_part_upload_progress = progress_bar.add_task(
-        f"[green]Uploading {local_file}:", start=True
+        f"[green]Uploading {_get_relpath_if_in_tempdir(local_file)}:", start=True
     )
 
     def upload(part_number: int, seek: int):
@@ -385,7 +419,7 @@ def _azure_multi_part_upload(
             local_file,
         )
         _file_part_upload(
-            url=multipart_upload.part_signed_urls[part_number].url,
+            url=multipart_upload.part_signed_urls[part_number].signed_url,
             file_path=local_file,
             seek=seek,
             length=multipart_info.part_size,
@@ -527,8 +561,7 @@ class MlFoundryArtifactsRepository:
     def log_artifacts(  # noqa: C901
         self, local_dir, artifact_path=None, progress=None
     ):
-        if progress is None:
-            progress = sys.stdout.isatty()
+        show_progress = _can_display_progress(progress)
 
         dest_path = artifact_path or ""
         dest_path = dest_path.lstrip(posixpath.sep)
@@ -565,14 +598,15 @@ class MlFoundryArtifactsRepository:
 
         with Progress(
             "[progress.description]{task.description}",
-            BarColumn(),
+            BarColumn(bar_width=None),
             "[progress.percentage]{task.percentage:>3.0f}%",
             DownloadColumn(),
             TransferSpeedColumn(),
             TimeRemainingColumn(),
             TimeElapsedColumn(),
             refresh_per_second=1,
-            disable=not progress,
+            disable=not show_progress,
+            expand=True,
         ) as progress_bar, ThreadPoolExecutor(
             max_workers=_MAX_WORKERS_FOR_UPLOAD
         ) as executor:
@@ -647,7 +681,11 @@ class MlFoundryArtifactsRepository:
             )[0]
 
         if progress_bar.disable:
-            logger.info("Uploading %s to %s", local_file, artifact_path)
+            logger.info(
+                "Uploading %s to %s",
+                _get_relpath_if_in_tempdir(local_file),
+                artifact_path,
+            )
 
         _signed_url_upload_file(
             signed_url=signed_url,
@@ -668,7 +706,9 @@ class MlFoundryArtifactsRepository:
     ):
         if progress_bar.disable:
             logger.info(
-                "Uploading %s to %s using multipart upload", local_file, artifact_path
+                "Uploading %s to %s using multipart upload",
+                _get_relpath_if_in_tempdir(local_file),
+                artifact_path,
             )
 
         multipart_upload = self.create_multipart_upload_for_identifier(
@@ -746,7 +786,7 @@ class MlFoundryArtifactsRepository:
         upload_path = posixpath.join(upload_path, os.path.basename(local_file))
         with Progress(
             "[progress.description]{task.description}",
-            BarColumn(),
+            BarColumn(bar_width=None),
             "[progress.percentage]{task.percentage:>3.0f}%",
             DownloadColumn(),
             TransferSpeedColumn(),
@@ -754,6 +794,7 @@ class MlFoundryArtifactsRepository:
             TimeElapsedColumn(),
             refresh_per_second=1,
             disable=True,
+            expand=True,
         ) as progress_bar:
             self._log_artifact(
                 local_file=local_file,
@@ -769,11 +810,11 @@ class MlFoundryArtifactsRepository:
 
     def download_artifacts(  # noqa: C901
         self,
-        artifact_path,
-        dst_path=None,
+        artifact_path: str,
+        dst_path: Optional[str] = None,
         overwrite: bool = False,
         progress: Optional[bool] = None,
-    ):
+    ) -> str:
         """
         Download an artifact file or directory to a local directory if applicable, and return a
         local path for it. The caller is responsible for managing the lifecycle of the downloaded artifacts.
@@ -791,8 +832,7 @@ class MlFoundryArtifactsRepository:
             str: Absolute path of the local filesystem location containing the desired artifacts.
         """
 
-        if progress is None:
-            progress = sys.stdout.isatty()
+        show_progress = _can_display_progress()
 
         is_dir_temp = False
         if dst_path is None:
@@ -829,7 +869,8 @@ class MlFoundryArtifactsRepository:
             TimeRemainingColumn(),
             TimeElapsedColumn(),
             refresh_per_second=1,
-            disable=not progress,
+            disable=not show_progress,
+            expand=True,
         )
 
         try:

truefoundry/ml/autogen/client/__init__.py

@@ -74,9 +74,6 @@ from truefoundry.ml.autogen.client.models.authorize_user_for_model_request_dto i
 from truefoundry.ml.autogen.client.models.authorize_user_for_model_version_request_dto import (
     AuthorizeUserForModelVersionRequestDto,
 )
-from truefoundry.ml.autogen.client.models.backfill_default_storage_integration_id_request_dto import (
-    BackfillDefaultStorageIntegrationIdRequestDto,
-)
 from truefoundry.ml.autogen.client.models.blob_storage_reference import (
     BlobStorageReference,
 )