truefoundry 0.2.10__py3-none-any.whl → 0.3.0rc1__py3-none-any.whl

truefoundry/deploy/lib/clients/shell_client.py

@@ -0,0 +1,13 @@
+import subprocess
+
+from truefoundry.logger import logger
+
+
+class Shell:
+    def execute_shell_command(self, command, ip=None):
+        logger.debug(f"executing command: {command}")
+        try:
+            p = subprocess.run(command, stdout=subprocess.PIPE, check=True, input=ip)
+            return p.stdout.decode("UTF-8")
+        except subprocess.CalledProcessError as cpe:
+            raise Exception(f"failed to execute: {command}, error: {cpe}") from cpe

truefoundry/deploy/lib/clients/utils.py

@@ -0,0 +1,41 @@
+import os
+import time
+from typing import Any, Callable, Generator, Optional
+
+from truefoundry.deploy.lib.const import HOST_ENV_NAME
+from truefoundry.deploy.lib.exceptions import BadRequestException
+
+
+def request_handling(res):
+    try:
+        status_code = res.status_code
+    except Exception as ex:
+        raise Exception("Unknown error occurred. Couldn't get status code.") from ex
+    if 200 <= status_code <= 299:
+        if res.content == b"":
+            return None
+        return res.json()
+    if 400 <= status_code <= 499:
+        try:
+            message = str(res.json()["message"])
+        except Exception:
+            message = res
+        raise BadRequestException(res.status_code, message)
+    if 500 <= status_code <= 599:
+        raise Exception(res.content)
+
+
+def poll_for_function(
+    func: Callable, poll_after_secs: int = 5, *args, **kwargs
+) -> Generator[Any, None, None]:
+    while True:
+        yield func(*args, **kwargs)
+        time.sleep(poll_after_secs)
+
+
+def resolve_base_url(host: Optional[str] = None) -> str:
+    if not host and not os.getenv(HOST_ENV_NAME):
+        raise ValueError(
+            f"Either `host` should be provided by --host <value>, or `{HOST_ENV_NAME}` env must be set"
+        )
+    return host or os.getenv(HOST_ENV_NAME)

truefoundry/deploy/lib/const.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+from truefoundry.deploy.io.rich_output_callback import RichOutputCallBack
+
+DEFAULT_BASE_URL = "https://app.truefoundry.com"
+API_SERVER_RELATIVE_PATH = "api/svc"
+DEFAULT_API_SERVER = f"{DEFAULT_BASE_URL.rstrip('/')}/{API_SERVER_RELATIVE_PATH}"
+DEFAULT_AUTH_UI = DEFAULT_BASE_URL
+DEFAULT_AUTH_SERVER = (
+    "https://auth-server.tfy-ctl-euwe1-production.production.truefoundry.com"
+)
+DEFAULT_TENANT_NAME = "truefoundry"
+DEFAULT_PROFILE_NAME = "default"
+HOST_ENV_NAME = "TFY_HOST"
+API_KEY_ENV_NAME = "TFY_API_KEY"
+
+_SFY_CONFIG_DIR = Path.home() / ".truefoundry"
+SFY_CONFIG_DIR = str(_SFY_CONFIG_DIR)  # as a directory
+CREDENTIAL_FILEPATH = _SFY_CONFIG_DIR / "credentials.json"
+
+OLD_SFY_PROFILES_FILEPATH = _SFY_CONFIG_DIR / "profiles.json"  # as a directory
+OLD_SFY_SESSIONS_FILEPATH = _SFY_CONFIG_DIR / "sessions.json"  # as a directory
+OLD_SESSION_FILEPATH = str(
+    Path.home() / ".truefoundry"
+)  # as a filepath, to be removed in future versions
+
+# Polling during login redirect
+MAX_POLLING_RETRY = 100
+POLLING_SLEEP_TIME_IN_SEC = 4
+
+# Refresh access token cutoff
+REFRESH_ACCESS_TOKEN_IN_SEC = 10 * 60
+
+ENTITY_JSON_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+RICH_OUTPUT_CALLBACK = RichOutputCallBack()
+
+VERSION_PREFIX = "v1"
+
+SFY_DEBUG_ENV_KEY = "SFY_DEBUG"
+SFY_INTERNAL_ENV_KEY = "SFY_INTERNAL"
+
+TFY_DEBUG_ENV_KEY = "TFY_DEBUG"
+TFY_INTERNAL_ENV_KEY = "TFY_INTERNAL"

truefoundry/deploy/lib/dao/application.py

@@ -0,0 +1,246 @@
+import shlex
+from typing import Any, Dict, Optional, Sequence, Union
+
+import truefoundry.deploy.lib.dao.version as version_lib
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.model.entity import (
+    Application,
+    Deployment,
+    TriggerJobResult,
+)
+from truefoundry.deploy.lib.util import (
+    find_list_paths,
+    get_application_fqn_from_deployment_fqn,
+)
+from truefoundry.logger import logger
+from truefoundry.pydantic_v1 import utils as pydantic_v1_utils
+
+
+def list_applications(
+    application_type: str,
+    workspace_fqn: Optional[str] = None,
+    client: Optional[ServiceFoundryServiceClient] = None,
+):
+    client = client or ServiceFoundryServiceClient()
+    if workspace_fqn:
+        workspace = client.get_id_from_fqn(fqn=workspace_fqn, fqn_type="workspace")
+        applications = client.list_applications(workspace_id=workspace["workspaceId"])
+    else:
+        applications = client.list_applications()
+
+    if application_type != "all":
+        applications = [
+            application
+            for application in applications
+            if application.deployment.manifest.type == application_type
+        ]
+    return applications
+
+
+def get_application(
+    application_fqn: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+):
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    application = client.get_application_info(
+        application_id=application["applicationId"]
+    )
+    return application
+
+
+def delete_application(
+    application_fqn: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Dict[str, Any]:
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    response = client.remove_application(application_id=application["applicationId"])
+    return response
+
+
+def redeploy_application(
+    application_fqn: str,
+    version: int,
+    wait: bool,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Deployment:
+    from truefoundry.deploy.v2.lib.deployable_patched_models import Application
+
+    client = client or ServiceFoundryServiceClient()
+
+    deployment_info = version_lib.get_version(
+        application_fqn=application_fqn, version=version
+    )
+
+    manifest = deployment_info.manifest.dict()
+
+    application_id = deployment_info.applicationId
+    application_info = client.get_application_info(application_id=application_id)
+    workspace_fqn = application_info.workspace.fqn
+
+    application = Application.parse_obj(manifest)
+    deployment = application.deploy(workspace_fqn=workspace_fqn, wait=wait)
+    return deployment
+
+
+def list_job_runs(
+    application_fqn: str,
+    max_results: Optional[int] = None,
+    offset: Optional[int] = None,
+    client: Optional[ServiceFoundryServiceClient] = None,
+):
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    response = client.list_job_runs(
+        application_id=application["applicationId"], limit=max_results, offset=offset
+    )
+    return response
+
+
+def get_job_run(
+    application_fqn: str,
+    job_run_name: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+):
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    response = client.get_job_run(
+        application_id=application["applicationId"], job_run_name=job_run_name
+    )
+    return response
+
+
+def trigger_job(
+    application_fqn: str,
+    component_name: str = None,
+    command: Optional[Union[str, Sequence[str]]] = None,
+    params: Optional[Dict[str, str]] = None,
+) -> TriggerJobResult:
+    """
+    Trigger a Job on TrueFoundry platform
+
+    Args:
+        application_fqn: Fully Qualified Name of the Deployed Job (without the version number)
+        component_name: Name of the component to trigger the job on. Required in case of type `application`, defaults to `None`.
+        command: command to run the job with, defaults to `None`. Can be a `str` or `List[str]`
+            When `None`, the job is triggered with configured command at the time of deployment.
+            When passed as a list, the command will be joined using `shlex.join`
+        params: A dict mapping from parameter names (as defined in the job spec) to string values
+
+    Returns:
+        TriggerJobResult: metadata returning status of job trigger
+    """
+    if params and command:
+        raise ValueError(
+            "`command` and `params` arguments are mutually exclusive. Please pass only one of them"
+        )
+
+    try:
+        # If user is passing in deployment fqn copied from UI, till we change the fqns on UI
+        application_fqn = get_application_fqn_from_deployment_fqn(application_fqn)
+        logger.warning(
+            "Detected version number in `application_fqn`. "
+            "Automatically discarding the version number. "
+            f"This automatic conversion will be removed in future. "
+            f"Please pass {application_fqn!r} as the value."
+        )
+    except ValueError:
+        pass
+    client = ServiceFoundryServiceClient()
+    _application_info = client.get_application_info_by_fqn(
+        application_fqn=application_fqn
+    )
+    application_info = client.get_application_info(
+        application_id=_application_info.applicationId
+    )
+    command_str = ""
+    message = ""
+    if not params and not command:
+        message = "Job triggered with pre-configured command"
+    elif command:
+        if not isinstance(command, str):
+            command_str = shlex.join(command).strip()
+        else:
+            command_str = command.strip()
+        message = f"Job triggered with command {command_str!r}"
+    elif params:
+        # Check if params has any non string values
+        for key, value in params.items():
+            if not isinstance(value, str):
+                raise ValueError(
+                    f"Invalid value {value!r} for key {key!r}. "
+                    "Only string values are allowed for `params`"
+                )
+        command_str = ""
+        message = f"Job triggered with params {params!r}"
+    result = client.trigger_job(
+        deployment_id=application_info.activeDeploymentId,
+        component_name=component_name,
+        command=command_str if command_str else None,
+        params=params if params else None,
+    )
+    previous_runs_url = f"{client.base_url.strip('/')}/deployments/{application_info.id}?tab=previousRuns"
+    logger.info(
+        f"{message}.\n"
+        f"You can check the status of your job run at {previous_runs_url}"
+    )
+    return result
+
+
+def get_patched_application_definition(
+    application: Application,
+    manifest_patch: Dict[str, Any],
+) -> Dict[str, Any]:
+    # TODO: define ApplicationPatch type but for now
+    # I am adding a manual check for name since name patch can
+    # create a new application
+    if (
+        manifest_patch.get("name")
+        and manifest_patch.get("name") != application.deployment.manifest.name
+    ):
+        raise Exception(
+            f"Cannot change name of application from `{application.deployment.manifest.name}` to `{manifest_patch.get('name')}`"
+        )
+
+    patch_list_paths = find_list_paths(manifest_patch)
+
+    for path in patch_list_paths:
+        logger.warn(
+            f"You are patching the value at {path}. Note that updating array-type objects will replace the entire object."
+        )
+
+    return pydantic_v1_utils.deep_update(
+        application.deployment.manifest.dict(), manifest_patch
+    )
+
+
+def terminate_job_run(
+    application_fqn: str,
+    job_run_name: str,
+):
+    try:
+        application_fqn = get_application_fqn_from_deployment_fqn(application_fqn)
+        logger.warning(
+            "Detected version number in `application_fqn`. "
+            "Automatically discarding the version number. "
+            f"This automatic conversion will be removed in future. "
+            f"Please pass {application_fqn!r} as the value."
+        )
+    except ValueError:
+        pass
+    client = ServiceFoundryServiceClient()
+    _application_info = client.get_application_info_by_fqn(
+        application_fqn=application_fqn
+    )
+    application_info = client.get_application_info(
+        application_id=_application_info.applicationId
+    )
+
+    response = client.terminate_job_run(
+        deployment_id=application_info.activeDeploymentId,
+        job_run_name=job_run_name,
+    )
+    return response

truefoundry/deploy/lib/dao/apply.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+from typing import Any, Dict, Iterator, Optional
+
+import yaml
+
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.model.entity import ApplyResult, Manifest
+from truefoundry.pydantic_v1 import ValidationError
+
+
+def _apply_manifest(
+    manifest: Dict[str, Any],
+    client: Optional[ServiceFoundryServiceClient] = None,
+    filename: Optional[str] = None,
+    index: Optional[int] = None,
+) -> ApplyResult:
+    client = client or ServiceFoundryServiceClient()
+
+    file_metadata = ""
+    if index is not None:
+        file_metadata += f" at index {index}"
+    if filename:
+        file_metadata += f" from file {filename}"
+
+    try:
+        manifest = Manifest.parse_obj(manifest)
+    except ValidationError as ex:
+        return ApplyResult(
+            success=False,
+            message=f"Failed to apply manifest{file_metadata}. Error: {ex}",
+        )
+
+    try:
+        client.apply(manifest.dict())
+        return ApplyResult(
+            success=True,
+            message=f"Successfully configured manifest {manifest.name} of type {manifest.type}",
+        )
+    except Exception as ex:
+        return ApplyResult(
+            success=False,
+            message=f"Failed to apply manifest {manifest.name} of type {manifest.type}. Error: {ex}",
+        )
+
+
+def apply_manifest(
+    manifest: Dict[str, Any],
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> ApplyResult:
+    return _apply_manifest(manifest=manifest, client=client)
+
+
+def apply_manifest_file(
+    filepath: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Iterator[ApplyResult]:
+    client = client or ServiceFoundryServiceClient()
+    filename = Path(filepath).name
+    try:
+        with open(filepath, "r") as f:
+            manifests_it = list(yaml.safe_load_all(f))
+    except Exception as ex:
+        yield ApplyResult(
+            success=False,
+            message=f"Failed to read file {filepath} as a valid YAML file. Error: {ex}",
+        )
+    else:
+        for index, manifest in enumerate(manifests_it):
+            if not isinstance(manifest, dict):
+                yield ApplyResult(
+                    success=False,
+                    message=f"Failed to apply manifest at index {index} from file {filename}. Error: A manifest must be a dict, got type {type(manifest)}",
+                )
+                continue
+
+            yield _apply_manifest(
+                manifest=manifest, client=client, filename=filename, index=index
+            )

truefoundry/deploy/lib/dao/version.py

@@ -0,0 +1,33 @@
+from typing import List, Optional
+
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.model.entity import Deployment
+
+
+def list_versions(
+    application_fqn: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> List[Deployment]:
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    versions = client.list_versions(application_id=application["applicationId"])
+    return versions
+
+
+def get_version(
+    application_fqn: str,
+    version: int,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Deployment:
+    client = client or ServiceFoundryServiceClient()
+    application = client.get_id_from_fqn(fqn=application_fqn, fqn_type="app")
+    versions = client.list_versions(
+        application_id=application["applicationId"], deployment_version=version
+    )
+    if len(versions) == 0:
+        raise ValueError(
+            f"Version {version!r} for Application with FQN {application_fqn!r} does not exist."
+        )
+    return versions[0]

truefoundry/deploy/lib/dao/workspace.py

@@ -0,0 +1,71 @@
+from typing import Optional
+
+from truefoundry.deploy.cli.console import console
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.messages import PROMPT_CREATING_NEW_WORKSPACE
+from truefoundry.deploy.lib.model.entity import Workspace, WorkspaceResources
+from truefoundry.logger import logger
+
+
+def create_workspace(
+    name: str,
+    cluster_name: str,
+    cpu_limit: Optional[float] = None,
+    memory_limit: Optional[int] = None,
+    ephemeral_storage_limit: Optional[int] = None,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Workspace:
+    client = client or ServiceFoundryServiceClient()
+
+    workspace_resources = WorkspaceResources(
+        cpu_limit=cpu_limit,
+        memory_limit=memory_limit,
+        ephemeral_storage_limit=ephemeral_storage_limit,
+    )
+
+    with console.status(PROMPT_CREATING_NEW_WORKSPACE.format(name), spinner="dots"):
+        workspace = client.create_workspace(
+            workspace_name=name,
+            cluster_name=cluster_name,
+            resources=workspace_resources,
+        )
+
+    url = f"{client.base_url.strip('/')}/workspaces"
+    logger.info(
+        "You can find your workspace: '%s' on the dashboard: %s", workspace.name, url
+    )
+
+    return workspace
+
+
+def list_workspaces(
+    cluster_name: Optional[str] = None,
+    client: Optional[ServiceFoundryServiceClient] = None,
+):
+    client = client or ServiceFoundryServiceClient()
+    workspaces = client.list_workspaces(cluster_id=cluster_name)
+    return workspaces
+
+
+def get_workspace_by_fqn(
+    workspace_fqn: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Workspace:
+    client = client or ServiceFoundryServiceClient()
+    workspaces = client.get_workspace_by_fqn(workspace_fqn=workspace_fqn)
+    if len(workspaces) == 0:
+        raise ValueError(f"Workspace with FQN {workspace_fqn!r} does not exist.")
+    workspace = workspaces[0]
+    return workspace
+
+
+def delete_workspace(
+    workspace_fqn: str,
+    client: Optional[ServiceFoundryServiceClient] = None,
+) -> Workspace:
+    client = client or ServiceFoundryServiceClient()
+    workspace = client.get_id_from_fqn(fqn=workspace_fqn, fqn_type="workspace")
+    deleted_workspace = client.remove_workspace(workspace_id=workspace["workspaceId"])
+    return deleted_workspace

truefoundry/deploy/lib/exceptions.py

@@ -0,0 +1,23 @@
+class BadRequestException(Exception):
+    def __init__(self, status_code, message=None):
+        super(BadRequestException, self).__init__()
+        self.status_code = status_code
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.message
+
+
+class ConfigurationException(Exception):
+    def __init__(self, message=None):
+        super(ConfigurationException, self).__init__()
+        self.message = message
+
+    def __str__(self):
+        return self.message
+
+    def __repr__(self):
+        return self.message

truefoundry/deploy/lib/logs_utils.py

@@ -0,0 +1,43 @@
+import re
+from datetime import datetime, timedelta, timezone
+
+from dateutil import parser
+from dateutil.tz import tzlocal
+
+from truefoundry.logger import logger
+
+time_duration_regex = re.compile("^[0-9]+[smhd]$")
+
+granularity_timedelta = {
+    "s": timedelta(seconds=1),
+    "m": timedelta(minutes=1),
+    "h": timedelta(hours=1),
+    "d": timedelta(days=1),
+}
+
+
+def get_timestamp_from_timestamp_or_duration(time_or_duration: str):
+    """
+    Returns timestamp in milliseconds
+    """
+    if time_duration_regex.match(time_or_duration):
+        number = int(time_or_duration[:-1])
+        granularity = time_or_duration[-1]
+        current_time = datetime.now(timezone.utc)
+        required_time = current_time - granularity_timedelta[granularity] * number
+        return required_time.timestamp() * 1000
+    else:
+        logger.debug(
+            f"Cannot parse: {time_or_duration} as duration, regex match failed"
+        )
+        try:
+            date_time_obj = parser.parse(time_or_duration)
+        except ValueError as ve:
+            logger.debug(f"Value not a valid timestamp or duration: {time_or_duration}")
+            raise ValueError(
+                f"Value not a valid timestamp or duration: {time_or_duration}"
+            ) from ve
+        if not date_time_obj.tzinfo:
+            date_time_obj = date_time_obj.replace(tzinfo=tzlocal())
+        date_time_obj = date_time_obj.astimezone(timezone.utc)
+        return date_time_obj.timestamp() * 1000

truefoundry/deploy/lib/messages.py

@@ -0,0 +1,12 @@
+TFY = "TFY"
+
+# TODO: probably create another `rich_messages.py` and apply all formatting there
+PROMPT_LOGOUT_SUCCESSFUL = """[green bold]Logged Out![/]"""
+PROMPT_ALREADY_LOGGED_OUT = """[yellow]You are already logged out[/]"""
+PROMPT_CREATING_NEW_WORKSPACE = """[yellow]Creating a new workspace {!r}[/]"""
+PROMPT_DELETED_WORKSPACE = """[green]Deleted workspace {!r}[/]"""
+PROMPT_DELETED_APPLICATION = """[green]Deleted Application {!r}[/]"""
+PROMPT_NO_WORKSPACES = f"""[yellow]No workspaces found. Either cluster name is wrong, or your cluster doesn't contain any workspaces. You can create one with [bold]{TFY} create workspace[/][/]"""
+PROMPT_NO_APPLICATIONS = f"""[yellow]No applications found. You can create one with [bold]{TFY} deploy[/] from within your application folder"""
+PROMPT_NO_VERSIONS = """[yellow]No application versions found."""
+PROMPT_APPLYING_MANIFEST = """[yellow]Applying manifest for file {!r}[/]"""