truefoundry 0.5.8rc2__tar.gz → 0.5.8rc4__tar.gz

truefoundry-0.5.8rc4/.github/workflows/integration-tests.yml

@@ -0,0 +1,87 @@
+name: Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request_review:
+    branches:
+      - main
+    types:
+      - submitted
+  workflow_dispatch:
+
+jobs:
+  integration-test-dev:
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_review' && github.event.review.state == 'approved')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv venv --python ${{ matrix.python-version }} --seed
+
+      - name: Install the project
+        run: uv sync --locked  --all-extras
+
+      - name: Run Integration Tests
+        env:
+          # Environment variables needed for tests
+          ITEST_TFY_HOST: ${{ secrets.ITEST_DEV_TFY_HOST }}
+          ITEST_TFY_API_KEY: ${{ secrets.ITEST_DEV_TFY_API_KEY }}
+          ITEST_CLUSTER_FQN: 'tfy-usea1-devtest'
+          ITEST_STORAGE_INTEGRATION_FQN: 'truefoundry:aws:tfy-usea1-ctl-devtest-internal:blob-storage:blob'
+          ITEST_TENANT_NAME: 'truefoundry'
+        run: |
+          uv run pytest -s -vv -m "integration" --durations=0 tests/itest/
+
+  integration-test-prod:
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_review' && github.event.review.state == 'approved')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        run: uv venv --python ${{ matrix.python-version }} --seed
+
+      - name: Install the project
+        run: uv sync --locked  --all-extras
+
+      - name: Run Integration Tests
+        env:
+          # Environment variables needed for tests
+          ITEST_TFY_HOST: ${{ secrets.ITEST_PROD_TFY_HOST }}
+          ITEST_TFY_API_KEY: ${{ secrets.ITEST_PROD_TFY_API_KEY }}
+          ITEST_CLUSTER_FQN: 'tfy-usea1-ctl-devtest'
+          ITEST_STORAGE_INTEGRATION_FQN: 'internal:aws:aws-2:blob-storage:truefoundry-default'
+          ITEST_TENANT_NAME: 'internal'
+        run: |
+          uv run pytest -s -vv -m "integration" --durations=0 tests/itest/

{truefoundry-0.5.8rc2 → truefoundry-0.5.8rc4}/.github/workflows/lint-and-test.yml

@@ -91,9 +91,9 @@ jobs:
 
       - name: Test TrueFoundry CLI invocations
         run: |
-          uv run tfy --help
-          uv run tfy deploy --help
-          uv run tfy ml --help
+          uv run --no-dev tfy --help
+          uv run --no-dev tfy deploy --help
+          uv run --no-dev tfy ml --help
 
   test:
     strategy:
@@ -129,4 +129,4 @@ jobs:
 
       - name: Run tests
         run: |
-          uv run pytest --durations=0 tests/
+          uv run pytest -m "not integration" --durations=0 tests/

{truefoundry-0.5.8rc2 → truefoundry-0.5.8rc4}/.github/workflows/release.yml

@@ -15,8 +15,11 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
 
-      - name: Enable dynamic versioning
-        run: uv run enable_dynamic_versioning.py
+      - name: Set the version
+        run: |
+          VERSION=$(uv run --python 3.10 --with "dunamai>=1.23,<2.0" dunamai from git --no-metadata --style pep440)
+          echo "Setting version to $VERSION"
+          uv run --python 3.10 --with "toml-cli>=0.7.0,<1.0.0" toml set --toml-path=pyproject.toml project.version $VERSION
 
       - name: Build project
         run: uv build --python 3.10

{truefoundry-0.5.8rc2 → truefoundry-0.5.8rc4}/.pre-commit-config.yaml

@@ -16,9 +16,6 @@ repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.9.6
   hooks:
-    - id: ruff-format
-      args:
-        - --config=pyproject.toml
     - id: ruff
       args:
         - --config=pyproject.toml

{truefoundry-0.5.8rc2 → truefoundry-0.5.8rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truefoundry
-Version: 0.5.8rc2
+Version: 0.5.8rc4
 Summary: TrueFoundry CLI
 Author-email: TrueFoundry Team <abhishek@truefoundry.com>
 Requires-Python: <3.13,>=3.8.1

{truefoundry-0.5.8rc2 → truefoundry-0.5.8rc4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truefoundry"
-dynamic = ["version"]
+version = "0.5.8rc4"  # This is a placeholder version, it will be replaced when releasing
 requires-python = ">=3.8.1,<3.13"
 description = "TrueFoundry CLI"
 authors = [{ name = "TrueFoundry Team", email = "abhishek@truefoundry.com" }]
@@ -88,15 +88,19 @@ dev = [
 ]
 
 [build-system]
-requires = ["hatchling", "uv-dynamic-versioning>=0.1.0,<1.0.0; python_version >= '3.9'"]
+requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[tool.hatch.version]
-source = "uv-dynamic-versioning"
-
 [tool.ruff]
 target-version = "py38"
-lint.select = [
+exclude = [
+  ".venv",
+  "venv",
+  "repos",
+]
+
+[tool.ruff.lint]
+select = [
   "E",     # pycodestyle errors
   "W",     # pycodestyle warnings
   "F",     # pyflakes
@@ -107,24 +111,22 @@ lint.select = [
   "T10",
   "ASYNC",
 ]
-lint.ignore = ["E501"]
-exclude = [
-  ".venv",
-  "venv",
-  "repos",
-  "truefoundry/ml/autogen/**",
-  "truefoundry/deploy/auto_gen/**",
-]
+ignore = ["E501"]
 
-[tool.ruff.format]
-quote-style = "double"
+[tool.ruff.lint.flake8-quotes]
+docstring-quotes = "double"
+inline-quotes = "double"
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401"]
 "truefoundry/ml/autogen/client/*" = ["B028", "B904", "C901", "C409", "F841"]
+"truefoundry/ml/autogen/models/*" = ["E721"]
 "truefoundry/ml/autogen/client/models/*" = ["F821"]
 "truefoundry/ml/autogen/client/api_client.py" = ["E721"]
 
+[tool.ruff.format]
+quote-style = "double"
+
 [tool.mypy]
 plugins = ["pydantic.v1.mypy"]
 install_types = true
@@ -172,7 +174,3 @@ env = [
 
 [tool.uv]
 default-groups = ["dev"]
-
-[tool.uv-dynamic-versioning]
-vcs = "git"
-strict = true

truefoundry-0.5.8rc4/tests/itest/api_client.py

@@ -0,0 +1,97 @@
+import os
+from pprint import pprint
+
+import requests
+
+
+class TruefoundryAPIClient:
+    """Client for interacting with Truefoundry API"""
+
+    def __init__(self):
+        self.host = os.getenv("ITEST_TFY_HOST")
+        self.api_key = os.getenv("ITEST_TFY_API_KEY")
+        self.headers = {
+            "accept": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+
+        if not self.host or not self.api_key:
+            raise ValueError(
+                "TFY_HOST and TFY_API_KEY environment variables are required"
+            )
+
+    def get_provider_account_id(self, provider_account_fqn: str) -> str:
+        url = f"{self.host}/api/svc/v1/provider-accounts"
+
+        try:
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+
+            # Parse the nested response
+            data = response.json()
+            if "providerAccounts" not in data:
+                raise ValueError("Unexpected API response format")
+
+            provider_accounts = data["providerAccounts"]
+
+            # Find matching account
+            for account in provider_accounts:
+                if account["fqn"] == provider_account_fqn:
+                    return account["id"]
+
+            raise ValueError(
+                f"Provider account with FQN {provider_account_fqn} not found. "
+                f"Available FQNs: {[acc['fqn'] for acc in provider_accounts]}"
+            )
+
+        except (ValueError, requests.exceptions.RequestException) as e:
+            print(f"Error in get provider account operation: {e}")
+            raise
+
+    def delete_provider_account(self, provider_account_fqn: str) -> None:
+        try:
+            provider_account_id = self.get_provider_account_id(provider_account_fqn)
+            pprint(provider_account_id)
+            url = f"{self.host}/api/svc/v1/provider-accounts/{provider_account_id}"
+            response = requests.delete(url, headers=self.headers)
+            response.raise_for_status()
+            assert response.status_code == 200
+
+        except (ValueError, requests.exceptions.RequestException) as e:
+            print(f"Error in delete provider account operation: {e}")
+            raise
+
+    def get_ml_repo_id(self, repo_name: str) -> str:
+        url = f"{self.host}/api/ml/api/2.0/mlflow/experiments/list"
+
+        try:
+            response = requests.get(
+                url, headers=self.headers, params={"filter_name": repo_name}
+            )
+            response.raise_for_status()
+            data = response.json()
+            if not data["experiments"]:
+                raise ValueError(f"Invalid API response structure: {data}")
+            if (
+                not data["experiments"][0]
+                or data["experiments"][0]["name"] != repo_name
+            ):
+                raise ValueError(f"ML repository '{repo_name}' not found. ")
+            return data["experiments"][0]["experiment_id"]
+
+        except (ValueError, requests.exceptions.RequestException) as e:
+            print(f"Error in get ML repository operation: {e}")
+            raise
+
+    def delete_ml_repo(self, repo_name: str) -> None:
+        try:
+            repo_id = self.get_ml_repo_id(repo_name)
+            url = f"{self.host}/api/ml/api/2.0/mlflow/experiments/hard-delete"
+            payload = {"experiment_id": repo_id}
+            response = requests.post(url, headers=self.headers, json=payload)
+            response.raise_for_status()
+            assert response.status_code == 200
+
+        except (ValueError, requests.exceptions.RequestException) as e:
+            print(f"Error in delete ML repository operation: {e}")
+            raise

truefoundry-0.5.8rc4/tests/itest/conftest.py

@@ -0,0 +1,118 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from tests.itest.utils import (
+    ITEST_EXAMPLES_DIR,
+    clean_cli_output,
+    modify_name_in_manifest_file,
+    override_field_in_yaml_file,
+)
+from truefoundry.cli.__main__ import create_truefoundry_cli
+
+runner = CliRunner()
+
+
+@dataclass
+class DeployTestState:
+    """Manages state for deployment tests"""
+
+    workspace_yaml_path: Path = (
+        Path(ITEST_EXAMPLES_DIR) / "deploy/internal_workspace.yaml"
+    )
+    workspace_yaml: dict = None
+    workspace_name: str = ""
+    workspace_fqn: str = ""
+    cluster_fqn: str = ""
+
+    def __post_init__(self):
+        self._set_cluster_fqn()
+        self._initialize_workspace()
+
+    def _set_cluster_fqn(self):
+        """Set cluster_fqn from environment variable"""
+        self.cluster_fqn = os.getenv("ITEST_CLUSTER_FQN")
+        if not self.cluster_fqn:
+            raise ValueError("ITEST_CLUSTER_FQN environment variable is required")
+        override_field_in_yaml_file(
+            self.workspace_yaml_path,
+            "cluster_fqn",
+            self.cluster_fqn,
+        )
+
+    def _initialize_workspace(self):
+        """Initialize workspace with unique name"""
+        # There might be other workspaces in the cluster with the same name (failed tests or coincidence)
+        # So we need to modify the workspace name to make it unique
+        modify_name_in_manifest_file(self.workspace_yaml_path, "itest-internal-ws")
+        with open(self.workspace_yaml_path) as f:
+            self.workspace_yaml = yaml.safe_load(f)
+            self.workspace_name = self.workspace_yaml["name"]
+            self.workspace_fqn = (
+                f"{self.workspace_yaml['cluster_fqn']}:{self.workspace_name}"
+            )
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cli_setup():
+    """Primary setup fixture for CLI authentication. Runs automatically for all tests."""
+    # Get TFY host and API key from environment variable
+    tfy_host = os.getenv("ITEST_TFY_HOST")
+    tfy_api_key = os.getenv("ITEST_TFY_API_KEY")
+    if not tfy_host or not tfy_api_key:
+        raise ValueError(
+            "ITEST_TFY_HOST and ITEST_TFY_API_KEY environment variables are required"
+        )
+
+    print("\n----- CLI Setup -----")
+    result = runner.invoke(
+        create_truefoundry_cli(),
+        ["login", "--host", tfy_host, "--api-key", tfy_api_key],
+    )
+    assert result.exit_code == 0, "CLI login failed"
+    yield
+    print("----- CLI Cleanup -----")
+
+
+@pytest.fixture(scope="session")
+def deploy_state():
+    """Initialize deployment test state"""
+    print("\n----- Initializing Deploy Test State -----")
+    return DeployTestState()
+
+
+@pytest.fixture(scope="session")
+def workspace_setup(deploy_state):
+    """Setup workspace for deployment tests"""
+    print("----- Workspace Setup -----")
+
+    # Create workspace
+    result = runner.invoke(
+        create_truefoundry_cli(), ["apply", "--file", deploy_state.workspace_yaml_path]
+    )
+    clean_output = clean_cli_output(result.output)
+    print(f"Workspace creation output: {clean_output}")
+    assert result.exit_code == 0
+    assert (
+        f"Successfully configured manifest {deploy_state.workspace_name} of type workspace"
+        in clean_output
+    )
+
+    yield deploy_state
+
+    # Cleanup workspace
+    print(f"\n----- Cleaning up workspace: {deploy_state.workspace_name} -----")
+    result = runner.invoke(
+        create_truefoundry_cli(),
+        ["delete", "workspace", "--workspace-fqn", deploy_state.workspace_fqn, "--yes"],
+    )
+    assert result.exit_code == 0
+
+
+def pytest_configure(config):
+    """Register custom markers."""
+    config.addinivalue_line("markers", "integration: mark test as an integration test")

truefoundry-0.5.8rc4/tests/itest/examples/apply/ml_repo/example1/tf.yaml

@@ -0,0 +1,7 @@
+collaborators:
+- role_id: mlf-project-admin
+  subject: team:everyone
+description: Hello World
+name: itest-mlrepo-example1
+storage_integration_fqn: truefoundry:aws:tfy-usea1-ctl-devtest-internal:blob-storage:blob
+type: ml-repo

truefoundry-0.5.8rc4/tests/itest/examples/apply/provider_account/example1/tf.yaml

@@ -0,0 +1,14 @@
+type: provider-account/aws
+name: itest-pa-example1
+aws_account_id: '526077812922'
+integrations:
+  - type: integration/blob-storage/aws/s3
+    authorized_subjects:
+      - team:everyone
+    name: blob3
+    storage_root: s3://tfy-dev-model-server-research-2022-10-16/cli-integration-tests
+    region: eu-west-1
+    auth_data:
+      type: access-key-based
+      access_key_id: AKIAXU7FOMC5OSAS6L5M
+      secret_access_key: CG1HVSqVZMPdxaQy1UUM/PT7TyE+chWe4P3cdPwE

truefoundry-0.5.8rc4/tests/itest/examples/apply/workspace/example1/tf.yaml

@@ -0,0 +1,7 @@
+cluster_fqn: tfy-usea1-devtest
+collaborators:
+- role_id: workspace-admin
+  subject: team:everyone
+environment_name: devtest
+name: itest-ws-example1
+type: workspace

truefoundry-0.5.8rc4/tests/itest/examples/deploy/internal_workspace.yaml

@@ -0,0 +1,7 @@
+cluster_fqn: tfy-usea1-devtest
+collaborators:
+- role_id: workspace-admin
+  subject: user:rushabh@truefoundry.com
+environment_name: devtest
+name: itest-internal-ws-a3f5
+type: workspace

truefoundry-0.5.8rc4/tests/itest/examples/deploy/job/example1/tf.yaml

@@ -0,0 +1,28 @@
+image:
+  build_source:
+    branch_name: main
+    ref: 71b5433ad0672f500f6c8fbbc2cc5c409ee7f3b3
+    repo_url: https://github.com/truefoundry/getting-started-examples
+    type: git
+  build_spec:
+    build_context_path: ./train-model/
+    command: python train.py
+    python_version: '3.11'
+    type: tfy-python-buildpack
+  type: build
+name: itest-job-example1
+resources:
+  cpu_limit: 0.5
+  cpu_request: 0.2
+  ephemeral_storage_limit: 2000
+  ephemeral_storage_request: 1000
+  memory_limit: 500
+  memory_request: 200
+  node:
+    capacity_type: spot_fallback_on_demand
+    type: node_selector
+retries: 0
+trigger:
+  type: manual
+trigger_on_deploy: false
+type: job

truefoundry-0.5.8rc4/tests/itest/examples/deploy/job/example2/tf.yaml

@@ -0,0 +1,31 @@
+image:
+  build_source:
+    branch_name: main
+    ref: 71b5433ad0672f500f6c8fbbc2cc5c409ee7f3b3
+    repo_url: https://github.com/truefoundry/getting-started-examples
+    type: git
+  build_spec:
+    build_context_path: ./train-model/
+    command: python train.py
+    python_version: '3.11'
+    type: tfy-python-buildpack
+  type: build
+name: itest-job-example2
+resources:
+  cpu_limit: 0.5
+  cpu_request: 0.2
+  ephemeral_storage_limit: 2000
+  ephemeral_storage_request: 1000
+  memory_limit: 500
+  memory_request: 200
+  node:
+    capacity_type: spot_fallback_on_demand
+    type: node_selector
+retries: 1
+trigger:
+  concurrency_policy: Forbid
+  schedule: '*/5 * * * *'
+  timezone: UTC
+  type: scheduled
+trigger_on_deploy: true
+type: job

truefoundry-0.5.8rc4/tests/itest/examples/deploy/service/example1/tf.yaml

@@ -0,0 +1,31 @@
+allow_interception: false
+image:
+  build_source:
+    branch_name: main
+    ref: 3df2537bded2b6761997ef0a4347951a687a2cb3
+    repo_url: https://github.com/truefoundry/emotion-classification-fastapi
+    type: git
+  build_spec:
+    build_context_path: ./
+    dockerfile_path: ./Dockerfile
+    type: dockerfile
+  type: build
+name: itest-service-example1
+ports:
+- app_protocol: http
+  expose: true
+  host: itest-service-example1-8000.tfy-usea1-ctl.devtest.truefoundry.tech
+  port: 8000
+  protocol: TCP
+replicas: 1
+resources:
+  cpu_limit: 0.5
+  cpu_request: 0.2
+  ephemeral_storage_limit: 2000
+  ephemeral_storage_request: 1000
+  memory_limit: 500
+  memory_request: 200
+  node:
+    capacity_type: spot_fallback_on_demand
+    type: node_selector
+type: service

truefoundry-0.5.8rc4/tests/itest/examples/deploy/service/example2/tf.yaml

@@ -0,0 +1,35 @@
+allow_interception: false
+image:
+  image_uri: testcontainers/helloworld
+  type: image
+labels:
+  tfy_openapi_path: openapi.json
+name: itest-service-example2
+ports:
+- app_protocol: http
+  expose: true
+  host: itest-service-example2-8080.tfy-usea1-ctl.devtest.truefoundry.tech
+  port: 8080
+  protocol: TCP
+readiness_probe:
+  config:
+    path: /ping
+    port: 8080
+    type: http
+  failure_threshold: 3
+  initial_delay_seconds: 30
+  period_seconds: 10
+  success_threshold: 1
+  timeout_seconds: 1
+replicas: 1
+resources:
+  cpu_limit: 0.2
+  cpu_request: 0.2
+  ephemeral_storage_limit: 500
+  ephemeral_storage_request: 500
+  memory_limit: 200
+  memory_request: 200
+  node:
+    capacity_type: spot_fallback_on_demand
+    type: node_selector
+type: service

truefoundry-0.5.8rc4/tests/itest/examples/deploy/workflow/example1/workflow.py

@@ -0,0 +1,48 @@
+from truefoundry.deploy import Resources
+from truefoundry.workflow import PythonTaskConfig, TaskPythonBuild, task, workflow
+
+cpu_task_config = PythonTaskConfig(
+    # `TaskPythonBuild` helps specify the details of your Python Code.
+    # These details will be used to templatize a DockerFile to build your Docker Image
+    image=TaskPythonBuild(
+        python_version="3.9",
+        # Pip packages to install.`truefoundry[workflow]` is a mandatory dependency
+        pip_packages=["truefoundry[workflow]==0.4.8"],
+        # Optionally pass the path to requirements.txt
+        # requirements_path="requirements.txt"
+    ),
+    resources=Resources(cpu_request=0.5),
+)
+
+
+# Task 1: Generate a list of numbers
+@task(task_config=cpu_task_config)
+def generate_numbers(n: int) -> list[int]:
+    return list(range(1, n + 1))
+
+
+# Task 2: Calculate the square of each number in the list
+@task(task_config=cpu_task_config)
+def square_numbers(numbers: list[int]) -> list[int]:
+    return [x**2 for x in numbers]
+
+
+# Task 3: Sum the squared numbers
+@task(task_config=cpu_task_config)
+def sum_squares(squared_numbers: list[int]) -> int:
+    return sum(squared_numbers)
+
+
+# Workflow: Orchestrate the tasks
+@workflow
+def simple_pipeline(n: int) -> int:
+    numbers = generate_numbers(n=n)
+    squared_numbers = square_numbers(numbers=numbers)
+    result = sum_squares(squared_numbers=squared_numbers)
+    return result
+
+
+# Runs the pipeline locally
+if __name__ == "__main__":
+    result = simple_pipeline(n=5)
+    print(f"The sum of squares from 1 to 5 is: {result}")