konduktor-nightly 0.1.0.dev20251128104812__py3-none-any.whl

konduktor/utils/validator.py

@@ -0,0 +1,461 @@
+"""This module contains a custom validator for the JSON Schema specification.
+
+The main motivation behind extending the existing JSON Schema validator is to
+allow for case-insensitive enum matching since this is currently not supported
+by the JSON Schema specification.
+"""
+
+import base64
+import json
+import os
+import re
+import subprocess
+import time
+from pathlib import Path
+from typing import Tuple
+
+import jsonschema
+import requests
+from colorama import Fore, Style
+from filelock import FileLock
+
+from konduktor import logging
+
+SCHEMA_VERSION = 'v1.32.0-standalone-strict'
+SCHEMA_CACHE_PATH = Path.home() / '.konduktor/schemas'
+SCHEMA_LOCK_PATH = SCHEMA_CACHE_PATH / '.lock'
+CACHE_MAX_AGE_SECONDS = 86400  # 24 hours
+
+# Schema URLs for different Kubernetes resources
+SCHEMA_URLS = {
+    'podspec': f'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{SCHEMA_VERSION}/podspec.json',
+    'deployment': f'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{SCHEMA_VERSION}/deployment.json',
+    'service': f'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{SCHEMA_VERSION}/service.json',
+    'horizontalpodautoscaler': f'https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/{SCHEMA_VERSION}/horizontalpodautoscaler-autoscaling-v2.json',
+}
+
+logger = logging.get_logger(__name__)
+
+
+def _skip_image_checks() -> bool:
+    val = os.getenv('KONDUKTOR_SKIP_IMAGE_CHECK', '')
+    return val.lower() in ('1', 'true', 'yes', 'y')
+
+
+def case_insensitive_enum(validator, enums, instance, schema):
+    del validator, schema  # Unused.
+    if instance.lower() not in [enum.lower() for enum in enums]:
+        yield jsonschema.ValidationError(f'{instance!r} is not one of {enums!r}')
+
+
+SchemaValidator = jsonschema.validators.extend(
+    jsonschema.Draft7Validator,
+    validators={'case_insensitive_enum': case_insensitive_enum},
+)
+
+
+def get_cached_schema(schema_type: str) -> dict:
+    """Get cached schema for a specific Kubernetes resource type."""
+    schema_url = SCHEMA_URLS.get(schema_type)
+    if not schema_url:
+        raise ValueError(f'Unknown schema type: {schema_type}')
+
+    schema_file = SCHEMA_CACHE_PATH / f'{schema_type}.json'
+    lock = FileLock(str(SCHEMA_LOCK_PATH))
+
+    with lock:
+        # Check if schema file exists and is fresh
+        if schema_file.exists():
+            age = time.time() - schema_file.stat().st_mtime
+            # if fresh
+            if age < CACHE_MAX_AGE_SECONDS:
+                with open(schema_file, 'r') as f:
+                    return json.load(f)
+
+        # Download schema
+        resp = requests.get(schema_url)
+        resp.raise_for_status()
+
+        SCHEMA_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+        with open(schema_file, 'w') as f:
+            f.write(resp.text)
+
+        return resp.json()
+
+
+def _validate_k8s_spec(spec: dict, schema_type: str, resource_name: str) -> None:
+    """Generic validation function for Kubernetes specs."""
+    schema = get_cached_schema(schema_type)
+
+    validator = jsonschema.Draft7Validator(schema)
+    errors = sorted(validator.iter_errors(spec), key=lambda e: e.path)
+
+    if not errors:
+        return
+
+    formatted = [
+        f'- {error.message}'
+        + (f" at path: {' → '.join(str(p) for p in error.path)}" if error.path else '')
+        for error in errors
+    ]
+
+    # Clean log
+    logger.debug('Invalid k8s %s spec/config:\n%s', resource_name, '\n'.join(formatted))
+
+    # Color only in CLI
+    formatted_colored = [
+        f'{Fore.RED}- {error.message}'
+        + (f" at path: {' → '.join(str(p) for p in error.path)}" if error.path else '')
+        + Style.RESET_ALL
+        for error in errors
+    ]
+
+    raise ValueError(
+        f'\n{Fore.RED}Invalid k8s {resource_name} spec/config: {Style.RESET_ALL}\n'
+        + '\n'.join(formatted_colored)
+    )
+
+
+def validate_pod_spec(pod_spec: dict) -> None:
+    """Validate a Kubernetes pod spec."""
+    _validate_k8s_spec(pod_spec, 'podspec', 'pod')
+
+
+def validate_deployment_spec(deployment_spec: dict) -> None:
+    """Validate a Kubernetes deployment spec."""
+    _validate_k8s_spec(deployment_spec, 'deployment', 'deployment')
+
+
+def validate_service_spec(service_spec: dict) -> None:
+    """Validate a Kubernetes service spec."""
+    _validate_k8s_spec(service_spec, 'service', 'service')
+
+
+def validate_horizontalpodautoscaler_spec(hpa_spec: dict) -> None:
+    """Validate a Kubernetes HorizontalPodAutoscaler spec."""
+    _validate_k8s_spec(hpa_spec, 'horizontalpodautoscaler', 'horizontalpodautoscaler')
+
+
+def validate_docker_image(image_id: str) -> Tuple[str, str]:
+    """Validate if a Docker image exists and is accessible.
+
+    Args:
+        image_id: The Docker image ID to validate
+        (e.g., 'ubuntu:latest', 'gcr.io/project/image:tag')
+
+    Returns:
+        Tuple of (status, message) where status is:
+        - 'valid': Image definitely exists
+        - 'warning': Couldn't validate, but might be valid
+        - 'invalid': Image definitely doesn't exist
+    """
+    if not image_id or not isinstance(image_id, str):
+        return 'invalid', 'Image ID must be a non-empty string'
+
+    # Basic format validation
+    if not _is_valid_docker_image_format(image_id):
+        return 'invalid', f'Invalid Docker image format: {image_id}'
+
+    # Try registry API validation first (works without Docker daemon)
+    registry_result = _validate_image_in_registry(image_id)
+    if registry_result[0] in ['valid', 'invalid']:
+        return registry_result
+
+    # If registry validation couldn't determine, try local Docker as fallback
+    if _can_pull_image_locally(image_id):
+        return 'valid', f"Docker image '{image_id}' validated locally"
+
+    # Return the registry result (warning)
+    return registry_result
+
+
+def _is_valid_docker_image_format(image_id: str) -> bool:
+    """Check if the image ID follows valid Docker image naming conventions."""
+    # Basic regex for Docker image names
+    # Supports: name:tag, registry/name:tag, registry/namespace/name:tag
+    pattern = (
+        r'^[a-zA-Z0-9][a-zA-Z0-9._-]*'
+        r'(?:\/[a-zA-Z0-9][a-zA-Z0-9._-]*)*'
+        r'(?::[a-zA-Z0-9._-]+)?$'
+    )
+    return bool(re.match(pattern, image_id))
+
+
+def _can_pull_image_locally(image_id: str) -> bool:
+    """Try to inspect the image manifest locally to check if it exists."""
+    try:
+        # Use docker manifest inspect instead of pull for faster validation
+        result = subprocess.run(
+            ['docker', 'manifest', 'inspect', image_id],
+            capture_output=True,
+            text=True,
+            timeout=30,  # 30 second timeout
+        )
+
+        # Debug logging
+        logger.debug(
+            f'Local Docker manifest inspect for {image_id}: '
+            f'returncode={result.returncode}, '
+            f"stdout='{result.stdout}', "
+            f"stderr='{result.stderr}'"
+        )
+
+        return result.returncode == 0
+    except (
+        subprocess.TimeoutExpired,
+        FileNotFoundError,
+        subprocess.SubprocessError,
+    ) as e:
+        # Docker not available or timeout
+        logger.debug(f'Local Docker manifest inspect failed for {image_id}: {e}')
+        return False
+
+
+def _validate_image_in_registry(image_id: str) -> Tuple[str, str]:
+    """Validate image exists in registry using API calls."""
+    try:
+        registry, repo, tag = _parse_image_components(image_id)
+
+        if registry == 'docker.io':
+            return _validate_dockerhub_image(repo, tag)
+        elif registry.endswith('gcr.io'):
+            return _validate_gcr_image(registry, repo, tag)
+        elif registry.endswith('ecr.') and '.amazonaws.com' in registry:
+            return _validate_ecr_image(registry, repo, tag)
+        elif registry == 'nvcr.io':
+            return _validate_nvcr_image(registry, repo, tag)
+        elif registry == 'ghcr.io':
+            return _validate_ghcr_image(registry, repo, tag)
+        elif registry == 'quay.io':
+            return _validate_quay_image(registry, repo, tag)
+        else:
+            # For other registries, we can't easily validate without credentials
+            # Return warning that we couldn't verify
+            return (
+                'warning',
+                f"Could not validate '{image_id}' in registry {registry} "
+                f'(not supported)',
+            )
+
+    except Exception as e:
+        logger.debug(f'Error validating image {image_id}: {e}')
+        return 'warning', f"Could not validate '{image_id}' due to validation error"
+
+
+def _parse_image_components(image_id: str) -> Tuple[str, str, str]:
+    """Parse image ID into registry, repository, and tag components."""
+    # Default to Docker Hub
+    if '/' not in image_id or '.' not in image_id.split('/')[0]:
+        registry = 'docker.io'
+        # For Docker Hub official images (single word), add 'library/' prefix
+        if ':' in image_id:
+            repo, tag = image_id.rsplit(':', 1)
+        else:
+            repo = image_id
+            tag = 'latest'
+        # Only add 'library/' prefix for single-word official images
+        if '/' not in repo:
+            repo = f'library/{repo}'
+    else:
+        parts = image_id.split('/')
+        if '.' in parts[0] or parts[0] in ['localhost']:
+            registry = parts[0]
+            repo = '/'.join(parts[1:])
+        else:
+            registry = 'docker.io'
+            repo = image_id
+
+        # Split repository and tag
+        if ':' in repo:
+            repo, tag = repo.rsplit(':', 1)
+        else:
+            tag = 'latest'
+
+    return registry, repo, tag
+
+
+def _validate_dockerhub_image(repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in Docker Hub using the official API."""
+    try:
+        # Use Docker Hub's official API v2 endpoint
+        # This endpoint checks if a specific tag exists for a repository
+        url = f'https://registry.hub.docker.com/v2/repositories/{repo}/tags/{tag}'
+
+        # Add User-Agent to avoid being blocked
+        headers = {'User-Agent': 'Konduktor-Docker-Validator/1.0'}
+
+        response = requests.get(url, headers=headers, timeout=10)
+
+        if response.status_code == 200:
+            return 'valid', f"Docker image '{repo}:{tag}' validated via Docker Hub"
+        else:
+            # API error, can't determine
+            return ('warning', f"Could not validate '{repo}:{tag}' in Docker Hub")
+
+    except requests.RequestException:
+        # Network error, can't determine
+        return (
+            'warning',
+            f"Could not validate '{repo}:{tag}' in Docker Hub " f'(network error)',
+        )
+
+
+def _validate_gcr_image(registry: str, repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in Google Container Registry."""
+    try:
+        # GCR manifest endpoint
+        url = f'https://{registry}/v2/{repo}/manifests/{tag}'
+        response = requests.get(url, timeout=10)
+
+        if response.status_code == 200:
+            return 'valid', f"Docker image '{repo}:{tag}' validated via {registry}"
+        else:
+            # API error, can't determine
+            return ('warning', f"Could not validate '{repo}:{tag}' in {registry} ")
+
+    except requests.RequestException:
+        # Network error, can't determine
+        return (
+            'warning',
+            f"Could not validate '{repo}:{tag}' in {registry} " f'(network error)',
+        )
+
+
+def _validate_ecr_image(registry: str, repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in Amazon ECR."""
+    # ECR requires AWS credentials and is complex to validate
+    # For now, return warning that we couldn't verify
+    return ('warning', f"Could not validate '{repo}:{tag}' in {registry}")
+
+
+def _validate_nvcr_image(registry: str, repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in NVIDIA Container Registry."""
+    # NVCR requires NVIDIA credentials and is complex to validate
+    # For now, return warning that we couldn't verify
+    return ('warning', f"Could not validate '{repo}:{tag}' in {registry}")
+
+
+def _validate_ghcr_image(registry: str, repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in GitHub Container Registry."""
+    try:
+        # Check if GITHUB_TOKEN is available
+        github_token = os.environ.get('GITHUB_TOKEN')
+
+        # If not in environment, try to get from konduktor secrets
+        if not github_token:
+            try:
+                # these imports are inside the try block to avoid circular import error
+                from konduktor.backends import constants as backend_constants
+                from konduktor.utils import common_utils, kubernetes_utils
+
+                context = kubernetes_utils.get_current_kube_config_context_name()
+                namespace = kubernetes_utils.get_kube_config_context_namespace(context)
+                user_hash = common_utils.get_user_hash()
+                label_selector = f'{backend_constants.SECRET_OWNER_LABEL}={user_hash}'
+                user_secrets = kubernetes_utils.list_secrets(
+                    namespace, context, label_filter=label_selector
+                )
+
+                for secret in user_secrets:
+                    kind = kubernetes_utils.get_secret_kind(secret)
+                    if kind == 'env' and secret.data and 'GITHUB_TOKEN' in secret.data:
+                        # Decode the base64 encoded token
+                        github_token = base64.b64decode(
+                            secret.data['GITHUB_TOKEN']
+                        ).decode()
+                        logger.debug('GITHUB_TOKEN found in konduktor secret')
+                        break
+
+            except Exception as e:
+                logger.debug(f'Failed to check konduktor secrets: {e}')
+
+        if not github_token:
+            return (
+                'warning',
+                'GITHUB_TOKEN unset, cannot verify this image. '
+                'To enable validation, either:\n'
+                '  1. Set GITHUB_TOKEN locally: export GITHUB_TOKEN=<token>\n'
+                '  2. Create a secret: konduktor secret create --kind=env '
+                '--inline GITHUB_TOKEN=<token> <name>\n'
+                'See: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry',
+            )
+
+        # Base64 encode the token
+        ghcr_token = base64.b64encode(github_token.encode()).decode()
+
+        # GHCR manifest endpoint
+        url = f'https://{registry}/v2/{repo}/manifests/{tag}'
+        headers = {'Authorization': f'Bearer {ghcr_token}'}
+        response = requests.get(url, headers=headers, timeout=10)
+
+        if response.status_code == 200:
+            return 'valid', f"Docker image '{repo}:{tag}' validated via {registry}"
+        else:
+            # API error, can't determine
+            return ('warning', f"Could not validate '{repo}:{tag}' in {registry}")
+
+    except requests.RequestException:
+        # Network error, can't determine
+        return (
+            'warning',
+            f"Could not validate '{repo}:{tag}' in {registry} " f'(network error)',
+        )
+
+
+def _validate_quay_image(registry: str, repo: str, tag: str) -> Tuple[str, str]:
+    """Validate image exists in Quay.io Container Registry."""
+    # Quay.io requires authentication and is complex to validate
+    # For now, return warning that we couldn't verify
+    return ('warning', f"Could not validate '{repo}:{tag}' in {registry}")
+
+
+# Track which images we've already warned about to avoid duplicate warnings
+_warned_images = set()
+
+
+def validate_and_warn_image(image_id: str, context: str = 'task') -> None:
+    """Validate Docker image and show appropriate warnings.
+
+    Args:
+        image_id: The Docker image ID to validate
+        context: Context for the validation (e.g., "task", "deployment")
+
+    """
+    if not image_id:
+        return
+
+    if _skip_image_checks():
+        logger.info(
+            'Skipping Docker image validation for %s',
+            image_id,
+        )
+        return
+
+    status, message = validate_docker_image(image_id)
+
+    if status == 'invalid':
+        # Invalid images should fail - they definitely don't exist
+        raise ValueError(
+            f'{message}\n'
+            f'This Docker image does not exist and will cause the {context} to fail.\n'
+            f"Please check that the image '{image_id}' is correct and accessible.\n"
+        )
+    elif status == 'warning':
+        # Only warn once per image per session for warnings
+        if image_id not in _warned_images:
+            _warned_images.add(image_id)
+
+            logger.warning(
+                f'⚠️  Basic public image validation using Docker Daemon failed.  ⚠️\n'
+                f'⚠️  {message} ⚠️\n'
+                f'⚠️  The {context} will be submitted anyway, but may be stuck '
+                f'PENDING forever. ⚠️\n'
+                f"⚠️  Check for 'ErrImagePull' or 'ImagePullBackOff' in "
+                f'kubectl get pods if issues occur. ⚠️'
+            )
+
+            # Add info about private registries
+            logger.info(
+                '⚠️  If pulling from a private registry, using ecr/nvcr, or not '
+                'logged into Docker, this is safe to ignore. ⚠️'
+            )

konduktor_nightly-0.1.0.dev20251128104812.dist-info/LICENSE

@@ -0,0 +1,91 @@
+# Trainy Software License
+## Version 1.0
+
+Last Updated: June 12, 2024
+Copyright © 2024 Trainy, Inc.
+
+This Trainy Software License (“License”), effective the date we first make the Software available to you (“Effective Date”) is between the individual or legal entity exercising permissions granted by this License (“you” or “your”) and Trainy, Inc. (“Trainy” or “we” or “us” or “our”) concerning your use of the work of authorship, whether in source or object form, made available under this License, as indicated by a copyright notice that is included in or attached to the work (the “Software”). 
+
+# Authority
+If you use the Software on behalf of another person or legal entity, (a) all references to “you” throughout this License will include that person or legal entity, (b) you represent that you are authorized to accept this License on that person’s or legal entity’s behalf, and (c) in the event you or the person or legal entity violates this License, the person or legal entity agrees to be responsible to us.
+ 
+# Licenses.
+
+## Copyright License.
+
+Subject to your compliance with this License, Trainy hereby grants you a non-exclusive, royalty-free copyright license to reproduce, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any purpose that is not a Restriction (as defined below).
+
+## Limited Patent License.
+Subject to your compliance with this License, the license grant above includes a license under our Applicable Patents. Notwithstanding the foregoing, If you initiate any claim, action or proceeding against any third party (including cross-claims or counterclaims) alleging that all or any portion of the Software infringes the patents of any entity or other person, the license granted under this Section 2.b will immediately terminate automatically without further action by either party. “Applicable Patents” are any of patents licensable by us that will be necessarily infringed by using the Software.
+
+## Restrictions.
+A “Restriction” means distributing or otherwise commercially making the Software available in a manner that provides functionality to address the same or substantially similar user requirements as the Software or includes the same or similar functionality as the Software. Without limiting the foregoing, you may use the Software for your internal purposes and for non-commercial purposes. We grant you no rights except as expressly set forth in this License.
+
+## Derivatives; Marking
+If you distribute any copies, improvements, modifications or derivatives (“Derivatives”) of the Software, this License will apply to you must include a copy of or a link to this License and include with such Derivatives any copyright, patent, trademark and attribution notices provided in or with the Software. You may reproduce our trademarks, trade names, service marks and other similar indicia (“Trademarks”) solely to the extent necessary to reproduce and/or link to applicable notices as provided in this Section 4. Except as provided in the prior sentence, we grant you no rights in or to our Trademarks, whether expressly, by implication, estoppel or otherwise.
+
+## Disclaimer. 
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES, EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+## Apache License.
+Subject to your compliance with this License, Trainy hereby grants you a license to use the Software under the Apache License, Version 2.0, currently available at http://www.apache.org/licenses/LICENSE-2.0 (the “Apache License”), provided that you hereby covenant and agree not to use the Software under the Apache License until the third anniversary of the Effective Date. On or after such date, you may continue to use the Software under the terms of this License, or you may use the Software under the terms and conditions of, and in compliance with, the Apache License. 
+
+--------------------------------------------------------------------------------
+
+The following files were modified from:
+
+Code in manifests/kube-prometheus-stack.values modified from
+https://github.com/prometheus-community/helm-charts
+Git Revision: db15dcf70d06da9c71454f7d9757b47b04d2c5ae
+
+Copyright 2024 Prometheus community Helm charts developers
+
+Code in manifests/kube-prometheus-stack.values modified from
+https://github.com/prometheus-community/helm-charts
+Git Revision: db15dcf70d06da9c71454f7d9757b47b04d2c5ae
+
+Code in:
+   manifests/manifests.yaml 
+   manifests/single-clusterqueue-setup.yaml
+modified from
+https://github.com/kubernetes-sigs/kueue/
+Git Revision: bf050331ad8917afa3e3ddc92e3c8153b2a654d2
+
+Copyright 2022 The Kubernetes Authors
+
+Code in grafana/ modified from
+https://github.com/NVIDIA/dcgm-exporter/
+Git Revision: 478fab1573c7a1f0db4358357733967e4ec52200
+
+Copyright 2022 NVIDIA
+
+The original files are licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+The modifications are proprietary and subject to the terms of the Trainy Software License Version 1.0
+Copyright 2024 Trainy Inc.
+
+Code is modified from https://github.com/skypilot-org/skypilot
+
+The original files are licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+The modifications are proprietary and subject to the terms of the Trainy Software License Version 1.0
+Copyright 2024 Trainy Inc.

konduktor_nightly-0.1.0.dev20251128104812.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.3
+Name: konduktor-nightly
+Version: 0.1.0.dev20251128104812
+Summary: GPU Cluster Health Management
+Author: Andrew Aikawa
+Author-email: asai@berkeley.edu
+Requires-Python: >=3.9,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: s3
+Requires-Dist: awscli[s3] (>=1.32.84,<2.0.0) ; extra == "s3"
+Requires-Dist: boto3[s3] (>=1.34.84,<2.0.0) ; extra == "s3"
+Requires-Dist: botocore[s3] (>=1.34.84,<2.0.0) ; extra == "s3"
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: colorama (>=0.4.6,<0.5.0)
+Requires-Dist: filelock (>=3.18.0,<4.0.0)
+Requires-Dist: google-api-python-client[gcp] (>=2.161.0,<3.0.0)
+Requires-Dist: google-cloud-storage[gcp] (>=3.0.0,<4.0.0)
+Requires-Dist: jinja2 (>=3.1.5,<4.0.0)
+Requires-Dist: jsonschema (>=4.23.0,<5.0.0)
+Requires-Dist: kr8s (>=0.20.1,<0.21.0)
+Requires-Dist: kubernetes (>=30.1.0,<31.0.0)
+Requires-Dist: posthog (>=3.7.4,<4.0.0)
+Requires-Dist: prettytable (>=3.12.0,<4.0.0)
+Requires-Dist: psutil (>=7.0.0,<8.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: rich (>=13.9.4,<14.0.0)
+Requires-Dist: websockets (>=15.0.1,<16.0.0)
+Description-Content-Type: text/markdown
+
+
+<p align="center">
+  <picture>
+    <img alt="Trainy Konduktor Logo" src="https://raw.githubusercontent.com/Trainy-ai/konduktor/main/docs/source/images/konduktor-logo-white-no-background.png" width="353" height="64" style="max-width: 100%;">
+  </picture>
+  <br/>
+  <br/>
+</p>
+
+Built on [Kubernetes](https://kubernetes.io). Konduktor uses existing open source tools to build a platform that makes it easy for ML Researchers to submit batch jobs and for administrative/infra teams to easily manage GPU clusters.
+
+## How it works
+
+Konduktor uses a combination of open source projects. Where tools exist with MIT, Apache, or another compatible open license, we want to use and even contribute to that tool. Where we see gaps in tooling, we build it.
+
+### Architecture
+
+Konduktor can be self-hosted and run on any certified Kubernetes distribution or managed by us. Contact us at founders@trainy.ai if you are just interested in the managed version. We're focused on tooling for clusters with NVIDIA cards for now but in the future we may expand to our scope to support other accelerators.
+
+<p align="center">
+  <img alt="architecture" src="https://raw.githubusercontent.com/Trainy-ai/konduktor/main/docs/source/images/architecture.png" width=80%>
+</p>
+
+For ML researchers
+- Konduktor CLI & SDK - user friendly batch job framework, where users only need to specify the resource requirements of their job and a script to launch that makes simple to scale work across multiple nodes. Works with most ML application frameworks out of the box.
+
+```
+num_nodes: 100
+
+resources:
+  accelerators: H100:8
+  cloud: kubernetes
+  labels:
+    kueue.x-k8s.io/queue-name: user-queue
+    kueue.x-k8s.io/priority-class: low-priority
+
+run: |
+  torchrun \
+  --nproc_per_node 8 \
+  --rdzv_id=1 --rdzv_endpoint=$master_addr:1234 \
+  --rdzv_backend=c10d --nnodes $num_nodes \
+  torch_ddp_benchmark.py --distributed-backend nccl
+```
+
+For cluster administrators
+- [DCGM Exporter](https://github.com/NVIDIA/dcgm-exporter), [GPU operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/), [Network Operator](https://github.com/Mellanox/network-operator) - For installing NVIDIA driver, container runtime, and exporting node health metrics.
+- [Kueue](https://kueue.sigs.k8s.io/docs/) - centralized creation of job queues, gang-scheduling, and resource quotas and sharing across projects.
+- [Prometheus](https://prometheus.io/) - For publishing metrics about node health and workload queues.
+- [OpenTelemetry](https://opentelemetry.io/) - For pushing logs from each node
+- [Grafana, Loki](https://grafana.com/) - Visualizations for metrics/logging solution.
+
+
+## Community & Support
+- [Discord](https://discord.com/invite/HQUBJSVgAP)
+- founders@trainy.ai
+
+## Contributor Guide
+
+Format your code with
+```
+poetry install --with dev
+bash format.sh
+```
+