konduktor-nightly 0.1.0.dev20251128104812__py3-none-any.whl

konduktor/controller/parse.py

@@ -0,0 +1,111 @@
+import os
+import re
+from typing import Any, Dict, List, Set
+
+import requests
+
+from konduktor import logging as konduktor_logging
+from konduktor.controller import constants
+
+# comma separated list of namespaces to watch for pod errors
+WATCHED_NAMESPACES: List[str] = os.environ.get('WATCHED_NAMESPACES', 'default').split(
+    ','
+)
+LOGS_SINCE: int = 10  # retrieves logs generated in the past 10 seconds
+LOG_ENDPOINT: str = os.environ.get(
+    'LOG_ENDPOINT',
+    # this assumes you have access to this endpoint by
+    # running as a deployment within the cluster
+    # for local development use 'http://localhost:3100' and
+    # kubectl port-forward svc/loki -n loki 3100:3100
+    'http://loki.loki.svc.cluster.local:3100',
+)
+QUERY_URL: str = '/loki/api/v1/query_range'
+
+logger = konduktor_logging.get_logger(__name__)
+
+
+def _query_range(pattern: str, **label_filters) -> List[Dict[str, Any]]:
+    """Send LogQL query_range to loki
+    https://grafana.com/docs/loki/latest/reference/loki-http-api/#query-logs-within-a-range-of-time
+
+    Args:
+        pattern (str): regex pattern to match loglines against
+
+    Returns:
+        List[Dict[str, Any]]: List of loglines
+    """
+    url = f'{LOG_ENDPOINT}{QUERY_URL}'
+    formatted_filters = ', '.join(
+        f'{key}="{value}"' for key, value in label_filters.items()
+    )
+    query = r'{' f'{formatted_filters}' r'}' f'|~ {pattern}'
+    params = {'query': query, 'since': f'{LOGS_SINCE}s'}
+    response = requests.get(url, params=params)
+    if response.status_code == 200:
+        data = response.json()
+        return data['data']['result']
+    elif response.status_code == 400:
+        logger.error(f'Bad Request: {response.status_code}')
+        logger.error(response.json())  # Optionally print the error details
+    else:
+        logger.error(f'loki query failed {params}')
+    return []
+
+
+def pod_errors() -> Set[str]:
+    logger.info('querying pod logs')
+    bad_nodes = set()
+    for regex in constants.POD_LOG_ERROR_REGEXES:
+        for namespace in WATCHED_NAMESPACES:
+            log_lines = _query_range(regex, k8s_namespace_name=namespace)
+            for line in log_lines:
+                log_node = line['stream']['k8s_node_name']
+                bad_nodes.add(log_node)
+    return bad_nodes
+
+
+def sxid_error(pattern: str, log_content: str) -> int:
+    """Regex pattern match for an xid error, from `log_content` otherwise return 0
+    Example Xid error from dmesg
+    [1235733.431527] NVRM: Xid (PCI:0000:4e:00): 79, pid='<unknown>', name=<unknown>, GPU has fallen off the bus.
+    Example sxid error from dmesg
+    [1235733.431527] nvidia-nvswitch3: SXid (PCI:0000:4e:00.0): 12028, Non-fatal, Link 32 egress non-posted PRIV error (First)
+    """  # noqa: E501
+
+    match = re.search(pattern, log_content)
+    if match:
+        return int(match.group(1))
+
+    return 0
+
+
+def is_sxid_error(log_content: str) -> bool:
+    """Returns (S)Xid error code, zero otherwise"""
+    error_code = sxid_error(r'SXid.*?: (\d+),', log_content) or sxid_error(
+        r'NVRM: Xid.*?: (\d+),', log_content
+    )
+    return error_code not in constants.ALLOWLISTED_NVSWITCH_SXID_ERRORS
+
+
+def dmesg_errors() -> Set[str]:
+    logger.info('checking dmesg logs')
+    pattern = ' or '.join(constants.DMESG_ERROR_REGEXES)
+    log_lines = _query_range(pattern, k8s_daemonset_name='dmesg')
+    bad_nodes = set()
+    for line in log_lines:
+        log_node, log_content = line['stream']['k8s_node_name'], line['values'][0][1]
+        if is_sxid_error(log_content):
+            logger.info(f'node `{log_node}` has (S)Xid error: {log_content}')
+        else:
+            logger.info(f'dmesg error on node `{log_node}`: {log_content}')
+        bad_nodes.add(log_node)
+    return bad_nodes
+
+
+if __name__ == '__main__':
+    import time
+
+    while True:
+        time.sleep(5)
+        print(dmesg_errors())

konduktor/dashboard/README.md

@@ -0,0 +1,30 @@
+### Prereqs: kubectl is configured with remote machine/cluster
+
+# OPTION 1 (Automated Setup)
+
+To open the dashboard, run this inside the root konduktor directory:
+```
+./start_dashboard.sh
+```
+
+# OPTION 2 (Manual Setup)
+
+## 1. Apply kubernetes manifest
+Inside manifests directory (one with dashboard_deployment.yaml):
+```
+kubectl apply -f dashboard_deployment.yaml
+```
+Then, wait a minute or two for the pods to finish setup
+
+## 2. Port forward frontend in a terminal
+```
+kubectl port-forward svc/frontend 5173:5173 -n konduktor-dashboard
+```
+
+## 3. Port forward grafana in a terminal
+```
+kubectl port-forward svc/kube-prometheus-stack-grafana 3000:80 -n prometheus
+```
+
+## 4. Open dashboard at http://localhost:5173/
+

konduktor/dashboard/backend/main.py

@@ -0,0 +1,169 @@
+from typing import Any, Dict, List
+
+import socketio
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from kubernetes import client
+from kubernetes.client.exceptions import ApiException
+
+from konduktor import logging as konduktor_logging
+from konduktor.kube_client import batch_api, core_api, crd_api
+
+from .sockets import socketio as sio
+
+logger = konduktor_logging.get_logger(__name__)
+
+# FastAPI app
+app = FastAPI()
+
+
+# CORS Configuration
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=['*'],  # Allow all origins
+    allow_credentials=True,
+    allow_methods=['*'],  # Allow all methods
+    allow_headers=['*'],  # Allow all headers
+)
+
+# Use Kubernetes API clients
+# Initialize BatchV1 and CoreV1 API (native kubernetes)
+batch_client = batch_api()
+core_client = core_api()
+# Initialize Kueue API
+crd_client = crd_api()
+
+
+@app.get('/')
+async def home():
+    return JSONResponse({'home': '/'})
+
+
+@app.delete('/deleteJob')
+async def delete_job(request: Request):
+    data = await request.json()
+    name = data.get('name', '')
+    namespace = data.get('namespace', 'default')
+
+    try:
+        delete_options = client.V1DeleteOptions(propagation_policy='Background')
+
+        crd_client.delete_namespaced_custom_object(
+            group='kueue.x-k8s.io',
+            version='v1beta1',
+            namespace=namespace,
+            plural='workloads',
+            name=name,
+            body=delete_options,
+        )
+        logger.debug(f"Kueue Workload '{name}' deleted successfully.")
+
+        return JSONResponse({'success': True, 'status': 200})
+
+    except ApiException as e:
+        logger.debug(f'Exception: {e}')
+        return JSONResponse({'error': str(e)}, status_code=e.status)
+
+
+@app.get('/getJobs')
+async def get_jobs():
+    rows = fetch_jobs()
+    return JSONResponse(rows)
+
+
+@app.get('/getNamespaces')
+async def get_namespaces():
+    try:
+        # Get the list of namespaces
+        namespaces = core_client.list_namespace()
+        # Extract the namespace names from the response
+        namespace_list = [ns.metadata.name for ns in namespaces.items]
+        return JSONResponse(namespace_list)
+    except ApiException as e:
+        logger.debug(f'Exception: {e}')
+        return JSONResponse({'error': str(e)}, status_code=e.status)
+
+
+@app.put('/updatePriority')
+async def update_priority(request: Request):
+    data = await request.json()
+    name = data.get('name', '')
+    namespace = data.get('namespace', 'default')
+    priority = data.get('priority', 0)
+
+    try:
+        job = crd_client.get_namespaced_custom_object(
+            group='kueue.x-k8s.io',
+            version='v1beta1',
+            namespace=namespace,
+            plural='workloads',
+            name=name,
+        )
+
+        job['spec']['priority'] = priority
+
+        crd_client.patch_namespaced_custom_object(
+            group='kueue.x-k8s.io',
+            version='v1beta1',
+            namespace=namespace,
+            plural='workloads',
+            name=name,
+            body=job,
+        )
+        return JSONResponse({'success': True, 'status': 200})
+
+    except ApiException as e:
+        logger.debug(f'Exception: {e}')
+        return JSONResponse({'error': str(e)}, status_code=e.status)
+
+
+# Get a listing of workloads in kueue
+def fetch_jobs():
+    listing = crd_client.list_namespaced_custom_object(
+        group='kueue.x-k8s.io',
+        version='v1beta1',
+        namespace='default',
+        plural='workloads',
+    )
+
+    return format_workloads(listing)
+
+
+def format_workloads(listing: Dict[str, Any]) -> List[Dict[str, Any]]:
+    if not listing:
+        return []
+
+    res = []
+
+    for job in listing['items']:
+        id = job['metadata']['uid']
+        name = job['metadata']['name']
+        created_at = job['metadata']['creationTimestamp']
+        namespace = job['metadata']['namespace']
+        localQueueName = job['spec'].get('queueName', 'Unknown')
+        priority = job['spec']['priority']
+        active = job['spec'].get('active', 0)
+        status = 'ADMITTED' if 'admission' in job.get('status', {}) else 'PENDING'
+
+        statusVal = 1 if 'admission' in job.get('status', {}) else 0
+        order = (statusVal * 10) + priority
+
+        res.append(
+            {
+                'id': id,
+                'name': name,
+                'namespace': namespace,
+                'localQueueName': localQueueName,
+                'priority': priority,
+                'status': status,
+                'active': active,
+                'created_at': created_at,
+                'order': order,
+            }
+        )
+
+    return res
+
+
+app = socketio.ASGIApp(sio, app)

konduktor/dashboard/backend/sockets.py

@@ -0,0 +1,154 @@
+import asyncio
+import datetime
+import os
+import time
+from typing import Dict, List
+
+import requests
+from socketio import AsyncServer  # Import the AsyncServer for ASGI compatibility
+
+from konduktor import logging as konduktor_logging
+
+# SocketIO configuration
+socketio = AsyncServer(
+    cors_allowed_origins='*', ping_interval=25, ping_timeout=60, async_mode='asgi'
+)
+
+logger = konduktor_logging.get_logger(__name__)
+
+# Global variables
+CLIENT_CONNECTED = False
+FIRST_RUN = True
+BACKGROUND_TASK_RUNNING = False
+LOG_CHECKPOINT_TIME = None
+SELECTED_NAMESPACES: list[str] = []
+
+# "http://loki.loki.svc.cluster.local:3100/loki/api/v1/query_range" for prod
+# "http://localhost:3100/loki/api/v1/query_range" for local
+LOGS_URL = os.environ.get('LOGS_URL', 'http://localhost:3100/loki/api/v1/query_range')
+
+
+def format_log_entry(entry: List[str], namespace: str) -> Dict[str, str]:
+    """
+    Formats a log entry and its corresponding namespace
+
+    Args:
+        entry (List[str]): A list of log entry strings to be formatted.
+        namespace (str): The namespace to apply to each log entry.
+
+    Returns:
+        Dict[str, str]: an object with the following properties:
+        timestamp, log (message), and namespace
+    """
+    timestamp_ns = entry[0]
+    log_message = entry[1]
+    timestamp_s = int(timestamp_ns) / 1e9
+    dt = datetime.datetime.utcfromtimestamp(timestamp_s)
+    human_readable_time = dt.strftime('%Y-%m-%d %H:%M:%S')
+    formatted_log = {
+        'timestamp': human_readable_time,
+        'log': log_message,
+        'namespace': namespace,
+    }
+    return formatted_log
+
+
+def get_logs(FIRST_RUN: bool) -> List[Dict[str, str]]:
+    global LOG_CHECKPOINT_TIME
+
+    logger.debug(f'Selected namespaces: {SELECTED_NAMESPACES}')
+
+    # Use the selected namespaces in the query
+    namespace_filter = (
+        '|'.join(SELECTED_NAMESPACES) if SELECTED_NAMESPACES else 'default'
+    )
+    query = f'{{k8s_namespace_name=~"{namespace_filter}"}}'
+
+    logger.debug(f'Loki logs query: {query}')
+
+    if FIRST_RUN:
+        # Calculate how many nanoseconds to look back when first time looking at logs
+        # (currently 1 hour)
+        now = int(time.time() * 1e9)
+        one_hour_ago = now - int(3600 * 1e9)
+        start_time = str(one_hour_ago)
+    else:
+        # calculate new start_time based on newest, last message
+        if LOG_CHECKPOINT_TIME is None:
+            LOG_CHECKPOINT_TIME = 0
+        start_time = str(int(LOG_CHECKPOINT_TIME) + 1)
+
+    params = {'query': query, 'start': start_time, 'limit': '300'}
+
+    url = LOGS_URL
+    response = requests.get(url, params=params)
+    formatted_logs = []
+
+    last = 0
+
+    if response.status_code == 200:
+        data = response.json()
+        rows = data['data']['result']
+
+        for row in rows:
+            namespace = row['stream']['k8s_namespace_name']
+            for value in row['values']:
+                last = max(int(value[0]), last)
+                formatted_logs.append(format_log_entry(value, namespace))
+
+    if formatted_logs:
+        # sort because sometimes loki API is wrong and logs are out of order
+        formatted_logs.sort(
+            key=lambda log: datetime.datetime.strptime(
+                log['timestamp'], '%Y-%m-%d %H:%M:%S'
+            )
+        )
+        LOG_CHECKPOINT_TIME = last
+
+    logger.debug(f'Formatted logs length: {len(formatted_logs)}')
+
+    return formatted_logs
+
+
+async def send_logs():
+    global CLIENT_CONNECTED, FIRST_RUN, BACKGROUND_TASK_RUNNING
+    while CLIENT_CONNECTED:
+        logs = get_logs(FIRST_RUN)
+
+        FIRST_RUN = False  # After the first successful fetch, set to False
+        if logs:
+            await socketio.emit('log_data', logs)
+
+        await asyncio.sleep(5)
+
+    # Background task is no longer running after the loop
+    BACKGROUND_TASK_RUNNING = False
+
+
+@socketio.event
+async def connect(sid, environ):
+    global CLIENT_CONNECTED, FIRST_RUN, BACKGROUND_TASK_RUNNING
+    CLIENT_CONNECTED = True
+    FIRST_RUN = True
+    logger.debug('Client connected')
+
+    # Start the background task only if it's not already running
+    if not BACKGROUND_TASK_RUNNING:
+        BACKGROUND_TASK_RUNNING = True
+        socketio.start_background_task(send_logs)
+
+
+@socketio.event
+async def update_namespaces(sid, namespaces):
+    global SELECTED_NAMESPACES
+    SELECTED_NAMESPACES = namespaces
+    logger.debug('Updated namespaces')
+
+
+@socketio.event
+async def disconnect(sid):
+    global CLIENT_CONNECTED, FIRST_RUN, BACKGROUND_TASK_RUNNING
+    CLIENT_CONNECTED = False
+    FIRST_RUN = True
+    BACKGROUND_TASK_RUNNING = False
+    logger.debug('Client disconnected')

konduktor/dashboard/frontend/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

konduktor/dashboard/frontend/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

konduktor/dashboard/frontend/app/api/jobs/route.js

@@ -0,0 +1,71 @@
+import { NextResponse } from 'next/server';
+
+const backendUrl = process.env.NODE_ENV === 'development'
+    ? 'http://127.0.0.1:5001' // Development API
+    : 'http://backend:5001' // Production API
+
+// GET request for jobs
+export async function GET() {
+    try {
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/getJobs`, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+        })
+    
+        const data = await response.json()
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server get error:", error);
+        return new NextResponse(error)
+    }
+}
+
+// DELETE request for job deletion
+export async function DELETE(req) {
+    try {
+        const { name, namespace } = await req.json(); // Parse the request body
+        
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/deleteJob`, {
+            method: 'DELETE',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ name, namespace })
+        })
+    
+        const data = await response.json()
+        console.log(`Server Component deleteJob: ${JSON.stringify(data)}`)
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server delete error:", error);
+        return new NextResponse(error)
+    }
+}
+
+// PUT request for updating job priority
+export async function PUT(req) {
+    try {
+        const { name, namespace, priority, priority_class_name } = await req.json(); // Parse the request body
+
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/updatePriority`, {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ name, namespace, priority, priority_class_name })
+        })
+    
+        const data = await response.json()
+        console.log(`Server Component updatePriority: ${JSON.stringify(data)}`)
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server update error:", error);
+        return new NextResponse(error)
+    }
+}
+  

konduktor/dashboard/frontend/app/api/namespaces/route.js

@@ -0,0 +1,69 @@
+import { NextResponse } from 'next/server';
+
+const backendUrl = process.env.NODE_ENV === 'development'
+    ? 'http://127.0.0.1:5001' // Development API
+    : 'http://backend:5001' // Production API
+
+// GET request for jobs
+export async function GET() {
+    try {
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/getNamespaces`, {
+            method: 'GET',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+        })
+    
+        const data = await response.json()
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server get error:", error);
+        return new NextResponse(error)
+    }
+}
+
+// DELETE request for job deletion
+export async function DELETE(req) {
+    try {
+        const { name, namespace } = await req.json(); // Parse the request body
+
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/deleteJob`, {
+            method: 'DELETE',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ name, namespace })
+        })
+
+        const data = await response.json()
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server delete error:", error);
+        return new NextResponse(error)
+    }
+}
+
+// PUT request for updating job priority
+export async function PUT(req) {
+    try {
+        const { name, namespace, priority, priority_class_name } = await req.json(); // Parse the request body
+
+        // Forward request to backend API
+        const response = await fetch(`${backendUrl}/updatePriority`, {
+            method: 'PUT',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({ name, namespace, priority, priority_class_name })
+        })
+
+        const data = await response.json()
+        return new NextResponse(JSON.stringify(data))
+    } catch (error) {
+        console.error("Server update error:", error);
+        return new NextResponse(error)
+    }
+}
+