dory-sdk 2.1.0__py3-none-any.whl

dory/k8s/client.py

@@ -0,0 +1,251 @@
+"""
+Kubernetes client wrapper.
+
+Provides simplified interface to Kubernetes API.
+"""
+
+import logging
+from typing import Any
+
+from dory.utils.errors import DoryK8sError
+
+logger = logging.getLogger(__name__)
+
+# Optional kubernetes import
+try:
+    from kubernetes import client, config
+    from kubernetes.client.rest import ApiException
+    K8S_AVAILABLE = True
+except ImportError:
+    K8S_AVAILABLE = False
+    client = None
+    config = None
+    ApiException = Exception
+
+
+class K8sClient:
+    """
+    Kubernetes API client wrapper.
+
+    Handles configuration loading and provides
+    simplified access to common operations.
+    """
+
+    def __init__(self, namespace: str | None = None):
+        """
+        Initialize Kubernetes client.
+
+        Args:
+            namespace: Kubernetes namespace (auto-detected if not provided)
+        """
+        self._namespace = namespace
+        self._core_api: Any = None
+        self._initialized = False
+
+    def _ensure_initialized(self) -> None:
+        """Initialize Kubernetes client if not already done."""
+        if self._initialized:
+            return
+
+        if not K8S_AVAILABLE:
+            raise DoryK8sError(
+                "Kubernetes client not available. "
+                "Install with: pip install kubernetes"
+            )
+
+        try:
+            # Try in-cluster config first
+            config.load_incluster_config()
+            logger.debug("Using in-cluster Kubernetes config")
+        except config.ConfigException:
+            try:
+                # Fall back to kubeconfig
+                config.load_kube_config()
+                logger.debug("Using kubeconfig")
+            except config.ConfigException as e:
+                raise DoryK8sError(f"Failed to load Kubernetes config: {e}", cause=e)
+
+        self._core_api = client.CoreV1Api()
+
+        # Auto-detect namespace if not provided
+        if not self._namespace:
+            import os
+            self._namespace = os.environ.get("POD_NAMESPACE", "default")
+
+        self._initialized = True
+
+    @property
+    def namespace(self) -> str:
+        """Get current namespace."""
+        self._ensure_initialized()
+        return self._namespace
+
+    @property
+    def core_api(self):
+        """Get CoreV1Api client."""
+        self._ensure_initialized()
+        return self._core_api
+
+    async def get_pod(self, name: str) -> dict[str, Any]:
+        """
+        Get pod details.
+
+        Args:
+            name: Pod name
+
+        Returns:
+            Pod details as dictionary
+
+        Raises:
+            DoryK8sError: If operation fails
+        """
+        self._ensure_initialized()
+
+        try:
+            pod = self._core_api.read_namespaced_pod(
+                name=name,
+                namespace=self._namespace,
+            )
+            return pod.to_dict()
+
+        except ApiException as e:
+            if e.status == 404:
+                raise DoryK8sError(f"Pod not found: {name}", cause=e)
+            raise DoryK8sError(f"Failed to get pod {name}: {e}", cause=e)
+
+    async def get_pod_annotations(self, name: str) -> dict[str, str]:
+        """
+        Get pod annotations.
+
+        Args:
+            name: Pod name
+
+        Returns:
+            Annotations dictionary
+        """
+        pod = await self.get_pod(name)
+        return pod.get("metadata", {}).get("annotations", {})
+
+    async def patch_pod_annotations(
+        self,
+        name: str,
+        annotations: dict[str, str],
+    ) -> None:
+        """
+        Patch pod annotations.
+
+        Args:
+            name: Pod name
+            annotations: Annotations to add/update
+        """
+        self._ensure_initialized()
+
+        body = {
+            "metadata": {
+                "annotations": annotations,
+            }
+        }
+
+        try:
+            self._core_api.patch_namespaced_pod(
+                name=name,
+                namespace=self._namespace,
+                body=body,
+            )
+            logger.debug(f"Patched annotations on pod {name}")
+
+        except ApiException as e:
+            raise DoryK8sError(f"Failed to patch pod {name}: {e}", cause=e)
+
+    async def get_configmap(self, name: str) -> dict[str, str] | None:
+        """
+        Get ConfigMap data.
+
+        Args:
+            name: ConfigMap name
+
+        Returns:
+            ConfigMap data, or None if not found
+        """
+        self._ensure_initialized()
+
+        try:
+            cm = self._core_api.read_namespaced_config_map(
+                name=name,
+                namespace=self._namespace,
+            )
+            return cm.data or {}
+
+        except ApiException as e:
+            if e.status == 404:
+                return None
+            raise DoryK8sError(f"Failed to get ConfigMap {name}: {e}", cause=e)
+
+    async def create_or_update_configmap(
+        self,
+        name: str,
+        data: dict[str, str],
+        labels: dict[str, str] | None = None,
+    ) -> None:
+        """
+        Create or update a ConfigMap.
+
+        Args:
+            name: ConfigMap name
+            data: ConfigMap data
+            labels: Optional labels
+        """
+        self._ensure_initialized()
+
+        configmap = client.V1ConfigMap(
+            metadata=client.V1ObjectMeta(
+                name=name,
+                namespace=self._namespace,
+                labels=labels,
+            ),
+            data=data,
+        )
+
+        try:
+            # Try create first
+            self._core_api.create_namespaced_config_map(
+                namespace=self._namespace,
+                body=configmap,
+            )
+            logger.debug(f"Created ConfigMap {name}")
+
+        except ApiException as e:
+            if e.status == 409:
+                # Already exists, update
+                self._core_api.replace_namespaced_config_map(
+                    name=name,
+                    namespace=self._namespace,
+                    body=configmap,
+                )
+                logger.debug(f"Updated ConfigMap {name}")
+            else:
+                raise DoryK8sError(f"Failed to create ConfigMap {name}: {e}", cause=e)
+
+    async def delete_configmap(self, name: str) -> bool:
+        """
+        Delete a ConfigMap.
+
+        Args:
+            name: ConfigMap name
+
+        Returns:
+            True if deleted, False if not found
+        """
+        self._ensure_initialized()
+
+        try:
+            self._core_api.delete_namespaced_config_map(
+                name=name,
+                namespace=self._namespace,
+            )
+            return True
+
+        except ApiException as e:
+            if e.status == 404:
+                return False
+            raise DoryK8sError(f"Failed to delete ConfigMap {name}: {e}", cause=e)

dory/k8s/pod_metadata.py

@@ -0,0 +1,182 @@
+"""
+Pod metadata extraction from Kubernetes environment.
+
+Retrieves pod information from:
+1. Downward API environment variables
+2. Kubernetes API
+3. /etc/podinfo files
+"""
+
+import logging
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PodMetadata:
+    """
+    Pod metadata extracted from environment.
+
+    Populated from:
+    - Environment variables (POD_NAME, POD_NAMESPACE, etc.)
+    - Downward API files (/etc/podinfo/)
+    - Kubernetes API (if available)
+    """
+
+    # Core identification
+    name: str = ""
+    namespace: str = "default"
+    uid: str = ""
+
+    # Node information
+    node_name: str = ""
+    service_account: str = ""
+
+    # Labels and annotations
+    labels: dict[str, str] = field(default_factory=dict)
+    annotations: dict[str, str] = field(default_factory=dict)
+
+    # Resource info
+    cpu_request: str = ""
+    cpu_limit: str = ""
+    memory_request: str = ""
+    memory_limit: str = ""
+
+    # Container info
+    container_name: str = ""
+    image: str = ""
+
+    @classmethod
+    def from_environment(cls) -> "PodMetadata":
+        """
+        Create PodMetadata from environment.
+
+        Reads from environment variables and downward API files.
+        """
+        metadata = cls()
+
+        # Read from environment variables
+        metadata.name = os.environ.get("POD_NAME", "")
+        metadata.namespace = os.environ.get("POD_NAMESPACE", "default")
+        metadata.uid = os.environ.get("POD_UID", "")
+        metadata.node_name = os.environ.get("NODE_NAME", "")
+        metadata.service_account = os.environ.get("SERVICE_ACCOUNT", "")
+        metadata.container_name = os.environ.get("CONTAINER_NAME", "")
+
+        # Try reading from downward API files
+        metadata._read_downward_api_files()
+
+        # Parse labels/annotations from environment
+        metadata._parse_labels_from_env()
+
+        logger.debug(f"Pod metadata: {metadata}")
+        return metadata
+
+    def _read_downward_api_files(self) -> None:
+        """Read metadata from downward API volume mounts."""
+        podinfo_path = Path("/etc/podinfo")
+
+        if not podinfo_path.exists():
+            return
+
+        # Read labels file
+        labels_file = podinfo_path / "labels"
+        if labels_file.exists():
+            self.labels = self._parse_labels_file(labels_file)
+
+        # Read annotations file
+        annotations_file = podinfo_path / "annotations"
+        if annotations_file.exists():
+            self.annotations = self._parse_labels_file(annotations_file)
+
+        # Read individual files
+        for attr, filename in [
+            ("name", "name"),
+            ("namespace", "namespace"),
+            ("uid", "uid"),
+            ("node_name", "nodeName"),
+        ]:
+            file_path = podinfo_path / filename
+            if file_path.exists():
+                try:
+                    value = file_path.read_text().strip()
+                    if value:
+                        setattr(self, attr, value)
+                except IOError:
+                    pass
+
+    def _parse_labels_file(self, path: Path) -> dict[str, str]:
+        """
+        Parse labels/annotations file.
+
+        Format: key="value"
+        """
+        result = {}
+        try:
+            content = path.read_text()
+            for line in content.strip().split("\n"):
+                if "=" in line:
+                    key, value = line.split("=", 1)
+                    # Remove quotes
+                    value = value.strip('"')
+                    result[key] = value
+        except IOError:
+            pass
+        return result
+
+    def _parse_labels_from_env(self) -> None:
+        """Parse labels from POD_LABELS environment variable."""
+        labels_env = os.environ.get("POD_LABELS", "")
+        if labels_env:
+            # Format: key1=value1,key2=value2
+            for pair in labels_env.split(","):
+                if "=" in pair:
+                    key, value = pair.split("=", 1)
+                    self.labels[key] = value
+
+    def get_processor_id(self) -> str:
+        """
+        Get processor ID from metadata.
+
+        Uses label if present, otherwise pod name.
+        """
+        # Try label first
+        processor_id = self.labels.get("dory.io/processor-id")
+        if processor_id:
+            return processor_id
+
+        # Fall back to pod name with suffix stripped
+        name = self.name
+        if name:
+            # Strip deployment suffix (e.g., myapp-7f8d9c6b-x4h2j -> myapp)
+            parts = name.rsplit("-", 2)
+            if len(parts) >= 2:
+                return parts[0]
+
+        return name or "unknown"
+
+    def is_migration(self) -> bool:
+        """Check if this is a migration restart."""
+        return self.annotations.get("dory.io/migration") == "true"
+
+    def get_previous_pod(self) -> str | None:
+        """Get previous pod name if this is a migration."""
+        return self.annotations.get("dory.io/previous-pod")
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "name": self.name,
+            "namespace": self.namespace,
+            "uid": self.uid,
+            "node_name": self.node_name,
+            "service_account": self.service_account,
+            "labels": self.labels,
+            "annotations": self.annotations,
+            "container_name": self.container_name,
+            "processor_id": self.get_processor_id(),
+        }

dory/logging/__init__.py

@@ -0,0 +1,9 @@
+"""Structured logging utilities."""
+
+from dory.logging.logger import setup_logging, get_logger, DoryLoggerAdapter
+
+__all__ = [
+    "setup_logging",
+    "get_logger",
+    "DoryLoggerAdapter",
+]

dory/logging/logger.py

@@ -0,0 +1,175 @@
+"""
+Structured logging for Dory SDK.
+
+Provides JSON-formatted logging suitable for Kubernetes
+and log aggregation systems.
+"""
+
+import json
+import logging
+import sys
+import time
+from typing import Any
+
+from dory.types import LogFormat
+
+
+class JsonFormatter(logging.Formatter):
+    """
+    JSON log formatter for structured logging.
+
+    Output format:
+    {
+        "timestamp": "2024-01-15T10:30:45.123456Z",
+        "level": "INFO",
+        "logger": "dory.core.app",
+        "message": "Processor starting",
+        "extra": {...}
+    }
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        """Format log record as JSON."""
+        log_dict = {
+            "timestamp": self._format_timestamp(record.created),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+
+        # Add exception info if present
+        if record.exc_info:
+            log_dict["exception"] = self.formatException(record.exc_info)
+
+        # Add extra fields
+        extra = {}
+        for key, value in record.__dict__.items():
+            if key not in (
+                "name", "msg", "args", "created", "levelname", "levelno",
+                "pathname", "filename", "module", "exc_info", "exc_text",
+                "stack_info", "lineno", "funcName", "relativeCreated",
+                "thread", "threadName", "processName", "process", "message",
+                "msecs", "taskName",
+            ):
+                extra[key] = self._serialize_value(value)
+
+        if extra:
+            log_dict["extra"] = extra
+
+        return json.dumps(log_dict)
+
+    def _format_timestamp(self, created: float) -> str:
+        """Format timestamp as ISO 8601."""
+        return time.strftime(
+            "%Y-%m-%dT%H:%M:%S",
+            time.gmtime(created)
+        ) + f".{int((created % 1) * 1000000):06d}Z"
+
+    def _serialize_value(self, value: Any) -> Any:
+        """Serialize value for JSON."""
+        if isinstance(value, (str, int, float, bool, type(None))):
+            return value
+        if isinstance(value, (list, tuple)):
+            return [self._serialize_value(v) for v in value]
+        if isinstance(value, dict):
+            return {k: self._serialize_value(v) for k, v in value.items()}
+        return str(value)
+
+
+class TextFormatter(logging.Formatter):
+    """
+    Human-readable text formatter.
+
+    Output format:
+    2024-01-15 10:30:45.123 [INFO] dory.core.app: Processor starting
+    """
+
+    def __init__(self):
+        super().__init__(
+            fmt="%(asctime)s.%(msecs)03d [%(levelname)s] %(name)s: %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+
+
+class DoryLoggerAdapter(logging.LoggerAdapter):
+    """
+    Logger adapter that adds context to all log messages.
+
+    Usage:
+        logger = DoryLoggerAdapter(
+            logging.getLogger(__name__),
+            {"processor_id": "my-processor", "pod": "my-pod-abc123"}
+        )
+        logger.info("Processing started")
+        # Output includes processor_id and pod in extra fields
+    """
+
+    def process(self, msg: str, kwargs: dict) -> tuple[str, dict]:
+        """Add extra context to log kwargs."""
+        extra = kwargs.get("extra", {})
+        extra.update(self.extra)
+        kwargs["extra"] = extra
+        return msg, kwargs
+
+
+def setup_logging(
+    level: str = "INFO",
+    format: str | LogFormat = LogFormat.JSON,
+) -> None:
+    """
+    Setup logging configuration for Dory SDK.
+
+    Args:
+        level: Log level (DEBUG, INFO, WARNING, ERROR)
+        format: Log format (json or text)
+    """
+    # Convert enum to string if needed
+    if isinstance(format, LogFormat):
+        format = format.value
+
+    # Get root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(getattr(logging, level.upper()))
+
+    # Remove existing handlers
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Create console handler
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(getattr(logging, level.upper()))
+
+    # Set formatter based on format
+    if format == "json":
+        handler.setFormatter(JsonFormatter())
+    else:
+        handler.setFormatter(TextFormatter())
+
+    root_logger.addHandler(handler)
+
+    # Set levels for noisy libraries
+    logging.getLogger("kubernetes").setLevel(logging.WARNING)
+    logging.getLogger("urllib3").setLevel(logging.WARNING)
+    logging.getLogger("aiohttp").setLevel(logging.WARNING)
+
+
+def get_logger(
+    name: str,
+    extra: dict[str, Any] | None = None,
+) -> logging.Logger | DoryLoggerAdapter:
+    """
+    Get a logger with optional extra context.
+
+    Args:
+        name: Logger name (usually __name__)
+        extra: Optional extra context to include in all logs
+
+    Returns:
+        Logger or LoggerAdapter
+    """
+    logger = logging.getLogger(name)
+
+    if extra:
+        return DoryLoggerAdapter(logger, extra)
+
+    return logger

dory/metrics/__init__.py

@@ -0,0 +1,7 @@
+"""Prometheus metrics collection."""
+
+from dory.metrics.collector import MetricsCollector
+
+__all__ = [
+    "MetricsCollector",
+]