airbyte-agent-airtable 0.1.5__py3-none-any.whl

airbyte_agent_airtable/_vendored/connector_sdk/logging/logger.py

@@ -0,0 +1,273 @@
+"""Request/response logging implementation."""
+
+import base64
+import json
+import time
+import uuid
+from pathlib import Path
+from typing import Any, Dict, Set
+
+from .types import LogSession, RequestLog
+
+# Headers to redact for security
+SENSITIVE_HEADERS: Set[str] = {
+    "authorization",
+    "bearer",
+    "api-key",
+    "x-api-key",
+    "token",
+    "secret",
+    "password",
+    "credential",
+}
+
+
+class RequestLogger:
+    """Captures HTTP request/response interactions to a JSON file.
+
+    Implements bounded logging with automatic rotation and flush-before-discard
+    to prevent unbounded memory growth in long-running processes.
+    """
+
+    def __init__(
+        self,
+        log_file: str | None = None,
+        connector_name: str | None = None,
+        max_logs: int | None = 10000,
+    ):
+        """
+        Initialize the request logger.
+
+        Args:
+            log_file: Path to write logs. If None, generates timestamped filename.
+            connector_name: Name of the connector being logged.
+            max_logs: Maximum number of logs to keep in memory before rotation.
+                Set to None for unlimited (not recommended for production).
+                Defaults to 10000.
+        """
+        if log_file is None:
+            timestamp = time.strftime("%Y%m%d_%H%M%S")
+            log_file = f".logs/session_{timestamp}.json"
+
+        self.log_file = Path(log_file)
+        self.log_file.parent.mkdir(parents=True, exist_ok=True)
+
+        self.session = LogSession(
+            session_id=str(uuid.uuid4()),
+            connector_name=connector_name,
+            max_logs=max_logs,
+        )
+        self._active_requests: Dict[str, Dict[str, Any]] = {}
+        # Store rotated logs that have been flushed from active buffer
+        self._rotated_logs: list[RequestLog] = []
+
+    def _redact_headers(self, headers: Dict[str, str]) -> Dict[str, str]:
+        """Redact sensitive headers."""
+        redacted = {}
+        for key, value in headers.items():
+            if any(sensitive in key.lower() for sensitive in SENSITIVE_HEADERS):
+                redacted[key] = "[REDACTED]"
+            else:
+                redacted[key] = value
+        return redacted
+
+    def _rotate_logs_if_needed(self) -> None:
+        """Rotate logs if max_logs limit is reached.
+
+        Moves oldest logs to _rotated_logs before removing them from active buffer.
+        This ensures logs are preserved for final save() without memory growth.
+        """
+        max_logs = self.session.max_logs
+        if max_logs is None:
+            # Unlimited logging, no rotation needed
+            return
+
+        current_count = len(self.session.logs)
+        if current_count >= max_logs:
+            # Calculate how many logs to rotate (keep buffer at ~90% to avoid thrashing)
+            num_to_rotate = max(1, current_count - int(max_logs * 0.9))
+
+            # Move oldest logs to rotated buffer
+            rotated = self.session.logs[:num_to_rotate]
+            self._rotated_logs.extend(rotated)
+
+            # Remove rotated logs from active buffer
+            self.session.logs = self.session.logs[num_to_rotate:]
+
+    def log_request(
+        self,
+        method: str,
+        url: str,
+        path: str,
+        headers: Dict[str, str] | None = None,
+        params: Dict[str, Any] | None = None,
+        body: Any | None = None,
+    ) -> str:
+        """
+        Log the start of an HTTP request.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            url: Full URL
+            path: Request path
+            headers: Request headers
+            params: Query parameters
+            body: Request body
+
+        Returns:
+            Request ID for correlating with response
+        """
+        request_id = str(uuid.uuid4())
+        self._active_requests[request_id] = {
+            "start_time": time.time(),
+            "method": method,
+            "url": url,
+            "path": path,
+            "headers": self._redact_headers(headers or {}),
+            "params": params,
+            "body": body,
+        }
+        return request_id
+
+    def log_response(
+        self,
+        request_id: str,
+        status_code: int,
+        response_body: Any | None = None,
+        response_headers: Dict[str, str] | None = None,
+    ) -> None:
+        """
+        Log a successful HTTP response.
+
+        Args:
+            request_id: ID returned from log_request
+            status_code: HTTP status code
+            response_body: Response body
+            response_headers: Response headers
+        """
+        if request_id not in self._active_requests:
+            return
+
+        request_data = self._active_requests.pop(request_id)
+        timing_ms = (time.time() - request_data["start_time"]) * 1000
+
+        # Convert bytes to base64 for JSON serialization
+        serializable_body = response_body
+        if isinstance(response_body, bytes):
+            serializable_body = {
+                "_binary": True,
+                "_base64": base64.b64encode(response_body).decode("utf-8"),
+            }
+
+        log_entry = RequestLog(
+            method=request_data["method"],
+            url=request_data["url"],
+            path=request_data["path"],
+            headers=request_data["headers"],
+            params=request_data["params"],
+            body=request_data["body"],
+            response_status=status_code,
+            response_body=serializable_body,
+            response_headers=response_headers or {},
+            timing_ms=timing_ms,
+        )
+
+        self.session.logs.append(log_entry)
+        self._rotate_logs_if_needed()
+
+    def log_error(
+        self,
+        request_id: str,
+        error: str,
+        status_code: int | None = None,
+    ) -> None:
+        """
+        Log an HTTP request error.
+
+        Args:
+            request_id: ID returned from log_request
+            error: Error message
+            status_code: HTTP status code if available
+        """
+        if request_id not in self._active_requests:
+            return
+
+        request_data = self._active_requests.pop(request_id)
+        timing_ms = (time.time() - request_data["start_time"]) * 1000
+
+        log_entry = RequestLog(
+            method=request_data["method"],
+            url=request_data["url"],
+            path=request_data["path"],
+            headers=request_data["headers"],
+            params=request_data["params"],
+            body=request_data["body"],
+            response_status=status_code,
+            timing_ms=timing_ms,
+            error=error,
+        )
+
+        self.session.logs.append(log_entry)
+        self._rotate_logs_if_needed()
+
+    def log_chunk_fetch(self, chunk: bytes) -> None:
+        """Log a chunk from streaming response.
+
+        Args:
+            chunk: Binary chunk data from streaming response
+        """
+        self.session.chunk_logs.append(chunk)
+
+    def save(self) -> None:
+        """Write the current session to the log file.
+
+        Includes both rotated logs and current active logs to ensure
+        no data loss during bounded logging.
+        """
+        # Combine rotated logs with current logs for complete session
+        all_logs = self._rotated_logs + self.session.logs
+
+        # Create a temporary session with all logs for serialization
+        session_data = self.session.model_dump(mode="json")
+        session_data["logs"] = [log.model_dump(mode="json") for log in all_logs]
+
+        with open(self.log_file, "w") as f:
+            json.dump(session_data, f, indent=2, default=str)
+
+    def close(self) -> None:
+        """Finalize and save the logging session."""
+        self.save()
+
+
+class NullLogger:
+    """No-op logger for when logging is disabled."""
+
+    def log_request(self, *args, **kwargs) -> str:
+        """No-op log_request."""
+        return ""
+
+    def log_response(
+        self,
+        request_id: str,
+        status_code: int,
+        response_body: Any | None = None,
+        response_headers: Dict[str, str] | None = None,
+    ) -> None:
+        """No-op log_response."""
+        pass
+
+    def log_error(self, *args, **kwargs) -> None:
+        """No-op log_error."""
+        pass
+
+    def log_chunk_fetch(self, chunk: bytes) -> None:
+        """No-op chunk logging for production."""
+        pass
+
+    def save(self) -> None:
+        """No-op save."""
+        pass
+
+    def close(self) -> None:
+        """No-op close."""
+        pass

airbyte_agent_airtable/_vendored/connector_sdk/logging/types.py

@@ -0,0 +1,93 @@
+"""Type definitions for request/response logging."""
+
+import base64
+from datetime import UTC, datetime
+from typing import Any, Dict, List
+
+from pydantic import BaseModel, ConfigDict, Field, field_serializer, field_validator
+
+
+def _utc_now() -> datetime:
+    """Get current UTC datetime (timezone-aware)."""
+    return datetime.now(UTC)
+
+
+def _encode_bytes(v: bytes) -> dict:
+    """Encode bytes as base64 for JSON serialization."""
+    return {"_binary": True, "_base64": base64.b64encode(v).decode("utf-8")}
+
+
+class RequestLog(BaseModel):
+    """Captures a single HTTP request/response interaction."""
+
+    model_config = ConfigDict()
+
+    timestamp: datetime = Field(default_factory=_utc_now)
+    method: str
+    url: str
+    path: str
+    headers: Dict[str, str] = Field(default_factory=dict)
+    params: Dict[str, Any] | None = None
+    body: Any | None = None
+    response_status: int | None = None
+    response_body: Any | None = None
+    response_headers: Dict[str, str] = Field(default_factory=dict)
+    timing_ms: float | None = None
+    error: str | None = None
+
+    @field_serializer("timestamp")
+    def serialize_datetime(self, value: datetime) -> str:
+        return value.isoformat()
+
+
+class LogSession(BaseModel):
+    """Collection of request logs with session metadata.
+
+    When max_logs is set, the session will maintain a bounded buffer of recent logs.
+    Older logs should be flushed to disk before being discarded (handled by RequestLogger).
+    """
+
+    model_config = ConfigDict()
+
+    session_id: str
+    started_at: datetime = Field(default_factory=_utc_now)
+    connector_name: str | None = None
+    logs: List[RequestLog] = Field(default_factory=list)
+    max_logs: int | None = Field(
+        default=10000,
+        description="Maximum number of logs to keep in memory. "
+        "When limit is reached, oldest logs should be flushed before removal. "
+        "Set to None for unlimited (not recommended for production).",
+    )
+    chunk_logs: List[bytes] = Field(
+        default_factory=list,
+        description="Captured chunks from streaming responses. Each chunk is logged when log_chunk_fetch() is called.",
+    )
+
+    @field_validator("chunk_logs", mode="before")
+    @classmethod
+    def decode_chunk_logs(cls, v: Any) -> List[bytes]:
+        """Decode chunk_logs from JSON representation back to bytes."""
+        if v is None or v == []:
+            return []
+        if isinstance(v, list):
+            result = []
+            for item in v:
+                if isinstance(item, bytes):
+                    result.append(item)
+                elif isinstance(item, dict) and item.get("_binary"):
+                    # Decode from {"_binary": True, "_base64": "..."} format
+                    result.append(base64.b64decode(item["_base64"]))
+                else:
+                    result.append(item)
+            return result
+        return v
+
+    @field_serializer("started_at")
+    def serialize_datetime(self, value: datetime) -> str:
+        return value.isoformat()
+
+    @field_serializer("chunk_logs")
+    def serialize_chunk_logs(self, value: List[bytes]) -> List[dict]:
+        """Serialize bytes chunks as base64 for JSON."""
+        return [_encode_bytes(chunk) for chunk in value]

airbyte_agent_airtable/_vendored/connector_sdk/observability/__init__.py

@@ -0,0 +1,11 @@
+"""Shared observability components for logging and telemetry."""
+
+from .models import OperationMetadata
+from .redactor import DataRedactor
+from .session import ObservabilitySession
+
+__all__ = [
+    "DataRedactor",
+    "ObservabilitySession",
+    "OperationMetadata",
+]

airbyte_agent_airtable/_vendored/connector_sdk/observability/config.py

@@ -0,0 +1,179 @@
+"""Unified configuration for connector-sdk."""
+
+import logging
+import os
+import tempfile
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+# New config location
+CONFIG_DIR = Path.home() / ".airbyte" / "connector-sdk"
+CONFIG_PATH = CONFIG_DIR / "config.yaml"
+
+# Legacy file locations (for migration)
+LEGACY_USER_ID_PATH = Path.home() / ".airbyte" / "ai_sdk_user_id"
+LEGACY_INTERNAL_MARKER_PATH = Path.home() / ".airbyte" / "internal_user"
+
+
+@dataclass
+class SDKConfig:
+    """Connector SDK configuration."""
+
+    user_id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    is_internal_user: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for YAML serialization."""
+        return {
+            "user_id": self.user_id,
+            "is_internal_user": self.is_internal_user,
+        }
+
+
+def _delete_legacy_files() -> None:
+    """
+    Delete legacy config files after successful migration.
+
+    Removes:
+    - ~/.airbyte/ai_sdk_user_id
+    - ~/.airbyte/internal_user
+    """
+    for legacy_path in [LEGACY_USER_ID_PATH, LEGACY_INTERNAL_MARKER_PATH]:
+        try:
+            if legacy_path.exists():
+                legacy_path.unlink()
+                logger.debug(f"Deleted legacy config file: {legacy_path}")
+        except Exception as e:
+            logger.debug(f"Could not delete legacy file {legacy_path}: {e}")
+
+
+def _migrate_legacy_config() -> SDKConfig | None:
+    """
+    Migrate from legacy file-based config to new YAML format.
+
+    Reads from:
+    - ~/.airbyte/ai_sdk_user_id (user_id)
+    - ~/.airbyte/internal_user (is_internal_user marker)
+
+    Returns SDKConfig if migration was successful, None otherwise.
+    """
+    user_id = None
+    is_internal = False
+
+    # Try to read legacy user_id
+    try:
+        if LEGACY_USER_ID_PATH.exists():
+            user_id = LEGACY_USER_ID_PATH.read_text().strip()
+            if not user_id:
+                user_id = None
+    except Exception:
+        pass
+
+    # Check legacy internal_user marker
+    try:
+        is_internal = LEGACY_INTERNAL_MARKER_PATH.exists()
+    except Exception:
+        pass
+
+    if user_id or is_internal:
+        return SDKConfig(
+            user_id=user_id or str(uuid.uuid4()),
+            is_internal_user=is_internal,
+        )
+
+    return None
+
+
+def load_config() -> SDKConfig:
+    """
+    Load SDK configuration from config file.
+
+    Checks (in order):
+    1. New config file at ~/.airbyte/connector-sdk/config.yaml
+    2. Legacy files at ~/.airbyte/ai_sdk_user_id and ~/.airbyte/internal_user
+    3. Creates new config with generated user_id if nothing exists
+
+    Environment variable AIRBYTE_INTERNAL_USER can override is_internal_user.
+
+    Returns:
+        SDKConfig with user_id and is_internal_user
+    """
+    config = None
+
+    # Try to load from new config file
+    try:
+        if CONFIG_PATH.exists():
+            content = CONFIG_PATH.read_text()
+            data = yaml.safe_load(content) or {}
+            config = SDKConfig(
+                user_id=data.get("user_id", str(uuid.uuid4())),
+                is_internal_user=data.get("is_internal_user", False),
+            )
+            # Always clean up legacy files if they exist (even if new config exists)
+            _delete_legacy_files()
+    except Exception as e:
+        logger.debug(f"Could not load config from {CONFIG_PATH}: {e}")
+
+    # Try to migrate from legacy files if new config doesn't exist
+    if config is None:
+        config = _migrate_legacy_config()
+        if config:
+            # Save migrated config to new location
+            try:
+                save_config(config)
+                logger.debug("Migrated legacy config to new location")
+                # Delete legacy files after successful migration
+                _delete_legacy_files()
+            except Exception as e:
+                logger.debug(f"Could not save migrated config: {e}")
+
+    # Create new config if nothing exists
+    if config is None:
+        config = SDKConfig()
+        try:
+            save_config(config)
+        except Exception as e:
+            logger.debug(f"Could not save new config: {e}")
+
+    # Environment variable override for is_internal_user
+    env_value = os.getenv("AIRBYTE_INTERNAL_USER", "").lower()
+    if env_value in ("true", "1", "yes"):
+        config.is_internal_user = True
+    elif env_value:
+        # Any other non-empty value (including "false", "0", "no") defaults to False
+        config.is_internal_user = False
+
+    return config
+
+
+def save_config(config: SDKConfig) -> None:
+    """
+    Save SDK configuration to config file.
+
+    Creates the config directory if it doesn't exist.
+    Uses atomic writes to prevent corruption from concurrent access.
+
+    Args:
+        config: SDKConfig to save
+    """
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Use atomic write: write to temp file then rename (atomic on POSIX)
+    fd, temp_path = tempfile.mkstemp(dir=CONFIG_DIR, suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w") as f:
+            yaml.dump(config.to_dict(), f, default_flow_style=False)
+        os.rename(temp_path, CONFIG_PATH)
+    except Exception:
+        # Clean up temp file on failure
+        try:
+            os.unlink(temp_path)
+        except OSError:
+            pass
+        raise

airbyte_agent_airtable/_vendored/connector_sdk/observability/models.py

@@ -0,0 +1,19 @@
+"""Shared operation metadata models."""
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Dict
+
+
+@dataclass
+class OperationMetadata:
+    """Shared operation metadata."""
+
+    entity: str
+    action: str
+    timestamp: datetime
+    timing_ms: float | None = None
+    status_code: int | None = None
+    error_type: str | None = None
+    error_message: str | None = None
+    params: Dict[str, Any] | None = None

airbyte_agent_airtable/_vendored/connector_sdk/observability/redactor.py

@@ -0,0 +1,81 @@
+"""Shared redaction logic for both logging and telemetry."""
+
+from typing import Any, Dict
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
+
+
+class DataRedactor:
+    """Shared redaction logic for both logging and telemetry."""
+
+    SENSITIVE_HEADER_PATTERNS = [
+        "authorization",
+        "api-key",
+        "x-api-key",
+        "token",
+        "bearer",
+        "secret",
+        "password",
+        "credential",
+    ]
+
+    SENSITIVE_PARAM_PATTERNS = [
+        "password",
+        "secret",
+        "api_key",
+        "apikey",
+        "token",
+        "credentials",
+        "auth",
+        "key",
+    ]
+
+    @staticmethod
+    def redact_headers(headers: Dict[str, str]) -> Dict[str, str]:
+        """Redact sensitive headers."""
+        redacted = {}
+        for key, value in headers.items():
+            if any(pattern in key.lower() for pattern in DataRedactor.SENSITIVE_HEADER_PATTERNS):
+                redacted[key] = "***REDACTED***"
+            else:
+                redacted[key] = value
+        return redacted
+
+    @staticmethod
+    def redact_params(params: Dict[str, Any]) -> Dict[str, Any]:
+        """Redact sensitive parameters."""
+        redacted = {}
+        for key, value in params.items():
+            if any(pattern in key.lower() for pattern in DataRedactor.SENSITIVE_PARAM_PATTERNS):
+                redacted[key] = "***REDACTED***"
+            else:
+                redacted[key] = value
+        return redacted
+
+    @staticmethod
+    def redact_url(url: str) -> str:
+        """Redact sensitive query params from URL."""
+        parsed = urlparse(url)
+        if not parsed.query:
+            return url
+
+        params = parse_qs(parsed.query)
+        redacted_params = {}
+
+        for key, values in params.items():
+            if any(pattern in key.lower() for pattern in DataRedactor.SENSITIVE_PARAM_PATTERNS):
+                redacted_params[key] = ["***REDACTED***"] * len(values)
+            else:
+                redacted_params[key] = values
+
+        # Reconstruct URL with redacted params
+        new_query = urlencode(redacted_params, doseq=True)
+        return urlunparse(
+            (
+                parsed.scheme,
+                parsed.netloc,
+                parsed.path,
+                parsed.params,
+                new_query,
+                parsed.fragment,
+            )
+        )