orgdatacore 2.0.0__py3-none-any.whl

orgdatacore/_exceptions.py

@@ -0,0 +1,82 @@
+"""Custom exceptions for orgdatacore.
+
+This module defines a hierarchy of exceptions for better error handling:
+
+    OrgDataError (base)
+    ├── DataLoadError      - Failed to load/parse data
+    ├── DataSourceError    - Data source operation failed
+    │   ├── GCSError       - GCS-specific errors
+    │   └── FileSourceError - File source errors (internal)
+    └── ConfigurationError - Invalid configuration
+
+Example:
+    try:
+        service.load_from_data_source(source)
+    except DataLoadError as e:
+        logger.error(f"Failed to load data: {e}")
+    except DataSourceError as e:
+        logger.error(f"Data source error: {e}")
+"""
+
+
+class OrgDataError(Exception):
+    """Base exception for all orgdatacore errors."""
+
+    pass
+
+
+class DataLoadError(OrgDataError):
+    """Failed to load or parse organizational data.
+
+    Raised when:
+    - JSON parsing fails
+    - Data structure is invalid
+    - Required fields are missing
+    """
+
+    pass
+
+
+class DataSourceError(OrgDataError):
+    """Data source operation failed.
+
+    Base class for data source specific errors.
+    """
+
+    pass
+
+
+class GCSError(DataSourceError):
+    """Google Cloud Storage operation failed.
+
+    Raised when:
+    - GCS connection fails
+    - Bucket/object not found
+    - Authentication fails
+    - Download fails
+    """
+
+    pass
+
+
+class FileSourceError(DataSourceError):
+    """File source operation failed (internal testing only).
+
+    Raised when:
+    - File not found
+    - Permission denied
+    - Read error
+    """
+
+    pass
+
+
+class ConfigurationError(OrgDataError):
+    """Invalid configuration provided.
+
+    Raised when:
+    - Required config fields are missing
+    - Config values are invalid
+    """
+
+    pass

orgdatacore/_gcs.py

@@ -0,0 +1,317 @@
+# pyright: reportUnknownMemberType=false, reportUnknownArgumentType=false
+"""
+Data source implementations for orgdatacore.
+
+This module provides the GCSDataSource for production use with Google Cloud Storage.
+
+IMPORTANT: The caller is responsible for managing the data source lifecycle.
+The reader returned by load() must be closed when done reading. When using
+Service.load_from_data_source(), the service handles closing the reader.
+
+For custom data sources (e.g., S3, Azure Blob, etc.), implement the DataSource
+protocol (just implement load(), watch(), __str__() methods).
+
+Example custom implementation:
+
+    from typing import BinaryIO, Callable, Optional
+    from io import BytesIO
+
+    class S3DataSource:  # No inheritance needed - just implement the methods
+        def __init__(self, bucket: str, key: str):
+            self.bucket = bucket
+            self.key = key
+
+        def load(self) -> BinaryIO:
+            import boto3
+            s3 = boto3.client('s3')
+            response = s3.get_object(Bucket=self.bucket, Key=self.key)
+            return BytesIO(response['Body'].read())
+
+        def watch(self, callback: Callable[[], Optional[Exception]]) -> Optional[Exception]:
+            # Your S3 watching logic here
+            return None
+
+        def __str__(self) -> str:
+            return f"s3://{self.bucket}/{self.key}"
+"""
+
+import threading
+import time
+from collections.abc import Callable
+from io import BytesIO
+from typing import Any, BinaryIO, TypeVar
+
+from ._exceptions import ConfigurationError, GCSError
+from ._log import get_logger
+from ._types import GCSConfig
+
+_T = TypeVar("_T")
+
+# Default retry configuration
+DEFAULT_MAX_RETRIES = 3
+DEFAULT_RETRY_DELAY = 1.0  # seconds
+DEFAULT_RETRY_BACKOFF = 2.0  # multiplier
+
+
+try:
+    from google.cloud import storage
+
+    _HAS_GCS = True
+except ImportError:
+    _HAS_GCS = False
+
+
+def _retry_with_backoff(
+    operation: Callable[[], _T],
+    max_retries: int = DEFAULT_MAX_RETRIES,
+    initial_delay: float = DEFAULT_RETRY_DELAY,
+    backoff: float = DEFAULT_RETRY_BACKOFF,
+    operation_name: str = "operation",
+) -> _T:
+    """Execute an operation with exponential backoff retry.
+
+    Args:
+        operation: Callable to execute.
+        max_retries: Maximum number of retry attempts.
+        initial_delay: Initial delay between retries in seconds.
+        backoff: Multiplier for delay after each retry.
+        operation_name: Name of operation for logging.
+
+    Returns:
+        Result of the operation.
+
+    Raises:
+        GCSError: If all retries are exhausted.
+    """
+    logger = get_logger()
+    delay = initial_delay
+    last_error: Exception | None = None
+
+    for attempt in range(max_retries + 1):
+        try:
+            return operation()
+        except Exception as e:
+            last_error = e
+            if attempt < max_retries:
+                logger.warning(
+                    f"{operation_name} failed, retrying",
+                    extra={
+                        "attempt": attempt + 1,
+                        "max_retries": max_retries,
+                        "delay": delay,
+                        "error": str(e),
+                    },
+                )
+                time.sleep(delay)
+                delay *= backoff
+            else:
+                logger.error(
+                    f"{operation_name} failed after all retries",
+                    extra={
+                        "attempts": max_retries + 1,
+                        "error": str(e),
+                    },
+                )
+
+    raise GCSError(
+        f"{operation_name} failed after {max_retries + 1} attempts: {last_error}"
+    )
+
+
+class GCSDataSource:
+    """GCS data source using google-cloud-storage.
+
+    Requires the google-cloud-storage package:
+        pip install orgdatacore[gcs]
+
+    Features:
+    - Automatic retry with exponential backoff for transient failures
+    - Structured logging for observability
+    - Proper error handling with custom exceptions
+
+    Example:
+        from orgdatacore import GCSConfig, GCSDataSource
+        from datetime import timedelta
+
+        config = GCSConfig(
+            bucket="your-bucket",
+            object_path="path/to/data.json",
+            project_id="your-project",
+            check_interval=timedelta(minutes=5),
+        )
+        source = GCSDataSource(config)
+        service.load_from_data_source(source)
+    """
+
+    def __init__(
+        self,
+        config: GCSConfig,
+        *,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        retry_delay: float = DEFAULT_RETRY_DELAY,
+        retry_backoff: float = DEFAULT_RETRY_BACKOFF,
+    ) -> None:
+        """Create a GCS data source.
+
+        Args:
+            config: GCS configuration.
+            max_retries: Maximum retry attempts for transient failures.
+            retry_delay: Initial delay between retries in seconds.
+            retry_backoff: Multiplier for delay after each retry.
+
+        Raises:
+            ImportError: If google-cloud-storage is not installed.
+            ConfigurationError: If configuration is invalid.
+        """
+        if not _HAS_GCS:
+            raise ImportError(
+                "google-cloud-storage is required for GCS support. "
+                "Install it with: pip install orgdatacore[gcs]"
+            )
+        if not config.bucket:
+            raise ConfigurationError("GCS bucket is required")
+        if not config.object_path:
+            raise ConfigurationError("GCS object_path is required")
+
+        self.config = config
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self.retry_backoff = retry_backoff
+        self._client: Any = None
+        self._stop_event = threading.Event()
+
+    def _get_client(self) -> Any:
+        """Get or create the GCS client."""
+        if self._client is None:
+            logger = get_logger()
+            logger.debug(
+                "Creating GCS client", extra={"project_id": self.config.project_id}
+            )
+
+            if self.config.credentials_json:
+                import json as _json
+
+                creds_info = _json.loads(self.config.credentials_json)
+                self._client = storage.Client.from_service_account_info(creds_info)
+            else:
+                self._client = storage.Client(project=self.config.project_id or None)
+        return self._client
+
+    def load(self) -> BinaryIO:
+        """Load and return a reader for the organizational data from GCS.
+
+        Returns:
+            File-like object containing the JSON data.
+
+        Raises:
+            GCSError: If loading fails after all retries.
+        """
+        logger = get_logger()
+        logger.debug(
+            "Loading from GCS",
+            extra={"bucket": self.config.bucket, "object": self.config.object_path},
+        )
+
+        def _download() -> BinaryIO:
+            client = self._get_client()
+            bucket = client.bucket(self.config.bucket)
+            blob = bucket.blob(self.config.object_path)
+            content = blob.download_as_bytes()
+            return BytesIO(content)
+
+        return _retry_with_backoff(
+            _download,
+            max_retries=self.max_retries,
+            initial_delay=self.retry_delay,
+            backoff=self.retry_backoff,
+            operation_name=f"GCS download gs://{self.config.bucket}/{self.config.object_path}",
+        )
+
+    def watch(self, callback: Callable[[], Exception | None]) -> Exception | None:
+        """Monitor for changes and call callback when data is updated.
+
+        Uses polling based on the configured check interval.
+        Detects changes by comparing object generation numbers.
+
+        Args:
+            callback: Function to call when data changes.
+
+        Returns:
+            Exception if watcher setup fails, None otherwise.
+        """
+        logger = get_logger()
+
+        # Reset stop event so watch() can be called again after stop_watching()
+        self._stop_event = threading.Event()
+
+        try:
+            client = self._get_client()
+            bucket = client.bucket(self.config.bucket)
+            blob = bucket.blob(self.config.object_path)
+
+            # Get initial generation
+            blob.reload()
+            last_generation = blob.generation
+
+            logger.info(
+                "Starting GCS watcher",
+                extra={
+                    "bucket": self.config.bucket,
+                    "object": self.config.object_path,
+                    "check_interval": str(self.config.check_interval),
+                    "initial_generation": last_generation,
+                },
+            )
+        except Exception as e:
+            logger.error("Failed to initialize GCS watcher", extra={"error": str(e)})
+            return GCSError(f"Failed to initialize GCS watcher: {e}")
+
+        def watcher() -> None:
+            nonlocal last_generation
+            interval = self.config.check_interval.total_seconds()
+
+            while not self._stop_event.is_set():
+                time.sleep(interval)
+                if self._stop_event.is_set():
+                    break
+
+                try:
+                    blob.reload()
+                    if blob.generation != last_generation:
+                        logger.info(
+                            "GCS object changed, triggering reload",
+                            extra={
+                                "old_generation": last_generation,
+                                "new_generation": blob.generation,
+                            },
+                        )
+                        last_generation = blob.generation
+                        err = callback()
+                        if err:
+                            logger.error(
+                                "Reload callback failed", extra={"error": str(err)}
+                            )
+                except Exception as e:
+                    logger.error("GCS watcher check failed", extra={"error": str(e)})
+
+        thread = threading.Thread(target=watcher, daemon=True, name="gcs-watcher")
+        thread.start()
+        return None
+
+    def stop_watching(self) -> None:
+        """Stop the GCS watcher."""
+        logger = get_logger()
+        logger.info("Stopping GCS watcher")
+        self._stop_event.set()
+
+    def stop(self) -> None:
+        """Stop the GCS watcher.
+
+        Alias for stop_watching() to support the standard stop interface
+        expected by AsyncService.stop_watcher().
+        """
+        self.stop_watching()
+
+    def __str__(self) -> str:
+        """Return a description of this data source."""
+        return f"gs://{self.config.bucket}/{self.config.object_path}"

orgdatacore/_internal/__init__.py

@@ -0,0 +1,5 @@
+"""Internal modules - not part of public API."""
+
+from .testing import FakeDataSource, FileDataSource, create_test_data_json
+
+__all__ = ["FileDataSource", "FakeDataSource", "create_test_data_json"]

orgdatacore/_internal/testing/__init__.py

@@ -0,0 +1,16 @@
+"""Internal testing utilities - not part of public API."""
+
+from .fake_gcs import FakeBlob, FakeBucket, FakeGCSClient, FakeGCSDataSource
+from .filesource import FileDataSource
+from .helpers import FakeDataSource, create_test_data, create_test_data_json
+
+__all__ = [
+    "FileDataSource",
+    "FakeDataSource",
+    "FakeGCSClient",
+    "FakeGCSDataSource",
+    "FakeBucket",
+    "FakeBlob",
+    "create_test_data_json",
+    "create_test_data",
+]

orgdatacore/_internal/testing/fake_gcs.py

@@ -0,0 +1,173 @@
+"""Fake GCS client implementation for testing.
+
+This module provides fake implementations of the google.cloud.storage
+Client, Bucket, and Blob classes for testing GCS-related code without
+requiring actual GCS connections.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from io import BytesIO
+from typing import BinaryIO
+
+
+class FakeBucket:
+    """Fake implementation of google.cloud.storage.Bucket."""
+
+    def __init__(self, name: str, client: FakeGCSClient | None = None) -> None:
+        self.name = name
+        self.client = client
+        self.blobs: dict[str, dict] = {}
+
+    def blob(self, name: str) -> FakeBlob:
+        """Get a blob reference."""
+        return FakeBlob(name, self)
+
+    def get_blob_data(self, name: str) -> dict | None:
+        """Get blob data from internal storage."""
+        return self.blobs.get(name)
+
+    def set_blob_data(self, name: str, data: dict) -> None:
+        """Set blob data in internal storage."""
+        self.blobs[name] = data
+
+    def add_blob(self, name: str, content: bytes, generation: int = 1) -> None:
+        """Add a blob with content for testing."""
+        self.blobs[name] = {
+            "content": content,
+            "generation": generation,
+            "updated": datetime.now(),
+        }
+
+    def update_blob(self, name: str, content: bytes) -> None:
+        """Update a blob's content and increment generation."""
+        if name not in self.blobs:
+            self.add_blob(name, content)
+            return
+        current = self.blobs[name]
+        self.blobs[name] = {
+            "content": content,
+            "generation": current["generation"] + 1,
+            "updated": datetime.now(),
+        }
+
+
+class FakeBlob:
+    """Fake implementation of google.cloud.storage.Blob."""
+
+    def __init__(self, name: str, bucket: FakeBucket) -> None:
+        self.name = name
+        self.bucket = bucket
+        self._generation = 1
+        self._updated = datetime.now()
+
+    @property
+    def generation(self) -> int:
+        """Return the generation number of the blob."""
+        return self._generation
+
+    @property
+    def updated(self) -> datetime:
+        """Return when the blob was last updated."""
+        return self._updated
+
+    def reload(self) -> None:
+        """Reload blob metadata from the fake storage."""
+        data = self.bucket.get_blob_data(self.name)
+        if data is None:
+            raise Exception(f"Blob {self.name} not found")
+        self._generation = data["generation"]
+        self._updated = data["updated"]
+
+    def download_as_bytes(self) -> bytes:
+        """Download the blob content as bytes."""
+        data = self.bucket.get_blob_data(self.name)
+        if data is None:
+            raise Exception(f"Blob {self.name} not found")
+        return data["content"]
+
+    def upload_from_string(self, content: str | bytes) -> None:
+        """Upload content to the blob."""
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        self.bucket.set_blob_data(
+            self.name,
+            {
+                "content": content,
+                "generation": self._generation + 1,
+                "updated": datetime.now(),
+            },
+        )
+        self._generation += 1
+        self._updated = datetime.now()
+
+
+class FakeGCSClient:
+    """Fake implementation of google.cloud.storage.Client."""
+
+    def __init__(self, project: str | None = None) -> None:
+        self.project = project
+        self.buckets: dict[str, FakeBucket] = {}
+
+    @classmethod
+    def from_service_account_json(cls, json_path: str) -> FakeGCSClient:
+        """Create a client from a service account JSON file (fake)."""
+        return cls(project="fake-project-from-json")
+
+    def bucket(self, name: str) -> FakeBucket:
+        """Get a bucket reference."""
+        if name not in self.buckets:
+            self.buckets[name] = FakeBucket(name, self)
+        return self.buckets[name]
+
+    def add_bucket(self, name: str) -> FakeBucket:
+        """Add a bucket for testing."""
+        bucket = FakeBucket(name, self)
+        self.buckets[name] = bucket
+        return bucket
+
+
+class FakeGCSDataSource:
+    """A fake GCS data source that uses in-memory storage.
+
+    This can be used directly in tests as a DataSource implementation.
+    """
+
+    def __init__(
+        self,
+        bucket: str,
+        object_path: str,
+        content: bytes | str = b"{}",
+    ) -> None:
+        self.bucket_name = bucket
+        self.object_path = object_path
+        self._client = FakeGCSClient()
+        self._bucket = self._client.add_bucket(bucket)
+
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        self._bucket.add_blob(object_path, content)
+
+        self._generation = 1
+        self._stop_watching = False
+
+    def load(self) -> BinaryIO:
+        """Load data from fake GCS."""
+        blob = self._bucket.blob(self.object_path)
+        content = blob.download_as_bytes()
+        return BytesIO(content)
+
+    def watch(self, callback) -> Exception | None:
+        """Watch for changes (no-op in fake implementation)."""
+        return None
+
+    def update_content(self, content: bytes | str) -> None:
+        """Update the blob content (for testing hot reload)."""
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        self._bucket.update_blob(self.object_path, content)
+        self._generation += 1
+
+    def __str__(self) -> str:
+        return f"gs://{self.bucket_name}/{self.object_path} (fake)"