classifyre-cli 0.4.2__py3-none-any.whl

src/sources/dependencies.py

@@ -0,0 +1,81 @@
+"""Helpers for optional source dependencies."""
+
+from __future__ import annotations
+
+import importlib
+import logging
+from types import ModuleType
+
+from src.utils.uv_sync import auto_install_enabled, sync_group
+
+logger = logging.getLogger(__name__)
+
+
+class MissingSourceDependencyError(RuntimeError):
+    """Raised when an optional source dependency is unavailable."""
+
+    def __init__(
+        self,
+        source_name: str,
+        dependencies: list[str],
+        uv_groups: list[str],
+        detail: str | None = None,
+    ) -> None:
+        self.source_name = source_name
+        self.dependencies = dependencies
+        self.uv_groups = uv_groups
+        self.detail = detail
+
+        deps = ", ".join(dependencies)
+        group_hint = " or ".join(f"`uv sync --group {group}`" for group in uv_groups)
+        message = (
+            f"{source_name} source requires optional dependencies ({deps}). "
+            f"Install with {group_hint}."
+        )
+        if detail:
+            message = f"{message} {detail}"
+        super().__init__(message)
+
+
+def _ordered_groups(groups: list[str]) -> list[str]:
+    return sorted(dict.fromkeys(groups))
+
+
+def require_module(
+    module_name: str,
+    source_name: str,
+    uv_groups: list[str],
+    detail: str | None = None,
+) -> ModuleType:
+    """Import a module or raise MissingSourceDependencyError with uv guidance."""
+    try:
+        return importlib.import_module(module_name)
+    except Exception as exc:  # pragma: no cover - environment dependent
+        detail_messages: list[str] = [f"Original error: {exc}"]
+
+        if auto_install_enabled() and uv_groups:
+            for group in _ordered_groups(uv_groups):
+                success, install_detail = sync_group(group)
+                if install_detail:
+                    detail_messages.append(install_detail)
+                if not success:
+                    continue
+
+                try:
+                    importlib.invalidate_caches()
+                    return importlib.import_module(module_name)
+                except Exception as retry_exc:  # pragma: no cover
+                    detail_messages.append(
+                        f"Module '{module_name}' still unavailable after installing '{group}': {retry_exc}"
+                    )
+
+        base_detail = detail or "Optional dependency import failed"
+        error_detail = (
+            f"{base_detail}. {'; '.join(detail_messages)}" if detail_messages else base_detail
+        )
+        raise MissingSourceDependencyError(
+            source_name=source_name,
+            dependencies=[module_name.split(".", maxsplit=1)[0]],
+            uv_groups=uv_groups,
+            detail=error_detail,
+        ) from exc

src/sources/google_cloud_storage/__init__.py

@@ -0,0 +1,3 @@
+from .source import GoogleCloudStorageSource
+
+__all__ = ["GoogleCloudStorageSource"]

src/sources/google_cloud_storage/source.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator
+from typing import Any
+
+from ...models.generated_input import GoogleCloudStorageInput
+from ..dependencies import require_module
+from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
+
+
+class GoogleCloudStorageSource(ObjectStorageSourceBase):
+    source_type = "google_cloud_storage"
+    provider_label = "GOOGLE_CLOUD_STORAGE"
+    input_model = GoogleCloudStorageInput
+
+    def _required_bucket(self) -> str:
+        bucket = str(self.config.required.bucket).strip()
+        if not bucket:
+            raise ValueError("required.bucket must be set")
+        return bucket
+
+    def _build_client(self) -> Any:
+        storage_module = require_module(
+            module_name="google.cloud.storage",
+            source_name="Google Cloud Storage",
+            uv_groups=["google-cloud-storage"],
+            detail="Google Cloud Storage requires google-cloud-storage.",
+        )
+
+        project = self._string_or_none(self._connection_option("project_id"))
+        credentials_json = self._masked_value("gcp_credentials_json")
+        credentials_file = self._string_or_none(self._connection_option("gcp_credentials_file"))
+
+        if credentials_json:
+            service_account_module = require_module(
+                module_name="google.oauth2.service_account",
+                source_name="Google Cloud Storage",
+                uv_groups=["google-cloud-storage"],
+                detail="Inline service account credentials require google-auth support.",
+            )
+            credentials = service_account_module.Credentials.from_service_account_info(
+                json.loads(credentials_json)
+            )
+            return storage_module.Client(project=project, credentials=credentials)
+
+        if credentials_file:
+            service_account_module = require_module(
+                module_name="google.oauth2.service_account",
+                source_name="Google Cloud Storage",
+                uv_groups=["google-cloud-storage"],
+                detail="File-based service account credentials require google-auth support.",
+            )
+            credentials = service_account_module.Credentials.from_service_account_file(
+                credentials_file
+            )
+            return storage_module.Client(project=project, credentials=credentials)
+
+        return storage_module.Client(project=project)
+
+    def _client(self) -> Any:
+        if self._cached_client is None:
+            self._cached_client = self._build_client()
+        return self._cached_client
+
+    def _list_objects(self) -> Iterator[ObjectRef]:
+        client = self._client()
+        bucket = self._required_bucket()
+        prefix = self._prefix()
+        max_keys = self._max_keys_per_page()
+        timeout = self._request_timeout_seconds()
+
+        blobs = client.list_blobs(
+            bucket_or_name=bucket,
+            prefix=prefix or None,
+            page_size=max_keys,
+            timeout=timeout,
+        )
+        for blob in blobs:
+            key = str(getattr(blob, "name", "") or "")
+            if not key or key.endswith("/"):
+                continue
+
+            size = int(getattr(blob, "size", 0) or 0)
+            if size == 0 and not self._include_empty_objects():
+                continue
+            if not self._object_matches_extension_filters(key):
+                continue
+
+            yield ObjectRef(
+                key=key,
+                size=size,
+                last_modified=self._parse_datetime(getattr(blob, "updated", None)),
+                etag=str(getattr(blob, "etag", "") or "") or None,
+                content_type_hint=str(getattr(blob, "content_type", "") or "") or None,
+            )
+
+    def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None, bool]:
+        client = self._client()
+        bucket = client.bucket(self._required_bucket())
+        blob = bucket.blob(ref.key)
+
+        max_bytes = self._max_object_bytes()
+        timeout = self._request_timeout_seconds()
+
+        if ref.size > max_bytes:
+            file_bytes = blob.download_as_bytes(start=0, end=max_bytes - 1, timeout=timeout)
+            return file_bytes, ref.content_type_hint, True
+
+        file_bytes = blob.download_as_bytes(timeout=timeout)
+        return file_bytes, ref.content_type_hint, False
+
+    def _external_url(self, key: str) -> str:
+        return f"gs://{self._required_bucket()}/{key}"

src/sources/hive/__init__.py

@@ -0,0 +1,3 @@
+from .source import HiveSource
+
+__all__ = ["HiveSource"]