classifyre-cli 0.4.2__py3-none-any.whl

src/sources/recipe_normalizer.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from copy import deepcopy
+from typing import Any
+
+_VALID_SAMPLING_STRATEGIES = {"RANDOM", "LATEST", "ALL"}
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    if isinstance(value, dict):
+        return value
+    return {}
+
+
+def _as_positive_int(value: Any) -> int | None:
+    try:
+        if value is None:
+            return None
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _normalize_sampling_strategy(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip().upper()
+    return normalized if normalized in _VALID_SAMPLING_STRATEGIES else None
+
+
+def _as_bool(value: Any) -> bool | None:
+    if isinstance(value, bool):
+        return value
+    return None
+
+
+def _pick(*values: Any) -> Any:
+    for value in values:
+        if value is not None:
+            return value
+    return None
+
+
+def _normalize_object_storage_shape(
+    normalized: dict[str, Any],
+    source_type_value: str,
+) -> None:
+    required = _as_dict(normalized.get("required"))
+    optional = _as_dict(normalized.get("optional"))
+    optional_connection = _as_dict(optional.get("connection"))
+    optional_scope = _as_dict(optional.get("scope"))
+
+    for key in (
+        "request_timeout_seconds",
+        "max_keys_per_page",
+        "max_object_bytes",
+    ):
+        value = normalized.get(key)
+        if value is None:
+            continue
+        optional_connection.setdefault(key, value)
+        normalized.pop(key, None)
+
+    for key in (
+        "prefix",
+        "include_extensions",
+        "exclude_extensions",
+        "include_empty_objects",
+        "include_object_metadata",
+        "include_content_preview",
+    ):
+        value = normalized.get(key)
+        if value is None:
+            continue
+        optional_scope.setdefault(key, value)
+        normalized.pop(key, None)
+
+    if source_type_value == "S3_COMPATIBLE_STORAGE":
+        if (bucket := normalized.pop("bucket", None)) is not None:
+            required.setdefault("bucket", bucket)
+        for key in ("endpoint_url", "region_name", "verify_ssl"):
+            value = normalized.get(key)
+            if value is None:
+                continue
+            optional_connection.setdefault(key, value)
+            normalized.pop(key, None)
+
+    if source_type_value == "AZURE_BLOB_STORAGE":
+        if (account_url := normalized.pop("account_url", None)) is not None:
+            required.setdefault("account_url", account_url)
+        if (container := normalized.pop("container", None)) is not None:
+            required.setdefault("container", container)
+
+    if source_type_value == "GOOGLE_CLOUD_STORAGE":
+        if (bucket := normalized.pop("bucket", None)) is not None:
+            required.setdefault("bucket", bucket)
+        for key in ("project_id", "gcp_credentials_file"):
+            value = normalized.get(key)
+            if value is None:
+                continue
+            optional_connection.setdefault(key, value)
+            normalized.pop(key, None)
+
+    required.pop("provider", None)
+    normalized["required"] = required
+    if optional_connection:
+        optional["connection"] = optional_connection
+    if optional_scope:
+        optional["scope"] = optional_scope
+    if optional:
+        normalized["optional"] = optional
+
+
+def normalize_source_recipe(
+    recipe: dict[str, Any],
+    source_type: str | None = None,
+) -> dict[str, Any]:
+    normalized = deepcopy(recipe)
+    source_type_value = str(source_type or normalized.get("type") or "").upper()
+
+    if source_type_value:
+        normalized["type"] = source_type_value
+
+    optional = _as_dict(normalized.get("optional"))
+    optional_sampling = _as_dict(optional.get("sampling"))
+    sampling = _as_dict(normalized.get("sampling"))
+
+    strategy = _pick(
+        _normalize_sampling_strategy(sampling.get("strategy")),
+        _normalize_sampling_strategy(optional_sampling.get("strategy")),
+        _normalize_sampling_strategy(optional_sampling.get("mode")),
+        "RANDOM",
+    )
+
+    sampling["strategy"] = strategy
+    # Strip removed fields so legacy recipes with limit/max_columns don't fail validation
+    sampling.pop("limit", None)
+    sampling.pop("max_columns", None)
+
+    for key in (
+        "order_by_column",
+        "fallback_to_random",
+        "rows_per_page",
+        "include_column_names",
+    ):
+        if key not in sampling and key in optional_sampling:
+            sampling[key] = optional_sampling[key]
+
+    sampling.pop("fetch_all_until_first_success", None)
+
+    normalized["sampling"] = sampling
+
+    optional.pop("sampling", None)
+
+    if optional:
+        normalized["optional"] = optional
+
+    if source_type_value == "WORDPRESS":
+        required = _as_dict(normalized.get("required"))
+        if isinstance(normalized.get("url"), str):
+            required.setdefault("url", normalized.pop("url"))
+        normalized["required"] = required
+        normalized.setdefault("masked", _as_dict(normalized.get("masked")))
+
+    if source_type_value == "SLACK":
+        required = _as_dict(normalized.get("required"))
+        if isinstance(normalized.get("workspace"), str):
+            required.setdefault("workspace", normalized.pop("workspace"))
+        normalized["required"] = required
+
+    if source_type_value in {
+        "S3_COMPATIBLE_STORAGE",
+        "AZURE_BLOB_STORAGE",
+        "GOOGLE_CLOUD_STORAGE",
+    }:
+        _normalize_object_storage_shape(normalized, source_type_value)
+
+    return normalized

src/sources/s3_compatible_storage/README.md

@@ -0,0 +1,66 @@
+# S3-Compatible Storage Source
+
+This source uses the `S3_COMPATIBLE_STORAGE` schema and reads credentials from `config.masked`.
+
+## Schema Field Mapping
+
+Use these schema paths when configuring any S3-compatible provider:
+
+- `config.required.bucket`: Bucket name (for example `testclassifyrebucket`)
+- `config.masked.aws_access_key_id`: Access key ID (not key name)
+- `config.masked.aws_secret_access_key`: Secret access key
+- `config.masked.aws_session_token`: Optional temporary session token
+- `config.optional.connection.endpoint_url`: Provider S3 endpoint URL
+- `config.optional.connection.region_name`: Region used for SigV4 signing
+- `config.optional.connection.verify_ssl`: TLS verification toggle (default `true`)
+- `config.optional.connection.request_timeout_seconds`: Network timeout for list/download calls
+- `config.optional.connection.max_keys_per_page`: Max objects per list page
+- `config.optional.connection.max_object_bytes`: Max bytes downloaded per object for preview/extraction
+- `config.optional.scope.prefix`: Optional prefix filter
+- `config.optional.scope.include_extensions`: Optional include extension filter
+- `config.optional.scope.exclude_extensions`: Optional exclude extension filter
+- `config.optional.scope.include_empty_objects`: Include zero-byte objects
+- `config.optional.scope.include_object_metadata`: Include object metadata
+- `config.optional.scope.include_content_preview`: Download content for MIME/text preview
+- `config.sampling.strategy`: `RANDOM`, `LATEST`, or `ALL`
+- `config.sampling.rows_per_page`: Item limit per sample run (default 100, ignored when strategy is `ALL`)
+
+## Backblaze B2 Example
+
+```json
+{
+  "type": "S3_COMPATIBLE_STORAGE",
+  "required": {
+    "bucket": "testclassifyrebucket"
+  },
+  "masked": {
+    "aws_access_key_id": "002b0e7121683000000000001",
+    "aws_secret_access_key": "K002ZQTPmV9xMTEGg/F3AGtDFzdZgnY"
+  },
+  "optional": {
+    "connection": {
+      "endpoint_url": "https://s3.us-west-002.backblazeb2.com",
+      "region_name": "us-west-002",
+      "verify_ssl": true,
+      "request_timeout_seconds": 30
+    },
+    "scope": {
+      "include_empty_objects": false,
+      "include_object_metadata": true,
+      "include_content_preview": true
+    }
+  },
+  "sampling": {
+    "strategy": "LATEST"
+  }
+}
+```
+
+## Common Misconfigurations
+
+- `InvalidRequest: The Credential is malformed`:
+  - `aws_access_key_id` is wrong (often secret key pasted into this field).
+- `InvalidAccessKeyId`:
+  - Access key ID does not exist for this B2 application key.
+- `SUCCESS` but `Listed 0 object(s)`:
+  - `optional.scope.prefix` does not match actual key paths, or extension filters exclude files.

src/sources/s3_compatible_storage/__init__.py

@@ -0,0 +1,3 @@
+from .source import S3CompatibleStorageSource
+
+__all__ = ["S3CompatibleStorageSource"]

src/sources/s3_compatible_storage/source.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import quote
+
+from ...models.generated_input import S3CompatibleStorageInput
+from ..dependencies import require_module
+from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
+
+logger = logging.getLogger(__name__)
+
+
+class S3CompatibleStorageSource(ObjectStorageSourceBase):
+    source_type = "s3_compatible_storage"
+    provider_label = "S3_COMPATIBLE_STORAGE"
+    input_model = S3CompatibleStorageInput
+
+    def _required_bucket(self) -> str:
+        bucket = str(self.config.required.bucket).strip()
+        if not bucket:
+            raise ValueError("required.bucket must be set")
+        return bucket
+
+    def _build_client(self) -> Any:
+        boto3 = require_module(
+            module_name="boto3",
+            source_name="S3 Compatible Storage",
+            uv_groups=["s3-compatible-storage"],
+            detail="S3-compatible storage requires boto3.",
+        )
+
+        kwargs: dict[str, Any] = {}
+        region_name = self._string_or_none(self._connection_option("region_name"))
+        endpoint_url = self._string_or_none(self._connection_option("endpoint_url"))
+        aws_access_key_id = self._masked_value("aws_access_key_id")
+        aws_secret_access_key = self._masked_value("aws_secret_access_key")
+        aws_session_token = self._masked_value("aws_session_token")
+
+        if region_name:
+            kwargs["region_name"] = region_name
+        if endpoint_url:
+            kwargs["endpoint_url"] = endpoint_url
+        if aws_access_key_id and aws_secret_access_key:
+            kwargs["aws_access_key_id"] = aws_access_key_id
+            kwargs["aws_secret_access_key"] = aws_secret_access_key
+            if aws_session_token:
+                kwargs["aws_session_token"] = aws_session_token
+
+        kwargs["verify"] = self._verify_ssl()
+
+        try:
+            botocore_config = require_module(
+                module_name="botocore.config",
+                source_name="S3 Compatible Storage",
+                uv_groups=["s3-compatible-storage"],
+                detail="S3-compatible storage uses botocore timeout configuration.",
+            )
+            timeout = int(self._request_timeout_seconds())
+            kwargs["config"] = botocore_config.Config(
+                connect_timeout=timeout,
+                read_timeout=timeout,
+            )
+        except Exception:
+            logger.debug("Could not initialize botocore timeout configuration; using defaults")
+
+        return boto3.client("s3", **kwargs)
+
+    def _client(self) -> Any:
+        if self._cached_client is None:
+            self._cached_client = self._build_client()
+        return self._cached_client
+
+    def _list_objects(self) -> Iterator[ObjectRef]:
+        client = self._client()
+        bucket = self._required_bucket()
+        prefix = self._prefix()
+        max_keys = self._max_keys_per_page()
+
+        continuation_token: str | None = None
+
+        while True:
+            params: dict[str, Any] = {
+                "Bucket": bucket,
+                "MaxKeys": max_keys,
+            }
+            if prefix:
+                params["Prefix"] = prefix
+            if continuation_token:
+                params["ContinuationToken"] = continuation_token
+
+            response = client.list_objects_v2(**params)
+            for item in response.get("Contents", []) or []:
+                key = str(item.get("Key") or "")
+                if not key or key.endswith("/"):
+                    continue
+
+                size = int(item.get("Size") or 0)
+                if size == 0 and not self._include_empty_objects():
+                    continue
+                if not self._object_matches_extension_filters(key):
+                    continue
+
+                yield ObjectRef(
+                    key=key,
+                    size=size,
+                    last_modified=self._parse_datetime(item.get("LastModified")),
+                    etag=str(item.get("ETag")).strip('"') if item.get("ETag") else None,
+                )
+
+            if not response.get("IsTruncated"):
+                break
+            continuation_token = response.get("NextContinuationToken")
+            if not continuation_token:
+                break
+
+    def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None, bool]:
+        client = self._client()
+        bucket = self._required_bucket()
+        max_bytes = self._max_object_bytes()
+
+        params: dict[str, Any] = {"Bucket": bucket, "Key": ref.key}
+        truncated = False
+        if ref.size > max_bytes:
+            params["Range"] = f"bytes=0-{max_bytes - 1}"
+            truncated = True
+
+        response = client.get_object(**params)
+        body = response["Body"]
+        try:
+            file_bytes = body.read()
+        finally:
+            try:
+                body.close()
+            except Exception:
+                logger.debug("Failed to close S3 response body")
+
+        content_type = response.get("ContentType")
+        return file_bytes, str(content_type) if content_type else None, truncated
+
+    def _external_url(self, key: str) -> str:
+        bucket = self._required_bucket()
+        endpoint_url = self._string_or_none(self._connection_option("endpoint_url"))
+        if endpoint_url:
+            endpoint = endpoint_url.rstrip("/")
+            encoded_bucket = quote(bucket, safe="")
+            encoded_key = quote(key, safe="/")
+            return f"{endpoint}/{encoded_bucket}/{encoded_key}"
+        return f"s3://{bucket}/{key}"

src/sources/servicedesk/__init__.py

@@ -0,0 +1,3 @@
+from .source import ServiceDeskSource
+
+__all__ = ["ServiceDeskSource"]