classifyre-cli 0.4.34__tar.gz → 0.4.35__tar.gz

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 268 packages in 168ms
+Resolved 271 packages in 177ms
 Checked 50 packages in 1ms

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.34
+Version: 0.4.35
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.34",
+  "version": "0.4.35",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.34"
+version = "0.4.35"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -207,7 +207,12 @@ spark = [
     # LTS) on the host; format JARs (Delta/Iceberg/Hudi) are resolved at runtime
     # via spark.jars.packages (see SPARK_* env vars). Pinned to the Spark 4.1
     # minor so the format-JAR coordinates below stay version-matched.
-    "pyspark>=4.1,<4.2",
+    #
+    # The `connect` extra pulls in pandas>=2.2, pyarrow, and grpcio, which the
+    # Spark Connect client (Spark Catalog via sc:// URLs) hard-requires at session
+    # build time. Classic/local Spark (Delta/Hudi/Iceberg) does not need them, but
+    # they share this group, so we ship the client deps once for all of them.
+    "pyspark[connect]>=4.1,<4.2",
 ]
 delta-lake = [
     { include-group = "spark" },

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/src/models/generated_input.py

@@ -50,6 +50,9 @@ class AssetType(StrEnum):
     HUDI = 'HUDI'
     SPARK_CATALOG = 'SPARK_CATALOG'
     KAFKA = 'KAFKA'
+    ELASTICSEARCH = 'ELASTICSEARCH'
+    OPENSEARCH = 'OPENSEARCH'
+    MEILISEARCH = 'MEILISEARCH'
 
 
 class DetectorType(StrEnum):
@@ -353,6 +356,9 @@ class Type(StrEnum):
     HUDI = 'HUDI'
     SPARK_CATALOG = 'SPARK_CATALOG'
     KAFKA = 'KAFKA'
+    ELASTICSEARCH = 'ELASTICSEARCH'
+    OPENSEARCH = 'OPENSEARCH'
+    MEILISEARCH = 'MEILISEARCH'
 
 
 class YouTubeRequired(BaseModel):
@@ -2865,6 +2871,9 @@ class Type19(StrEnum):
     HUDI = 'HUDI'
     SPARK_CATALOG = 'SPARK_CATALOG'
     KAFKA = 'KAFKA'
+    ELASTICSEARCH = 'ELASTICSEARCH'
+    OPENSEARCH = 'OPENSEARCH'
+    MEILISEARCH = 'MEILISEARCH'
 
 
 class ConfluenceInput(CoreInput):
@@ -3491,25 +3500,68 @@ class IcebergInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
-class KafkaRequired(BaseModel):
+class NoAuthentication(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
     )
+    auth_mode: Literal['NONE']
     bootstrap_servers: str = Field(
         ..., description='Comma-separated Kafka bootstrap servers (host:port)'
     )
 
 
-class KafkaMasked(BaseModel):
+class SASL(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['SASL']
+    bootstrap_servers: str = Field(
+        ..., description='Comma-separated Kafka bootstrap servers (host:port)'
+    )
+
+
+class ClientCertificateMTLS(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['CLIENT_CERT']
+    bootstrap_servers: str = Field(
+        ..., description='Comma-separated Kafka bootstrap servers (host:port)'
+    )
+
+
+class NoAuthentication1(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+
+
+class SASL1(BaseModel):
     """
-    Optional SASL credentials.
+    SASL username/password credentials.
     """
 
     model_config = ConfigDict(
         extra='forbid',
     )
-    sasl_username: str | None = Field(None, description='SASL username')
-    sasl_password: str | None = Field(None, description='SASL password')
+    sasl_username: str = Field(..., description='SASL username')
+    sasl_password: str = Field(..., description='SASL password')
+
+
+class ClientCertificateMTLS1(BaseModel):
+    """
+    mTLS client certificate credentials.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    ssl_certfile: str = Field(
+        ..., description='PEM-encoded client certificate (access cert)'
+    )
+    ssl_keyfile: str = Field(
+        ..., description='PEM-encoded client private key (access key)'
+    )
 
 
 class KafkaOptionalConnection(BaseModel):
@@ -3523,7 +3575,8 @@ class KafkaOptionalConnection(BaseModel):
     security_protocol: KafkaSecurityProtocol | None = 'PLAINTEXT'
     sasl_mechanism: KafkaSaslMechanism | None = 'PLAIN'
     ssl_ca: str | None = Field(
-        None, description='PEM-encoded CA certificate for TLS verification'
+        None,
+        description="PEM-encoded CA certificate for TLS verification (optional for client-certificate auth; validates the broker's certificate)",
     )
     request_timeout_ms: int | None = Field(
         30000, description='Client request timeout in milliseconds', ge=1000
@@ -3562,8 +3615,12 @@ class KafkaInput(CoreInput):
     type: Literal['KAFKA'] | None = Field(
         None, description='Type of the asset or source'
     )
-    required: KafkaRequired
-    masked: KafkaMasked | None = None
+    required: NoAuthentication | SASL | ClientCertificateMTLS = Field(
+        ..., title='KafkaRequired'
+    )
+    masked: NoAuthentication1 | SASL1 | ClientCertificateMTLS1 | None = Field(
+        None, title='KafkaMasked'
+    )
     optional: KafkaOptional | None = None
     detectors: list[Detector] | None = Field(
         None, description='Detectors to run on ingested content'
@@ -3576,6 +3633,254 @@ class KafkaInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class NoAuthentication2(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['NONE']
+    url: AnyUrl = Field(
+        ..., description='Base URL of the cluster (e.g. https://localhost:9200)'
+    )
+
+
+class BasicUsernamePassword(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['BASIC']
+    url: AnyUrl = Field(
+        ..., description='Base URL of the cluster (e.g. https://localhost:9200)'
+    )
+
+
+class APIKeyBearerToken(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['API_KEY']
+    url: AnyUrl = Field(
+        ..., description='Base URL of the cluster (e.g. https://localhost:9200)'
+    )
+
+
+class NoAuthentication3(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+
+
+class BasicUsernamePassword1(BaseModel):
+    """
+    Basic auth credentials.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    username: str = Field(..., description='Basic auth username')
+    password: str = Field(..., description='Basic auth password')
+
+
+class APIKeyBearerToken1(BaseModel):
+    """
+    API key / bearer token credential.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    api_key: str = Field(
+        ..., description='API key or bearer token, sent as an Authorization header'
+    )
+
+
+class SearchEngineOptionalConnection(BaseModel):
+    """
+    Cluster connection controls.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    verify_ssl: bool | None = Field(
+        True, description='TLS certificate verification toggle'
+    )
+    request_timeout_seconds: float | None = Field(
+        30,
+        description='Network timeout in seconds for cluster API calls',
+        ge=1.0,
+        le=300.0,
+    )
+
+
+class SearchEngineOptionalScope(BaseModel):
+    """
+    Index selection scope.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_indices: list[str] | None = Field(
+        None, description='Optional index allowlist'
+    )
+    exclude_indices: list[str] | None = Field(None, description='Index denylist')
+    include_system_indices: bool | None = Field(
+        False, description='Include system indices (names starting with .)'
+    )
+    index_limit: int | None = Field(
+        None, description='Optional cap on number of index assets', ge=1
+    )
+
+
+class ElasticsearchOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    connection: SearchEngineOptionalConnection | None = None
+    scope: SearchEngineOptionalScope | None = None
+
+
+class ElasticsearchInput(CoreInput):
+    type: Literal['ELASTICSEARCH'] | None = Field(
+        None, description='Type of the asset or source'
+    )
+    required: NoAuthentication2 | BasicUsernamePassword | APIKeyBearerToken = Field(
+        ..., title='ElasticsearchRequired'
+    )
+    masked: NoAuthentication3 | BasicUsernamePassword1 | APIKeyBearerToken1 | None = (
+        Field(None, title='ElasticsearchMasked')
+    )
+    optional: ElasticsearchOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
+class OpenSearchOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    connection: SearchEngineOptionalConnection | None = None
+    scope: SearchEngineOptionalScope | None = None
+
+
+class OpenSearchInput(CoreInput):
+    type: Literal['OPENSEARCH'] | None = Field(
+        None, description='Type of the asset or source'
+    )
+    required: NoAuthentication2 | BasicUsernamePassword | APIKeyBearerToken = Field(
+        ..., title='OpenSearchRequired'
+    )
+    masked: NoAuthentication3 | BasicUsernamePassword1 | APIKeyBearerToken1 | None = (
+        Field(None, title='OpenSearchMasked')
+    )
+    optional: OpenSearchOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
+class NoAuthentication4(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['NONE']
+    url: AnyUrl = Field(
+        ...,
+        description='Base URL of the Meilisearch instance (e.g. http://localhost:7700)',
+    )
+
+
+class APIKeyBearerToken2(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    auth_mode: Literal['API_KEY']
+    url: AnyUrl = Field(
+        ...,
+        description='Base URL of the Meilisearch instance (e.g. http://localhost:7700)',
+    )
+
+
+class NoAuthentication5(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+
+
+class APIKeyBearerToken3(BaseModel):
+    """
+    Meilisearch API key or master key, sent as an Authorization: Bearer header. Meilisearch has no separate username/password authentication mode.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    api_key: str = Field(..., description='API key or master key')
+
+
+class MeilisearchOptionalScope(BaseModel):
+    """
+    Index selection scope.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    include_indices: list[str] | None = Field(
+        None, description='Optional index allowlist (matches index uid)'
+    )
+    exclude_indices: list[str] | None = Field(
+        None, description='Index denylist (matches index uid)'
+    )
+    index_limit: int | None = Field(
+        None, description='Optional cap on number of index assets', ge=1
+    )
+
+
+class MeilisearchOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    connection: SearchEngineOptionalConnection | None = None
+    scope: MeilisearchOptionalScope | None = None
+
+
+class MeilisearchInput(CoreInput):
+    type: Literal['MEILISEARCH'] | None = Field(
+        None, description='Type of the asset or source'
+    )
+    required: NoAuthentication4 | APIKeyBearerToken2 = Field(
+        ..., title='MeilisearchRequired'
+    )
+    masked: NoAuthentication5 | APIKeyBearerToken3 | None = Field(
+        None, title='MeilisearchMasked'
+    )
+    optional: MeilisearchOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class YouTubeInput(CoreInput):
     type: Literal['YOUTUBE'] | None = Field(
         None, description='Type of the asset or source'
@@ -3624,6 +3929,9 @@ class SourceInput(
         | HudiInput
         | SparkCatalogInput
         | KafkaInput
+        | ElasticsearchInput
+        | OpenSearchInput
+        | MeilisearchInput
     ]
 ):
     root: (
@@ -3655,6 +3963,9 @@ class SourceInput(
         | HudiInput
         | SparkCatalogInput
         | KafkaInput
+        | ElasticsearchInput
+        | OpenSearchInput
+        | MeilisearchInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',

classifyre_cli-0.4.35/src/sources/elasticsearch/__init__.py

@@ -0,0 +1,3 @@
+from .source import ElasticsearchSource
+
+__all__ = ["ElasticsearchSource"]

classifyre_cli-0.4.35/src/sources/elasticsearch/source.py

@@ -0,0 +1,31 @@
+"""Elasticsearch source — discovers indices and samples documents.
+
+Uses plain REST calls (``requests``) rather than the ``elasticsearch-py``
+client, since only read-only cluster/index/search endpoints are needed and
+those are stable across Elasticsearch versions. Shared with ``OpenSearchSource``
+via :mod:`src.sources.search_engine_base` — see that module for the REST logic.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ...models.generated_input import ElasticsearchInput
+from ..base import BaseSource
+from ..search_engine_base import SearchEngineSourceMixin
+
+
+class ElasticsearchSource(SearchEngineSourceMixin, BaseSource):
+    source_type = "elasticsearch"
+    ENGINE_LABEL = "Elasticsearch"
+
+    def __init__(
+        self,
+        recipe: dict[str, Any],
+        source_id: str | None = None,
+        runner_id: str | None = None,
+    ) -> None:
+        super().__init__(recipe, source_id, runner_id)
+        self.config = ElasticsearchInput.model_validate(recipe)
+        self.runner_id = runner_id or "local-run"
+        self._index_lookup: dict[str, str] = {}

{classifyre_cli-0.4.34 → classifyre_cli-0.4.35}/src/sources/kafka/source.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import logging
 import ssl as ssl_module
+import tempfile
 from collections.abc import AsyncGenerator
 from datetime import UTC, datetime
 from typing import Any
@@ -67,6 +68,7 @@ class KafkaSource(BaseSource):
     def _client_kwargs(self) -> dict[str, Any]:
         kwargs: dict[str, Any] = {"bootstrap_servers": self._bootstrap_servers()}
         connection = self._connection()
+        ssl_ca = getattr(connection, "ssl_ca", None) if connection is not None else None
         if connection is not None:
             protocol = getattr(connection, "security_protocol", None)
             if protocol is not None:
@@ -80,17 +82,34 @@ class KafkaSource(BaseSource):
                 )
             if getattr(connection, "request_timeout_ms", None):
                 kwargs["request_timeout_ms"] = int(connection.request_timeout_ms)
-            if getattr(connection, "ssl_ca", None):
-                context = ssl_module.create_default_context(cadata=connection.ssl_ca)
-                kwargs["ssl_context"] = context
         masked = self.config.masked
-        if masked is not None:
-            if getattr(masked, "sasl_username", None):
-                kwargs["sasl_plain_username"] = masked.sasl_username
-            if getattr(masked, "sasl_password", None):
-                kwargs["sasl_plain_password"] = masked.sasl_password
+        if getattr(masked, "sasl_username", None):
+            kwargs["sasl_plain_username"] = masked.sasl_username
+        if getattr(masked, "sasl_password", None):
+            kwargs["sasl_plain_password"] = masked.sasl_password
+        ssl_certfile = getattr(masked, "ssl_certfile", None)
+        ssl_keyfile = getattr(masked, "ssl_keyfile", None)
+        if ssl_ca or ssl_certfile:
+            context = ssl_module.create_default_context(cadata=ssl_ca)
+            if ssl_certfile and ssl_keyfile:
+                self._load_client_cert_chain(context, ssl_certfile, ssl_keyfile)
+            kwargs["ssl_context"] = context
         return kwargs
 
+    @staticmethod
+    def _load_client_cert_chain(
+        context: ssl_module.SSLContext, certfile: str, keyfile: str
+    ) -> None:
+        with (
+            tempfile.NamedTemporaryFile("w", suffix=".pem") as cert_tmp,
+            tempfile.NamedTemporaryFile("w", suffix=".pem") as key_tmp,
+        ):
+            cert_tmp.write(certfile)
+            cert_tmp.flush()
+            key_tmp.write(keyfile)
+            key_tmp.flush()
+            context.load_cert_chain(certfile=cert_tmp.name, keyfile=key_tmp.name)
+
     def _make_consumer(self, **extra: Any) -> Any:
         kwargs = {**self._client_kwargs(), "enable_auto_commit": False, **extra}
         return self._kafka.KafkaConsumer(**kwargs)

classifyre_cli-0.4.35/src/sources/meilisearch/__init__.py

@@ -0,0 +1,3 @@
+from .source import MeilisearchSource
+
+__all__ = ["MeilisearchSource"]