classifyre-cli 0.4.45__py3-none-any.whl → 0.4.46__py3-none-any.whl

{classifyre_cli-0.4.45.dist-info → classifyre_cli-0.4.46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.45
+Version: 0.4.46
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.45.dist-info → classifyre_cli-0.4.46.dist-info}/RECORD

@@ -31,7 +31,7 @@ src/detectors/threat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 src/detectors/threat/code_security_detector.py,sha256=SdjhvTSU10wHVxP2EwyzoTGMq8-J8CtwypayHVHdMUs,7049
 src/detectors/threat/yara_detector.py,sha256=VuJ6Tu3o3wMhv2bskSx2YaB7sTILuQaZlK0uWEiLqe0,6549
 src/models/generated_detectors.py,sha256=bDTV8bBG2IJhC37lYu32t_-VTEf-pTPqB_Q-yTMclIo,47709
-src/models/generated_input.py,sha256=kU6vivkgnexaDRhmlYraeSfKzZ9deIQ3b-6y24kDu88,112653
+src/models/generated_input.py,sha256=XA81Wk9H5vFhtRIcExXrPP3WNENDjkeRILzImGIODcQ,115008
 src/models/generated_single_asset_scan_results.py,sha256=BRHBLb9wAKghAF60KPJEYPio2i2tklmbt5kMTLLt2qY,7761
 src/outputs/__init__.py,sha256=41SESY2nC1y4zF0CTWn1dnTh9AK7qMdRxmRSN_GuWss,126
 src/outputs/base.py,sha256=mRFTwqkaWLqyS55OkdbO49yBnSHXlaIGd_bzLyNGj9E,1556
@@ -79,6 +79,8 @@ src/sources/jira/__init__.py,sha256=p3G96i94D6ZrdHKE6bzgVtYvO8h2-Aet4zERRYdJAyA,
 src/sources/jira/source.py,sha256=OApvZQTQC6sV4qKN3jZMpaA2gTdC0U-ky2VTgL2G3ts,24559
 src/sources/kafka/__init__.py,sha256=BJoJs6vgviCpcyJXuUf70i46XqRnBrqcT9dJJZtHC5o,59
 src/sources/kafka/source.py,sha256=3WAJGobP85HHSozH2noaSnsJmEtTkD22AcBrmvaYfy4,17610
+src/sources/local_folder/__init__.py,sha256=WYo83Wg7ONugPZzisjwPkaWe0TADD7AetTpSXs2k908,71
+src/sources/local_folder/source.py,sha256=ed6DdgqIt-w8thchYs6bQVp89lr8sa9dyg6ms5hI3Vc,5472
 src/sources/meilisearch/__init__.py,sha256=2ZESbCcM68x8B7qF_bRUt0Aah32pdzje_YndSHizAjM,71
 src/sources/meilisearch/source.py,sha256=2Gm3WA_1llxmmdrRkmDyOZfMg_Oo8I16sbrGXM8AXyk,14487
 src/sources/mongodb/__init__.py,sha256=IcSTuVMB3JOXAjYpwFcFvRuc2a7m6xgZ1VG5AfM4eJU,63
@@ -130,7 +132,7 @@ src/utils/resources.py,sha256=nE3YBKNBCFMnFErQcO9DKkatHPFQfFt-OsXCpoEz6sA,2125
 src/utils/transcription.py,sha256=_wr0DuPnzBb8tR4IgmMcS0PDSgJ5YbvIBZSx0_iXgIs,14067
 src/utils/uv_sync.py,sha256=VZ0TZ7t3_pSjxOb20BxUs2XLVjigXnVqa5y69XWlnE8,9307
 src/utils/validation.py,sha256=ab6OzB-GeZBMr413U10jG2FDiiWXCil-_e70T0BXwFE,1694
-classifyre_cli-0.4.45.dist-info/METADATA,sha256=BHEwscKkcXC-TzeG6b9DWnPxt1jJAukaEfB_vMuASlE,4222
-classifyre_cli-0.4.45.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-classifyre_cli-0.4.45.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
-classifyre_cli-0.4.45.dist-info/RECORD,,
+classifyre_cli-0.4.46.dist-info/METADATA,sha256=jWJZvJ0w4g_B0KYyTJiG9iMkpU74DBiQceAJItsWmHU,4222
+classifyre_cli-0.4.46.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+classifyre_cli-0.4.46.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
+classifyre_cli-0.4.46.dist-info/RECORD,,

src/models/generated_input.py

@@ -51,6 +51,7 @@ class AssetType(StrEnum):
     ELASTICSEARCH = 'ELASTICSEARCH'
     OPENSEARCH = 'OPENSEARCH'
     MEILISEARCH = 'MEILISEARCH'
+    LOCAL_FOLDER = 'LOCAL_FOLDER'
 
 
 class DetectorType(StrEnum):
@@ -355,6 +356,7 @@ class Type(StrEnum):
     ELASTICSEARCH = 'ELASTICSEARCH'
     OPENSEARCH = 'OPENSEARCH'
     MEILISEARCH = 'MEILISEARCH'
+    LOCAL_FOLDER = 'LOCAL_FOLDER'
 
 
 class YouTubeRequired(BaseModel):
@@ -644,6 +646,64 @@ class S3CompatibleStorageOptional(BaseModel):
     scope: ObjectStorageOptionalScope | None = None
 
 
+class LocalFolderRequired(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    path: str = Field(
+        ...,
+        description='Absolute path of the local folder to scan (for example, /Users/jane/Documents)',
+    )
+
+
+class LocalFolderMasked(BaseModel):
+    """
+    Local folder scans run with the desktop application's filesystem permissions; no credentials are required.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+
+
+class LocalFolderOptionalTraversal(BaseModel):
+    """
+    Directory traversal controls.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    follow_symlinks: bool | None = Field(
+        False,
+        description='Follow symbolic links while traversing (loops are not detected; leave off unless needed)',
+    )
+    include_hidden: bool | None = Field(
+        False,
+        description='Include hidden files and dot-directories (names starting with a dot)',
+    )
+    max_depth: int | None = Field(
+        None,
+        description='Maximum directory depth below the selected folder (unset scans all levels)',
+        ge=1,
+        le=64,
+    )
+    max_file_bytes: int | None = Field(
+        10485760,
+        description='Maximum bytes read per file for MIME detection and text extraction',
+        ge=1024,
+        le=104857600,
+    )
+
+
+class LocalFolderOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    traversal: LocalFolderOptionalTraversal | None = None
+    scope: ObjectStorageOptionalScope | None = None
+
+
 class AzureBlobStorageRequired(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
@@ -2100,6 +2160,24 @@ class S3CompatibleStorageInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class LocalFolderInput(CoreInput):
+    type: Literal['LOCAL_FOLDER'] | None = Field(
+        None, description='Type of the asset or source'
+    )
+    required: LocalFolderRequired
+    masked: LocalFolderMasked | None = None
+    optional: LocalFolderOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class AzureBlobStorageInput(CoreInput):
     type: Literal['AZURE_BLOB_STORAGE'] | None = Field(
         None, description='Type of the asset or source'
@@ -2531,7 +2609,7 @@ class ConfluenceOptionalConnection(BaseModel):
     )
 
 
-class Type18(StrEnum):
+class Type19(StrEnum):
     """
     Filter spaces by space type
     """
@@ -2568,7 +2646,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
     keys: list[str] | None = Field(
         None, description='Filter spaces by keys (up to 250)', max_length=250
     )
-    type: Type18 | None = Field(None, description='Filter spaces by space type')
+    type: Type19 | None = Field(None, description='Filter spaces by space type')
     status: Status | None = Field(None, description='Filter spaces by status')
     labels: list[str] | None = Field(
         None,
@@ -2834,7 +2912,7 @@ class ServiceDeskOptional(BaseModel):
     content: ServiceDeskOptionalContent | None = None
 
 
-class Type19(StrEnum):
+class Type20(StrEnum):
     """
     Type of the asset or source
     """
@@ -2868,6 +2946,7 @@ class Type19(StrEnum):
     ELASTICSEARCH = 'ELASTICSEARCH'
     OPENSEARCH = 'OPENSEARCH'
     MEILISEARCH = 'MEILISEARCH'
+    LOCAL_FOLDER = 'LOCAL_FOLDER'
 
 
 class ConfluenceInput(CoreInput):
@@ -3682,6 +3761,7 @@ class SourceInput(
     RootModel[
         SlackInput
         | S3CompatibleStorageInput
+        | LocalFolderInput
         | AzureBlobStorageInput
         | GoogleCloudStorageInput
         | PostgreSQLInput
@@ -3714,6 +3794,7 @@ class SourceInput(
     root: (
         SlackInput
         | S3CompatibleStorageInput
+        | LocalFolderInput
         | AzureBlobStorageInput
         | GoogleCloudStorageInput
         | PostgreSQLInput

src/sources/local_folder/__init__.py

@@ -0,0 +1,3 @@
+from .source import LocalFolderSource
+
+__all__ = ["LocalFolderSource"]

src/sources/local_folder/source.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import logging
+import os
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+from ...models.generated_input import LocalFolderInput
+from ..object_storage.base import ObjectRef, ObjectStorageSourceBase
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFolderSource(ObjectStorageSourceBase):
+    """Scans a folder on the local filesystem (desktop application deployments).
+
+    Reuses the object-storage pipeline: each file becomes an ObjectRef whose key
+    is the path relative to the scanned root, so sampling, MIME resolution, text
+    extraction, file metadata, and embedded-image child assets all come from
+    ObjectStorageSourceBase.
+    """
+
+    source_type = "local_folder"
+    provider_label = "LOCAL_FOLDER"
+    input_model = LocalFolderInput
+
+    DEFAULT_MAX_FILE_BYTES = 10 * 1024 * 1024
+
+    _resolved_root: Path | None = None
+
+    def _root(self) -> Path:
+        if self._resolved_root is not None:
+            return self._resolved_root
+        raw = str(self.config.required.path).strip()
+        if not raw:
+            raise ValueError("required.path must be set")
+        root = Path(raw).expanduser()
+        if not root.is_absolute():
+            raise ValueError(f"required.path must be an absolute path, got: {raw}")
+        if not root.exists():
+            raise ValueError(f"Folder does not exist: {root}")
+        if not root.is_dir():
+            raise ValueError(f"Path is not a folder: {root}")
+        self._resolved_root = root.resolve()
+        return self._resolved_root
+
+    def _traversal_option(self, key: str, default: Any = None) -> Any:
+        optional = self.config.optional
+        traversal = getattr(optional, "traversal", None) if optional else None
+        if traversal is not None:
+            value = getattr(traversal, key, None)
+            if value is not None:
+                return value
+        return default
+
+    def _follow_symlinks(self) -> bool:
+        return bool(self._traversal_option("follow_symlinks", False))
+
+    def _include_hidden(self) -> bool:
+        return bool(self._traversal_option("include_hidden", False))
+
+    def _max_depth(self) -> int | None:
+        value = self._traversal_option("max_depth")
+        try:
+            return int(value) if value is not None else None
+        except (TypeError, ValueError):
+            return None
+
+    def _max_file_bytes(self) -> int:
+        value = self._traversal_option("max_file_bytes", self.DEFAULT_MAX_FILE_BYTES)
+        try:
+            parsed = int(value)
+        except (TypeError, ValueError):
+            return self.DEFAULT_MAX_FILE_BYTES
+        return max(parsed, 1024)
+
+    def _is_hidden(self, name: str) -> bool:
+        return name.startswith(".")
+
+    def _walk(self, directory: Path, depth: int) -> Iterator[ObjectRef]:
+        max_depth = self._max_depth()
+        follow_symlinks = self._follow_symlinks()
+        include_hidden = self._include_hidden()
+
+        try:
+            entries = sorted(os.scandir(directory), key=lambda entry: entry.name)
+        except OSError as exc:
+            logger.warning("Skipping unreadable directory %s: %s", directory, exc)
+            return
+
+        for entry in entries:
+            if not include_hidden and self._is_hidden(entry.name):
+                continue
+            try:
+                if entry.is_dir(follow_symlinks=follow_symlinks):
+                    if max_depth is None or depth < max_depth:
+                        yield from self._walk(Path(entry.path), depth + 1)
+                    continue
+                if not entry.is_file(follow_symlinks=follow_symlinks):
+                    continue
+                stat = entry.stat(follow_symlinks=follow_symlinks)
+            except OSError as exc:
+                logger.warning("Skipping unreadable entry %s: %s", entry.path, exc)
+                continue
+
+            key = Path(entry.path).relative_to(self._root()).as_posix()
+            if not self._object_matches_extension_filters(key):
+                continue
+            prefix = self._prefix()
+            if prefix and not key.startswith(prefix):
+                continue
+            if stat.st_size == 0 and not self._include_empty_objects():
+                continue
+
+            yield ObjectRef(
+                key=key,
+                size=stat.st_size,
+                last_modified=datetime.fromtimestamp(stat.st_mtime, tz=UTC),
+                # Change signature: mtime+size, so edits produce a new checksum
+                # without hashing file bytes during discovery.
+                etag=f"{stat.st_mtime_ns:x}-{stat.st_size:x}",
+            )
+
+    def _list_objects(self) -> Iterator[ObjectRef]:
+        yield from self._walk(self._root(), depth=0)
+
+    def _download_object(self, ref: ObjectRef) -> tuple[bytes, str | None]:
+        file_path = self._root() / ref.key
+        max_bytes = self._max_file_bytes()
+        with open(file_path, "rb") as handle:
+            file_bytes = handle.read(max_bytes)
+        if ref.size > max_bytes:
+            logger.info(
+                "Truncated %s to %d of %d bytes for content extraction",
+                file_path,
+                max_bytes,
+                ref.size,
+            )
+        return file_bytes, None
+
+    def _external_url(self, key: str) -> str:
+        absolute = (self._root() / key).as_posix()
+        return f"file://{quote(absolute)}"