classifyre-cli 0.4.14__tar.gz → 0.4.16__tar.gz

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 268 packages in 224ms
+Resolved 255 packages in 216ms
 Checked 50 packages in 1ms

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.14
+Version: 0.4.16
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.14",
+  "version": "0.4.16",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.14"
+version = "0.4.16"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -21,6 +21,18 @@ dependencies = [
 # Local editable install for development — ignored when published to PyPI
 classifyre-schemas = { path = "../../packages/schemas", editable = true }
 en-core-web-sm = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }
+# CPU-only torch on Linux. The default PyPI Linux wheels bundle the full NVIDIA
+# CUDA stack (~5-7 GB) which is dead weight on our CPU-only CLI jobs (no GPU
+# requested). Pull torch/torchvision from the PyTorch CPU index on Linux so the
+# nvidia-* packages drop out entirely. macOS/other platforms already use CPU
+# wheels from PyPI. (A GPU image can override this in the future.)
+torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
+torchvision = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
+
+[[tool.uv.index]]
+name = "pytorch-cpu"
+url = "https://download.pytorch.org/whl/cpu"
+explicit = true
 
 [project.scripts]
 classifyre = "src.main:main"
@@ -82,6 +94,7 @@ custom = [
     "transformers>=4.50.0,<6.0.0",
     "torch>=2.12.0",
     "pillow>=12.2.0",
+    "timm>=1.0.27",
     "setfit>=1.1.3",
     "datasets>=4.8.5",
     "scikit-learn>=1.7.2",
@@ -167,6 +180,9 @@ azure-blob-storage = [
 google-cloud-storage = [
     "google-cloud-storage>=3.10.1,<4.0.0",
 ]
+email = [
+    "imap-tools>=1.10.0,<2.0.0",
+]
 otel = [
     "opentelemetry-sdk>=1.42.0",
     "opentelemetry-exporter-otlp-proto-http>=1.27.0",

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/src/detectors/custom/runners/_object_detection.py

@@ -48,6 +48,7 @@ class ObjectDetectionRunner(BaseRunner):
             raise MissingDependencyError(
                 "object_detection",
                 ["custom", "detectors"],
+                ["custom", "detectors"],
                 f"ObjectDetectionRunner requires additional dependencies: {exc}",
             ) from exc
 

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/src/main.py

@@ -197,6 +197,27 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     has_detectors = bool(pipeline.detectors)
 
                     if has_detectors:
+                        # Warm this run's optional dependency groups once, here in
+                        # the parent process, before the worker pool spawns — so the
+                        # pool's worker processes don't each race on their own
+                        # `uv sync` against the shared venv. Best-effort: the
+                        # lock-protected require_module path remains the safety net.
+                        from .utils.dependency_groups import recipe_uv_groups
+                        from .utils.uv_sync import warm_groups
+
+                        warm = recipe_uv_groups(recipe)
+                        if warm:
+                            logger.info(
+                                "Warming optional dependency groups: %s",
+                                ", ".join(sorted(warm)),
+                            )
+                            warm_ok, warm_detail = warm_groups(warm)
+                            if not warm_ok:
+                                logger.warning(
+                                    "Dependency warm-up incomplete (workers will retry): %s",
+                                    warm_detail,
+                                )
+
                         worker_pool = DetectorWorkerPool(max_workers=pool_workers)
                         pipeline = DetectorPipeline.from_recipe(
                             recipe,
@@ -351,6 +372,16 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                         total_assets,
                         output_batch_count,
                     )
+
+                    # Phase 1: emit source-derived relationship edges (best-effort).
+                    if hasattr(source, "collect_relationships") and hasattr(sink, "emit_edges"):
+                        try:
+                            edges = await source.collect_relationships()
+                            if edges:
+                                await sink.emit_edges(edges)
+                                logger.info("Emitted %d source-derived relationship edges", len(edges))
+                        except Exception as rel_error:
+                            logger.warning("Relationship emission failed (non-fatal): %s", rel_error)
                 except Exception as extraction_error:
                     if _is_timeout_error(extraction_error):
                         logger.warning(

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/src/models/generated_input.py

@@ -43,6 +43,7 @@ class AssetType(StrEnum):
     SERVICEDESK = 'SERVICEDESK'
     SQLITE = 'SQLITE'
     NOTION = 'NOTION'
+    EMAIL = 'EMAIL'
 
 
 class SourceCategory(StrEnum):
@@ -238,6 +239,85 @@ class WordPressOptional(BaseModel):
     content: WordPressOptionalContent | None = None
 
 
+class EmailRequired(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    host: str = Field(
+        ..., description='IMAP server host (e.g. imap.gmail.com, outlook.office365.com)'
+    )
+    port: int | None = Field(
+        993, description='IMAP server port (993 for IMAPS)', ge=1, le=65535
+    )
+
+
+class EmailMasked(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    username: str = Field(
+        ..., description='Mailbox login (usually the full email address)'
+    )
+    password: str = Field(
+        ...,
+        description='Account or app-specific password (use an app password for Gmail/Outlook)',
+    )
+
+
+class EmailOptionalConnection(BaseModel):
+    """
+    Transport-level connection controls.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    use_ssl: bool | None = Field(
+        True,
+        description='Connect over implicit TLS (IMAPS). Disable only for STARTTLS/plain servers.',
+    )
+    timeout_seconds: int | None = Field(
+        30, description='Socket timeout for IMAP operations', ge=1
+    )
+
+
+class EmailOptionalScope(BaseModel):
+    """
+    Which messages and attachments to ingest.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    folders: list[str] | None = Field(['INBOX'], description='Mailbox folders to scan')
+    since_date: str | None = Field(
+        None,
+        description='Only fetch messages on/after this date (ISO 8601 date, e.g. 2026-01-01)',
+    )
+    before_date: str | None = Field(
+        None, description='Only fetch messages before this date (ISO 8601 date)'
+    )
+    unseen_only: bool | None = Field(
+        False, description='Only fetch unread (UNSEEN) messages'
+    )
+    include_attachments: bool | None = Field(
+        True, description='Emit attachments as separate assets linked to the email'
+    )
+    max_attachment_size_bytes: int | None = Field(
+        None,
+        description='Skip downloading attachment bytes above this size (still emits a metadata-only asset). Unset means no limit.',
+        ge=0,
+    )
+
+
+class EmailOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    connection: EmailOptionalConnection | None = None
+    scope: EmailOptionalScope | None = None
+
+
 class SlackRequired(BaseModel):
     model_config = ConfigDict(
         extra='forbid',
@@ -1872,6 +1952,7 @@ class Type(StrEnum):
     SERVICEDESK = 'SERVICEDESK'
     SQLITE = 'SQLITE'
     NOTION = 'NOTION'
+    EMAIL = 'EMAIL'
 
 
 class SlackInput(CoreInput):
@@ -1892,6 +1973,22 @@ class SlackInput(CoreInput):
     resources: ResourceOverrides | None = None
 
 
+class EmailInput(CoreInput):
+    type: Literal['EMAIL'] = Field('EMAIL', description='Type of the asset or source')
+    required: EmailRequired
+    masked: EmailMasked
+    optional: EmailOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    resources: ResourceOverrides | None = None
+
+
 class S3CompatibleStorageInput(CoreInput):
     type: Literal['S3_COMPATIBLE_STORAGE'] = Field(
         'S3_COMPATIBLE_STORAGE', description='Type of the asset or source'
@@ -2331,7 +2428,7 @@ class ConfluenceOptionalConnection(BaseModel):
     )
 
 
-class Type16(StrEnum):
+class Type17(StrEnum):
     """
     Filter spaces by space type
     """
@@ -2368,7 +2465,7 @@ class ConfluenceOptionalScopeSpaces(BaseModel):
     keys: list[str] | None = Field(
         None, description='Filter spaces by keys (up to 250)', max_length=250
     )
-    type: Type16 | None = Field(None, description='Filter spaces by space type')
+    type: Type17 | None = Field(None, description='Filter spaces by space type')
     status: Status | None = Field(None, description='Filter spaces by status')
     labels: list[str] | None = Field(
         None,
@@ -2634,7 +2731,7 @@ class ServiceDeskOptional(BaseModel):
     content: ServiceDeskOptionalContent | None = None
 
 
-class Type17(StrEnum):
+class Type18(StrEnum):
     """
     Type of the asset or source
     """
@@ -2660,6 +2757,7 @@ class Type17(StrEnum):
     SERVICEDESK = 'SERVICEDESK'
     SQLITE = 'SQLITE'
     NOTION = 'NOTION'
+    EMAIL = 'EMAIL'
 
 
 class ConfluenceInput(CoreInput):
@@ -2920,6 +3018,7 @@ class SourceInput(
         | ServiceDeskInput
         | SQLiteInput
         | NotionInput
+        | EmailInput
     ]
 ):
     root: (
@@ -2944,6 +3043,7 @@ class SourceInput(
         | ServiceDeskInput
         | SQLiteInput
         | NotionInput
+        | EmailInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/src/outputs/rest.py

@@ -97,6 +97,31 @@ class BulkIngestAssetsRequest(BaseModel):
     skip_findings: bool = Field(False, serialization_alias="skipFindings")
 
 
+class IngestEdge(BaseModel):
+    """A source-derived relationship edge for the investigation graph.
+
+    Identify endpoints by UUID (from_id / to_id) or by asset hash
+    (from_hash / to_hash — the API resolves hashes to UUIDs).
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    from_type: str = Field(serialization_alias="fromType")
+    from_id: str | None = Field(None, serialization_alias="fromId")
+    from_hash: str | None = Field(None, serialization_alias="fromHash")
+    to_type: str = Field(serialization_alias="toType")
+    to_id: str | None = Field(None, serialization_alias="toId")
+    to_hash: str | None = Field(None, serialization_alias="toHash")
+    relation_type: str = Field(serialization_alias="relationType")
+    confidence: float = 1.0
+
+
+class BulkIngestEdgesRequest(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    edges: list[IngestEdge]
+
+
 class FinalizeIngestRunRequest(BaseModel):
     model_config = ConfigDict(populate_by_name=True)
 
@@ -252,6 +277,30 @@ class RestOutputSink:
                 update_error,
             )
 
+    async def emit_edges(self, edges: list[IngestEdge]) -> None:
+        """Bulk-upsert source-derived relationship edges to the investigation graph.
+
+        Idempotent — safe to call multiple times with overlapping data.
+        Silently skips if the list is empty.
+        """
+        if not edges:
+            return
+
+        _edge_batch = 500
+        for i in range(0, len(edges), _edge_batch):
+            chunk = edges[i : i + _edge_batch]
+            payload = BulkIngestEdgesRequest(edges=chunk)
+            try:
+                self._request_json(
+                    "POST",
+                    "/graph/edges",
+                    payload.model_dump(mode="json", by_alias=True),
+                )
+                logger.debug("Emitted %d source-derived edges to graph", len(chunk))
+            except Exception as exc:
+                # Edge emission is best-effort: log and continue.
+                logger.warning("Failed to emit edges to graph: %s", exc)
+
     async def register_discovered_assets(self, hashes: list[str]) -> None:
         runner_id = self._require_runner_id()
         for i in range(0, len(hashes), 500):

{classifyre_cli-0.4.14 → classifyre_cli-0.4.16}/src/sources/base.py

@@ -4,6 +4,7 @@ from collections.abc import AsyncGenerator, Generator
 from typing import TYPE_CHECKING, Any
 
 from ..models.generated_single_asset_scan_results import DetectionResult, SingleAssetScanResults
+from ..outputs.rest import IngestEdge
 
 if TYPE_CHECKING:
     from ..utils.file_parser import ParsedBytes
@@ -315,3 +316,15 @@ class BaseSource(ABC):
         this and map those identifiers back to their original URLs.
         """
         return normalize_http_url(link)
+
+
+    async def collect_relationships(self) -> list[IngestEdge]:
+        """Return source-derived relationship edges for the investigation graph.
+
+        Connectors override this to emit typed edges (READS, ATTACHED_TO,
+        SENT_TO, OWNS, ACCESSED, etc.) discovered during extraction. The caller
+        (main.py) will forward these to ``RestOutputSink.emit_edges()``.
+
+        Default: no relationships (empty list).
+        """
+        return []

classifyre_cli-0.4.16/src/sources/email/__init__.py

@@ -0,0 +1,3 @@
+from .source import EmailSource
+
+__all__ = ["EmailSource"]