classifyre-cli 0.4.48__tar.gz → 0.4.50__tar.gz

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 276 packages in 141ms
+Resolved 276 packages in 178ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: classifyre-cli
-Version: 0.4.48
+Version: 0.4.50
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.48",
+  "version": "0.4.50",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.48"
+version = "0.4.50"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_base.py

@@ -47,6 +47,24 @@ _IMAGE_INPUT_CONTENT_TYPES = [*_IMAGE_CONTENT_TYPES, "application/pdf"]
 
 logger = logging.getLogger(__name__)
 
+# Cap per-label entity spans embedded in each finding's metadata. Every finding
+# carries a copy of the pipeline result (the API builds extraction rows from it),
+# so an uncapped regex with thousands of matches would blow up the scan payload
+# quadratically.
+_MAX_EMBEDDED_SPANS_PER_LABEL = 25
+
+
+def _slim_pipeline_result(result: PipelineResult) -> dict[str, Any]:
+    """model_dump with entity span lists capped for embedding in finding metadata."""
+    dump = result.model_dump()
+    entities = dump.get("entities")
+    if isinstance(entities, dict):
+        for label, spans in entities.items():
+            if isinstance(spans, list) and len(spans) > _MAX_EMBEDDED_SPANS_PER_LABEL:
+                entities[label] = spans[:_MAX_EMBEDDED_SPANS_PER_LABEL]
+                dump.setdefault("metadata", {})[f"truncated_spans:{label}"] = len(spans)
+    return dump
+
 
 def _load_input_images(content: bytes, content_type: str, pil: Any) -> list[tuple[int, Any]]:
     """Return ``(page_index, PIL.Image)`` tuples for an image or renderable file.
@@ -139,6 +157,7 @@ class BaseRunner(ABC):
     def _result_to_findings(self, text: str, result: PipelineResult) -> list[DetectionResult]:
         findings: list[DetectionResult] = []
         runner_type = result.metadata.get("runner", "GLINER2")
+        pipeline_result_dump = _slim_pipeline_result(result)
 
         for label, spans in result.entities.items():
             for span in spans:
@@ -162,7 +181,7 @@ class BaseRunner(ABC):
                 meta: dict[str, Any] = {
                     "runner": runner_type,
                     "entity_label": label,
-                    "pipeline_result": result.model_dump(),
+                    "pipeline_result": pipeline_result_dump,
                 }
                 if "groups" in span:
                     meta["capture_groups"] = span["groups"]
@@ -197,7 +216,7 @@ class BaseRunner(ABC):
                         "runner": runner_type,
                         "task": task,
                         "label": label,
-                        "pipeline_result": result.model_dump(),
+                        "pipeline_result": pipeline_result_dump,
                     },
                 )
             )

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_feature_extraction.py

@@ -60,20 +60,44 @@ class FeatureExtractionRunner(BaseRunner):
         self._schema = schema
         self._detector_key = detector_key
         self._detector_name = detector_name
-        ensure_torch("feature_extraction", ["custom", "detectors"])
-        transformers = require_module("transformers", "feature_extraction", ["custom", "detectors"])
-        truncation = schema.truncation if schema.truncation is not None else True
-        tokenizer_kwargs: dict[str, Any] = {"truncation": truncation}
-        if schema.max_length is not None:
-            tokenizer_kwargs["max_length"] = schema.max_length
-        pipeline_kwargs: dict[str, Any] = {
-            "model": schema.model,
-            "device": schema.device or "cpu",
-            "tokenizer_kwargs": tokenizer_kwargs,
-        }
-        if schema.model_revision:
-            pipeline_kwargs["revision"] = schema.model_revision
-        self._pipe: Any = transformers.pipeline("feature-extraction", **pipeline_kwargs)
+        # Model loading is deferred to first detect() so the parent process
+        # (which only routes when a worker pool is active) never pays the
+        # torch import + model memory cost.
+        self._pipe: Any | None = None
+        self._load_error: str | None = None
+
+    def _ensure_pipeline(self) -> Any | None:
+        if self._pipe is not None:
+            return self._pipe
+        if self._load_error is not None:
+            return None
+        schema = self._schema
+        try:
+            ensure_torch("feature_extraction", ["custom", "detectors"])
+            transformers = require_module(
+                "transformers", "feature_extraction", ["custom", "detectors"]
+            )
+            truncation = schema.truncation if schema.truncation is not None else True
+            tokenizer_kwargs: dict[str, Any] = {"truncation": truncation}
+            if schema.max_length is not None:
+                tokenizer_kwargs["max_length"] = schema.max_length
+            pipeline_kwargs: dict[str, Any] = {
+                "model": schema.model,
+                "device": schema.device or "cpu",
+                "tokenizer_kwargs": tokenizer_kwargs,
+            }
+            if schema.model_revision:
+                pipeline_kwargs["revision"] = schema.model_revision
+            self._pipe = transformers.pipeline("feature-extraction", **pipeline_kwargs)
+            return self._pipe
+        except Exception as exc:
+            # Raise on the first failure so the scan records one structured
+            # error; later assets skip quietly via the cached _load_error.
+            self._load_error = str(exc)
+            raise RuntimeError(
+                f"feature_extraction model '{schema.model}' failed to load for "
+                f"detector '{self._detector_key}': {exc}"
+            ) from exc
 
     def run(self, text: str) -> None:  # type: ignore[override]  # pragma: no cover
         raise NotImplementedError("FeatureExtractionRunner uses detect() directly")
@@ -86,6 +110,9 @@ class FeatureExtractionRunner(BaseRunner):
         text = content.strip()
         if not text:
             return []
+        pipe = self._ensure_pipeline()
+        if pipe is None:
+            return []
 
         schema = self._schema
         pooling = str(schema.pooling_strategy or "mean")
@@ -96,7 +123,7 @@ class FeatureExtractionRunner(BaseRunner):
         results: list[DetectionResult] = []
         try:
             for chunk, offset in _chunk_text_with_offsets(text, chunk_size, chunk_overlap):
-                raw: list[list[list[float]]] = self._pipe(chunk) or []
+                raw: list[list[list[float]]] = pipe(chunk) or []
                 if not raw or not raw[0]:
                     continue
                 embedding = _pool_hidden(raw[0], pooling, normalize)

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_gliner2.py

@@ -18,9 +18,20 @@ from ....models.generated_detectors import (
 )
 from ...dependencies import MissingDependencyError, require_module
 from ._base import _DEFAULT_GLINER2_MODEL, BaseRunner
+from ._text_classification import _chunk_text
 
 logger = logging.getLogger(__name__)
 
+# GLiNER2's encoder window is a few hundred tokens; anything past it is silently
+# dropped by a plain extract_entities call. Above this char threshold we use the
+# library's extract_entities_long (chunked) API when available, and run
+# classification per-chunk with max-confidence aggregation.
+_LONG_TEXT_CHAR_THRESHOLD = 3000
+_CLS_CHUNK_SIZE = 3000
+_CLS_CHUNK_OVERLAP = 200
+# Bound per-asset CPU cost on pathological inputs.
+_MAX_CLS_CHUNKS = 50
+
 
 class GLiNER2Runner(BaseRunner):
     """Execute a GLiNER2 pipeline: single-model pass for entities + classification.
@@ -38,6 +49,7 @@ class GLiNER2Runner(BaseRunner):
         self._detector_key = detector_key
         self._detector_name = detector_name
         self._model: Any | None = None
+        self._load_error: str | None = None
         self._setfit_models: dict[str, Any] | None = None
         self._setfit_labels: dict[str, list[str]] = {}
         self._artifact_dir: Path | None = None
@@ -70,21 +82,15 @@ class GLiNER2Runner(BaseRunner):
 
         try:
             if entity_schema:
-                raw = model.extract_entities(
-                    text,
-                    entity_schema,
-                    threshold=0.0,
-                    include_confidence=True,
-                    include_spans=True,
-                )
+                raw = self._extract_entities(model, text, entity_schema)
                 raw_entities = _normalise_entity_output(raw, text)
 
             for task_name, labels in classification_tasks.items():
                 setfit = self._get_setfit_model(task_name)
                 if setfit is not None:
-                    raw_cls = self._run_setfit(setfit, task_name, text)
+                    raw_cls: Any = self._run_setfit(setfit, task_name, text)
                 else:
-                    raw_cls = model.classify(text, labels, threshold=0.0)
+                    raw_cls = self._classify_chunked(model, text, labels)
                 raw_classification[task_name] = _normalise_classification_output(raw_cls)
 
         except Exception as exc:  # pragma: no cover - runtime specific
@@ -113,9 +119,50 @@ class GLiNER2Runner(BaseRunner):
             },
         )
 
+    def _extract_entities(self, model: Any, text: str, entity_schema: dict[str, str]) -> Any:
+        """Extract entities, using the library's chunked long-document API for large texts."""
+        kwargs: dict[str, Any] = {
+            "threshold": 0.0,
+            "include_confidence": True,
+            "include_spans": True,
+        }
+        extract_long = getattr(model, "extract_entities_long", None)
+        if extract_long is not None and len(text) > _LONG_TEXT_CHAR_THRESHOLD:
+            try:
+                return extract_long(text, entity_schema, **kwargs)
+            except TypeError:
+                logger.debug(
+                    "extract_entities_long signature mismatch for detector '%s', "
+                    "falling back to extract_entities",
+                    self._detector_key,
+                )
+        return model.extract_entities(text, entity_schema, **kwargs)
+
+    def _classify_chunked(self, model: Any, text: str, labels: list[str]) -> dict[str, object]:
+        """Classify text, chunking long inputs and keeping the max-confidence label."""
+        chunks = _chunk_text(text, _CLS_CHUNK_SIZE, _CLS_CHUNK_OVERLAP)[:_MAX_CLS_CHUNKS]
+        if len(chunks) == 1:
+            return _normalise_classification_output(model.classify(text, labels, threshold=0.0))
+
+        best: dict[str, object] = {}
+        for chunk in chunks:
+            outcome = _normalise_classification_output(model.classify(chunk, labels, threshold=0.0))
+            if outcome and float(outcome.get("confidence", 0.0)) > float(
+                best.get("confidence", -1.0)
+            ):
+                best = outcome
+        return best
+
     def _load_model(self) -> Any | None:
         if self._model is not None:
             return self._model
+        if self._load_error is not None:
+            logger.debug(
+                "GLiNER2 model previously failed to load for detector '%s': %s",
+                self._detector_key,
+                self._load_error,
+            )
+            return None
 
         if self._artifact_dir is not None:
             gliner_path = self._artifact_dir / "gliner2"
@@ -153,13 +200,14 @@ class GLiNER2Runner(BaseRunner):
         except MissingDependencyError:
             raise
         except Exception as exc:  # pragma: no cover - environment specific
-            logger.warning(
-                "Failed to load GLiNER2 model '%s' for detector '%s': %s",
-                model_name,
-                self._detector_key,
-                exc,
-            )
-            return None
+            # Raise on the first failure so the scan records one structured error
+            # instead of silently reporting zero findings; later assets skip
+            # quietly via the cached _load_error above.
+            self._load_error = str(exc)
+            raise RuntimeError(
+                f"GLiNER2 model '{model_name}' failed to load for detector "
+                f"'{self._detector_key}': {exc}"
+            ) from exc
 
     def _get_setfit_model(self, task_name: str) -> Any | None:
         """Return the SetFit model for task_name if a trained artifact exists."""

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_image_classification.py

@@ -31,24 +31,46 @@ class ImageClassificationRunner(BaseRunner):
         self._schema = schema
         self._detector_key = detector_key
         self._detector_name = detector_name
-        ensure_torch("image_classification", ["custom", "detectors"])
-        transformers = require_module(
-            "transformers", "image_classification", ["custom", "detectors"]
-        )
-        self._pil = require_module("PIL.Image", "image_classification", ["custom", "detectors"])
-        model_id = schema.model or _DEFAULT_IMAGE_CLASSIFICATION_MODEL
-        pipeline_kwargs: dict[str, Any] = {
-            "model": model_id,
-            "device": schema.device or "cpu",
-        }
-        if schema.model_revision:
-            pipeline_kwargs["revision"] = schema.model_revision
-        if schema.top_k is not None:
-            pipeline_kwargs["top_k"] = schema.top_k
-        if schema.function_to_apply is not None:
-            pipeline_kwargs["function_to_apply"] = str(schema.function_to_apply)
-        self._pipe: Any = transformers.pipeline("image-classification", **pipeline_kwargs)
-        self._model_id = model_id
+        self._model_id = schema.model or _DEFAULT_IMAGE_CLASSIFICATION_MODEL
+        # Model loading is deferred to first detect() so the parent process
+        # (which only routes when a worker pool is active) never pays the
+        # torch import + model memory cost.
+        self._pipe: Any | None = None
+        self._pil: Any | None = None
+        self._load_error: str | None = None
+
+    def _ensure_pipeline(self) -> Any | None:
+        if self._pipe is not None:
+            return self._pipe
+        if self._load_error is not None:
+            return None
+        schema = self._schema
+        try:
+            ensure_torch("image_classification", ["custom", "detectors"])
+            transformers = require_module(
+                "transformers", "image_classification", ["custom", "detectors"]
+            )
+            self._pil = require_module("PIL.Image", "image_classification", ["custom", "detectors"])
+            pipeline_kwargs: dict[str, Any] = {
+                "model": self._model_id,
+                "device": schema.device or "cpu",
+            }
+            if schema.model_revision:
+                pipeline_kwargs["revision"] = schema.model_revision
+            if schema.top_k is not None:
+                pipeline_kwargs["top_k"] = schema.top_k
+            if schema.function_to_apply is not None:
+                pipeline_kwargs["function_to_apply"] = str(schema.function_to_apply)
+            self._pipe = transformers.pipeline("image-classification", **pipeline_kwargs)
+            return self._pipe
+        except Exception as exc:
+            # Raise on the first failure so the scan records one structured
+            # error; later assets skip quietly via the cached _load_error.
+            self._load_error = str(exc)
+            raise RuntimeError(
+                f"image_classification model '{self._model_id}' failed to load for "
+                f"detector '{self._detector_key}': {exc}"
+            ) from exc
 
     def run(self, text: str) -> None:  # type: ignore[override]  # pragma: no cover
         raise NotImplementedError("ImageClassificationRunner uses detect() directly")
@@ -58,6 +80,10 @@ class ImageClassificationRunner(BaseRunner):
             logger.warning("image_classification: received string content, expected bytes")
             return []
 
+        pipe = self._ensure_pipeline()
+        if pipe is None:
+            return []
+
         # image/* opens directly; PDFs are rasterised to one image per page.
         images = _load_input_images(content, content_type, self._pil)
         if not images:
@@ -69,7 +95,7 @@ class ImageClassificationRunner(BaseRunner):
         results: list[DetectionResult] = []
         for page_index, image in images:
             try:
-                predictions: list[dict[str, Any]] = self._pipe(image) or []
+                predictions: list[dict[str, Any]] = pipe(image) or []
                 for pred in predictions:
                     label: str = pred.get("label", "unknown")
                     score: float = float(pred.get("score", 0.0))

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_llm.py

@@ -39,6 +39,10 @@ _VISION_CONTENT_TYPES = [*_IMAGE_CONTENT_TYPES, "application/pdf"]
 # token cost and request size for multi-page PDFs.
 _MAX_VISION_IMAGES = 20
 
+# Hard request timeout so a hung provider endpoint can never stall a pool
+# worker for litellm's ~10-minute default. Overridable per deployment.
+_COMPLETION_TIMEOUT_SECONDS = float(os.environ.get("CLASSIFYRE_LLM_TIMEOUT_SECONDS", "90"))
+
 
 class LLMRunner(BaseRunner):
     """AI detector — sends content to a configured LLM provider for classification + extraction."""
@@ -136,6 +140,7 @@ class LLMRunner(BaseRunner):
                 max_tokens=self._max_tokens(),
                 messages=messages,
                 response_format={"type": "json_object"},
+                timeout=_COMPLETION_TIMEOUT_SECONDS,
             )
             raw = response.choices[0].message.content or "{}"
             parsed = self._parse_json(raw)

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_object_detection.py

@@ -30,26 +30,51 @@ class ObjectDetectionRunner(BaseRunner):
         self._schema = schema
         self._detector_key = detector_key
         self._detector_name = detector_name
-        ensure_torch("object_detection", ["custom", "detectors"])
-        transformers = require_module("transformers", "object_detection", ["custom", "detectors"])
-        self._pil = require_module("PIL.Image", "object_detection", ["custom", "detectors"])
-        pipeline_kwargs: dict[str, Any] = {
-            "model": schema.model,
-            "device": schema.device or "cpu",
-        }
-        if schema.model_revision:
-            pipeline_kwargs["revision"] = schema.model_revision
-        nms = getattr(schema.nms_threshold, "root", schema.nms_threshold)
-        if nms is not None:
-            pipeline_kwargs["threshold"] = nms
+        # Model loading is deferred to first detect() so the parent process
+        # (which only routes when a worker pool is active) never pays the
+        # torch import + model memory cost.
+        self._pipe: Any | None = None
+        self._pil: Any | None = None
+        self._load_error: str | None = None
+
+    def _ensure_pipeline(self) -> Any | None:
+        if self._pipe is not None:
+            return self._pipe
+        if self._load_error is not None:
+            return None
+        schema = self._schema
         try:
-            self._pipe: Any = transformers.pipeline("object-detection", **pipeline_kwargs)
-        except ImportError as exc:
-            raise MissingDependencyError(
-                "object_detection",
-                ["custom", "detectors"],
-                ["custom", "detectors"],
-                f"ObjectDetectionRunner requires additional dependencies: {exc}",
+            ensure_torch("object_detection", ["custom", "detectors"])
+            transformers = require_module(
+                "transformers", "object_detection", ["custom", "detectors"]
+            )
+            self._pil = require_module("PIL.Image", "object_detection", ["custom", "detectors"])
+            pipeline_kwargs: dict[str, Any] = {
+                "model": schema.model,
+                "device": schema.device or "cpu",
+            }
+            if schema.model_revision:
+                pipeline_kwargs["revision"] = schema.model_revision
+            nms = getattr(schema.nms_threshold, "root", schema.nms_threshold)
+            if nms is not None:
+                pipeline_kwargs["threshold"] = nms
+            try:
+                self._pipe = transformers.pipeline("object-detection", **pipeline_kwargs)
+            except ImportError as exc:
+                raise MissingDependencyError(
+                    "object_detection",
+                    ["custom", "detectors"],
+                    ["custom", "detectors"],
+                    f"ObjectDetectionRunner requires additional dependencies: {exc}",
+                ) from exc
+            return self._pipe
+        except Exception as exc:
+            # Raise on the first failure so the scan records one structured
+            # error; later assets skip quietly via the cached _load_error.
+            self._load_error = str(exc)
+            raise RuntimeError(
+                f"object_detection model '{schema.model}' failed to load for "
+                f"detector '{self._detector_key}': {exc}"
             ) from exc
 
     def run(self, text: str) -> None:  # type: ignore[override]  # pragma: no cover
@@ -60,6 +85,10 @@ class ObjectDetectionRunner(BaseRunner):
             logger.warning("object_detection: received string content, expected bytes")
             return []
 
+        pipe = self._ensure_pipeline()
+        if pipe is None:
+            return []
+
         # image/* opens directly; PDFs are rasterised to one image per page.
         images = _load_input_images(content, content_type, self._pil)
         if not images:
@@ -71,7 +100,7 @@ class ObjectDetectionRunner(BaseRunner):
         results: list[DetectionResult] = []
         for page_index, image in images:
             try:
-                detections: list[dict[str, Any]] = self._pipe(image) or []
+                detections: list[dict[str, Any]] = pipe(image) or []
                 for det in detections:
                     label: str = det.get("label", "unknown")
                     score: float = float(det.get("score", 0.0))

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_regex.py

@@ -18,6 +18,11 @@ from ._base import BaseRunner
 
 logger = logging.getLogger(__name__)
 
+# Hard per-pattern span cap: a pathological pattern on a large document would
+# otherwise materialise millions of spans before the detector-level
+# max_findings cap applies.
+_MAX_SPANS_PER_PATTERN = 1000
+
 
 def _load_regex_engine() -> tuple[Any, bool]:
     """Try to load google-re2, fall back to stdlib re."""
@@ -130,6 +135,15 @@ class RegexRunner(BaseRunner):
                     span["groups"] = match.groups()
 
                 spans.append(span)
+                if len(spans) >= _MAX_SPANS_PER_PATTERN:
+                    logger.warning(
+                        "Pattern '%s' in detector '%s' hit the %d-span cap; "
+                        "remaining matches dropped",
+                        name,
+                        self._detector_key,
+                        _MAX_SPANS_PER_PATTERN,
+                    )
+                    break
             if spans:
                 entities[name] = spans
 

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/detectors/custom/runners/_text_classification.py

@@ -33,21 +33,43 @@ class TextClassificationRunner(BaseRunner):
         self._schema = schema
         self._detector_key = detector_key
         self._detector_name = detector_name
-        ensure_torch("text_classification", ["custom", "detectors"])
-        transformers = require_module(
-            "transformers", "text_classification", ["custom", "detectors"]
-        )
-        pipeline_kwargs: dict[str, Any] = {
-            "model": schema.model,
-            "device": schema.device or "cpu",
-        }
-        if schema.model_revision:
-            pipeline_kwargs["revision"] = schema.model_revision
-        if schema.top_k is not None:
-            pipeline_kwargs["top_k"] = schema.top_k
-        if schema.function_to_apply is not None:
-            pipeline_kwargs["function_to_apply"] = str(schema.function_to_apply)
-        self._pipe: Any = transformers.pipeline("text-classification", **pipeline_kwargs)
+        # Model loading is deferred to first detect() so the parent process
+        # (which only routes when a worker pool is active) never pays the
+        # torch import + model memory cost.
+        self._pipe: Any | None = None
+        self._load_error: str | None = None
+
+    def _ensure_pipeline(self) -> Any | None:
+        if self._pipe is not None:
+            return self._pipe
+        if self._load_error is not None:
+            return None
+        schema = self._schema
+        try:
+            ensure_torch("text_classification", ["custom", "detectors"])
+            transformers = require_module(
+                "transformers", "text_classification", ["custom", "detectors"]
+            )
+            pipeline_kwargs: dict[str, Any] = {
+                "model": schema.model,
+                "device": schema.device or "cpu",
+            }
+            if schema.model_revision:
+                pipeline_kwargs["revision"] = schema.model_revision
+            if schema.top_k is not None:
+                pipeline_kwargs["top_k"] = schema.top_k
+            if schema.function_to_apply is not None:
+                pipeline_kwargs["function_to_apply"] = str(schema.function_to_apply)
+            self._pipe = transformers.pipeline("text-classification", **pipeline_kwargs)
+            return self._pipe
+        except Exception as exc:
+            # Raise on the first failure so the scan records one structured
+            # error; later assets skip quietly via the cached _load_error.
+            self._load_error = str(exc)
+            raise RuntimeError(
+                f"text_classification model '{schema.model}' failed to load for "
+                f"detector '{self._detector_key}': {exc}"
+            ) from exc
 
     def run(self, text: str) -> None:  # type: ignore[override]  # pragma: no cover
         raise NotImplementedError("TextClassificationRunner uses detect() directly")
@@ -60,6 +82,9 @@ class TextClassificationRunner(BaseRunner):
         text = content.strip()
         if not text:
             return []
+        pipe = self._ensure_pipeline()
+        if pipe is None:
+            return []
 
         schema = self._schema
         chunk_size: int | None = getattr(schema.chunk_size, "root", schema.chunk_size)
@@ -74,7 +99,7 @@ class TextClassificationRunner(BaseRunner):
                 call_kwargs: dict[str, Any] = {"truncation": True}
                 if max_length is not None:
                     call_kwargs["max_length"] = max_length
-                raw = self._pipe(chunk, **call_kwargs) or []
+                raw = pipe(chunk, **call_kwargs) or []
                 preds: list[dict[str, Any]] = raw[0] if raw and isinstance(raw[0], list) else raw
                 for pred in preds:
                     label: str = pred.get("label", "unknown")

{classifyre_cli-0.4.48 → classifyre_cli-0.4.50}/src/main.py

@@ -182,10 +182,14 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     from .pipeline.worker_pool import (
                         DetectorWorkerPool,
                         compute_pool_workers,
+                        recipe_has_ml_custom_detectors,
                     )
 
+                    # ML-backed custom detectors load a model copy per worker
+                    # process; budget ~2GB/worker for those recipes instead of 1GB.
                     pool_workers = compute_pool_workers(
                         override=args.max_pool_workers,
+                        per_worker_mb=2048 if recipe_has_ml_custom_detectors(recipe) else 1024,
                     )
                     worker_pool: DetectorWorkerPool | None = None