classifyre-cli 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

{classifyre_cli-0.6.2.dist-info → classifyre_cli-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.2
+Version: 0.6.4
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.2.dist-info → classifyre_cli-0.6.4.dist-info}/RECORD

@@ -1,18 +1,18 @@
 src/__init__.py,sha256=a9znVQMuKTEdH6nB8kUs7jMmPV1U6IFXDzCU5RnzxFo,23
 src/config.py,sha256=c4v6RbUQuWNIaCySaL5isNmBcUnYZimaHBW7Ki2InpU,2688
-src/main.py,sha256=FznnPftGLJb_awTo1NXyTIcmPNYxwfhGP00ObmKKQlo,57046
+src/main.py,sha256=jvEIppZQgN2Jbw2OajJHukkSmcEnCNjUC9Dyhkdnqk4,58495
 src/telemetry.py,sha256=fbSmuJ_t-kSthfIh3E7ESVkloH22kzl8lQKY370YtuU,3089
 src/augmentation/__init__.py,sha256=pfb4zDjcyo62kHtvRl9QJ-xV28K0A_DTT_kJoF559Lc,147
 src/augmentation/contract.py,sha256=V0XQn7gk--vRSFTt7f5nC2fh8568KQR_oZ2hg3Tf20I,2716
 src/augmentation/preview.py,sha256=Bq58MfmD7bzYlRVLzhUUOqBjMavfpSurauTvSUHKsQM,9753
 src/augmentation/runner.py,sha256=0pnpYD5Nh00PaxrfS9pNk2rkGM4BnjwFUMZoVxar44w,11698
 src/augmentation/sdk.py,sha256=zpUudbk_ffVkBa6odajFCUzDE0Fuph6L0ApJeQoD0I8,17419
-src/augmentation/session.py,sha256=5TvvNQvrmrOgKA0-bvbHMaLhST8ZCCGNsIYCoMBiz3I,29362
+src/augmentation/session.py,sha256=KXoRGbfSlb3P-SNIVD2XEW_JJS97sOT_BySQygQewN0,23009
 src/detectors/__init__.py,sha256=BWrUF_GtSsfs1nQMvG9UhOpUokqwJ0-5Dyg498VMs4E,3475
 src/detectors/base.py,sha256=bXcPmc_rZG5ONWm_ZgGLT-QpLqK8zXsdAUsguMb3FMg,2998
 src/detectors/config.py,sha256=CCrzp8Js8tQ1E34_Pl9ChY8JFjvUalXbnJJ-QWk54d4,1683
 src/detectors/dependencies.py,sha256=C5FBnkBMWCnx34WkOrM7yTO8zcOV_zP0RbyvMtw8A20,3851
-src/detectors/engine_version.py,sha256=smcw4YTusu6EcclPAURJozqr7aGatdctzvJ8tMaGjb0,2297
+src/detectors/engine_version.py,sha256=xnSpYD0o-buNSnzuxuszR1s_4M81urdkm2ZPwXhzaLI,4030
 src/detectors/errors.py,sha256=bpmaSD21kb48Sy4OoWmPuYR4Miq3PFKNvZ1_d5_ApVc,812
 src/detectors/broken_links/__init__.py,sha256=maN50pAfC9IJGNUUIHGVM4YnjTAozRI43APLUCQrJcE,77
 src/detectors/broken_links/detector.py,sha256=k1ocQAI-cOuf6HSuipvKfdKyvpjOq5vAvIR1PmSjT3Y,9332
@@ -21,9 +21,10 @@ src/detectors/custom/__init__.py,sha256=HfTzYuQFd7XJbIAhWndqScxbbL0AS7vaOc7p3dkF
 src/detectors/custom/detector.py,sha256=SMZxVmDsj7L5UvF3eVg3qlKwFcmwo1xcvQXEULiLmzA,2407
 src/detectors/custom/extractor.py,sha256=N7LySJBR88o5hueYSZd-efhXjMT5LenWRgthoxdf9tY,9848
 src/detectors/custom/trainer.py,sha256=xS-eg8Ic6qr_rtMvSlrePFVwrzt9hwC56awSw6gTLOM,11358
-src/detectors/custom/runners/__init__.py,sha256=Gi8jMtIv_hIlaBOXjDmMte-cFLrH22fMEuQVW0FL_GQ,1528
+src/detectors/custom/runners/__init__.py,sha256=alVRBfRhTk9q89kyvfS8eHZtSKeoL2BYrN9YxhzijzM,1607
 src/detectors/custom/runners/_base.py,sha256=-iStmAy_iqQUGrM-ZJiTbqWGh-T1h7rFR2uGNmadcAY,8485
-src/detectors/custom/runners/_factory.py,sha256=0Fkpe5eZqCgsQ1XHsPvpbP2oMIC3i43o5AFZPxHqej8,2227
+src/detectors/custom/runners/_custom_detector.py,sha256=aWVufimAKEvnP0RiTqksQqLVmTBuACiqicMYZ85ixIs,2483
+src/detectors/custom/runners/_factory.py,sha256=PpASlTeWbsqxU-WM0CV2vnFxPfZhvdmZBUe2yDrshX0,2553
 src/detectors/custom/runners/_gliner2.py,sha256=VJMab3YUBh8s6lkSLH6-Dty5WjXwGbV_cvy3YmgJtwc,18331
 src/detectors/custom/runners/_image_classification.py,sha256=_4pTRBxQ-TwfHWdVat4rI6bTX3IHPB0Ei-e9qSlFeyE,5658
 src/detectors/custom/runners/_llm.py,sha256=i47_FuUhYSjhxc31G6QcbyEpGuBLkzbuvY3GSPvOzgA,22824
@@ -31,6 +32,14 @@ src/detectors/custom/runners/_object_detection.py,sha256=QVRimQE0TjMQa6X4-FWfbEH
 src/detectors/custom/runners/_regex.py,sha256=j2a0TqHrpXQkNsmilLTb_vG7Ds_A_R4T4lVPBvaWv3U,5818
 src/detectors/custom/runners/_tag.py,sha256=A8_OSs0ly9g7asMHOpXzl4Dr3q5Hiu54vS5F1UDG7No,5947
 src/detectors/custom/runners/_text_classification.py,sha256=1vj_AIzmYBm3iBALnJDc0J6G0l86RyOSpxk0xf9-5PQ,8464
+src/detectors/custom_detector/__init__.py,sha256=Dr3NXUBcNqMttLWSE8IX5NYL-KNjyLbdVDdP2vNupWA,330
+src/detectors/custom_detector/contract.py,sha256=aJEBCkAu4uwAr0EoV7AnrJQrbwSGJr3aDgT111lfDaQ,1541
+src/detectors/custom_detector/files.py,sha256=dbKeKRYF3nI0mV1HcaRUzj9MV0JjnJRt7Hn0QpUQpfI,3821
+src/detectors/custom_detector/preview.py,sha256=STXLiHQ_WKRGtd_9FQN2n_dtRtwitYJCH4HmbpGlY3M,10481
+src/detectors/custom_detector/results.py,sha256=Y6sExAYkvmhemgfsJ6EWyoNd02O70eQ0Z96Mwxx7MNU,10228
+src/detectors/custom_detector/runner.py,sha256=NMK8sfQUOVwoSrbUF7fx2iBIYvPO_BngNf8kisvJn7c,9462
+src/detectors/custom_detector/sdk.py,sha256=QepinCIP2l-sBrKY0wSRdVawReEOFbuoCQ5AkcR91c4,12729
+src/detectors/custom_detector/session.py,sha256=gQv_VJY0HsKDv8zr1ABukPPAKTTRos0KZAPHVkNJmoI,17193
 src/detectors/pii/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 src/detectors/pii/detector.py,sha256=LFNvP-SvK0nsHZsYv1TFzn9CE-M439YB3qt8ehBlcuA,45044
 src/detectors/secrets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -39,15 +48,16 @@ src/detectors/threat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 src/detectors/threat/code_security_detector.py,sha256=SdjhvTSU10wHVxP2EwyzoTGMq8-J8CtwypayHVHdMUs,7049
 src/detectors/threat/yara_detector.py,sha256=VuJ6Tu3o3wMhv2bskSx2YaB7sTILuQaZlK0uWEiLqe0,6549
 src/file_evaluation/__init__.py,sha256=S2IF_CeZtxbfcVKcEamdyW1mZVb-fHAuWqt8QE999Rg,141
-src/file_evaluation/runner.py,sha256=losdZP_xu2ArPSWNajqJtlQl7wEveeF_uN8jmUkpxNE,13401
+src/file_evaluation/asset_fixture.py,sha256=6fcEXlIxYYVE7xJpGtH28Zhmwjdh4ewxqfJlCJSOZ8Y,5192
+src/file_evaluation/runner.py,sha256=_hZ9ewrnnU-QkUmK19MErA57AZ06TQqAzncoeVc9eOM,17379
 src/graph/__init__.py,sha256=9Uhsf3oCiG_w49rwoHQ7fTtsER4WyYTrEIBCE8RsqYc,585
 src/graph/edges.py,sha256=ct9tDheI3pklYwX0mtiSet6Sdy_9E4ATEwa5pqdHsOA,15942
-src/models/generated_detectors.py,sha256=0IZFBhSi_cEtqlCBnGBTwjwjOKPvG2MVp2vfX5tzJYw,54300
+src/models/generated_detectors.py,sha256=cpJLlneiViU4cvD6-o-j5FwWXT1TvvElkoHNTtwgKhg,63159
 src/models/generated_input.py,sha256=EE4ESBNBm9h5cwq4nKICvJGM3FAZlSYwVhyTOxzWF0o,205362
-src/models/generated_single_asset_scan_results.py,sha256=yTwkgC9X-GPT6YuhEanO5GlXegn_2FjqKoplcoTCtEs,16049
+src/models/generated_single_asset_scan_results.py,sha256=zJ78x8GsjpiOBWG5aOQHHVBHdjQeosmCPQ2etwYxAv4,16508
 src/notebook/__init__.py,sha256=IbgAR0i8AKkvBn9QFPwOy5iJUUeDk-Vkcnhoz2nSHBc,985
-src/notebook/child.py,sha256=5vamvuLvyjETdOrjg0cZGtSnr3iD9oRY6XdIYHGEpew,8712
-src/notebook/cli.py,sha256=y2wIL7ePh9StX-2Zh8LzFj1DuPetRUmQ9fzZVhY_Ob8,12606
+src/notebook/child.py,sha256=7ZXQ6dBwonPahl2g_tXmlSa-XiXUlBgGJwCU60BA7-U,8944
+src/notebook/cli.py,sha256=jrfvd3ffktleIhHZb7-dm-3bUiaMBfoYiXcfYSIaQ2I,14925
 src/notebook/cohort.py,sha256=L2bcx5cvYeBHfJBboQPh2Bw9hsRDBhZ4GUwCieIUKP4,7137
 src/notebook/contract.py,sha256=S2_oyHBWqPxmJ3s_5soh7RXE0VBudw8SUOsanggNPsU,10331
 src/notebook/execute.py,sha256=pfkphtwWxmGsJtnJzcXwqBtFx-c8feH9oyl7X_Yb7Uk,21019
@@ -55,7 +65,7 @@ src/notebook/files.py,sha256=3Ef1r_udTM1k8AKNL-wlE135Bczo6Er_s8a90IZ1xUI,6584
 src/notebook/groups.py,sha256=bl4Rqc4ubjI7kuUCSKAQBTwswtHZlpipFuLh8vc4irU,4366
 src/notebook/outputs.py,sha256=GcfnUYitRwrfZlrZ_qPNuNhHwtqxl9_UXSX4VT1cIgI,5194
 src/notebook/packages.py,sha256=mkxFOSmQNja7eqSpXkpe1LesPYfb7MmMt11kkR0QRCU,5827
-src/notebook/protocol.py,sha256=Cpxneux0g-6d0VbBZASUEWqjPYyDk0BwWy_gTqeg51c,7869
+src/notebook/protocol.py,sha256=X9T5zkyh1YEw_bUtvD-GVaF4v4s1ZgBQUI-MSql1hEY,9277
 src/notebook/redact.py,sha256=OYp7ugsmIuscgW6oJFb-K-HxQlUI3sSkeR3IkqAt_2o,981
 src/notebook/scaffold.py,sha256=_vDjiuJEbCIc-4HAa19y9zQ8RU69oMEWaWNchWa62Jw,2283
 src/notebook/sdk.py,sha256=zELFrZC03y48_x3KtiwAsnThE0LJV2GvW9c8gYSy2Q0,40557
@@ -67,12 +77,13 @@ src/outputs/factory.py,sha256=ritKokn9SM6s51MSCPJhLKQMwhKF4OBRatsntGYi22w,5397
 src/outputs/file.py,sha256=6y585mxJS04MgTcMu5KPkJt_hm9qLjnbPEzD4MgrmI4,2594
 src/outputs/rest.py,sha256=Yew1iPPR1j1ljYlLQhILYNeOt5Jj6cpB11-RnRDX-VY,25848
 src/pipeline/__init__.py,sha256=BCWl3hwwLkmZc2vQ2fEyUdhis4TPK930HEgXTog8NjA,742
+src/pipeline/asset_payload_server.py,sha256=6LPEcybDk6gKYw6cSVXRKAHmZhkczwkjt3YgE6kN644,9773
 src/pipeline/content_provider.py,sha256=XiSOED7Lw0S663ybZo4iveQRceg5f51tdU9tyv2erSg,1078
 src/pipeline/detector_breaker.py,sha256=Mn1F7K7LM7jIJpgbZEJVmApAxLRrK-W1LRgpP8ZPuMw,7969
-src/pipeline/detector_pipeline.py,sha256=yLkTWdRsv_lN3S4QzhZo9ano-jcmLWV123OikyLu8eY,60123
+src/pipeline/detector_pipeline.py,sha256=X6FC58QAYgtBE_83SM2AnHCIb9QMhF4kR7A0wgMi-dg,65851
 src/pipeline/parsed_content_provider.py,sha256=rHTvvBvjTQtCJF3HT8GC1oSZV3eQ2PbgZ2tCtLIPVOE,4382
 src/pipeline/payload_window.py,sha256=H5aDNuaLxei_zI0Hk0FQsiGNCX6ixcXa4ffV3Inglp0,25254
-src/pipeline/scan_cache.py,sha256=a28icQ_8zs-2z5vXrV77XcdS7EIY8P5fg28XjgdW70A,24240
+src/pipeline/scan_cache.py,sha256=0rZwP1Hv29yPWY50OqfreLhtE7Rn8HS5sB3ZEZpBsmg,26475
 src/pipeline/text_artifact.py,sha256=Ltu6wbpxnE8CdStC4E41y5VzhXFEgv00OhsI4Bv0YJo,1656
 src/pipeline/worker_pool.py,sha256=Yjr_2LXxV2nYNcXNN4gFRPK9KN15GP_IkkAzaplKaTc,10543
 src/sources/__init__.py,sha256=wYksO_0IcvlGXLouWapKyqzL0jG0El4rDDGRfwa4Zqk,3350
@@ -182,7 +193,7 @@ src/utils/hashing.py,sha256=T0GL1FWtG32QJID5RN3X-TzHHkolnNK6EfADvX6sdlE,2490
 src/utils/legacy_office.py,sha256=LCDiU7E-ZUxHgtQieihQWCXNXOtjiUpJEExWjme-dPI,9205
 src/utils/payload.py,sha256=ewvSABOLSiIGnrq5A9wfOeFZnVJ_TXtSmEOxDYcE58U,6159
 src/utils/range_reader.py,sha256=tIT4oicfPT4B_hJyijVmB-SA8-HOj4Gol0Fv0Crw6s0,8661
-src/utils/redaction.py,sha256=elhNyCGR9DxzS8FSTDsPuDTwVIsEku-pd4vJv8l0SKw,11621
+src/utils/redaction.py,sha256=9LYP_R58plMe_6fvpoOCnM0FcTHh-W7p4iOthExNygQ,12552
 src/utils/resources.py,sha256=2ejFjz9r_Kv_d-sUGD0oj54U-ZIUSxDAu7aD4YEv4FQ,2630
 src/utils/source_files.py,sha256=OKZSW64cur0HTuWPRkV1sJg_Z2LFGWzUsC853gGlFBY,5033
 src/utils/sql_lineage.py,sha256=uvbiH9SQVDflToFsJqFs32rVwV2K2FG6VFRxIIJZxkI,8024
@@ -192,7 +203,7 @@ src/utils/urn.py,sha256=lzq5hEi0schfHh83Gs3oFQoI2qsk3klCev_aatMyI_E,11189
 src/utils/uv_sync.py,sha256=VZ0TZ7t3_pSjxOb20BxUs2XLVjigXnVqa5y69XWlnE8,9307
 src/utils/validation.py,sha256=5NHsgyZwyEONRu9_ADxUAQNPvbMnpMhKL-hpgFEVMm8,3837
 src/utils/video_processing.py,sha256=B6Br9ybFEa_ioglJGo0P9Fmp1e2B95C24joAktwrjww,11293
-classifyre_cli-0.6.2.dist-info/METADATA,sha256=7rRt9wjX1hGvk4B-Mfl07P0vxaAthJ7d_2roXOfKeNo,5850
-classifyre_cli-0.6.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-classifyre_cli-0.6.2.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
-classifyre_cli-0.6.2.dist-info/RECORD,,
+classifyre_cli-0.6.4.dist-info/METADATA,sha256=-0x1bhBf5oDBn34RF62GKWi8Xw7C_RfPyr3YT29jmVw,5850
+classifyre_cli-0.6.4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+classifyre_cli-0.6.4.dist-info/entry_points.txt,sha256=edxllyMiIFfajgCZ3dTRACXYGC34ZX1AYTezMSR5E_8,45
+classifyre_cli-0.6.4.dist-info/RECORD,,

src/augmentation/session.py

@@ -18,7 +18,6 @@ silently-degraded run visible.
 from __future__ import annotations
 
 import asyncio
-import base64
 import json
 import logging
 from collections.abc import Mapping
@@ -29,6 +28,7 @@ from ..graph.edges import edge_from_payload
 from ..notebook.child import ChildProcessError, NotebookChildProcess
 from ..notebook.groups import warm_declared_groups
 from ..notebook.packages import install as install_packages
+from ..pipeline.asset_payload_server import AssetPayloadServer
 from ..utils.hashing import unhash_id
 from ..utils.urn import normalize_urn_or_none
 from .contract import validate_augmentation_notebook
@@ -143,10 +143,6 @@ class AugmentationSession:
         self._warnings: list[str] = []
         self._warning_set: set[str] = set()
         self.stats = AugmentationStats()
-        # Run-scoped memo of fetched payloads, keyed by fetch candidate id.
-        # Bounded: one in-flight asset's worth per entry, each capped at
-        # max_payload_bytes, dropped by evict_asset_cache().
-        self._memo: dict[str, dict[str, Any]] = {}
 
     # -- construction ----------------------------------------------------------
 
@@ -295,7 +291,6 @@ class AugmentationSession:
                 slot.child.terminate()
             except Exception:
                 pass
-        self._memo.clear()
 
     # -- per asset -----------------------------------------------------------------
 
@@ -314,163 +309,34 @@ class AugmentationSession:
             "mime_type": None,
         }
 
-    def _candidates(self, asset: Any) -> list[str]:
-        seen: list[str] = []
-        for candidate in (getattr(asset, "external_url", ""), getattr(asset, "hash", "")):
-            value = str(candidate or "").strip()
-            if value and value not in seen:
-                seen.append(value)
-        return seen
+    @property
+    def _payloads(self) -> AssetPayloadServer:
+        """The source's shared payload server.
+
+        Shared with the code detectors that run on the same asset later in the
+        scan, so a payload augmentation fetched is not fetched again (G1 R20).
+        """
+        return AssetPayloadServer.shared(
+            self._source, max_payload_bytes=self._limits["max_payload_bytes"]
+        )
 
     async def _serve_need(self, asset: Any, memo_key: str, frame: dict[str, Any]) -> Any:
-        op = str(frame.get("op") or "")
-        if op == "payload":
-            fetched = await self._fetch_payload(asset)
-            if fetched is None:
-                return {"bytes_b64": None, "mime": None}
-            raw, mime = fetched
-            return {"bytes_b64": base64.b64encode(raw).decode("ascii"), "mime": mime}
-        if op == "raw_pages":
-            pages = await self._fetch_raw_pages(asset)
-            return {"pages": pages}
-        if op == "text":
-            pages = await self._fetch_text_pages(asset)
-            whole = "".join(pages)
-            return {"text": whole, "pages": pages}
-        raise ChildProcessError(f"Unknown payload op {op!r}")
+        try:
+            return await self._payloads.serve(asset, frame)
+        except ValueError as exc:
+            raise ChildProcessError(str(exc)) from exc
 
     async def _fetch_payload(self, asset: Any) -> tuple[bytes, str] | None:
         """Raw bytes for file-shaped sources; None for row-shaped ones."""
-        memo = self._memo_for(asset)
-        if "payload" in memo:
-            cached = memo["payload"]
-            return cached if cached is None else (bytes(cached[0]), str(cached[1]))
-        fetch = getattr(self._source, "fetch_content_bytes", None)
-        result: tuple[bytes, str] | None = None
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    result = await fetch(candidate)
-                except Exception as exc:
-                    logger.debug("Augmentation payload fetch failed for %s: %s", candidate, exc)
-                    continue
-                if result is not None:
-                    break
-        if result is not None:
-            raw, mime = result
-            cap = self._limits["max_payload_bytes"]
-            if len(raw) > cap:
-                logger.debug("Augmentation payload for %s truncated to %d bytes", candidate, cap)
-                raw = raw[:cap]
-            result = (raw, mime)
-            self._remember_bytes(asset, raw, mime)
-        memo["payload"] = result
-        return result
+        return await self._payloads.payload(asset)
 
     async def _fetch_raw_pages(self, asset: Any) -> list[str]:
         """The connector's own raw representation, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "raw_pages" in memo:
-            return [str(page) for page in memo["raw_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for raw, _text in fetch(candidate):
-                        if not raw:
-                            continue
-                        chunk = str(raw)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation raw fetch failed for %s: %s", candidate, exc)
-                    continue
-        memo["raw_pages"] = pages
-        return pages
+        return await self._payloads.raw_pages(asset)
 
     async def _fetch_text_pages(self, asset: Any) -> list[str]:
         """Extracted text, page by page (capped)."""
-        memo = self._memo_for(asset)
-        if "text_pages" in memo:
-            return [str(page) for page in memo["text_pages"]]
-        pages: list[str] = []
-        fetch = getattr(self._source, "fetch_content_pages", None)
-        cap = self._limits["max_payload_bytes"]
-        used = 0
-        if callable(fetch):
-            for candidate in self._candidates(asset):
-                try:
-                    async for _raw, text in fetch(candidate):
-                        if not text:
-                            continue
-                        chunk = str(text)
-                        if used + len(chunk) > cap:
-                            chunk = chunk[: max(0, cap - used)]
-                        if chunk:
-                            pages.append(chunk)
-                            used += len(chunk)
-                        if used >= cap or len(pages) >= 50:
-                            break
-                    if pages:
-                        break
-                except Exception as exc:
-                    logger.debug("Augmentation text fetch failed for %s: %s", candidate, exc)
-                    continue
-        if not pages:
-            provider = getattr(self._source, "fetch_content_bytes", None)
-            if callable(provider):
-                fetched = await self._fetch_payload(asset)
-                if fetched is not None:
-                    raw, mime = fetched
-                    try:
-                        text_pages = await asyncio.to_thread(
-                            list, self._source.iter_asset_pages(raw, mime)
-                        )
-                        for page in text_pages[:50]:
-                            chunk = str(page)
-                            if used + len(chunk) > cap:
-                                chunk = chunk[: max(0, cap - used)]
-                            if chunk:
-                                pages.append(chunk)
-                                used += len(chunk)
-                            if used >= cap:
-                                break
-                    except Exception as exc:
-                        logger.debug("Augmentation text fallback failed: %s", exc)
-        memo["text_pages"] = pages
-        return pages
-
-    def _memo_for(self, asset: Any) -> dict[str, Any]:
-        key = str(getattr(asset, "hash", "") or "")
-        return self._memo.setdefault(key, {})
-
-    def _remember_bytes(self, asset: Any, raw: bytes, mime: str) -> None:
-        """Mirror fetched bytes onto the source so the pipeline reuses them.
-
-        ``ParsedContentProvider.fetch_bytes`` consults this memo first instead
-        of re-downloading — the fetch augmentation already paid for is not
-        paid twice. Registered under every candidate id the pipeline tries
-        (external URL, then hash), and dropped by ``evict`` at the end of the
-        asset.
-        """
-        remember = getattr(self._source, "remember_augmentation_bytes", None)
-        if not callable(remember):
-            return
-        for candidate in self._candidates(asset):
-            try:
-                remember(candidate, raw, mime)
-            except Exception:
-                pass
+        return await self._payloads.text_pages(asset)
 
     async def augment(self, asset: Any) -> Any:
         """Enrich one asset. Never raises: failure degrades to a warning."""
@@ -718,13 +584,4 @@ class AugmentationSession:
 
     def evict(self, asset: Any) -> None:
         """Drop one asset's payload memo. Called alongside evict_asset_cache."""
-        self._memo.pop(str(getattr(asset, "hash", "") or ""), None)
-        forget = getattr(self._source, "forget_augmentation_bytes", None)
-        if not callable(forget):
-            return
-        # The bytes were mirrored under every candidate id; drop them all.
-        for candidate in self._candidates(asset):
-            try:
-                forget(candidate)
-            except Exception:
-                pass
+        self._payloads.evict(asset)

src/detectors/custom/runners/__init__.py

@@ -13,6 +13,7 @@ from ._base import (
     BaseRunner,
     _resolve_pipeline_severity,
 )
+from ._custom_detector import CustomDetectorRunner
 from ._factory import create_runner
 from ._gliner2 import (
     GLiNER2Runner,
@@ -35,6 +36,7 @@ __all__ = [
     "_IMAGE_CONTENT_TYPES",
     "_TEXT_CONTENT_TYPES",
     "BaseRunner",
+    "CustomDetectorRunner",
     "GLiNER2Runner",
     "ImageClassificationRunner",
     "LLMRunner",

src/detectors/custom/runners/_custom_detector.py

@@ -0,0 +1,72 @@
+"""Runner for the ``CODE_DETECTOR`` pipeline type: a code detector.
+
+Like ``TAG`` it never runs over a page of text: ``detect()`` returns nothing and
+no content type is advertised, so the text, binary and link passes never
+schedule it and the worker pool never receives it. The detector pipeline
+recognises it by this class and calls it once per asset, after every other
+detector, through the :class:`CustomDetectorSession` it owns.
+
+The session is created lazily on first use and bound to the scan's source,
+because it needs the source to serve payload and holds child processes that
+must not exist in the worker pool's processes.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ....models.generated_detectors import CodeDetectorPipelineSchema, PipelineResult
+from ....models.generated_single_asset_scan_results import DetectionResult
+from ._base import BaseRunner
+
+
+class CustomDetectorRunner(BaseRunner):
+    """Asset-scoped runner: judged per asset by a notebook, never per page."""
+
+    asset_scoped = True
+
+    def __init__(
+        self,
+        schema: CodeDetectorPipelineSchema,
+        detector_key: str = "",
+        detector_name: str = "",
+    ) -> None:
+        self.schema = schema
+        self._detector_key = detector_key
+        self._detector_name = detector_name
+        self._session: Any = None
+
+    def run(self, text: str) -> PipelineResult:
+        return PipelineResult(metadata={"runner": "CODE_DETECTOR"})
+
+    def detect(self, content: str | bytes, content_type: str) -> list[DetectionResult]:
+        """Never called per page. The pipeline calls ``session().detect(asset)``."""
+        return []
+
+    def get_supported_content_types(self) -> list[str]:
+        return []
+
+    @property
+    def deterministic(self) -> bool:
+        return self.schema.deterministic is not False
+
+    @property
+    def needs_findings(self) -> bool:
+        return bool(self.schema.needs_findings)
+
+    def session(self, source: Any) -> Any:
+        """The run's session, created on first use."""
+        if self._session is None:
+            from ...custom_detector.session import CustomDetectorSession
+
+            self._session = CustomDetectorSession(
+                key=self._detector_key,
+                name=self._detector_name,
+                schema=self.schema,
+                source=source,
+            )
+        return self._session
+
+    def close(self) -> None:
+        if self._session is not None:
+            self._session.close()

src/detectors/custom/runners/_factory.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from ....models.generated_detectors import (
+    CodeDetectorPipelineSchema,
     GLiNER2PipelineSchema,
     ImageClassificationPipelineSchema,
     LLMPipelineSchema,
@@ -12,6 +13,7 @@ from ....models.generated_detectors import (
     TextClassificationPipelineSchema,
 )
 from ._base import BaseRunner
+from ._custom_detector import CustomDetectorRunner
 from ._gliner2 import GLiNER2Runner
 from ._image_classification import ImageClassificationRunner
 from ._llm import LLMRunner
@@ -30,6 +32,7 @@ def create_runner(
         | ImageClassificationPipelineSchema
         | ObjectDetectionPipelineSchema
         | TagPipelineSchema
+        | CodeDetectorPipelineSchema
     ),
     detector_key: str = "",
     detector_name: str = "",
@@ -50,5 +53,8 @@ def create_runner(
     # detector that is not supposed to run anything at all.
     if isinstance(schema, TagPipelineSchema):
         return TagRunner(schema, detector_key, detector_name)
+    # Same reason as TAG: a code detector must never fall through to GLiNER2.
+    if isinstance(schema, CodeDetectorPipelineSchema):
+        return CustomDetectorRunner(schema, detector_key, detector_name)
     # GLiNER2PipelineSchema is the default / backward-compat path
     return GLiNER2Runner(schema, detector_key, detector_name)

src/detectors/custom_detector/__init__.py

@@ -0,0 +1,9 @@
+"""Code detectors: the ``CODE_DETECTOR`` pipeline type.
+
+A notebook that defines ``detect(asset, ctx)`` and yields ``Finding`` objects,
+run in an isolated child process once per asset, after every other detector.
+"""
+
+from .session import CustomDetectorSession, DetectOutcome
+
+__all__ = ["CustomDetectorSession", "DetectOutcome"]

src/detectors/custom_detector/contract.py

@@ -0,0 +1,45 @@
+"""The interface a code detector's notebook must cover.
+
+A code detector answers one question -- "what is wrong with this asset?" -- so
+``detect(asset, ctx)`` is required and ``setup(ctx)`` optional. Checked at the
+three moments the connector contract is: on save (the API's cheap check),
+before a preview, and before every scan, because the recipe travels to a job
+that has to defend itself.
+
+Only the shape is checked, through the shared ``validate_module`` /
+``validate_notebook``. Isolation is the child process and its scrubbed
+environment, not anything an AST can decide.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+
+from ...notebook.contract import ContractReport, validate_module, validate_notebook
+from ...notebook.serialize import ModuleSource
+
+REQUIRED_FUNCTIONS = ("detect",)
+
+#: ``setup(ctx)`` runs once per worker before the first asset: load a list,
+#: a model, a lookup table into ``ctx.state``.
+OPTIONAL_FUNCTIONS = ("setup",)
+
+
+def validate_detector_module(module: ModuleSource) -> ContractReport:
+    """Parse the assembled module and check ``detect()`` exists."""
+    return validate_module(module, required=REQUIRED_FUNCTIONS)
+
+
+def validate_detector_notebook(
+    cells: Iterable[Mapping[str, object] | object],
+) -> ContractReport:
+    """Validate a code detector's cells."""
+    return validate_notebook(cells, required=REQUIRED_FUNCTIONS)
+
+
+__all__ = [
+    "OPTIONAL_FUNCTIONS",
+    "REQUIRED_FUNCTIONS",
+    "validate_detector_module",
+    "validate_detector_notebook",
+]

src/detectors/custom_detector/files.py

@@ -0,0 +1,102 @@
+"""Materialize a code detector's uploaded files for ``ctx.file(name)``.
+
+Downloaded by the parent, never by the child: the child is started without the
+API URL or the callback key, and handing it either would undo the isolation it
+exists for. The API injects ``files_runtime`` -- one entry per file, with a
+content hash and a download URL -- into the pipeline schema at dispatch.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+import shutil
+import tempfile
+from collections.abc import Iterable, Mapping
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+import requests
+
+from ...outputs.base import DEFAULT_REST_TIMEOUT_SEC
+from ...outputs.rest import INTERNAL_KEY_HEADER
+from ...utils.source_files import CHUNK_BYTES, api_base_url
+
+logger = logging.getLogger(__name__)
+
+#: A directory prepared by the caller (the preview job's init container, a
+#: test) wins over downloading.
+FILES_DIR_ENV = "CLASSIFYRE_DETECTOR_FILES_DIR"
+
+
+def _safe_name(name: str) -> str:
+    base = Path(name).name or "file"
+    return re.sub(r"[^A-Za-z0-9._ -]", "_", base)[:200] or "file"
+
+
+def _entries(files_runtime: Iterable[Any] | None) -> list[dict[str, Any]]:
+    entries: list[dict[str, Any]] = []
+    for raw in files_runtime or []:
+        entry = raw.model_dump() if hasattr(raw, "model_dump") else raw
+        if isinstance(entry, Mapping) and entry.get("id") and entry.get("name"):
+            entries.append(dict(entry))
+    return entries
+
+
+def materialize_detector_files(
+    detector_id: str | None,
+    files_runtime: Iterable[Any] | None,
+) -> Path | None:
+    """Download every file into a fresh temp dir; None when there are none.
+
+    A file that fails to download is logged and skipped: a rule that reads one
+    list out of three should still see the two that arrived, and a rule that
+    needs the missing one fails loudly on ``ctx.file(name)`` with its name.
+    """
+    prepared = os.environ.get(FILES_DIR_ENV)
+    if prepared and Path(prepared).is_dir():
+        return Path(prepared)
+    entries = _entries(files_runtime)
+    if not entries:
+        return None
+    destination = Path(tempfile.mkdtemp(prefix="classifyre-detector-files-"))
+    headers = {"Connection": "close"}
+    key = os.environ.get("CLASSIFYRE_INTERNAL_KEY", "").strip()
+    if key:
+        headers[INTERNAL_KEY_HEADER] = key
+    written = 0
+    with requests.Session() as session:
+        for entry in entries:
+            url = str(entry.get("url") or "").strip()
+            if not url:
+                if not detector_id:
+                    continue
+                url = (
+                    f"{api_base_url()}/custom-detectors/{quote(detector_id, safe='')}"
+                    f"/files/{quote(str(entry['id']), safe='')}/content"
+                )
+            target = destination / _safe_name(str(entry["name"]))
+            try:
+                with session.get(
+                    url, headers=headers, stream=True, timeout=(30, DEFAULT_REST_TIMEOUT_SEC)
+                ) as response:
+                    response.raise_for_status()
+                    with target.open("wb") as handle:
+                        for chunk in response.iter_content(chunk_size=CHUNK_BYTES):
+                            if chunk:
+                                handle.write(chunk)
+            except (requests.RequestException, OSError) as exc:
+                logger.warning("Could not download detector file %s: %s", entry.get("name"), exc)
+                target.unlink(missing_ok=True)
+                continue
+            written += 1
+    if not written:
+        shutil.rmtree(destination, ignore_errors=True)
+        return None
+    logger.info("Made %d detector file(s) available to ctx.files", written)
+    return destination
+
+
+__all__ = ["FILES_DIR_ENV", "materialize_detector_files"]