classifyre-cli 0.5.3__tar.gz → 0.5.5__tar.gz

classifyre_cli-0.5.5/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 323 packages in 136ms
+Checked 51 packages in 4ms

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.5.3
+Version: 0.5.5
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured
-Requires-Python: >=3.12
+Requires-Python: <3.15,>=3.12
 Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: classifyre-schemas
 Requires-Dist: email-validator>=2.3.0

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.5.3",
+  "version": "0.5.5",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/pyproject.toml

@@ -1,9 +1,14 @@
 [project]
 name = "classifyre-cli"
-version = "0.5.3"
+version = "0.5.5"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
-requires-python = ">=3.12"
+# Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably
+# linux/aarch64, which required-environments forces uv to resolve), so an
+# unbounded range makes `uv lock` fail for the python>=3.15 split and breaks
+# the release workflow and Dependabot's uv updates. Lift to <3.16 once
+# onnxruntime ships cp315 wheels for all required-environments platforms.
+requires-python = ">=3.12,<3.15"
 license = { text = "MIT" }
 keywords = ["data", "metadata", "ingestion", "unstructured", "pii", "secrets"]
 dependencies = [

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/src/detectors/custom/runners/_llm.py

@@ -3,12 +3,15 @@
 from __future__ import annotations
 
 import base64
+import functools
 import json
 import logging
 import os
 import random
+import re
 import time
 from datetime import UTC, datetime
+from pathlib import Path
 from typing import Any
 
 # Quiet litellm's import-time provider preload warnings (bedrock/sagemaker need
@@ -22,6 +25,7 @@ from ....models.generated_single_asset_scan_results import (
 )
 from ....utils.file_to_images import render_to_images, supported_mime_type
 from ...dependencies import require_module
+from ...errors import ProviderRefusedError
 from ._base import _IMAGE_CONTENT_TYPES, _TEXT_CONTENT_TYPES, BaseRunner, _resolve_pipeline_severity
 
 logger = logging.getLogger(__name__)
@@ -48,11 +52,61 @@ _COMPLETION_TIMEOUT_SECONDS = float(os.environ.get("CLASSIFYRE_LLM_TIMEOUT_SECON
 # Retry budget for transient provider failures (429 rate limits, 5xx, timeouts).
 # Exponential backoff with jitter, honouring Retry-After when the provider
 # sends one, so a saturated endpoint is backed off from instead of hammered.
-_MAX_COMPLETION_ATTEMPTS = max(1, int(os.environ.get("CLASSIFYRE_LLM_MAX_ATTEMPTS", "4")))
+# The attempt count is read per call (see `_max_attempts`), so a detector's own
+# `budget.max_attempts` can override it.
+_DEFAULT_MAX_COMPLETION_ATTEMPTS = 4
 _RETRY_BACKOFF_BASE_SECONDS = float(os.environ.get("CLASSIFYRE_LLM_RETRY_BASE_SECONDS", "2"))
 _RETRY_BACKOFF_MAX_SECONDS = 60.0
 _RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504}
 
+# Used only if the shared marker list cannot be read, so a packaging mistake
+# degrades to "the known worst offender still fails fast" rather than to "every
+# quota refusal is retried on every asset again".
+_FALLBACK_QUOTA_MARKERS = (
+    "insufficient_credit",
+    "insufficient balance",
+    "out of credits",
+    "resource_exhausted",
+    "resource exhausted",
+    "quota exceeded",
+    "free-models-per-day",
+    "insufficient_quota",
+)
+
+# Billing-exhaustion signals checked before the generic substring scan, with
+# word-boundary anchors so "billing" inside an unrelated word cannot trip them
+# while "you have insufficient_credit" always does. Mirrors the head of the
+# shared quota list in provider_refusal_markers.json.
+_ANCHORED_BILLING_EXHAUSTION_MARKERS = (
+    "insufficient_credit",
+    "insufficient balance",
+    "out of credits",
+)
+
+
+def _billing_exhaustion_match(text: str) -> str | None:
+    """The anchored billing marker in *text*, or None when absent."""
+    for marker in _ANCHORED_BILLING_EXHAUSTION_MARKERS:
+        if re.search(rf"(?<![a-z0-9_]){re.escape(marker)}(?![a-z0-9_])", text):
+            return marker
+    return None
+
+
+@functools.lru_cache(maxsize=1)
+def _quota_markers() -> tuple[str, ...]:
+    """Substrings that make a refusal non-retryable; shared with the API."""
+    try:
+        import schemas
+
+        path = Path(schemas.__file__).parent / "provider_refusal_markers.json"
+        markers = json.loads(path.read_text(encoding="utf-8")).get("quota", [])
+        loaded = tuple(str(m).strip().lower() for m in markers if str(m).strip())
+        if loaded:
+            return loaded
+    except Exception as exc:  # pragma: no cover - packaging failure
+        logger.warning("provider_refusal_markers.json unavailable (%s); using fallback", exc)
+    return _FALLBACK_QUOTA_MARKERS
+
 
 class LLMCompletionError(RuntimeError):
     """Provider call failed after retries.
@@ -63,6 +117,14 @@ class LLMCompletionError(RuntimeError):
     """
 
 
+class LLMProviderRefusedError(LLMCompletionError, ProviderRefusedError):
+    """The provider refused in a way no retry can fix (quota, auth, billing).
+
+    Raised on the first such response, never after retries. The pipeline
+    disables the detector for the rest of the run when it sees one.
+    """
+
+
 class LLMRunner(BaseRunner):
     """AI detector — sends content to a configured LLM provider for classification + extraction."""
 
@@ -153,6 +215,10 @@ class LLMRunner(BaseRunner):
             response = self._complete_with_backoff(messages)
             raw = response.choices[0].message.content or "{}"
             parsed = self._parse_json(raw)
+        except LLMProviderRefusedError:
+            # Already names the detector, the model and the refusal. No stack
+            # trace: it is expected, and it will be the same on every asset.
+            raise
         except Exception as exc:
             logger.error(
                 "llm detector error (detector=%s, model=%s): %s",
@@ -173,7 +239,8 @@ class LLMRunner(BaseRunner):
 
     def _complete_with_backoff(self, messages: list[dict[str, Any]]) -> Any:
         schema = self._schema
-        for attempt in range(1, _MAX_COMPLETION_ATTEMPTS + 1):
+        max_attempts = self._max_attempts()
+        for attempt in range(1, max_attempts + 1):
             try:
                 return self._litellm.completion(
                     model=self._model_string(),
@@ -184,9 +251,26 @@ class LLMRunner(BaseRunner):
                     messages=messages,
                     response_format={"type": "json_object"},
                     timeout=_COMPLETION_TIMEOUT_SECONDS,
+                    # One retry layer, and it is this loop. litellm hands the
+                    # OpenAI client its own default of 2 retries, which hid
+                    # ~30 s of backoff inside every single attempt: a daily-quota
+                    # refusal measured 30.2 s before this loop ever saw it.
+                    max_retries=0,
                 )
             except Exception as exc:
-                if attempt >= _MAX_COMPLETION_ATTEMPTS or not self._is_retryable(exc):
+                refusal = self._refusal_reason(exc)
+                if refusal is not None:
+                    logger.warning(
+                        "llm provider refused (detector=%s, model=%s): %s — not retrying",
+                        self._detector_key,
+                        self._runtime.model,
+                        refusal,
+                    )
+                    raise LLMProviderRefusedError(
+                        f"LLM provider refused detector '{self._detector_key}' "
+                        f"(model={self._runtime.model}, {refusal}): {exc}"
+                    ) from None
+                if attempt >= max_attempts or not self._is_retryable(exc):
                     raise
                 delay = min(
                     _RETRY_BACKOFF_MAX_SECONDS,
@@ -202,13 +286,55 @@ class LLMRunner(BaseRunner):
                     self._detector_key,
                     self._runtime.model,
                     attempt,
-                    _MAX_COMPLETION_ATTEMPTS,
+                    max_attempts,
                     delay,
                     exc,
                 )
                 time.sleep(delay)
         raise LLMCompletionError("unreachable")  # pragma: no cover
 
+    def _max_attempts(self) -> int:
+        """The detector's own budget first, then the deployment default."""
+        budget = getattr(self._schema, "budget", None)
+        configured = getattr(budget, "max_attempts", None)
+        if isinstance(configured, int) and configured >= 1:
+            return configured
+        raw = os.environ.get("CLASSIFYRE_LLM_MAX_ATTEMPTS", "")
+        try:
+            return max(1, int(raw)) if raw.strip() else _DEFAULT_MAX_COMPLETION_ATTEMPTS
+        except ValueError:
+            return _DEFAULT_MAX_COMPLETION_ATTEMPTS
+
+    def _refusal_reason(self, exc: Exception) -> str | None:
+        """Why retrying this error cannot help, or None when it might.
+
+        A 429 is usually "slow down" and is retried. It is a refusal only when the
+        provider says the allowance itself is gone — a daily or credit quota — and
+        then every further call until the reset is the same answer, paid for again.
+        """
+        status = getattr(exc, "status_code", None)
+        if status in (401, 403):
+            return "authentication refused"
+        if status == 402:
+            return "payment required"
+        for type_name, reason in (
+            ("AuthenticationError", "authentication refused"),
+            ("PermissionDeniedError", "permission denied"),
+            ("BudgetExceededError", "budget exceeded"),
+        ):
+            error_type = getattr(self._litellm, type_name, None)
+            if isinstance(error_type, type) and isinstance(exc, error_type):
+                return reason
+
+        text = _error_text(exc).lower()
+        # Anchored billing-exhaustion markers first: a spent balance is the
+        # most specific "retrying cannot help" signal in the shared list.
+        if _billing_exhaustion_match(text) is not None:
+            return "quota exhausted"
+        if any(marker in text for marker in _quota_markers()):
+            return "quota exhausted"
+        return None
+
     def _is_retryable(self, exc: Exception) -> bool:
         retryable_types = tuple(
             t
@@ -223,6 +349,9 @@ class LLMRunner(BaseRunner):
         )
         if retryable_types and isinstance(exc, retryable_types):
             return True
+        # Only the listed statuses are retried: an unknown 4xx (e.g. a Gemini
+        # safety-block 400) is a final answer for this input, not a busy
+        # provider. Retry stays for 5xx, plain 429s and network errors.
         return getattr(exc, "status_code", None) in _RETRYABLE_STATUS_CODES
 
     def _vision_enabled(self) -> bool:
@@ -391,6 +520,16 @@ class LLMRunner(BaseRunner):
         return {str(k): v for k, v in raw.items()} if isinstance(raw, dict) else {}
 
 
+def _error_text(exc: Exception) -> str:
+    """Everything a provider exception says about itself, for marker matching."""
+    parts = [str(exc), str(getattr(exc, "message", "") or "")]
+    response = getattr(exc, "response", None)
+    body = getattr(response, "text", None)
+    if isinstance(body, str):
+        parts.append(body)
+    return " ".join(parts)
+
+
 def _retry_after_seconds(exc: Exception) -> float | None:
     """Best-effort extraction of a Retry-After hint from a provider exception."""
     response = getattr(exc, "response", None)

classifyre_cli-0.5.5/src/detectors/errors.py

@@ -0,0 +1,19 @@
+"""Detector errors whose *type* the pipeline acts on.
+
+Kept free of heavy imports so the pipeline can recognise them without loading
+any engine, and kept to a single string argument so they survive pickling
+across the detector worker-pool process boundary with their class intact.
+"""
+
+from __future__ import annotations
+
+
+class ProviderRefusedError(RuntimeError):
+    """A provider refused the request in a way retrying cannot fix.
+
+    Quota exhausted for the day, a rejected key, billing. The next asset will be
+    refused the same way, so the pipeline disables the detector for the rest of
+    the run on the first one instead of paying for the same answer on every
+    asset — which is what held the only runner slot for 10 hours a day when a
+    free-tier LLM provider started refusing its key.
+    """

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/src/main.py

@@ -768,6 +768,22 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                                 f" asset(s) after retries: {preview}"
                             )
 
+                    # A detector the breaker disabled is not a crash and not a
+                    # clean run; the API reports it from the per-asset
+                    # `breaker_open[...]` outcomes, and this is the log line an
+                    # operator reads first.
+                    for degraded in pipeline.breaker_summary():
+                        logger.warning(
+                            "Detector %s was disabled mid-run (%s): %s. %d payload(s) "
+                            "attempted, %d skipped; the skipped assets keep their "
+                            "findings and are retried on the next run.",
+                            degraded["label"],
+                            degraded["cause"],
+                            degraded["reason"],
+                            degraded["attempted_payloads"],
+                            degraded["skipped_payloads"],
+                        )
+
                     # Persist the advanced AUTOMATIC sampling cursor (no-op for
                     # other strategies, which return None). Only on the normal
                     # completion path — a timed-out run must not advance it.
@@ -777,6 +793,10 @@ async def run_command_async(args: argparse.Namespace, recipe: dict[str, Any]) ->
                     # A connector that covered a slice must say so before
                     # finish(), which is what decides whether absence retires
                     # an asset.
+                    cohort_stats = getattr(source, "cohort_stats", None)
+                    if cohort_stats and hasattr(sink, "set_cohort_stats"):
+                        sink.set_cohort_stats(cohort_stats)
+
                     if getattr(source, "partial_coverage", False) and hasattr(
                         sink, "set_partial_coverage"
                     ):

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/src/models/generated_detectors.py

@@ -966,6 +966,37 @@ class DetectorScope(BaseModel):
     )
 
 
+class DetectorBudget(BaseModel):
+    """
+    How much of a run this detector may spend failing.
+
+    Without a budget a detector keeps being called after it has stopped working: an LLM detector on a provider that refused its key retried every asset for 15-40 seconds and held the only runner slot for 10 hours a day, failing on 451 of 451 assets. Every run already disables a detector after a non-retryable provider refusal (quota, auth, billing) and after 10 consecutive failures; these fields tighten or loosen that per detector.
+
+    A disabled detector records an ERROR outcome on the assets it skipped, so their existing findings are kept and the assets are retried on the next run. Changing the budget never invalidates the scan cache.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    max_attempts: int | None = Field(
+        None,
+        description='LLM detectors only: provider calls per asset, including the first. Overrides CLASSIFYRE_LLM_MAX_ATTEMPTS. A non-retryable refusal is never retried regardless.',
+        ge=1,
+        le=10,
+    )
+    max_consecutive_failures: int | None = Field(
+        None,
+        description='Disable the detector for the rest of the run after this many consecutive failed payloads. Defaults to 10.',
+        ge=1,
+        le=10000,
+    )
+    max_wall_clock_seconds: int | None = Field(
+        None,
+        description='Disable the detector for the rest of the run once it has spent this long, summed over every payload it ran on. Unset means no limit.',
+        ge=1,
+    )
+
+
 class Type1(StrEnum):
     GLINER2 = 'GLINER2'
 
@@ -978,6 +1009,10 @@ class GLiNER2PipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['GLINER2'] = 'GLINER2'
     entities: dict[str, PipelineEntityDefinition] | None = None
     classification: dict[str, PipelineClassificationDefinition] | None = None
@@ -997,6 +1032,10 @@ class RegexPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['REGEX'] = 'REGEX'
     patterns: dict[str, RegexPatternDefinition] | None = None
     validation: PipelineValidationConfig | None = None
@@ -1105,6 +1144,10 @@ class LLMPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['LLM'] = 'LLM'
     system_prompt: str = Field(
         ...,
@@ -1205,6 +1248,10 @@ class TextClassificationPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['TEXT_CLASSIFICATION']
     model: str = Field(
         ...,
@@ -1279,6 +1326,10 @@ class ImageClassificationPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['IMAGE_CLASSIFICATION']
     model: str | None = Field(
         None,
@@ -1335,6 +1386,10 @@ class ObjectDetectionPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['OBJECT_DETECTION']
     model: str = Field(
         ...,
@@ -1399,6 +1454,10 @@ class TagPipelineSchema(BaseModel):
         None,
         description='Restrict this detector to part of a source (asset kind, content type, or an asset-metadata predicate). Null runs it on everything the source produces.',
     )
+    budget: DetectorBudget | None = Field(
+        None,
+        description='Failure and time budget for this detector within one run. Null uses the defaults: stop after a non-retryable provider refusal or 10 consecutive failures.',
+    )
     type: Literal['TAG'] = 'TAG'
     label: str | None = Field(
         None,

{classifyre_cli-0.5.3 → classifyre_cli-0.5.5}/src/models/generated_input.py

@@ -4631,6 +4631,24 @@ class Variables(RootModel[str]):
     root: str = Field(..., max_length=8192)
 
 
+class CohortWeightsMode(StrEnum):
+    """
+    auto: measured from what each band yields. fixed: the declared or fixed split, always.
+    """
+
+    auto = 'auto'
+    fixed = 'fixed'
+
+
+class CustomCohortBandWeights(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    newest: float | None = Field(None, ge=0.0)
+    oldest: float | None = Field(None, ge=0.0)
+    random: float | None = Field(None, ge=0.0)
+
+
 class NotebookPackage(BaseModel):
     """
     A Python distribution installed before the notebook runs.
@@ -5688,38 +5706,40 @@ class GitInput(CoreInput):
     augmentation: AugmentationConfig | None = None
 
 
-class CustomOptional(BaseModel):
+class CustomCohortWeights(BaseModel):
+    """
+    How ctx.cohort() splits each run's budget across its bands. auto shifts the split toward the bands whose keys produce HIGH/CRITICAL findings, within guards: the declared split stands until every band has 200 visits, no band drops below the floor, and no band moves more than 15 points per run. fixed always uses the bands the notebook declares, or the ones given here.
+    """
+
     model_config = ConfigDict(
         extra='forbid',
     )
-    variables: (
-        dict[constr(pattern=r'^[A-Za-z_][A-Za-z0-9_]{0,62}$'), Variables] | None
-    ) = Field(
-        {},
-        description='Non-secret values the notebook reads with ctx.var("name"), such as a base URL or an account id. Stored in clear text. Keys must be valid Python identifiers.',
-        max_length=64,
-        validate_default=True,
+    mode: CohortWeightsMode | None = Field(
+        'auto',
+        description='auto: measured from what each band yields. fixed: the declared or fixed split, always.',
+        title='CohortWeightsMode',
     )
-    limits: CustomOptionalLimits | None = None
-    packages: list[NotebookPackage] | None = Field(
-        [],
-        description="Python packages installed into the run environment before any cell executes. Installed with uv; the base image's own dependencies are always present and do not need listing.",
-        max_length=50,
-        validate_default=True,
+    floor: float | None = Field(
+        None,
+        description='The least share any band keeps under auto, as a fraction. Never below 0.1.',
+        ge=0.1,
+        le=0.5,
     )
-    local_folders: list[NotebookLocalFolder] | None = Field(
-        [],
-        description='Folders the notebook reads with ctx.folder("name"). Not available in Kubernetes deployments, where files are uploaded to the source instead. This is a convenience, not a sandbox: the notebook process runs as you and can open any path you can.',
-        max_length=10,
-        validate_default=True,
+    fixed: dict[str, CustomCohortBandWeights] | None = Field(
+        None,
+        description='Per cohort name, the band split to use under fixed, e.g. {"register": {"newest": 60, "oldest": 30, "random": 10}}.',
+    )
+    effective: dict[str, CustomCohortBandWeights] | None = Field(
+        None,
+        description="Set by the platform on each run's recipe: the split each cohort runs with. Not stored on the source.",
     )
 
 
-class CustomInput(CoreInput):
-    type: Literal['CUSTOM'] = Field(..., description='Type of the asset or source')
-    required: CustomRequired
-    masked: CustomMasked | None = None
-    optional: CustomOptional | None = None
+class YouTubeInput(CoreInput):
+    type: Literal['YOUTUBE'] = Field(..., description='Type of the asset or source')
+    required: YouTubeRequired
+    masked: YouTubeMasked | None = None
+    optional: YouTubeOptional | None = None
     detectors: list[Detector] | None = Field(
         None, description='Detectors to run on ingested content'
     )
@@ -5737,11 +5757,15 @@ class CustomInput(CoreInput):
     augmentation: AugmentationConfig | None = None
 
 
-class YouTubeInput(CoreInput):
-    type: Literal['YOUTUBE'] = Field(..., description='Type of the asset or source')
-    required: YouTubeRequired
-    masked: YouTubeMasked | None = None
-    optional: YouTubeOptional | None = None
+class RedditInput(CoreInput):
+    type: Literal['REDDIT'] = Field(..., description='Type of the asset or source')
+    required: (
+        RedditRequiredReadOnly | RedditRequiredScript | RedditRequiredRefreshToken
+    ) = Field(..., title='RedditRequired')
+    masked: RedditMaskedReadOnly | RedditMaskedScript | RedditMaskedRefreshToken = (
+        Field(..., title='RedditMasked')
+    )
+    optional: RedditOptional | None = None
     detectors: list[Detector] | None = Field(
         None, description='Detectors to run on ingested content'
     )
@@ -5759,15 +5783,39 @@ class YouTubeInput(CoreInput):
     augmentation: AugmentationConfig | None = None
 
 
-class RedditInput(CoreInput):
-    type: Literal['REDDIT'] = Field(..., description='Type of the asset or source')
-    required: (
-        RedditRequiredReadOnly | RedditRequiredScript | RedditRequiredRefreshToken
-    ) = Field(..., title='RedditRequired')
-    masked: RedditMaskedReadOnly | RedditMaskedScript | RedditMaskedRefreshToken = (
-        Field(..., title='RedditMasked')
+class CustomOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
     )
-    optional: RedditOptional | None = None
+    variables: (
+        dict[constr(pattern=r'^[A-Za-z_][A-Za-z0-9_]{0,62}$'), Variables] | None
+    ) = Field(
+        {},
+        description='Non-secret values the notebook reads with ctx.var("name"), such as a base URL or an account id. Stored in clear text. Keys must be valid Python identifiers.',
+        max_length=64,
+        validate_default=True,
+    )
+    limits: CustomOptionalLimits | None = None
+    packages: list[NotebookPackage] | None = Field(
+        [],
+        description="Python packages installed into the run environment before any cell executes. Installed with uv; the base image's own dependencies are always present and do not need listing.",
+        max_length=50,
+        validate_default=True,
+    )
+    local_folders: list[NotebookLocalFolder] | None = Field(
+        [],
+        description='Folders the notebook reads with ctx.folder("name"). Not available in Kubernetes deployments, where files are uploaded to the source instead. This is a convenience, not a sandbox: the notebook process runs as you and can open any path you can.',
+        max_length=10,
+        validate_default=True,
+    )
+    cohort_weights: CustomCohortWeights | None = None
+
+
+class CustomInput(CoreInput):
+    type: Literal['CUSTOM'] = Field(..., description='Type of the asset or source')
+    required: CustomRequired
+    masked: CustomMasked | None = None
+    optional: CustomOptional | None = None
     detectors: list[Detector] | None = Field(
         None, description='Detectors to run on ingested content'
     )