classifyre-cli 0.4.121__tar.gz → 0.4.123__tar.gz

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 309 packages in 236ms
+Resolved 323 packages in 9ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.4.121
+Version: 0.4.123
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.121",
+  "version": "0.4.123",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.121"
+version = "0.4.123"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -45,6 +45,13 @@ build-backend = "hatchling.build"
 packages = ["src"]
 
 [dependency-groups]
+# Notebook authoring only. `run_cell` is what makes a bare `df` on a cell's last
+# line display as a table instead of being discarded, and what gives rich
+# outputs a display protocol. Production never installs this: a productized
+# notebook is an ordinary Python module run with `python workflow.py`.
+interactive = [
+    "ipython>=8.30.0",
+]
 dev = [
     "ruff>=0.15.13",
     "mypy>=1.13.0",
@@ -340,6 +347,12 @@ ignore_missing_imports = true
 module = ["bs4.*", "lxml.*"]
 ignore_missing_imports = true
 
+# Notebook authoring extras: present only when the `interactive` group is
+# installed, so the executor imports them defensively.
+[[tool.mypy.overrides]]
+module = ["IPython.*", "matplotlib.*"]
+ignore_missing_imports = true
+
 [[tool.mypy.overrides]]
 module = [
     "neo4j.*",

classifyre_cli-0.4.123/scripts/generate_sdk_completions.py

@@ -0,0 +1,206 @@
+"""Generate the notebook editor's SDK completions from the SDK itself.
+
+The editor has to offer `ctx.secret(...)` and `Asset(...)` with the right
+arguments, and a hand-written list of those would be wrong the first time
+someone adds a parameter. So the list is derived by introspecting the runtime
+objects a notebook actually receives -- add a method to ``Context`` and the
+editor offers it, with its real signature and its real docstring.
+
+Run from apps/cli:
+
+    uv run python scripts/generate_sdk_completions.py
+
+``tests/test_sdk_completions.py`` fails when the checked-in file drifts from
+what this would produce, so the two cannot silently diverge.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import inspect
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+CLI_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(CLI_ROOT))
+
+from src.notebook.contract import (  # noqa: E402
+    OPTIONAL_FUNCTIONS,
+    REQUIRED_FUNCTIONS,
+)
+from src.notebook.sdk import Asset, Context  # noqa: E402
+
+OUTPUT_PATH = (
+    CLI_ROOT.parents[1] / "packages" / "schemas" / "src" / "schemas" / "notebook_completions.json"
+)
+
+
+def _summary(obj: Any) -> str:
+    """The whole docstring, which is what the editor shows on demand.
+
+    Not just the first paragraph: the part of a docstring that changes what an
+    author writes is often the caveat below it -- that reading ``ctx.offset``
+    hands paging to the notebook, say -- and truncating to a one-liner is
+    exactly how that gets lost.
+    """
+    return (inspect.getdoc(obj) or "").strip()
+
+
+def _signature(func: Any, *, drop_self: bool = True) -> str:
+    try:
+        signature = inspect.signature(func)
+    except (TypeError, ValueError):
+        return "()"
+    parameters = [
+        parameter
+        for name, parameter in signature.parameters.items()
+        if not (drop_self and name == "self")
+    ]
+    return str(signature.replace(parameters=parameters))
+
+
+def _return_type(func: Any) -> str:
+    """The annotated return type, rendered as it appears in the source."""
+    try:
+        annotation = inspect.signature(func).return_annotation
+    except (TypeError, ValueError):
+        return ""
+    if annotation is inspect.Signature.empty:
+        return ""
+    return str(annotation).strip("'")
+
+
+def _snippet(name: str, func: Any) -> str:
+    """A tab-stop snippet, so calling a method lands the cursor in its first arg."""
+    try:
+        signature = inspect.signature(func)
+    except (TypeError, ValueError):
+        return f"{name}()"
+    required = [
+        parameter_name
+        for parameter_name, parameter in signature.parameters.items()
+        if parameter_name != "self"
+        and parameter.default is inspect.Parameter.empty
+        and parameter.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
+    ]
+    if not required:
+        return f"{name}()"
+    placeholders = ", ".join(
+        f"${{{index}:{parameter}}}" for index, parameter in enumerate(required, start=1)
+    )
+    return f"{name}({placeholders})"
+
+
+#: Public on Context because the runtime reads them, but not part of what a
+#: notebook author writes -- offering them would be noise at best and a
+#: misleading suggestion at worst.
+RUNTIME_ONLY_MEMBERS = frozenset({"set_offset", "offset_consumed", "next_cursor"})
+
+
+def _members(cls: type) -> list[dict[str, Any]]:
+    members: list[dict[str, Any]] = []
+    for name, member in sorted(vars(cls).items()):
+        if name.startswith("_") or name in RUNTIME_ONLY_MEMBERS:
+            continue
+        if isinstance(member, property):
+            members.append(
+                {
+                    "label": name,
+                    "kind": "property",
+                    # A property is read, not called, so showing its getter's
+                    # "()" would suggest otherwise. Only the type it yields.
+                    "detail": _return_type(member.fget) if member.fget else "",
+                    "documentation": _summary(member),
+                    "insertText": name,
+                }
+            )
+        elif inspect.isfunction(member) or isinstance(member, staticmethod):
+            func = member.__func__ if isinstance(member, staticmethod) else member
+            members.append(
+                {
+                    "label": name,
+                    "kind": "method",
+                    "detail": _signature(func),
+                    "documentation": _summary(func),
+                    "insertText": _snippet(name, func),
+                }
+            )
+    return members
+
+
+def _fields(cls: type) -> list[dict[str, Any]]:
+    """Dataclass fields, offered as keyword arguments when constructing one."""
+    annotations = getattr(cls, "__annotations__", {})
+    dataclass_fields = getattr(cls, "__dataclass_fields__", {})
+    fields: list[dict[str, Any]] = []
+    for name in annotations:
+        if name.startswith("_"):
+            continue
+        field = dataclass_fields.get(name)
+        # dataclasses signal "no default" with MISSING, not Parameter.empty --
+        # the wrong sentinel marks every field optional.
+        required = field is not None and (
+            field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING
+        )
+        fields.append(
+            {
+                "label": name,
+                "kind": "field",
+                "detail": str(annotations[name]),
+                "required": required,
+                "insertText": f"{name}=",
+            }
+        )
+    return fields
+
+
+def build() -> dict[str, Any]:
+    return {
+        "$comment": (
+            "Generated by apps/cli/scripts/generate_sdk_completions.py from "
+            "apps/cli/src/notebook/sdk.py. Do not edit by hand."
+        ),
+        "module": "classifyre",
+        "globals": [
+            {
+                "label": "ctx",
+                "kind": "variable",
+                "detail": "Context",
+                "documentation": _summary(Context),
+                "insertText": "ctx",
+            },
+            {
+                "label": "Asset",
+                "kind": "class",
+                "detail": "Asset(...)",
+                "documentation": _summary(Asset),
+                "insertText": "Asset",
+            },
+        ],
+        "objects": {
+            "ctx": {"type": "Context", "members": _members(Context)},
+        },
+        "classes": {
+            "Asset": {
+                "documentation": _summary(Asset),
+                "fields": _fields(Asset),
+            },
+        },
+        "contract": {
+            "required": list(REQUIRED_FUNCTIONS),
+            "optional": list(OPTIONAL_FUNCTIONS),
+        },
+    }
+
+
+def main() -> int:
+    payload = build()
+    OUTPUT_PATH.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    print(f"Wrote {OUTPUT_PATH.relative_to(CLI_ROOT.parents[1])}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/src/main.py

@@ -743,14 +743,24 @@ def main() -> None:
     parser = argparse.ArgumentParser(description="Classifyre Metadata Extraction CLI")
     parser.add_argument(
         "command",
-        choices=["test", "extract", "discover", "evaluate-file", "train", "dropbox-auth"],
+        choices=[
+            "test",
+            "extract",
+            "discover",
+            "notebook",
+            "evaluate-file",
+            "train",
+            "dropbox-auth",
+        ],
         help="Command to run",
     )
     parser.add_argument(
         "recipe",
         nargs="?",
         default=None,
-        help="Path to recipe JSON (or file path for evaluate-file)",
+        help=(
+            "Path to recipe JSON (a file path for evaluate-file, an execution request for notebook)"
+        ),
     )
     parser.add_argument("--debug", action="store_true", help="Enable debug logging")
     parser.add_argument(
@@ -856,6 +866,11 @@ def main() -> None:
     if args.command == "dropbox-auth":
         sys.exit(run_dropbox_auth_command(args))
 
+    if args.command == "notebook":
+        from .notebook.cli import run_notebook_command
+
+        sys.exit(run_notebook_command(args))
+
     if args.command == "evaluate-file":
         run_evaluate_file_command(args)
         return

{classifyre_cli-0.4.121 → classifyre_cli-0.4.123}/src/models/generated_input.py

@@ -14,6 +14,7 @@ from pydantic import (
     EmailStr,
     Field,
     RootModel,
+    constr,
 )
 
 
@@ -59,6 +60,7 @@ class AssetType(StrEnum):
     HUGGING_FACE = 'HUGGING_FACE'
     SANDBOX = 'SANDBOX'
     GIT = 'GIT'
+    CUSTOM = 'CUSTOM'
 
 
 class DetectorType(StrEnum):
@@ -391,6 +393,7 @@ class Type(StrEnum):
     HUGGING_FACE = 'HUGGING_FACE'
     SANDBOX = 'SANDBOX'
     GIT = 'GIT'
+    CUSTOM = 'CUSTOM'
 
 
 class YouTubeRequired(BaseModel):
@@ -3473,6 +3476,7 @@ class Type22(StrEnum):
     HUGGING_FACE = 'HUGGING_FACE'
     SANDBOX = 'SANDBOX'
     GIT = 'GIT'
+    CUSTOM = 'CUSTOM'
 
 
 class ConfluenceInput(CoreInput):
@@ -5278,6 +5282,151 @@ class GitInput(CoreInput):
     )
 
 
+class NotebookCodeCell(BaseModel):
+    """
+    A Python cell. Must remain valid standard Python -- IPython magics (%time, !pip) are rejected.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: str = Field(
+        ...,
+        description='Stable cell identifier, unique within the notebook',
+        pattern='^[A-Za-z0-9_-]{1,64}$',
+    )
+    type: Literal['code']
+    source: str = Field(
+        ..., description='Python source of this cell', max_length=100000
+    )
+
+
+class NotebookMarkdownCell(BaseModel):
+    """
+    Prose. Never executed; serialized as comments when the notebook is exported to Python.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    id: str = Field(
+        ...,
+        description='Stable cell identifier, unique within the notebook',
+        pattern='^[A-Za-z0-9_-]{1,64}$',
+    )
+    type: Literal['markdown']
+    source: str = Field(
+        ..., description='Markdown source of this cell', max_length=100000
+    )
+
+
+class CustomNotebook(BaseModel):
+    """
+    Cells run in document order in one fresh process per execution. There is no persistent kernel: state is rebuilt from the current cell sources every time.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    revision: int | None = Field(
+        1,
+        description='Monotonic revision, bumped on every save. Used for optimistic locking so two editors cannot silently overwrite each other.',
+        ge=1,
+    )
+    cells: list[NotebookCodeCell | NotebookMarkdownCell] = Field(
+        ...,
+        description='Ordered notebook cells. The assembled code cells must define test_connection() and extract().',
+        max_length=200,
+        min_length=1,
+    )
+
+
+class CustomRequired(BaseModel):
+    """
+    The Python notebook that implements this connector.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    notebook: CustomNotebook
+
+
+class Secrets(RootModel[str]):
+    root: str = Field(..., max_length=8192)
+
+
+class CustomMasked(BaseModel):
+    """
+    Encrypted key/value pairs, stored the same way as any other source credential.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    secrets: dict[constr(pattern=r'^[A-Za-z_][A-Za-z0-9_]{0,62}$'), Secrets] | None = (
+        Field(
+            {},
+            description='Secret values the notebook reads with ctx.secret("name"). Encrypted at rest and redacted from logs and cell output. Keys must be valid Python identifiers.',
+            max_length=64,
+            validate_default=True,
+        )
+    )
+
+
+class CustomOptionalLimits(BaseModel):
+    """
+    Bounds applied to every notebook execution and scan.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    max_assets: int | None = Field(
+        None,
+        description='Stop a scan after this many assets. Unset means no cap.',
+        ge=1,
+        le=1000000,
+    )
+    timeout_seconds: int | None = Field(
+        900,
+        description='Kill an execution that runs longer than this. Cells cannot be interrupted from inside Python, so this is the real stop button.',
+        ge=10,
+        le=86400,
+    )
+    max_output_bytes: int | None = Field(
+        2097152,
+        description='Total serialized cell output kept per execution. Larger outputs are truncated rather than stored.',
+        ge=1024,
+        le=52428800,
+    )
+
+
+class Variables(RootModel[str]):
+    root: str = Field(..., max_length=8192)
+
+
+class NotebookPackage(BaseModel):
+    """
+    A Python distribution installed before the notebook runs.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    name: str = Field(
+        ...,
+        description="Distribution name as published on the index (for example 'pandas').",
+        pattern='^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$',
+    )
+    version: str | None = Field(
+        None,
+        description="Optional version or specifier: '2.2.0', '>=2.0', '~=1.4'. Empty means latest.",
+        max_length=64,
+        pattern='^$|^(==|>=|<=|~=|!=|>|<).+$|^[0-9][A-Za-z0-9._*+!-]*$',
+    )
+
+
 class YouTubeInput(CoreInput):
     type: Literal['YOUTUBE'] = Field(..., description='Type of the asset or source')
     required: YouTubeRequired
@@ -5413,6 +5562,48 @@ class IcebergInput(CoreInput):
     )
 
 
+class CustomOptional(BaseModel):
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    variables: (
+        dict[constr(pattern=r'^[A-Za-z_][A-Za-z0-9_]{0,62}$'), Variables] | None
+    ) = Field(
+        {},
+        description='Non-secret values the notebook reads with ctx.var("name"), such as a base URL or an account id. Stored in clear text. Keys must be valid Python identifiers.',
+        max_length=64,
+        validate_default=True,
+    )
+    limits: CustomOptionalLimits | None = None
+    packages: list[NotebookPackage] | None = Field(
+        [],
+        description="Python packages installed into the run environment before any cell executes. Installed with uv; the base image's own dependencies are always present and do not need listing.",
+        max_length=50,
+        validate_default=True,
+    )
+
+
+class CustomInput(CoreInput):
+    type: Literal['CUSTOM'] = Field(..., description='Type of the asset or source')
+    required: CustomRequired
+    masked: CustomMasked | None = None
+    optional: CustomOptional | None = None
+    detectors: list[Detector] | None = Field(
+        None, description='Detectors to run on ingested content'
+    )
+    custom_detectors: list[CustomDetectorSelection] | None = Field(
+        None,
+        description='Reusable custom detector IDs selected from the custom detector catalog.',
+    )
+    sampling: SamplingConfig
+    scan_cache: ScanCacheConfig | None = None
+    resources: ResourceOverrides | None = None
+    cleanup_removed_detector_findings: bool | None = Field(
+        True,
+        description='When enabled (default), findings produced by detectors that are no longer configured on this source (removed or disabled) are automatically resolved at the start of the next run, keeping the findings list in step with the current detector set.',
+    )
+
+
 class SourceInput(
     RootModel[
         SandboxInput
@@ -5452,6 +5643,7 @@ class SourceInput(
         | DropboxInput
         | HuggingFaceInput
         | GitInput
+        | CustomInput
     ]
 ):
     root: (
@@ -5492,6 +5684,7 @@ class SourceInput(
         | DropboxInput
         | HuggingFaceInput
         | GitInput
+        | CustomInput
     ) = Field(
         ...,
         description='Merged configuration schema with all source types and common definitions',

classifyre_cli-0.4.123/src/notebook/__init__.py

@@ -0,0 +1,29 @@
+"""Notebook runtime for the CUSTOM source.
+
+A notebook is a list of cells, not a running kernel. Every execution assembles
+the current cell sources into one ordinary Python module, runs it in a fresh
+process, serializes what the target cell produced, and exits. Nothing survives
+between executions except the cell sources themselves -- which is what keeps
+"what you see" and "what runs" the same thing.
+"""
+
+from .contract import ContractReport, ContractViolation, validate_notebook
+from .protocol import ExecutionMode, ExecutionRequest, ExecutionResponse
+from .redact import Redactor
+from .scaffold import cell_for_function, scaffold_cells
+from .serialize import CellSpan, ModuleSource, to_module_source
+
+__all__ = [
+    "CellSpan",
+    "ContractReport",
+    "ContractViolation",
+    "ExecutionMode",
+    "ExecutionRequest",
+    "ExecutionResponse",
+    "ModuleSource",
+    "Redactor",
+    "cell_for_function",
+    "scaffold_cells",
+    "to_module_source",
+    "validate_notebook",
+]