classifyre-cli 0.4.124__tar.gz → 0.4.126__tar.gz

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/.turbo/turbo-build.log

@@ -1,3 +1,3 @@
 $ uv sync
-Resolved 323 packages in 147ms
+Resolved 323 packages in 199ms
 Checked 51 packages in 1ms

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.4.124
+Version: 0.4.126
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.4.124",
+  "version": "0.4.126",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.4.124"
+version = "0.4.126"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 requires-python = ">=3.12"

classifyre_cli-0.4.126/scripts/generate_runtime_packages.py

@@ -0,0 +1,244 @@
+"""Generate the list of packages a notebook can import without declaring them.
+
+The CLI image already carries a lot: requests and lxml are always there, and
+pdfplumber, duckdb, boto3, psycopg2 and the rest arrive the first time something
+imports them. None of that was visible to a connector author, so people
+re-declared packages that were already installed -- at a different version than
+the lock had resolved.
+
+So the list is derived rather than written: names and pins come from
+``pyproject.toml``, resolved versions come from ``uv.lock``. Add a dependency and
+the notebook editor offers it; bump it and the editor shows the new version.
+
+Two judgements are encoded here and nowhere else:
+
+* ``INCLUDED_GROUPS`` -- which optional groups are worth advertising. Connector
+  and file-format groups are; the ML detector groups are not, because putting a
+  multi-gigabyte ``uv sync`` behind a one-line ``import`` is a trap rather than a
+  feature.
+* ``MODULE_ALIASES`` -- the import name, where it differs from the distribution
+  name. This is what makes ``import bs4`` resolve back to ``beautifulsoup4`` for
+  the group-warming path in ``src/notebook/groups.py``.
+
+Run from apps/cli:
+
+    uv run python scripts/generate_runtime_packages.py
+
+``tests/test_runtime_packages.py`` fails when the checked-in file drifts from
+what this would produce, so the two cannot silently diverge.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+from pathlib import Path
+from typing import Any
+
+CLI_ROOT = Path(__file__).resolve().parents[1]
+PYPROJECT_PATH = CLI_ROOT / "pyproject.toml"
+LOCK_PATH = CLI_ROOT / "uv.lock"
+OUTPUT_PATH = (
+    CLI_ROOT.parents[1]
+    / "packages"
+    / "schemas"
+    / "src"
+    / "schemas"
+    / "notebook_runtime_packages.json"
+)
+
+#: Optional dependency groups worth telling a connector author about: database
+#: drivers, object stores, SaaS clients and the file-format stack. Each is
+#: installed on demand, so listing one is a promise that importing it works --
+#: `src/notebook/groups.py` is what keeps that promise.
+INCLUDED_GROUPS = (
+    "file-processing",
+    "regex",
+    "llm",
+    "postgresql",
+    "mysql",
+    "mssql",
+    "oracle",
+    "hive",
+    "databricks",
+    "snowflake",
+    "mongodb",
+    "neo4j",
+    "tableau",
+    "s3-compatible-storage",
+    "azure-blob-storage",
+    "microsoft-graph",
+    "google-workspace",
+    "google-cloud-storage",
+    "dropbox",
+    "hugging-face",
+    "slack",
+    "email",
+    "youtube",
+    "reddit",
+    "kafka",
+    "delta-lake",
+    "iceberg",
+)
+
+#: Everything else in `[dependency-groups]`, listed rather than inferred so that
+#: adding a group is a deliberate decision in one place. The ML groups pull
+#: torch, transformers or docling -- gigabytes that an author would trigger by
+#: typing one import. `dev`, `interactive` and `otel` are tooling, not a
+#: connector's vocabulary; `detectors` and `ocr` are aggregates of the rest.
+EXCLUDED_GROUPS = (
+    "dev",
+    "interactive",
+    "detectors",
+    "ocr",
+    "otel",
+    "privacy",
+    "security",
+    "threat-ml",
+    "content",
+    "quality",
+    "classification",
+    "custom",
+    "transcription",
+    "video",
+)
+
+#: Import name, where it is not just the distribution name with '-' as '_'.
+MODULE_ALIASES: dict[str, tuple[str, ...]] = {
+    "beautifulsoup4": ("bs4",),
+    "pillow": ("PIL",),
+    "python-docx": ("docx",),
+    "opencv-python-headless": ("cv2",),
+    "psycopg2-binary": ("psycopg2",),
+    "google-re2": ("re2",),
+    "databricks-sql-connector": ("databricks.sql",),
+    "snowflake-connector-python": ("snowflake.connector",),
+    # Namespace packages: the dotted path is the whole import name, so that
+    # `import azure` matches both of these and `import azure.identity` matches
+    # only the one that provides it.
+    "azure-storage-blob": ("azure.storage.blob",),
+    "azure-identity": ("azure.identity",),
+    "google-api-python-client": ("googleapiclient",),
+    "google-auth": ("google.auth",),
+    "google-cloud-storage": ("google.cloud.storage",),
+    "msgraph-sdk": ("msgraph",),
+    "en-core-web-sm": ("en_core_web_sm",),
+    "classifyre-schemas": ("schemas",),
+}
+
+#: Distributions that exist in the lock but are not something a notebook imports
+#: directly: they are transports or model data pulled in by something else.
+SKIPPED_DISTRIBUTIONS = frozenset({"classifyre-schemas", "en-core-web-sm"})
+
+#: A requirement string is only ever read for its name here -- the version comes
+#: from the lock, which is what actually gets installed.
+_NAME_OF_REQUIREMENT = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)")
+
+
+def _canonical(name: str) -> str:
+    """PEP 503 normalization, so pyproject and uv.lock spellings line up."""
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def _requirement_name(requirement: Any) -> str | None:
+    """The distribution named by one `[dependency-groups]` entry.
+
+    Entries are usually strings; an `{include-group = ...}` table pulls in
+    another group, which the caller already resolves by listing groups
+    explicitly, so it is skipped rather than expanded.
+    """
+    if not isinstance(requirement, str):
+        return None
+    match = _NAME_OF_REQUIREMENT.match(requirement)
+    return _canonical(match.group(1)) if match else None
+
+
+def _locked_versions() -> dict[str, str]:
+    """Resolved version per distribution, from uv.lock.
+
+    A name can appear twice when an index pin produces a local build (`+cpu`);
+    the plain version is the one to show, since it is what the wheel reports.
+    """
+    lock = tomllib.loads(LOCK_PATH.read_text(encoding="utf-8"))
+    versions: dict[str, list[str]] = {}
+    for package in lock.get("package", []):
+        name = _canonical(str(package.get("name", "")))
+        version = str(package.get("version", ""))
+        if name and version:
+            versions.setdefault(name, []).append(version)
+
+    resolved: dict[str, str] = {}
+    for name, candidates in versions.items():
+        plain = [version for version in candidates if "+" not in version]
+        resolved[name] = sorted(plain or candidates)[-1]
+    return resolved
+
+
+def _modules(name: str) -> list[str]:
+    return list(MODULE_ALIASES.get(name, (name.replace("-", "_"),)))
+
+
+def build() -> dict[str, Any]:
+    pyproject = tomllib.loads(PYPROJECT_PATH.read_text(encoding="utf-8"))
+    versions = _locked_versions()
+
+    groups = pyproject.get("dependency-groups", {})
+    unknown = sorted(set(groups) - set(INCLUDED_GROUPS) - set(EXCLUDED_GROUPS))
+    if unknown:
+        raise SystemExit(
+            f"pyproject.toml has dependency groups this script has no verdict on: {unknown}. "
+            "Add each to INCLUDED_GROUPS or EXCLUDED_GROUPS in "
+            "scripts/generate_runtime_packages.py."
+        )
+
+    # A distribution can appear in several groups (pillow, pyarrow, boto3). The
+    # first one wins, and `always` wins over any group, so the availability an
+    # author is shown is the strongest one that holds.
+    entries: dict[str, dict[str, Any]] = {}
+
+    for requirement in pyproject.get("project", {}).get("dependencies", []):
+        name = _requirement_name(requirement)
+        if not name or name in SKIPPED_DISTRIBUTIONS:
+            continue
+        entries[name] = {
+            "name": name,
+            "version": versions.get(name, ""),
+            "modules": _modules(name),
+            "availability": "always",
+        }
+
+    for group in INCLUDED_GROUPS:
+        for requirement in groups.get(group, []):
+            name = _requirement_name(requirement)
+            if not name or name in SKIPPED_DISTRIBUTIONS or name in entries:
+                continue
+            entries[name] = {
+                "name": name,
+                "version": versions.get(name, ""),
+                "modules": _modules(name),
+                "availability": "on-demand",
+                "group": group,
+            }
+
+    return {
+        "$comment": (
+            "Generated by apps/cli/scripts/generate_runtime_packages.py from "
+            "apps/cli/pyproject.toml and apps/cli/uv.lock. Do not edit by hand."
+        ),
+        "pythonVersion": str(pyproject.get("project", {}).get("requires-python", "")),
+        "packages": [entries[name] for name in sorted(entries)],
+    }
+
+
+def main() -> int:
+    payload = build()
+    OUTPUT_PATH.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    print(
+        f"Wrote {OUTPUT_PATH.relative_to(CLI_ROOT.parents[1])} ({len(payload['packages'])} packages)"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/scripts/generate_sdk_completions.py

@@ -30,7 +30,14 @@ from src.notebook.contract import (  # noqa: E402
     OPTIONAL_FUNCTIONS,
     REQUIRED_FUNCTIONS,
 )
-from src.notebook.sdk import Asset, Context  # noqa: E402
+from src.notebook.sdk import (  # noqa: E402
+    Asset,
+    Context,
+    NotebookFile,
+    ParsedContent,
+    pages,
+    parse,
+)
 
 OUTPUT_PATH = (
     CLI_ROOT.parents[1] / "packages" / "schemas" / "src" / "schemas" / "notebook_completions.json"
@@ -156,6 +163,22 @@ def _fields(cls: type) -> list[dict[str, Any]]:
     return fields
 
 
+#: Module-level helpers a notebook can call directly. Offered alongside `ctx`
+#: and `Asset` because `namespace()` pre-binds them, so they are as available as
+#: the import line says they are.
+MODULE_FUNCTIONS = (("parse", parse), ("pages", pages))
+
+
+def _function_entry(name: str, func: Any) -> dict[str, Any]:
+    return {
+        "label": name,
+        "kind": "function",
+        "detail": _signature(func, drop_self=False),
+        "documentation": _summary(func),
+        "insertText": _snippet(name, func),
+    }
+
+
 def build() -> dict[str, Any]:
     return {
         "$comment": (
@@ -178,6 +201,7 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Asset),
                 "insertText": "Asset",
             },
+            *(_function_entry(name, func) for name, func in MODULE_FUNCTIONS),
         ],
         "objects": {
             "ctx": {"type": "Context", "members": _members(Context)},
@@ -187,6 +211,17 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Asset),
                 "fields": _fields(Asset),
             },
+            # Returned by `parse()` and reached as `ctx.file(...)`, so an author
+            # who never writes their name still needs their members offered.
+            "ParsedContent": {
+                "documentation": _summary(ParsedContent),
+                "fields": _fields(ParsedContent),
+            },
+            "NotebookFile": {
+                "documentation": _summary(NotebookFile),
+                "fields": _fields(NotebookFile),
+                "members": _members(NotebookFile),
+            },
         },
         "contract": {
             "required": list(REQUIRED_FUNCTIONS),

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/src/models/generated_input.py

@@ -5427,6 +5427,27 @@ class NotebookPackage(BaseModel):
     )
 
 
+class NotebookLocalFolder(BaseModel):
+    """
+    A folder on the machine running the desktop application.
+    """
+
+    model_config = ConfigDict(
+        extra='forbid',
+    )
+    name: str = Field(
+        ...,
+        description='How the notebook refers to it: ctx.folder("name"). Must be a valid Python identifier.',
+        pattern='^[A-Za-z_][A-Za-z0-9_]{0,62}$',
+    )
+    path: str = Field(
+        ...,
+        description='Absolute path to the folder, for example /Users/me/dumps.',
+        max_length=4096,
+        min_length=1,
+    )
+
+
 class YouTubeInput(CoreInput):
     type: Literal['YOUTUBE'] = Field(..., description='Type of the asset or source')
     required: YouTubeRequired
@@ -5581,6 +5602,12 @@ class CustomOptional(BaseModel):
         max_length=50,
         validate_default=True,
     )
+    local_folders: list[NotebookLocalFolder] | None = Field(
+        [],
+        description='Desktop only. Folders on this machine the notebook reads with ctx.folder("name"). Not available in Kubernetes deployments, where files are uploaded to the source instead. This is a convenience, not a sandbox: the notebook process runs as you and can open any path you can.',
+        max_length=10,
+        validate_default=True,
+    )
 
 
 class CustomInput(CoreInput):

{classifyre_cli-0.4.124 → classifyre_cli-0.4.126}/src/notebook/cli.py

@@ -11,11 +11,14 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 import signal
 import sys
 from typing import Any
 
 from .execute import execute_notebook
+from .files import local_folders
+from .groups import warm_declared_groups
 from .packages import install as install_packages
 from .protocol import (
     RESULT_PREFIX,
@@ -46,12 +49,24 @@ def _string_map(value: Any) -> dict[str, str]:
     return {str(key): str(item) for key, item in value.items() if item is not None}
 
 
+#: Where the API put this source's uploaded files before starting us. Set for an
+#: execution that runs cells; absent for one that only validates.
+NOTEBOOK_FILES_DIR_ENV = "CLASSIFYRE_NOTEBOOK_FILES_DIR"
+
+
 def build_context(recipe: dict[str, Any], *, should_abort: Any = None) -> Context:
-    """The `ctx` a notebook sees, assembled from its recipe."""
+    """The `ctx` a notebook sees, assembled from its recipe.
+
+    Authoring and scanning have to agree here: a cell that reads ``ctx.files``
+    while someone is writing it must see the same files the scan will, or the
+    notebook is tested against a different source than it runs against.
+    """
     return Context(
         variables=_string_map(_section(recipe, "optional").get("variables")),
         secrets=_string_map(_section(recipe, "masked").get("secrets")),
         sampling=_section(recipe, "sampling"),
+        files_dir=os.environ.get(NOTEBOOK_FILES_DIR_ENV) or None,
+        folders=local_folders(recipe),
         should_abort=should_abort,
     )
 
@@ -129,6 +144,10 @@ def run_request(raw_request: dict[str, Any]) -> ExecutionResponse:
     sys.stderr = RedactingStream(sys.stderr, redactor)
 
     warm_interactive_group()
+    # What the notebook imports from the runtime's own optional groups. Warmed
+    # before the declared packages below because `uv sync --frozen` prunes
+    # anything `uv pip install` added -- the reverse order uninstalls them.
+    warm_declared_groups(cells)
 
     # Before any cell: a connector's client library has to be importable by the
     # first line that imports it, and resolving on every Run would make the

classifyre_cli-0.4.126/src/notebook/files.py

@@ -0,0 +1,180 @@
+"""Reading and parsing files from a notebook.
+
+A connector that fetches files should not have to know how to read a PDF, an
+.eml, a .docx or a Parquet file -- every other source in this system delegates
+that to ``src/utils/file_parser.py``, and a notebook has more reason to than
+most. So the parser is exposed rather than reimplemented: ``parse()`` is a thin
+adapter over ``file_parser.parse_file`` / ``parse_bytes``, and ``pages()`` over
+its streaming path.
+
+Two deliberate choices:
+
+* The result is a small ``ParsedContent`` of our own rather than the parser's
+  ``ParsedFile`` / ``ParsedBytes``. Those two differ (``encoding`` on one,
+  ``raw_content`` on the other) and are internal shapes free to change; a
+  notebook that is someone's saved work should not break when they do.
+* ``file_parser`` is imported inside the functions. Importing it pulls the
+  ``file-processing`` group in at first use, and a notebook that never touches a
+  file should never pay for that.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:  # the parser and its heavy dependencies stay a runtime import
+    from ..utils.file_parser import ParsedBytes, ParsedFile
+
+logger = logging.getLogger(__name__)
+
+#: Text pages are read in this many lines, tabular payloads in this many rows.
+DEFAULT_PAGE_SIZE = 100
+
+
+@dataclass
+class ParsedContent:
+    """What one file turned out to be.
+
+    ``error`` is populated rather than raised: a corrupt attachment in the
+    middle of a long scan should cost that one asset, not the run.
+    """
+
+    #: Detected content type, e.g. ``application/pdf``.
+    mime_type: str
+    #: Extracted text, ready to hand to ``Asset(content=...)``. Empty when the
+    #: payload has no text in it, or when ``error`` is set.
+    text: str
+    #: Whether the payload is binary rather than plain text.
+    is_binary: bool
+    size_bytes: int
+    #: Why extraction produced nothing, when it did not. ``None`` on success.
+    error: str | None = None
+
+    def __bool__(self) -> bool:
+        return self.error is None
+
+
+def local_folders(recipe: dict[str, Any]) -> dict[str, str]:
+    """The desktop-only folders a source declared, as ``{name: path}``.
+
+    Read straight from the recipe rather than handed over separately: unlike an
+    uploaded file there is nothing to fetch, only a path the notebook may open.
+    Both the scan runner and the authoring CLI read it here so that ``ctx.folder``
+    means the same thing in the editor and in a real run.
+    """
+    optional = recipe.get("optional") if isinstance(recipe, dict) else None
+    declared = optional.get("local_folders") if isinstance(optional, dict) else None
+    if not isinstance(declared, list):
+        return {}
+    folders: dict[str, str] = {}
+    for entry in declared:
+        if isinstance(entry, dict) and entry.get("name") and entry.get("path"):
+            folders[str(entry["name"])] = str(entry["path"])
+    return folders
+
+
+def _as_source(source: Any) -> Any:
+    """Unwrap the things a notebook is likely to pass.
+
+    ``ctx.file(...)`` hands back a ``NotebookFile``; accepting it directly saves
+    every caller from writing ``.path``.
+    """
+    path = getattr(source, "path", None)
+    return path if isinstance(path, Path) else source
+
+
+def _name_of(source: Any, name: str) -> str:
+    if name:
+        return name
+    if isinstance(source, str | Path):
+        return Path(source).name
+    return str(getattr(source, "name", "") or "")
+
+
+def parse(source: Any, *, name: str = "", mime_type: str | None = None) -> ParsedContent:
+    """Detect a payload's type and extract its text.
+
+    Accepts a path, raw bytes, an open binary file, or a ``NotebookFile`` from
+    ``ctx.files``. ``mime_type`` is a hint: it is trusted over sniffing when the
+    caller already knows, which matters for a payload downloaded from an API
+    that declared one.
+
+        for file in ctx.files:
+            parsed = parse(file)
+            yield Asset(id=file.name, content=parsed.text, content_bytes=file.read_bytes())
+    """
+    from ..utils.file_parser import TextExtractionCoverageError, parse_bytes, parse_file
+
+    resolved = _as_source(source)
+    file_name = _name_of(resolved, name)
+
+    # `ParsedFile` and `ParsedBytes` are separate shapes that happen to share
+    # the four fields below; the union is what lets both land on ParsedContent.
+    result: ParsedFile | ParsedBytes
+    try:
+        if isinstance(resolved, str | Path):
+            result = parse_file(Path(resolved))
+        else:
+            result = parse_bytes(resolved, declared_mime_type=mime_type, file_name=file_name)
+    except (OSError, TextExtractionCoverageError, ValueError) as exc:
+        logger.warning("Could not parse %s: %s", file_name or "payload", exc)
+        return ParsedContent(
+            mime_type=mime_type or "application/octet-stream",
+            text="",
+            is_binary=True,
+            size_bytes=0,
+            error=str(exc),
+        )
+
+    return ParsedContent(
+        mime_type=result.mime_type,
+        text=result.text_content or "",
+        is_binary=bool(result.is_binary),
+        size_bytes=int(result.file_size_bytes or 0),
+        error=result.parse_error,
+    )
+
+
+def pages(
+    source: Any,
+    *,
+    name: str = "",
+    mime_type: str | None = None,
+    page_size: int = DEFAULT_PAGE_SIZE,
+) -> Iterator[str]:
+    """Read a payload in pages instead of all at once.
+
+    For a Parquet file or a large CSV, ``parse()`` materialises the whole thing
+    as one string. This yields rows (tabular) or lines (everything else) a page
+    at a time, so a multi-gigabyte dump can become many assets without ever
+    being held whole.
+
+        for index, page in enumerate(pages(ctx.file("dump.parquet"))):
+            yield Asset(id=f"dump-{index}", content=page, kind="table")
+    """
+    from ..utils.file_parser import iter_file_pages, resolve_mime_type
+
+    resolved = _as_source(source)
+    file_name = _name_of(resolved, name)
+
+    handle = None
+    try:
+        if isinstance(resolved, str | Path):
+            handle = Path(resolved).open("rb")
+            payload: Any = handle
+        else:
+            payload = resolved
+
+        resolved_mime = resolve_mime_type(
+            payload, declared_mime_type=mime_type, file_name=file_name
+        )
+        yield from iter_file_pages(
+            payload, resolved_mime, batch_size=page_size, file_name=file_name
+        )
+    finally:
+        if handle is not None:
+            handle.close()