whiff-python 0.1.0__cp311-abi3-win_amd64.whl

whiff/scratch.py

@@ -0,0 +1,294 @@
+"""Opt-in local staging for path extraction.
+
+``stage="local"`` copies a regular file into a private directory before the
+native call opens it, then rewrites the report so it still describes the
+original path. Times are second precision. Fields this module does not read
+(birth time, allocated size, sparse flag, owner names, ACLs, extended
+attributes) are omitted and named in ``diagnostics.warnings``.
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+import shutil
+import stat
+import tempfile
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from whiff.errors import (
+    AccessError,
+    InvalidOptionError,
+    NativeError,
+    NotFoundError,
+    UnsupportedInputError,
+    WhiffError,
+)
+
+_DEFAULT_MAX_BYTES = 1_073_741_824
+_DIR_MODE = 0o700
+_FILE_MODE = 0o600
+
+_OMITTED = (
+    'stage="local": filesystem fields taken from the original at second precision; '
+    "created, allocated_bytes, sparse, owner names, acl, and xattrs omitted"
+)
+_UNREAD = 'stage="local": original file could not be re-read; filesystem fields describe the copy'
+
+
+@dataclass(frozen=True)
+class Staged:
+    original: str
+    staged: str
+
+
+def scratch_dir() -> str:
+    return os.environ.get("WHIFF_SCRATCH_DIR") or str(Path(tempfile.gettempdir()) / "whiff-scratch")
+
+
+def scratch_max_bytes() -> int:
+    raw = os.environ.get("WHIFF_SCRATCH_MAX_BYTES")
+    if raw is None:
+        return _configured_max
+    try:
+        value = int(raw)
+    except ValueError:
+        return _configured_max
+    return value if value > 0 else _configured_max
+
+
+_configured_max = _DEFAULT_MAX_BYTES
+
+
+def configure(*, directory: str | None = None, max_bytes: int | None = None) -> None:
+    global _configured_max
+    if directory is not None:
+        os.environ["WHIFF_SCRATCH_DIR"] = directory
+    if max_bytes is not None:
+        if isinstance(max_bytes, bool) or not isinstance(max_bytes, int) or max_bytes < 1:
+            raise InvalidOptionError("scratch_max_bytes must be a positive integer")
+        _configured_max = max_bytes
+        os.environ["WHIFF_SCRATCH_MAX_BYTES"] = str(max_bytes)
+
+
+def stage(path: str, mode: str) -> Staged | None:
+    """Copy ``path`` when ``mode`` is ``local``. Return None when staging is off."""
+    if mode == "never":
+        return None
+    if mode != "local":
+        raise InvalidOptionError(f"invalid stage: {mode!r}")
+    return _stage_local(path)
+
+
+def cleanup(tag: Staged | None) -> None:
+    if tag is None:
+        return
+    shutil.rmtree(os.path.dirname(tag.staged), ignore_errors=True)
+
+
+def restore_report(report: dict[str, Any], tag: Staged | None) -> dict[str, Any]:
+    if tag is None:
+        return report
+    try:
+        info = os.stat(tag.original)
+    except OSError:
+        return _warn(_put(report, ("source", "path"), tag.original), _UNREAD)
+    updated = _put(report, ("source", "path"), tag.original)
+    updated = _restore_temporal(updated, info)
+    updated = _restore_system(updated, info)
+    updated = _restore_security(updated, info)
+    return _warn(updated, _OMITTED)
+
+
+def restore_error(err: WhiffError, tag: Staged | None) -> WhiffError:
+    if tag is None:
+        return err
+    message = err.message.replace(tag.staged, tag.original)
+    err.message = message
+    err.path = tag.original
+    err.args = (err._format(),)
+    return err
+
+
+def _stage_local(path: str) -> Staged:
+    try:
+        info = os.stat(path)
+    except FileNotFoundError as err:
+        raise NotFoundError(f"stat: no such file: {err.strerror}", path=path) from err
+    except PermissionError as err:
+        raise AccessError(f"stat: permission denied: {err.strerror}", path=path) from err
+    except OSError as err:
+        raise NativeError(f"stat: {err.strerror or err}", path=path) from err
+    if not stat.S_ISREG(info.st_mode):
+        kind = "directory" if stat.S_ISDIR(info.st_mode) else "non-regular file"
+        raise UnsupportedInputError(
+            f"scratch staging requires a regular file, got {kind}", path=path
+        )
+    limit = scratch_max_bytes()
+    if info.st_size > limit:
+        raise InvalidOptionError(
+            f"file size {info.st_size} exceeds scratch_max_bytes {limit}",
+            path=path,
+            details={"size": info.st_size, "scratch_max_bytes": limit},
+        )
+    root = _ensure_root(scratch_dir())
+    job = _job_dir(root)
+    try:
+        _verify_root(root, job)
+        staged = os.path.join(job, os.path.basename(path) or "input")
+        _copy(path, staged, info)
+    except Exception:
+        shutil.rmtree(job, ignore_errors=True)
+        raise
+    return Staged(original=path, staged=staged)
+
+
+def _ensure_root(root: str) -> str:
+    if os.path.islink(root):
+        raise NativeError("scratch_dir is a symlink, not a directory", path=root)
+    if os.path.exists(root):
+        if not os.path.isdir(root):
+            raise NativeError("scratch_dir is not a directory", path=root)
+        return root
+    os.makedirs(root, mode=_DIR_MODE, exist_ok=False)
+    os.chmod(root, _DIR_MODE)
+    return root
+
+
+def _job_dir(root: str) -> str:
+    last_error: Exception | None = None
+    for _ in range(3):
+        path = os.path.join(root, "whiff-" + secrets.token_urlsafe(12))
+        try:
+            os.mkdir(path, _DIR_MODE)
+        except FileExistsError:
+            continue
+        except OSError as err:
+            last_error = err
+            break
+        os.chmod(path, _DIR_MODE)
+        return path
+    raise NativeError(
+        f"could not create a unique scratch job directory: {last_error}",
+        path=root,
+    )
+
+
+def _verify_root(root: str, job: str) -> None:
+    if os.name != "posix":
+        return
+    root_stat = os.lstat(root)
+    job_stat = os.lstat(job)
+    if root_stat.st_uid != job_stat.st_uid:
+        raise AccessError("scratch_dir is owned by another user", path=root)
+    if (root_stat.st_mode & 0o022) != 0 and (root_stat.st_mode & 0o1000) == 0:
+        raise AccessError("scratch_dir is writable by group or other users", path=root)
+
+
+def _copy(source: str, dest: str, info: os.stat_result) -> None:
+    shutil.copyfile(source, dest)
+    os.chmod(dest, _FILE_MODE)
+    os.utime(dest, (info.st_atime, info.st_mtime))
+
+
+def _restore_temporal(report: dict[str, Any], info: os.stat_result) -> dict[str, Any]:
+    def update(section: dict[str, Any]) -> dict[str, Any]:
+        section = dict(section)
+        section["modified"] = _rfc3339(info.st_mtime)
+        section["accessed"] = _rfc3339(info.st_atime)
+        section.pop("created", None)
+        if os.name == "posix":
+            section["changed"] = _rfc3339(info.st_ctime)
+        else:
+            section.pop("changed", None)
+        return section
+
+    return _update(report, ("temporal", "filesystem"), update)
+
+
+def _restore_system(report: dict[str, Any], info: os.stat_result) -> dict[str, Any]:
+    def update(section: dict[str, Any]) -> dict[str, Any]:
+        section = dict(section)
+        section["device"] = info.st_dev
+        section["inode"] = info.st_ino
+        section["links"] = info.st_nlink
+        section.pop("allocated_bytes", None)
+        section.pop("sparse", None)
+        return section
+
+    return _update(report, ("system",), update)
+
+
+def _restore_security(report: dict[str, Any], info: os.stat_result) -> dict[str, Any]:
+    def update(section: dict[str, Any]) -> dict[str, Any]:
+        section = dict(section)
+        mode = info.st_mode & 0o7777
+        _put_if(section, "mode", "0" + format(mode, "o"))
+        _put_if(section, "permissions", _permissions(mode))
+        _put_if(section, "setuid", (mode & 0o4000) != 0)
+        _put_if(section, "setgid", (mode & 0o2000) != 0)
+        _put_if(section, "sticky", (mode & 0o1000) != 0)
+        if "owner" in section:
+            section["owner"] = {"uid": info.st_uid, "gid": info.st_gid}
+        section.pop("acl", None)
+        section.pop("xattrs", None)
+        return section
+
+    return _update(report, ("security",), update)
+
+
+def _put_if(section: dict[str, Any], key: str, value: object) -> None:
+    if key in section:
+        section[key] = value
+
+
+def _permissions(mode: int) -> str:
+    bits = (0o400, 0o200, 0o100, 0o040, 0o020, 0o010, 0o004, 0o002, 0o001)
+    chars = "rwxrwxrwx"
+    return "".join(char if mode & bit else "-" for bit, char in zip(bits, chars, strict=True))
+
+
+def _rfc3339(timestamp: float) -> str:
+    return datetime.fromtimestamp(int(timestamp), UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _update(report: dict[str, Any], path: tuple[str, ...], fn) -> dict[str, Any]:
+    current = _get(report, path)
+    if not isinstance(current, dict):
+        return report
+    return _put(report, path, fn(current))
+
+
+def _warn(report: dict[str, Any], warning: str) -> dict[str, Any]:
+    diagnostics = report.get("diagnostics")
+    diagnostics = dict(diagnostics) if isinstance(diagnostics, dict) else {}
+    warnings = list(diagnostics.get("warnings") or [])
+    warnings.append(warning)
+    diagnostics["warnings"] = warnings
+    updated = dict(report)
+    updated["diagnostics"] = diagnostics
+    return updated
+
+
+def _get(report: dict[str, Any], path: tuple[str, ...]) -> object:
+    current: object = report
+    for key in path:
+        if not isinstance(current, dict) or key not in current:
+            return None
+        current = current[key]
+    return current
+
+
+def _put(report: dict[str, Any], path: tuple[str, ...], value: object) -> dict[str, Any]:
+    if len(path) == 1:
+        updated = dict(report)
+        updated[path[0]] = value
+        return updated
+    child = report.get(path[0])
+    child = dict(child) if isinstance(child, dict) else {}
+    updated = dict(report)
+    updated[path[0]] = _put(child, path[1:], value)
+    return updated

whiff_python-0.1.0.dist-info/METADATA

@@ -0,0 +1,183 @@
+Metadata-Version: 2.4
+Name: whiff-python
+Version: 0.1.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Rust
+Classifier: Typing :: Typed
+Requires-Dist: pytest>=8 ; extra == 'dev'
+Requires-Dist: ruff>=0.8 ; extra == 'dev'
+Provides-Extra: dev
+License-File: LICENSE
+Summary: Identify a file or bytes and return a whiff/1 report.
+Keywords: metadata,file-format,detection,whiff
+Author: Sin Anima
+License-Expression: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Upstream, https://github.com/baoulo/whiff
+
+# whiff
+
+Python bindings for [whiff](https://github.com/baoulo/whiff). Point it at a file or at bytes and it returns one [`whiff/1`](https://github.com/baoulo/whiff/blob/main/docs/schema/whiff-1.schema.json) report: format, hashes, security and rights signals, content, and format-specific metadata.
+
+Identification and extraction run in the Rust crate (0.8.0, the `common` feature set). Python extractors and analyzers can add fields afterwards. They do not run inside native extraction, and they do not run for embedded children.
+
+```python
+import whiff
+
+report = whiff.extract("invoice.pdf")
+report["identification"]["format"]  # "pdf"
+
+report = whiff.extract(data, name="invoice.pdf", hashes=["sha256"])
+view = whiff.llm("contract.pdf", tokens=800)
+```
+
+Empty optional fields are omitted. The vendored schema is `whiff.schema()`.
+
+## Install
+
+Import the package as `whiff`. The PyPI project `whiff` already belongs to an unrelated library, so this distribution is named `whiff-python`. A GitHub release tagged with the `version` in `pyproject.toml` (an optional `v` prefix is fine) publishes the wheels. From a checkout, with Rust 1.89 or later:
+
+```sh
+maturin develop
+```
+
+That builds the `common` formats. A source build of every collection is:
+
+```sh
+maturin develop --features full
+```
+
+`geo`, `science`, `data`, `cad`, `forensics`, and `games` are the individual cargo features. Wheels are built with `common` only.
+
+## Extract
+
+`extract` accepts a path, `bytes`, or a binary file object. A file object is read up to `max_read_bytes` and then treated as bytes. `name` (or `filename`, not both) drives extension detection for in-memory input.
+
+Corrupt or partial files still return a report. Problems inside the file are listed in `diagnostics`. The call raises `WhiffError` when the path cannot be inspected or an option is invalid:
+
+```python
+try:
+    report = whiff.extract(path)
+except whiff.NotFoundError:
+    ...
+except whiff.AccessError:
+    ...
+except whiff.InvalidOptionError as err:
+    print(err.message)
+```
+
+`NotFoundError` is also a `FileNotFoundError`. `AccessError` is also a `PermissionError`. IO messages include the full path.
+
+`llm` returns upstream `Report::llm_view`. Python plugins do not run for it.
+
+```python
+results = whiff.extract_many(
+    paths,
+    max_concurrency=4,
+    timeout=300_000,
+    profile="minimal",
+    hashes="none",
+)
+```
+
+Results keep input order. With `on_error="collect"` (the default) each item is `(path, report)` or `(path, error)`. With `on_error="halt"` the first error is raised and `err.details["partial"]` holds the pairs completed before it. `timeout` is per path, in milliseconds. A timeout does not stop native work that has already started.
+
+An `Engine` holds native options and is safe to share across threads:
+
+```python
+engine = whiff.Engine(profile="minimal", hashes="none")
+engine.extract(path)
+```
+
+Native calls take a process-wide slot (default 8). A caller that waits longer than `native_wait_ms` (default 60000) raises `OverloadedError`. Change the limits with `whiff.configure(native_max_concurrency=4, native_wait_ms=10_000)`.
+
+`stage="local"` copies a regular file into a private scratch directory before taking a slot, for slow or network filesystems. The copy is removed afterwards. The report still describes the original path. Times are second precision, and fields Python does not read (birth time, allocated size, owner names, ACLs, extended attributes) are omitted, with a `diagnostics.warnings` entry. Files larger than `scratch_max_bytes` (default 1 GiB) are rejected.
+
+## Options
+
+An option a function does not use is rejected.
+
+| Option | Values | Default |
+|---|---|---|
+| `profile` | `minimal`, `standard`, `full`, `llm` | `standard` |
+| `hashes` | `all`, `none`, or a list of `sha256` / `blake3` | `all` |
+| `privacy` | `strict`, `off` | `strict` |
+| `only` / `skip` | extractor ids or groups | none |
+| `format` | format id to force | detect |
+| `provenance` | record which extractor set each field | `False` |
+| `limits` | dict of native limits; see below | profile limits |
+| `stage` | `never`, `local` | `never` |
+| `name` / `filename` / `mime` | strings (`bytes` and file objects) | none |
+| `tokens` | 1..100000 | `800` (`llm` only) |
+| `mode` | `fallback`, `augment`, `override` | `fallback` |
+| `extractors` / `analyzers` | plugin classes or instances | none |
+| `strict` | plugin failure raises | `False` |
+| `plugin_timeout_ms` | 1..600000 | `5000` |
+| `override_fields` | dotted paths, for example `["content.title"]` | `[]` |
+| `max_concurrency` / `on_error` / `timeout` | `extract_many` | `4` / `collect` / `300000` |
+
+`limits` keys: `max_read_bytes` and `max_decompressed_bytes` (1..4 GiB), `hash_max_bytes` (0..4 GiB), `max_parse_bytes` (0..256 MiB), `max_text_bytes` (0..64 MiB), `max_depth` (0..8), `max_children` (0..500), `file_timeout_ms` and `extractor_timeout_ms` (1..600000).
+
+`privacy="strict"` reduces personal data in native structured fields to counts and scrubs plugin patches. Upstream redacts secrets either way. `privacy="off"` leaves plugin patches unchanged.
+
+## Plugins
+
+```python
+class Trade(whiff.Extractor):
+    id = "myapp.trade"
+    version = 1
+
+    def supports(self, source, report) -> bool:
+        return source.extension == ".trade"
+
+    def extract(self, source, report) -> dict:
+        return {"format": {"myapp.trade": {"trade_id": "T-1001"}}}
+
+
+report = whiff.extract("positions/2026-10-28.trade", extractors=[Trade])
+```
+
+An analyzer subclasses `whiff.Analyzer` and implements `analyze(report, source)`. Analyzers run after extractors. Their patches always merge in `augment` mode.
+
+| Mode | When an extractor runs | Existing native values |
+|---|---|---|
+| `fallback` | native extraction did not cover a readable file | kept, except `content.*` |
+| `augment` | `supports` returns true | kept |
+| `override` | `supports` returns true | replaced for paths in `override_fields` |
+
+`whiff`, `integrity`, and `embedded` cannot be patched. `diagnostics` accepts only extra warnings. `security.risks` are appended with `source` set to `python:<id>`. `metadata` and `format` keys are namespaces. One plugin cannot replace a value another plugin wrote unless that path is declared.
+
+A failed plugin is recorded in `diagnostics.extractors` with `status` `error`, and the call still returns the report. With `strict=True` the call raises. Plugins run on a worker thread. They are not a security boundary.
+
+`whiff.merge` applies the same rules to a patch dict.
+
+## Development
+
+```sh
+cargo test
+cargo clippy --all-targets -- -D warnings
+maturin develop
+ruff check src tests
+pytest
+```
+
+Golden reports are compared with `probes/rust_extract`, which links the same `whiff` 0.8.0 crate. Design notes: [docs/adr/0001-baseline.md](docs/adr/0001-baseline.md).
+
+## Limitations
+
+- `extract` of `bytes` copies the buffer once into native memory.
+- There is no streaming reader. A file object is read up to the read cap.
+- Python callbacks do not run for embedded children, and `llm` does not run them.
+- `stage="local"` copies the whole file first.
+- A native `file_timeout` can leave a Rust worker running after the call returns.
+- IO error messages include the full path.
+
+## License
+
+MIT. Upstream whiff is also MIT.
+

whiff_python-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+whiff/__init__.py,sha256=FlXzk5La2EAmFEpa63QUgrBw_GMXHCs6uFTweB2TDck,1412
+whiff/_native.pyd,sha256=EEyljJWAfoBQWHUOX7_UZwSPFWRTdFgptOSaztH1QBk,27445760
+whiff/_native.pyi,sha256=8DBhwUI3HBf3GCoPwjNNN3v_4VYbJlxzZlIwShGhBU0,470
+whiff/engine.py,sha256=CLiDTHMhwy68kkg8IbIBl_JVr-ObPrEk4gpu3-IDOM4,14158
+whiff/errors.py,sha256=ZSK50TZufBAhx3go5fd3BOC7pxYFqpoGXNwd25XL7gY,4996
+whiff/input.py,sha256=88mnZgT4_7YkmSJ8YLCZh2SymeIG2ARD-u4aVi1krKY,1613
+whiff/limiter.py,sha256=prTlHegotUxtarDugMecSn8Bs-BtsA9NEt-B5ALk57Q,4249
+whiff/merge.py,sha256=1mutnMGCYM7qscy19yHWEC4eVYEW9yA7Aq97kN4yWwA,14397
+whiff/options.py,sha256=DX0wYFaAbYq-UCDLW-bn_mA0Kvsji5pDOr5SeafeP5c,8313
+whiff/plugins.py,sha256=eSu5CVEp8j5T7SeDwarvkWpMbtSTzaJMmQCxcQVHI5w,10677
+whiff/privacy.py,sha256=Gk7YerZzfrfRHODR11NolpYsJbFcZOtDNd8K_m1iGFg,6094
+whiff/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+whiff/report.py,sha256=GzZOyuxWHKtutGMyjEqEzSO0J8qHP2fCBdXBR4pILeA,2901
+whiff/schemas/whiff-1.schema.json,sha256=5_dfDz8-7zn7nfrdnwQlYkwtETZ2PqANQEq49qWa2iI,46452
+whiff/scratch.py,sha256=aWN5TF-lrYuNV7M-tocJ88aO0mfPTCItRQN8ZmHvPyM,10023
+whiff_python-0.1.0.dist-info/METADATA,sha256=VfhfVX-1we6EciBJ_QpjjWCHmG31Wwu_KCxY1dgsciY,8209
+whiff_python-0.1.0.dist-info/WHEEL,sha256=DRguzO_JHkJTcrXDweuHyM66sVwM00caBYZ3J_zMn0w,96
+whiff_python-0.1.0.dist-info/licenses/LICENSE,sha256=tFTWErWxMCMtJ-_hJwDjOrCZXO30u6hYoYR_5dzUMF8,1087
+whiff_python-0.1.0.dist-info/sboms/whiff_native.cyclonedx.json,sha256=1tXqSRMzGGsWb8qZvsp3zDKXVSQufJGXI71cSVbd9Ys,180490
+whiff_python-0.1.0.dist-info/RECORD,,

whiff_python-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.16.0)
+Root-Is-Purelib: false
+Tag: cp311-abi3-win_amd64

whiff_python-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sin Anima
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.