data-director-inventory 0.1.0a3__py3-none-any.whl

data_director_inventory-0.1.0a3.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.5
+Name: data-director-inventory
+Version: 0.1.0-alpha.3
+Summary: Describe a dataset's files where they live: size, digest, PRONOM format and table fields, without uploading them
+Project-URL: Homepage, https://data-director-agent.github.io/data-director/
+Project-URL: Documentation, https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/
+Project-URL: Repository, https://github.com/data-director-agent/data-director/tree/main/inventory
+Project-URL: Changelog, https://github.com/data-director-agent/data-director/blob/main/inventory/CHANGELOG.md
+Project-URL: Issues, https://github.com/data-director-agent/data-director/issues
+Author-email: Joe Heffer <j.heffer@sheffield.ac.uk>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: FAIR,Frictionless,PRONOM,checksum,data management,inventory,research data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: System :: Archiving
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Requires-Dist: frictionless<6,>=5.18
+Requires-Dist: opf-fido==1.6.1
+Description-Content-Type: text/markdown
+
+# Data Director inventory
+
+[![CI](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml/badge.svg)](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml) [![Coverage](https://codecov.io/gh/data-director-agent/data-director/graph/badge.svg?flag=inventory)](https://codecov.io/gh/data-director-agent/data-director?flags[0]=inventory)
+
+`dd-inventory` describes a dataset's files where they live. It reads every file under a folder
+and writes a `FileInventory`, a JSON document of the core contract, that gives for each file:
+
+- its path relative to the folder;
+- its size and its SHA-256 digest;
+- its media type and every [PRONOM](https://www.nationalarchives.gov.uk/PRONOM/) identifier it
+  matched, found with [fido](https://github.com/openpreserve/fido), and whether the match was by
+  byte signature, container signature or extension only;
+- for a CSV or TSV table, its fields as a [Frictionless](https://framework.frictionlessdata.io/)
+  Table Schema inferred from a sample of rows.
+
+The inventory holds facts about the files and never their content. Research data are not
+uploaded to Data Director: the researcher adds the inventory to their project instead
+([Projects](https://github.com/data-director-agent/data-director/blob/main/docs/specs/projects.md)). The digests let the files deposited
+later be shown to be the files described.
+
+```console
+$ uv run dd-inventory path/to/dataset -o inventory.json
+```
+
+The package installs two names for the same command, `dd-inventory` and `data-director-inventory`,
+so that `uvx data-director-inventory path/to/dataset` works. It is published to [PyPI](https://pypi.org/p/data-director-inventory) and, for trying a release first, TestPyPI ([RELEASING.md](https://github.com/data-director-agent/data-director/blob/main/RELEASING.md)).
+
+A researcher without a checkout runs it with `uvx` and attaches the file to a conversation in the
+app; the user guide's [Making an inventory](https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/)
+says how.
+
+Hidden files and folders (names beginning with `.`) are skipped, symbolic links are not
+followed, and both are counted in `inventory_limitations`. If fido cannot be loaded, formats are
+guessed from file extensions and the inventory says so.
+
+To read an inventory in TypeScript, as the app does to show one to the researcher, use
+[`@data-director/inventory`](https://github.com/data-director-agent/data-director/blob/main/inventory/browser/README.md) in `browser/`, which is not part of this Python
+package.
+
+The package does not depend on the contract or any other part of Data Director
+([System and boundaries](https://github.com/data-director-agent/data-director/blob/main/docs/specs/system.md)). Its tests check
+what it writes against the contract's `FileInventory` schema.
+
+| Module | What it does |
+|---|---|
+| `scan.py` | Walks the folder, digests each file and assembles the inventory. |
+| `identify.py` | The one module that calls fido, with the extension fallback. |
+| `tables.py` | Infers a table's fields with Frictionless. |
+| `cli.py` | The `dd-inventory` command. |

data_director_inventory-0.1.0a3.dist-info/RECORD

@@ -0,0 +1,11 @@
+dd_inventory/__init__.py,sha256=hp_trp3Pam6GMuOOENTkEsSRQUD1lUfIXimLgsKLIRg,497
+dd_inventory/cli.py,sha256=IpKNZpqRL8tmDEmup6pemmpDt5eT7quVR2y9PfmAjq4,1562
+dd_inventory/identify.py,sha256=cw3ZGZX9S98e_Mh8EwMDONDVy3HTHwDuHkqG039UUs4,5249
+dd_inventory/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dd_inventory/scan.py,sha256=tabMZCclE802M7N7IFo1XnJHhxaPqdC2Ti9_gN08k9c,4279
+dd_inventory/tables.py,sha256=g5yS9IsXWbndBKLl4BOlpaKCqAmIrHdRGKL33BcgB_I,1658
+data_director_inventory-0.1.0a3.dist-info/METADATA,sha256=OAbPEgMwBj0lGsQn09OyhM3V-SADkTEDlTItx7ISfx8,4558
+data_director_inventory-0.1.0a3.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+data_director_inventory-0.1.0a3.dist-info/entry_points.txt,sha256=AkvceYfOpf-5a2II6BpKABtdaD4NTgHID9KdgmWRJBw,103
+data_director_inventory-0.1.0a3.dist-info/licenses/LICENSE,sha256=6gZU8sLgglMXGyB_q9vTYivSiRPHrFMLadPc7ydtLbg,1084
+data_director_inventory-0.1.0a3.dist-info/RECORD,,

data_director_inventory-0.1.0a3.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

data_director_inventory-0.1.0a3.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+data-director-inventory = dd_inventory.cli:main
+dd-inventory = dd_inventory.cli:main

data_director_inventory-0.1.0a3.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 The University of Sheffield
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dd_inventory/__init__.py

@@ -0,0 +1,11 @@
+"""Describe a dataset's files where they live, without sending them anywhere
+(docs/specs/projects.md).
+
+`dd-inventory ROOT` walks a folder and writes a `FileInventory`: for each file its path relative
+to ROOT, its size, its SHA-256 digest, its format identified against PRONOM, and for a table its
+fields. The inventory holds facts about the files and never their content.
+"""
+
+from dd_inventory.scan import TOOL, make_inventory, tool_version
+
+__all__ = ["TOOL", "make_inventory", "tool_version"]

dd_inventory/cli.py

@@ -0,0 +1,46 @@
+"""`dd-inventory ROOT [-o FILE]`: write the inventory of a dataset's files as JSON.
+
+Run it where the data live. It reads every file under ROOT and sends nothing anywhere; the
+researcher adds the JSON it writes to their project.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from dd_inventory.scan import make_inventory, tool_version
+
+EXIT_OK, EXIT_USAGE = 0, 2
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="dd-inventory",
+        description=(
+            "Describe the files under ROOT (paths, sizes, SHA-256 digests, PRONOM formats and "
+            "table fields) as a Data Director FileInventory. No file content is written."
+        ),
+    )
+    parser.add_argument("root", type=Path, metavar="ROOT", help="the dataset's root folder")
+    parser.add_argument(
+        "-o", "--output", type=Path, help="write the inventory to this file, not standard output"
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {tool_version()}")
+    args = parser.parse_args(argv)
+    if not args.root.is_dir():
+        parser.print_usage(sys.stderr)
+        sys.stderr.write(f"dd-inventory: {args.root} is not a folder\n")
+        return EXIT_USAGE
+    text = json.dumps(make_inventory(args.root), indent=2, ensure_ascii=False) + "\n"
+    if args.output:
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        sys.stdout.write(text)
+    return EXIT_OK
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dd_inventory/identify.py

@@ -0,0 +1,127 @@
+"""Identify a file's format against PRONOM with fido. The one module that knows fido.
+
+fido reads the start and end of a file and matches them against PRONOM's byte signatures, then
+its container signatures for ZIP and OLE2 files, and falls back to the file's extension. Each
+step can match several formats, so every PRONOM identifier (PUID) it returns is kept, with the
+step that found them.
+
+Where fido cannot be loaded, `ExtensionIdentifier` guesses a media type from the extension and
+gives no PUID (docs/specs/projects.md, dependencies).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import mimetypes
+import warnings
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any, Protocol
+from xml.etree import ElementTree
+
+# fido's step names are the core's `IdentificationMethod` values; an empty match is `none`.
+SIGNATURE, CONTAINER, EXTENSION, NONE = "signature", "container", "extension", "none"
+
+
+@dataclass(frozen=True)
+class Identification:
+    method: str
+    pronom_ids: list[str] = field(default_factory=list)
+    media_type: str | None = None
+
+
+class Identifier(Protocol):
+    # What identified the formats, recorded in the inventory.
+    description: str
+
+    def identify(self, path: Path) -> Identification: ...
+
+
+class FidoUnavailableError(Exception):
+    """fido could not be loaded; the reason is recorded in the inventory's limitations."""
+
+
+@contextlib.contextmanager
+def _quiet_fido() -> Iterator[None]:
+    """fido tests an XML element's truth value, which Python 3.12 deprecates, and writes notices
+    (an empty file, an unreadable one) to stderr itself. Neither is the caller's to see."""
+    with warnings.catch_warnings(), contextlib.redirect_stderr(io.StringIO()):
+        warnings.simplefilter("ignore", DeprecationWarning)
+        yield
+
+
+def guess_media_type(path: Path) -> str | None:
+    return mimetypes.guess_type(path.name, strict=False)[0]
+
+
+class ExtensionIdentifier:
+    description = "file extensions only"
+
+    def identify(self, path: Path) -> Identification:
+        media_type = guess_media_type(path)
+        return Identification(EXTENSION if media_type else NONE, [], media_type)
+
+
+class FidoIdentifier:
+    """fido's own steps, in the order its `identify_file` takes them, on a file this opens and
+    closes: `identify_file` leaves every file it reads open."""
+
+    def __init__(self) -> None:
+        try:
+            from fido import CONFIG_DIR
+            from fido.fido import Fido
+            from fido.package import OlePackage, ZipPackage
+            from fido.versions import get_local_versions
+
+            # The library's defaults name a signature file fido 1.6.1 does not ship; its CLI
+            # reads the files from versions.xml, and so does this.
+            versions = get_local_versions()
+            with _quiet_fido():
+                self._fido = Fido(
+                    quiet=True,
+                    format_files=[versions.pronom_signature, versions.fido_extension_signature],
+                )
+                self._containers = ElementTree.parse(
+                    Path(CONFIG_DIR) / versions.pronom_container_signature
+                )
+            pronom = versions.pronom_version
+        except Exception as exc:  # fido raises bare OSError, ParseError, ...
+            raise FidoUnavailableError(f"fido could not be loaded: {exc}") from exc
+        self._packages = {"zip": ("ZIP", ZipPackage), "ole": ("OLE2", OlePackage)}
+        self.description = f"fido {version('opf-fido')} with PRONOM signatures v{pronom}"
+
+    def identify(self, path: Path) -> Identification:
+        fido = self._fido
+        with _quiet_fido():
+            size = path.stat().st_size
+            with path.open("rb") as fh:
+                bof, eof, _read = fido.get_buffers(fh, size, seekable=True)
+            # An empty file matches RTF's signature, so fido goes by its extension instead.
+            matches = fido.match_formats(bof, eof) if size else []
+            package = self._packages.get(fido.container_type(matches))
+            if package is not None:
+                kind, cls = package
+                inside = fido.match_container(kind, cls, str(path), self._containers)
+                if inside:
+                    return self._found(path, inside, CONTAINER)
+            if matches:
+                return self._found(path, matches, SIGNATURE)
+            return self._found(path, fido.match_extensions(str(path)), EXTENSION)
+
+    @staticmethod
+    def _found(path: Path, matches: list[tuple[Any, Any]], method: str) -> Identification:
+        puids = [p.text for f, _sig in matches if (p := f.find("puid")) is not None and p.text]
+        mimes = [m.text for f, _sig in matches if (m := f.find("mime")) is not None and m.text]
+        media_type = mimes[0] if mimes else guess_media_type(path)
+        return Identification(method if puids else NONE, puids, media_type)
+
+
+def default_identifier() -> tuple[Identifier, str | None]:
+    """fido, or the extension fallback and why fido was not used."""
+    try:
+        return FidoIdentifier(), None
+    except FidoUnavailableError as exc:
+        return ExtensionIdentifier(), str(exc)

dd_inventory/scan.py

@@ -0,0 +1,119 @@
+"""Walk a dataset's root folder and make its `FileInventory` (docs/specs/projects.md).
+
+The inventory is a plain JSON document of the core contract class `FileInventory`; this package
+does not need the contract to make one. Paths are relative to the root and sorted, so two runs
+over the same files differ only in `inventory_made_at`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from dd_inventory.identify import Identifier, default_identifier
+from dd_inventory.tables import TableError, is_table, table_fields
+
+TOOL = "dd-inventory"
+SCHEMA_CLASS = "FileInventory"
+
+
+def tool_version() -> str:
+    return version("data-director-inventory")
+
+
+def sha256(path: Path) -> str:
+    with path.open("rb") as fh:
+        return hashlib.file_digest(fh, "sha256").hexdigest()
+
+
+def _walk(root: Path) -> tuple[list[Path], list[str]]:
+    """Every regular file under `root`, and what was left out. Hidden files and links are not
+    followed: a link may point outside the dataset, and a hidden file is rarely part of it."""
+    found: list[Path] = []
+    hidden = links = 0
+    for folder, dirs, names in os.walk(root, followlinks=False):
+        here = Path(folder)
+        for d in list(dirs):
+            if d.startswith(".") or (here / d).is_symlink():
+                hidden += d.startswith(".")
+                links += not d.startswith(".")
+                dirs.remove(d)
+        for name in names:
+            path = here / name
+            if name.startswith("."):
+                hidden += 1
+            elif path.is_symlink():
+                links += 1
+            elif path.is_file():
+                found.append(path)
+    skipped = []
+    if hidden:
+        skipped.append(
+            f"{hidden} hidden file(s) or folder(s), whose names begin with '.', were skipped."
+        )
+    if links:
+        skipped.append(f"{links} symbolic link(s) were not followed.")
+    return sorted(found, key=lambda p: p.relative_to(root).as_posix()), skipped
+
+
+def make_inventory(
+    root: Path, *, identifier: Identifier | None = None, now: datetime | None = None
+) -> dict[str, Any]:
+    """The `FileInventory` of every file under `root`, as a JSON-ready dict."""
+    root = root.resolve()
+    if not root.is_dir():
+        raise NotADirectoryError(root)
+    limitations: list[str] = []
+    if identifier is None:
+        identifier, unavailable = default_identifier()
+        if unavailable:
+            limitations.append(f"{unavailable}; formats were guessed from file extensions only.")
+    paths, skipped = _walk(root)
+    limitations += skipped
+    files = []
+    for path in paths:
+        try:
+            files.append(_describe(root, path, identifier, limitations))
+        except OSError as exc:
+            limitations.append(
+                f"{path.relative_to(root).as_posix()} could not be read: {exc.strerror}."
+            )
+    limitations.append(
+        "Table fields are inferred from a sample of rows; a later row may not fit the type given."
+    )
+    made_at = (now or datetime.now(UTC)).isoformat(timespec="seconds")
+    return {
+        "schema_class": SCHEMA_CLASS,
+        "inventory_files": files,
+        "inventory_tool": TOOL,
+        "inventory_tool_version": tool_version(),
+        "inventory_identified_with": identifier.description,
+        "inventory_made_at": made_at,
+        "inventory_limitations": limitations,
+    }
+
+
+def _describe(
+    root: Path, path: Path, identifier: Identifier, limitations: list[str]
+) -> dict[str, Any]:
+    relative = PurePosixPath(path.relative_to(root).as_posix())
+    found = identifier.identify(path)
+    entry: dict[str, Any] = {
+        "file_path": str(relative),
+        "file_size": path.stat().st_size,
+        "file_sha256": sha256(path),
+    }
+    if found.media_type:
+        entry["file_media_type"] = found.media_type
+    entry["file_pronom_ids"] = found.pronom_ids
+    entry["file_identified_by"] = found.method
+    if is_table(path):
+        try:
+            entry["file_fields"] = table_fields(path)
+        except TableError as exc:
+            limitations.append(f"The fields of {relative} could not be inferred: {exc}")
+    return entry

dd_inventory/tables.py

@@ -0,0 +1,48 @@
+"""Infer a table's fields as a Frictionless Table Schema.
+
+Only delimited text is described: CSV and TSV. Frictionless infers each field's type and format
+from a sample of rows, so a type is a guess that later rows may contradict.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+# Rows Frictionless reads to infer types; its own default is 100.
+SAMPLE_ROWS = 1000
+TABLE_SUFFIXES = {".csv": ",", ".tsv": "\t", ".tab": "\t"}
+
+
+class TableError(Exception):
+    """The file looked like a table but its fields could not be inferred."""
+
+
+def is_table(path: Path) -> bool:
+    return path.suffix.lower() in TABLE_SUFFIXES
+
+
+def table_fields(path: Path) -> list[dict[str, Any]]:
+    """The table's fields as `TableField` objects: name, field_type and, if any, field_format."""
+    from frictionless import Detector, Dialect, Resource, formats
+
+    control = formats.CsvControl(delimiter=TABLE_SUFFIXES[path.suffix.lower()])
+    try:
+        resource = Resource(
+            path=path.name,
+            basepath=str(path.parent),
+            format="csv",
+            dialect=Dialect(controls=[control]),
+            detector=Detector(sample_size=SAMPLE_ROWS),
+        )
+        resource.infer()
+        schema = resource.schema
+    except Exception as exc:  # Frictionless raises its own errors and the reader's
+        raise TableError(f"{path.name}: {exc}") from exc
+    fields = []
+    for f in schema.fields:
+        entry: dict[str, Any] = {"name": f.name, "field_type": f.type}
+        if f.format and f.format != "default":
+            entry["field_format"] = f.format
+        fields.append(entry)
+    return fields