data-director-inventory 0.1.0a3__tar.gz

data_director_inventory-0.1.0a3/.gitignore

@@ -0,0 +1,240 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+.report.json
+# Playwright traces from workbench/browser_tests (pytest-playwright --output)
+test-results/
+# The lock export the dependency audit reads (.github/workflows/audit.yml)
+requirements-audit.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+# Local development settings and model keys (docs/specs/policy-and-identity.md).
+config/local/
+# Per-agent env files under agents/ hold API keys; the bare .env rule misses *.env names.
+agents/**/*.env
+agents/**/.env.*
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+
+# Inspect AI evaluation logs (docs/specs/evaluation.md)
+logs/
+
+# MkDocs build output
+/site/
+
+# A local copy of the app profile (app/config/profile.yaml.example) for editing.
+app/config/profile.yaml
+
+# Coding agents
+.claude/worktrees/

data_director_inventory-0.1.0a3/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+
+## [0.1.0-alpha.3](https://github.com/data-director-agent/data-director/compare/inventory-v0.1.0-alpha.2...inventory-v0.1.0-alpha.3) (2026-11-06)
+
+
+### Features
+
+* **app:** show an inventory in readable form ([e4461f1](https://github.com/data-director-agent/data-director/commit/e4461f1b6314bcdb938cc9a6f5d3f05b1809886c))
+* **inventory:** add a TypeScript library for reading an inventory ([aacfbea](https://github.com/data-director-agent/data-director/commit/aacfbeafef963525a366fc318ef913925e61b378))
+
+
+### Bug Fixes
+
+* **inventory:** complete the package metadata for PyPI ([20918d7](https://github.com/data-director-agent/data-director/commit/20918d7da918d9742a303179c6cd3bbaf3ec3f09))
+
+
+### Documentation
+
+* describe the inventory library and the inventory dialog, and wire them into CI ([ced5cca](https://github.com/data-director-agent/data-director/commit/ced5ccac2af755851622639ea363f17877aaaf29))
+* **inventory:** complete the package metadata for PyPI ([4c3f782](https://github.com/data-director-agent/data-director/commit/4c3f782bdefef97dfc7dd205ced16d80329e458b))
+
+## [0.1.0-alpha.2](https://github.com/data-director-agent/data-director/compare/inventory-v0.1.0-alpha.1...inventory-v0.1.0-alpha.2) (2026-11-06)
+
+
+### Features
+
+* **inventory:** publish to TestPyPI and add a data-director-inventory command ([#562](https://github.com/data-director-agent/data-director/issues/562)) ([d89c15d](https://github.com/data-director-agent/data-director/commit/d89c15d0b1223bfe70ad7e2163d7fe4cb80f3ec2)), closes [#552](https://github.com/data-director-agent/data-director/issues/552)
+
+## [0.1.0-alpha.1](https://github.com/data-director-agent/data-director/compare/inventory-v0.1.0-alpha...inventory-v0.1.0-alpha.1) (2026-11-05)
+
+
+### Features
+
+* **app:** attach a folder's inventory in the chat ([#463](https://github.com/data-director-agent/data-director/issues/463)) ([12470f8](https://github.com/data-director-agent/data-director/commit/12470f86a9c7fd7508d5e084c97f0d2895d9a573))
+* **inventory:** inventory a dataset's files where they live ([#393](https://github.com/data-director-agent/data-director/issues/393)) ([2238ce6](https://github.com/data-director-agent/data-director/commit/2238ce657e8158931229f4339cafc83a857bbb03))
+
+
+### Bug Fixes
+
+* **docs:** publish the inventory changelog ([#466](https://github.com/data-director-agent/data-director/issues/466)) ([e238600](https://github.com/data-director-agent/data-director/commit/e238600fadfe267f3af84a11d779088cf42b50d2))
+
+
+### Documentation
+
+* replace decision records with living specs ([#434](https://github.com/data-director-agent/data-director/issues/434)) ([a19caba](https://github.com/data-director-agent/data-director/commit/a19caba60b1f1fc0049c0a68c29feb2354b97db5))

data_director_inventory-0.1.0a3/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 The University of Sheffield
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data_director_inventory-0.1.0a3/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.5
+Name: data-director-inventory
+Version: 0.1.0-alpha.3
+Summary: Describe a dataset's files where they live: size, digest, PRONOM format and table fields, without uploading them
+Project-URL: Homepage, https://data-director-agent.github.io/data-director/
+Project-URL: Documentation, https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/
+Project-URL: Repository, https://github.com/data-director-agent/data-director/tree/main/inventory
+Project-URL: Changelog, https://github.com/data-director-agent/data-director/blob/main/inventory/CHANGELOG.md
+Project-URL: Issues, https://github.com/data-director-agent/data-director/issues
+Author-email: Joe Heffer <j.heffer@sheffield.ac.uk>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: FAIR,Frictionless,PRONOM,checksum,data management,inventory,research data
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: System :: Archiving
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Requires-Dist: frictionless<6,>=5.18
+Requires-Dist: opf-fido==1.6.1
+Description-Content-Type: text/markdown
+
+# Data Director inventory
+
+[![CI](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml/badge.svg)](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml) [![Coverage](https://codecov.io/gh/data-director-agent/data-director/graph/badge.svg?flag=inventory)](https://codecov.io/gh/data-director-agent/data-director?flags[0]=inventory)
+
+`dd-inventory` describes a dataset's files where they live. It reads every file under a folder
+and writes a `FileInventory`, a JSON document of the core contract, that gives for each file:
+
+- its path relative to the folder;
+- its size and its SHA-256 digest;
+- its media type and every [PRONOM](https://www.nationalarchives.gov.uk/PRONOM/) identifier it
+  matched, found with [fido](https://github.com/openpreserve/fido), and whether the match was by
+  byte signature, container signature or extension only;
+- for a CSV or TSV table, its fields as a [Frictionless](https://framework.frictionlessdata.io/)
+  Table Schema inferred from a sample of rows.
+
+The inventory holds facts about the files and never their content. Research data are not
+uploaded to Data Director: the researcher adds the inventory to their project instead
+([Projects](https://github.com/data-director-agent/data-director/blob/main/docs/specs/projects.md)). The digests let the files deposited
+later be shown to be the files described.
+
+```console
+$ uv run dd-inventory path/to/dataset -o inventory.json
+```
+
+The package installs two names for the same command, `dd-inventory` and `data-director-inventory`,
+so that `uvx data-director-inventory path/to/dataset` works. It is published to [PyPI](https://pypi.org/p/data-director-inventory) and, for trying a release first, TestPyPI ([RELEASING.md](https://github.com/data-director-agent/data-director/blob/main/RELEASING.md)).
+
+A researcher without a checkout runs it with `uvx` and attaches the file to a conversation in the
+app; the user guide's [Making an inventory](https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/)
+says how.
+
+Hidden files and folders (names beginning with `.`) are skipped, symbolic links are not
+followed, and both are counted in `inventory_limitations`. If fido cannot be loaded, formats are
+guessed from file extensions and the inventory says so.
+
+To read an inventory in TypeScript, as the app does to show one to the researcher, use
+[`@data-director/inventory`](https://github.com/data-director-agent/data-director/blob/main/inventory/browser/README.md) in `browser/`, which is not part of this Python
+package.
+
+The package does not depend on the contract or any other part of Data Director
+([System and boundaries](https://github.com/data-director-agent/data-director/blob/main/docs/specs/system.md)). Its tests check
+what it writes against the contract's `FileInventory` schema.
+
+| Module | What it does |
+|---|---|
+| `scan.py` | Walks the folder, digests each file and assembles the inventory. |
+| `identify.py` | The one module that calls fido, with the extension fallback. |
+| `tables.py` | Infers a table's fields with Frictionless. |
+| `cli.py` | The `dd-inventory` command. |

data_director_inventory-0.1.0a3/README.md

@@ -0,0 +1,49 @@
+# Data Director inventory
+
+[![CI](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml/badge.svg)](https://github.com/data-director-agent/data-director/actions/workflows/ci.yml) [![Coverage](https://codecov.io/gh/data-director-agent/data-director/graph/badge.svg?flag=inventory)](https://codecov.io/gh/data-director-agent/data-director?flags[0]=inventory)
+
+`dd-inventory` describes a dataset's files where they live. It reads every file under a folder
+and writes a `FileInventory`, a JSON document of the core contract, that gives for each file:
+
+- its path relative to the folder;
+- its size and its SHA-256 digest;
+- its media type and every [PRONOM](https://www.nationalarchives.gov.uk/PRONOM/) identifier it
+  matched, found with [fido](https://github.com/openpreserve/fido), and whether the match was by
+  byte signature, container signature or extension only;
+- for a CSV or TSV table, its fields as a [Frictionless](https://framework.frictionlessdata.io/)
+  Table Schema inferred from a sample of rows.
+
+The inventory holds facts about the files and never their content. Research data are not
+uploaded to Data Director: the researcher adds the inventory to their project instead
+([Projects](https://github.com/data-director-agent/data-director/blob/main/docs/specs/projects.md)). The digests let the files deposited
+later be shown to be the files described.
+
+```console
+$ uv run dd-inventory path/to/dataset -o inventory.json
+```
+
+The package installs two names for the same command, `dd-inventory` and `data-director-inventory`,
+so that `uvx data-director-inventory path/to/dataset` works. It is published to [PyPI](https://pypi.org/p/data-director-inventory) and, for trying a release first, TestPyPI ([RELEASING.md](https://github.com/data-director-agent/data-director/blob/main/RELEASING.md)).
+
+A researcher without a checkout runs it with `uvx` and attaches the file to a conversation in the
+app; the user guide's [Making an inventory](https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/)
+says how.
+
+Hidden files and folders (names beginning with `.`) are skipped, symbolic links are not
+followed, and both are counted in `inventory_limitations`. If fido cannot be loaded, formats are
+guessed from file extensions and the inventory says so.
+
+To read an inventory in TypeScript, as the app does to show one to the researcher, use
+[`@data-director/inventory`](https://github.com/data-director-agent/data-director/blob/main/inventory/browser/README.md) in `browser/`, which is not part of this Python
+package.
+
+The package does not depend on the contract or any other part of Data Director
+([System and boundaries](https://github.com/data-director-agent/data-director/blob/main/docs/specs/system.md)). Its tests check
+what it writes against the contract's `FileInventory` schema.
+
+| Module | What it does |
+|---|---|
+| `scan.py` | Walks the folder, digests each file and assembles the inventory. |
+| `identify.py` | The one module that calls fido, with the extension fallback. |
+| `tables.py` | Infers a table's fields with Frictionless. |
+| `cli.py` | The `dd-inventory` command. |

data_director_inventory-0.1.0a3/pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["hatchling>=1.32.4"]
+build-backend = "hatchling.build"
+
+[project]
+name = "data-director-inventory"
+version = "0.1.0-alpha.3"
+description = "Describe a dataset's files where they live: size, digest, PRONOM format and table fields, without uploading them"
+readme = "README.md"
+requires-python = ">=3.14"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Joe Heffer", email = "j.heffer@sheffield.ac.uk" }]
+keywords = ["research data", "inventory", "FAIR", "PRONOM", "checksum", "data management", "Frictionless"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Science/Research",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.14",
+  "Topic :: Scientific/Engineering :: Information Analysis",
+  "Topic :: System :: Archiving",
+  "Typing :: Typed",
+]
+
+# No workspace member: the command runs without the contract (docs/specs/system.md). What it
+# writes is a core `FileInventory`, which the tests check against the contract's schema.
+dependencies = [
+  # fido 1.6.1's library defaults name a signature file its wheel does not ship; identify.py
+  # reads the shipped versions.xml instead. Check that still holds before allowing a newer fido.
+  "opf-fido==1.6.1",
+  "frictionless>=5.18,<6",
+]
+
+[project.scripts]
+dd-inventory = "dd_inventory.cli:main"
+# So that `uvx data-director-inventory path/to/dataset` works: uvx runs the script named like the
+# package.
+data-director-inventory = "dd_inventory.cli:main"
+
+[project.urls]
+Homepage = "https://data-director-agent.github.io/data-director/"
+Documentation = "https://data-director-agent.github.io/data-director/user-guide/making-an-inventory/"
+Repository = "https://github.com/data-director-agent/data-director/tree/main/inventory"
+Changelog = "https://github.com/data-director-agent/data-director/blob/main/inventory/CHANGELOG.md"
+Issues = "https://github.com/data-director-agent/data-director/issues"
+
+# What this package's tests import from the workspace (docs/specs/system.md, scripts/check_boundaries.py).
+[dependency-groups]
+dev = ["data-director-sdk"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dd_inventory"]
+
+# inventory/browser is the TypeScript package for the browser (its README); it is not part of the
+# Python distribution.
+[tool.hatch.build.targets.sdist]
+exclude = ["browser"]

data_director_inventory-0.1.0a3/src/dd_inventory/__init__.py

@@ -0,0 +1,11 @@
+"""Describe a dataset's files where they live, without sending them anywhere
+(docs/specs/projects.md).
+
+`dd-inventory ROOT` walks a folder and writes a `FileInventory`: for each file its path relative
+to ROOT, its size, its SHA-256 digest, its format identified against PRONOM, and for a table its
+fields. The inventory holds facts about the files and never their content.
+"""
+
+from dd_inventory.scan import TOOL, make_inventory, tool_version
+
+__all__ = ["TOOL", "make_inventory", "tool_version"]

data_director_inventory-0.1.0a3/src/dd_inventory/cli.py

@@ -0,0 +1,46 @@
+"""`dd-inventory ROOT [-o FILE]`: write the inventory of a dataset's files as JSON.
+
+Run it where the data live. It reads every file under ROOT and sends nothing anywhere; the
+researcher adds the JSON it writes to their project.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from dd_inventory.scan import make_inventory, tool_version
+
+EXIT_OK, EXIT_USAGE = 0, 2
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="dd-inventory",
+        description=(
+            "Describe the files under ROOT (paths, sizes, SHA-256 digests, PRONOM formats and "
+            "table fields) as a Data Director FileInventory. No file content is written."
+        ),
+    )
+    parser.add_argument("root", type=Path, metavar="ROOT", help="the dataset's root folder")
+    parser.add_argument(
+        "-o", "--output", type=Path, help="write the inventory to this file, not standard output"
+    )
+    parser.add_argument("--version", action="version", version=f"%(prog)s {tool_version()}")
+    args = parser.parse_args(argv)
+    if not args.root.is_dir():
+        parser.print_usage(sys.stderr)
+        sys.stderr.write(f"dd-inventory: {args.root} is not a folder\n")
+        return EXIT_USAGE
+    text = json.dumps(make_inventory(args.root), indent=2, ensure_ascii=False) + "\n"
+    if args.output:
+        args.output.write_text(text, encoding="utf-8")
+    else:
+        sys.stdout.write(text)
+    return EXIT_OK
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data_director_inventory-0.1.0a3/src/dd_inventory/identify.py

@@ -0,0 +1,127 @@
+"""Identify a file's format against PRONOM with fido. The one module that knows fido.
+
+fido reads the start and end of a file and matches them against PRONOM's byte signatures, then
+its container signatures for ZIP and OLE2 files, and falls back to the file's extension. Each
+step can match several formats, so every PRONOM identifier (PUID) it returns is kept, with the
+step that found them.
+
+Where fido cannot be loaded, `ExtensionIdentifier` guesses a media type from the extension and
+gives no PUID (docs/specs/projects.md, dependencies).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import mimetypes
+import warnings
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from importlib.metadata import version
+from pathlib import Path
+from typing import Any, Protocol
+from xml.etree import ElementTree
+
+# fido's step names are the core's `IdentificationMethod` values; an empty match is `none`.
+SIGNATURE, CONTAINER, EXTENSION, NONE = "signature", "container", "extension", "none"
+
+
+@dataclass(frozen=True)
+class Identification:
+    method: str
+    pronom_ids: list[str] = field(default_factory=list)
+    media_type: str | None = None
+
+
+class Identifier(Protocol):
+    # What identified the formats, recorded in the inventory.
+    description: str
+
+    def identify(self, path: Path) -> Identification: ...
+
+
+class FidoUnavailableError(Exception):
+    """fido could not be loaded; the reason is recorded in the inventory's limitations."""
+
+
+@contextlib.contextmanager
+def _quiet_fido() -> Iterator[None]:
+    """fido tests an XML element's truth value, which Python 3.12 deprecates, and writes notices
+    (an empty file, an unreadable one) to stderr itself. Neither is the caller's to see."""
+    with warnings.catch_warnings(), contextlib.redirect_stderr(io.StringIO()):
+        warnings.simplefilter("ignore", DeprecationWarning)
+        yield
+
+
+def guess_media_type(path: Path) -> str | None:
+    return mimetypes.guess_type(path.name, strict=False)[0]
+
+
+class ExtensionIdentifier:
+    description = "file extensions only"
+
+    def identify(self, path: Path) -> Identification:
+        media_type = guess_media_type(path)
+        return Identification(EXTENSION if media_type else NONE, [], media_type)
+
+
+class FidoIdentifier:
+    """fido's own steps, in the order its `identify_file` takes them, on a file this opens and
+    closes: `identify_file` leaves every file it reads open."""
+
+    def __init__(self) -> None:
+        try:
+            from fido import CONFIG_DIR
+            from fido.fido import Fido
+            from fido.package import OlePackage, ZipPackage
+            from fido.versions import get_local_versions
+
+            # The library's defaults name a signature file fido 1.6.1 does not ship; its CLI
+            # reads the files from versions.xml, and so does this.
+            versions = get_local_versions()
+            with _quiet_fido():
+                self._fido = Fido(
+                    quiet=True,
+                    format_files=[versions.pronom_signature, versions.fido_extension_signature],
+                )
+                self._containers = ElementTree.parse(
+                    Path(CONFIG_DIR) / versions.pronom_container_signature
+                )
+            pronom = versions.pronom_version
+        except Exception as exc:  # fido raises bare OSError, ParseError, ...
+            raise FidoUnavailableError(f"fido could not be loaded: {exc}") from exc
+        self._packages = {"zip": ("ZIP", ZipPackage), "ole": ("OLE2", OlePackage)}
+        self.description = f"fido {version('opf-fido')} with PRONOM signatures v{pronom}"
+
+    def identify(self, path: Path) -> Identification:
+        fido = self._fido
+        with _quiet_fido():
+            size = path.stat().st_size
+            with path.open("rb") as fh:
+                bof, eof, _read = fido.get_buffers(fh, size, seekable=True)
+            # An empty file matches RTF's signature, so fido goes by its extension instead.
+            matches = fido.match_formats(bof, eof) if size else []
+            package = self._packages.get(fido.container_type(matches))
+            if package is not None:
+                kind, cls = package
+                inside = fido.match_container(kind, cls, str(path), self._containers)
+                if inside:
+                    return self._found(path, inside, CONTAINER)
+            if matches:
+                return self._found(path, matches, SIGNATURE)
+            return self._found(path, fido.match_extensions(str(path)), EXTENSION)
+
+    @staticmethod
+    def _found(path: Path, matches: list[tuple[Any, Any]], method: str) -> Identification:
+        puids = [p.text for f, _sig in matches if (p := f.find("puid")) is not None and p.text]
+        mimes = [m.text for f, _sig in matches if (m := f.find("mime")) is not None and m.text]
+        media_type = mimes[0] if mimes else guess_media_type(path)
+        return Identification(method if puids else NONE, puids, media_type)
+
+
+def default_identifier() -> tuple[Identifier, str | None]:
+    """fido, or the extension fallback and why fido was not used."""
+    try:
+        return FidoIdentifier(), None
+    except FidoUnavailableError as exc:
+        return ExtensionIdentifier(), str(exc)

data_director_inventory-0.1.0a3/src/dd_inventory/scan.py

@@ -0,0 +1,119 @@
+"""Walk a dataset's root folder and make its `FileInventory` (docs/specs/projects.md).
+
+The inventory is a plain JSON document of the core contract class `FileInventory`; this package
+does not need the contract to make one. Paths are relative to the root and sorted, so two runs
+over the same files differ only in `inventory_made_at`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from datetime import UTC, datetime
+from importlib.metadata import version
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from dd_inventory.identify import Identifier, default_identifier
+from dd_inventory.tables import TableError, is_table, table_fields
+
+TOOL = "dd-inventory"
+SCHEMA_CLASS = "FileInventory"
+
+
+def tool_version() -> str:
+    return version("data-director-inventory")
+
+
+def sha256(path: Path) -> str:
+    with path.open("rb") as fh:
+        return hashlib.file_digest(fh, "sha256").hexdigest()
+
+
+def _walk(root: Path) -> tuple[list[Path], list[str]]:
+    """Every regular file under `root`, and what was left out. Hidden files and links are not
+    followed: a link may point outside the dataset, and a hidden file is rarely part of it."""
+    found: list[Path] = []
+    hidden = links = 0
+    for folder, dirs, names in os.walk(root, followlinks=False):
+        here = Path(folder)
+        for d in list(dirs):
+            if d.startswith(".") or (here / d).is_symlink():
+                hidden += d.startswith(".")
+                links += not d.startswith(".")
+                dirs.remove(d)
+        for name in names:
+            path = here / name
+            if name.startswith("."):
+                hidden += 1
+            elif path.is_symlink():
+                links += 1
+            elif path.is_file():
+                found.append(path)
+    skipped = []
+    if hidden:
+        skipped.append(
+            f"{hidden} hidden file(s) or folder(s), whose names begin with '.', were skipped."
+        )
+    if links:
+        skipped.append(f"{links} symbolic link(s) were not followed.")
+    return sorted(found, key=lambda p: p.relative_to(root).as_posix()), skipped
+
+
+def make_inventory(
+    root: Path, *, identifier: Identifier | None = None, now: datetime | None = None
+) -> dict[str, Any]:
+    """The `FileInventory` of every file under `root`, as a JSON-ready dict."""
+    root = root.resolve()
+    if not root.is_dir():
+        raise NotADirectoryError(root)
+    limitations: list[str] = []
+    if identifier is None:
+        identifier, unavailable = default_identifier()
+        if unavailable:
+            limitations.append(f"{unavailable}; formats were guessed from file extensions only.")
+    paths, skipped = _walk(root)
+    limitations += skipped
+    files = []
+    for path in paths:
+        try:
+            files.append(_describe(root, path, identifier, limitations))
+        except OSError as exc:
+            limitations.append(
+                f"{path.relative_to(root).as_posix()} could not be read: {exc.strerror}."
+            )
+    limitations.append(
+        "Table fields are inferred from a sample of rows; a later row may not fit the type given."
+    )
+    made_at = (now or datetime.now(UTC)).isoformat(timespec="seconds")
+    return {
+        "schema_class": SCHEMA_CLASS,
+        "inventory_files": files,
+        "inventory_tool": TOOL,
+        "inventory_tool_version": tool_version(),
+        "inventory_identified_with": identifier.description,
+        "inventory_made_at": made_at,
+        "inventory_limitations": limitations,
+    }
+
+
+def _describe(
+    root: Path, path: Path, identifier: Identifier, limitations: list[str]
+) -> dict[str, Any]:
+    relative = PurePosixPath(path.relative_to(root).as_posix())
+    found = identifier.identify(path)
+    entry: dict[str, Any] = {
+        "file_path": str(relative),
+        "file_size": path.stat().st_size,
+        "file_sha256": sha256(path),
+    }
+    if found.media_type:
+        entry["file_media_type"] = found.media_type
+    entry["file_pronom_ids"] = found.pronom_ids
+    entry["file_identified_by"] = found.method
+    if is_table(path):
+        try:
+            entry["file_fields"] = table_fields(path)
+        except TableError as exc:
+            limitations.append(f"The fields of {relative} could not be inferred: {exc}")
+    return entry

data_director_inventory-0.1.0a3/src/dd_inventory/tables.py

@@ -0,0 +1,48 @@
+"""Infer a table's fields as a Frictionless Table Schema.
+
+Only delimited text is described: CSV and TSV. Frictionless infers each field's type and format
+from a sample of rows, so a type is a guess that later rows may contradict.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+# Rows Frictionless reads to infer types; its own default is 100.
+SAMPLE_ROWS = 1000
+TABLE_SUFFIXES = {".csv": ",", ".tsv": "\t", ".tab": "\t"}
+
+
+class TableError(Exception):
+    """The file looked like a table but its fields could not be inferred."""
+
+
+def is_table(path: Path) -> bool:
+    return path.suffix.lower() in TABLE_SUFFIXES
+
+
+def table_fields(path: Path) -> list[dict[str, Any]]:
+    """The table's fields as `TableField` objects: name, field_type and, if any, field_format."""
+    from frictionless import Detector, Dialect, Resource, formats
+
+    control = formats.CsvControl(delimiter=TABLE_SUFFIXES[path.suffix.lower()])
+    try:
+        resource = Resource(
+            path=path.name,
+            basepath=str(path.parent),
+            format="csv",
+            dialect=Dialect(controls=[control]),
+            detector=Detector(sample_size=SAMPLE_ROWS),
+        )
+        resource.infer()
+        schema = resource.schema
+    except Exception as exc:  # Frictionless raises its own errors and the reader's
+        raise TableError(f"{path.name}: {exc}") from exc
+    fields = []
+    for f in schema.fields:
+        entry: dict[str, Any] = {"name": f.name, "field_type": f.type}
+        if f.format and f.format != "default":
+            entry["field_format"] = f.format
+        fields.append(entry)
+    return fields

data_director_inventory-0.1.0a3/tests/data/soil-survey/README.md

@@ -0,0 +1,3 @@
+# Soil survey (test fixture)
+
+Soil chemistry readings from three sites, made up for the inventory tests.

data_director_inventory-0.1.0a3/tests/data/soil-survey/data/readings.csv

@@ -0,0 +1,4 @@
+site,sampled_on,ph,nitrate_mg_kg
+A,2026-03-01,6.4,12.5
+B,2026-03-02,7.1,9.0
+C,2026-03-03,5.8,15.25

data_director_inventory-0.1.0a3/tests/data/soil-survey/data/sites.tsv

@@ -0,0 +1,4 @@
+site	latitude	longitude	name
+A	53.38	-1.47	Crookes
+B	53.40	-1.50	Walkley
+C	53.36	-1.49	Ecclesall

data_director_inventory-0.1.0a3/tests/data/soil-survey/docs/protocol.pdf

@@ -0,0 +1,21 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 72 72] >>
+endobj
+xref
+0 4
+0000000000 65535 f 
+0000000009 00000 n 
+0000000058 00000 n 
+0000000115 00000 n 
+trailer
+<< /Size 4 /Root 1 0 R >>
+startxref
+184
+%%EOF

data_director_inventory-0.1.0a3/tests/test_inventory.py

@@ -0,0 +1,200 @@
+"""The inventory command describes a dataset's files without their content
+(docs/specs/projects.md)."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import zipfile
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from dd_inventory import cli, identify, make_inventory, scan
+from dd_inventory.identify import ExtensionIdentifier, FidoUnavailableError
+from dd_inventory.tables import TableError
+from dd_sdk.contract.classes import ClassSchema
+from dd_sdk.contract.models import FileInventory
+
+Inventory = dict[str, Any]
+FIXTURE = Path(__file__).parent / "data" / "soil-survey"
+
+
+@pytest.fixture(scope="module")
+def inventory() -> Inventory:
+    return make_inventory(FIXTURE)
+
+
+def _by_path(inventory: Inventory) -> dict[str, dict[str, Any]]:
+    return {f["file_path"]: f for f in inventory["inventory_files"]}
+
+
+@pytest.fixture
+def dataset(tmp_path: Path) -> Path:
+    root = tmp_path / "soil-survey"
+    shutil.copytree(FIXTURE, root)
+    return root
+
+
+def test_the_inventory_is_a_core_file_inventory(inventory: Inventory) -> None:
+    assert ClassSchema.of(FileInventory).errors(inventory) == []
+    FileInventory.model_validate(inventory)
+    assert inventory["inventory_tool"] == "dd-inventory"
+    assert inventory["inventory_identified_with"].startswith("fido 1.6.1 with PRONOM signatures v")
+
+
+def test_every_file_is_listed_by_relative_path_in_order_with_its_size_and_digest(
+    inventory: Inventory,
+) -> None:
+    files = inventory["inventory_files"]
+    paths = [f["file_path"] for f in files]
+    assert paths == sorted(paths)
+    assert set(paths) == {
+        "README.md",
+        "data/readings.csv",
+        "data/sites.tsv",
+        "docs/protocol.pdf",
+        "docs/site-map.png",
+    }
+    for f in files:
+        data = (FIXTURE / f["file_path"]).read_bytes()
+        assert f["file_size"] == len(data)
+        assert f["file_sha256"] == hashlib.sha256(data).hexdigest()
+
+
+def test_no_file_content_and_no_absolute_path_is_written(inventory: Inventory) -> None:
+    text = json.dumps(inventory)
+    assert str(FIXTURE.resolve()) not in text
+    assert "Crookes" not in text  # a value from sites.tsv
+    assert "nitrate" in text  # a field name is a fact about the table, and is kept
+
+
+def test_formats_with_a_byte_signature_are_identified_from_content(inventory: Inventory) -> None:
+    files = _by_path(inventory)
+    assert files["docs/protocol.pdf"]["file_pronom_ids"] == ["fmt/18"]  # PDF 1.4
+    assert files["docs/protocol.pdf"]["file_identified_by"] == "signature"
+    assert files["docs/protocol.pdf"]["file_media_type"] == "application/pdf"
+    assert files["docs/site-map.png"]["file_identified_by"] == "signature"
+    assert files["data/readings.csv"]["file_pronom_ids"] == ["x-fmt/18"]
+    assert files["data/readings.csv"]["file_identified_by"] == "extension"
+
+
+def test_an_ambiguous_match_keeps_every_candidate(dataset: Path) -> None:
+    (dataset / "old-notes.doc").write_bytes(b"not a Word document")
+    found = _by_path(make_inventory(dataset))["old-notes.doc"]
+    assert found["file_identified_by"] == "extension"
+    assert len(found["file_pronom_ids"]) > 1
+
+
+def test_a_file_nothing_recognises_has_no_identifier(dataset: Path) -> None:
+    (dataset / "readings.xyzzy").write_bytes(b"\x00\x01\x02")
+    found = _by_path(make_inventory(dataset))["readings.xyzzy"]
+    assert found["file_pronom_ids"] == []
+    assert found["file_identified_by"] == "none"
+    assert "file_media_type" not in found
+
+
+def test_a_table_has_its_fields_inferred(inventory: Inventory) -> None:
+    files = _by_path(inventory)
+    assert files["data/readings.csv"]["file_fields"] == [
+        {"name": "site", "field_type": "string"},
+        {"name": "sampled_on", "field_type": "date"},
+        {"name": "ph", "field_type": "number"},
+        {"name": "nitrate_mg_kg", "field_type": "number"},
+    ]
+    assert [f["name"] for f in files["data/sites.tsv"]["file_fields"]] == [
+        "site",
+        "latitude",
+        "longitude",
+        "name",
+    ]
+    assert "file_fields" not in files["docs/protocol.pdf"]
+
+
+def test_a_table_whose_fields_cannot_be_inferred_is_listed_and_said_so(
+    dataset: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    def unreadable(path: Path) -> list[dict[str, Any]]:
+        raise TableError(f"{path.name}: not a table")
+
+    monkeypatch.setattr(scan, "table_fields", unreadable)
+    (dataset / "data" / "broken.csv").write_bytes(b"\xff\xfe\x00\x00" * 4)
+    made = make_inventory(dataset)
+    assert "data/broken.csv" in _by_path(made)
+    assert "file_fields" not in _by_path(made)["data/broken.csv"]
+    assert any("data/broken.csv" in note for note in made["inventory_limitations"])
+
+
+def test_hidden_files_and_links_are_skipped_and_counted(dataset: Path) -> None:
+    (dataset / ".DS_Store").write_bytes(b"\x00")
+    (dataset / ".git").mkdir()
+    (dataset / ".git" / "config").write_text("[core]\n")
+    (dataset / "elsewhere.csv").symlink_to(FIXTURE / "data" / "readings.csv")
+    made = make_inventory(dataset)
+    paths = set(_by_path(made))
+    assert not {p for p in paths if p.startswith(".") or "/." in p}
+    assert "elsewhere.csv" not in paths
+    notes = " ".join(made["inventory_limitations"])
+    assert "2 hidden" in notes
+    assert "1 symbolic link" in notes
+
+
+def test_without_fido_formats_are_guessed_from_extensions(
+    dataset: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    def unavailable() -> None:
+        raise FidoUnavailableError("fido could not be loaded: no signatures")
+
+    monkeypatch.setattr(identify, "FidoIdentifier", unavailable)
+    made = make_inventory(dataset, now=datetime(2026, 10, 30, tzinfo=UTC))
+    assert ClassSchema.of(FileInventory).errors(made) == []
+    assert made["inventory_identified_with"] == ExtensionIdentifier.description
+    assert made["inventory_made_at"] == "2026-10-30T00:00:00+00:00"
+    pdf = _by_path(made)["docs/protocol.pdf"]
+    assert pdf == {**pdf, "file_pronom_ids": [], "file_identified_by": "extension"}
+    assert pdf["file_media_type"] == "application/pdf"
+    assert "fido could not be loaded" in made["inventory_limitations"][0]
+
+
+def test_the_command_writes_the_inventory_to_a_file(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    out = tmp_path / "inventory.json"
+    assert cli.main([str(FIXTURE), "-o", str(out)]) == cli.EXIT_OK
+    assert json.loads(out.read_text("utf-8"))["schema_class"] == "FileInventory"
+    assert capsys.readouterr().out == ""
+
+
+def test_the_command_writes_to_standard_output_by_default(
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    assert cli.main([str(FIXTURE)]) == cli.EXIT_OK
+    assert len(json.loads(capsys.readouterr().out)["inventory_files"]) == 5
+
+
+def test_the_command_refuses_a_root_that_is_not_a_folder(
+    tmp_path: Path, capsys: pytest.CaptureFixture[str]
+) -> None:
+    assert cli.main([str(tmp_path / "missing")]) == cli.EXIT_USAGE
+    assert "is not a folder" in capsys.readouterr().err
+    with pytest.raises(NotADirectoryError):
+        make_inventory(FIXTURE / "README.md")
+
+
+WORD_CONTENT_TYPES = (
+    '<?xml version="1.0"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/'
+    'content-types"><Override PartName="/word/document.xml" ContentType="application/'
+    'vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/></Types>'
+)
+
+
+def test_a_zip_based_format_is_identified_by_its_container_signature(dataset: Path) -> None:
+    with zipfile.ZipFile(dataset / "docs" / "consent-form.docx", "w") as docx:
+        docx.writestr("[Content_Types].xml", WORD_CONTENT_TYPES)
+        docx.writestr("word/document.xml", '<?xml version="1.0"?><document/>')
+    found = _by_path(make_inventory(dataset))["docs/consent-form.docx"]
+    assert found["file_identified_by"] == "container"
+    assert found["file_pronom_ids"] == ["fmt/412"]  # Microsoft Word for Windows 2007 onwards