vcti-path-format 1.2.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vcti_path_format-2.0.0/PKG-INFO +331 -0
- vcti_path_format-2.0.0/README.md +305 -0
- vcti_path_format-2.0.0/pyproject.toml +71 -0
- vcti_path_format-2.0.0/src/vcti/pathformat/core/__init__.py +54 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/descriptor.py +16 -0
- vcti_path_format-2.0.0/src/vcti/pathformat/core/errors.py +60 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/evaluator/__init__.py +8 -9
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/evaluator/base.py +46 -0
- vcti_path_format-2.0.0/src/vcti/pathformat/core/evaluator/heuristic.py +522 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/feature_validator/__init__.py +4 -2
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/feature_validator/base.py +39 -3
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/feature_validator/extension.py +15 -1
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/feature_validator/magic_bytes.py +27 -5
- vcti_path_format-2.0.0/src/vcti/pathformat/core/feature_validator/path_kind.py +84 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/identifier.py +72 -22
- vcti_path_format-2.0.0/src/vcti_path_format.egg-info/PKG-INFO +331 -0
- vcti_path_format-2.0.0/src/vcti_path_format.egg-info/SOURCES.txt +36 -0
- vcti_path_format-2.0.0/src/vcti_path_format.egg-info/requires.txt +12 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_descriptor.py +23 -3
- vcti_path_format-2.0.0/tests/test_docs.py +201 -0
- vcti_path_format-2.0.0/tests/test_errors.py +71 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_evaluator_base.py +4 -4
- vcti_path_format-2.0.0/tests/test_examples.py +268 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_extension_validator.py +16 -2
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_feature_validator_base.py +56 -16
- vcti_path_format-2.0.0/tests/test_heuristic_evaluator.py +805 -0
- vcti_path_format-2.0.0/tests/test_identifier.py +304 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_integration.py +4 -4
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_magic_bytes_validator.py +43 -2
- vcti_path_format-2.0.0/tests/test_path_kind_validator.py +64 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_registry.py +3 -3
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/tests/test_version.py +3 -3
- vcti_path_format-1.2.0/PKG-INFO +0 -274
- vcti_path_format-1.2.0/README.md +0 -257
- vcti_path_format-1.2.0/pyproject.toml +0 -41
- vcti_path_format-1.2.0/src/vcti/pathformat/__init__.py +0 -29
- vcti_path_format-1.2.0/src/vcti/pathformat/evaluator/heuristic.py +0 -367
- vcti_path_format-1.2.0/src/vcti/pathformat/feature_validator/registry.py +0 -53
- vcti_path_format-1.2.0/src/vcti_path_format.egg-info/PKG-INFO +0 -274
- vcti_path_format-1.2.0/src/vcti_path_format.egg-info/SOURCES.txt +0 -33
- vcti_path_format-1.2.0/src/vcti_path_format.egg-info/requires.txt +0 -9
- vcti_path_format-1.2.0/src/vcti_path_format.egg-info/zip-safe +0 -1
- vcti_path_format-1.2.0/tests/test_heuristic_evaluator.py +0 -291
- vcti_path_format-1.2.0/tests/test_identifier.py +0 -90
- vcti_path_format-1.2.0/tests/test_validator_registry.py +0 -45
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/LICENSE +0 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/setup.cfg +0 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/py.typed +0 -0
- {vcti_path_format-1.2.0/src/vcti/pathformat → vcti_path_format-2.0.0/src/vcti/pathformat/core}/registry.py +0 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/src/vcti_path_format.egg-info/dependency_links.txt +0 -0
- {vcti_path_format-1.2.0 → vcti_path_format-2.0.0}/src/vcti_path_format.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vcti-path-format
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: File format identification framework with heuristic evaluators and feature validators for Python
|
|
5
|
+
Author: Visual Collaboration Technologies Inc.
|
|
6
|
+
License-Expression: LicenseRef-Proprietary
|
|
7
|
+
Project-URL: Repository, https://github.com/vcollab/vcti-python-path-format
|
|
8
|
+
Project-URL: Changelog, https://github.com/vcollab/vcti-python-path-format/blob/main/CHANGELOG.md
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
13
|
+
Requires-Python: <3.15,>=3.12
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: vcti-plugin-catalog>=2.0.0
|
|
17
|
+
Requires-Dist: vcti-lookup>=2.0.0
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Requires-Dist: pytest; extra == "test"
|
|
20
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
21
|
+
Provides-Extra: lint
|
|
22
|
+
Requires-Dist: ruff; extra == "lint"
|
|
23
|
+
Provides-Extra: typecheck
|
|
24
|
+
Requires-Dist: mypy; extra == "typecheck"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# Path Format
|
|
28
|
+
|
|
29
|
+
File format identification framework with heuristic evaluators and feature validators for Python.
|
|
30
|
+
|
|
31
|
+
## Overview
|
|
32
|
+
|
|
33
|
+
`vcti-path-format` answers "what format is this file or folder, and how sure
|
|
34
|
+
are we?" Each format is a `FormatDescriptor` held in a `FormatRegistry`; the
|
|
35
|
+
descriptor's evaluator runs feature validators — magic bytes, extension, or
|
|
36
|
+
checks of your own — against a path. Validators are either *gates*, which
|
|
37
|
+
rule a format out when they fail, or *evidence*, which only lowers
|
|
38
|
+
confidence, and they are grouped into tiers of increasing cost so callers
|
|
39
|
+
can stop at cheap checks. `FormatIdentifier` evaluates a path against every
|
|
40
|
+
registered format and ranks the results on a five-level `MatchConfidence`
|
|
41
|
+
scale.
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install "vcti-path-format>=2.0.0"
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### In `pyproject.toml` dependencies
|
|
50
|
+
|
|
51
|
+
```toml
|
|
52
|
+
dependencies = [
|
|
53
|
+
"vcti-path-format>=2.0.0",
|
|
54
|
+
]
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## Quick Start
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from pathlib import Path
|
|
63
|
+
from vcti.pathformat.core import (
|
|
64
|
+
FormatDescriptor,
|
|
65
|
+
FormatIdentifier,
|
|
66
|
+
FormatRegistry,
|
|
67
|
+
HeuristicEvaluator,
|
|
68
|
+
MatchConfidence,
|
|
69
|
+
PathKind,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
# Define a format descriptor with validators
|
|
73
|
+
hdf5_descriptor = FormatDescriptor(
|
|
74
|
+
id="hdf5-file",
|
|
75
|
+
name="HDF5 File",
|
|
76
|
+
evaluator=(
|
|
77
|
+
HeuristicEvaluator()
|
|
78
|
+
.check_path_kind(PathKind.FILE) # PRECONDITION: rules out directories
|
|
79
|
+
.check_magic_bytes(b"\x89HDF\r\n\x1a\n") # GATE
|
|
80
|
+
.check_extension([".h5", ".hdf5", ".he5"]) # EVIDENCE
|
|
81
|
+
),
|
|
82
|
+
attributes={"path_type": "file", "structure": "hdf5"},
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# Register in a format registry
|
|
86
|
+
registry = FormatRegistry()
|
|
87
|
+
registry.register(hdf5_descriptor)
|
|
88
|
+
|
|
89
|
+
# Identify a file
|
|
90
|
+
identifier = FormatIdentifier(registry)
|
|
91
|
+
results = identifier.identify_file_format(Path("data.h5"))
|
|
92
|
+
|
|
93
|
+
for result in results:
|
|
94
|
+
print(f"{result.descriptor.name}: {result.confidence.name}")
|
|
95
|
+
|
|
96
|
+
# Get best match above a confidence threshold
|
|
97
|
+
best = identifier.get_best_match(
|
|
98
|
+
Path("data.h5"),
|
|
99
|
+
min_confidence=MatchConfidence.LIKELY,
|
|
100
|
+
)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## Core Concepts
|
|
106
|
+
|
|
107
|
+
### FormatDescriptor
|
|
108
|
+
|
|
109
|
+
Extends `Descriptor[Evaluator]` from vcti-plugin-catalog. Wraps an evaluator
|
|
110
|
+
with format metadata and attributes.
|
|
111
|
+
|
|
112
|
+
### FormatRegistry
|
|
113
|
+
|
|
114
|
+
Extends `Registry[FormatDescriptor]`. Central catalog of known formats with
|
|
115
|
+
attribute-based filtering via `registry.lookup`.
|
|
116
|
+
|
|
117
|
+
### FormatIdentifier
|
|
118
|
+
|
|
119
|
+
Evaluates a path against all (or filtered) registered formats and returns
|
|
120
|
+
results sorted by confidence, then by descriptor `priority`.
|
|
121
|
+
|
|
122
|
+
### HeuristicEvaluator
|
|
123
|
+
|
|
124
|
+
Builder-pattern evaluator that runs feature validators and turns their
|
|
125
|
+
results into a confidence:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from vcti.pathformat.core import HeuristicEvaluator, PathKind
|
|
129
|
+
|
|
130
|
+
evaluator = (
|
|
131
|
+
HeuristicEvaluator()
|
|
132
|
+
.check_path_kind(PathKind.FILE) # PRECONDITION
|
|
133
|
+
.check_magic_bytes(b"\x89PNG\r\n\x1a\n") # GATE
|
|
134
|
+
.check_extension([".png"]) # EVIDENCE
|
|
135
|
+
)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Each validator plays a **role**:
|
|
139
|
+
|
|
140
|
+
| Role | Failure | Success |
|
|
141
|
+
|------|---------|---------|
|
|
142
|
+
| `PRECONDITION` | Rules the format out | No evidence — the format merely *applies* (e.g. "is a file") |
|
|
143
|
+
| `GATE` | Rules the format out | Strong evidence of a match (e.g. the signature) |
|
|
144
|
+
| `EVIDENCE` | Lowers confidence | Raises confidence |
|
|
145
|
+
|
|
146
|
+
A validator that cannot obtain evidence — the file cannot be read — returns
|
|
147
|
+
an *inconclusive* result instead of a failure: an inconclusive
|
|
148
|
+
PRECONDITION or GATE gives `CANT_EVALUATE`, not `CERTAINLY_NOT`.
|
|
149
|
+
|
|
150
|
+
**Confidence rules.** A failed PRECONDITION or GATE gives `CERTAINLY_NOT`,
|
|
151
|
+
an inconclusive one `CANT_EVALUATE`, and either stops evaluation. Otherwise
|
|
152
|
+
the evaluator's confidence policy decides. The default policy:
|
|
153
|
+
|
|
154
|
+
- All passed + GATE present -> `DEFINITE`
|
|
155
|
+
- GATE passed, some EVIDENCE failed -> `LIKELY`
|
|
156
|
+
- All passed + no GATE -> `LIKELY`
|
|
157
|
+
- Some EVIDENCE failed, no GATE -> `UNLIKELY`
|
|
158
|
+
- No GATE and no conclusive EVIDENCE -> `CANT_EVALUATE`
|
|
159
|
+
|
|
160
|
+
### Feature Validators
|
|
161
|
+
|
|
162
|
+
| Validator | Default role | Tier | Checks |
|
|
163
|
+
|-----------|--------------|------|--------|
|
|
164
|
+
| `PathKindValidator` | PRECONDITION | IDENTIFICATION | Path is a file, or a directory (`check_path_kind`) |
|
|
165
|
+
| `MagicBytesValidator` | GATE | IDENTIFICATION | File signature bytes (`check_magic_bytes`) |
|
|
166
|
+
| `ExtensionValidator` | EVIDENCE | IDENTIFICATION | File extension (`check_extension`) |
|
|
167
|
+
|
|
168
|
+
Every descriptor is evaluated against every path, so a file format also
|
|
169
|
+
meets directories and a directory format meets files. `check_path_kind()`
|
|
170
|
+
rules those out before any other check reads the path; it looks at the
|
|
171
|
+
filesystem, not at descriptor attributes.
|
|
172
|
+
|
|
173
|
+
Custom validators implement the `FeatureValidator` protocol — no base class
|
|
174
|
+
needed. See [docs/extending.md](docs/extending.md) for a worked example and
|
|
175
|
+
for when to return a failed versus an inconclusive result.
|
|
176
|
+
|
|
177
|
+
---
|
|
178
|
+
|
|
179
|
+
## Deciding What Counts
|
|
180
|
+
|
|
181
|
+
Which evidence matters is a property of the format, so the descriptor's
|
|
182
|
+
author decides it:
|
|
183
|
+
|
|
184
|
+
- **`role=`** on any builder method overrides a validator's default role —
|
|
185
|
+
e.g. make an extension a GATE for a format it defines.
|
|
186
|
+
- **`policy=`** on `HeuristicEvaluator` replaces the default confidence
|
|
187
|
+
policy with a function of the results.
|
|
188
|
+
- **`priority=`** on `FormatDescriptor` ranks formats that reach the same
|
|
189
|
+
confidence — e.g. CGNS above the plain HDF5 it is built on.
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
from vcti.pathformat.core import FormatDescriptor, HeuristicEvaluator, ValidatorRole
|
|
193
|
+
|
|
194
|
+
erf_descriptor = FormatDescriptor(
|
|
195
|
+
id="erfh5",
|
|
196
|
+
name="ERF HDF5",
|
|
197
|
+
evaluator=(
|
|
198
|
+
HeuristicEvaluator()
|
|
199
|
+
.check_magic_bytes(b"\x89HDF\r\n\x1a\n")
|
|
200
|
+
.check_extension([".erfh5"], role=ValidatorRole.GATE)
|
|
201
|
+
),
|
|
202
|
+
priority=10, # wins over a generic HDF5 descriptor when both match
|
|
203
|
+
)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Why the package leaves these choices to you is in
|
|
207
|
+
[docs/design.md](docs/design.md); writing a policy is in
|
|
208
|
+
[docs/extending.md](docs/extending.md).
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## Validation Tiers
|
|
213
|
+
|
|
214
|
+
Control evaluation depth with `max_tier`:
|
|
215
|
+
|
|
216
|
+
| Tier | Cost | Examples |
|
|
217
|
+
|------|------|---------|
|
|
218
|
+
| `IDENTIFICATION` | Cheap | Path kind, magic bytes, file extension |
|
|
219
|
+
| `STRUCTURE` | Medium | Schema validation, header parsing |
|
|
220
|
+
| `SEMANTIC` | Expensive | Content analysis, business logic |
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
from vcti.pathformat.core import ValidationTier
|
|
224
|
+
|
|
225
|
+
# Only run cheap checks
|
|
226
|
+
results = identifier.identify_file_format(path, max_tier=ValidationTier.IDENTIFICATION)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## Evaluator Caching
|
|
232
|
+
|
|
233
|
+
Each `HeuristicEvaluator` caches its reports, keyed by the resolved path,
|
|
234
|
+
its modification time and size, and `max_tier`, so a changed file is
|
|
235
|
+
evaluated afresh. `HeuristicEvaluator(cache_size=0)` disables the cache,
|
|
236
|
+
`evaluate(path, use_cache=False)` bypasses it once, and `cache_info()` /
|
|
237
|
+
`clear_cache()` inspect and reset it. Sizing, the cases the key cannot see
|
|
238
|
+
(an edit inside an evaluated directory) and thread safety are in
|
|
239
|
+
[docs/production-guide.md](docs/production-guide.md).
|
|
240
|
+
|
|
241
|
+
---
|
|
242
|
+
|
|
243
|
+
## Pre-filtering with Rules
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
from vcti.lookup import Rule
|
|
247
|
+
|
|
248
|
+
# Only evaluate formats with structure="hdf5"
|
|
249
|
+
results = identifier.identify_file_format(
|
|
250
|
+
path,
|
|
251
|
+
rules=[Rule("structure", "==", "hdf5")],
|
|
252
|
+
)
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
## Error Handling
|
|
258
|
+
|
|
259
|
+
A path that cannot be identified raises a standard exception, so one
|
|
260
|
+
`except (ValueError, OSError)` covers it:
|
|
261
|
+
|
|
262
|
+
| Exception | When |
|
|
263
|
+
|-----------|------|
|
|
264
|
+
| `FileNotFoundError` (an `OSError`) | Path does not exist; `.filename` holds the path |
|
|
265
|
+
| `UnsupportedPathError` (a `ValueError`) | Path is neither a regular file nor a directory, e.g. a socket or named pipe; `.path` holds the path |
|
|
266
|
+
|
|
267
|
+
An unreadable file is not an error either: `MagicBytesValidator` reports
|
|
268
|
+
it as inconclusive, so the formats it gates come out `CANT_EVALUATE` rather
|
|
269
|
+
than ruled out.
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
try:
|
|
273
|
+
results = identifier.identify_file_format(path)
|
|
274
|
+
except (ValueError, OSError) as e:
|
|
275
|
+
print(f"Cannot identify {path}: {e}")
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
A format whose evaluator fails does not stop the others. Its result is
|
|
279
|
+
`CANT_EVALUATE`, the error is attached as `result.error`, and a WARNING is
|
|
280
|
+
logged:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
for result in identifier.identify_file_format(path):
|
|
284
|
+
if result.error is not None:
|
|
285
|
+
print(f"{result.descriptor.id} failed: {result.error}")
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Pass `strict=True` to `identify_file_format()` or `get_best_match()` to
|
|
289
|
+
raise the error instead — useful in the tests of a descriptor package.
|
|
290
|
+
|
|
291
|
+
Errors from evaluation itself derive from `EvaluatorError`. All exception
|
|
292
|
+
classes are importable from `vcti.pathformat.core`:
|
|
293
|
+
|
|
294
|
+
| Exception | When |
|
|
295
|
+
|-----------|------|
|
|
296
|
+
| `EvaluatorError` | Base class for evaluator errors |
|
|
297
|
+
| `ValidationError` | A validator raised an unexpected exception; raised by `FormatIdentifier` only with `strict=True` |
|
|
298
|
+
| `InvalidValidatorError` | Invalid validator or arguments passed to a builder method |
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
## Ecosystem
|
|
303
|
+
|
|
304
|
+
This package is the identification engine in a three-repo system:
|
|
305
|
+
|
|
306
|
+
| Package | Role |
|
|
307
|
+
|---------|------|
|
|
308
|
+
| **vcti-path-format** | Framework: evaluators, validators, registry, identifier |
|
|
309
|
+
| [vcti-path-format-attributes](https://pypi.org/project/vcti-path-format-attributes/) | Vocabulary: standardized attribute enums |
|
|
310
|
+
| [vcti-path-format-descriptors](https://pypi.org/project/vcti-path-format-descriptors/) | Built-in format definitions (HDF5, CAX, etc.) |
|
|
311
|
+
|
|
312
|
+
---
|
|
313
|
+
|
|
314
|
+
## Dependencies
|
|
315
|
+
|
|
316
|
+
- [vcti-plugin-catalog](https://pypi.org/project/vcti-plugin-catalog/) (>=2.0.0) — descriptor/registry framework
|
|
317
|
+
- [vcti-lookup](https://pypi.org/project/vcti-lookup/) (>=2.0.0) — attribute-based filtering
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
|
|
321
|
+
## Documentation
|
|
322
|
+
|
|
323
|
+
| If you want to… | Read |
|
|
324
|
+
|---|---|
|
|
325
|
+
| Get started using the package | Quick Start above |
|
|
326
|
+
| See practical, real-world usage | [docs/patterns.md](docs/patterns.md) |
|
|
327
|
+
| Understand the architecture and design decisions | [docs/design.md](docs/design.md) |
|
|
328
|
+
| Run it in a long-lived or multi-threaded service | [docs/production-guide.md](docs/production-guide.md) |
|
|
329
|
+
| See how the related path-format repositories fit together | [docs/ecosystem.md](docs/ecosystem.md) |
|
|
330
|
+
| Navigate and understand the source | [docs/source-guide.md](docs/source-guide.md) |
|
|
331
|
+
| Add custom validators or evaluators | [docs/extending.md](docs/extending.md) |
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
# Path Format
|
|
2
|
+
|
|
3
|
+
File format identification framework with heuristic evaluators and feature validators for Python.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
`vcti-path-format` answers "what format is this file or folder, and how sure
|
|
8
|
+
are we?" Each format is a `FormatDescriptor` held in a `FormatRegistry`; the
|
|
9
|
+
descriptor's evaluator runs feature validators — magic bytes, extension, or
|
|
10
|
+
checks of your own — against a path. Validators are either *gates*, which
|
|
11
|
+
rule a format out when they fail, or *evidence*, which only lowers
|
|
12
|
+
confidence, and they are grouped into tiers of increasing cost so callers
|
|
13
|
+
can stop at cheap checks. `FormatIdentifier` evaluates a path against every
|
|
14
|
+
registered format and ranks the results on a five-level `MatchConfidence`
|
|
15
|
+
scale.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install "vcti-path-format>=2.0.0"
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
### In `pyproject.toml` dependencies
|
|
24
|
+
|
|
25
|
+
```toml
|
|
26
|
+
dependencies = [
|
|
27
|
+
"vcti-path-format>=2.0.0",
|
|
28
|
+
]
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## Quick Start
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from vcti.pathformat.core import (
|
|
38
|
+
FormatDescriptor,
|
|
39
|
+
FormatIdentifier,
|
|
40
|
+
FormatRegistry,
|
|
41
|
+
HeuristicEvaluator,
|
|
42
|
+
MatchConfidence,
|
|
43
|
+
PathKind,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
# Define a format descriptor with validators
|
|
47
|
+
hdf5_descriptor = FormatDescriptor(
|
|
48
|
+
id="hdf5-file",
|
|
49
|
+
name="HDF5 File",
|
|
50
|
+
evaluator=(
|
|
51
|
+
HeuristicEvaluator()
|
|
52
|
+
.check_path_kind(PathKind.FILE) # PRECONDITION: rules out directories
|
|
53
|
+
.check_magic_bytes(b"\x89HDF\r\n\x1a\n") # GATE
|
|
54
|
+
.check_extension([".h5", ".hdf5", ".he5"]) # EVIDENCE
|
|
55
|
+
),
|
|
56
|
+
attributes={"path_type": "file", "structure": "hdf5"},
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Register in a format registry
|
|
60
|
+
registry = FormatRegistry()
|
|
61
|
+
registry.register(hdf5_descriptor)
|
|
62
|
+
|
|
63
|
+
# Identify a file
|
|
64
|
+
identifier = FormatIdentifier(registry)
|
|
65
|
+
results = identifier.identify_file_format(Path("data.h5"))
|
|
66
|
+
|
|
67
|
+
for result in results:
|
|
68
|
+
print(f"{result.descriptor.name}: {result.confidence.name}")
|
|
69
|
+
|
|
70
|
+
# Get best match above a confidence threshold
|
|
71
|
+
best = identifier.get_best_match(
|
|
72
|
+
Path("data.h5"),
|
|
73
|
+
min_confidence=MatchConfidence.LIKELY,
|
|
74
|
+
)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## Core Concepts
|
|
80
|
+
|
|
81
|
+
### FormatDescriptor
|
|
82
|
+
|
|
83
|
+
Extends `Descriptor[Evaluator]` from vcti-plugin-catalog. Wraps an evaluator
|
|
84
|
+
with format metadata and attributes.
|
|
85
|
+
|
|
86
|
+
### FormatRegistry
|
|
87
|
+
|
|
88
|
+
Extends `Registry[FormatDescriptor]`. Central catalog of known formats with
|
|
89
|
+
attribute-based filtering via `registry.lookup`.
|
|
90
|
+
|
|
91
|
+
### FormatIdentifier
|
|
92
|
+
|
|
93
|
+
Evaluates a path against all (or filtered) registered formats and returns
|
|
94
|
+
results sorted by confidence, then by descriptor `priority`.
|
|
95
|
+
|
|
96
|
+
### HeuristicEvaluator
|
|
97
|
+
|
|
98
|
+
Builder-pattern evaluator that runs feature validators and turns their
|
|
99
|
+
results into a confidence:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from vcti.pathformat.core import HeuristicEvaluator, PathKind
|
|
103
|
+
|
|
104
|
+
evaluator = (
|
|
105
|
+
HeuristicEvaluator()
|
|
106
|
+
.check_path_kind(PathKind.FILE) # PRECONDITION
|
|
107
|
+
.check_magic_bytes(b"\x89PNG\r\n\x1a\n") # GATE
|
|
108
|
+
.check_extension([".png"]) # EVIDENCE
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Each validator plays a **role**:
|
|
113
|
+
|
|
114
|
+
| Role | Failure | Success |
|
|
115
|
+
|------|---------|---------|
|
|
116
|
+
| `PRECONDITION` | Rules the format out | No evidence — the format merely *applies* (e.g. "is a file") |
|
|
117
|
+
| `GATE` | Rules the format out | Strong evidence of a match (e.g. the signature) |
|
|
118
|
+
| `EVIDENCE` | Lowers confidence | Raises confidence |
|
|
119
|
+
|
|
120
|
+
A validator that cannot obtain evidence — the file cannot be read — returns
|
|
121
|
+
an *inconclusive* result instead of a failure: an inconclusive
|
|
122
|
+
PRECONDITION or GATE gives `CANT_EVALUATE`, not `CERTAINLY_NOT`.
|
|
123
|
+
|
|
124
|
+
**Confidence rules.** A failed PRECONDITION or GATE gives `CERTAINLY_NOT`,
|
|
125
|
+
an inconclusive one `CANT_EVALUATE`, and either stops evaluation. Otherwise
|
|
126
|
+
the evaluator's confidence policy decides. The default policy:
|
|
127
|
+
|
|
128
|
+
- All passed + GATE present -> `DEFINITE`
|
|
129
|
+
- GATE passed, some EVIDENCE failed -> `LIKELY`
|
|
130
|
+
- All passed + no GATE -> `LIKELY`
|
|
131
|
+
- Some EVIDENCE failed, no GATE -> `UNLIKELY`
|
|
132
|
+
- No GATE and no conclusive EVIDENCE -> `CANT_EVALUATE`
|
|
133
|
+
|
|
134
|
+
### Feature Validators
|
|
135
|
+
|
|
136
|
+
| Validator | Default role | Tier | Checks |
|
|
137
|
+
|-----------|--------------|------|--------|
|
|
138
|
+
| `PathKindValidator` | PRECONDITION | IDENTIFICATION | Path is a file, or a directory (`check_path_kind`) |
|
|
139
|
+
| `MagicBytesValidator` | GATE | IDENTIFICATION | File signature bytes (`check_magic_bytes`) |
|
|
140
|
+
| `ExtensionValidator` | EVIDENCE | IDENTIFICATION | File extension (`check_extension`) |
|
|
141
|
+
|
|
142
|
+
Every descriptor is evaluated against every path, so a file format also
|
|
143
|
+
meets directories and a directory format meets files. `check_path_kind()`
|
|
144
|
+
rules those out before any other check reads the path; it looks at the
|
|
145
|
+
filesystem, not at descriptor attributes.
|
|
146
|
+
|
|
147
|
+
Custom validators implement the `FeatureValidator` protocol — no base class
|
|
148
|
+
needed. See [docs/extending.md](docs/extending.md) for a worked example and
|
|
149
|
+
for when to return a failed versus an inconclusive result.
|
|
150
|
+
|
|
151
|
+
---
|
|
152
|
+
|
|
153
|
+
## Deciding What Counts
|
|
154
|
+
|
|
155
|
+
Which evidence matters is a property of the format, so the descriptor's
|
|
156
|
+
author decides it:
|
|
157
|
+
|
|
158
|
+
- **`role=`** on any builder method overrides a validator's default role —
|
|
159
|
+
e.g. make an extension a GATE for a format it defines.
|
|
160
|
+
- **`policy=`** on `HeuristicEvaluator` replaces the default confidence
|
|
161
|
+
policy with a function of the results.
|
|
162
|
+
- **`priority=`** on `FormatDescriptor` ranks formats that reach the same
|
|
163
|
+
confidence — e.g. CGNS above the plain HDF5 it is built on.
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
from vcti.pathformat.core import FormatDescriptor, HeuristicEvaluator, ValidatorRole
|
|
167
|
+
|
|
168
|
+
erf_descriptor = FormatDescriptor(
|
|
169
|
+
id="erfh5",
|
|
170
|
+
name="ERF HDF5",
|
|
171
|
+
evaluator=(
|
|
172
|
+
HeuristicEvaluator()
|
|
173
|
+
.check_magic_bytes(b"\x89HDF\r\n\x1a\n")
|
|
174
|
+
.check_extension([".erfh5"], role=ValidatorRole.GATE)
|
|
175
|
+
),
|
|
176
|
+
priority=10, # wins over a generic HDF5 descriptor when both match
|
|
177
|
+
)
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Why the package leaves these choices to you is in
|
|
181
|
+
[docs/design.md](docs/design.md); writing a policy is in
|
|
182
|
+
[docs/extending.md](docs/extending.md).
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Validation Tiers
|
|
187
|
+
|
|
188
|
+
Control evaluation depth with `max_tier`:
|
|
189
|
+
|
|
190
|
+
| Tier | Cost | Examples |
|
|
191
|
+
|------|------|---------|
|
|
192
|
+
| `IDENTIFICATION` | Cheap | Path kind, magic bytes, file extension |
|
|
193
|
+
| `STRUCTURE` | Medium | Schema validation, header parsing |
|
|
194
|
+
| `SEMANTIC` | Expensive | Content analysis, business logic |
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
from vcti.pathformat.core import ValidationTier
|
|
198
|
+
|
|
199
|
+
# Only run cheap checks
|
|
200
|
+
results = identifier.identify_file_format(path, max_tier=ValidationTier.IDENTIFICATION)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## Evaluator Caching
|
|
206
|
+
|
|
207
|
+
Each `HeuristicEvaluator` caches its reports, keyed by the resolved path,
|
|
208
|
+
its modification time and size, and `max_tier`, so a changed file is
|
|
209
|
+
evaluated afresh. `HeuristicEvaluator(cache_size=0)` disables the cache,
|
|
210
|
+
`evaluate(path, use_cache=False)` bypasses it once, and `cache_info()` /
|
|
211
|
+
`clear_cache()` inspect and reset it. Sizing, the cases the key cannot see
|
|
212
|
+
(an edit inside an evaluated directory) and thread safety are in
|
|
213
|
+
[docs/production-guide.md](docs/production-guide.md).
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Pre-filtering with Rules
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
from vcti.lookup import Rule
|
|
221
|
+
|
|
222
|
+
# Only evaluate formats with structure="hdf5"
|
|
223
|
+
results = identifier.identify_file_format(
|
|
224
|
+
path,
|
|
225
|
+
rules=[Rule("structure", "==", "hdf5")],
|
|
226
|
+
)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
---
|
|
230
|
+
|
|
231
|
+
## Error Handling
|
|
232
|
+
|
|
233
|
+
A path that cannot be identified raises a standard exception, so one
|
|
234
|
+
`except (ValueError, OSError)` covers it:
|
|
235
|
+
|
|
236
|
+
| Exception | When |
|
|
237
|
+
|-----------|------|
|
|
238
|
+
| `FileNotFoundError` (an `OSError`) | Path does not exist; `.filename` holds the path |
|
|
239
|
+
| `UnsupportedPathError` (a `ValueError`) | Path is neither a regular file nor a directory, e.g. a socket or named pipe; `.path` holds the path |
|
|
240
|
+
|
|
241
|
+
An unreadable file is not an error either: `MagicBytesValidator` reports
|
|
242
|
+
it as inconclusive, so the formats it gates come out `CANT_EVALUATE` rather
|
|
243
|
+
than ruled out.
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
try:
|
|
247
|
+
results = identifier.identify_file_format(path)
|
|
248
|
+
except (ValueError, OSError) as e:
|
|
249
|
+
print(f"Cannot identify {path}: {e}")
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
A format whose evaluator fails does not stop the others. Its result is
|
|
253
|
+
`CANT_EVALUATE`, the error is attached as `result.error`, and a WARNING is
|
|
254
|
+
logged:
|
|
255
|
+
|
|
256
|
+
```python
|
|
257
|
+
for result in identifier.identify_file_format(path):
|
|
258
|
+
if result.error is not None:
|
|
259
|
+
print(f"{result.descriptor.id} failed: {result.error}")
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
Pass `strict=True` to `identify_file_format()` or `get_best_match()` to
|
|
263
|
+
raise the error instead — useful in the tests of a descriptor package.
|
|
264
|
+
|
|
265
|
+
Errors from evaluation itself derive from `EvaluatorError`. All exception
|
|
266
|
+
classes are importable from `vcti.pathformat.core`:
|
|
267
|
+
|
|
268
|
+
| Exception | When |
|
|
269
|
+
|-----------|------|
|
|
270
|
+
| `EvaluatorError` | Base class for evaluator errors |
|
|
271
|
+
| `ValidationError` | A validator raised an unexpected exception; raised by `FormatIdentifier` only with `strict=True` |
|
|
272
|
+
| `InvalidValidatorError` | Invalid validator or arguments passed to a builder method |
|
|
273
|
+
|
|
274
|
+
---
|
|
275
|
+
|
|
276
|
+
## Ecosystem
|
|
277
|
+
|
|
278
|
+
This package is the identification engine in a three-repo system:
|
|
279
|
+
|
|
280
|
+
| Package | Role |
|
|
281
|
+
|---------|------|
|
|
282
|
+
| **vcti-path-format** | Framework: evaluators, validators, registry, identifier |
|
|
283
|
+
| [vcti-path-format-attributes](https://pypi.org/project/vcti-path-format-attributes/) | Vocabulary: standardized attribute enums |
|
|
284
|
+
| [vcti-path-format-descriptors](https://pypi.org/project/vcti-path-format-descriptors/) | Built-in format definitions (HDF5, CAX, etc.) |
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## Dependencies
|
|
289
|
+
|
|
290
|
+
- [vcti-plugin-catalog](https://pypi.org/project/vcti-plugin-catalog/) (>=2.0.0) — descriptor/registry framework
|
|
291
|
+
- [vcti-lookup](https://pypi.org/project/vcti-lookup/) (>=2.0.0) — attribute-based filtering
|
|
292
|
+
|
|
293
|
+
---
|
|
294
|
+
|
|
295
|
+
## Documentation
|
|
296
|
+
|
|
297
|
+
| If you want to… | Read |
|
|
298
|
+
|---|---|
|
|
299
|
+
| Get started using the package | Quick Start above |
|
|
300
|
+
| See practical, real-world usage | [docs/patterns.md](docs/patterns.md) |
|
|
301
|
+
| Understand the architecture and design decisions | [docs/design.md](docs/design.md) |
|
|
302
|
+
| Run it in a long-lived or multi-threaded service | [docs/production-guide.md](docs/production-guide.md) |
|
|
303
|
+
| See how the related path-format repositories fit together | [docs/ecosystem.md](docs/ecosystem.md) |
|
|
304
|
+
| Navigate and understand the source | [docs/source-guide.md](docs/source-guide.md) |
|
|
305
|
+
| Add custom validators or evaluators | [docs/extending.md](docs/extending.md) |
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vcti-path-format"
|
|
7
|
+
version = "2.0.0"
|
|
8
|
+
description = "File format identification framework with heuristic evaluators and feature validators for Python"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{name = "Visual Collaboration Technologies Inc."}
|
|
12
|
+
]
|
|
13
|
+
license = "LicenseRef-Proprietary"
|
|
14
|
+
license-files = ["LICENSE"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Programming Language :: Python :: 3.12",
|
|
18
|
+
"Programming Language :: Python :: 3.13",
|
|
19
|
+
"Programming Language :: Python :: 3.14",
|
|
20
|
+
]
|
|
21
|
+
requires-python = ">=3.12,<3.15"
|
|
22
|
+
dependencies = [
|
|
23
|
+
"vcti-plugin-catalog>=2.0.0",
|
|
24
|
+
"vcti-lookup>=2.0.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Repository = "https://github.com/vcollab/vcti-python-path-format"
|
|
29
|
+
Changelog = "https://github.com/vcollab/vcti-python-path-format/blob/main/CHANGELOG.md"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
where = ["src"]
|
|
33
|
+
include = ["vcti.pathformat.core", "vcti.pathformat.core.*"]
|
|
34
|
+
|
|
35
|
+
[tool.setuptools.package-data]
|
|
36
|
+
"vcti.pathformat.core" = ["py.typed"]
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
test = ["pytest", "pytest-cov"]
|
|
40
|
+
lint = ["ruff"]
|
|
41
|
+
typecheck = ["mypy"]
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
addopts = "--cov=vcti.pathformat.core --cov-report=term-missing --cov-fail-under=95"
|
|
45
|
+
|
|
46
|
+
[tool.mypy]
|
|
47
|
+
python_version = "3.12"
|
|
48
|
+
strict = true
|
|
49
|
+
files = ["src"]
|
|
50
|
+
namespace_packages = true
|
|
51
|
+
explicit_package_bases = true
|
|
52
|
+
mypy_path = ["src"]
|
|
53
|
+
|
|
54
|
+
[tool.coverage.run]
|
|
55
|
+
branch = true
|
|
56
|
+
|
|
57
|
+
[tool.coverage.report]
|
|
58
|
+
exclude_also = [
|
|
59
|
+
"raise NotImplementedError",
|
|
60
|
+
"if TYPE_CHECKING:",
|
|
61
|
+
"if __name__ == .__main__.:",
|
|
62
|
+
"@(abc\\.)?abstractmethod",
|
|
63
|
+
"^\\s*\\.\\.\\.\\s*$",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
target-version = "py312"
|
|
68
|
+
line-length = 99
|
|
69
|
+
|
|
70
|
+
[tool.ruff.lint]
|
|
71
|
+
select = ["E", "F", "W", "I", "UP"]
|