vcti-fileloader 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vcti_fileloader-1.0.0/LICENSE +8 -0
- vcti_fileloader-1.0.0/PKG-INFO +270 -0
- vcti_fileloader-1.0.0/README.md +239 -0
- vcti_fileloader-1.0.0/pyproject.toml +59 -0
- vcti_fileloader-1.0.0/setup.cfg +4 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/__init__.py +38 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/descriptor.py +50 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/loader.py +202 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/py.typed +0 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/registry.py +24 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/setup.py +22 -0
- vcti_fileloader-1.0.0/src/vcti/fileloader/validator.py +22 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/PKG-INFO +270 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/SOURCES.txt +18 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/dependency_links.txt +1 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/requires.txt +13 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/top_level.txt +1 -0
- vcti_fileloader-1.0.0/src/vcti_fileloader.egg-info/zip-safe +1 -0
- vcti_fileloader-1.0.0/tests/test_loader.py +639 -0
- vcti_fileloader-1.0.0/tests/test_version.py +15 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
Copyright (c) 2018-2026 Visual Collaboration Technologies Inc.
|
|
2
|
+
All Rights Reserved.
|
|
3
|
+
|
|
4
|
+
This software is proprietary and confidential. Unauthorized copying,
|
|
5
|
+
distribution, or use of this software, via any medium, is strictly
|
|
6
|
+
prohibited. Access is granted only to authorized VCollab developers
|
|
7
|
+
and individuals explicitly authorized by Visual Collaboration
|
|
8
|
+
Technologies Inc.
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vcti-fileloader
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: File loader framework with pluggable descriptors, validators, and a registry for format-specific data loading
|
|
5
|
+
Author: Visual Collaboration Technologies Inc.
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Project-URL: Homepage, https://github.com/vcollab/vcti-python-fileloader
|
|
8
|
+
Project-URL: Repository, https://github.com/vcollab/vcti-python-fileloader
|
|
9
|
+
Project-URL: Issues, https://github.com/vcollab/vcti-python-fileloader/issues
|
|
10
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: Other/Proprietary License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Typing :: Typed
|
|
17
|
+
Requires-Python: <3.15,>=3.12
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: numpy>=1.24
|
|
21
|
+
Requires-Dist: vcti-plugin-catalog>=1.0.0
|
|
22
|
+
Requires-Dist: vcti-array-tree>=1.0.0
|
|
23
|
+
Provides-Extra: test
|
|
24
|
+
Requires-Dist: pytest; extra == "test"
|
|
25
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
26
|
+
Provides-Extra: lint
|
|
27
|
+
Requires-Dist: ruff; extra == "lint"
|
|
28
|
+
Provides-Extra: typecheck
|
|
29
|
+
Requires-Dist: mypy; extra == "typecheck"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# vcti-fileloader
|
|
33
|
+
|
|
34
|
+
A protocol-based framework for loading hierarchical scientific and engineering
|
|
35
|
+
data from files. It defines a **standard interface** that all file-format loaders
|
|
36
|
+
must implement, and a **registry** for discovering and managing them at runtime.
|
|
37
|
+
|
|
38
|
+
This package is fully typed (`py.typed`) and safe for strict type checkers
|
|
39
|
+
(mypy `--strict`, pyright).
|
|
40
|
+
|
|
41
|
+
## Why this package exists
|
|
42
|
+
|
|
43
|
+
Applications that work with simulation and CAE data need to load many file
|
|
44
|
+
formats — HDF5, VTK, OpenFOAM, proprietary binary, etc. Each format has its
|
|
45
|
+
own library, its own API, and its own way of representing a tree of nodes,
|
|
46
|
+
metadata, and heavy data arrays.
|
|
47
|
+
|
|
48
|
+
vcti-fileloader solves this by defining a **single, uniform protocol** that
|
|
49
|
+
every loader plugin implements. Application code programs against the protocol,
|
|
50
|
+
not the format. Adding support for a new file format means writing a new loader
|
|
51
|
+
plugin — no changes to application code.
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
┌─────────────────────────────────────────────────────┐
|
|
55
|
+
│ Application Code │
|
|
56
|
+
│ (uses Loader protocol + Registry) │
|
|
57
|
+
└──────────────┬──────────────────────┬───────────────┘
|
|
58
|
+
│ │
|
|
59
|
+
┌───────▼───────┐ ┌───────▼───────┐
|
|
60
|
+
│ HDF5 Loader │ │ VTK Loader │ ... (one per format)
|
|
61
|
+
│ (plugin pkg) │ │ (plugin pkg) │
|
|
62
|
+
└───────────────┘ └───────────────┘
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Key Concepts
|
|
66
|
+
|
|
67
|
+
### Loader (Protocol)
|
|
68
|
+
|
|
69
|
+
The central interface. Any class that implements these methods satisfies
|
|
70
|
+
the protocol — no base class inheritance required (PEP 544 structural subtyping):
|
|
71
|
+
|
|
72
|
+
| Method | Purpose |
|
|
73
|
+
|--------|---------|
|
|
74
|
+
| `load(path, **options)` | Open a file and return an opaque data handle |
|
|
75
|
+
| `unload(data)` | Release file handles and memory (**idempotent**) |
|
|
76
|
+
| `can_load(path)` | Lightweight check — can this loader handle the file? |
|
|
77
|
+
| `load_tree(data)` | Extract the node hierarchy as a NumPy structured array |
|
|
78
|
+
| `load_node_info(data)` | Extract lightweight node metadata (name, type) |
|
|
79
|
+
| `load_attributes(data, node_ids)` | Extract key-value attributes per node |
|
|
80
|
+
| `load_dataset(data, node_id)` | Extract a heavy data array as a `DataNode` |
|
|
81
|
+
|
|
82
|
+
Each loader also carries optional **validator** and **setup** hooks:
|
|
83
|
+
|
|
84
|
+
- `LoaderValidator.validate()` — returns `True` if all runtime dependencies
|
|
85
|
+
(e.g., h5py, vtk) are available.
|
|
86
|
+
- `LoaderSetup.setup()` — configures paths, environment variables, or
|
|
87
|
+
component versions before first use.
|
|
88
|
+
|
|
89
|
+
### LoaderDescriptor
|
|
90
|
+
|
|
91
|
+
Wraps a `Loader` instance with registry metadata — a unique `id`, a
|
|
92
|
+
human-readable `name`, and filterable `attributes` (e.g., `supported_formats`).
|
|
93
|
+
|
|
94
|
+
### LoaderRegistry
|
|
95
|
+
|
|
96
|
+
A typed registry of `LoaderDescriptor` entries. Register loaders at startup,
|
|
97
|
+
then look them up by id or query by attributes at runtime.
|
|
98
|
+
|
|
99
|
+
**Key behaviours** (inherited from `vcti-plugin-catalog`):
|
|
100
|
+
|
|
101
|
+
- `register()` raises `DuplicateEntryError` if the id already exists.
|
|
102
|
+
- `get()` raises `EntryNotFoundError` if the id is not found.
|
|
103
|
+
- `find()` returns `None` instead of raising for missing ids.
|
|
104
|
+
- `lookup` property provides attribute-based filtering via `vcti-lookup`.
|
|
105
|
+
|
|
106
|
+
### NodeID
|
|
107
|
+
|
|
108
|
+
Type alias (`int`) for node identifiers used across the protocol, exported
|
|
109
|
+
from the package for use in type annotations.
|
|
110
|
+
|
|
111
|
+
### DataNode
|
|
112
|
+
|
|
113
|
+
`load_dataset()` returns a `DataNode` from `vcti-array-tree`. A DataNode has:
|
|
114
|
+
|
|
115
|
+
- `.data` — the NumPy array containing the heavy data.
|
|
116
|
+
- `.attributes` — a `dict[str, Any]` of metadata for the node.
|
|
117
|
+
|
|
118
|
+
See the [vcti-array-tree documentation](https://pypi.org/project/vcti-array-tree/)
|
|
119
|
+
for full details.
|
|
120
|
+
|
|
121
|
+
## Data Flow
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
Register ──► Discover ──► Validate & Setup ──► Load ──► Query ──► Unload
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
1. **Register** — Each loader plugin registers a `LoaderDescriptor` with
|
|
128
|
+
the shared `LoaderRegistry`.
|
|
129
|
+
2. **Discover** — Application code looks up a loader by id or filters by
|
|
130
|
+
attributes (e.g., find all loaders that support `"hdf5-file"`).
|
|
131
|
+
3. **Validate & Setup** — Call `validator.validate()` and `setup.setup()`
|
|
132
|
+
to ensure the runtime environment is ready.
|
|
133
|
+
4. **Load** — `loader.load(path)` opens the file and returns an opaque handle.
|
|
134
|
+
5. **Query** — Use `load_tree`, `load_node_info`, `load_attributes`, and
|
|
135
|
+
`load_dataset` to extract structure and data from the handle.
|
|
136
|
+
6. **Unload** — `loader.unload(handle)` releases resources. This is
|
|
137
|
+
**idempotent** — calling it twice on the same handle must not raise.
|
|
138
|
+
|
|
139
|
+
## Lifecycle Contracts
|
|
140
|
+
|
|
141
|
+
- Call `can_load(path)` **before** `load()` to prevent `UnsupportedFormatError`.
|
|
142
|
+
- Call `validator.validate()` and `setup.setup()` **before** the first `load()`.
|
|
143
|
+
- `unload()` is **idempotent** — safe to call multiple times on the same handle.
|
|
144
|
+
- After `unload()`, calling any `load_*` method on that handle is **undefined**.
|
|
145
|
+
- `load()` may be called multiple times with different paths; each returns
|
|
146
|
+
an independent handle.
|
|
147
|
+
|
|
148
|
+
## Installation
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
pip install vcti-fileloader>=1.0.0
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### In `pyproject.toml` dependencies
|
|
155
|
+
|
|
156
|
+
```toml
|
|
157
|
+
dependencies = [
|
|
158
|
+
"vcti-fileloader>=1.0.0",
|
|
159
|
+
]
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## Quick Start
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
from pathlib import Path
|
|
166
|
+
|
|
167
|
+
from vcti.fileloader import LoaderDescriptor, LoaderRegistry
|
|
168
|
+
|
|
169
|
+
# At startup: register available loaders
|
|
170
|
+
registry = LoaderRegistry()
|
|
171
|
+
registry.register(LoaderDescriptor(
|
|
172
|
+
id="hdf5-h5py-loader",
|
|
173
|
+
name="HDF5 Loader (h5py)",
|
|
174
|
+
loader=my_h5py_loader, # implements Loader protocol
|
|
175
|
+
attributes={"supported_formats": ["hdf5-file"]},
|
|
176
|
+
))
|
|
177
|
+
|
|
178
|
+
# At runtime: discover, validate, load
|
|
179
|
+
desc = registry.get("hdf5-h5py-loader")
|
|
180
|
+
desc.loader.validator.validate() # check dependencies
|
|
181
|
+
desc.loader.setup.setup() # configure environment
|
|
182
|
+
|
|
183
|
+
handle = desc.loader.load(Path("simulation.h5"))
|
|
184
|
+
tree = desc.loader.load_tree(handle) # node hierarchy
|
|
185
|
+
info = desc.loader.load_node_info(handle) # node names and types
|
|
186
|
+
attrs = desc.loader.load_attributes(handle) # per-node attributes
|
|
187
|
+
node = desc.loader.load_dataset(handle, node_id=1) # heavy data array
|
|
188
|
+
desc.loader.unload(handle) # release resources
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Error Handling
|
|
192
|
+
|
|
193
|
+
All exceptions inherit from `LoaderError`, so callers can catch broadly
|
|
194
|
+
or handle specific failure modes:
|
|
195
|
+
|
|
196
|
+
| Exception | When to raise / catch |
|
|
197
|
+
|-----------|----------------------|
|
|
198
|
+
| `LoaderError` | Base — catches any loader failure |
|
|
199
|
+
| `LoadError` | File cannot be opened or parsed (I/O errors, corrupt files) |
|
|
200
|
+
| `UnloadError` | Resource cleanup failed |
|
|
201
|
+
| `UnsupportedFormatError` | Loader does not recognise the file format. Prefer `can_load()` first |
|
|
202
|
+
| `ValidationError` | `validator.validate()` detected missing dependencies |
|
|
203
|
+
| `SetupError` | `setup.setup()` could not configure the environment |
|
|
204
|
+
|
|
205
|
+
**Distinguishing `LoadError` vs `UnsupportedFormatError`:** Use
|
|
206
|
+
`UnsupportedFormatError` when the loader does not recognise the format at
|
|
207
|
+
all (wrong extension, unknown magic bytes). Use `LoadError` when the
|
|
208
|
+
format is recognised but the content cannot be read (truncated file,
|
|
209
|
+
incompatible version, permission error).
|
|
210
|
+
|
|
211
|
+
### Error handling example
|
|
212
|
+
|
|
213
|
+
```python
|
|
214
|
+
from vcti.fileloader import (
|
|
215
|
+
LoaderError,
|
|
216
|
+
LoadError,
|
|
217
|
+
UnsupportedFormatError,
|
|
218
|
+
ValidationError,
|
|
219
|
+
SetupError,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# Validate before loading
|
|
223
|
+
if not desc.loader.validator.validate():
|
|
224
|
+
raise ValidationError("Missing h5py — install with: pip install h5py")
|
|
225
|
+
|
|
226
|
+
if not desc.loader.setup.setup():
|
|
227
|
+
raise SetupError("Could not configure HDF5 library paths")
|
|
228
|
+
|
|
229
|
+
# Load with error handling
|
|
230
|
+
path = Path("data.h5")
|
|
231
|
+
if not desc.loader.can_load(path):
|
|
232
|
+
print(f"Loader {desc.id} cannot handle {path}")
|
|
233
|
+
else:
|
|
234
|
+
try:
|
|
235
|
+
handle = desc.loader.load(path)
|
|
236
|
+
tree = desc.loader.load_tree(handle)
|
|
237
|
+
except LoadError as e:
|
|
238
|
+
print(f"Failed to read file: {e}")
|
|
239
|
+
except LoaderError as e:
|
|
240
|
+
print(f"Unexpected loader error: {e}")
|
|
241
|
+
finally:
|
|
242
|
+
desc.loader.unload(handle) # safe even if load() failed partially
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
## What this package does NOT do
|
|
246
|
+
|
|
247
|
+
- **No concrete loaders** — This is the interface only. Actual file reading
|
|
248
|
+
(HDF5, VTK, etc.) lives in separate loader plugin packages.
|
|
249
|
+
- **No data transformation** — Data is returned as-is from the loader.
|
|
250
|
+
- **No caching** — Caching strategies belong at the application level.
|
|
251
|
+
|
|
252
|
+
## Further Reading
|
|
253
|
+
|
|
254
|
+
- [Common Patterns](docs/patterns.md) — Validator/setup implementation,
|
|
255
|
+
multi-loader registration, error handling, and naming conventions.
|
|
256
|
+
- [Design & Concepts](docs/design.md) — Architecture, protocol rationale,
|
|
257
|
+
and package boundaries.
|
|
258
|
+
|
|
259
|
+
## Dependencies
|
|
260
|
+
|
|
261
|
+
- [numpy](https://numpy.org/) (>=1.24)
|
|
262
|
+
- [vcti-plugin-catalog](https://pypi.org/project/vcti-plugin-catalog/) (>=1.0.0) — Descriptor and Registry base classes
|
|
263
|
+
- [vcti-array-tree](https://pypi.org/project/vcti-array-tree/) (>=1.0.0) — `DataNode` returned by `load_dataset`
|
|
264
|
+
|
|
265
|
+
## Versioning
|
|
266
|
+
|
|
267
|
+
This package follows [Semantic Versioning](https://semver.org/). Breaking
|
|
268
|
+
changes to the `Loader` protocol (adding required methods, changing
|
|
269
|
+
signatures) will only occur in major version bumps. Downstream loader
|
|
270
|
+
plugins should pin to a compatible major version (e.g., `vcti-fileloader>=1.0,<2`).
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# vcti-fileloader
|
|
2
|
+
|
|
3
|
+
A protocol-based framework for loading hierarchical scientific and engineering
|
|
4
|
+
data from files. It defines a **standard interface** that all file-format loaders
|
|
5
|
+
must implement, and a **registry** for discovering and managing them at runtime.
|
|
6
|
+
|
|
7
|
+
This package is fully typed (`py.typed`) and safe for strict type checkers
|
|
8
|
+
(mypy `--strict`, pyright).
|
|
9
|
+
|
|
10
|
+
## Why this package exists
|
|
11
|
+
|
|
12
|
+
Applications that work with simulation and CAE data need to load many file
|
|
13
|
+
formats — HDF5, VTK, OpenFOAM, proprietary binary, etc. Each format has its
|
|
14
|
+
own library, its own API, and its own way of representing a tree of nodes,
|
|
15
|
+
metadata, and heavy data arrays.
|
|
16
|
+
|
|
17
|
+
vcti-fileloader solves this by defining a **single, uniform protocol** that
|
|
18
|
+
every loader plugin implements. Application code programs against the protocol,
|
|
19
|
+
not the format. Adding support for a new file format means writing a new loader
|
|
20
|
+
plugin — no changes to application code.
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
┌─────────────────────────────────────────────────────┐
|
|
24
|
+
│ Application Code │
|
|
25
|
+
│ (uses Loader protocol + Registry) │
|
|
26
|
+
└──────────────┬──────────────────────┬───────────────┘
|
|
27
|
+
│ │
|
|
28
|
+
┌───────▼───────┐ ┌───────▼───────┐
|
|
29
|
+
│ HDF5 Loader │ │ VTK Loader │ ... (one per format)
|
|
30
|
+
│ (plugin pkg) │ │ (plugin pkg) │
|
|
31
|
+
└───────────────┘ └───────────────┘
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Key Concepts
|
|
35
|
+
|
|
36
|
+
### Loader (Protocol)
|
|
37
|
+
|
|
38
|
+
The central interface. Any class that implements these methods satisfies
|
|
39
|
+
the protocol — no base class inheritance required (PEP 544 structural subtyping):
|
|
40
|
+
|
|
41
|
+
| Method | Purpose |
|
|
42
|
+
|--------|---------|
|
|
43
|
+
| `load(path, **options)` | Open a file and return an opaque data handle |
|
|
44
|
+
| `unload(data)` | Release file handles and memory (**idempotent**) |
|
|
45
|
+
| `can_load(path)` | Lightweight check — can this loader handle the file? |
|
|
46
|
+
| `load_tree(data)` | Extract the node hierarchy as a NumPy structured array |
|
|
47
|
+
| `load_node_info(data)` | Extract lightweight node metadata (name, type) |
|
|
48
|
+
| `load_attributes(data, node_ids)` | Extract key-value attributes per node |
|
|
49
|
+
| `load_dataset(data, node_id)` | Extract a heavy data array as a `DataNode` |
|
|
50
|
+
|
|
51
|
+
Each loader also carries optional **validator** and **setup** hooks:
|
|
52
|
+
|
|
53
|
+
- `LoaderValidator.validate()` — returns `True` if all runtime dependencies
|
|
54
|
+
(e.g., h5py, vtk) are available.
|
|
55
|
+
- `LoaderSetup.setup()` — configures paths, environment variables, or
|
|
56
|
+
component versions before first use.
|
|
57
|
+
|
|
58
|
+
### LoaderDescriptor
|
|
59
|
+
|
|
60
|
+
Wraps a `Loader` instance with registry metadata — a unique `id`, a
|
|
61
|
+
human-readable `name`, and filterable `attributes` (e.g., `supported_formats`).
|
|
62
|
+
|
|
63
|
+
### LoaderRegistry
|
|
64
|
+
|
|
65
|
+
A typed registry of `LoaderDescriptor` entries. Register loaders at startup,
|
|
66
|
+
then look them up by id or query by attributes at runtime.
|
|
67
|
+
|
|
68
|
+
**Key behaviours** (inherited from `vcti-plugin-catalog`):
|
|
69
|
+
|
|
70
|
+
- `register()` raises `DuplicateEntryError` if the id already exists.
|
|
71
|
+
- `get()` raises `EntryNotFoundError` if the id is not found.
|
|
72
|
+
- `find()` returns `None` instead of raising for missing ids.
|
|
73
|
+
- `lookup` property provides attribute-based filtering via `vcti-lookup`.
|
|
74
|
+
|
|
75
|
+
### NodeID
|
|
76
|
+
|
|
77
|
+
Type alias (`int`) for node identifiers used across the protocol, exported
|
|
78
|
+
from the package for use in type annotations.
|
|
79
|
+
|
|
80
|
+
### DataNode
|
|
81
|
+
|
|
82
|
+
`load_dataset()` returns a `DataNode` from `vcti-array-tree`. A DataNode has:
|
|
83
|
+
|
|
84
|
+
- `.data` — the NumPy array containing the heavy data.
|
|
85
|
+
- `.attributes` — a `dict[str, Any]` of metadata for the node.
|
|
86
|
+
|
|
87
|
+
See the [vcti-array-tree documentation](https://pypi.org/project/vcti-array-tree/)
|
|
88
|
+
for full details.
|
|
89
|
+
|
|
90
|
+
## Data Flow
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
Register ──► Discover ──► Validate & Setup ──► Load ──► Query ──► Unload
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
1. **Register** — Each loader plugin registers a `LoaderDescriptor` with
|
|
97
|
+
the shared `LoaderRegistry`.
|
|
98
|
+
2. **Discover** — Application code looks up a loader by id or filters by
|
|
99
|
+
attributes (e.g., find all loaders that support `"hdf5-file"`).
|
|
100
|
+
3. **Validate & Setup** — Call `validator.validate()` and `setup.setup()`
|
|
101
|
+
to ensure the runtime environment is ready.
|
|
102
|
+
4. **Load** — `loader.load(path)` opens the file and returns an opaque handle.
|
|
103
|
+
5. **Query** — Use `load_tree`, `load_node_info`, `load_attributes`, and
|
|
104
|
+
`load_dataset` to extract structure and data from the handle.
|
|
105
|
+
6. **Unload** — `loader.unload(handle)` releases resources. This is
|
|
106
|
+
**idempotent** — calling it twice on the same handle must not raise.
|
|
107
|
+
|
|
108
|
+
## Lifecycle Contracts
|
|
109
|
+
|
|
110
|
+
- Call `can_load(path)` **before** `load()` to prevent `UnsupportedFormatError`.
|
|
111
|
+
- Call `validator.validate()` and `setup.setup()` **before** the first `load()`.
|
|
112
|
+
- `unload()` is **idempotent** — safe to call multiple times on the same handle.
|
|
113
|
+
- After `unload()`, calling any `load_*` method on that handle is **undefined**.
|
|
114
|
+
- `load()` may be called multiple times with different paths; each returns
|
|
115
|
+
an independent handle.
|
|
116
|
+
|
|
117
|
+
## Installation
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
pip install vcti-fileloader>=1.0.0
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### In `pyproject.toml` dependencies
|
|
124
|
+
|
|
125
|
+
```toml
|
|
126
|
+
dependencies = [
|
|
127
|
+
"vcti-fileloader>=1.0.0",
|
|
128
|
+
]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Quick Start
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from pathlib import Path
|
|
135
|
+
|
|
136
|
+
from vcti.fileloader import LoaderDescriptor, LoaderRegistry
|
|
137
|
+
|
|
138
|
+
# At startup: register available loaders
|
|
139
|
+
registry = LoaderRegistry()
|
|
140
|
+
registry.register(LoaderDescriptor(
|
|
141
|
+
id="hdf5-h5py-loader",
|
|
142
|
+
name="HDF5 Loader (h5py)",
|
|
143
|
+
loader=my_h5py_loader, # implements Loader protocol
|
|
144
|
+
attributes={"supported_formats": ["hdf5-file"]},
|
|
145
|
+
))
|
|
146
|
+
|
|
147
|
+
# At runtime: discover, validate, load
|
|
148
|
+
desc = registry.get("hdf5-h5py-loader")
|
|
149
|
+
desc.loader.validator.validate() # check dependencies
|
|
150
|
+
desc.loader.setup.setup() # configure environment
|
|
151
|
+
|
|
152
|
+
handle = desc.loader.load(Path("simulation.h5"))
|
|
153
|
+
tree = desc.loader.load_tree(handle) # node hierarchy
|
|
154
|
+
info = desc.loader.load_node_info(handle) # node names and types
|
|
155
|
+
attrs = desc.loader.load_attributes(handle) # per-node attributes
|
|
156
|
+
node = desc.loader.load_dataset(handle, node_id=1) # heavy data array
|
|
157
|
+
desc.loader.unload(handle) # release resources
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## Error Handling
|
|
161
|
+
|
|
162
|
+
All exceptions inherit from `LoaderError`, so callers can catch broadly
|
|
163
|
+
or handle specific failure modes:
|
|
164
|
+
|
|
165
|
+
| Exception | When to raise / catch |
|
|
166
|
+
|-----------|----------------------|
|
|
167
|
+
| `LoaderError` | Base — catches any loader failure |
|
|
168
|
+
| `LoadError` | File cannot be opened or parsed (I/O errors, corrupt files) |
|
|
169
|
+
| `UnloadError` | Resource cleanup failed |
|
|
170
|
+
| `UnsupportedFormatError` | Loader does not recognise the file format. Prefer `can_load()` first |
|
|
171
|
+
| `ValidationError` | `validator.validate()` detected missing dependencies |
|
|
172
|
+
| `SetupError` | `setup.setup()` could not configure the environment |
|
|
173
|
+
|
|
174
|
+
**Distinguishing `LoadError` vs `UnsupportedFormatError`:** Use
|
|
175
|
+
`UnsupportedFormatError` when the loader does not recognise the format at
|
|
176
|
+
all (wrong extension, unknown magic bytes). Use `LoadError` when the
|
|
177
|
+
format is recognised but the content cannot be read (truncated file,
|
|
178
|
+
incompatible version, permission error).
|
|
179
|
+
|
|
180
|
+
### Error handling example
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
from vcti.fileloader import (
|
|
184
|
+
LoaderError,
|
|
185
|
+
LoadError,
|
|
186
|
+
UnsupportedFormatError,
|
|
187
|
+
ValidationError,
|
|
188
|
+
SetupError,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
# Validate before loading
|
|
192
|
+
if not desc.loader.validator.validate():
|
|
193
|
+
raise ValidationError("Missing h5py — install with: pip install h5py")
|
|
194
|
+
|
|
195
|
+
if not desc.loader.setup.setup():
|
|
196
|
+
raise SetupError("Could not configure HDF5 library paths")
|
|
197
|
+
|
|
198
|
+
# Load with error handling
|
|
199
|
+
path = Path("data.h5")
|
|
200
|
+
if not desc.loader.can_load(path):
|
|
201
|
+
print(f"Loader {desc.id} cannot handle {path}")
|
|
202
|
+
else:
|
|
203
|
+
try:
|
|
204
|
+
handle = desc.loader.load(path)
|
|
205
|
+
tree = desc.loader.load_tree(handle)
|
|
206
|
+
except LoadError as e:
|
|
207
|
+
print(f"Failed to read file: {e}")
|
|
208
|
+
except LoaderError as e:
|
|
209
|
+
print(f"Unexpected loader error: {e}")
|
|
210
|
+
finally:
|
|
211
|
+
desc.loader.unload(handle) # safe even if load() failed partially
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
## What this package does NOT do
|
|
215
|
+
|
|
216
|
+
- **No concrete loaders** — This is the interface only. Actual file reading
|
|
217
|
+
(HDF5, VTK, etc.) lives in separate loader plugin packages.
|
|
218
|
+
- **No data transformation** — Data is returned as-is from the loader.
|
|
219
|
+
- **No caching** — Caching strategies belong at the application level.
|
|
220
|
+
|
|
221
|
+
## Further Reading
|
|
222
|
+
|
|
223
|
+
- [Common Patterns](docs/patterns.md) — Validator/setup implementation,
|
|
224
|
+
multi-loader registration, error handling, and naming conventions.
|
|
225
|
+
- [Design & Concepts](docs/design.md) — Architecture, protocol rationale,
|
|
226
|
+
and package boundaries.
|
|
227
|
+
|
|
228
|
+
## Dependencies
|
|
229
|
+
|
|
230
|
+
- [numpy](https://numpy.org/) (>=1.24)
|
|
231
|
+
- [vcti-plugin-catalog](https://pypi.org/project/vcti-plugin-catalog/) (>=1.0.0) — Descriptor and Registry base classes
|
|
232
|
+
- [vcti-array-tree](https://pypi.org/project/vcti-array-tree/) (>=1.0.0) — `DataNode` returned by `load_dataset`
|
|
233
|
+
|
|
234
|
+
## Versioning
|
|
235
|
+
|
|
236
|
+
This package follows [Semantic Versioning](https://semver.org/). Breaking
|
|
237
|
+
changes to the `Loader` protocol (adding required methods, changing
|
|
238
|
+
signatures) will only occur in major version bumps. Downstream loader
|
|
239
|
+
plugins should pin to a compatible major version (e.g., `vcti-fileloader>=1.0,<2`).
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "vcti-fileloader"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "File loader framework with pluggable descriptors, validators, and a registry for format-specific data loading"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
authors = [
|
|
11
|
+
{name = "Visual Collaboration Technologies Inc."}
|
|
12
|
+
]
|
|
13
|
+
license = {text = "Proprietary"}
|
|
14
|
+
requires-python = ">=3.12,<3.15"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 5 - Production/Stable",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: Other/Proprietary License",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Programming Language :: Python :: 3.14",
|
|
22
|
+
"Typing :: Typed",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
dependencies = [
|
|
26
|
+
"numpy>=1.24",
|
|
27
|
+
"vcti-plugin-catalog>=1.0.0",
|
|
28
|
+
"vcti-array-tree>=1.0.0",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/vcollab/vcti-python-fileloader"
|
|
33
|
+
Repository = "https://github.com/vcollab/vcti-python-fileloader"
|
|
34
|
+
Issues = "https://github.com/vcollab/vcti-python-fileloader/issues"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
test = ["pytest", "pytest-cov"]
|
|
38
|
+
lint = ["ruff"]
|
|
39
|
+
typecheck = ["mypy"]
|
|
40
|
+
|
|
41
|
+
[tool.setuptools.packages.find]
|
|
42
|
+
where = ["src"]
|
|
43
|
+
include = ["vcti.fileloader", "vcti.fileloader.*"]
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.package-data]
|
|
46
|
+
"vcti.fileloader" = ["py.typed"]
|
|
47
|
+
|
|
48
|
+
[tool.setuptools]
|
|
49
|
+
zip-safe = true
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
addopts = "--cov=vcti.fileloader --cov-report=term-missing --cov-fail-under=95"
|
|
53
|
+
|
|
54
|
+
[tool.ruff]
|
|
55
|
+
target-version = "py312"
|
|
56
|
+
line-length = 99
|
|
57
|
+
|
|
58
|
+
[tool.ruff.lint]
|
|
59
|
+
select = ["E", "F", "W", "I", "UP"]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
|
|
2
|
+
# See LICENSE for details.
|
|
3
|
+
"""vcti.fileloader — File loader framework with pluggable descriptors and registry."""
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
|
|
7
|
+
from .descriptor import LoaderDescriptor
|
|
8
|
+
from .loader import (
|
|
9
|
+
Loader,
|
|
10
|
+
LoaderError,
|
|
11
|
+
LoadError,
|
|
12
|
+
NodeID,
|
|
13
|
+
SetupError,
|
|
14
|
+
UnloadError,
|
|
15
|
+
UnsupportedFormatError,
|
|
16
|
+
ValidationError,
|
|
17
|
+
)
|
|
18
|
+
from .registry import LoaderRegistry
|
|
19
|
+
from .setup import LoaderSetup
|
|
20
|
+
from .validator import LoaderValidator
|
|
21
|
+
|
|
22
|
+
__version__ = version("vcti-fileloader")
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"__version__",
|
|
26
|
+
"LoadError",
|
|
27
|
+
"Loader",
|
|
28
|
+
"LoaderDescriptor",
|
|
29
|
+
"LoaderError",
|
|
30
|
+
"LoaderRegistry",
|
|
31
|
+
"LoaderSetup",
|
|
32
|
+
"LoaderValidator",
|
|
33
|
+
"NodeID",
|
|
34
|
+
"SetupError",
|
|
35
|
+
"UnloadError",
|
|
36
|
+
"UnsupportedFormatError",
|
|
37
|
+
"ValidationError",
|
|
38
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright Visual Collaboration Technologies Inc. All Rights Reserved.
|
|
2
|
+
# See LICENSE for details.
|
|
3
|
+
"""Loader descriptor — metadata + loader instance for the registry."""
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from vcti.plugincatalog import Descriptor
|
|
8
|
+
|
|
9
|
+
from .loader import Loader
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LoaderDescriptor(Descriptor[Loader]):
|
|
13
|
+
"""Descriptor pairing a Loader with metadata for the registry.
|
|
14
|
+
|
|
15
|
+
Extends Descriptor[Loader] from vcti-plugin-catalog. The `instance`
|
|
16
|
+
attribute holds the Loader; `attributes` holds filterable metadata
|
|
17
|
+
like supported formats.
|
|
18
|
+
|
|
19
|
+
Example:
|
|
20
|
+
>>> desc = LoaderDescriptor(
|
|
21
|
+
... id="hdf5-h5py-loader",
|
|
22
|
+
... name="HDF5 Loader (h5py)",
|
|
23
|
+
... loader=H5pyLoader(),
|
|
24
|
+
... attributes={"supported_formats": ["hdf5-file"]},
|
|
25
|
+
... )
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
id: str,
|
|
31
|
+
name: str,
|
|
32
|
+
loader: Loader,
|
|
33
|
+
description: str | None = None,
|
|
34
|
+
attributes: dict[str, Any] | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
super().__init__(
|
|
37
|
+
id=id,
|
|
38
|
+
name=name,
|
|
39
|
+
instance=loader,
|
|
40
|
+
description=description,
|
|
41
|
+
attributes=attributes,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def loader(self) -> Loader:
|
|
46
|
+
"""Get the loader instance."""
|
|
47
|
+
return self.instance
|
|
48
|
+
|
|
49
|
+
def __repr__(self) -> str:
|
|
50
|
+
return f"<LoaderDescriptor id={self.id!r} name={self.name!r}>"
|