mx8fs 2.0.0.43__tar.gz → 2.0.0.45__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/PKG-INFO +150 -1
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/README.md +149 -0
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/__init__.py +7 -0
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/cache.py +5 -3
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/comparer.py +14 -11
- mx8fs-2.0.0.45/mx8fs/file_io.py +1457 -0
- mx8fs-2.0.0.45/mx8fs/handler.py +424 -0
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/indexed_storage.py +34 -24
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/lock.py +3 -3
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/storage.py +71 -55
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/pyproject.toml +1 -1
- mx8fs-2.0.0.43/mx8fs/file_io.py +0 -1198
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/cli.py +0 -0
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/index.py +0 -0
- {mx8fs-2.0.0.43 → mx8fs-2.0.0.45}/mx8fs/migration.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: mx8fs
|
|
3
|
-
Version: 2.0.0.
|
|
3
|
+
Version: 2.0.0.45
|
|
4
4
|
Summary: MX8 Filing system
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Tom Weiss
|
|
@@ -305,6 +305,114 @@ current state is captured before the next mx8fs mutation.
|
|
|
305
305
|
blob = f.read()
|
|
306
306
|
```
|
|
307
307
|
|
|
308
|
+
Both handlers accept keyword-only `versioned=False`, `version_id=None` and `expected_version=None`.
|
|
309
|
+
Use `versioned=True` to track the current read or successful write in `handler.version`. Historical
|
|
310
|
+
`version_id` reads and conditional `expected_version` writes enable version tracking automatically.
|
|
311
|
+
`version_id` also accepts a listed `FileVersionMetadata` object to reuse its metadata without another lookup.
|
|
312
|
+
The revision is the S3 ETag or local nanosecond mtime; HTTPS current reads have no revision.
|
|
313
|
+
|
|
314
|
+
```python
|
|
315
|
+
handler = GzipFileHandler(path, "rt", encoding="utf-8", versioned=True)
|
|
316
|
+
with handler as source:
|
|
317
|
+
text = source.read()
|
|
318
|
+
|
|
319
|
+
writer = GzipFileHandler(path, "wt", encoding="utf-8", expected_version=handler.version)
|
|
320
|
+
with writer as destination:
|
|
321
|
+
destination.write(text)
|
|
322
|
+
new_version = writer.version
|
|
323
|
+
```
|
|
324
|
+
|
|
325
|
+
Versioned writes buffer until successful context exit. Gzip finishes before the underlying binary handler
|
|
326
|
+
publishes, so a body or compression-finalization failure leaves the destination untouched. Local versioned
|
|
327
|
+
writes preserve local history and publish with atomic replacement. Conditional writes use the existing
|
|
328
|
+
mtime checks locally and S3 `IfMatch`; local compare-and-replace retains the concurrency limitation described
|
|
329
|
+
above. Versioned S3 writes use `PutObject` and its single-request size limit; default stream writes retain
|
|
330
|
+
managed uploads. `GzipFileHandler` also accepts `compression_level=9` (0–9), preserving its existing default.
|
|
331
|
+
|
|
332
|
+
`JsonFileHandler(path, mode="r", *, compression=None, compression_level=1, version_id=None,
|
|
333
|
+
expected_version=None, decoder=json.loads, encoder=json.dumps)` wraps these handlers for JSON documents.
|
|
334
|
+
It supports `r` and `w`, returns itself from the context, and exposes `read()`, `write(value)` and `.version`.
|
|
335
|
+
Compression accepts `None` or `"gzip"`; the supplied path is used unchanged. JSON I/O always enables version
|
|
336
|
+
tracking and successful-exit publication, including when compression is disabled. Historical version IDs
|
|
337
|
+
and listed `FileVersionMetadata` objects work in the same way as the binary and gzip handlers.
|
|
338
|
+
|
|
339
|
+
```python
|
|
340
|
+
from mx8fs import JsonFileHandler
|
|
341
|
+
|
|
342
|
+
reader = JsonFileHandler("/tmp/settings.json.gz", compression="gzip")
|
|
343
|
+
with reader as source:
|
|
344
|
+
settings = source.read()
|
|
345
|
+
|
|
346
|
+
with JsonFileHandler(
|
|
347
|
+
"/tmp/settings.json.gz", "w", compression="gzip", expected_version=reader.version
|
|
348
|
+
) as destination:
|
|
349
|
+
destination.write(settings)
|
|
350
|
+
new_version = destination.version
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
The decoder receives UTF-8 JSON text; the encoder returns JSON text. Supplying model conversion functions
|
|
354
|
+
preserves custom validation and serialization without adding a Pydantic dependency to the handler.
|
|
355
|
+
`JsonFileStorage` supplies its existing model hooks through `FileHandler`, which owns logical filename
|
|
356
|
+
selection, fallback, history and version routing. The storage classes do not implement those file policies.
|
|
357
|
+
|
|
358
|
+
`TextFileHandler(path, mode="r", *, encoding="utf-8", newline="auto", compression=None, compression_level=1,
|
|
359
|
+
version_id=None, expected_version=None)` owns text conversion. Its `read(size=-1)` and `write(text)` methods
|
|
360
|
+
use `TextIOWrapper` for encoding and newline handling. It returns itself from its context and exposes
|
|
361
|
+
`.version`. Text is flushed before gzip finalization and binary publication. `JsonFileHandler` delegates
|
|
362
|
+
text I/O to this handler, preserving JSON text without newline translation.
|
|
363
|
+
|
|
364
|
+
The default `newline="auto"` preserves the existing text-read behaviour: local reads normalize
|
|
365
|
+
line endings to `\n`, while S3 and HTTPS preserve their original line endings. Explicit `newline=None`
|
|
366
|
+
normalizes on every backend; `newline=""` preserves line endings on every backend. The text handler
|
|
367
|
+
owns this policy, including reads through the legacy helpers.
|
|
368
|
+
|
|
369
|
+
### Typed file operations
|
|
370
|
+
|
|
371
|
+
`FileHandler(path, data_type, *, compression=None, compression_level=1, fallback_uncompressed=False,
|
|
372
|
+
encoding="utf-8", newline="auto", decoder=None, encoder=None)` is the complete logical-file API.
|
|
373
|
+
Use `bytes` for binary data, `str` for text, and JSON types such as `dict` or `list` for JSON values.
|
|
374
|
+
Pydantic model types use their JSON validation and serialization methods; custom codecs can override them
|
|
375
|
+
without adding a Pydantic dependency to the handler.
|
|
376
|
+
|
|
377
|
+
The path is logical: `FileHandler("/tmp/settings.json", dict, compression="gzip")` writes
|
|
378
|
+
`/tmp/settings.json.gz`. `primary_path` exposes that physical destination. Low-level binary, gzip, text and
|
|
379
|
+
JSON stream handlers continue to use their supplied physical paths unchanged.
|
|
380
|
+
|
|
381
|
+
```python
|
|
382
|
+
from mx8fs import FileHandler
|
|
383
|
+
|
|
384
|
+
file = FileHandler("/tmp/settings.json", data_type=dict, compression="gzip", fallback_uncompressed=True)
|
|
385
|
+
settings, revision = file.read_with_version()
|
|
386
|
+
file.write(settings, expected_version=revision)
|
|
387
|
+
metadata = file.metadata()
|
|
388
|
+
versions = file.history()
|
|
389
|
+
old_settings = file.read(version_id=versions[-1])
|
|
390
|
+
|
|
391
|
+
folder = FileHandler("/tmp", data_type=dict, compression="gzip", fallback_uncompressed=True)
|
|
392
|
+
keys = folder.list("json")
|
|
393
|
+
files = folder.list_with_metadata("json", include_deleted=True)
|
|
394
|
+
child = folder.joinpath("settings.json")
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
The handler provides `read(version_id=None)`, `read_with_version(version_id=None)`,
|
|
398
|
+
`write(value, expected_version=None)`, `exists()`, `delete()`, `current_version()`,
|
|
399
|
+
`metadata(include_deleted=False)`, `history()`, `restore_version(version_id)`, `undelete()` and `get_lock()`.
|
|
400
|
+
Version IDs are keyword-only on reads and accept historical ID strings or listed `FileVersionMetadata`.
|
|
401
|
+
`write()` returns the published revision; `.version` records the revision of the last opened or published
|
|
402
|
+
content. HTTPS supports current reads and existence checks, without concurrency versions or portable metadata.
|
|
403
|
+
|
|
404
|
+
Directory listings accept `prefix=""` and return extension-free logical keys. The prefix is passed to
|
|
405
|
+
the backend to limit S3 listing work. Single-file metadata uses the logical filename,
|
|
406
|
+
including its data extension. Gzip/plain listings and history are merged by the handler, with opaque
|
|
407
|
+
source-aware tokens. Fallback happens only while opening an absent file or historical version, before
|
|
408
|
+
JSON or model decoding. Normal gzip writes publish before retiring plain current files; conditional
|
|
409
|
+
updates retain the representation that was read.
|
|
410
|
+
|
|
411
|
+
`open_read()` yields a `FileReader` containing a deferred `read()` and the revision of those exact bytes.
|
|
412
|
+
Indexed storage uses it to retain the source revision even when model validation fails. File metadata,
|
|
413
|
+
version parsing and backend operations remain inside the handler; storage owns keys and models, while
|
|
414
|
+
indexed storage owns SQL projections and reconciliation.
|
|
415
|
+
|
|
308
416
|
### Locking
|
|
309
417
|
|
|
310
418
|
- `class Waiter(wait_period: float, time_out_seconds: float)`: Helper for periodic waits with a timeout.
|
|
@@ -403,6 +511,47 @@ Install JSON storage dependencies with `pip install "mx8fs[json-storage]"`.
|
|
|
403
511
|
store.delete(u.key)
|
|
404
512
|
```
|
|
405
513
|
|
|
514
|
+
Optional gzip storage reuses these handlers:
|
|
515
|
+
|
|
516
|
+
```python
|
|
517
|
+
UserStorage = json_file_storage_factory(
|
|
518
|
+
extension="json", model=User, key_field="key",
|
|
519
|
+
compression="gzip", compression_level=1, fallback_uncompressed=True,
|
|
520
|
+
)
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
The factory configures `FileHandler` for compression and fallback, using the same `JsonFileStorage` class
|
|
524
|
+
with or without gzip. Indexed storage uses that same handler API. Its read fallback catches
|
|
525
|
+
`FileNotFoundError` while opening gzip and then opens the uncompressed filename; decoding and validation
|
|
526
|
+
failures propagate. A separate gzip storage subclass is not required.
|
|
527
|
+
|
|
528
|
+
Defaults remain uncompressed, with plain fallback disabled. Gzip mode appends `.gz` to the extension and
|
|
529
|
+
uses compression level 1 unless configured otherwise. With `fallback_uncompressed=True`, reads prefer gzip
|
|
530
|
+
and fall back only on absence; corrupt gzip, invalid JSON and permission errors propagate. Listings
|
|
531
|
+
deduplicate logical keys, key allocation checks both names, and deletion removes both current representations.
|
|
532
|
+
Normal writes finish publishing gzip before deleting the plain current object.
|
|
533
|
+
|
|
534
|
+
Local Unix file modes are not configurable through the storage API. Creating a gzip destination uses
|
|
535
|
+
normal creation permissions and the process umask; it does not inherit the plain file's mode. Under
|
|
536
|
+
umask `022`, migrating a `0600` plain file can therefore create a `0644` gzip file. Use a restrictive
|
|
537
|
+
process umask when creating private local records. An existing gzip destination retains its own mode.
|
|
538
|
+
|
|
539
|
+
Gzip-mode concurrency tokens identify the representation with `gzip:` or `plain:`. Pass them unchanged to
|
|
540
|
+
`update_if_version()`. Conditional mutations retain the representation read: a plain mutation stays plain
|
|
541
|
+
until a normal save. A concurrent gzip publication causes a plain mutation to report a version conflict so
|
|
542
|
+
`mutate()` can reread and recompute. Normal saves and deletes are not cross-object transactions; coordinate
|
|
543
|
+
them with `get_lock()` when the application needs serialization.
|
|
544
|
+
|
|
545
|
+
Locks use the logical filename so plain and gzip callers serialize access to the same record.
|
|
546
|
+
With local history enabled, migration captures the plain state before publishing gzip so recovery
|
|
547
|
+
selects the newly saved value.
|
|
548
|
+
|
|
549
|
+
History combines both filenames and supplies tagged `version_id` values for `read_version()` and
|
|
550
|
+
`restore_version()`. Deleted listing metadata includes a tagged `latest_readable_version` when available.
|
|
551
|
+
Existing untagged historical IDs are accepted, trying gzip then plain only if that version is absent.
|
|
552
|
+
Restoring or undeleting writes a new gzip current version. Indexed storage supports the same factory options;
|
|
553
|
+
run `rebuild_index()` when switching an existing indexed namespace to gzip to refresh its source versions.
|
|
554
|
+
|
|
406
555
|
### Indexed JSON Storage
|
|
407
556
|
|
|
408
557
|
`json_index_factory` adds a SQL secondary index while keeping JSON files canonical. Local storage uses a hidden SQLite database at `<base_path>/.mx8fs-index.sqlite3`; indexed S3 storage uses Aurora DSQL.
|
|
@@ -281,6 +281,114 @@ current state is captured before the next mx8fs mutation.
|
|
|
281
281
|
blob = f.read()
|
|
282
282
|
```
|
|
283
283
|
|
|
284
|
+
Both handlers accept keyword-only `versioned=False`, `version_id=None` and `expected_version=None`.
|
|
285
|
+
Use `versioned=True` to track the current read or successful write in `handler.version`. Historical
|
|
286
|
+
`version_id` reads and conditional `expected_version` writes enable version tracking automatically.
|
|
287
|
+
`version_id` also accepts a listed `FileVersionMetadata` object to reuse its metadata without another lookup.
|
|
288
|
+
The revision is the S3 ETag or local nanosecond mtime; HTTPS current reads have no revision.
|
|
289
|
+
|
|
290
|
+
```python
|
|
291
|
+
handler = GzipFileHandler(path, "rt", encoding="utf-8", versioned=True)
|
|
292
|
+
with handler as source:
|
|
293
|
+
text = source.read()
|
|
294
|
+
|
|
295
|
+
writer = GzipFileHandler(path, "wt", encoding="utf-8", expected_version=handler.version)
|
|
296
|
+
with writer as destination:
|
|
297
|
+
destination.write(text)
|
|
298
|
+
new_version = writer.version
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Versioned writes buffer until successful context exit. Gzip finishes before the underlying binary handler
|
|
302
|
+
publishes, so a body or compression-finalization failure leaves the destination untouched. Local versioned
|
|
303
|
+
writes preserve local history and publish with atomic replacement. Conditional writes use the existing
|
|
304
|
+
mtime checks locally and S3 `IfMatch`; local compare-and-replace retains the concurrency limitation described
|
|
305
|
+
above. Versioned S3 writes use `PutObject` and its single-request size limit; default stream writes retain
|
|
306
|
+
managed uploads. `GzipFileHandler` also accepts `compression_level=9` (0–9), preserving its existing default.
|
|
307
|
+
|
|
308
|
+
`JsonFileHandler(path, mode="r", *, compression=None, compression_level=1, version_id=None,
|
|
309
|
+
expected_version=None, decoder=json.loads, encoder=json.dumps)` wraps these handlers for JSON documents.
|
|
310
|
+
It supports `r` and `w`, returns itself from the context, and exposes `read()`, `write(value)` and `.version`.
|
|
311
|
+
Compression accepts `None` or `"gzip"`; the supplied path is used unchanged. JSON I/O always enables version
|
|
312
|
+
tracking and successful-exit publication, including when compression is disabled. Historical version IDs
|
|
313
|
+
and listed `FileVersionMetadata` objects work in the same way as the binary and gzip handlers.
|
|
314
|
+
|
|
315
|
+
```python
|
|
316
|
+
from mx8fs import JsonFileHandler
|
|
317
|
+
|
|
318
|
+
reader = JsonFileHandler("/tmp/settings.json.gz", compression="gzip")
|
|
319
|
+
with reader as source:
|
|
320
|
+
settings = source.read()
|
|
321
|
+
|
|
322
|
+
with JsonFileHandler(
|
|
323
|
+
"/tmp/settings.json.gz", "w", compression="gzip", expected_version=reader.version
|
|
324
|
+
) as destination:
|
|
325
|
+
destination.write(settings)
|
|
326
|
+
new_version = destination.version
|
|
327
|
+
```
|
|
328
|
+
|
|
329
|
+
The decoder receives UTF-8 JSON text; the encoder returns JSON text. Supplying model conversion functions
|
|
330
|
+
preserves custom validation and serialization without adding a Pydantic dependency to the handler.
|
|
331
|
+
`JsonFileStorage` supplies its existing model hooks through `FileHandler`, which owns logical filename
|
|
332
|
+
selection, fallback, history and version routing. The storage classes do not implement those file policies.
|
|
333
|
+
|
|
334
|
+
`TextFileHandler(path, mode="r", *, encoding="utf-8", newline="auto", compression=None, compression_level=1,
|
|
335
|
+
version_id=None, expected_version=None)` owns text conversion. Its `read(size=-1)` and `write(text)` methods
|
|
336
|
+
use `TextIOWrapper` for encoding and newline handling. It returns itself from its context and exposes
|
|
337
|
+
`.version`. Text is flushed before gzip finalization and binary publication. `JsonFileHandler` delegates
|
|
338
|
+
text I/O to this handler, preserving JSON text without newline translation.
|
|
339
|
+
|
|
340
|
+
The default `newline="auto"` preserves the existing text-read behaviour: local reads normalize
|
|
341
|
+
line endings to `\n`, while S3 and HTTPS preserve their original line endings. Explicit `newline=None`
|
|
342
|
+
normalizes on every backend; `newline=""` preserves line endings on every backend. The text handler
|
|
343
|
+
owns this policy, including reads through the legacy helpers.
|
|
344
|
+
|
|
345
|
+
### Typed file operations
|
|
346
|
+
|
|
347
|
+
`FileHandler(path, data_type, *, compression=None, compression_level=1, fallback_uncompressed=False,
|
|
348
|
+
encoding="utf-8", newline="auto", decoder=None, encoder=None)` is the complete logical-file API.
|
|
349
|
+
Use `bytes` for binary data, `str` for text, and JSON types such as `dict` or `list` for JSON values.
|
|
350
|
+
Pydantic model types use their JSON validation and serialization methods; custom codecs can override them
|
|
351
|
+
without adding a Pydantic dependency to the handler.
|
|
352
|
+
|
|
353
|
+
The path is logical: `FileHandler("/tmp/settings.json", dict, compression="gzip")` writes
|
|
354
|
+
`/tmp/settings.json.gz`. `primary_path` exposes that physical destination. Low-level binary, gzip, text and
|
|
355
|
+
JSON stream handlers continue to use their supplied physical paths unchanged.
|
|
356
|
+
|
|
357
|
+
```python
|
|
358
|
+
from mx8fs import FileHandler
|
|
359
|
+
|
|
360
|
+
file = FileHandler("/tmp/settings.json", data_type=dict, compression="gzip", fallback_uncompressed=True)
|
|
361
|
+
settings, revision = file.read_with_version()
|
|
362
|
+
file.write(settings, expected_version=revision)
|
|
363
|
+
metadata = file.metadata()
|
|
364
|
+
versions = file.history()
|
|
365
|
+
old_settings = file.read(version_id=versions[-1])
|
|
366
|
+
|
|
367
|
+
folder = FileHandler("/tmp", data_type=dict, compression="gzip", fallback_uncompressed=True)
|
|
368
|
+
keys = folder.list("json")
|
|
369
|
+
files = folder.list_with_metadata("json", include_deleted=True)
|
|
370
|
+
child = folder.joinpath("settings.json")
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
The handler provides `read(version_id=None)`, `read_with_version(version_id=None)`,
|
|
374
|
+
`write(value, expected_version=None)`, `exists()`, `delete()`, `current_version()`,
|
|
375
|
+
`metadata(include_deleted=False)`, `history()`, `restore_version(version_id)`, `undelete()` and `get_lock()`.
|
|
376
|
+
Version IDs are keyword-only on reads and accept historical ID strings or listed `FileVersionMetadata`.
|
|
377
|
+
`write()` returns the published revision; `.version` records the revision of the last opened or published
|
|
378
|
+
content. HTTPS supports current reads and existence checks, without concurrency versions or portable metadata.
|
|
379
|
+
|
|
380
|
+
Directory listings accept `prefix=""` and return extension-free logical keys. The prefix is passed to
|
|
381
|
+
the backend to limit S3 listing work. Single-file metadata uses the logical filename,
|
|
382
|
+
including its data extension. Gzip/plain listings and history are merged by the handler, with opaque
|
|
383
|
+
source-aware tokens. Fallback happens only while opening an absent file or historical version, before
|
|
384
|
+
JSON or model decoding. Normal gzip writes publish before retiring plain current files; conditional
|
|
385
|
+
updates retain the representation that was read.
|
|
386
|
+
|
|
387
|
+
`open_read()` yields a `FileReader` containing a deferred `read()` and the revision of those exact bytes.
|
|
388
|
+
Indexed storage uses it to retain the source revision even when model validation fails. File metadata,
|
|
389
|
+
version parsing and backend operations remain inside the handler; storage owns keys and models, while
|
|
390
|
+
indexed storage owns SQL projections and reconciliation.
|
|
391
|
+
|
|
284
392
|
### Locking
|
|
285
393
|
|
|
286
394
|
- `class Waiter(wait_period: float, time_out_seconds: float)`: Helper for periodic waits with a timeout.
|
|
@@ -379,6 +487,47 @@ Install JSON storage dependencies with `pip install "mx8fs[json-storage]"`.
|
|
|
379
487
|
store.delete(u.key)
|
|
380
488
|
```
|
|
381
489
|
|
|
490
|
+
Optional gzip storage reuses these handlers:
|
|
491
|
+
|
|
492
|
+
```python
|
|
493
|
+
UserStorage = json_file_storage_factory(
|
|
494
|
+
extension="json", model=User, key_field="key",
|
|
495
|
+
compression="gzip", compression_level=1, fallback_uncompressed=True,
|
|
496
|
+
)
|
|
497
|
+
```
|
|
498
|
+
|
|
499
|
+
The factory configures `FileHandler` for compression and fallback, using the same `JsonFileStorage` class
|
|
500
|
+
with or without gzip. Indexed storage uses that same handler API. Its read fallback catches
|
|
501
|
+
`FileNotFoundError` while opening gzip and then opens the uncompressed filename; decoding and validation
|
|
502
|
+
failures propagate. A separate gzip storage subclass is not required.
|
|
503
|
+
|
|
504
|
+
Defaults remain uncompressed, with plain fallback disabled. Gzip mode appends `.gz` to the extension and
|
|
505
|
+
uses compression level 1 unless configured otherwise. With `fallback_uncompressed=True`, reads prefer gzip
|
|
506
|
+
and fall back only on absence; corrupt gzip, invalid JSON and permission errors propagate. Listings
|
|
507
|
+
deduplicate logical keys, key allocation checks both names, and deletion removes both current representations.
|
|
508
|
+
Normal writes finish publishing gzip before deleting the plain current object.
|
|
509
|
+
|
|
510
|
+
Local Unix file modes are not configurable through the storage API. Creating a gzip destination uses
|
|
511
|
+
normal creation permissions and the process umask; it does not inherit the plain file's mode. Under
|
|
512
|
+
umask `022`, migrating a `0600` plain file can therefore create a `0644` gzip file. Use a restrictive
|
|
513
|
+
process umask when creating private local records. An existing gzip destination retains its own mode.
|
|
514
|
+
|
|
515
|
+
Gzip-mode concurrency tokens identify the representation with `gzip:` or `plain:`. Pass them unchanged to
|
|
516
|
+
`update_if_version()`. Conditional mutations retain the representation read: a plain mutation stays plain
|
|
517
|
+
until a normal save. A concurrent gzip publication causes a plain mutation to report a version conflict so
|
|
518
|
+
`mutate()` can reread and recompute. Normal saves and deletes are not cross-object transactions; coordinate
|
|
519
|
+
them with `get_lock()` when the application needs serialization.
|
|
520
|
+
|
|
521
|
+
Locks use the logical filename so plain and gzip callers serialize access to the same record.
|
|
522
|
+
With local history enabled, migration captures the plain state before publishing gzip so recovery
|
|
523
|
+
selects the newly saved value.
|
|
524
|
+
|
|
525
|
+
History combines both filenames and supplies tagged `version_id` values for `read_version()` and
|
|
526
|
+
`restore_version()`. Deleted listing metadata includes a tagged `latest_readable_version` when available.
|
|
527
|
+
Existing untagged historical IDs are accepted, trying gzip then plain only if that version is absent.
|
|
528
|
+
Restoring or undeleting writes a new gzip current version. Indexed storage supports the same factory options;
|
|
529
|
+
run `rebuild_index()` when switching an existing indexed namespace to gzip to refresh its source versions.
|
|
530
|
+
|
|
382
531
|
### Indexed JSON Storage
|
|
383
532
|
|
|
384
533
|
`json_index_factory` adds a SQL secondary index while keeping JSON files canonical. Local storage uses a hidden SQLite database at `<base_path>/.mx8fs-index.sqlite3`; indexed S3 storage uses Aurora DSQL.
|
|
@@ -21,6 +21,8 @@ from .file_io import (
|
|
|
21
21
|
FileVersionDeletedError,
|
|
22
22
|
FileVersionMetadata,
|
|
23
23
|
GzipFileHandler,
|
|
24
|
+
JsonFileHandler,
|
|
25
|
+
TextFileHandler,
|
|
24
26
|
VersioningNotEnabledError,
|
|
25
27
|
VersioningNotSupportedError,
|
|
26
28
|
VersionMismatchError,
|
|
@@ -46,6 +48,7 @@ from .file_io import (
|
|
|
46
48
|
update_file_if_version_matches,
|
|
47
49
|
write_file,
|
|
48
50
|
)
|
|
51
|
+
from .handler import FileHandler, FileReader
|
|
49
52
|
from .lock import FileLock, Waiter
|
|
50
53
|
from .storage import JsonFileStorage, json_file_storage_factory
|
|
51
54
|
|
|
@@ -111,6 +114,7 @@ def __getattr__(name: str) -> Any:
|
|
|
111
114
|
__all__ = [
|
|
112
115
|
"BinaryFileHandler",
|
|
113
116
|
"GzipFileHandler",
|
|
117
|
+
"JsonFileHandler",
|
|
114
118
|
"cache_to_disk_binary",
|
|
115
119
|
"cache_to_disk",
|
|
116
120
|
"copy_file",
|
|
@@ -128,6 +132,9 @@ __all__ = [
|
|
|
128
132
|
"get_folders",
|
|
129
133
|
"json_file_storage_factory",
|
|
130
134
|
"JsonFileStorage",
|
|
135
|
+
"FileHandler",
|
|
136
|
+
"FileReader",
|
|
137
|
+
"TextFileHandler",
|
|
131
138
|
"list_files",
|
|
132
139
|
"list_file_versions",
|
|
133
140
|
"list_files_with_metadata",
|
|
@@ -26,7 +26,8 @@ from json.decoder import JSONDecodeError
|
|
|
26
26
|
from time import time
|
|
27
27
|
from typing import Any
|
|
28
28
|
|
|
29
|
-
from .file_io import BinaryFileHandler
|
|
29
|
+
from .file_io import BinaryFileHandler
|
|
30
|
+
from .handler import FileHandler
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
def get_cache_filename(path: str, name: str, extension: str, expiration_seconds: int = 0, **kwargs: dict) -> str:
|
|
@@ -207,9 +208,10 @@ def cache_to_disk(
|
|
|
207
208
|
**clean_kwargs,
|
|
208
209
|
)
|
|
209
210
|
|
|
211
|
+
cache_file = FileHandler(filename, str)
|
|
210
212
|
try:
|
|
211
213
|
# Try to read the cached result from disk
|
|
212
|
-
result =
|
|
214
|
+
result = cache_file.read()
|
|
213
215
|
|
|
214
216
|
_do_logging(
|
|
215
217
|
log_group, "Cache hit", result, args, clean_kwargs, filename, func, expiration_seconds, level
|
|
@@ -221,7 +223,7 @@ def cache_to_disk(
|
|
|
221
223
|
log_group, "Cache miss", None, args, clean_kwargs, filename, func, expiration_seconds, level
|
|
222
224
|
)
|
|
223
225
|
result = func(*args, **kwargs)
|
|
224
|
-
|
|
226
|
+
cache_file.write(result)
|
|
225
227
|
|
|
226
228
|
return result
|
|
227
229
|
|
|
@@ -26,7 +26,7 @@ from logging import getLogger
|
|
|
26
26
|
from tempfile import NamedTemporaryFile
|
|
27
27
|
from typing import Any
|
|
28
28
|
|
|
29
|
-
from
|
|
29
|
+
from .handler import FileHandler
|
|
30
30
|
|
|
31
31
|
logger = getLogger("mx8.comparer")
|
|
32
32
|
|
|
@@ -179,15 +179,14 @@ class ResultsComparer:
|
|
|
179
179
|
differences = Differences()
|
|
180
180
|
|
|
181
181
|
# Obfuscate the test file content before diffing
|
|
182
|
-
test_content =
|
|
182
|
+
test_content = FileHandler(test, str).read()
|
|
183
183
|
obfuscated_test_content = self._obfuscate_text(test_content)
|
|
184
|
+
correct_file = FileHandler(correct, str)
|
|
184
185
|
|
|
185
186
|
if self._create_test_data:
|
|
186
|
-
|
|
187
|
-
os.makedirs(os.path.dirname(correct), exist_ok=True)
|
|
188
|
-
write_file(correct, obfuscated_test_content)
|
|
187
|
+
correct_file.write(obfuscated_test_content)
|
|
189
188
|
|
|
190
|
-
correct_content =
|
|
189
|
+
correct_content = correct_file.read()
|
|
191
190
|
|
|
192
191
|
if diff := get_diff(correct_content, obfuscated_test_content):
|
|
193
192
|
differences.append({"file": diff})
|
|
@@ -197,22 +196,26 @@ class ResultsComparer:
|
|
|
197
196
|
def get_dict_differences(self, test: str, correct: str) -> Differences:
|
|
198
197
|
"""Compare a test file with a correct file."""
|
|
199
198
|
# Load and obfuscate the test file
|
|
200
|
-
test_dict =
|
|
199
|
+
test_dict = FileHandler(test, object).read()
|
|
201
200
|
obfuscated_test_dict = self._obfuscate_dict(test_dict)
|
|
202
201
|
|
|
202
|
+
correct_file = FileHandler(
|
|
203
|
+
correct,
|
|
204
|
+
object,
|
|
205
|
+
encoder=lambda value: json.dumps(value, indent=4, ensure_ascii=False).strip(),
|
|
206
|
+
)
|
|
203
207
|
self._differences.clear()
|
|
204
208
|
if self._create_test_data:
|
|
205
209
|
try:
|
|
206
|
-
correct_dict =
|
|
210
|
+
correct_dict = correct_file.read()
|
|
207
211
|
self._compare_dicts(correct_dict, obfuscated_test_dict)
|
|
208
212
|
assert self._differences == [], "The files should be identical"
|
|
209
213
|
except (FileNotFoundError, AssertionError):
|
|
210
214
|
# Save the obfuscated test file as the correct file
|
|
211
|
-
|
|
212
|
-
write_file(correct, json.dumps(obfuscated_test_dict, indent=4, ensure_ascii=False).strip())
|
|
215
|
+
correct_file.write(obfuscated_test_dict)
|
|
213
216
|
self._differences.clear()
|
|
214
217
|
else:
|
|
215
|
-
correct_dict =
|
|
218
|
+
correct_dict = correct_file.read()
|
|
216
219
|
self._compare_dicts(correct_dict, obfuscated_test_dict)
|
|
217
220
|
|
|
218
221
|
return self._differences
|