umgodi-json 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- umgodi_json-0.1.0/.gitignore +50 -0
- umgodi_json-0.1.0/API.md +239 -0
- umgodi_json-0.1.0/GettingStarted.md +280 -0
- umgodi_json-0.1.0/PKG-INFO +274 -0
- umgodi_json-0.1.0/README.md +249 -0
- umgodi_json-0.1.0/assets/jsonpit-sync-logo.svg +132 -0
- umgodi_json-0.1.0/assets/logo-128x128.png +0 -0
- umgodi_json-0.1.0/assets/logo-192x192.png +0 -0
- umgodi_json-0.1.0/assets/logo-512x512.png +0 -0
- umgodi_json-0.1.0/assets/logo.png +0 -0
- umgodi_json-0.1.0/assets/logo.svg +132 -0
- umgodi_json-0.1.0/assets/umgodi-json-sync-banner.svg +125 -0
- umgodi_json-0.1.0/assets/umgodi-json-sync-compact.svg +15 -0
- umgodi_json-0.1.0/assets/umgodi-json-sync.svg +139 -0
- umgodi_json-0.1.0/doc/CR/CR024_AIA_to_RAIkeep_Ephemeral_Flag_Self_Cleanup.md +70 -0
- umgodi_json-0.1.0/doc/CR/CR040_AIA_to_RAIkeep_and_jsonpit_Prohibit-Read-Modify-Write.md +145 -0
- umgodi_json-0.1.0/doc/CR/CR041_jsonpit_to_RAIkeep_Restore-Clean-Change-Filenames.md +106 -0
- umgodi_json-0.1.0/doc/CR/CR043_AfricaStage_to_RAIkeep_Improve-pits-seed-array-error-message.md +90 -0
- umgodi_json-0.1.0/doc/CR/CR044_AIA_and_jsonpit_to_RAIkeep_Auto-Detect-Cloud-Drives-and-Init-Config.md +362 -0
- umgodi_json-0.1.0/doc/CR/CR047_jsonpit_and_pits_Require-Existing-Patch-Flag-and-Seed-Commit-Count.md +107 -0
- umgodi_json-0.1.0/doc/CR/CR049_AIA_and_jsonpit_to_RAIkeep_Live-ID-Validation_and_Zip-Image-Import.md +244 -0
- umgodi_json-0.1.0/doc/CROSS_ENGINE_PARITY_SPECIFICATION.md +223 -0
- umgodi_json-0.1.0/doc/EcholotConceptResearch.md +74 -0
- umgodi_json-0.1.0/doc/JSONPIT_VISUAL_IDENTITY_AND_ICON_SPEC.md +93 -0
- umgodi_json-0.1.0/doc/PREFLIGHT.md +24 -0
- umgodi_json-0.1.0/doc/umgodi-class-diagram.puml +312 -0
- umgodi_json-0.1.0/doc/umgodi-class-diagram.svg +1 -0
- umgodi_json-0.1.0/doc/uml/jsonpitCD.png +0 -0
- umgodi_json-0.1.0/doc/uml/jsonpitCD.puml +78 -0
- umgodi_json-0.1.0/doc/uml/jsonpitCD.svg +1 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.png +0 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.puml +75 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.svg +1 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.png +0 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.puml +141 -0
- umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.svg +1 -0
- umgodi_json-0.1.0/pyproject.toml +79 -0
- umgodi_json-0.1.0/scripts/build_standalone.py +48 -0
- umgodi_json-0.1.0/scripts/preflight.py +106 -0
- umgodi_json-0.1.0/tests/__init__.py +1 -0
- umgodi_json-0.1.0/tests/fixtures/Person.pit +1 -0
- umgodi_json-0.1.0/tests/run_all.py +93 -0
- umgodi_json-0.1.0/tests/sequence_runner.py +338 -0
- umgodi_json-0.1.0/tests/suites/01_persistence_and_crud.json5 +152 -0
- umgodi_json-0.1.0/tests/suites/02_changes_and_receipts.json5 +124 -0
- umgodi_json-0.1.0/tests/suites/03_tombstones_and_projections.json5 +178 -0
- umgodi_json-0.1.0/tests/suites/04_compaction_and_events.json5 +109 -0
- umgodi_json-0.1.0/tests/suites/05_process_flags_and_leases.json5 +126 -0
- umgodi_json-0.1.0/tests/suites/06_split_master_recovery.json5 +115 -0
- umgodi_json-0.1.0/tests/suites/cr040_and_cross_parity.json5 +90 -0
- umgodi_json-0.1.0/tests/test_audit.py +323 -0
- umgodi_json-0.1.0/tests/test_canonical.py +58 -0
- umgodi_json-0.1.0/tests/test_change_file.py +97 -0
- umgodi_json-0.1.0/tests/test_cli.py +324 -0
- umgodi_json-0.1.0/tests/test_cli_export.py +88 -0
- umgodi_json-0.1.0/tests/test_cli_style.py +122 -0
- umgodi_json-0.1.0/tests/test_cr043_seed_shapes.py +189 -0
- umgodi_json-0.1.0/tests/test_cr044_missing_config.py +160 -0
- umgodi_json-0.1.0/tests/test_cr047_strict_patch.py +317 -0
- umgodi_json-0.1.0/tests/test_cr049_live_id_validation.py +107 -0
- umgodi_json-0.1.0/tests/test_cr081_seed_dispositions.py +175 -0
- umgodi_json-0.1.0/tests/test_cr082_cloud_export.py +195 -0
- umgodi_json-0.1.0/tests/test_csharp_compatibility.py +51 -0
- umgodi_json-0.1.0/tests/test_delete_property_projection.py +201 -0
- umgodi_json-0.1.0/tests/test_equal_timestamp_ordering.py +100 -0
- umgodi_json-0.1.0/tests/test_inqaba_boundary.py +33 -0
- umgodi_json-0.1.0/tests/test_live_references.py +505 -0
- umgodi_json-0.1.0/tests/test_maintain.py +377 -0
- umgodi_json-0.1.0/tests/test_master_ticket.py +83 -0
- umgodi_json-0.1.0/tests/test_pit_item.py +91 -0
- umgodi_json-0.1.0/tests/test_sequence_isolation.py +42 -0
- umgodi_json-0.1.0/tests/test_sequence_suites.py +26 -0
- umgodi_json-0.1.0/tests/test_sparse_delta.py +196 -0
- umgodi_json-0.1.0/umgodi/__init__.py +76 -0
- umgodi_json-0.1.0/umgodi/audit.py +209 -0
- umgodi_json-0.1.0/umgodi/canonical.py +105 -0
- umgodi_json-0.1.0/umgodi/changes.py +206 -0
- umgodi_json-0.1.0/umgodi/cli.py +1817 -0
- umgodi_json-0.1.0/umgodi/cli_style.py +112 -0
- umgodi_json-0.1.0/umgodi/config.py +7 -0
- umgodi_json-0.1.0/umgodi/exceptions.py +44 -0
- umgodi_json-0.1.0/umgodi/flags.py +272 -0
- umgodi_json-0.1.0/umgodi/fs.py +177 -0
- umgodi_json-0.1.0/umgodi/history.py +202 -0
- umgodi_json-0.1.0/umgodi/icons.py +63 -0
- umgodi_json-0.1.0/umgodi/item.py +800 -0
- umgodi_json-0.1.0/umgodi/store.py +1480 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
.Python
|
|
8
|
+
build/
|
|
9
|
+
develop-eggs/
|
|
10
|
+
dist/
|
|
11
|
+
downloads/
|
|
12
|
+
eggs/
|
|
13
|
+
.eggs/
|
|
14
|
+
lib/
|
|
15
|
+
lib64/
|
|
16
|
+
parts/
|
|
17
|
+
sdist/
|
|
18
|
+
var/
|
|
19
|
+
wheels/
|
|
20
|
+
share/python-wheels/
|
|
21
|
+
*.egg-info/
|
|
22
|
+
.installed.cfg
|
|
23
|
+
*.egg
|
|
24
|
+
MANIFEST
|
|
25
|
+
|
|
26
|
+
# Virtual environments
|
|
27
|
+
.venv/
|
|
28
|
+
venv/
|
|
29
|
+
ENV/
|
|
30
|
+
env/
|
|
31
|
+
|
|
32
|
+
# IDE / Editor
|
|
33
|
+
.vscode/
|
|
34
|
+
.idea/
|
|
35
|
+
*.swp
|
|
36
|
+
*.swo
|
|
37
|
+
*~
|
|
38
|
+
.DS_Store
|
|
39
|
+
|
|
40
|
+
# Testing & Type checking
|
|
41
|
+
.pytest_cache/
|
|
42
|
+
.mypy_cache/
|
|
43
|
+
.ruff_cache/
|
|
44
|
+
.coverage
|
|
45
|
+
htmlcov/
|
|
46
|
+
|
|
47
|
+
# Local test scratch / temp
|
|
48
|
+
scratch/
|
|
49
|
+
tmp/
|
|
50
|
+
*.tmp
|
umgodi_json-0.1.0/API.md
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# umgodi API Reference (v0.1.0)
|
|
2
|
+
|
|
3
|
+
The `umgodi-json` distribution provides the public `umgodi` API and uses Inqaba for shared filesystem and configuration operations.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Architectural Diagrams
|
|
8
|
+
|
|
9
|
+
### Core Replicated Storage Architecture
|
|
10
|
+
<p align="center">
|
|
11
|
+
<img src="https://raw.githubusercontent.com/Burkhardt/umgodi-json/main/doc/uml/jsonpitCD.png" alt="Core Replicated Storage Architecture" width="700"/>
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
### Storage Maintenance & Recovery Audit Architecture
|
|
15
|
+
<p align="center">
|
|
16
|
+
<img src="https://raw.githubusercontent.com/Burkhardt/umgodi-json/main/doc/uml/jsonpit_opsCD.png" alt="Operations & Recovery Audit Architecture" width="700"/>
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## Pit Lifecycle & Storage
|
|
22
|
+
|
|
23
|
+
- <details>
|
|
24
|
+
<summary><code>Pit</code>: Live eventually consistent distributed JSON store</summary>
|
|
25
|
+
|
|
26
|
+
Coordinates one live public instance per canonical pit path, implements `MutableMapping[str, PitItem]` and the `ContextManager` protocol, observes cloud-drive changes, emits sparse delta change files, and coordinates master leases.
|
|
27
|
+
|
|
28
|
+
**Key Methods:**
|
|
29
|
+
- `__enter__()` / `__exit__()`: Context manager for atomic batch sessions and guaranteed flag cleanup.
|
|
30
|
+
- `add(item: PitItem, refresh_modified: bool = True) -> bool`: Adds or updates a living entity. Validates against prohibited template markers (`{` or `<`) under CR049.
|
|
31
|
+
- `add_historical(item: PitItem) -> bool`: Replays an immutable historical fragment without modifying timestamps or rejecting legacy IDs.
|
|
32
|
+
- `get(item_id: str, default: Any = None, at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Retrieves the projected living state of an entity, with optional time-travel projection.
|
|
33
|
+
- `delete_item(item_id: str, by: str | None = None, backdate_100: bool = False) -> bool`: Tombstones an entity across the distributed cluster.
|
|
34
|
+
- `rename_id(old_key: str, new_key: str, by: str | None = None) -> bool`: Atomically migrates an entity identity.
|
|
35
|
+
- `seed_from_file(file_path: Path | str, require_existing: bool = False) -> int`: Ingests an external JSON/JSON5 file (single entity, keyed map, or array) with whole-batch ID preflight and optional CR047 strict patch enforcement.
|
|
36
|
+
- `maintain(options: PitMaintenanceOptions = None, apply: bool = False) -> PitMaintenanceResult`: Reconciles change files, manages 10-minute receipt grace periods, prunes expired process flags (`--older-than`), and repairs legacy event extensions.
|
|
37
|
+
- `save(force: bool = False, pretty: bool = False) -> None`: Persists the unified canonical `.pit` file in-place using CR022 sibling writes (`.pit.tmp` followed by atomic rename).
|
|
38
|
+
</details>
|
|
39
|
+
|
|
40
|
+
- <details>
|
|
41
|
+
<summary><code>Pit.maintain(...)</code>, <code>PitMaintenanceOptions</code>, and <code>PitMaintenanceResult</code></summary>
|
|
42
|
+
|
|
43
|
+
`maintain()` inventories pending change/receipt cleanup without mutation. It checks the canonical parent first, returning deferred status for missing targets without creating directories. `maintain(apply=True)` applies canonical reconciliation and eligible change-first/receipt-second retirement (10-minute grace period).
|
|
44
|
+
|
|
45
|
+
**Maintenance Options:**
|
|
46
|
+
- `apply: bool`: Authorizes modifications to disk (default False: report-only preview).
|
|
47
|
+
- `prune_process_flags: bool`: Authorizes pruning of expired PID-window flag files. Requires `apply=True` and `older_than`.
|
|
48
|
+
- `older_than: timedelta`: Duration threshold for flag pruning (e.g. `01:00:00`, `7.00:00:00`).
|
|
49
|
+
- `repair_legacy_extensions: bool`: Migrates legacy `{stem}_{sha256}.event` files to clean `{stem}.event` in `Events/`.
|
|
50
|
+
- `archive_events: bool`: Compacts event logs into immutable archives.
|
|
51
|
+
|
|
52
|
+
**Maintenance Result (`to_dict()` for 100% C# parity):**
|
|
53
|
+
- Exposes `PitFile`, `Applied`, `Succeeded`, `ChangeFilesObserved/Merged/Removed`, `ReceiptsCreated/Removed`, `ProcessFlagsActive/Expired/Pruned`, `LegacyArtifactsObserved/Repaired`, `Deferred`, and `Failures`.
|
|
54
|
+
</details>
|
|
55
|
+
|
|
56
|
+
- <details>
|
|
57
|
+
<summary><code>JsonPitBase</code>: Shared storage and path foundation</summary>
|
|
58
|
+
|
|
59
|
+
Base class providing pit path resolution, cloud drive detection, subscriber identity, process leasing options (`read_only`, `unflagged`, `retain_window`), and canonical serialization.
|
|
60
|
+
</details>
|
|
61
|
+
|
|
62
|
+
- <details>
|
|
63
|
+
<summary><code>PitStore</code>: Thread-safe repository of open Pit instances</summary>
|
|
64
|
+
|
|
65
|
+
Factory and cache ensuring only one active `Pit` instance exists per canonical file path within a single process, preventing memory divergence.
|
|
66
|
+
</details>
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## Entity Domain Model & History
|
|
71
|
+
|
|
72
|
+
- <details>
|
|
73
|
+
<summary><code>PitItem</code>: Polymorphic living entity and sparse delta emitter</summary>
|
|
74
|
+
|
|
75
|
+
Represents an individual entity within a Pit. Implements dictionary-like attribute access while protecting immutable engine attributes (`id`, `modified`, `deleted`) from accidental mutation (CR040). When bound to a `Pit`, mutations automatically stream sparse delta change files without triggering read-modify-write anti-patterns.
|
|
76
|
+
|
|
77
|
+
**Core Attributes:**
|
|
78
|
+
- `id`: Non-empty string identifier. Rejects unresolved template markers (`{` or `<`) at write boundaries (CR049).
|
|
79
|
+
- `modified`: 7-digit UTC timestamp (`datetime`) of the latest fragment.
|
|
80
|
+
- `deleted`: Boolean tombstone flag.
|
|
81
|
+
- `note`: Optional human- or agent-readable string note.
|
|
82
|
+
|
|
83
|
+
**Sparse Mutation Methods:**
|
|
84
|
+
- `set_property(patch: dict[str, Any] | str) -> bool`: Atomically applies a property delta and emits a sparse change file.
|
|
85
|
+
- `delete_property(property_name: str) -> None`: Emits a property-level null tombstone without rewriting living state.
|
|
86
|
+
- `delete_property_path(property_path: str) -> None`: Traverses dot-delimited paths to tombstone nested properties.
|
|
87
|
+
- `extend_with(obj: dict[str, Any]) -> bool`: Merges attributes from a dictionary.
|
|
88
|
+
- `to_dict() -> dict[str, Any]` / `to_json() -> str`: Canonical JSON serialization.
|
|
89
|
+
</details>
|
|
90
|
+
|
|
91
|
+
- <details>
|
|
92
|
+
<summary><code>PitItems</code>: Immutable fragment stack and time-travel projector</summary>
|
|
93
|
+
|
|
94
|
+
Maintains an entity's ordered history of timestamped fragments.
|
|
95
|
+
|
|
96
|
+
**Key Methods:**
|
|
97
|
+
- `push(item: PitItem) -> PitItems`: Returns a new `PitItems` stack with the fragment prepended.
|
|
98
|
+
- `latest_fragment() -> PitItem | None`: Returns the most recently written fragment.
|
|
99
|
+
- `project_state(at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Merges historical fragments from oldest to newest up to timestamp `at` to compute accurate point-in-time living state.
|
|
100
|
+
</details>
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## Distributed Protocol & Cloud Safety
|
|
105
|
+
|
|
106
|
+
- <details>
|
|
107
|
+
<summary><code>ChangeFile</code>: Shortened cloud-safe change artifact</summary>
|
|
108
|
+
|
|
109
|
+
Writes sparse delta mutations into `Changes/` using the 4-character hex checksum format:
|
|
110
|
+
`{UtcTicks}_{ExactProcessIdentity}_{4charHex}.json`
|
|
111
|
+
Guards against partially synchronized cloud writes (e.g. OneDrive) without exceeding path length limits. Validates checksums on read and maintains backwards compatibility with legacy 64-character SHA-suffixed change files.
|
|
112
|
+
</details>
|
|
113
|
+
|
|
114
|
+
- <details>
|
|
115
|
+
<summary><code>ReceiptFile</code>: Durable round-trip confirmation</summary>
|
|
116
|
+
|
|
117
|
+
Pairs with change files using the `.receipt` extension. Enforces a 10-minute grace period before allowing retired change files to be compacted or purged.
|
|
118
|
+
</details>
|
|
119
|
+
|
|
120
|
+
- <details>
|
|
121
|
+
<summary><code>MasterFlagFile</code> & <code>ProcessFlagFile</code>: Cloud drive lease coordination</summary>
|
|
122
|
+
|
|
123
|
+
- `Master.flag`: 60-second lease renewed every 30 seconds by the active master writer.
|
|
124
|
+
- `{Machine}-{Process}-{PID}.flag`: Ephemeral process activity flag with heartbeat updates. Enforces deterministic self-cleanup on process exit (CR024).
|
|
125
|
+
</details>
|
|
126
|
+
|
|
127
|
+
- <details>
|
|
128
|
+
<summary><code>EventFile</code>: Clean event compaction naming & migration</summary>
|
|
129
|
+
|
|
130
|
+
Emits compacted historical events under clean logical stems (`{LogicalStem}.event`) without SHA-256 hash suffixes. Automatically recognizes legacy 64-char hash filenames and supports in-place migration.
|
|
131
|
+
</details>
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Configuration & Cloud Discovery
|
|
136
|
+
|
|
137
|
+
- <details>
|
|
138
|
+
<summary><code>OsConfig</code>: Cross-platform cloud drive discovery</summary>
|
|
139
|
+
|
|
140
|
+
Discovers synchronization folders for `Dropbox`, `OneDrive`, `GoogleDrive`, and `ICloudDrive`. Reads primarily from `~/.config/RAIkeep.json5` with fallback to `~/.config/jsonpit.json5` or `$JSONPIT_CONFIG`.
|
|
141
|
+
</details>
|
|
142
|
+
|
|
143
|
+
- <details>
|
|
144
|
+
<summary><code>missing_configuration_diagnostic()</code>: CR044 operator guidance</summary>
|
|
145
|
+
|
|
146
|
+
When cloud drives or configuration files are absent, formats clear actionable instructions directing the operator to run `amafu init` to bootstrap configuration.
|
|
147
|
+
</details>
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
## Audit Engine & Recovery Inspection
|
|
152
|
+
|
|
153
|
+
- <details>
|
|
154
|
+
<summary><code>PitAudit</code>: Read-only inspection of durable event logs</summary>
|
|
155
|
+
|
|
156
|
+
Provides read-only access to durable JsonPit recovery events under `Events/` without opening a `Pit`, creating process activity flags, acquiring leases, or mutating disk state (CR003). Inspects both loose `*.event` files and immutable `Events_*.zip` compaction archives.
|
|
157
|
+
|
|
158
|
+
**Key Methods:**
|
|
159
|
+
- `PitAudit.inspect(pit_directory: Path | str, machine_filter: str = "all", min_level: LogLevel = LogLevel.TRACE) -> PitAuditReadResult`: Reads, validates, deduplicates by `EventId`, filters by machine/severity, and orders events deterministically by machine, UTC time, and event identity.
|
|
160
|
+
</details>
|
|
161
|
+
|
|
162
|
+
- <details>
|
|
163
|
+
<summary><code>PitAuditEvent</code>: Durable recovery event domain model</summary>
|
|
164
|
+
|
|
165
|
+
Encapsulates a single durable recovery event:
|
|
166
|
+
- `file_name: str`: Source file or archive entry name.
|
|
167
|
+
- `content: dict[str, Any]`: Raw deserialized event JSON object.
|
|
168
|
+
- `machine: str`: Originating machine hostname.
|
|
169
|
+
- `utc_time: datetime`: Event occurrence timestamp in UTC.
|
|
170
|
+
- `level: LogLevel`: Severity level.
|
|
171
|
+
- `stage: str`: Pit lifecycle stage (e.g. `ChangeFilesPublished`, `CleanupPending`, `Completed`).
|
|
172
|
+
- `event_id: str`: Unique GUID event identifier used for cross-archive deduplication.
|
|
173
|
+
- `message: str`: Human-readable summary message.
|
|
174
|
+
</details>
|
|
175
|
+
|
|
176
|
+
- <details>
|
|
177
|
+
<summary><code>LogLevel</code>: Event severity enumeration</summary>
|
|
178
|
+
|
|
179
|
+
Matches .NET `LogLevel` integer values and names:
|
|
180
|
+
- `TRACE = 0` (`Trace`)
|
|
181
|
+
- `DEBUG = 1` (`Debug`)
|
|
182
|
+
- `INFORMATION = 2` (`Information` / `info`)
|
|
183
|
+
- `WARNING = 3` (`Warning` / `warn`)
|
|
184
|
+
- `ERROR = 4` (`Error` / `err`)
|
|
185
|
+
- `CRITICAL = 5` (`Critical` / `fatal`)
|
|
186
|
+
- `from_string(name: str) -> LogLevel`: Case-insensitive parsing supporting names, aliases, and integer strings.
|
|
187
|
+
- `display_name -> str`: Capitalized canonical name matching C# output formatting.
|
|
188
|
+
</details>
|
|
189
|
+
|
|
190
|
+
- <details>
|
|
191
|
+
<summary><code>PitAuditReadResult</code>: Filtered events and diagnostic issues</summary>
|
|
192
|
+
|
|
193
|
+
Contains:
|
|
194
|
+
- `events: list[PitAuditEvent]`: Filtered and ordinally sorted events.
|
|
195
|
+
- `issues: list[str]`: Physical corruption, empty file, or conflicting event identity warnings.
|
|
196
|
+
- `succeeded: bool`: Returns `True` if no diagnostic issues were discovered.
|
|
197
|
+
</details>
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## Exception Hierarchy
|
|
202
|
+
|
|
203
|
+
All `umgodi` exceptions derive from `JsonPitError`:
|
|
204
|
+
|
|
205
|
+
```text
|
|
206
|
+
JsonPitError
|
|
207
|
+
├── PitNotFoundError # Target pit directory or file does not exist
|
|
208
|
+
├── PitCorruptError # Payload failed checksum or schema validation
|
|
209
|
+
├── PitConcurrencyError # Conflict acquiring or renewing master lease
|
|
210
|
+
├── PitInstanceConflictError # Multiple live Pit instances for the same file path
|
|
211
|
+
├── ProtectedAttributeError # Client attempted to mutate 'id', 'modified', or 'deleted' (CR040)
|
|
212
|
+
├── TombstoneError # Attempted to delete protected engine attributes
|
|
213
|
+
└── StrictPatchValidationError # Missing or tombstoned entity under --id-exists / --patch (CR047)
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
## CR077 mutation contract (4.6.0)
|
|
218
|
+
|
|
219
|
+
Live handles remain ordinary PitItems. Use indexing, set_property, extend_with,
|
|
220
|
+
merge, delete_property and delete_property_path; save remains the persistence
|
|
221
|
+
boundary. Re-adding an owned live handle flushes pending differences without
|
|
222
|
+
replaying its full projection. PitItem(existing_item) is detached; adding that
|
|
223
|
+
copy as client data is rejected when it carries lifecycle metadata. A matching
|
|
224
|
+
Id assertion in a patch is supported. JSON-array extend validates every row before
|
|
225
|
+
acceptance and creates one combined sparse mutation. Vikela is optional developer
|
|
226
|
+
tooling and is not a jsonpit runtime dependency.
|
|
227
|
+
|
|
228
|
+
## CR081 CLI receipts and lifecycle failures
|
|
229
|
+
|
|
230
|
+
`jpit seed` and `jpit put` report source-entry written/unchanged counts after
|
|
231
|
+
explicit Save and context exit. A failed local save or required owned-window
|
|
232
|
+
cleanup propagates to a nonzero CLI exit without a success receipt. `close()`
|
|
233
|
+
now raises OSError if the owned activity flag remains after an unsuccessful
|
|
234
|
+
release; `retain_window=True` preserves its explicit release opt-out.
|
|
235
|
+
|
|
236
|
+
`add()` matches C# latest-fragment comparison ignoring Modified before refreshing
|
|
237
|
+
timestamps. Identical latest fragments return False. Projected-state matches with
|
|
238
|
+
a different latest fragment still append; historical and tombstone semantics
|
|
239
|
+
remain covered by parity tests.
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# Getting Started with Python umgodi-json (v0.1.0)
|
|
2
|
+
|
|
3
|
+
This guide provides a practical walkthrough for using **`umgodi`** from Python—either embedded as a library in your services, AI agent frameworks, and data pipelines, or as the standalone **`jpit`** CLI for command-line workflows.
|
|
4
|
+
|
|
5
|
+
It is based on `umgodi` v0.1.0, featuring 100% lockstep parity with C# `JsonPit` in [RAIkeep](https://github.com/Burkhardt/RAIkeep).
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. Purpose and Mental Model
|
|
10
|
+
|
|
11
|
+
`umgodi` is a cloud-first, file-based, distributed replicated storage engine for `Id`-identified items that need to be shared, synchronized, and reloaded across machines and processes without introducing a database daemon.
|
|
12
|
+
|
|
13
|
+
### The Core Concepts:
|
|
14
|
+
- **`Pit`**: A named container directory stored on disk (e.g. `Person/Person.pit`).
|
|
15
|
+
- **`PitItem`**: An `Id`-identified JSON object with lifecycle metadata (`Modified`, `Deleted`, `Note`) and arbitrary properties.
|
|
16
|
+
- **`PitItems`**: The immutable version history of sparse change fragments for an item key.
|
|
17
|
+
|
|
18
|
+
### Mental Model:
|
|
19
|
+
- A `Pit` is closer to a synchronized document store than a SQL table.
|
|
20
|
+
- Updates are **append-only sparse fragments**, not blind in-memory overwrites.
|
|
21
|
+
- Coordination happens through synchronized cloud storage (OneDrive, Dropbox, GoogleDrive, ICloudDrive) using lease flags and change files.
|
|
22
|
+
- **[Asynchronous persistence with eventual durability](https://github.com/Burkhardt/RAIkeep/blob/main/MANIFESTO.md#asynchronous-persistence-with-eventual-durability)**: request handling happens in memory; synchronization is explicit via `save()` or context manager exit.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 3. Installing the CLI or Python Library
|
|
27
|
+
|
|
28
|
+
For the CLI, install one pipx environment that supplies both commands:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pipx install "umgodi-json==0.1.0"
|
|
32
|
+
# For an existing environment:
|
|
33
|
+
pipx upgrade umgodi-json
|
|
34
|
+
jpit --version
|
|
35
|
+
umgodi --version
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The outputs are `jpit v0.1.0` and `umgodi v0.1.0`. If a separate `jpit` pipx
|
|
39
|
+
environment already exists, run `pipx uninstall jpit`, followed by
|
|
40
|
+
`pipx install --force "umgodi-json==0.1.0"` to restore both command links.
|
|
41
|
+
|
|
42
|
+
Ubuntu/Debian and Homebrew may reject system-wide pip installs as externally
|
|
43
|
+
managed. Use pipx for the CLI, or an isolated virtual environment for library use:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
python3 -m venv .venv
|
|
47
|
+
source .venv/bin/activate
|
|
48
|
+
python -m pip install "umgodi-json==0.1.0"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## 4. Configuration & Simplified Cloud Shortcuts
|
|
54
|
+
|
|
55
|
+
`umgodi` automatically reads your cloud provider roots from `~/.config/RAIkeep.json5` (with fallback to `~/.config/jsonpit.json5` or `$JSONPIT_CONFIG`).
|
|
56
|
+
|
|
57
|
+
### Zero-Friction Setup via [`amafu`](https://www.nuget.org/packages/Amafu)
|
|
58
|
+
|
|
59
|
+
On macOS and Linux, enable your cloud clients (iCloud, OneDrive, Google Drive, Dropbox), then use [`amafu`](https://www.nuget.org/packages/Amafu) to auto-detect your local sync folders and create clean, unified shortcuts in `~/.CloudStorage/`:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
# 1. Preview detected roots and proposed shortcuts:
|
|
63
|
+
amafu detect --create-links --dry-run
|
|
64
|
+
|
|
65
|
+
# 2. Initialize configuration AND create shortcuts in ~/.CloudStorage/:
|
|
66
|
+
amafu init --create-links
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
If you already have a configuration and just want to create the shortcuts:
|
|
70
|
+
```bash
|
|
71
|
+
amafu detect --create-links
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
This creates convenient, unified symbolic links:
|
|
75
|
+
- `~/.CloudStorage/ICloudDrive` $\rightarrow$ `~/Library/Mobile Documents/com~apple~CloudDocs/`
|
|
76
|
+
- `~/.CloudStorage/GoogleDrive` $\rightarrow$ `~/Library/CloudStorage/GoogleDrive-.../My Drive/`
|
|
77
|
+
- `~/.CloudStorage/OneDrive` $\rightarrow$ `~/Library/CloudStorage/OneDrive-.../`
|
|
78
|
+
- `~/.CloudStorage/Dropbox` $\rightarrow$ `~/Library/CloudStorage/Dropbox/`
|
|
79
|
+
|
|
80
|
+
You never have to search for obscure operating system cloud paths again. Your real files stay safely in their original cloud folders, and your Python code or terminal sessions can easily use `~/.CloudStorage/<provider>` or reference the provider name directly (e.g. `cloud="ICloudDrive"`).
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## 5. First Steps with the Python Library
|
|
85
|
+
|
|
86
|
+
### Basic End-to-End Example
|
|
87
|
+
Here is the recommended Python idiom using the `Pit.open()` context manager:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from umgodi import Pit, PitItem
|
|
91
|
+
|
|
92
|
+
# 1. Open the Pit using the context manager
|
|
93
|
+
# (Acquires a process activity window, loads state, and auto-saves on exit)
|
|
94
|
+
with Pit.open("person", cloud="GoogleDrive") as people:
|
|
95
|
+
|
|
96
|
+
# 2. Create an entity with Id
|
|
97
|
+
max_person = PitItem("Max")
|
|
98
|
+
max_person["Email"] = "max@example.org"
|
|
99
|
+
max_person["Phone"] = "+27-82-000-0000"
|
|
100
|
+
max_person["ComPref"] = ["WhatsApp", "Email"]
|
|
101
|
+
|
|
102
|
+
# 3. Add to Pit
|
|
103
|
+
people.add(max_person)
|
|
104
|
+
print(f"Added {max_person.id} to Pit '{people.pit_name}'")
|
|
105
|
+
|
|
106
|
+
# --- On context manager exit, people.save() is called automatically ---
|
|
107
|
+
|
|
108
|
+
# 4. Reopen and query
|
|
109
|
+
with Pit.open("person", cloud="GoogleDrive", read_only=True) as people:
|
|
110
|
+
loaded_max = people["Max"]
|
|
111
|
+
print(f"Loaded: {loaded_max.id} -> Email: {loaded_max['Email']}, Phone: {loaded_max['Phone']}")
|
|
112
|
+
print(f"Channels: {loaded_max['ComPref']}")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Updating an Existing Item
|
|
116
|
+
Looking up an entity returns a **live reference**. Mutating that reference records sparse modifications directly:
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
with Pit.open("person", cloud="GoogleDrive") as people:
|
|
120
|
+
max_person = people["Max"]
|
|
121
|
+
|
|
122
|
+
# Direct indexer updates append sparse fragments
|
|
123
|
+
max_person["Phone"] = "+27-82-111-2222"
|
|
124
|
+
max_person["Instagram"] = "max.africastage"
|
|
125
|
+
|
|
126
|
+
# Nested dictionaries are deep-merged
|
|
127
|
+
max_person["Address"] = {
|
|
128
|
+
"Street": "42 Long Street",
|
|
129
|
+
"City": "Cape Town",
|
|
130
|
+
"Country": "South Africa"
|
|
131
|
+
}
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
### Removing Attributes (Tombstones)
|
|
135
|
+
`umgodi` never mutates history destructively. Deleting a property appends a **null tombstone** that masks previous values in the projection while preserving time travel:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
with Pit.open("person", cloud="GoogleDrive") as people:
|
|
139
|
+
max_person = people["Max"]
|
|
140
|
+
|
|
141
|
+
# Remove a top-level property
|
|
142
|
+
max_person.delete_property("Instagram")
|
|
143
|
+
|
|
144
|
+
# Remove a nested path
|
|
145
|
+
max_person.delete_property_path("Address.Street")
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
### Deleting an Entity
|
|
149
|
+
```python
|
|
150
|
+
with Pit.open("person", cloud="GoogleDrive") as people:
|
|
151
|
+
# Tombstones the entire entity
|
|
152
|
+
del people["Max"]
|
|
153
|
+
# or: people.delete_item("Max", by="Operator")
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
### Time Travel and Historical Auditing
|
|
157
|
+
Because every modification is an immutable timestamped fragment, you can inspect historical states:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
with Pit.open("person", cloud="GoogleDrive", read_only=True) as people:
|
|
161
|
+
# Inspect raw version history
|
|
162
|
+
history = people.history["Max"]
|
|
163
|
+
for frag in history.history:
|
|
164
|
+
print(f"[{frag.timestamp}] {frag.data}")
|
|
165
|
+
|
|
166
|
+
# Reconstruct the entity as it existed at a past point in time
|
|
167
|
+
past_state = people.get_at("Max", at="2026-10-01T12:00:00Z")
|
|
168
|
+
if past_state:
|
|
169
|
+
print("Past Phone:", past_state.get("Phone"))
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## 6. Alternative: First Steps with the `jpit` CLI
|
|
175
|
+
|
|
176
|
+
If you prefer to operate directly from the command line without writing Python code, the `jpit` CLI provides high-performance data seeding, mutation, and inspection.
|
|
177
|
+
|
|
178
|
+
### Step 1: Seed Initial Data
|
|
179
|
+
Create a JSON or JSON5 file (`people.json5`):
|
|
180
|
+
|
|
181
|
+
```json5
|
|
182
|
+
[
|
|
183
|
+
{
|
|
184
|
+
Id: "Max",
|
|
185
|
+
Email: "max@example.org",
|
|
186
|
+
Phone: "+27-82-000-0000",
|
|
187
|
+
ComPref: ["WhatsApp", "Email"]
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
Id: "Adele",
|
|
191
|
+
Email: "adele@example.org",
|
|
192
|
+
Role: "Software Architect"
|
|
193
|
+
}
|
|
194
|
+
]
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Seed the entities into a Pit named `person`:
|
|
198
|
+
```bash
|
|
199
|
+
jpit seed person people.json5
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Step 2: Read Entities
|
|
203
|
+
Get one entity formatted as JSON:
|
|
204
|
+
```bash
|
|
205
|
+
jpit get person Max
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Get only a specific property:
|
|
209
|
+
```bash
|
|
210
|
+
jpit get person Max --property Email
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
List all active entities in the pit:
|
|
214
|
+
```bash
|
|
215
|
+
jpit list person
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### Step 3: Sparse Mutations (Anti-Read-Modify-Write)
|
|
219
|
+
Update attributes without rewriting the entire entity:
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
jpit set person Max Phone "+27-82-111-2222"
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Add a nested object:
|
|
226
|
+
```bash
|
|
227
|
+
jpit set person Max Address '{"City": "Cape Town", "Country": "South Africa"}'
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Tombstone an attribute:
|
|
231
|
+
```bash
|
|
232
|
+
jpit del-prop person Max Email
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
### Step 4: Semantic Search ("Pit-Grep")
|
|
236
|
+
Search across living entity states using ripgrep-style speed:
|
|
237
|
+
|
|
238
|
+
```bash
|
|
239
|
+
jpit grep "Cape Town" person
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Filter by property and output structured JSON:
|
|
243
|
+
```bash
|
|
244
|
+
jpit grep --property Role "Architect" person --json
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
### Step 5: Exporting & Piping with `jq`
|
|
248
|
+
Pipe living state directly into `jq`:
|
|
249
|
+
```bash
|
|
250
|
+
jpit export person --jq '.[].Id'
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
---
|
|
254
|
+
|
|
255
|
+
## 7. 100% C# / `pits` Cross-Language Interoperability
|
|
256
|
+
|
|
257
|
+
Every Pit created or updated by Python `umgodi` is 100% binary- and schema-compatible with C# `JsonPit` and the `pits` CLI.
|
|
258
|
+
|
|
259
|
+
Write an entity attribute with Python `jpit`:
|
|
260
|
+
```bash
|
|
261
|
+
jpit set person Max Status "Online"
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
Read it immediately with C# `pits`:
|
|
265
|
+
```bash
|
|
266
|
+
pits get person Max
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Both engines enforce identical timestamp parsing, canonical key ordering, lease flag acquisition, receipt grace periods, and sparse delta projection.
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## 8. Alternative Installation & Troubleshooting
|
|
274
|
+
|
|
275
|
+
### Shell PATH troubleshooting
|
|
276
|
+
If your shell reports `command not found: jpit`, ensure that `~/.local/bin` is in your shell `PATH`:
|
|
277
|
+
```bash
|
|
278
|
+
echo 'export PATH="$HOME/.local/bin:$HOME/.dotnet/tools:$PATH"' >> ~/.zshrc
|
|
279
|
+
source ~/.zshrc
|
|
280
|
+
```
|