umgodi-json 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. umgodi_json-0.1.0/.gitignore +50 -0
  2. umgodi_json-0.1.0/API.md +239 -0
  3. umgodi_json-0.1.0/GettingStarted.md +280 -0
  4. umgodi_json-0.1.0/PKG-INFO +274 -0
  5. umgodi_json-0.1.0/README.md +249 -0
  6. umgodi_json-0.1.0/assets/jsonpit-sync-logo.svg +132 -0
  7. umgodi_json-0.1.0/assets/logo-128x128.png +0 -0
  8. umgodi_json-0.1.0/assets/logo-192x192.png +0 -0
  9. umgodi_json-0.1.0/assets/logo-512x512.png +0 -0
  10. umgodi_json-0.1.0/assets/logo.png +0 -0
  11. umgodi_json-0.1.0/assets/logo.svg +132 -0
  12. umgodi_json-0.1.0/assets/umgodi-json-sync-banner.svg +125 -0
  13. umgodi_json-0.1.0/assets/umgodi-json-sync-compact.svg +15 -0
  14. umgodi_json-0.1.0/assets/umgodi-json-sync.svg +139 -0
  15. umgodi_json-0.1.0/doc/CR/CR024_AIA_to_RAIkeep_Ephemeral_Flag_Self_Cleanup.md +70 -0
  16. umgodi_json-0.1.0/doc/CR/CR040_AIA_to_RAIkeep_and_jsonpit_Prohibit-Read-Modify-Write.md +145 -0
  17. umgodi_json-0.1.0/doc/CR/CR041_jsonpit_to_RAIkeep_Restore-Clean-Change-Filenames.md +106 -0
  18. umgodi_json-0.1.0/doc/CR/CR043_AfricaStage_to_RAIkeep_Improve-pits-seed-array-error-message.md +90 -0
  19. umgodi_json-0.1.0/doc/CR/CR044_AIA_and_jsonpit_to_RAIkeep_Auto-Detect-Cloud-Drives-and-Init-Config.md +362 -0
  20. umgodi_json-0.1.0/doc/CR/CR047_jsonpit_and_pits_Require-Existing-Patch-Flag-and-Seed-Commit-Count.md +107 -0
  21. umgodi_json-0.1.0/doc/CR/CR049_AIA_and_jsonpit_to_RAIkeep_Live-ID-Validation_and_Zip-Image-Import.md +244 -0
  22. umgodi_json-0.1.0/doc/CROSS_ENGINE_PARITY_SPECIFICATION.md +223 -0
  23. umgodi_json-0.1.0/doc/EcholotConceptResearch.md +74 -0
  24. umgodi_json-0.1.0/doc/JSONPIT_VISUAL_IDENTITY_AND_ICON_SPEC.md +93 -0
  25. umgodi_json-0.1.0/doc/PREFLIGHT.md +24 -0
  26. umgodi_json-0.1.0/doc/umgodi-class-diagram.puml +312 -0
  27. umgodi_json-0.1.0/doc/umgodi-class-diagram.svg +1 -0
  28. umgodi_json-0.1.0/doc/uml/jsonpitCD.png +0 -0
  29. umgodi_json-0.1.0/doc/uml/jsonpitCD.puml +78 -0
  30. umgodi_json-0.1.0/doc/uml/jsonpitCD.svg +1 -0
  31. umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.png +0 -0
  32. umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.puml +75 -0
  33. umgodi_json-0.1.0/doc/uml/jsonpit_opsCD.svg +1 -0
  34. umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.png +0 -0
  35. umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.puml +141 -0
  36. umgodi_json-0.1.0/doc/uml/jsonpit_unifiedCD.svg +1 -0
  37. umgodi_json-0.1.0/pyproject.toml +79 -0
  38. umgodi_json-0.1.0/scripts/build_standalone.py +48 -0
  39. umgodi_json-0.1.0/scripts/preflight.py +106 -0
  40. umgodi_json-0.1.0/tests/__init__.py +1 -0
  41. umgodi_json-0.1.0/tests/fixtures/Person.pit +1 -0
  42. umgodi_json-0.1.0/tests/run_all.py +93 -0
  43. umgodi_json-0.1.0/tests/sequence_runner.py +338 -0
  44. umgodi_json-0.1.0/tests/suites/01_persistence_and_crud.json5 +152 -0
  45. umgodi_json-0.1.0/tests/suites/02_changes_and_receipts.json5 +124 -0
  46. umgodi_json-0.1.0/tests/suites/03_tombstones_and_projections.json5 +178 -0
  47. umgodi_json-0.1.0/tests/suites/04_compaction_and_events.json5 +109 -0
  48. umgodi_json-0.1.0/tests/suites/05_process_flags_and_leases.json5 +126 -0
  49. umgodi_json-0.1.0/tests/suites/06_split_master_recovery.json5 +115 -0
  50. umgodi_json-0.1.0/tests/suites/cr040_and_cross_parity.json5 +90 -0
  51. umgodi_json-0.1.0/tests/test_audit.py +323 -0
  52. umgodi_json-0.1.0/tests/test_canonical.py +58 -0
  53. umgodi_json-0.1.0/tests/test_change_file.py +97 -0
  54. umgodi_json-0.1.0/tests/test_cli.py +324 -0
  55. umgodi_json-0.1.0/tests/test_cli_export.py +88 -0
  56. umgodi_json-0.1.0/tests/test_cli_style.py +122 -0
  57. umgodi_json-0.1.0/tests/test_cr043_seed_shapes.py +189 -0
  58. umgodi_json-0.1.0/tests/test_cr044_missing_config.py +160 -0
  59. umgodi_json-0.1.0/tests/test_cr047_strict_patch.py +317 -0
  60. umgodi_json-0.1.0/tests/test_cr049_live_id_validation.py +107 -0
  61. umgodi_json-0.1.0/tests/test_cr081_seed_dispositions.py +175 -0
  62. umgodi_json-0.1.0/tests/test_cr082_cloud_export.py +195 -0
  63. umgodi_json-0.1.0/tests/test_csharp_compatibility.py +51 -0
  64. umgodi_json-0.1.0/tests/test_delete_property_projection.py +201 -0
  65. umgodi_json-0.1.0/tests/test_equal_timestamp_ordering.py +100 -0
  66. umgodi_json-0.1.0/tests/test_inqaba_boundary.py +33 -0
  67. umgodi_json-0.1.0/tests/test_live_references.py +505 -0
  68. umgodi_json-0.1.0/tests/test_maintain.py +377 -0
  69. umgodi_json-0.1.0/tests/test_master_ticket.py +83 -0
  70. umgodi_json-0.1.0/tests/test_pit_item.py +91 -0
  71. umgodi_json-0.1.0/tests/test_sequence_isolation.py +42 -0
  72. umgodi_json-0.1.0/tests/test_sequence_suites.py +26 -0
  73. umgodi_json-0.1.0/tests/test_sparse_delta.py +196 -0
  74. umgodi_json-0.1.0/umgodi/__init__.py +76 -0
  75. umgodi_json-0.1.0/umgodi/audit.py +209 -0
  76. umgodi_json-0.1.0/umgodi/canonical.py +105 -0
  77. umgodi_json-0.1.0/umgodi/changes.py +206 -0
  78. umgodi_json-0.1.0/umgodi/cli.py +1817 -0
  79. umgodi_json-0.1.0/umgodi/cli_style.py +112 -0
  80. umgodi_json-0.1.0/umgodi/config.py +7 -0
  81. umgodi_json-0.1.0/umgodi/exceptions.py +44 -0
  82. umgodi_json-0.1.0/umgodi/flags.py +272 -0
  83. umgodi_json-0.1.0/umgodi/fs.py +177 -0
  84. umgodi_json-0.1.0/umgodi/history.py +202 -0
  85. umgodi_json-0.1.0/umgodi/icons.py +63 -0
  86. umgodi_json-0.1.0/umgodi/item.py +800 -0
  87. umgodi_json-0.1.0/umgodi/store.py +1480 -0
@@ -0,0 +1,50 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # Distribution / packaging
7
+ .Python
8
+ build/
9
+ develop-eggs/
10
+ dist/
11
+ downloads/
12
+ eggs/
13
+ .eggs/
14
+ lib/
15
+ lib64/
16
+ parts/
17
+ sdist/
18
+ var/
19
+ wheels/
20
+ share/python-wheels/
21
+ *.egg-info/
22
+ .installed.cfg
23
+ *.egg
24
+ MANIFEST
25
+
26
+ # Virtual environments
27
+ .venv/
28
+ venv/
29
+ ENV/
30
+ env/
31
+
32
+ # IDE / Editor
33
+ .vscode/
34
+ .idea/
35
+ *.swp
36
+ *.swo
37
+ *~
38
+ .DS_Store
39
+
40
+ # Testing & Type checking
41
+ .pytest_cache/
42
+ .mypy_cache/
43
+ .ruff_cache/
44
+ .coverage
45
+ htmlcov/
46
+
47
+ # Local test scratch / temp
48
+ scratch/
49
+ tmp/
50
+ *.tmp
@@ -0,0 +1,239 @@
1
+ # umgodi API Reference (v0.1.0)
2
+
3
+ The `umgodi-json` distribution provides the public `umgodi` API and uses Inqaba for shared filesystem and configuration operations.
4
+
5
+ ---
6
+
7
+ ## Architectural Diagrams
8
+
9
+ ### Core Replicated Storage Architecture
10
+ <p align="center">
11
+ <img src="https://raw.githubusercontent.com/Burkhardt/umgodi-json/main/doc/uml/jsonpitCD.png" alt="Core Replicated Storage Architecture" width="700"/>
12
+ </p>
13
+
14
+ ### Storage Maintenance & Recovery Audit Architecture
15
+ <p align="center">
16
+ <img src="https://raw.githubusercontent.com/Burkhardt/umgodi-json/main/doc/uml/jsonpit_opsCD.png" alt="Operations & Recovery Audit Architecture" width="700"/>
17
+ </p>
18
+
19
+ ---
20
+
21
+ ## Pit Lifecycle & Storage
22
+
23
+ - <details>
24
+ <summary><code>Pit</code>: Live eventually consistent distributed JSON store</summary>
25
+
26
+ Coordinates one live public instance per canonical pit path, implements `MutableMapping[str, PitItem]` and the `ContextManager` protocol, observes cloud-drive changes, emits sparse delta change files, and coordinates master leases.
27
+
28
+ **Key Methods:**
29
+ - `__enter__()` / `__exit__()`: Context manager for atomic batch sessions and guaranteed flag cleanup.
30
+ - `add(item: PitItem, refresh_modified: bool = True) -> bool`: Adds or updates a living entity. Validates against prohibited template markers (`{` or `<`) under CR049.
31
+ - `add_historical(item: PitItem) -> bool`: Replays an immutable historical fragment without modifying timestamps or rejecting legacy IDs.
32
+ - `get(item_id: str, default: Any = None, at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Retrieves the projected living state of an entity, with optional time-travel projection.
33
+ - `delete_item(item_id: str, by: str | None = None, backdate_100: bool = False) -> bool`: Tombstones an entity across the distributed cluster.
34
+ - `rename_id(old_key: str, new_key: str, by: str | None = None) -> bool`: Atomically migrates an entity identity.
35
+ - `seed_from_file(file_path: Path | str, require_existing: bool = False) -> int`: Ingests an external JSON/JSON5 file (single entity, keyed map, or array) with whole-batch ID preflight and optional CR047 strict patch enforcement.
36
+ - `maintain(options: PitMaintenanceOptions = None, apply: bool = False) -> PitMaintenanceResult`: Reconciles change files, manages 10-minute receipt grace periods, prunes expired process flags (`--older-than`), and repairs legacy event extensions.
37
+ - `save(force: bool = False, pretty: bool = False) -> None`: Persists the unified canonical `.pit` file in-place using CR022 sibling writes (`.pit.tmp` followed by atomic rename).
38
+ </details>
39
+
40
+ - <details>
41
+ <summary><code>Pit.maintain(...)</code>, <code>PitMaintenanceOptions</code>, and <code>PitMaintenanceResult</code></summary>
42
+
43
+ `maintain()` inventories pending change/receipt cleanup without mutation. It checks the canonical parent first, returning deferred status for missing targets without creating directories. `maintain(apply=True)` applies canonical reconciliation and eligible change-first/receipt-second retirement (10-minute grace period).
44
+
45
+ **Maintenance Options:**
46
+ - `apply: bool`: Authorizes modifications to disk (default False: report-only preview).
47
+ - `prune_process_flags: bool`: Authorizes pruning of expired PID-window flag files. Requires `apply=True` and `older_than`.
48
+ - `older_than: timedelta`: Duration threshold for flag pruning (e.g. `01:00:00`, `7.00:00:00`).
49
+ - `repair_legacy_extensions: bool`: Migrates legacy `{stem}_{sha256}.event` files to clean `{stem}.event` in `Events/`.
50
+ - `archive_events: bool`: Compacts event logs into immutable archives.
51
+
52
+ **Maintenance Result (`to_dict()` for 100% C# parity):**
53
+ - Exposes `PitFile`, `Applied`, `Succeeded`, `ChangeFilesObserved/Merged/Removed`, `ReceiptsCreated/Removed`, `ProcessFlagsActive/Expired/Pruned`, `LegacyArtifactsObserved/Repaired`, `Deferred`, and `Failures`.
54
+ </details>
55
+
56
+ - <details>
57
+ <summary><code>JsonPitBase</code>: Shared storage and path foundation</summary>
58
+
59
+ Base class providing pit path resolution, cloud drive detection, subscriber identity, process leasing options (`read_only`, `unflagged`, `retain_window`), and canonical serialization.
60
+ </details>
61
+
62
+ - <details>
63
+ <summary><code>PitStore</code>: Thread-safe repository of open Pit instances</summary>
64
+
65
+ Factory and cache ensuring only one active `Pit` instance exists per canonical file path within a single process, preventing memory divergence.
66
+ </details>
67
+
68
+ ---
69
+
70
+ ## Entity Domain Model & History
71
+
72
+ - <details>
73
+ <summary><code>PitItem</code>: Polymorphic living entity and sparse delta emitter</summary>
74
+
75
+ Represents an individual entity within a Pit. Implements dictionary-like attribute access while protecting immutable engine attributes (`id`, `modified`, `deleted`) from accidental mutation (CR040). When bound to a `Pit`, mutations automatically stream sparse delta change files without triggering read-modify-write anti-patterns.
76
+
77
+ **Core Attributes:**
78
+ - `id`: Non-empty string identifier. Rejects unresolved template markers (`{` or `<`) at write boundaries (CR049).
79
+ - `modified`: 7-digit UTC timestamp (`datetime`) of the latest fragment.
80
+ - `deleted`: Boolean tombstone flag.
81
+ - `note`: Optional human- or agent-readable string note.
82
+
83
+ **Sparse Mutation Methods:**
84
+ - `set_property(patch: dict[str, Any] | str) -> bool`: Atomically applies a property delta and emits a sparse change file.
85
+ - `delete_property(property_name: str) -> None`: Emits a property-level null tombstone without rewriting living state.
86
+ - `delete_property_path(property_path: str) -> None`: Traverses dot-delimited paths to tombstone nested properties.
87
+ - `extend_with(obj: dict[str, Any]) -> bool`: Merges attributes from a dictionary.
88
+ - `to_dict() -> dict[str, Any]` / `to_json() -> str`: Canonical JSON serialization.
89
+ </details>
90
+
91
+ - <details>
92
+ <summary><code>PitItems</code>: Immutable fragment stack and time-travel projector</summary>
93
+
94
+ Maintains an entity's ordered history of timestamped fragments.
95
+
96
+ **Key Methods:**
97
+ - `push(item: PitItem) -> PitItems`: Returns a new `PitItems` stack with the fragment prepended.
98
+ - `latest_fragment() -> PitItem | None`: Returns the most recently written fragment.
99
+ - `project_state(at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Merges historical fragments from oldest to newest up to timestamp `at` to compute accurate point-in-time living state.
100
+ </details>
101
+
102
+ ---
103
+
104
+ ## Distributed Protocol & Cloud Safety
105
+
106
+ - <details>
107
+ <summary><code>ChangeFile</code>: Shortened cloud-safe change artifact</summary>
108
+
109
+ Writes sparse delta mutations into `Changes/` using the 4-character hex checksum format:
110
+ `{UtcTicks}_{ExactProcessIdentity}_{4charHex}.json`
111
+ Guards against partially synchronized cloud writes (e.g. OneDrive) without exceeding path length limits. Validates checksums on read and maintains backwards compatibility with legacy 64-character SHA-suffixed change files.
112
+ </details>
113
+
114
+ - <details>
115
+ <summary><code>ReceiptFile</code>: Durable round-trip confirmation</summary>
116
+
117
+ Pairs with change files using the `.receipt` extension. Enforces a 10-minute grace period before allowing retired change files to be compacted or purged.
118
+ </details>
119
+
120
+ - <details>
121
+ <summary><code>MasterFlagFile</code> & <code>ProcessFlagFile</code>: Cloud drive lease coordination</summary>
122
+
123
+ - `Master.flag`: 60-second lease renewed every 30 seconds by the active master writer.
124
+ - `{Machine}-{Process}-{PID}.flag`: Ephemeral process activity flag with heartbeat updates. Enforces deterministic self-cleanup on process exit (CR024).
125
+ </details>
126
+
127
+ - <details>
128
+ <summary><code>EventFile</code>: Clean event compaction naming & migration</summary>
129
+
130
+ Emits compacted historical events under clean logical stems (`{LogicalStem}.event`) without SHA-256 hash suffixes. Automatically recognizes legacy 64-char hash filenames and supports in-place migration.
131
+ </details>
132
+
133
+ ---
134
+
135
+ ## Configuration & Cloud Discovery
136
+
137
+ - <details>
138
+ <summary><code>OsConfig</code>: Cross-platform cloud drive discovery</summary>
139
+
140
+ Discovers synchronization folders for `Dropbox`, `OneDrive`, `GoogleDrive`, and `ICloudDrive`. Reads primarily from `~/.config/RAIkeep.json5` with fallback to `~/.config/jsonpit.json5` or `$JSONPIT_CONFIG`.
141
+ </details>
142
+
143
+ - <details>
144
+ <summary><code>missing_configuration_diagnostic()</code>: CR044 operator guidance</summary>
145
+
146
+ When cloud drives or configuration files are absent, formats clear actionable instructions directing the operator to run `amafu init` to bootstrap configuration.
147
+ </details>
148
+
149
+ ---
150
+
151
+ ## Audit Engine & Recovery Inspection
152
+
153
+ - <details>
154
+ <summary><code>PitAudit</code>: Read-only inspection of durable event logs</summary>
155
+
156
+ Provides read-only access to durable JsonPit recovery events under `Events/` without opening a `Pit`, creating process activity flags, acquiring leases, or mutating disk state (CR003). Inspects both loose `*.event` files and immutable `Events_*.zip` compaction archives.
157
+
158
+ **Key Methods:**
159
+ - `PitAudit.inspect(pit_directory: Path | str, machine_filter: str = "all", min_level: LogLevel = LogLevel.TRACE) -> PitAuditReadResult`: Reads, validates, deduplicates by `EventId`, filters by machine/severity, and orders events deterministically by machine, UTC time, and event identity.
160
+ </details>
161
+
162
+ - <details>
163
+ <summary><code>PitAuditEvent</code>: Durable recovery event domain model</summary>
164
+
165
+ Encapsulates a single durable recovery event:
166
+ - `file_name: str`: Source file or archive entry name.
167
+ - `content: dict[str, Any]`: Raw deserialized event JSON object.
168
+ - `machine: str`: Originating machine hostname.
169
+ - `utc_time: datetime`: Event occurrence timestamp in UTC.
170
+ - `level: LogLevel`: Severity level.
171
+ - `stage: str`: Pit lifecycle stage (e.g. `ChangeFilesPublished`, `CleanupPending`, `Completed`).
172
+ - `event_id: str`: Unique GUID event identifier used for cross-archive deduplication.
173
+ - `message: str`: Human-readable summary message.
174
+ </details>
175
+
176
+ - <details>
177
+ <summary><code>LogLevel</code>: Event severity enumeration</summary>
178
+
179
+ Matches .NET `LogLevel` integer values and names:
180
+ - `TRACE = 0` (`Trace`)
181
+ - `DEBUG = 1` (`Debug`)
182
+ - `INFORMATION = 2` (`Information` / `info`)
183
+ - `WARNING = 3` (`Warning` / `warn`)
184
+ - `ERROR = 4` (`Error` / `err`)
185
+ - `CRITICAL = 5` (`Critical` / `fatal`)
186
+ - `from_string(name: str) -> LogLevel`: Case-insensitive parsing supporting names, aliases, and integer strings.
187
+ - `display_name -> str`: Capitalized canonical name matching C# output formatting.
188
+ </details>
189
+
190
+ - <details>
191
+ <summary><code>PitAuditReadResult</code>: Filtered events and diagnostic issues</summary>
192
+
193
+ Contains:
194
+ - `events: list[PitAuditEvent]`: Filtered and ordinally sorted events.
195
+ - `issues: list[str]`: Physical corruption, empty file, or conflicting event identity warnings.
196
+ - `succeeded: bool`: Returns `True` if no diagnostic issues were discovered.
197
+ </details>
198
+
199
+ ---
200
+
201
+ ## Exception Hierarchy
202
+
203
+ All `umgodi` exceptions derive from `JsonPitError`:
204
+
205
+ ```text
206
+ JsonPitError
207
+ ├── PitNotFoundError # Target pit directory or file does not exist
208
+ ├── PitCorruptError # Payload failed checksum or schema validation
209
+ ├── PitConcurrencyError # Conflict acquiring or renewing master lease
210
+ ├── PitInstanceConflictError # Multiple live Pit instances for the same file path
211
+ ├── ProtectedAttributeError # Client attempted to mutate 'id', 'modified', or 'deleted' (CR040)
212
+ ├── TombstoneError # Attempted to delete protected engine attributes
213
+ └── StrictPatchValidationError # Missing or tombstoned entity under --id-exists / --patch (CR047)
214
+ ```
215
+
216
+
217
+ ## CR077 mutation contract (4.6.0)
218
+
219
+ Live handles remain ordinary PitItems. Use indexing, set_property, extend_with,
220
+ merge, delete_property and delete_property_path; save remains the persistence
221
+ boundary. Re-adding an owned live handle flushes pending differences without
222
+ replaying its full projection. PitItem(existing_item) is detached; adding that
223
+ copy as client data is rejected when it carries lifecycle metadata. A matching
224
+ Id assertion in a patch is supported. JSON-array extend validates every row before
225
+ acceptance and creates one combined sparse mutation. Vikela is optional developer
226
+ tooling and is not a jsonpit runtime dependency.
227
+
228
+ ## CR081 CLI receipts and lifecycle failures
229
+
230
+ `jpit seed` and `jpit put` report source-entry written/unchanged counts after
231
+ explicit Save and context exit. A failed local save or required owned-window
232
+ cleanup propagates to a nonzero CLI exit without a success receipt. `close()`
233
+ now raises OSError if the owned activity flag remains after an unsuccessful
234
+ release; `retain_window=True` preserves its explicit release opt-out.
235
+
236
+ `add()` matches C# latest-fragment comparison ignoring Modified before refreshing
237
+ timestamps. Identical latest fragments return False. Projected-state matches with
238
+ a different latest fragment still append; historical and tombstone semantics
239
+ remain covered by parity tests.
@@ -0,0 +1,280 @@
1
+ # Getting Started with Python umgodi-json (v0.1.0)
2
+
3
+ This guide provides a practical walkthrough for using **`umgodi`** from Python—either embedded as a library in your services, AI agent frameworks, and data pipelines, or as the standalone **`jpit`** CLI for command-line workflows.
4
+
5
+ It is based on `umgodi` v0.1.0, featuring 100% lockstep parity with C# `JsonPit` in [RAIkeep](https://github.com/Burkhardt/RAIkeep).
6
+
7
+ ---
8
+
9
+ ## 1. Purpose and Mental Model
10
+
11
+ `umgodi` is a cloud-first, file-based, distributed replicated storage engine for `Id`-identified items that need to be shared, synchronized, and reloaded across machines and processes without introducing a database daemon.
12
+
13
+ ### The Core Concepts:
14
+ - **`Pit`**: A named container directory stored on disk (e.g. `Person/Person.pit`).
15
+ - **`PitItem`**: An `Id`-identified JSON object with lifecycle metadata (`Modified`, `Deleted`, `Note`) and arbitrary properties.
16
+ - **`PitItems`**: The immutable version history of sparse change fragments for an item key.
17
+
18
+ ### Mental Model:
19
+ - A `Pit` is closer to a synchronized document store than a SQL table.
20
+ - Updates are **append-only sparse fragments**, not blind in-memory overwrites.
21
+ - Coordination happens through synchronized cloud storage (OneDrive, Dropbox, GoogleDrive, ICloudDrive) using lease flags and change files.
22
+ - **[Asynchronous persistence with eventual durability](https://github.com/Burkhardt/RAIkeep/blob/main/MANIFESTO.md#asynchronous-persistence-with-eventual-durability)**: request handling happens in memory; synchronization is explicit via `save()` or context manager exit.
23
+
24
+ ---
25
+
26
+ ## 3. Installing the CLI or Python Library
27
+
28
+ For the CLI, install one pipx environment that supplies both commands:
29
+
30
+ ```bash
31
+ pipx install "umgodi-json==0.1.0"
32
+ # For an existing environment:
33
+ pipx upgrade umgodi-json
34
+ jpit --version
35
+ umgodi --version
36
+ ```
37
+
38
+ The outputs are `jpit v0.1.0` and `umgodi v0.1.0`. If a separate `jpit` pipx
39
+ environment already exists, run `pipx uninstall jpit`, followed by
40
+ `pipx install --force "umgodi-json==0.1.0"` to restore both command links.
41
+
42
+ Ubuntu/Debian and Homebrew may reject system-wide pip installs as externally
43
+ managed. Use pipx for the CLI, or an isolated virtual environment for library use:
44
+
45
+ ```bash
46
+ python3 -m venv .venv
47
+ source .venv/bin/activate
48
+ python -m pip install "umgodi-json==0.1.0"
49
+ ```
50
+
51
+ ---
52
+
53
+ ## 4. Configuration & Simplified Cloud Shortcuts
54
+
55
+ `umgodi` automatically reads your cloud provider roots from `~/.config/RAIkeep.json5` (with fallback to `~/.config/jsonpit.json5` or `$JSONPIT_CONFIG`).
56
+
57
+ ### Zero-Friction Setup via [`amafu`](https://www.nuget.org/packages/Amafu)
58
+
59
+ On macOS and Linux, enable your cloud clients (iCloud, OneDrive, Google Drive, Dropbox), then use [`amafu`](https://www.nuget.org/packages/Amafu) to auto-detect your local sync folders and create clean, unified shortcuts in `~/.CloudStorage/`:
60
+
61
+ ```bash
62
+ # 1. Preview detected roots and proposed shortcuts:
63
+ amafu detect --create-links --dry-run
64
+
65
+ # 2. Initialize configuration AND create shortcuts in ~/.CloudStorage/:
66
+ amafu init --create-links
67
+ ```
68
+
69
+ If you already have a configuration and just want to create the shortcuts:
70
+ ```bash
71
+ amafu detect --create-links
72
+ ```
73
+
74
+ This creates convenient, unified symbolic links:
75
+ - `~/.CloudStorage/ICloudDrive` $\rightarrow$ `~/Library/Mobile Documents/com~apple~CloudDocs/`
76
+ - `~/.CloudStorage/GoogleDrive` $\rightarrow$ `~/Library/CloudStorage/GoogleDrive-.../My Drive/`
77
+ - `~/.CloudStorage/OneDrive` $\rightarrow$ `~/Library/CloudStorage/OneDrive-.../`
78
+ - `~/.CloudStorage/Dropbox` $\rightarrow$ `~/Library/CloudStorage/Dropbox/`
79
+
80
+ You never have to search for obscure operating system cloud paths again. Your real files stay safely in their original cloud folders, and your Python code or terminal sessions can easily use `~/.CloudStorage/<provider>` or reference the provider name directly (e.g. `cloud="ICloudDrive"`).
81
+
82
+ ---
83
+
84
+ ## 5. First Steps with the Python Library
85
+
86
+ ### Basic End-to-End Example
87
+ Here is the recommended Python idiom using the `Pit.open()` context manager:
88
+
89
+ ```python
90
+ from umgodi import Pit, PitItem
91
+
92
+ # 1. Open the Pit using the context manager
93
+ # (Acquires a process activity window, loads state, and auto-saves on exit)
94
+ with Pit.open("person", cloud="GoogleDrive") as people:
95
+
96
+ # 2. Create an entity with Id
97
+ max_person = PitItem("Max")
98
+ max_person["Email"] = "max@example.org"
99
+ max_person["Phone"] = "+27-82-000-0000"
100
+ max_person["ComPref"] = ["WhatsApp", "Email"]
101
+
102
+ # 3. Add to Pit
103
+ people.add(max_person)
104
+ print(f"Added {max_person.id} to Pit '{people.pit_name}'")
105
+
106
+ # --- On context manager exit, people.save() is called automatically ---
107
+
108
+ # 4. Reopen and query
109
+ with Pit.open("person", cloud="GoogleDrive", read_only=True) as people:
110
+ loaded_max = people["Max"]
111
+ print(f"Loaded: {loaded_max.id} -> Email: {loaded_max['Email']}, Phone: {loaded_max['Phone']}")
112
+ print(f"Channels: {loaded_max['ComPref']}")
113
+ ```
114
+
115
+ ### Updating an Existing Item
116
+ Looking up an entity returns a **live reference**. Mutating that reference records sparse modifications directly:
117
+
118
+ ```python
119
+ with Pit.open("person", cloud="GoogleDrive") as people:
120
+ max_person = people["Max"]
121
+
122
+ # Direct indexer updates append sparse fragments
123
+ max_person["Phone"] = "+27-82-111-2222"
124
+ max_person["Instagram"] = "max.africastage"
125
+
126
+ # Nested dictionaries are deep-merged
127
+ max_person["Address"] = {
128
+ "Street": "42 Long Street",
129
+ "City": "Cape Town",
130
+ "Country": "South Africa"
131
+ }
132
+ ```
133
+
134
+ ### Removing Attributes (Tombstones)
135
+ `umgodi` never mutates history destructively. Deleting a property appends a **null tombstone** that masks previous values in the projection while preserving time travel:
136
+
137
+ ```python
138
+ with Pit.open("person", cloud="GoogleDrive") as people:
139
+ max_person = people["Max"]
140
+
141
+ # Remove a top-level property
142
+ max_person.delete_property("Instagram")
143
+
144
+ # Remove a nested path
145
+ max_person.delete_property_path("Address.Street")
146
+ ```
147
+
148
+ ### Deleting an Entity
149
+ ```python
150
+ with Pit.open("person", cloud="GoogleDrive") as people:
151
+ # Tombstones the entire entity
152
+ del people["Max"]
153
+ # or: people.delete_item("Max", by="Operator")
154
+ ```
155
+
156
+ ### Time Travel and Historical Auditing
157
+ Because every modification is an immutable timestamped fragment, you can inspect historical states:
158
+
159
+ ```python
160
+ with Pit.open("person", cloud="GoogleDrive", read_only=True) as people:
161
+ # Inspect raw version history
162
+ history = people.history["Max"]
163
+ for frag in history.history:
164
+ print(f"[{frag.timestamp}] {frag.data}")
165
+
166
+ # Reconstruct the entity as it existed at a past point in time
167
+ past_state = people.get_at("Max", at="2026-10-01T12:00:00Z")
168
+ if past_state:
169
+ print("Past Phone:", past_state.get("Phone"))
170
+ ```
171
+
172
+ ---
173
+
174
+ ## 6. Alternative: First Steps with the `jpit` CLI
175
+
176
+ If you prefer to operate directly from the command line without writing Python code, the `jpit` CLI provides high-performance data seeding, mutation, and inspection.
177
+
178
+ ### Step 1: Seed Initial Data
179
+ Create a JSON or JSON5 file (`people.json5`):
180
+
181
+ ```json5
182
+ [
183
+ {
184
+ Id: "Max",
185
+ Email: "max@example.org",
186
+ Phone: "+27-82-000-0000",
187
+ ComPref: ["WhatsApp", "Email"]
188
+ },
189
+ {
190
+ Id: "Adele",
191
+ Email: "adele@example.org",
192
+ Role: "Software Architect"
193
+ }
194
+ ]
195
+ ```
196
+
197
+ Seed the entities into a Pit named `person`:
198
+ ```bash
199
+ jpit seed person people.json5
200
+ ```
201
+
202
+ ### Step 2: Read Entities
203
+ Get one entity formatted as JSON:
204
+ ```bash
205
+ jpit get person Max
206
+ ```
207
+
208
+ Get only a specific property:
209
+ ```bash
210
+ jpit get person Max --property Email
211
+ ```
212
+
213
+ List all active entities in the pit:
214
+ ```bash
215
+ jpit list person
216
+ ```
217
+
218
+ ### Step 3: Sparse Mutations (Anti-Read-Modify-Write)
219
+ Update attributes without rewriting the entire entity:
220
+
221
+ ```bash
222
+ jpit set person Max Phone "+27-82-111-2222"
223
+ ```
224
+
225
+ Add a nested object:
226
+ ```bash
227
+ jpit set person Max Address '{"City": "Cape Town", "Country": "South Africa"}'
228
+ ```
229
+
230
+ Tombstone an attribute:
231
+ ```bash
232
+ jpit del-prop person Max Email
233
+ ```
234
+
235
+ ### Step 4: Semantic Search ("Pit-Grep")
236
+ Search across living entity states using ripgrep-style speed:
237
+
238
+ ```bash
239
+ jpit grep "Cape Town" person
240
+ ```
241
+
242
+ Filter by property and output structured JSON:
243
+ ```bash
244
+ jpit grep --property Role "Architect" person --json
245
+ ```
246
+
247
+ ### Step 5: Exporting & Piping with `jq`
248
+ Pipe living state directly into `jq`:
249
+ ```bash
250
+ jpit export person --jq '.[].Id'
251
+ ```
252
+
253
+ ---
254
+
255
+ ## 7. 100% C# / `pits` Cross-Language Interoperability
256
+
257
+ Every Pit created or updated by Python `umgodi` is 100% binary- and schema-compatible with C# `JsonPit` and the `pits` CLI.
258
+
259
+ Write an entity attribute with Python `jpit`:
260
+ ```bash
261
+ jpit set person Max Status "Online"
262
+ ```
263
+
264
+ Read it immediately with C# `pits`:
265
+ ```bash
266
+ pits get person Max
267
+ ```
268
+
269
+ Both engines enforce identical timestamp parsing, canonical key ordering, lease flag acquisition, receipt grace periods, and sparse delta projection.
270
+
271
+ ---
272
+
273
+ ## 8. Alternative Installation & Troubleshooting
274
+
275
+ ### Shell PATH troubleshooting
276
+ If your shell reports `command not found: jpit`, ensure that `~/.local/bin` is in your shell `PATH`:
277
+ ```bash
278
+ echo 'export PATH="$HOME/.local/bin:$HOME/.dotnet/tools:$PATH"' >> ~/.zshrc
279
+ source ~/.zshrc
280
+ ```