jsonpit 4.4.4__tar.gz → 4.4.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jsonpit-4.4.6/API.md +200 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/PKG-INFO +33 -6
- {jsonpit-4.4.4 → jsonpit-4.4.6}/README.md +31 -4
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/__init__.py +22 -6
- jsonpit-4.4.6/jsonpit/audit.py +209 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/changes.py +17 -14
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/cli.py +468 -17
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/exceptions.py +4 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/item.py +76 -9
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/store.py +322 -4
- {jsonpit-4.4.4 → jsonpit-4.4.6}/pyproject.toml +3 -2
- {jsonpit-4.4.4 → jsonpit-4.4.6}/.gitignore +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/canonical.py +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/config.py +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/flags.py +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/fs.py +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/history.py +0 -0
- {jsonpit-4.4.4 → jsonpit-4.4.6}/jsonpit/icons.py +0 -0
jsonpit-4.4.6/API.md
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# jsonpit API Reference (v4.4.6)
|
|
2
|
+
|
|
3
|
+
This document provides a foldable, searchable reference for the public `jsonpit` Python API. Designed for 100% C# JsonPit parity, zero third-party runtime dependencies, and Smalltalk-grade object-oriented architecture.
|
|
4
|
+
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
## Pit Lifecycle & Storage
|
|
8
|
+
|
|
9
|
+
- <details>
|
|
10
|
+
<summary><code>Pit</code>: Live eventually consistent distributed JSON store</summary>
|
|
11
|
+
|
|
12
|
+
Coordinates one live public instance per canonical pit path, implements `MutableMapping[str, PitItem]` and the `ContextManager` protocol, observes cloud-drive changes, emits sparse delta change files, and coordinates master leases.
|
|
13
|
+
|
|
14
|
+
**Key Methods:**
|
|
15
|
+
- `__enter__()` / `__exit__()`: Context manager for atomic batch sessions and guaranteed flag cleanup.
|
|
16
|
+
- `add(item: PitItem, refresh_modified: bool = True) -> bool`: Adds or updates a living entity. Validates against prohibited template markers (`{` or `<`) under CR049.
|
|
17
|
+
- `add_historical(item: PitItem) -> bool`: Replays an immutable historical fragment without modifying timestamps or rejecting legacy IDs.
|
|
18
|
+
- `get(item_id: str, default: Any = None, at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Retrieves the projected living state of an entity, with optional time-travel projection.
|
|
19
|
+
- `delete_item(item_id: str, by: str | None = None, backdate_100: bool = False) -> bool`: Tombstones an entity across the distributed cluster.
|
|
20
|
+
- `rename_id(old_key: str, new_key: str, by: str | None = None) -> bool`: Atomically migrates an entity identity.
|
|
21
|
+
- `seed_from_file(file_path: Path | str, require_existing: bool = False) -> int`: Ingests an external JSON/JSON5 file (single entity, keyed map, or array) with whole-batch ID preflight and optional CR047 strict patch enforcement.
|
|
22
|
+
- `maintain(options: PitMaintenanceOptions = None, apply: bool = False) -> PitMaintenanceResult`: Reconciles change files, manages 10-minute receipt grace periods, prunes expired process flags (`--older-than`), and repairs legacy event extensions.
|
|
23
|
+
- `save(force: bool = False, pretty: bool = False) -> None`: Persists the unified canonical `.pit` file in-place using CR022 sibling writes (`.pit.tmp` followed by atomic rename).
|
|
24
|
+
</details>
|
|
25
|
+
|
|
26
|
+
- <details>
|
|
27
|
+
<summary><code>Pit.maintain(...)</code>, <code>PitMaintenanceOptions</code>, and <code>PitMaintenanceResult</code></summary>
|
|
28
|
+
|
|
29
|
+
`maintain()` inventories pending change/receipt cleanup without mutation. It checks the canonical parent first, returning deferred status for missing targets without creating directories. `maintain(apply=True)` applies canonical reconciliation and eligible change-first/receipt-second retirement (10-minute grace period).
|
|
30
|
+
|
|
31
|
+
**Maintenance Options:**
|
|
32
|
+
- `apply: bool`: Authorizes modifications to disk (default False: report-only preview).
|
|
33
|
+
- `prune_process_flags: bool`: Authorizes pruning of expired PID-window flag files. Requires `apply=True` and `older_than`.
|
|
34
|
+
- `older_than: timedelta`: Duration threshold for flag pruning (e.g. `01:00:00`, `7.00:00:00`).
|
|
35
|
+
- `repair_legacy_extensions: bool`: Migrates legacy `{stem}_{sha256}.event` files to clean `{stem}.event` in `Events/`.
|
|
36
|
+
- `archive_events: bool`: Compacts event logs into immutable archives.
|
|
37
|
+
|
|
38
|
+
**Maintenance Result (`to_dict()` for 100% C# parity):**
|
|
39
|
+
- Exposes `PitFile`, `Applied`, `Succeeded`, `ChangeFilesObserved/Merged/Removed`, `ReceiptsCreated/Removed`, `ProcessFlagsActive/Expired/Pruned`, `LegacyArtifactsObserved/Repaired`, `Deferred`, and `Failures`.
|
|
40
|
+
</details>
|
|
41
|
+
|
|
42
|
+
- <details>
|
|
43
|
+
<summary><code>JsonPitBase</code>: Shared storage and path foundation</summary>
|
|
44
|
+
|
|
45
|
+
Base class providing pit path resolution, cloud drive detection, subscriber identity, process leasing options (`read_only`, `unflagged`, `retain_window`), and canonical serialization.
|
|
46
|
+
</details>
|
|
47
|
+
|
|
48
|
+
- <details>
|
|
49
|
+
<summary><code>PitStore</code>: Thread-safe repository of open Pit instances</summary>
|
|
50
|
+
|
|
51
|
+
Factory and cache ensuring only one active `Pit` instance exists per canonical file path within a single process, preventing memory divergence.
|
|
52
|
+
</details>
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Entity Domain Model & History
|
|
57
|
+
|
|
58
|
+
- <details>
|
|
59
|
+
<summary><code>PitItem</code>: Polymorphic living entity and sparse delta emitter</summary>
|
|
60
|
+
|
|
61
|
+
Represents an individual entity within a Pit. Implements dictionary-like attribute access while protecting immutable engine attributes (`id`, `modified`, `deleted`) from accidental mutation (CR040). When bound to a `Pit`, mutations automatically stream sparse delta change files without triggering read-modify-write anti-patterns.
|
|
62
|
+
|
|
63
|
+
**Core Attributes:**
|
|
64
|
+
- `id`: Non-empty string identifier. Rejects unresolved template markers (`{` or `<`) at write boundaries (CR049).
|
|
65
|
+
- `modified`: 7-digit UTC timestamp (`datetime`) of the latest fragment.
|
|
66
|
+
- `deleted`: Boolean tombstone flag.
|
|
67
|
+
- `note`: Optional human- or agent-readable string note.
|
|
68
|
+
|
|
69
|
+
**Sparse Mutation Methods:**
|
|
70
|
+
- `set_property(patch: dict[str, Any] | str) -> bool`: Atomically applies a property delta and emits a sparse change file.
|
|
71
|
+
- `delete_property(property_name: str) -> None`: Emits a property-level null tombstone without rewriting living state.
|
|
72
|
+
- `delete_property_path(property_path: str) -> None`: Traverses dot-delimited paths to tombstone nested properties.
|
|
73
|
+
- `extend_with(obj: dict[str, Any]) -> bool`: Merges attributes from a dictionary.
|
|
74
|
+
- `to_dict() -> dict[str, Any]` / `to_json() -> str`: Canonical JSON serialization.
|
|
75
|
+
</details>
|
|
76
|
+
|
|
77
|
+
- <details>
|
|
78
|
+
<summary><code>PitItems</code>: Immutable fragment stack and time-travel projector</summary>
|
|
79
|
+
|
|
80
|
+
Maintains an entity's ordered history of timestamped fragments.
|
|
81
|
+
|
|
82
|
+
**Key Methods:**
|
|
83
|
+
- `push(item: PitItem) -> PitItems`: Returns a new `PitItems` stack with the fragment prepended.
|
|
84
|
+
- `latest_fragment() -> PitItem | None`: Returns the most recently written fragment.
|
|
85
|
+
- `project_state(at: datetime = None, with_deleted: bool = False) -> PitItem | None`: Merges historical fragments from oldest to newest up to timestamp `at` to compute accurate point-in-time living state.
|
|
86
|
+
</details>
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Distributed Protocol & Cloud Safety
|
|
91
|
+
|
|
92
|
+
- <details>
|
|
93
|
+
<summary><code>ChangeFile</code>: Shortened cloud-safe change artifact</summary>
|
|
94
|
+
|
|
95
|
+
Writes sparse delta mutations into `Changes/` using the 4-character hex checksum format:
|
|
96
|
+
`{UtcTicks}_{ExactProcessIdentity}_{4charHex}.json`
|
|
97
|
+
Guards against partially synchronized cloud writes (e.g. OneDrive) without exceeding path length limits. Validates checksums on read and maintains backwards compatibility with legacy 64-character SHA-suffixed change files.
|
|
98
|
+
</details>
|
|
99
|
+
|
|
100
|
+
- <details>
|
|
101
|
+
<summary><code>ReceiptFile</code>: Durable round-trip confirmation</summary>
|
|
102
|
+
|
|
103
|
+
Pairs with change files using the `.receipt` extension. Enforces a 10-minute grace period before allowing retired change files to be compacted or purged.
|
|
104
|
+
</details>
|
|
105
|
+
|
|
106
|
+
- <details>
|
|
107
|
+
<summary><code>MasterFlagFile</code> & <code>ProcessFlagFile</code>: Cloud drive lease coordination</summary>
|
|
108
|
+
|
|
109
|
+
- `Master.flag`: 60-second lease renewed every 30 seconds by the active master writer.
|
|
110
|
+
- `{Machine}-{Process}-{PID}.flag`: Ephemeral process activity flag with heartbeat updates. Enforces deterministic self-cleanup on process exit (CR024).
|
|
111
|
+
</details>
|
|
112
|
+
|
|
113
|
+
- <details>
|
|
114
|
+
<summary><code>EventFile</code>: Clean event compaction naming & migration</summary>
|
|
115
|
+
|
|
116
|
+
Emits compacted historical events under clean logical stems (`{LogicalStem}.event`) without SHA-256 hash suffixes. Automatically recognizes legacy 64-char hash filenames and supports in-place migration.
|
|
117
|
+
</details>
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## Configuration & Cloud Discovery
|
|
122
|
+
|
|
123
|
+
- <details>
|
|
124
|
+
<summary><code>OsConfig</code>: Cross-platform cloud drive discovery</summary>
|
|
125
|
+
|
|
126
|
+
Discovers synchronization folders for `Dropbox`, `OneDrive`, `GoogleDrive`, and `ICloudDrive`. Reads primarily from `~/.config/RAIkeep.json5` with fallback to `~/.config/jsonpit.json5` or `$JSONPIT_CONFIG`.
|
|
127
|
+
</details>
|
|
128
|
+
|
|
129
|
+
- <details>
|
|
130
|
+
<summary><code>missing_configuration_diagnostic()</code>: CR044 operator guidance</summary>
|
|
131
|
+
|
|
132
|
+
When cloud drives or configuration files are absent, formats clear actionable instructions directing the operator to run `amafu init` to bootstrap configuration.
|
|
133
|
+
</details>
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## Audit Engine & Recovery Inspection
|
|
138
|
+
|
|
139
|
+
- <details>
|
|
140
|
+
<summary><code>PitAudit</code>: Read-only inspection of durable event logs</summary>
|
|
141
|
+
|
|
142
|
+
Provides read-only access to durable JsonPit recovery events under `Events/` without opening a `Pit`, creating process activity flags, acquiring leases, or mutating disk state (CR003, coordinated v3.13.2). Inspects both loose `*.event` files and immutable `Events_*.zip` compaction archives.
|
|
143
|
+
|
|
144
|
+
**Key Methods:**
|
|
145
|
+
- `PitAudit.inspect(pit_directory: Path | str, machine_filter: str = "all", min_level: LogLevel = LogLevel.TRACE) -> PitAuditReadResult`: Reads, validates, deduplicates by `EventId`, filters by machine/severity, and orders events deterministically by machine, UTC time, and event identity.
|
|
146
|
+
</details>
|
|
147
|
+
|
|
148
|
+
- <details>
|
|
149
|
+
<summary><code>PitAuditEvent</code>: Durable recovery event domain model</summary>
|
|
150
|
+
|
|
151
|
+
Encapsulates a single durable recovery event:
|
|
152
|
+
- `file_name: str`: Source file or archive entry name.
|
|
153
|
+
- `content: dict[str, Any]`: Raw deserialized event JSON object.
|
|
154
|
+
- `machine: str`: Originating machine hostname.
|
|
155
|
+
- `utc_time: datetime`: Event occurrence timestamp in UTC.
|
|
156
|
+
- `level: LogLevel`: Severity level.
|
|
157
|
+
- `stage: str`: Pit lifecycle stage (e.g. `ChangeFilesPublished`, `CleanupPending`, `Completed`).
|
|
158
|
+
- `event_id: str`: Unique GUID event identifier used for cross-archive deduplication.
|
|
159
|
+
- `message: str`: Human-readable summary message.
|
|
160
|
+
</details>
|
|
161
|
+
|
|
162
|
+
- <details>
|
|
163
|
+
<summary><code>LogLevel</code>: Event severity enumeration</summary>
|
|
164
|
+
|
|
165
|
+
Matches .NET `LogLevel` integer values and names:
|
|
166
|
+
- `TRACE = 0` (`Trace`)
|
|
167
|
+
- `DEBUG = 1` (`Debug`)
|
|
168
|
+
- `INFORMATION = 2` (`Information` / `info`)
|
|
169
|
+
- `WARNING = 3` (`Warning` / `warn`)
|
|
170
|
+
- `ERROR = 4` (`Error` / `err`)
|
|
171
|
+
- `CRITICAL = 5` (`Critical` / `fatal`)
|
|
172
|
+
- `from_string(name: str) -> LogLevel`: Case-insensitive parsing supporting names, aliases, and integer strings.
|
|
173
|
+
- `display_name -> str`: Capitalized canonical name matching C# output formatting.
|
|
174
|
+
</details>
|
|
175
|
+
|
|
176
|
+
- <details>
|
|
177
|
+
<summary><code>PitAuditReadResult</code>: Filtered events and diagnostic issues</summary>
|
|
178
|
+
|
|
179
|
+
Contains:
|
|
180
|
+
- `events: list[PitAuditEvent]`: Filtered and ordinally sorted events.
|
|
181
|
+
- `issues: list[str]`: Physical corruption, empty file, or conflicting event identity warnings.
|
|
182
|
+
- `succeeded: bool`: Returns `True` if no diagnostic issues were discovered.
|
|
183
|
+
</details>
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## Exception Hierarchy
|
|
188
|
+
|
|
189
|
+
All `jsonpit` exceptions derive from `JsonPitError`:
|
|
190
|
+
|
|
191
|
+
```text
|
|
192
|
+
JsonPitError
|
|
193
|
+
├── PitNotFoundError # Target pit directory or file does not exist
|
|
194
|
+
├── PitCorruptError # Payload failed checksum or schema validation
|
|
195
|
+
├── PitConcurrencyError # Conflict acquiring or renewing master lease
|
|
196
|
+
├── PitInstanceConflictError # Multiple live Pit instances for the same file path
|
|
197
|
+
├── ProtectedAttributeError # Client attempted to mutate 'id', 'modified', or 'deleted' (CR040)
|
|
198
|
+
├── TombstoneError # Attempted to delete protected engine attributes
|
|
199
|
+
└── StrictPatchValidationError # Missing or tombstoned entity under --require-existing (CR047)
|
|
200
|
+
```
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: jsonpit
|
|
3
|
-
Version: 4.4.
|
|
3
|
+
Version: 4.4.6
|
|
4
4
|
Summary: Daemon-free distributed storage over Cloud Drives for developers and AI agents — multi-process, immutable history, zero dependencies.
|
|
5
5
|
Author: Dr. Rainer Burkhardt
|
|
6
6
|
License-Expression: Apache-2.0
|
|
7
|
-
Classifier: Development Status ::
|
|
7
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
8
8
|
Classifier: Intended Audience :: Developers
|
|
9
9
|
Classifier: Operating System :: OS Independent
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -31,14 +31,14 @@ The canonical **reference (lead) implementation** is C# `JsonPit` and the `pits`
|
|
|
31
31
|
|
|
32
32
|
---
|
|
33
33
|
|
|
34
|
-
## Cross-Tool & Cross-Language Version Alignment (`v4.4.
|
|
34
|
+
## Cross-Tool & Cross-Language Version Alignment (`v4.4.6`)
|
|
35
35
|
|
|
36
|
-
`jsonpit` and the `jpit` CLI are version-synchronized to **`v4.4.
|
|
36
|
+
`jsonpit` and the `jpit` CLI are version-synchronized to **`v4.4.6`** in direct lockstep with C# `JsonPit` and `pits 4.4.6` in [RAIkeep](https://github.com/Burkhardt/RAIkeep).
|
|
37
37
|
|
|
38
38
|
### Why Harmonized Versioning is Essential:
|
|
39
39
|
- **Instant Compatibility Clarity:** Operators, DevOps pipelines, and autonomous AI agents immediately know whether `jpit` and `pits` share the exact same distributed protocol revision. If both tools report `4.4.x`, they guarantee identical master leasing rules, receipt mechanisms, 7-digit UTC timestamp parsing, and CR041 clean change file conventions.
|
|
40
40
|
- **Elimination of Multi-Language Matrix Confusion:** In heterogeneous environments (e.g. C# backend daemons paired with Python agent orchestrators and data science tooling), unified SemVer tags eliminate complex cross-compatibility lookup tables.
|
|
41
|
-
- **Joint Protocol Governance:** Architectural changes
|
|
41
|
+
- **Joint Protocol Governance:** Architectural changes—such as CR040 anti-Read-Modify-Write rules, CR041 clean change file formatting, CR044 missing configuration diagnostics, CR047 strict patch validation, and [CR049](doc/CR/CR049_AIA_and_jsonpit_to_RAIkeep_Live-ID-Validation_and_Zip-Image-Import.md) live ID validation with 4-character change checksums—are certified across both languages under the exact same version milestone before release.
|
|
42
42
|
|
|
43
43
|
---
|
|
44
44
|
|
|
@@ -54,6 +54,23 @@ The canonical **reference (lead) implementation** is C# `JsonPit` and the `pits`
|
|
|
54
54
|
|
|
55
55
|
---
|
|
56
56
|
|
|
57
|
+
## Object-Oriented Architecture
|
|
58
|
+
|
|
59
|
+
`jsonpit` models distributed state through rich, encapsulated domain classes implementing standard Python protocols (`MutableMapping`, `ContextManager`):
|
|
60
|
+
|
|
61
|
+
- **`Pit`**: Living collection and coordinator of distributed leases, change files, and storage compaction.
|
|
62
|
+
- **`PitItem`**: Polymorphic living entity. Emits pure sparse deltas without read-modify-write cycles (CR040).
|
|
63
|
+
- **`PitItems`**: Immutable fragment stack managing value history and accurate point-in-time state projection.
|
|
64
|
+
- **`JsonPitBase`**: Shared foundation for storage resolution, subscriber roles, and cloud synchronization.
|
|
65
|
+
|
|
66
|
+
<p align="center">
|
|
67
|
+
<img src="doc/uml/jsonpitCD.svg" alt="jsonpit Class Diagram" width="720"/>
|
|
68
|
+
</p>
|
|
69
|
+
|
|
70
|
+
> 📖 **Full API Reference:** For comprehensive method signatures, exception contracts, and protocol details, see **[API.md](API.md)**.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
57
74
|
## Installation & Upgrade
|
|
58
75
|
|
|
59
76
|
### As a Global CLI Tool (`jpit`) via `pipx` (Recommended)
|
|
@@ -65,7 +82,7 @@ pipx install jsonpit
|
|
|
65
82
|
# (or via companion alias)
|
|
66
83
|
pipx install jpit
|
|
67
84
|
|
|
68
|
-
# Upgrade to the latest lockstep release (v4.4.
|
|
85
|
+
# Upgrade to the latest lockstep release (v4.4.6)
|
|
69
86
|
pipx upgrade jsonpit
|
|
70
87
|
# (or)
|
|
71
88
|
pipx upgrade jpit
|
|
@@ -180,6 +197,16 @@ echo '{id: "AlanKay", dynabook: true}' | jpit put Person
|
|
|
180
197
|
# Set individual properties or tombstone an entity:
|
|
181
198
|
jpit set Person AlanKay Status "Visionary"
|
|
182
199
|
jpit del Person ObsoleteEntity
|
|
200
|
+
|
|
201
|
+
# Automated maintenance and dead flag pruning (100% C# pits maintain parity):
|
|
202
|
+
jpit maintain Activity -c OneDrive -r AIA
|
|
203
|
+
jpit maintain Activity -c OneDrive -r AIA --apply --prune-process-flags --older-than 01:00:00 --json
|
|
204
|
+
jpit maintain --wwwa -c OneDrive -r AIA --apply --prune-process-flags --older-than 7.00:00:00
|
|
205
|
+
|
|
206
|
+
# Read-only audit of durable recovery events (100% C# pits audit parity):
|
|
207
|
+
jpit audit Activity -c OneDrive -r AIA
|
|
208
|
+
jpit audit Activity -c OneDrive -r AIA --machine local --level Warning
|
|
209
|
+
jpit audit --wwwa -c OneDrive -r AIA --json
|
|
183
210
|
```
|
|
184
211
|
|
|
185
212
|
---
|
|
@@ -9,14 +9,14 @@ The canonical **reference (lead) implementation** is C# `JsonPit` and the `pits`
|
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
12
|
-
## Cross-Tool & Cross-Language Version Alignment (`v4.4.
|
|
12
|
+
## Cross-Tool & Cross-Language Version Alignment (`v4.4.6`)
|
|
13
13
|
|
|
14
|
-
`jsonpit` and the `jpit` CLI are version-synchronized to **`v4.4.
|
|
14
|
+
`jsonpit` and the `jpit` CLI are version-synchronized to **`v4.4.6`** in direct lockstep with C# `JsonPit` and `pits 4.4.6` in [RAIkeep](https://github.com/Burkhardt/RAIkeep).
|
|
15
15
|
|
|
16
16
|
### Why Harmonized Versioning is Essential:
|
|
17
17
|
- **Instant Compatibility Clarity:** Operators, DevOps pipelines, and autonomous AI agents immediately know whether `jpit` and `pits` share the exact same distributed protocol revision. If both tools report `4.4.x`, they guarantee identical master leasing rules, receipt mechanisms, 7-digit UTC timestamp parsing, and CR041 clean change file conventions.
|
|
18
18
|
- **Elimination of Multi-Language Matrix Confusion:** In heterogeneous environments (e.g. C# backend daemons paired with Python agent orchestrators and data science tooling), unified SemVer tags eliminate complex cross-compatibility lookup tables.
|
|
19
|
-
- **Joint Protocol Governance:** Architectural changes
|
|
19
|
+
- **Joint Protocol Governance:** Architectural changes—such as CR040 anti-Read-Modify-Write rules, CR041 clean change file formatting, CR044 missing configuration diagnostics, CR047 strict patch validation, and [CR049](doc/CR/CR049_AIA_and_jsonpit_to_RAIkeep_Live-ID-Validation_and_Zip-Image-Import.md) live ID validation with 4-character change checksums—are certified across both languages under the exact same version milestone before release.
|
|
20
20
|
|
|
21
21
|
---
|
|
22
22
|
|
|
@@ -32,6 +32,23 @@ The canonical **reference (lead) implementation** is C# `JsonPit` and the `pits`
|
|
|
32
32
|
|
|
33
33
|
---
|
|
34
34
|
|
|
35
|
+
## Object-Oriented Architecture
|
|
36
|
+
|
|
37
|
+
`jsonpit` models distributed state through rich, encapsulated domain classes implementing standard Python protocols (`MutableMapping`, `ContextManager`):
|
|
38
|
+
|
|
39
|
+
- **`Pit`**: Living collection and coordinator of distributed leases, change files, and storage compaction.
|
|
40
|
+
- **`PitItem`**: Polymorphic living entity. Emits pure sparse deltas without read-modify-write cycles (CR040).
|
|
41
|
+
- **`PitItems`**: Immutable fragment stack managing value history and accurate point-in-time state projection.
|
|
42
|
+
- **`JsonPitBase`**: Shared foundation for storage resolution, subscriber roles, and cloud synchronization.
|
|
43
|
+
|
|
44
|
+
<p align="center">
|
|
45
|
+
<img src="doc/uml/jsonpitCD.svg" alt="jsonpit Class Diagram" width="720"/>
|
|
46
|
+
</p>
|
|
47
|
+
|
|
48
|
+
> 📖 **Full API Reference:** For comprehensive method signatures, exception contracts, and protocol details, see **[API.md](API.md)**.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
35
52
|
## Installation & Upgrade
|
|
36
53
|
|
|
37
54
|
### As a Global CLI Tool (`jpit`) via `pipx` (Recommended)
|
|
@@ -43,7 +60,7 @@ pipx install jsonpit
|
|
|
43
60
|
# (or via companion alias)
|
|
44
61
|
pipx install jpit
|
|
45
62
|
|
|
46
|
-
# Upgrade to the latest lockstep release (v4.4.
|
|
63
|
+
# Upgrade to the latest lockstep release (v4.4.6)
|
|
47
64
|
pipx upgrade jsonpit
|
|
48
65
|
# (or)
|
|
49
66
|
pipx upgrade jpit
|
|
@@ -158,6 +175,16 @@ echo '{id: "AlanKay", dynabook: true}' | jpit put Person
|
|
|
158
175
|
# Set individual properties or tombstone an entity:
|
|
159
176
|
jpit set Person AlanKay Status "Visionary"
|
|
160
177
|
jpit del Person ObsoleteEntity
|
|
178
|
+
|
|
179
|
+
# Automated maintenance and dead flag pruning (100% C# pits maintain parity):
|
|
180
|
+
jpit maintain Activity -c OneDrive -r AIA
|
|
181
|
+
jpit maintain Activity -c OneDrive -r AIA --apply --prune-process-flags --older-than 01:00:00 --json
|
|
182
|
+
jpit maintain --wwwa -c OneDrive -r AIA --apply --prune-process-flags --older-than 7.00:00:00
|
|
183
|
+
|
|
184
|
+
# Read-only audit of durable recovery events (100% C# pits audit parity):
|
|
185
|
+
jpit audit Activity -c OneDrive -r AIA
|
|
186
|
+
jpit audit Activity -c OneDrive -r AIA --machine local --level Warning
|
|
187
|
+
jpit audit --wwwa -c OneDrive -r AIA --json
|
|
161
188
|
```
|
|
162
189
|
|
|
163
190
|
---
|
|
@@ -10,26 +10,41 @@ from .exceptions import (
|
|
|
10
10
|
PitInstanceConflictError,
|
|
11
11
|
PitNotFoundError,
|
|
12
12
|
ProtectedAttributeError,
|
|
13
|
+
StrictPatchValidationError,
|
|
13
14
|
TombstoneError,
|
|
14
15
|
)
|
|
16
|
+
from .audit import (
|
|
17
|
+
LogLevel,
|
|
18
|
+
PitAudit,
|
|
19
|
+
PitAuditEvent,
|
|
20
|
+
PitAuditReadResult,
|
|
21
|
+
)
|
|
15
22
|
from .config import DEFAULT_CONFIG_FILE_LOCATION, OsConfig, missing_configuration_diagnostic
|
|
16
23
|
from .history import PitItems
|
|
17
24
|
from .icons import Icons
|
|
18
25
|
from .item import PitItem
|
|
19
|
-
from .store import
|
|
26
|
+
from .store import (
|
|
27
|
+
Pit,
|
|
28
|
+
PitMaintenanceOptions,
|
|
29
|
+
PitMaintenanceResult,
|
|
30
|
+
PitStore,
|
|
31
|
+
parse_and_validate_seed_payload,
|
|
32
|
+
)
|
|
20
33
|
|
|
21
|
-
|
|
22
|
-
import importlib.metadata
|
|
23
|
-
__version__ = importlib.metadata.version("jsonpit")
|
|
24
|
-
except Exception:
|
|
25
|
-
__version__ = "4.4.4"
|
|
34
|
+
__version__ = "4.4.6"
|
|
26
35
|
|
|
27
36
|
__all__ = [
|
|
28
37
|
"Pit",
|
|
29
38
|
"PitStore",
|
|
39
|
+
"PitMaintenanceOptions",
|
|
40
|
+
"PitMaintenanceResult",
|
|
30
41
|
"parse_and_validate_seed_payload",
|
|
31
42
|
"PitItem",
|
|
32
43
|
"PitItems",
|
|
44
|
+
"LogLevel",
|
|
45
|
+
"PitAuditEvent",
|
|
46
|
+
"PitAuditReadResult",
|
|
47
|
+
"PitAudit",
|
|
33
48
|
"OsConfig",
|
|
34
49
|
"DEFAULT_CONFIG_FILE_LOCATION",
|
|
35
50
|
"missing_configuration_diagnostic",
|
|
@@ -40,5 +55,6 @@ __all__ = [
|
|
|
40
55
|
"PitConcurrencyError",
|
|
41
56
|
"PitInstanceConflictError",
|
|
42
57
|
"ProtectedAttributeError",
|
|
58
|
+
"StrictPatchValidationError",
|
|
43
59
|
"TombstoneError",
|
|
44
60
|
]
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Read-only audit access to durable JsonPit recovery events (CR003, coordinated v3.13.2).
|
|
3
|
+
Inspects immutable event logs and compact archives without opening a Pit,
|
|
4
|
+
creating process activity windows, or mutating storage.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
import datetime
|
|
11
|
+
from enum import IntEnum
|
|
12
|
+
import json
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
import zipfile
|
|
16
|
+
|
|
17
|
+
from .canonical import format_iso_timestamp, parse_iso_timestamp, utc_ticks_to_datetime
|
|
18
|
+
from .flags import get_machine_name
|
|
19
|
+
from .fs import safe_read_text
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LogLevel(IntEnum):
|
|
23
|
+
"""Standard severity levels for Pit audit events matching .NET LogLevel."""
|
|
24
|
+
TRACE = 0
|
|
25
|
+
DEBUG = 1
|
|
26
|
+
INFORMATION = 2
|
|
27
|
+
WARNING = 3
|
|
28
|
+
ERROR = 4
|
|
29
|
+
CRITICAL = 5
|
|
30
|
+
|
|
31
|
+
@classmethod
|
|
32
|
+
def from_string(cls, name: str) -> LogLevel:
|
|
33
|
+
norm = name.strip().lower()
|
|
34
|
+
if norm in ("trace", "0"):
|
|
35
|
+
return cls.TRACE
|
|
36
|
+
if norm in ("debug", "1"):
|
|
37
|
+
return cls.DEBUG
|
|
38
|
+
if norm in ("information", "info", "2"):
|
|
39
|
+
return cls.INFORMATION
|
|
40
|
+
if norm in ("warning", "warn", "3"):
|
|
41
|
+
return cls.WARNING
|
|
42
|
+
if norm in ("error", "err", "4"):
|
|
43
|
+
return cls.ERROR
|
|
44
|
+
if norm in ("critical", "crit", "fatal", "5"):
|
|
45
|
+
return cls.CRITICAL
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f"Unknown log level '{name}'. Valid levels: Trace, Debug, Information, Warning, Error, Critical."
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def display_name(self) -> str:
|
|
52
|
+
if self == LogLevel.INFORMATION:
|
|
53
|
+
return "Information"
|
|
54
|
+
return self.name.capitalize()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass
|
|
58
|
+
class PitAuditEvent:
|
|
59
|
+
"""One durable JsonPit audit event as read back from a pit's Events directory."""
|
|
60
|
+
file_name: str
|
|
61
|
+
content: dict[str, Any]
|
|
62
|
+
machine: str
|
|
63
|
+
utc_time: datetime.datetime
|
|
64
|
+
level: LogLevel
|
|
65
|
+
stage: str
|
|
66
|
+
event_id: str
|
|
67
|
+
message: str
|
|
68
|
+
|
|
69
|
+
@classmethod
|
|
70
|
+
def from_dict(cls, file_name: str, content: dict[str, Any]) -> PitAuditEvent:
|
|
71
|
+
machine = str(content.get("Machine") or "")
|
|
72
|
+
|
|
73
|
+
# Parse timestamp from UtcTicks or UtcTime
|
|
74
|
+
ticks = content.get("UtcTicks")
|
|
75
|
+
if ticks is not None and isinstance(ticks, (int, float)):
|
|
76
|
+
utc_time = utc_ticks_to_datetime(int(ticks))
|
|
77
|
+
elif content.get("UtcTime"):
|
|
78
|
+
try:
|
|
79
|
+
utc_time = parse_iso_timestamp(str(content["UtcTime"]))
|
|
80
|
+
except Exception:
|
|
81
|
+
utc_time = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
82
|
+
else:
|
|
83
|
+
utc_time = datetime.datetime.min.replace(tzinfo=datetime.timezone.utc)
|
|
84
|
+
|
|
85
|
+
lvl_raw = str(content.get("Level") or "Trace")
|
|
86
|
+
try:
|
|
87
|
+
level = LogLevel.from_string(lvl_raw)
|
|
88
|
+
except ValueError:
|
|
89
|
+
level = LogLevel.TRACE
|
|
90
|
+
|
|
91
|
+
stage = str(content.get("Stage") or "")
|
|
92
|
+
event_id = str(content.get("EventId") or "")
|
|
93
|
+
message = str(content.get("Message") or "")
|
|
94
|
+
|
|
95
|
+
return cls(
|
|
96
|
+
file_name=file_name,
|
|
97
|
+
content=content,
|
|
98
|
+
machine=machine,
|
|
99
|
+
utc_time=utc_time,
|
|
100
|
+
level=level,
|
|
101
|
+
stage=stage,
|
|
102
|
+
event_id=event_id,
|
|
103
|
+
message=message,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class PitAuditReadResult:
|
|
109
|
+
"""Read result containing filtered events and physical diagnostic issues."""
|
|
110
|
+
events: list[PitAuditEvent] = field(default_factory=list)
|
|
111
|
+
issues: list[str] = field(default_factory=list)
|
|
112
|
+
|
|
113
|
+
@property
|
|
114
|
+
def succeeded(self) -> bool:
|
|
115
|
+
return len(self.issues) == 0
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class PitAudit:
|
|
119
|
+
"""Read-only audit inspector for durable JsonPit event logs (Events/)."""
|
|
120
|
+
|
|
121
|
+
@staticmethod
|
|
122
|
+
def inspect(
|
|
123
|
+
pit_directory: Path | str,
|
|
124
|
+
machine_filter: str = "all",
|
|
125
|
+
min_level: LogLevel = LogLevel.TRACE,
|
|
126
|
+
) -> PitAuditReadResult:
|
|
127
|
+
p_dir = Path(pit_directory)
|
|
128
|
+
events_dir = p_dir / "Events"
|
|
129
|
+
issues: list[str] = []
|
|
130
|
+
|
|
131
|
+
if not events_dir.is_dir():
|
|
132
|
+
return PitAuditReadResult(events=[], issues=[])
|
|
133
|
+
|
|
134
|
+
raw_events: dict[str, dict[str, Any]] = {}
|
|
135
|
+
|
|
136
|
+
# 1. Inspect loose event files (*.event)
|
|
137
|
+
for ef in sorted(events_dir.glob("*.event"), key=lambda p: p.name):
|
|
138
|
+
try:
|
|
139
|
+
text = safe_read_text(ef)
|
|
140
|
+
if not text or not text.strip():
|
|
141
|
+
issues.append(f"Empty loose event '{ef.name}'")
|
|
142
|
+
continue
|
|
143
|
+
parsed = json.loads(text)
|
|
144
|
+
if not isinstance(parsed, dict):
|
|
145
|
+
issues.append(f"Invalid loose event '{ef.name}': content is not a JSON object")
|
|
146
|
+
continue
|
|
147
|
+
raw_events[ef.name] = parsed
|
|
148
|
+
except Exception as ex:
|
|
149
|
+
issues.append(f"Unreadable loose event '{ef.name}': {ex}")
|
|
150
|
+
|
|
151
|
+
# 2. Inspect immutable zip archives (Events_*.zip)
|
|
152
|
+
for zf_path in sorted(events_dir.glob("Events_*.zip"), key=lambda p: p.name):
|
|
153
|
+
try:
|
|
154
|
+
with zipfile.ZipFile(zf_path, "r") as zf:
|
|
155
|
+
for info in zf.infolist():
|
|
156
|
+
if info.filename.endswith(".event"):
|
|
157
|
+
try:
|
|
158
|
+
with zf.open(info) as z_entry:
|
|
159
|
+
text = z_entry.read().decode("utf-8")
|
|
160
|
+
parsed = json.loads(text)
|
|
161
|
+
if isinstance(parsed, dict):
|
|
162
|
+
if info.filename in raw_events:
|
|
163
|
+
if raw_events[info.filename] != parsed:
|
|
164
|
+
issues.append(
|
|
165
|
+
f"Conflicting event content for '{info.filename}' in '{zf_path.name}'."
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
raw_events[info.filename] = parsed
|
|
169
|
+
except Exception as ex:
|
|
170
|
+
issues.append(f"Invalid event '{info.filename}' in '{zf_path.name}': {ex}")
|
|
171
|
+
except Exception as ex:
|
|
172
|
+
issues.append(f"Invalid event archive '{zf_path.name}': {ex}")
|
|
173
|
+
|
|
174
|
+
# 3. Deduplicate events by EventId
|
|
175
|
+
identities: dict[str, PitAuditEvent] = {}
|
|
176
|
+
distinct: list[PitAuditEvent] = []
|
|
177
|
+
for file_name in sorted(raw_events.keys()):
|
|
178
|
+
content = raw_events[file_name]
|
|
179
|
+
ev = PitAuditEvent.from_dict(file_name, content)
|
|
180
|
+
if not ev.event_id:
|
|
181
|
+
distinct.append(ev)
|
|
182
|
+
continue
|
|
183
|
+
|
|
184
|
+
if ev.event_id not in identities:
|
|
185
|
+
identities[ev.event_id] = ev
|
|
186
|
+
distinct.append(ev)
|
|
187
|
+
else:
|
|
188
|
+
existing = identities[ev.event_id]
|
|
189
|
+
if existing.content != ev.content:
|
|
190
|
+
issues.append(
|
|
191
|
+
f"Conflicting event content for EventId '{ev.event_id}' in "
|
|
192
|
+
f"'{existing.file_name}' and '{ev.file_name}'."
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
# 4. Filter by machine
|
|
196
|
+
m_filt = (machine_filter or "all").strip().lower()
|
|
197
|
+
events = distinct
|
|
198
|
+
if m_filt != "all":
|
|
199
|
+
target_machine = get_machine_name().lower() if m_filt == "local" else m_filt
|
|
200
|
+
events = [e for e in events if e.machine.lower() == target_machine]
|
|
201
|
+
|
|
202
|
+
# 5. Filter by severity level
|
|
203
|
+
if min_level > LogLevel.TRACE:
|
|
204
|
+
events = [e for e in events if e.level >= min_level]
|
|
205
|
+
|
|
206
|
+
# 6. Order deterministically: Machine, UtcTime, EventId
|
|
207
|
+
events.sort(key=lambda e: (e.machine, e.utc_time, e.event_id))
|
|
208
|
+
|
|
209
|
+
return PitAuditReadResult(events=events, issues=issues)
|