context-compiler 0.8.3__tar.gz → 0.9.0.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/PKG-INFO +96 -52
  2. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/README.md +95 -51
  3. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/05_llm_prompt_drift_vs_state.py +2 -2
  4. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/06_llm_context_compaction.py +3 -3
  5. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/08_llm_replacement_precondition.py +24 -25
  6. context_compiler-0.8.3/demos/09_llm_pending_clarification.py → context_compiler-0.9.0.dev0/demos/09_llm_confirmation_passthrough.py +39 -44
  7. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/README.md +10 -4
  8. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/common.py +11 -7
  9. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/run_demo.py +1 -1
  10. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/01_persistent_guardrails.py +4 -4
  11. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/03_ambiguity_with_clarification.py +1 -1
  12. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/04_tool_governance_denylist.py +2 -2
  13. context_compiler-0.8.3/examples/06_step_sequence_and_checkpoint.py → context_compiler-0.9.0.dev0/examples/06_step_sequence_and_state_restore.py +4 -4
  14. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/README.md +4 -3
  15. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/_util.py +4 -4
  16. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/pyproject.toml +1 -1
  17. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/__init__.py +5 -6
  18. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/controller.py +2 -2
  19. context_compiler-0.9.0.dev0/src/context_compiler/engine.py +373 -0
  20. context_compiler-0.9.0.dev0/src/context_compiler/grammar.py +434 -0
  21. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/repl.py +6 -86
  22. context_compiler-0.8.3/src/context_compiler/engine.py +0 -713
  23. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/.gitignore +0 -0
  24. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/LICENSE +0 -0
  25. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/01_llm_contradiction_clarify.py +0 -0
  26. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/02_llm_constraint_guardrail.py +0 -0
  27. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/03_llm_premise_guardrail.py +0 -0
  28. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/04_llm_tool_denylist_guardrail.py +0 -0
  29. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/07_llm_prompt_vs_state.py +0 -0
  30. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/__init__.py +0 -0
  31. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/llm_client.py +0 -0
  32. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/02_configuration_and_correction.py +0 -0
  33. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/05_llm_integration_pattern.py +0 -0
  34. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/07_single_policy_correction.py +0 -0
  35. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/08_controller_preview_diff.py +0 -0
  36. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/host_support/__init__.py +0 -0
  37. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/host_support/provider_mode.py +0 -0
  38. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/const.py +0 -0
  39. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/decision_helpers.py +0 -0
  40. {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: context-compiler
3
- Version: 0.8.3
3
+ Version: 0.9.0.dev0
4
4
  Summary: Deterministic conversational state engine for LLM applications.
5
5
  Project-URL: Homepage, https://github.com/rlippmann/context-compiler
6
6
  Project-URL: Repository, https://github.com/rlippmann/context-compiler
@@ -35,27 +35,30 @@ Description-Content-Type: text/markdown
35
35
  [![License](https://img.shields.io/pypi/l/context-compiler)](https://pypi.org/project/context-compiler/)
36
36
 
37
37
  Context Compiler is a deterministic conversational state authority for LLM applications.
38
- It handles explicit state changes, clarification and confirmation flows,
39
- checkpoint restore, and structured authoritative state for the host.
38
+ It handles canonical directive execution, semantic validation, deterministic
39
+ clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
40
+ the host.
40
41
 
41
42
  ## What Context Compiler provides
42
43
 
43
44
  Context Compiler gives hosts fixed state rules:
44
45
 
45
- - handle explicit user state changes with deterministic rules
46
+ - handle canonical explicit state changes with deterministic rules
46
47
  - clarification instead of silent overwrite for blocked/ambiguous changes
47
- - pending confirmation flows that must resolve before anything else changes
48
- - export and import checkpoints to restore saved state and pending confirmation flow
48
+ - preserve supported pending continuation when explicit confirmation is required
49
+ - export and import authoritative state for host-managed persistence
49
50
  - produce structured authoritative state for downstream host decisions
50
51
 
51
52
  The model generates responses. The compiler owns state.
53
+ Human-facing normalization, malformed-input recovery, and intent drafting belong
54
+ outside core.
52
55
 
53
56
  ## How the compiler metaphor works
54
57
 
55
- Like a compiler, it parses input, validates it, applies fixed rules, and
58
+ Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
56
59
  produces a stable result the host can use. It treats important instructions as
57
60
  structured state instead of temporary prompt text. It is not source-code
58
- compilation and not a reasoning model.
61
+ compilation, not a reasoning model, and not a natural-language repair layer.
59
62
 
60
63
  ## 10-Second Example
61
64
 
@@ -83,33 +86,52 @@ instead of relying on memory of earlier conversation text.
83
86
 
84
87
  Context Compiler makes state-change rules explicit so behavior stays repeatable.
85
88
 
86
- **Explicit directive**
89
+ The architecture has three layers:
90
+
91
+ - syntax classification decides whether input is a canonical directive, invalid
92
+ directive-shaped syntax, or ordinary passthrough
93
+ - semantic evaluation decides whether a canonical directive updates state,
94
+ clarifies, or no-ops
95
+ - semantic continuation optionally preserves a deterministic blocked transition
96
+ that needs explicit `yes` / `no`
97
+
98
+ ### Explicit directive
99
+
87
100
  ```text
88
101
  set premise concise replies
89
102
  ```
103
+
90
104
  - Base model: silently accepts / rewrites
91
105
  - Context Compiler: applies a repeatable state update
92
106
 
93
- **Single-directive grammar**
107
+ ### Single-directive grammar
108
+
94
109
  ```text
95
110
  use docker and prohibit peanuts
96
111
  ```
112
+
97
113
  - Without an authority layer: host/model behavior varies
98
- - Context Compiler: returns `clarify`, keeps authoritative state unchanged, and asks for separate directives
114
+ - Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
115
+
116
+ ### State-dependent operation
99
117
 
100
- **State-dependent operation**
101
118
  ```text
102
119
  clear state
103
120
  use podman instead of docker
104
121
  ```
122
+
105
123
  - Without explicit state transition rules: behavior depends on host/model handling
106
- - Context Compiler: returns `clarify` before changing state
124
+ - Context Compiler: applies the deterministic resulting transition when
125
+ `docker` is absent and `use podman` is otherwise valid; other semantic
126
+ conflicts may still clarify
127
+
128
+ ### Lifecycle enforcement
107
129
 
108
- **Lifecycle enforcement**
109
130
  ```text
110
131
  clear state
111
132
  change premise to formal tone
112
133
  ```
134
+
113
135
  - Without explicit transition checks: behavior depends on host/model handling
114
136
  - Context Compiler: asks for clarification and keeps saved state unchanged
115
137
 
@@ -171,7 +193,7 @@ For runnable application-layer examples, see
171
193
  [`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
172
194
  That companion repository shows enforcement points built on compiler state,
173
195
  including retrieval filtering, schema selection, tool gating, execution
174
- authorization, gateway middleware, checkpoint continuation, and prompt
196
+ authorization, gateway middleware, runtime continuation handling, and prompt
175
197
  construction.
176
198
 
177
199
  ## Does it Work?
@@ -197,13 +219,13 @@ pip install context-compiler
197
219
  context-compiler
198
220
  ```
199
221
 
200
- Preload options keep saved rules separate from confirmation state in progress:
222
+ Preload options load authoritative state:
223
+
201
224
  - `--initial-state-json` / `--initial-state-file` load saved state
202
225
  (via exported state JSON).
203
- - `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
204
- continuation checkpoint (saved state + pending confirmation state).
205
226
 
206
227
  REPL commands (controller layer, not engine directives):
228
+
207
229
  - `state` shows current saved state.
208
230
  - `preview <input>` runs deterministic dry-run without mutating live state.
209
231
  - `step <input>` is an explicit alias of normal bare-input step behavior.
@@ -219,23 +241,25 @@ for non-interactive usage.
219
241
  context-compiler --json < input.txt
220
242
  ```
221
243
 
222
- Preload options keep saved rules separate from confirmation state in progress:
244
+ Preload options load authoritative state:
245
+
223
246
  - `--initial-state-json` / `--initial-state-file` load saved state
224
247
  (via exported state JSON).
225
- - `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
226
- continuation checkpoint (saved state + pending confirmation state).
227
248
 
228
249
  ## Installation
229
250
 
230
251
  Requirements:
252
+
231
253
  - Python 3.11+
232
254
 
233
255
  Install:
256
+
234
257
  ```bash
235
258
  pip install context-compiler
236
259
  ```
237
260
 
238
261
  Packaging notes:
262
+
239
263
  - Base install includes the core authority-layer engine and CLI.
240
264
  - Example and demo source files are available in the repository and source distribution.
241
265
  - To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
@@ -261,11 +285,11 @@ class Decision(TypedDict):
261
285
 
262
286
  Meaning:
263
287
 
264
- | kind | host behavior |
265
- |:-----------:|-----------------------------------------------|
266
- | passthrough | forward user input to LLM |
267
- | update | authoritative state mutated; host may call LLM with updated state |
268
- | clarify | show `prompt_to_user` and do not call the LLM |
288
+ | kind | host behavior |
289
+ | --- | --- |
290
+ | passthrough | forward user input to LLM |
291
+ | update | authoritative state mutated; host may call LLM with updated state |
292
+ | clarify | show `prompt_to_user` and do not call the LLM |
269
293
 
270
294
  For normal app code, prefer the exported decision helpers (`is_clarify`,
271
295
  `is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
@@ -277,12 +301,11 @@ reference.
277
301
  Common API entry points:
278
302
 
279
303
  - engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
280
- `engine.has_pending_clarification()`
304
+ `engine.premise`, `engine.policies`, `engine.export_json(...)`,
305
+ `engine.import_json(...)`
281
306
  - decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
282
307
  `get_clarify_prompt(...)`, `get_decision_state(...)`
283
- - state helpers: `get_premise_value(...)`, `get_policy_items(...)`
284
- - state and checkpoint transport: `export_json(...)`, `import_json(...)`,
285
- `export_checkpoint(...)`, `import_checkpoint(...)`
308
+ - state transport: `engine.export_json(...)`, `engine.import_json(...)`
286
309
  - controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
287
310
 
288
311
  ### Controller API (Reusable Outside REPL)
@@ -312,12 +335,15 @@ authoritative in future turns.
312
335
  - Policies are per-item (`use` or `prohibit`)
313
336
  - State changes only through explicit directives
314
337
  - No inference or semantic reasoning
338
+ - Non-canonical input normalization is outside the core state contract
315
339
 
316
340
  Identical input sequences always produce identical state.
317
341
 
318
- The internal structure of the state is intentionally opaque to host applications.
319
- For normal reads, prefer `get_premise_value(state)` and
320
- `get_policy_items(state, ...)` over direct key traversal.
342
+ For live engine-owned reads, use `engine.premise` and `engine.policies`.
343
+ `engine.policies` returns a caller-owned copy.
344
+
345
+ `engine.state` remains the public snapshot/serialization boundary when host code
346
+ needs the full authoritative state object.
321
347
 
322
348
  ---
323
349
 
@@ -359,23 +385,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
359
385
  Context Compiler enforces explicit directive and state rules. Domain reasoning
360
386
  still belongs to the host and model workflow.
361
387
 
362
- ---
363
-
364
- ## Checkpoint Contract
388
+ If a user says something non-canonical such as a near miss, alternate phrasing,
389
+ or a failed replacement request that would need reinterpretation, that
390
+ normalization is outside core and must happen before canonical directives reach
391
+ the compiler.
365
392
 
366
- `export_json()` / `import_json()` and the checkpoint APIs serve different boundaries:
393
+ ---
367
394
 
368
- - `export_json()` / `import_json()` transport **authoritative state only**
369
- - checkpoint APIs transport **serialized continuation**:
370
- - authoritative state
371
- - pending confirmation flow state
395
+ ## Persistence Contract
372
396
 
373
- Use state JSON when you only need authoritative state. Use checkpoint APIs when
374
- you also need resumable continuation state across process or request
375
- boundaries.
397
+ `export_json()` / `import_json()` are the current persistence boundary.
376
398
 
377
- For the checkpoint object shape, API-level usage notes, and serialization
378
- details, see [docs/api-reference.md](docs/api-reference.md#checkpoint-apis).
399
+ - They transport **authoritative state only**
400
+ - Hosts own any broader interaction or session workflow around that state
401
+ - Pending continuation, if supported by the active engine contract, remains a
402
+ runtime semantic concept rather than a documented persisted artifact
379
403
 
380
404
  ---
381
405
 
@@ -401,6 +425,11 @@ Replacement:
401
425
  User: use podman instead of docker
402
426
  ```
403
427
 
428
+ If `docker` is absent from saved state, that does not make the directive
429
+ pending. The user's intended resulting state is still unambiguous, so the
430
+ replacement follows the deterministic `use podman` transition when otherwise
431
+ semantically valid.
432
+
404
433
  Removal and reset:
405
434
 
406
435
  ```text
@@ -410,9 +439,15 @@ User: clear state
410
439
  ```
411
440
 
412
441
  Grammar invariant: one input may contain at most one canonical directive.
413
- If another canonical directive start appears later in the same input, the
414
- input is invalid and Context Compiler returns `clarify` without mutating
415
- authoritative state or creating pending confirmation state.
442
+ Directive-shaped invalid input is outside the canonical language, and
443
+ `clarify` is reserved for canonical directives that later fail semantic
444
+ evaluation against authoritative state.
445
+
446
+ Pending continuation is a separate runtime layer. It may exist only after a
447
+ canonical directive reaches a supported semantic `clarify` case. It never
448
+ repairs malformed syntax or reinterprets non-canonical input as a directive.
449
+ An absent source item in a canonical replacement directive is not, by itself,
450
+ such a `clarify` case.
416
451
 
417
452
  Examples:
418
453
 
@@ -442,9 +477,8 @@ set premise "use docker and prohibit peanuts"
442
477
  Quotes do not create protected literal regions inside a recognized directive
443
478
  payload.
444
479
 
445
- Conflicting directives also trigger clarification instead of changing state.
446
-
447
- For full directive grammar and edge-case behavior, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
480
+ For the normative grammar, classification rules, and syntax-versus-semantics
481
+ boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
448
482
 
449
483
  ---
450
484
 
@@ -465,6 +499,11 @@ allowed, when clarification is required, and how continuation state is
465
499
  restored. For runnable application-layer examples, see
466
500
  [`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
467
501
 
502
+ Human-facing interpretation is a separate concern. If you want to recognize
503
+ non-canonical phrasing, recover from malformed input, narrow user intent, or
504
+ turn a failed replacement request into a different canonical directive, do that
505
+ before calling core.
506
+
468
507
  **Why not just use a plain dict?**
469
508
  A plain dict can hold state for prompt construction, schema selection, tool
470
509
  gating, and other host behavior.
@@ -492,6 +531,7 @@ overwriting state.
492
531
  - Identical input sequences produce identical compiler state.
493
532
  - Model responses never modify compiler state.
494
533
  - Ambiguous directives trigger clarification instead of changing state.
534
+ - Syntax errors never create pending continuation.
495
535
 
496
536
  Behavioral tests and Hypothesis-based property tests verify these invariants.
497
537
 
@@ -519,6 +559,10 @@ These docs cover the design and milestone details:
519
559
  [`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
520
560
  These fixtures serve as the behavioral contract for compiler semantics across implementations.
521
561
 
562
+ ## Development Process
563
+
564
+ Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
565
+
522
566
  ---
523
567
 
524
568
  ## License
@@ -5,27 +5,30 @@
5
5
  [![License](https://img.shields.io/pypi/l/context-compiler)](https://pypi.org/project/context-compiler/)
6
6
 
7
7
  Context Compiler is a deterministic conversational state authority for LLM applications.
8
- It handles explicit state changes, clarification and confirmation flows,
9
- checkpoint restore, and structured authoritative state for the host.
8
+ It handles canonical directive execution, semantic validation, deterministic
9
+ clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
10
+ the host.
10
11
 
11
12
  ## What Context Compiler provides
12
13
 
13
14
  Context Compiler gives hosts fixed state rules:
14
15
 
15
- - handle explicit user state changes with deterministic rules
16
+ - handle canonical explicit state changes with deterministic rules
16
17
  - clarification instead of silent overwrite for blocked/ambiguous changes
17
- - pending confirmation flows that must resolve before anything else changes
18
- - export and import checkpoints to restore saved state and pending confirmation flow
18
+ - preserve supported pending continuation when explicit confirmation is required
19
+ - export and import authoritative state for host-managed persistence
19
20
  - produce structured authoritative state for downstream host decisions
20
21
 
21
22
  The model generates responses. The compiler owns state.
23
+ Human-facing normalization, malformed-input recovery, and intent drafting belong
24
+ outside core.
22
25
 
23
26
  ## How the compiler metaphor works
24
27
 
25
- Like a compiler, it parses input, validates it, applies fixed rules, and
28
+ Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
26
29
  produces a stable result the host can use. It treats important instructions as
27
30
  structured state instead of temporary prompt text. It is not source-code
28
- compilation and not a reasoning model.
31
+ compilation, not a reasoning model, and not a natural-language repair layer.
29
32
 
30
33
  ## 10-Second Example
31
34
 
@@ -53,33 +56,52 @@ instead of relying on memory of earlier conversation text.
53
56
 
54
57
  Context Compiler makes state-change rules explicit so behavior stays repeatable.
55
58
 
56
- **Explicit directive**
59
+ The architecture has three layers:
60
+
61
+ - syntax classification decides whether input is a canonical directive, invalid
62
+ directive-shaped syntax, or ordinary passthrough
63
+ - semantic evaluation decides whether a canonical directive updates state,
64
+ clarifies, or no-ops
65
+ - semantic continuation optionally preserves a deterministic blocked transition
66
+ that needs explicit `yes` / `no`
67
+
68
+ ### Explicit directive
69
+
57
70
  ```text
58
71
  set premise concise replies
59
72
  ```
73
+
60
74
  - Base model: silently accepts / rewrites
61
75
  - Context Compiler: applies a repeatable state update
62
76
 
63
- **Single-directive grammar**
77
+ ### Single-directive grammar
78
+
64
79
  ```text
65
80
  use docker and prohibit peanuts
66
81
  ```
82
+
67
83
  - Without an authority layer: host/model behavior varies
68
- - Context Compiler: returns `clarify`, keeps authoritative state unchanged, and asks for separate directives
84
+ - Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
85
+
86
+ ### State-dependent operation
69
87
 
70
- **State-dependent operation**
71
88
  ```text
72
89
  clear state
73
90
  use podman instead of docker
74
91
  ```
92
+
75
93
  - Without explicit state transition rules: behavior depends on host/model handling
76
- - Context Compiler: returns `clarify` before changing state
94
+ - Context Compiler: applies the deterministic resulting transition when
95
+ `docker` is absent and `use podman` is otherwise valid; other semantic
96
+ conflicts may still clarify
97
+
98
+ ### Lifecycle enforcement
77
99
 
78
- **Lifecycle enforcement**
79
100
  ```text
80
101
  clear state
81
102
  change premise to formal tone
82
103
  ```
104
+
83
105
  - Without explicit transition checks: behavior depends on host/model handling
84
106
  - Context Compiler: asks for clarification and keeps saved state unchanged
85
107
 
@@ -141,7 +163,7 @@ For runnable application-layer examples, see
141
163
  [`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
142
164
  That companion repository shows enforcement points built on compiler state,
143
165
  including retrieval filtering, schema selection, tool gating, execution
144
- authorization, gateway middleware, checkpoint continuation, and prompt
166
+ authorization, gateway middleware, runtime continuation handling, and prompt
145
167
  construction.
146
168
 
147
169
  ## Does it Work?
@@ -167,13 +189,13 @@ pip install context-compiler
167
189
  context-compiler
168
190
  ```
169
191
 
170
- Preload options keep saved rules separate from confirmation state in progress:
192
+ Preload options load authoritative state:
193
+
171
194
  - `--initial-state-json` / `--initial-state-file` load saved state
172
195
  (via exported state JSON).
173
- - `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
174
- continuation checkpoint (saved state + pending confirmation state).
175
196
 
176
197
  REPL commands (controller layer, not engine directives):
198
+
177
199
  - `state` shows current saved state.
178
200
  - `preview <input>` runs deterministic dry-run without mutating live state.
179
201
  - `step <input>` is an explicit alias of normal bare-input step behavior.
@@ -189,23 +211,25 @@ for non-interactive usage.
189
211
  context-compiler --json < input.txt
190
212
  ```
191
213
 
192
- Preload options keep saved rules separate from confirmation state in progress:
214
+ Preload options load authoritative state:
215
+
193
216
  - `--initial-state-json` / `--initial-state-file` load saved state
194
217
  (via exported state JSON).
195
- - `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
196
- continuation checkpoint (saved state + pending confirmation state).
197
218
 
198
219
  ## Installation
199
220
 
200
221
  Requirements:
222
+
201
223
  - Python 3.11+
202
224
 
203
225
  Install:
226
+
204
227
  ```bash
205
228
  pip install context-compiler
206
229
  ```
207
230
 
208
231
  Packaging notes:
232
+
209
233
  - Base install includes the core authority-layer engine and CLI.
210
234
  - Example and demo source files are available in the repository and source distribution.
211
235
  - To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
@@ -231,11 +255,11 @@ class Decision(TypedDict):
231
255
 
232
256
  Meaning:
233
257
 
234
- | kind | host behavior |
235
- |:-----------:|-----------------------------------------------|
236
- | passthrough | forward user input to LLM |
237
- | update | authoritative state mutated; host may call LLM with updated state |
238
- | clarify | show `prompt_to_user` and do not call the LLM |
258
+ | kind | host behavior |
259
+ | --- | --- |
260
+ | passthrough | forward user input to LLM |
261
+ | update | authoritative state mutated; host may call LLM with updated state |
262
+ | clarify | show `prompt_to_user` and do not call the LLM |
239
263
 
240
264
  For normal app code, prefer the exported decision helpers (`is_clarify`,
241
265
  `is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
@@ -247,12 +271,11 @@ reference.
247
271
  Common API entry points:
248
272
 
249
273
  - engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
250
- `engine.has_pending_clarification()`
274
+ `engine.premise`, `engine.policies`, `engine.export_json(...)`,
275
+ `engine.import_json(...)`
251
276
  - decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
252
277
  `get_clarify_prompt(...)`, `get_decision_state(...)`
253
- - state helpers: `get_premise_value(...)`, `get_policy_items(...)`
254
- - state and checkpoint transport: `export_json(...)`, `import_json(...)`,
255
- `export_checkpoint(...)`, `import_checkpoint(...)`
278
+ - state transport: `engine.export_json(...)`, `engine.import_json(...)`
256
279
  - controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
257
280
 
258
281
  ### Controller API (Reusable Outside REPL)
@@ -282,12 +305,15 @@ authoritative in future turns.
282
305
  - Policies are per-item (`use` or `prohibit`)
283
306
  - State changes only through explicit directives
284
307
  - No inference or semantic reasoning
308
+ - Non-canonical input normalization is outside the core state contract
285
309
 
286
310
  Identical input sequences always produce identical state.
287
311
 
288
- The internal structure of the state is intentionally opaque to host applications.
289
- For normal reads, prefer `get_premise_value(state)` and
290
- `get_policy_items(state, ...)` over direct key traversal.
312
+ For live engine-owned reads, use `engine.premise` and `engine.policies`.
313
+ `engine.policies` returns a caller-owned copy.
314
+
315
+ `engine.state` remains the public snapshot/serialization boundary when host code
316
+ needs the full authoritative state object.
291
317
 
292
318
  ---
293
319
 
@@ -329,23 +355,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
329
355
  Context Compiler enforces explicit directive and state rules. Domain reasoning
330
356
  still belongs to the host and model workflow.
331
357
 
332
- ---
333
-
334
- ## Checkpoint Contract
358
+ If a user says something non-canonical such as a near miss, alternate phrasing,
359
+ or a failed replacement request that would need reinterpretation, that
360
+ normalization is outside core and must happen before canonical directives reach
361
+ the compiler.
335
362
 
336
- `export_json()` / `import_json()` and the checkpoint APIs serve different boundaries:
363
+ ---
337
364
 
338
- - `export_json()` / `import_json()` transport **authoritative state only**
339
- - checkpoint APIs transport **serialized continuation**:
340
- - authoritative state
341
- - pending confirmation flow state
365
+ ## Persistence Contract
342
366
 
343
- Use state JSON when you only need authoritative state. Use checkpoint APIs when
344
- you also need resumable continuation state across process or request
345
- boundaries.
367
+ `export_json()` / `import_json()` are the current persistence boundary.
346
368
 
347
- For the checkpoint object shape, API-level usage notes, and serialization
348
- details, see [docs/api-reference.md](docs/api-reference.md#checkpoint-apis).
369
+ - They transport **authoritative state only**
370
+ - Hosts own any broader interaction or session workflow around that state
371
+ - Pending continuation, if supported by the active engine contract, remains a
372
+ runtime semantic concept rather than a documented persisted artifact
349
373
 
350
374
  ---
351
375
 
@@ -371,6 +395,11 @@ Replacement:
371
395
  User: use podman instead of docker
372
396
  ```
373
397
 
398
+ If `docker` is absent from saved state, that does not make the directive
399
+ pending. The user's intended resulting state is still unambiguous, so the
400
+ replacement follows the deterministic `use podman` transition when otherwise
401
+ semantically valid.
402
+
374
403
  Removal and reset:
375
404
 
376
405
  ```text
@@ -380,9 +409,15 @@ User: clear state
380
409
  ```
381
410
 
382
411
  Grammar invariant: one input may contain at most one canonical directive.
383
- If another canonical directive start appears later in the same input, the
384
- input is invalid and Context Compiler returns `clarify` without mutating
385
- authoritative state or creating pending confirmation state.
412
+ Directive-shaped invalid input is outside the canonical language, and
413
+ `clarify` is reserved for canonical directives that later fail semantic
414
+ evaluation against authoritative state.
415
+
416
+ Pending continuation is a separate runtime layer. It may exist only after a
417
+ canonical directive reaches a supported semantic `clarify` case. It never
418
+ repairs malformed syntax or reinterprets non-canonical input as a directive.
419
+ An absent source item in a canonical replacement directive is not, by itself,
420
+ such a `clarify` case.
386
421
 
387
422
  Examples:
388
423
 
@@ -412,9 +447,8 @@ set premise "use docker and prohibit peanuts"
412
447
  Quotes do not create protected literal regions inside a recognized directive
413
448
  payload.
414
449
 
415
- Conflicting directives also trigger clarification instead of changing state.
416
-
417
- For full directive grammar and edge-case behavior, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
450
+ For the normative grammar, classification rules, and syntax-versus-semantics
451
+ boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
418
452
 
419
453
  ---
420
454
 
@@ -435,6 +469,11 @@ allowed, when clarification is required, and how continuation state is
435
469
  restored. For runnable application-layer examples, see
436
470
  [`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
437
471
 
472
+ Human-facing interpretation is a separate concern. If you want to recognize
473
+ non-canonical phrasing, recover from malformed input, narrow user intent, or
474
+ turn a failed replacement request into a different canonical directive, do that
475
+ before calling core.
476
+
438
477
  **Why not just use a plain dict?**
439
478
  A plain dict can hold state for prompt construction, schema selection, tool
440
479
  gating, and other host behavior.
@@ -462,6 +501,7 @@ overwriting state.
462
501
  - Identical input sequences produce identical compiler state.
463
502
  - Model responses never modify compiler state.
464
503
  - Ambiguous directives trigger clarification instead of changing state.
504
+ - Syntax errors never create pending continuation.
465
505
 
466
506
  Behavioral tests and Hypothesis-based property tests verify these invariants.
467
507
 
@@ -489,6 +529,10 @@ These docs cover the design and milestone details:
489
529
  [`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
490
530
  These fixtures serve as the behavioral contract for compiler semantics across implementations.
491
531
 
532
+ ## Development Process
533
+
534
+ Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
535
+
492
536
  ---
493
537
 
494
538
  ## License
@@ -4,7 +4,7 @@ import argparse
4
4
  import re
5
5
 
6
6
  import demos.llm_client as llm_client
7
- from context_compiler import create_engine, get_premise_value
7
+ from context_compiler import create_engine
8
8
  from demos.common import (
9
9
  build_baseline_messages,
10
10
  build_mediated_messages_from_transcript,
@@ -257,7 +257,7 @@ def _run_demo(turns: int = _DEFAULT_TURNS) -> None:
257
257
  compact_output = f"[no call] clarification required: {compacted_prompt}"
258
258
  print_model_output("Compiler-mediated + compact", compact_output)
259
259
  else:
260
- premise_value = get_premise_value(compacted_state)
260
+ premise_value = compacted_state["premise"]
261
261
  if (
262
262
  premise_value is not None
263
263
  and _ORIGINAL_DIRECTIVE not in compacted_turns
@@ -1,6 +1,6 @@
1
1
  """Demo 6: host-side prompt replacement from authoritative step-derived state."""
2
2
 
3
- from context_compiler import create_engine, get_premise_value
3
+ from context_compiler import create_engine
4
4
  from demos.common import compact_user_turns, is_verbose, print_info_report
5
5
 
6
6
  DEMO_NAME = "06_context_compaction — superseded directives eliminated"
@@ -44,7 +44,7 @@ def _compile_premise(turns: list[str]) -> str:
44
44
  for turn in turns:
45
45
  decision = engine.step(turn)
46
46
  assert decision["kind"] == "update"
47
- compiled_premise = get_premise_value(engine.state)
47
+ compiled_premise = engine.premise
48
48
  assert compiled_premise is not None
49
49
  return compiled_premise
50
50
 
@@ -149,7 +149,7 @@ def main() -> None:
149
149
  compiled_context = f"- premise: {compiled_premise}"
150
150
  compacted_turns, compacted_state, compacted_prompt = compact_user_turns(transcript_turns)
151
151
  assert compacted_prompt is None
152
- assert get_premise_value(compacted_state) == FINAL_PREMISE
152
+ assert compacted_state["premise"] == FINAL_PREMISE
153
153
  compacted_context = "\n".join(f"User: {turn}" for turn in compacted_turns)
154
154
  baseline_prompt = _build_baseline_prompt(transcript_turns)
155
155
  compiled_prompt = _build_compiled_prompt(compiled_premise)