context-compiler 0.8.3__tar.gz → 0.9.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/PKG-INFO +96 -52
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/README.md +95 -51
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/05_llm_prompt_drift_vs_state.py +2 -2
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/06_llm_context_compaction.py +3 -3
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/08_llm_replacement_precondition.py +24 -25
- context_compiler-0.8.3/demos/09_llm_pending_clarification.py → context_compiler-0.9.0.dev0/demos/09_llm_confirmation_passthrough.py +39 -44
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/README.md +10 -4
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/common.py +11 -7
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/run_demo.py +1 -1
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/01_persistent_guardrails.py +4 -4
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/03_ambiguity_with_clarification.py +1 -1
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/04_tool_governance_denylist.py +2 -2
- context_compiler-0.8.3/examples/06_step_sequence_and_checkpoint.py → context_compiler-0.9.0.dev0/examples/06_step_sequence_and_state_restore.py +4 -4
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/README.md +4 -3
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/_util.py +4 -4
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/pyproject.toml +1 -1
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/__init__.py +5 -6
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/controller.py +2 -2
- context_compiler-0.9.0.dev0/src/context_compiler/engine.py +373 -0
- context_compiler-0.9.0.dev0/src/context_compiler/grammar.py +434 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/repl.py +6 -86
- context_compiler-0.8.3/src/context_compiler/engine.py +0 -713
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/.gitignore +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/LICENSE +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/01_llm_contradiction_clarify.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/02_llm_constraint_guardrail.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/03_llm_premise_guardrail.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/04_llm_tool_denylist_guardrail.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/07_llm_prompt_vs_state.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/__init__.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/llm_client.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/02_configuration_and_correction.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/05_llm_integration_pattern.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/07_single_policy_correction.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/examples/08_controller_preview_diff.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/host_support/__init__.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/host_support/provider_mode.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/const.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/decision_helpers.py +0 -0
- {context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/src/context_compiler/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: context-compiler
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0.dev0
|
|
4
4
|
Summary: Deterministic conversational state engine for LLM applications.
|
|
5
5
|
Project-URL: Homepage, https://github.com/rlippmann/context-compiler
|
|
6
6
|
Project-URL: Repository, https://github.com/rlippmann/context-compiler
|
|
@@ -35,27 +35,30 @@ Description-Content-Type: text/markdown
|
|
|
35
35
|
[](https://pypi.org/project/context-compiler/)
|
|
36
36
|
|
|
37
37
|
Context Compiler is a deterministic conversational state authority for LLM applications.
|
|
38
|
-
It handles
|
|
39
|
-
|
|
38
|
+
It handles canonical directive execution, semantic validation, deterministic
|
|
39
|
+
clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
|
|
40
|
+
the host.
|
|
40
41
|
|
|
41
42
|
## What Context Compiler provides
|
|
42
43
|
|
|
43
44
|
Context Compiler gives hosts fixed state rules:
|
|
44
45
|
|
|
45
|
-
- handle explicit
|
|
46
|
+
- handle canonical explicit state changes with deterministic rules
|
|
46
47
|
- clarification instead of silent overwrite for blocked/ambiguous changes
|
|
47
|
-
-
|
|
48
|
-
- export and import
|
|
48
|
+
- preserve supported pending continuation when explicit confirmation is required
|
|
49
|
+
- export and import authoritative state for host-managed persistence
|
|
49
50
|
- produce structured authoritative state for downstream host decisions
|
|
50
51
|
|
|
51
52
|
The model generates responses. The compiler owns state.
|
|
53
|
+
Human-facing normalization, malformed-input recovery, and intent drafting belong
|
|
54
|
+
outside core.
|
|
52
55
|
|
|
53
56
|
## How the compiler metaphor works
|
|
54
57
|
|
|
55
|
-
Like a compiler, it parses
|
|
58
|
+
Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
|
|
56
59
|
produces a stable result the host can use. It treats important instructions as
|
|
57
60
|
structured state instead of temporary prompt text. It is not source-code
|
|
58
|
-
compilation and not a
|
|
61
|
+
compilation, not a reasoning model, and not a natural-language repair layer.
|
|
59
62
|
|
|
60
63
|
## 10-Second Example
|
|
61
64
|
|
|
@@ -83,33 +86,52 @@ instead of relying on memory of earlier conversation text.
|
|
|
83
86
|
|
|
84
87
|
Context Compiler makes state-change rules explicit so behavior stays repeatable.
|
|
85
88
|
|
|
86
|
-
|
|
89
|
+
The architecture has three layers:
|
|
90
|
+
|
|
91
|
+
- syntax classification decides whether input is a canonical directive, invalid
|
|
92
|
+
directive-shaped syntax, or ordinary passthrough
|
|
93
|
+
- semantic evaluation decides whether a canonical directive updates state,
|
|
94
|
+
clarifies, or no-ops
|
|
95
|
+
- semantic continuation optionally preserves a deterministic blocked transition
|
|
96
|
+
that needs explicit `yes` / `no`
|
|
97
|
+
|
|
98
|
+
### Explicit directive
|
|
99
|
+
|
|
87
100
|
```text
|
|
88
101
|
set premise concise replies
|
|
89
102
|
```
|
|
103
|
+
|
|
90
104
|
- Base model: silently accepts / rewrites
|
|
91
105
|
- Context Compiler: applies a repeatable state update
|
|
92
106
|
|
|
93
|
-
|
|
107
|
+
### Single-directive grammar
|
|
108
|
+
|
|
94
109
|
```text
|
|
95
110
|
use docker and prohibit peanuts
|
|
96
111
|
```
|
|
112
|
+
|
|
97
113
|
- Without an authority layer: host/model behavior varies
|
|
98
|
-
- Context Compiler:
|
|
114
|
+
- Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
|
|
115
|
+
|
|
116
|
+
### State-dependent operation
|
|
99
117
|
|
|
100
|
-
**State-dependent operation**
|
|
101
118
|
```text
|
|
102
119
|
clear state
|
|
103
120
|
use podman instead of docker
|
|
104
121
|
```
|
|
122
|
+
|
|
105
123
|
- Without explicit state transition rules: behavior depends on host/model handling
|
|
106
|
-
- Context Compiler:
|
|
124
|
+
- Context Compiler: applies the deterministic resulting transition when
|
|
125
|
+
`docker` is absent and `use podman` is otherwise valid; other semantic
|
|
126
|
+
conflicts may still clarify
|
|
127
|
+
|
|
128
|
+
### Lifecycle enforcement
|
|
107
129
|
|
|
108
|
-
**Lifecycle enforcement**
|
|
109
130
|
```text
|
|
110
131
|
clear state
|
|
111
132
|
change premise to formal tone
|
|
112
133
|
```
|
|
134
|
+
|
|
113
135
|
- Without explicit transition checks: behavior depends on host/model handling
|
|
114
136
|
- Context Compiler: asks for clarification and keeps saved state unchanged
|
|
115
137
|
|
|
@@ -171,7 +193,7 @@ For runnable application-layer examples, see
|
|
|
171
193
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
172
194
|
That companion repository shows enforcement points built on compiler state,
|
|
173
195
|
including retrieval filtering, schema selection, tool gating, execution
|
|
174
|
-
authorization, gateway middleware,
|
|
196
|
+
authorization, gateway middleware, runtime continuation handling, and prompt
|
|
175
197
|
construction.
|
|
176
198
|
|
|
177
199
|
## Does it Work?
|
|
@@ -197,13 +219,13 @@ pip install context-compiler
|
|
|
197
219
|
context-compiler
|
|
198
220
|
```
|
|
199
221
|
|
|
200
|
-
Preload options
|
|
222
|
+
Preload options load authoritative state:
|
|
223
|
+
|
|
201
224
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
202
225
|
(via exported state JSON).
|
|
203
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
204
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
205
226
|
|
|
206
227
|
REPL commands (controller layer, not engine directives):
|
|
228
|
+
|
|
207
229
|
- `state` shows current saved state.
|
|
208
230
|
- `preview <input>` runs deterministic dry-run without mutating live state.
|
|
209
231
|
- `step <input>` is an explicit alias of normal bare-input step behavior.
|
|
@@ -219,23 +241,25 @@ for non-interactive usage.
|
|
|
219
241
|
context-compiler --json < input.txt
|
|
220
242
|
```
|
|
221
243
|
|
|
222
|
-
Preload options
|
|
244
|
+
Preload options load authoritative state:
|
|
245
|
+
|
|
223
246
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
224
247
|
(via exported state JSON).
|
|
225
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
226
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
227
248
|
|
|
228
249
|
## Installation
|
|
229
250
|
|
|
230
251
|
Requirements:
|
|
252
|
+
|
|
231
253
|
- Python 3.11+
|
|
232
254
|
|
|
233
255
|
Install:
|
|
256
|
+
|
|
234
257
|
```bash
|
|
235
258
|
pip install context-compiler
|
|
236
259
|
```
|
|
237
260
|
|
|
238
261
|
Packaging notes:
|
|
262
|
+
|
|
239
263
|
- Base install includes the core authority-layer engine and CLI.
|
|
240
264
|
- Example and demo source files are available in the repository and source distribution.
|
|
241
265
|
- To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
|
|
@@ -261,11 +285,11 @@ class Decision(TypedDict):
|
|
|
261
285
|
|
|
262
286
|
Meaning:
|
|
263
287
|
|
|
264
|
-
| kind
|
|
265
|
-
|
|
266
|
-
| passthrough | forward user input to LLM
|
|
267
|
-
| update
|
|
268
|
-
| clarify
|
|
288
|
+
| kind | host behavior |
|
|
289
|
+
| --- | --- |
|
|
290
|
+
| passthrough | forward user input to LLM |
|
|
291
|
+
| update | authoritative state mutated; host may call LLM with updated state |
|
|
292
|
+
| clarify | show `prompt_to_user` and do not call the LLM |
|
|
269
293
|
|
|
270
294
|
For normal app code, prefer the exported decision helpers (`is_clarify`,
|
|
271
295
|
`is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
|
|
@@ -277,12 +301,11 @@ reference.
|
|
|
277
301
|
Common API entry points:
|
|
278
302
|
|
|
279
303
|
- engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
|
|
280
|
-
`engine.
|
|
304
|
+
`engine.premise`, `engine.policies`, `engine.export_json(...)`,
|
|
305
|
+
`engine.import_json(...)`
|
|
281
306
|
- decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
|
|
282
307
|
`get_clarify_prompt(...)`, `get_decision_state(...)`
|
|
283
|
-
- state
|
|
284
|
-
- state and checkpoint transport: `export_json(...)`, `import_json(...)`,
|
|
285
|
-
`export_checkpoint(...)`, `import_checkpoint(...)`
|
|
308
|
+
- state transport: `engine.export_json(...)`, `engine.import_json(...)`
|
|
286
309
|
- controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
|
|
287
310
|
|
|
288
311
|
### Controller API (Reusable Outside REPL)
|
|
@@ -312,12 +335,15 @@ authoritative in future turns.
|
|
|
312
335
|
- Policies are per-item (`use` or `prohibit`)
|
|
313
336
|
- State changes only through explicit directives
|
|
314
337
|
- No inference or semantic reasoning
|
|
338
|
+
- Non-canonical input normalization is outside the core state contract
|
|
315
339
|
|
|
316
340
|
Identical input sequences always produce identical state.
|
|
317
341
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
342
|
+
For live engine-owned reads, use `engine.premise` and `engine.policies`.
|
|
343
|
+
`engine.policies` returns a caller-owned copy.
|
|
344
|
+
|
|
345
|
+
`engine.state` remains the public snapshot/serialization boundary when host code
|
|
346
|
+
needs the full authoritative state object.
|
|
321
347
|
|
|
322
348
|
---
|
|
323
349
|
|
|
@@ -359,23 +385,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
|
|
|
359
385
|
Context Compiler enforces explicit directive and state rules. Domain reasoning
|
|
360
386
|
still belongs to the host and model workflow.
|
|
361
387
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
388
|
+
If a user says something non-canonical such as a near miss, alternate phrasing,
|
|
389
|
+
or a failed replacement request that would need reinterpretation, that
|
|
390
|
+
normalization is outside core and must happen before canonical directives reach
|
|
391
|
+
the compiler.
|
|
365
392
|
|
|
366
|
-
|
|
393
|
+
---
|
|
367
394
|
|
|
368
|
-
|
|
369
|
-
- checkpoint APIs transport **serialized continuation**:
|
|
370
|
-
- authoritative state
|
|
371
|
-
- pending confirmation flow state
|
|
395
|
+
## Persistence Contract
|
|
372
396
|
|
|
373
|
-
|
|
374
|
-
you also need resumable continuation state across process or request
|
|
375
|
-
boundaries.
|
|
397
|
+
`export_json()` / `import_json()` are the current persistence boundary.
|
|
376
398
|
|
|
377
|
-
|
|
378
|
-
|
|
399
|
+
- They transport **authoritative state only**
|
|
400
|
+
- Hosts own any broader interaction or session workflow around that state
|
|
401
|
+
- Pending continuation, if supported by the active engine contract, remains a
|
|
402
|
+
runtime semantic concept rather than a documented persisted artifact
|
|
379
403
|
|
|
380
404
|
---
|
|
381
405
|
|
|
@@ -401,6 +425,11 @@ Replacement:
|
|
|
401
425
|
User: use podman instead of docker
|
|
402
426
|
```
|
|
403
427
|
|
|
428
|
+
If `docker` is absent from saved state, that does not make the directive
|
|
429
|
+
pending. The user's intended resulting state is still unambiguous, so the
|
|
430
|
+
replacement follows the deterministic `use podman` transition when otherwise
|
|
431
|
+
semantically valid.
|
|
432
|
+
|
|
404
433
|
Removal and reset:
|
|
405
434
|
|
|
406
435
|
```text
|
|
@@ -410,9 +439,15 @@ User: clear state
|
|
|
410
439
|
```
|
|
411
440
|
|
|
412
441
|
Grammar invariant: one input may contain at most one canonical directive.
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
442
|
+
Directive-shaped invalid input is outside the canonical language, and
|
|
443
|
+
`clarify` is reserved for canonical directives that later fail semantic
|
|
444
|
+
evaluation against authoritative state.
|
|
445
|
+
|
|
446
|
+
Pending continuation is a separate runtime layer. It may exist only after a
|
|
447
|
+
canonical directive reaches a supported semantic `clarify` case. It never
|
|
448
|
+
repairs malformed syntax or reinterprets non-canonical input as a directive.
|
|
449
|
+
An absent source item in a canonical replacement directive is not, by itself,
|
|
450
|
+
such a `clarify` case.
|
|
416
451
|
|
|
417
452
|
Examples:
|
|
418
453
|
|
|
@@ -442,9 +477,8 @@ set premise "use docker and prohibit peanuts"
|
|
|
442
477
|
Quotes do not create protected literal regions inside a recognized directive
|
|
443
478
|
payload.
|
|
444
479
|
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
For full directive grammar and edge-case behavior, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
480
|
+
For the normative grammar, classification rules, and syntax-versus-semantics
|
|
481
|
+
boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
448
482
|
|
|
449
483
|
---
|
|
450
484
|
|
|
@@ -465,6 +499,11 @@ allowed, when clarification is required, and how continuation state is
|
|
|
465
499
|
restored. For runnable application-layer examples, see
|
|
466
500
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
467
501
|
|
|
502
|
+
Human-facing interpretation is a separate concern. If you want to recognize
|
|
503
|
+
non-canonical phrasing, recover from malformed input, narrow user intent, or
|
|
504
|
+
turn a failed replacement request into a different canonical directive, do that
|
|
505
|
+
before calling core.
|
|
506
|
+
|
|
468
507
|
**Why not just use a plain dict?**
|
|
469
508
|
A plain dict can hold state for prompt construction, schema selection, tool
|
|
470
509
|
gating, and other host behavior.
|
|
@@ -492,6 +531,7 @@ overwriting state.
|
|
|
492
531
|
- Identical input sequences produce identical compiler state.
|
|
493
532
|
- Model responses never modify compiler state.
|
|
494
533
|
- Ambiguous directives trigger clarification instead of changing state.
|
|
534
|
+
- Syntax errors never create pending continuation.
|
|
495
535
|
|
|
496
536
|
Behavioral tests and Hypothesis-based property tests verify these invariants.
|
|
497
537
|
|
|
@@ -519,6 +559,10 @@ These docs cover the design and milestone details:
|
|
|
519
559
|
[`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
|
|
520
560
|
These fixtures serve as the behavioral contract for compiler semantics across implementations.
|
|
521
561
|
|
|
562
|
+
## Development Process
|
|
563
|
+
|
|
564
|
+
Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
|
|
565
|
+
|
|
522
566
|
---
|
|
523
567
|
|
|
524
568
|
## License
|
|
@@ -5,27 +5,30 @@
|
|
|
5
5
|
[](https://pypi.org/project/context-compiler/)
|
|
6
6
|
|
|
7
7
|
Context Compiler is a deterministic conversational state authority for LLM applications.
|
|
8
|
-
It handles
|
|
9
|
-
|
|
8
|
+
It handles canonical directive execution, semantic validation, deterministic
|
|
9
|
+
clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
|
|
10
|
+
the host.
|
|
10
11
|
|
|
11
12
|
## What Context Compiler provides
|
|
12
13
|
|
|
13
14
|
Context Compiler gives hosts fixed state rules:
|
|
14
15
|
|
|
15
|
-
- handle explicit
|
|
16
|
+
- handle canonical explicit state changes with deterministic rules
|
|
16
17
|
- clarification instead of silent overwrite for blocked/ambiguous changes
|
|
17
|
-
-
|
|
18
|
-
- export and import
|
|
18
|
+
- preserve supported pending continuation when explicit confirmation is required
|
|
19
|
+
- export and import authoritative state for host-managed persistence
|
|
19
20
|
- produce structured authoritative state for downstream host decisions
|
|
20
21
|
|
|
21
22
|
The model generates responses. The compiler owns state.
|
|
23
|
+
Human-facing normalization, malformed-input recovery, and intent drafting belong
|
|
24
|
+
outside core.
|
|
22
25
|
|
|
23
26
|
## How the compiler metaphor works
|
|
24
27
|
|
|
25
|
-
Like a compiler, it parses
|
|
28
|
+
Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
|
|
26
29
|
produces a stable result the host can use. It treats important instructions as
|
|
27
30
|
structured state instead of temporary prompt text. It is not source-code
|
|
28
|
-
compilation and not a
|
|
31
|
+
compilation, not a reasoning model, and not a natural-language repair layer.
|
|
29
32
|
|
|
30
33
|
## 10-Second Example
|
|
31
34
|
|
|
@@ -53,33 +56,52 @@ instead of relying on memory of earlier conversation text.
|
|
|
53
56
|
|
|
54
57
|
Context Compiler makes state-change rules explicit so behavior stays repeatable.
|
|
55
58
|
|
|
56
|
-
|
|
59
|
+
The architecture has three layers:
|
|
60
|
+
|
|
61
|
+
- syntax classification decides whether input is a canonical directive, invalid
|
|
62
|
+
directive-shaped syntax, or ordinary passthrough
|
|
63
|
+
- semantic evaluation decides whether a canonical directive updates state,
|
|
64
|
+
clarifies, or no-ops
|
|
65
|
+
- semantic continuation optionally preserves a deterministic blocked transition
|
|
66
|
+
that needs explicit `yes` / `no`
|
|
67
|
+
|
|
68
|
+
### Explicit directive
|
|
69
|
+
|
|
57
70
|
```text
|
|
58
71
|
set premise concise replies
|
|
59
72
|
```
|
|
73
|
+
|
|
60
74
|
- Base model: silently accepts / rewrites
|
|
61
75
|
- Context Compiler: applies a repeatable state update
|
|
62
76
|
|
|
63
|
-
|
|
77
|
+
### Single-directive grammar
|
|
78
|
+
|
|
64
79
|
```text
|
|
65
80
|
use docker and prohibit peanuts
|
|
66
81
|
```
|
|
82
|
+
|
|
67
83
|
- Without an authority layer: host/model behavior varies
|
|
68
|
-
- Context Compiler:
|
|
84
|
+
- Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
|
|
85
|
+
|
|
86
|
+
### State-dependent operation
|
|
69
87
|
|
|
70
|
-
**State-dependent operation**
|
|
71
88
|
```text
|
|
72
89
|
clear state
|
|
73
90
|
use podman instead of docker
|
|
74
91
|
```
|
|
92
|
+
|
|
75
93
|
- Without explicit state transition rules: behavior depends on host/model handling
|
|
76
|
-
- Context Compiler:
|
|
94
|
+
- Context Compiler: applies the deterministic resulting transition when
|
|
95
|
+
`docker` is absent and `use podman` is otherwise valid; other semantic
|
|
96
|
+
conflicts may still clarify
|
|
97
|
+
|
|
98
|
+
### Lifecycle enforcement
|
|
77
99
|
|
|
78
|
-
**Lifecycle enforcement**
|
|
79
100
|
```text
|
|
80
101
|
clear state
|
|
81
102
|
change premise to formal tone
|
|
82
103
|
```
|
|
104
|
+
|
|
83
105
|
- Without explicit transition checks: behavior depends on host/model handling
|
|
84
106
|
- Context Compiler: asks for clarification and keeps saved state unchanged
|
|
85
107
|
|
|
@@ -141,7 +163,7 @@ For runnable application-layer examples, see
|
|
|
141
163
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
142
164
|
That companion repository shows enforcement points built on compiler state,
|
|
143
165
|
including retrieval filtering, schema selection, tool gating, execution
|
|
144
|
-
authorization, gateway middleware,
|
|
166
|
+
authorization, gateway middleware, runtime continuation handling, and prompt
|
|
145
167
|
construction.
|
|
146
168
|
|
|
147
169
|
## Does it Work?
|
|
@@ -167,13 +189,13 @@ pip install context-compiler
|
|
|
167
189
|
context-compiler
|
|
168
190
|
```
|
|
169
191
|
|
|
170
|
-
Preload options
|
|
192
|
+
Preload options load authoritative state:
|
|
193
|
+
|
|
171
194
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
172
195
|
(via exported state JSON).
|
|
173
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
174
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
175
196
|
|
|
176
197
|
REPL commands (controller layer, not engine directives):
|
|
198
|
+
|
|
177
199
|
- `state` shows current saved state.
|
|
178
200
|
- `preview <input>` runs deterministic dry-run without mutating live state.
|
|
179
201
|
- `step <input>` is an explicit alias of normal bare-input step behavior.
|
|
@@ -189,23 +211,25 @@ for non-interactive usage.
|
|
|
189
211
|
context-compiler --json < input.txt
|
|
190
212
|
```
|
|
191
213
|
|
|
192
|
-
Preload options
|
|
214
|
+
Preload options load authoritative state:
|
|
215
|
+
|
|
193
216
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
194
217
|
(via exported state JSON).
|
|
195
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
196
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
197
218
|
|
|
198
219
|
## Installation
|
|
199
220
|
|
|
200
221
|
Requirements:
|
|
222
|
+
|
|
201
223
|
- Python 3.11+
|
|
202
224
|
|
|
203
225
|
Install:
|
|
226
|
+
|
|
204
227
|
```bash
|
|
205
228
|
pip install context-compiler
|
|
206
229
|
```
|
|
207
230
|
|
|
208
231
|
Packaging notes:
|
|
232
|
+
|
|
209
233
|
- Base install includes the core authority-layer engine and CLI.
|
|
210
234
|
- Example and demo source files are available in the repository and source distribution.
|
|
211
235
|
- To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
|
|
@@ -231,11 +255,11 @@ class Decision(TypedDict):
|
|
|
231
255
|
|
|
232
256
|
Meaning:
|
|
233
257
|
|
|
234
|
-
| kind
|
|
235
|
-
|
|
236
|
-
| passthrough | forward user input to LLM
|
|
237
|
-
| update
|
|
238
|
-
| clarify
|
|
258
|
+
| kind | host behavior |
|
|
259
|
+
| --- | --- |
|
|
260
|
+
| passthrough | forward user input to LLM |
|
|
261
|
+
| update | authoritative state mutated; host may call LLM with updated state |
|
|
262
|
+
| clarify | show `prompt_to_user` and do not call the LLM |
|
|
239
263
|
|
|
240
264
|
For normal app code, prefer the exported decision helpers (`is_clarify`,
|
|
241
265
|
`is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
|
|
@@ -247,12 +271,11 @@ reference.
|
|
|
247
271
|
Common API entry points:
|
|
248
272
|
|
|
249
273
|
- engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
|
|
250
|
-
`engine.
|
|
274
|
+
`engine.premise`, `engine.policies`, `engine.export_json(...)`,
|
|
275
|
+
`engine.import_json(...)`
|
|
251
276
|
- decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
|
|
252
277
|
`get_clarify_prompt(...)`, `get_decision_state(...)`
|
|
253
|
-
- state
|
|
254
|
-
- state and checkpoint transport: `export_json(...)`, `import_json(...)`,
|
|
255
|
-
`export_checkpoint(...)`, `import_checkpoint(...)`
|
|
278
|
+
- state transport: `engine.export_json(...)`, `engine.import_json(...)`
|
|
256
279
|
- controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
|
|
257
280
|
|
|
258
281
|
### Controller API (Reusable Outside REPL)
|
|
@@ -282,12 +305,15 @@ authoritative in future turns.
|
|
|
282
305
|
- Policies are per-item (`use` or `prohibit`)
|
|
283
306
|
- State changes only through explicit directives
|
|
284
307
|
- No inference or semantic reasoning
|
|
308
|
+
- Non-canonical input normalization is outside the core state contract
|
|
285
309
|
|
|
286
310
|
Identical input sequences always produce identical state.
|
|
287
311
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
312
|
+
For live engine-owned reads, use `engine.premise` and `engine.policies`.
|
|
313
|
+
`engine.policies` returns a caller-owned copy.
|
|
314
|
+
|
|
315
|
+
`engine.state` remains the public snapshot/serialization boundary when host code
|
|
316
|
+
needs the full authoritative state object.
|
|
291
317
|
|
|
292
318
|
---
|
|
293
319
|
|
|
@@ -329,23 +355,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
|
|
|
329
355
|
Context Compiler enforces explicit directive and state rules. Domain reasoning
|
|
330
356
|
still belongs to the host and model workflow.
|
|
331
357
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
358
|
+
If a user says something non-canonical such as a near miss, alternate phrasing,
|
|
359
|
+
or a failed replacement request that would need reinterpretation, that
|
|
360
|
+
normalization is outside core and must happen before canonical directives reach
|
|
361
|
+
the compiler.
|
|
335
362
|
|
|
336
|
-
|
|
363
|
+
---
|
|
337
364
|
|
|
338
|
-
|
|
339
|
-
- checkpoint APIs transport **serialized continuation**:
|
|
340
|
-
- authoritative state
|
|
341
|
-
- pending confirmation flow state
|
|
365
|
+
## Persistence Contract
|
|
342
366
|
|
|
343
|
-
|
|
344
|
-
you also need resumable continuation state across process or request
|
|
345
|
-
boundaries.
|
|
367
|
+
`export_json()` / `import_json()` are the current persistence boundary.
|
|
346
368
|
|
|
347
|
-
|
|
348
|
-
|
|
369
|
+
- They transport **authoritative state only**
|
|
370
|
+
- Hosts own any broader interaction or session workflow around that state
|
|
371
|
+
- Pending continuation, if supported by the active engine contract, remains a
|
|
372
|
+
runtime semantic concept rather than a documented persisted artifact
|
|
349
373
|
|
|
350
374
|
---
|
|
351
375
|
|
|
@@ -371,6 +395,11 @@ Replacement:
|
|
|
371
395
|
User: use podman instead of docker
|
|
372
396
|
```
|
|
373
397
|
|
|
398
|
+
If `docker` is absent from saved state, that does not make the directive
|
|
399
|
+
pending. The user's intended resulting state is still unambiguous, so the
|
|
400
|
+
replacement follows the deterministic `use podman` transition when otherwise
|
|
401
|
+
semantically valid.
|
|
402
|
+
|
|
374
403
|
Removal and reset:
|
|
375
404
|
|
|
376
405
|
```text
|
|
@@ -380,9 +409,15 @@ User: clear state
|
|
|
380
409
|
```
|
|
381
410
|
|
|
382
411
|
Grammar invariant: one input may contain at most one canonical directive.
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
412
|
+
Directive-shaped invalid input is outside the canonical language, and
|
|
413
|
+
`clarify` is reserved for canonical directives that later fail semantic
|
|
414
|
+
evaluation against authoritative state.
|
|
415
|
+
|
|
416
|
+
Pending continuation is a separate runtime layer. It may exist only after a
|
|
417
|
+
canonical directive reaches a supported semantic `clarify` case. It never
|
|
418
|
+
repairs malformed syntax or reinterprets non-canonical input as a directive.
|
|
419
|
+
An absent source item in a canonical replacement directive is not, by itself,
|
|
420
|
+
such a `clarify` case.
|
|
386
421
|
|
|
387
422
|
Examples:
|
|
388
423
|
|
|
@@ -412,9 +447,8 @@ set premise "use docker and prohibit peanuts"
|
|
|
412
447
|
Quotes do not create protected literal regions inside a recognized directive
|
|
413
448
|
payload.
|
|
414
449
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
For full directive grammar and edge-case behavior, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
450
|
+
For the normative grammar, classification rules, and syntax-versus-semantics
|
|
451
|
+
boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
418
452
|
|
|
419
453
|
---
|
|
420
454
|
|
|
@@ -435,6 +469,11 @@ allowed, when clarification is required, and how continuation state is
|
|
|
435
469
|
restored. For runnable application-layer examples, see
|
|
436
470
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
437
471
|
|
|
472
|
+
Human-facing interpretation is a separate concern. If you want to recognize
|
|
473
|
+
non-canonical phrasing, recover from malformed input, narrow user intent, or
|
|
474
|
+
turn a failed replacement request into a different canonical directive, do that
|
|
475
|
+
before calling core.
|
|
476
|
+
|
|
438
477
|
**Why not just use a plain dict?**
|
|
439
478
|
A plain dict can hold state for prompt construction, schema selection, tool
|
|
440
479
|
gating, and other host behavior.
|
|
@@ -462,6 +501,7 @@ overwriting state.
|
|
|
462
501
|
- Identical input sequences produce identical compiler state.
|
|
463
502
|
- Model responses never modify compiler state.
|
|
464
503
|
- Ambiguous directives trigger clarification instead of changing state.
|
|
504
|
+
- Syntax errors never create pending continuation.
|
|
465
505
|
|
|
466
506
|
Behavioral tests and Hypothesis-based property tests verify these invariants.
|
|
467
507
|
|
|
@@ -489,6 +529,10 @@ These docs cover the design and milestone details:
|
|
|
489
529
|
[`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
|
|
490
530
|
These fixtures serve as the behavioral contract for compiler semantics across implementations.
|
|
491
531
|
|
|
532
|
+
## Development Process
|
|
533
|
+
|
|
534
|
+
Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
|
|
535
|
+
|
|
492
536
|
---
|
|
493
537
|
|
|
494
538
|
## License
|
{context_compiler-0.8.3 → context_compiler-0.9.0.dev0}/demos/05_llm_prompt_drift_vs_state.py
RENAMED
|
@@ -4,7 +4,7 @@ import argparse
|
|
|
4
4
|
import re
|
|
5
5
|
|
|
6
6
|
import demos.llm_client as llm_client
|
|
7
|
-
from context_compiler import create_engine
|
|
7
|
+
from context_compiler import create_engine
|
|
8
8
|
from demos.common import (
|
|
9
9
|
build_baseline_messages,
|
|
10
10
|
build_mediated_messages_from_transcript,
|
|
@@ -257,7 +257,7 @@ def _run_demo(turns: int = _DEFAULT_TURNS) -> None:
|
|
|
257
257
|
compact_output = f"[no call] clarification required: {compacted_prompt}"
|
|
258
258
|
print_model_output("Compiler-mediated + compact", compact_output)
|
|
259
259
|
else:
|
|
260
|
-
premise_value =
|
|
260
|
+
premise_value = compacted_state["premise"]
|
|
261
261
|
if (
|
|
262
262
|
premise_value is not None
|
|
263
263
|
and _ORIGINAL_DIRECTIVE not in compacted_turns
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Demo 6: host-side prompt replacement from authoritative step-derived state."""
|
|
2
2
|
|
|
3
|
-
from context_compiler import create_engine
|
|
3
|
+
from context_compiler import create_engine
|
|
4
4
|
from demos.common import compact_user_turns, is_verbose, print_info_report
|
|
5
5
|
|
|
6
6
|
DEMO_NAME = "06_context_compaction — superseded directives eliminated"
|
|
@@ -44,7 +44,7 @@ def _compile_premise(turns: list[str]) -> str:
|
|
|
44
44
|
for turn in turns:
|
|
45
45
|
decision = engine.step(turn)
|
|
46
46
|
assert decision["kind"] == "update"
|
|
47
|
-
compiled_premise =
|
|
47
|
+
compiled_premise = engine.premise
|
|
48
48
|
assert compiled_premise is not None
|
|
49
49
|
return compiled_premise
|
|
50
50
|
|
|
@@ -149,7 +149,7 @@ def main() -> None:
|
|
|
149
149
|
compiled_context = f"- premise: {compiled_premise}"
|
|
150
150
|
compacted_turns, compacted_state, compacted_prompt = compact_user_turns(transcript_turns)
|
|
151
151
|
assert compacted_prompt is None
|
|
152
|
-
assert
|
|
152
|
+
assert compacted_state["premise"] == FINAL_PREMISE
|
|
153
153
|
compacted_context = "\n".join(f"User: {turn}" for turn in compacted_turns)
|
|
154
154
|
baseline_prompt = _build_baseline_prompt(transcript_turns)
|
|
155
155
|
compiled_prompt = _build_compiled_prompt(compiled_premise)
|