context-compiler 0.8.2__tar.gz → 0.9.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/PKG-INFO +130 -47
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/README.md +129 -46
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/05_llm_prompt_drift_vs_state.py +2 -2
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/06_llm_context_compaction.py +8 -7
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/08_llm_replacement_precondition.py +24 -25
- context_compiler-0.8.2/demos/09_llm_pending_clarification.py → context_compiler-0.9.0.dev0/demos/09_llm_confirmation_passthrough.py +39 -44
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/README.md +10 -4
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/common.py +13 -9
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/run_demo.py +1 -1
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/01_persistent_guardrails.py +4 -4
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/03_ambiguity_with_clarification.py +1 -1
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/04_tool_governance_denylist.py +2 -2
- context_compiler-0.9.0.dev0/examples/06_step_sequence_and_state_restore.py +31 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/README.md +5 -5
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/_util.py +4 -22
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/pyproject.toml +1 -1
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/__init__.py +5 -14
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/controller.py +2 -2
- context_compiler-0.9.0.dev0/src/context_compiler/engine.py +373 -0
- context_compiler-0.9.0.dev0/src/context_compiler/grammar.py +434 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/repl.py +6 -86
- context_compiler-0.8.2/examples/06_transcript_replay.py +0 -28
- context_compiler-0.8.2/src/context_compiler/engine.py +0 -685
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/.gitignore +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/LICENSE +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/01_llm_contradiction_clarify.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/02_llm_constraint_guardrail.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/03_llm_premise_guardrail.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/04_llm_tool_denylist_guardrail.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/07_llm_prompt_vs_state.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/__init__.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/llm_client.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/02_configuration_and_correction.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/05_llm_integration_pattern.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/07_single_policy_correction.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/examples/08_controller_preview_diff.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/host_support/__init__.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/host_support/provider_mode.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/const.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/decision_helpers.py +0 -0
- {context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/src/context_compiler/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: context-compiler
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0.dev0
|
|
4
4
|
Summary: Deterministic conversational state engine for LLM applications.
|
|
5
5
|
Project-URL: Homepage, https://github.com/rlippmann/context-compiler
|
|
6
6
|
Project-URL: Repository, https://github.com/rlippmann/context-compiler
|
|
@@ -35,27 +35,30 @@ Description-Content-Type: text/markdown
|
|
|
35
35
|
[](https://pypi.org/project/context-compiler/)
|
|
36
36
|
|
|
37
37
|
Context Compiler is a deterministic conversational state authority for LLM applications.
|
|
38
|
-
It handles
|
|
39
|
-
|
|
38
|
+
It handles canonical directive execution, semantic validation, deterministic
|
|
39
|
+
clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
|
|
40
|
+
the host.
|
|
40
41
|
|
|
41
42
|
## What Context Compiler provides
|
|
42
43
|
|
|
43
44
|
Context Compiler gives hosts fixed state rules:
|
|
44
45
|
|
|
45
|
-
- handle explicit
|
|
46
|
+
- handle canonical explicit state changes with deterministic rules
|
|
46
47
|
- clarification instead of silent overwrite for blocked/ambiguous changes
|
|
47
|
-
-
|
|
48
|
-
- export and import
|
|
48
|
+
- preserve supported pending continuation when explicit confirmation is required
|
|
49
|
+
- export and import authoritative state for host-managed persistence
|
|
49
50
|
- produce structured authoritative state for downstream host decisions
|
|
50
51
|
|
|
51
52
|
The model generates responses. The compiler owns state.
|
|
53
|
+
Human-facing normalization, malformed-input recovery, and intent drafting belong
|
|
54
|
+
outside core.
|
|
52
55
|
|
|
53
56
|
## How the compiler metaphor works
|
|
54
57
|
|
|
55
|
-
Like a compiler, it parses
|
|
58
|
+
Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
|
|
56
59
|
produces a stable result the host can use. It treats important instructions as
|
|
57
60
|
structured state instead of temporary prompt text. It is not source-code
|
|
58
|
-
compilation and not a
|
|
61
|
+
compilation, not a reasoning model, and not a natural-language repair layer.
|
|
59
62
|
|
|
60
63
|
## 10-Second Example
|
|
61
64
|
|
|
@@ -83,26 +86,52 @@ instead of relying on memory of earlier conversation text.
|
|
|
83
86
|
|
|
84
87
|
Context Compiler makes state-change rules explicit so behavior stays repeatable.
|
|
85
88
|
|
|
86
|
-
|
|
89
|
+
The architecture has three layers:
|
|
90
|
+
|
|
91
|
+
- syntax classification decides whether input is a canonical directive, invalid
|
|
92
|
+
directive-shaped syntax, or ordinary passthrough
|
|
93
|
+
- semantic evaluation decides whether a canonical directive updates state,
|
|
94
|
+
clarifies, or no-ops
|
|
95
|
+
- semantic continuation optionally preserves a deterministic blocked transition
|
|
96
|
+
that needs explicit `yes` / `no`
|
|
97
|
+
|
|
98
|
+
### Explicit directive
|
|
99
|
+
|
|
87
100
|
```text
|
|
88
101
|
set premise concise replies
|
|
89
102
|
```
|
|
103
|
+
|
|
90
104
|
- Base model: silently accepts / rewrites
|
|
91
105
|
- Context Compiler: applies a repeatable state update
|
|
92
106
|
|
|
93
|
-
|
|
107
|
+
### Single-directive grammar
|
|
108
|
+
|
|
109
|
+
```text
|
|
110
|
+
use docker and prohibit peanuts
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
- Without an authority layer: host/model behavior varies
|
|
114
|
+
- Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
|
|
115
|
+
|
|
116
|
+
### State-dependent operation
|
|
117
|
+
|
|
94
118
|
```text
|
|
95
119
|
clear state
|
|
96
120
|
use podman instead of docker
|
|
97
121
|
```
|
|
122
|
+
|
|
98
123
|
- Without explicit state transition rules: behavior depends on host/model handling
|
|
99
|
-
- Context Compiler:
|
|
124
|
+
- Context Compiler: applies the deterministic resulting transition when
|
|
125
|
+
`docker` is absent and `use podman` is otherwise valid; other semantic
|
|
126
|
+
conflicts may still clarify
|
|
127
|
+
|
|
128
|
+
### Lifecycle enforcement
|
|
100
129
|
|
|
101
|
-
**Lifecycle enforcement**
|
|
102
130
|
```text
|
|
103
131
|
clear state
|
|
104
132
|
change premise to formal tone
|
|
105
133
|
```
|
|
134
|
+
|
|
106
135
|
- Without explicit transition checks: behavior depends on host/model handling
|
|
107
136
|
- Context Compiler: asks for clarification and keeps saved state unchanged
|
|
108
137
|
|
|
@@ -164,7 +193,7 @@ For runnable application-layer examples, see
|
|
|
164
193
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
165
194
|
That companion repository shows enforcement points built on compiler state,
|
|
166
195
|
including retrieval filtering, schema selection, tool gating, execution
|
|
167
|
-
authorization, gateway middleware,
|
|
196
|
+
authorization, gateway middleware, runtime continuation handling, and prompt
|
|
168
197
|
construction.
|
|
169
198
|
|
|
170
199
|
## Does it Work?
|
|
@@ -190,13 +219,13 @@ pip install context-compiler
|
|
|
190
219
|
context-compiler
|
|
191
220
|
```
|
|
192
221
|
|
|
193
|
-
Preload options
|
|
222
|
+
Preload options load authoritative state:
|
|
223
|
+
|
|
194
224
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
195
225
|
(via exported state JSON).
|
|
196
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
197
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
198
226
|
|
|
199
227
|
REPL commands (controller layer, not engine directives):
|
|
228
|
+
|
|
200
229
|
- `state` shows current saved state.
|
|
201
230
|
- `preview <input>` runs deterministic dry-run without mutating live state.
|
|
202
231
|
- `step <input>` is an explicit alias of normal bare-input step behavior.
|
|
@@ -212,23 +241,25 @@ for non-interactive usage.
|
|
|
212
241
|
context-compiler --json < input.txt
|
|
213
242
|
```
|
|
214
243
|
|
|
215
|
-
Preload options
|
|
244
|
+
Preload options load authoritative state:
|
|
245
|
+
|
|
216
246
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
217
247
|
(via exported state JSON).
|
|
218
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
219
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
220
248
|
|
|
221
249
|
## Installation
|
|
222
250
|
|
|
223
251
|
Requirements:
|
|
252
|
+
|
|
224
253
|
- Python 3.11+
|
|
225
254
|
|
|
226
255
|
Install:
|
|
256
|
+
|
|
227
257
|
```bash
|
|
228
258
|
pip install context-compiler
|
|
229
259
|
```
|
|
230
260
|
|
|
231
261
|
Packaging notes:
|
|
262
|
+
|
|
232
263
|
- Base install includes the core authority-layer engine and CLI.
|
|
233
264
|
- Example and demo source files are available in the repository and source distribution.
|
|
234
265
|
- To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
|
|
@@ -254,11 +285,11 @@ class Decision(TypedDict):
|
|
|
254
285
|
|
|
255
286
|
Meaning:
|
|
256
287
|
|
|
257
|
-
| kind
|
|
258
|
-
|
|
259
|
-
| passthrough | forward user input to LLM
|
|
260
|
-
| update
|
|
261
|
-
| clarify
|
|
288
|
+
| kind | host behavior |
|
|
289
|
+
| --- | --- |
|
|
290
|
+
| passthrough | forward user input to LLM |
|
|
291
|
+
| update | authoritative state mutated; host may call LLM with updated state |
|
|
292
|
+
| clarify | show `prompt_to_user` and do not call the LLM |
|
|
262
293
|
|
|
263
294
|
For normal app code, prefer the exported decision helpers (`is_clarify`,
|
|
264
295
|
`is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
|
|
@@ -270,13 +301,11 @@ reference.
|
|
|
270
301
|
Common API entry points:
|
|
271
302
|
|
|
272
303
|
- engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
|
|
273
|
-
`engine.
|
|
304
|
+
`engine.premise`, `engine.policies`, `engine.export_json(...)`,
|
|
305
|
+
`engine.import_json(...)`
|
|
274
306
|
- decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
|
|
275
307
|
`get_clarify_prompt(...)`, `get_decision_state(...)`
|
|
276
|
-
- state
|
|
277
|
-
- transcript APIs: `compile_transcript(...)`, `engine.apply_transcript(...)`
|
|
278
|
-
- state and checkpoint transport: `export_json(...)`, `import_json(...)`,
|
|
279
|
-
`export_checkpoint(...)`, `import_checkpoint(...)`
|
|
308
|
+
- state transport: `engine.export_json(...)`, `engine.import_json(...)`
|
|
280
309
|
- controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
|
|
281
310
|
|
|
282
311
|
### Controller API (Reusable Outside REPL)
|
|
@@ -306,12 +335,15 @@ authoritative in future turns.
|
|
|
306
335
|
- Policies are per-item (`use` or `prohibit`)
|
|
307
336
|
- State changes only through explicit directives
|
|
308
337
|
- No inference or semantic reasoning
|
|
338
|
+
- Non-canonical input normalization is outside the core state contract
|
|
309
339
|
|
|
310
340
|
Identical input sequences always produce identical state.
|
|
311
341
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
342
|
+
For live engine-owned reads, use `engine.premise` and `engine.policies`.
|
|
343
|
+
`engine.policies` returns a caller-owned copy.
|
|
344
|
+
|
|
345
|
+
`engine.state` remains the public snapshot/serialization boundary when host code
|
|
346
|
+
needs the full authoritative state object.
|
|
315
347
|
|
|
316
348
|
---
|
|
317
349
|
|
|
@@ -353,23 +385,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
|
|
|
353
385
|
Context Compiler enforces explicit directive and state rules. Domain reasoning
|
|
354
386
|
still belongs to the host and model workflow.
|
|
355
387
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
388
|
+
If a user says something non-canonical such as a near miss, alternate phrasing,
|
|
389
|
+
or a failed replacement request that would need reinterpretation, that
|
|
390
|
+
normalization is outside core and must happen before canonical directives reach
|
|
391
|
+
the compiler.
|
|
359
392
|
|
|
360
|
-
|
|
393
|
+
---
|
|
361
394
|
|
|
362
|
-
|
|
363
|
-
- checkpoint APIs transport **serialized continuation**:
|
|
364
|
-
- authoritative state
|
|
365
|
-
- pending confirmation flow state
|
|
395
|
+
## Persistence Contract
|
|
366
396
|
|
|
367
|
-
|
|
368
|
-
you also need resumable continuation state across process or request
|
|
369
|
-
boundaries.
|
|
397
|
+
`export_json()` / `import_json()` are the current persistence boundary.
|
|
370
398
|
|
|
371
|
-
|
|
372
|
-
|
|
399
|
+
- They transport **authoritative state only**
|
|
400
|
+
- Hosts own any broader interaction or session workflow around that state
|
|
401
|
+
- Pending continuation, if supported by the active engine contract, remains a
|
|
402
|
+
runtime semantic concept rather than a documented persisted artifact
|
|
373
403
|
|
|
374
404
|
---
|
|
375
405
|
|
|
@@ -395,6 +425,11 @@ Replacement:
|
|
|
395
425
|
User: use podman instead of docker
|
|
396
426
|
```
|
|
397
427
|
|
|
428
|
+
If `docker` is absent from saved state, that does not make the directive
|
|
429
|
+
pending. The user's intended resulting state is still unambiguous, so the
|
|
430
|
+
replacement follows the deterministic `use podman` transition when otherwise
|
|
431
|
+
semantically valid.
|
|
432
|
+
|
|
398
433
|
Removal and reset:
|
|
399
434
|
|
|
400
435
|
```text
|
|
@@ -403,9 +438,47 @@ User: reset policies
|
|
|
403
438
|
User: clear state
|
|
404
439
|
```
|
|
405
440
|
|
|
406
|
-
|
|
441
|
+
Grammar invariant: one input may contain at most one canonical directive.
|
|
442
|
+
Directive-shaped invalid input is outside the canonical language, and
|
|
443
|
+
`clarify` is reserved for canonical directives that later fail semantic
|
|
444
|
+
evaluation against authoritative state.
|
|
445
|
+
|
|
446
|
+
Pending continuation is a separate runtime layer. It may exist only after a
|
|
447
|
+
canonical directive reaches a supported semantic `clarify` case. It never
|
|
448
|
+
repairs malformed syntax or reinterprets non-canonical input as a directive.
|
|
449
|
+
An absent source item in a canonical replacement directive is not, by itself,
|
|
450
|
+
such a `clarify` case.
|
|
451
|
+
|
|
452
|
+
Examples:
|
|
407
453
|
|
|
408
|
-
|
|
454
|
+
```text
|
|
455
|
+
Valid:
|
|
456
|
+
use docker
|
|
457
|
+
use podman instead of docker
|
|
458
|
+
clear state
|
|
459
|
+
|
|
460
|
+
Invalid:
|
|
461
|
+
use docker and prohibit peanuts
|
|
462
|
+
set premise vegetarian and use docker
|
|
463
|
+
clear state then set premise new project
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
Quote behavior follows the current grammar literally:
|
|
467
|
+
|
|
468
|
+
```text
|
|
469
|
+
Passthrough:
|
|
470
|
+
"use docker and prohibit peanuts"
|
|
471
|
+
|
|
472
|
+
Invalid:
|
|
473
|
+
use "docker and prohibit peanuts"
|
|
474
|
+
set premise "use docker and prohibit peanuts"
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
Quotes do not create protected literal regions inside a recognized directive
|
|
478
|
+
payload.
|
|
479
|
+
|
|
480
|
+
For the normative grammar, classification rules, and syntax-versus-semantics
|
|
481
|
+
boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
409
482
|
|
|
410
483
|
---
|
|
411
484
|
|
|
@@ -426,6 +499,11 @@ allowed, when clarification is required, and how continuation state is
|
|
|
426
499
|
restored. For runnable application-layer examples, see
|
|
427
500
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
428
501
|
|
|
502
|
+
Human-facing interpretation is a separate concern. If you want to recognize
|
|
503
|
+
non-canonical phrasing, recover from malformed input, narrow user intent, or
|
|
504
|
+
turn a failed replacement request into a different canonical directive, do that
|
|
505
|
+
before calling core.
|
|
506
|
+
|
|
429
507
|
**Why not just use a plain dict?**
|
|
430
508
|
A plain dict can hold state for prompt construction, schema selection, tool
|
|
431
509
|
gating, and other host behavior.
|
|
@@ -453,6 +531,7 @@ overwriting state.
|
|
|
453
531
|
- Identical input sequences produce identical compiler state.
|
|
454
532
|
- Model responses never modify compiler state.
|
|
455
533
|
- Ambiguous directives trigger clarification instead of changing state.
|
|
534
|
+
- Syntax errors never create pending continuation.
|
|
456
535
|
|
|
457
536
|
Behavioral tests and Hypothesis-based property tests verify these invariants.
|
|
458
537
|
|
|
@@ -480,6 +559,10 @@ These docs cover the design and milestone details:
|
|
|
480
559
|
[`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
|
|
481
560
|
These fixtures serve as the behavioral contract for compiler semantics across implementations.
|
|
482
561
|
|
|
562
|
+
## Development Process
|
|
563
|
+
|
|
564
|
+
Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
|
|
565
|
+
|
|
483
566
|
---
|
|
484
567
|
|
|
485
568
|
## License
|
|
@@ -5,27 +5,30 @@
|
|
|
5
5
|
[](https://pypi.org/project/context-compiler/)
|
|
6
6
|
|
|
7
7
|
Context Compiler is a deterministic conversational state authority for LLM applications.
|
|
8
|
-
It handles
|
|
9
|
-
|
|
8
|
+
It handles canonical directive execution, semantic validation, deterministic
|
|
9
|
+
clarify decisions, runtime semantic continuation boundaries, and structured authoritative state for
|
|
10
|
+
the host.
|
|
10
11
|
|
|
11
12
|
## What Context Compiler provides
|
|
12
13
|
|
|
13
14
|
Context Compiler gives hosts fixed state rules:
|
|
14
15
|
|
|
15
|
-
- handle explicit
|
|
16
|
+
- handle canonical explicit state changes with deterministic rules
|
|
16
17
|
- clarification instead of silent overwrite for blocked/ambiguous changes
|
|
17
|
-
-
|
|
18
|
-
- export and import
|
|
18
|
+
- preserve supported pending continuation when explicit confirmation is required
|
|
19
|
+
- export and import authoritative state for host-managed persistence
|
|
19
20
|
- produce structured authoritative state for downstream host decisions
|
|
20
21
|
|
|
21
22
|
The model generates responses. The compiler owns state.
|
|
23
|
+
Human-facing normalization, malformed-input recovery, and intent drafting belong
|
|
24
|
+
outside core.
|
|
22
25
|
|
|
23
26
|
## How the compiler metaphor works
|
|
24
27
|
|
|
25
|
-
Like a compiler, it parses
|
|
28
|
+
Like a compiler, it parses canonical directives, validates them, applies fixed rules, and
|
|
26
29
|
produces a stable result the host can use. It treats important instructions as
|
|
27
30
|
structured state instead of temporary prompt text. It is not source-code
|
|
28
|
-
compilation and not a
|
|
31
|
+
compilation, not a reasoning model, and not a natural-language repair layer.
|
|
29
32
|
|
|
30
33
|
## 10-Second Example
|
|
31
34
|
|
|
@@ -53,26 +56,52 @@ instead of relying on memory of earlier conversation text.
|
|
|
53
56
|
|
|
54
57
|
Context Compiler makes state-change rules explicit so behavior stays repeatable.
|
|
55
58
|
|
|
56
|
-
|
|
59
|
+
The architecture has three layers:
|
|
60
|
+
|
|
61
|
+
- syntax classification decides whether input is a canonical directive, invalid
|
|
62
|
+
directive-shaped syntax, or ordinary passthrough
|
|
63
|
+
- semantic evaluation decides whether a canonical directive updates state,
|
|
64
|
+
clarifies, or no-ops
|
|
65
|
+
- semantic continuation optionally preserves a deterministic blocked transition
|
|
66
|
+
that needs explicit `yes` / `no`
|
|
67
|
+
|
|
68
|
+
### Explicit directive
|
|
69
|
+
|
|
57
70
|
```text
|
|
58
71
|
set premise concise replies
|
|
59
72
|
```
|
|
73
|
+
|
|
60
74
|
- Base model: silently accepts / rewrites
|
|
61
75
|
- Context Compiler: applies a repeatable state update
|
|
62
76
|
|
|
63
|
-
|
|
77
|
+
### Single-directive grammar
|
|
78
|
+
|
|
79
|
+
```text
|
|
80
|
+
use docker and prohibit peanuts
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
- Without an authority layer: host/model behavior varies
|
|
84
|
+
- Context Compiler: treats this as invalid directive-shaped syntax, keeps authoritative state unchanged, and does not create pending continuation
|
|
85
|
+
|
|
86
|
+
### State-dependent operation
|
|
87
|
+
|
|
64
88
|
```text
|
|
65
89
|
clear state
|
|
66
90
|
use podman instead of docker
|
|
67
91
|
```
|
|
92
|
+
|
|
68
93
|
- Without explicit state transition rules: behavior depends on host/model handling
|
|
69
|
-
- Context Compiler:
|
|
94
|
+
- Context Compiler: applies the deterministic resulting transition when
|
|
95
|
+
`docker` is absent and `use podman` is otherwise valid; other semantic
|
|
96
|
+
conflicts may still clarify
|
|
97
|
+
|
|
98
|
+
### Lifecycle enforcement
|
|
70
99
|
|
|
71
|
-
**Lifecycle enforcement**
|
|
72
100
|
```text
|
|
73
101
|
clear state
|
|
74
102
|
change premise to formal tone
|
|
75
103
|
```
|
|
104
|
+
|
|
76
105
|
- Without explicit transition checks: behavior depends on host/model handling
|
|
77
106
|
- Context Compiler: asks for clarification and keeps saved state unchanged
|
|
78
107
|
|
|
@@ -134,7 +163,7 @@ For runnable application-layer examples, see
|
|
|
134
163
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
135
164
|
That companion repository shows enforcement points built on compiler state,
|
|
136
165
|
including retrieval filtering, schema selection, tool gating, execution
|
|
137
|
-
authorization, gateway middleware,
|
|
166
|
+
authorization, gateway middleware, runtime continuation handling, and prompt
|
|
138
167
|
construction.
|
|
139
168
|
|
|
140
169
|
## Does it Work?
|
|
@@ -160,13 +189,13 @@ pip install context-compiler
|
|
|
160
189
|
context-compiler
|
|
161
190
|
```
|
|
162
191
|
|
|
163
|
-
Preload options
|
|
192
|
+
Preload options load authoritative state:
|
|
193
|
+
|
|
164
194
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
165
195
|
(via exported state JSON).
|
|
166
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
167
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
168
196
|
|
|
169
197
|
REPL commands (controller layer, not engine directives):
|
|
198
|
+
|
|
170
199
|
- `state` shows current saved state.
|
|
171
200
|
- `preview <input>` runs deterministic dry-run without mutating live state.
|
|
172
201
|
- `step <input>` is an explicit alias of normal bare-input step behavior.
|
|
@@ -182,23 +211,25 @@ for non-interactive usage.
|
|
|
182
211
|
context-compiler --json < input.txt
|
|
183
212
|
```
|
|
184
213
|
|
|
185
|
-
Preload options
|
|
214
|
+
Preload options load authoritative state:
|
|
215
|
+
|
|
186
216
|
- `--initial-state-json` / `--initial-state-file` load saved state
|
|
187
217
|
(via exported state JSON).
|
|
188
|
-
- `--initial-checkpoint-json` / `--initial-checkpoint-file` restore full
|
|
189
|
-
continuation checkpoint (saved state + pending confirmation state).
|
|
190
218
|
|
|
191
219
|
## Installation
|
|
192
220
|
|
|
193
221
|
Requirements:
|
|
222
|
+
|
|
194
223
|
- Python 3.11+
|
|
195
224
|
|
|
196
225
|
Install:
|
|
226
|
+
|
|
197
227
|
```bash
|
|
198
228
|
pip install context-compiler
|
|
199
229
|
```
|
|
200
230
|
|
|
201
231
|
Packaging notes:
|
|
232
|
+
|
|
202
233
|
- Base install includes the core authority-layer engine and CLI.
|
|
203
234
|
- Example and demo source files are available in the repository and source distribution.
|
|
204
235
|
- To run the demos from this repository, clone the repo and install `context-compiler[demos]`.
|
|
@@ -224,11 +255,11 @@ class Decision(TypedDict):
|
|
|
224
255
|
|
|
225
256
|
Meaning:
|
|
226
257
|
|
|
227
|
-
| kind
|
|
228
|
-
|
|
229
|
-
| passthrough | forward user input to LLM
|
|
230
|
-
| update
|
|
231
|
-
| clarify
|
|
258
|
+
| kind | host behavior |
|
|
259
|
+
| --- | --- |
|
|
260
|
+
| passthrough | forward user input to LLM |
|
|
261
|
+
| update | authoritative state mutated; host may call LLM with updated state |
|
|
262
|
+
| clarify | show `prompt_to_user` and do not call the LLM |
|
|
232
263
|
|
|
233
264
|
For normal app code, prefer the exported decision helpers (`is_clarify`,
|
|
234
265
|
`is_update`, `is_passthrough`, `get_clarify_prompt`, `get_decision_state`)
|
|
@@ -240,13 +271,11 @@ reference.
|
|
|
240
271
|
Common API entry points:
|
|
241
272
|
|
|
242
273
|
- engine lifecycle: `create_engine(...)`, `engine.step(...)`, `engine.state`,
|
|
243
|
-
`engine.
|
|
274
|
+
`engine.premise`, `engine.policies`, `engine.export_json(...)`,
|
|
275
|
+
`engine.import_json(...)`
|
|
244
276
|
- decision helpers: `is_clarify(...)`, `is_update(...)`, `is_passthrough(...)`,
|
|
245
277
|
`get_clarify_prompt(...)`, `get_decision_state(...)`
|
|
246
|
-
- state
|
|
247
|
-
- transcript APIs: `compile_transcript(...)`, `engine.apply_transcript(...)`
|
|
248
|
-
- state and checkpoint transport: `export_json(...)`, `import_json(...)`,
|
|
249
|
-
`export_checkpoint(...)`, `import_checkpoint(...)`
|
|
278
|
+
- state transport: `engine.export_json(...)`, `engine.import_json(...)`
|
|
250
279
|
- controller APIs: `preview(...)`, `step(...)`, `state_diff(...)`
|
|
251
280
|
|
|
252
281
|
### Controller API (Reusable Outside REPL)
|
|
@@ -276,12 +305,15 @@ authoritative in future turns.
|
|
|
276
305
|
- Policies are per-item (`use` or `prohibit`)
|
|
277
306
|
- State changes only through explicit directives
|
|
278
307
|
- No inference or semantic reasoning
|
|
308
|
+
- Non-canonical input normalization is outside the core state contract
|
|
279
309
|
|
|
280
310
|
Identical input sequences always produce identical state.
|
|
281
311
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
312
|
+
For live engine-owned reads, use `engine.premise` and `engine.policies`.
|
|
313
|
+
`engine.policies` returns a caller-owned copy.
|
|
314
|
+
|
|
315
|
+
`engine.state` remains the public snapshot/serialization boundary when host code
|
|
316
|
+
needs the full authoritative state object.
|
|
285
317
|
|
|
286
318
|
---
|
|
287
319
|
|
|
@@ -323,23 +355,21 @@ Hosts define what policy items and premise mean in context. Common patterns incl
|
|
|
323
355
|
Context Compiler enforces explicit directive and state rules. Domain reasoning
|
|
324
356
|
still belongs to the host and model workflow.
|
|
325
357
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
358
|
+
If a user says something non-canonical such as a near miss, alternate phrasing,
|
|
359
|
+
or a failed replacement request that would need reinterpretation, that
|
|
360
|
+
normalization is outside core and must happen before canonical directives reach
|
|
361
|
+
the compiler.
|
|
329
362
|
|
|
330
|
-
|
|
363
|
+
---
|
|
331
364
|
|
|
332
|
-
|
|
333
|
-
- checkpoint APIs transport **serialized continuation**:
|
|
334
|
-
- authoritative state
|
|
335
|
-
- pending confirmation flow state
|
|
365
|
+
## Persistence Contract
|
|
336
366
|
|
|
337
|
-
|
|
338
|
-
you also need resumable continuation state across process or request
|
|
339
|
-
boundaries.
|
|
367
|
+
`export_json()` / `import_json()` are the current persistence boundary.
|
|
340
368
|
|
|
341
|
-
|
|
342
|
-
|
|
369
|
+
- They transport **authoritative state only**
|
|
370
|
+
- Hosts own any broader interaction or session workflow around that state
|
|
371
|
+
- Pending continuation, if supported by the active engine contract, remains a
|
|
372
|
+
runtime semantic concept rather than a documented persisted artifact
|
|
343
373
|
|
|
344
374
|
---
|
|
345
375
|
|
|
@@ -365,6 +395,11 @@ Replacement:
|
|
|
365
395
|
User: use podman instead of docker
|
|
366
396
|
```
|
|
367
397
|
|
|
398
|
+
If `docker` is absent from saved state, that does not make the directive
|
|
399
|
+
pending. The user's intended resulting state is still unambiguous, so the
|
|
400
|
+
replacement follows the deterministic `use podman` transition when otherwise
|
|
401
|
+
semantically valid.
|
|
402
|
+
|
|
368
403
|
Removal and reset:
|
|
369
404
|
|
|
370
405
|
```text
|
|
@@ -373,9 +408,47 @@ User: reset policies
|
|
|
373
408
|
User: clear state
|
|
374
409
|
```
|
|
375
410
|
|
|
376
|
-
|
|
411
|
+
Grammar invariant: one input may contain at most one canonical directive.
|
|
412
|
+
Directive-shaped invalid input is outside the canonical language, and
|
|
413
|
+
`clarify` is reserved for canonical directives that later fail semantic
|
|
414
|
+
evaluation against authoritative state.
|
|
415
|
+
|
|
416
|
+
Pending continuation is a separate runtime layer. It may exist only after a
|
|
417
|
+
canonical directive reaches a supported semantic `clarify` case. It never
|
|
418
|
+
repairs malformed syntax or reinterprets non-canonical input as a directive.
|
|
419
|
+
An absent source item in a canonical replacement directive is not, by itself,
|
|
420
|
+
such a `clarify` case.
|
|
421
|
+
|
|
422
|
+
Examples:
|
|
377
423
|
|
|
378
|
-
|
|
424
|
+
```text
|
|
425
|
+
Valid:
|
|
426
|
+
use docker
|
|
427
|
+
use podman instead of docker
|
|
428
|
+
clear state
|
|
429
|
+
|
|
430
|
+
Invalid:
|
|
431
|
+
use docker and prohibit peanuts
|
|
432
|
+
set premise vegetarian and use docker
|
|
433
|
+
clear state then set premise new project
|
|
434
|
+
```
|
|
435
|
+
|
|
436
|
+
Quote behavior follows the current grammar literally:
|
|
437
|
+
|
|
438
|
+
```text
|
|
439
|
+
Passthrough:
|
|
440
|
+
"use docker and prohibit peanuts"
|
|
441
|
+
|
|
442
|
+
Invalid:
|
|
443
|
+
use "docker and prohibit peanuts"
|
|
444
|
+
set premise "use docker and prohibit peanuts"
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
Quotes do not create protected literal regions inside a recognized directive
|
|
448
|
+
payload.
|
|
449
|
+
|
|
450
|
+
For the normative grammar, classification rules, and syntax-versus-semantics
|
|
451
|
+
boundary, see [DirectiveGrammarSpec.md](docs/DirectiveGrammarSpec.md).
|
|
379
452
|
|
|
380
453
|
---
|
|
381
454
|
|
|
@@ -396,6 +469,11 @@ allowed, when clarification is required, and how continuation state is
|
|
|
396
469
|
restored. For runnable application-layer examples, see
|
|
397
470
|
[`context-compiler-example-integrations`](https://github.com/rlippmann/context-compiler-example-integrations).
|
|
398
471
|
|
|
472
|
+
Human-facing interpretation is a separate concern. If you want to recognize
|
|
473
|
+
non-canonical phrasing, recover from malformed input, narrow user intent, or
|
|
474
|
+
turn a failed replacement request into a different canonical directive, do that
|
|
475
|
+
before calling core.
|
|
476
|
+
|
|
399
477
|
**Why not just use a plain dict?**
|
|
400
478
|
A plain dict can hold state for prompt construction, schema selection, tool
|
|
401
479
|
gating, and other host behavior.
|
|
@@ -423,6 +501,7 @@ overwriting state.
|
|
|
423
501
|
- Identical input sequences produce identical compiler state.
|
|
424
502
|
- Model responses never modify compiler state.
|
|
425
503
|
- Ambiguous directives trigger clarification instead of changing state.
|
|
504
|
+
- Syntax errors never create pending continuation.
|
|
426
505
|
|
|
427
506
|
Behavioral tests and Hypothesis-based property tests verify these invariants.
|
|
428
507
|
|
|
@@ -450,6 +529,10 @@ These docs cover the design and milestone details:
|
|
|
450
529
|
[`tests/fixtures/`](tests/fixtures/) defines the cross-language conformance tests.
|
|
451
530
|
These fixtures serve as the behavioral contract for compiler semantics across implementations.
|
|
452
531
|
|
|
532
|
+
## Development Process
|
|
533
|
+
|
|
534
|
+
Most of this project and related projects were implemented with Codex across many development sessions, including substantial implementation, refactoring, and cross-language porting work. ChatGPT was used separately for design discussion, review, and planning. Conformance harnesses and tests were used to verify behavioral consistency rather than treating model output as the correctness check.
|
|
535
|
+
|
|
453
536
|
---
|
|
454
537
|
|
|
455
538
|
## License
|
{context_compiler-0.8.2 → context_compiler-0.9.0.dev0}/demos/05_llm_prompt_drift_vs_state.py
RENAMED
|
@@ -4,7 +4,7 @@ import argparse
|
|
|
4
4
|
import re
|
|
5
5
|
|
|
6
6
|
import demos.llm_client as llm_client
|
|
7
|
-
from context_compiler import create_engine
|
|
7
|
+
from context_compiler import create_engine
|
|
8
8
|
from demos.common import (
|
|
9
9
|
build_baseline_messages,
|
|
10
10
|
build_mediated_messages_from_transcript,
|
|
@@ -257,7 +257,7 @@ def _run_demo(turns: int = _DEFAULT_TURNS) -> None:
|
|
|
257
257
|
compact_output = f"[no call] clarification required: {compacted_prompt}"
|
|
258
258
|
print_model_output("Compiler-mediated + compact", compact_output)
|
|
259
259
|
else:
|
|
260
|
-
premise_value =
|
|
260
|
+
premise_value = compacted_state["premise"]
|
|
261
261
|
if (
|
|
262
262
|
premise_value is not None
|
|
263
263
|
and _ORIGINAL_DIRECTIVE not in compacted_turns
|