annoscript 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- annoscript/__init__.py +1 -0
- annoscript/_backend_profiles.py +17 -0
- annoscript/_client_common.py +287 -0
- annoscript/_data.py +37 -0
- annoscript/_dispatch_pool.py +47 -0
- annoscript/_env.py +19 -0
- annoscript/_examples.py +34 -0
- annoscript/_expressions.py +93 -0
- annoscript/_plan_viewer.html +953 -0
- annoscript/_remote_jobs.py +211 -0
- annoscript/_request_adapters.py +71 -0
- annoscript/_speculation.py +321 -0
- annoscript/_test_artifacts.py +33 -0
- annoscript/_test_evaluation_http.py +117 -0
- annoscript/_test_ir_runtime.py +138 -0
- annoscript/_test_job_process.py +166 -0
- annoscript/_test_plan_runtime.py +331 -0
- annoscript/_test_remote_client.py +46 -0
- annoscript/_test_remote_server.py +75 -0
- annoscript/_test_synthetic.py +67 -0
- annoscript/anno.py +270 -0
- annoscript/backends/__init__.py +13 -0
- annoscript/backends/_batch_wire.py +26 -0
- annoscript/backends/_bounded_grammar.py +97 -0
- annoscript/backends/_example_layout.py +23 -0
- annoscript/backends/_generation_policy.py +116 -0
- annoscript/backends/_grammar.py +83 -0
- annoscript/backends/_preview.py +13 -0
- annoscript/backends/_prompt.py +115 -0
- annoscript/backends/_shared_rules.py +38 -0
- annoscript/backends/adapters/__init__.py +17 -0
- annoscript/backends/adapters/chat.py +39 -0
- annoscript/backends/adapters/decision.py +119 -0
- annoscript/backends/base.py +427 -0
- annoscript/backends/config.py +67 -0
- annoscript/backends/decision.py +150 -0
- annoscript/backends/llm.py +338 -0
- annoscript/client.py +247 -0
- annoscript/client_tests.py +17 -0
- annoscript/compiler.py +708 -0
- annoscript/cross_version_tests.py +68 -0
- annoscript/fixtures/request-adapters-v1.json +4741 -0
- annoscript/formatter.py +171 -0
- annoscript/grammar_tests.py +176 -0
- annoscript/inspect.py +77 -0
- annoscript/ir.py +458 -0
- annoscript/job_cross_version_tests.py +118 -0
- annoscript/job_protocol.py +180 -0
- annoscript/jobs.py +255 -0
- annoscript/metrics.py +153 -0
- annoscript/p0_grammar_tests.py +147 -0
- annoscript/passes/__init__.py +40 -0
- annoscript/passes/_analysis.py +178 -0
- annoscript/passes/control_flow.py +68 -0
- annoscript/passes/dead_assignments.py +44 -0
- annoscript/passes/folding.py +104 -0
- annoscript/passes/propagation.py +53 -0
- annoscript/passes/temporaries.py +45 -0
- annoscript/plan.py +542 -0
- annoscript/plan_formatter.py +452 -0
- annoscript/plan_lowering.py +571 -0
- annoscript/preview.py +115 -0
- annoscript/remote.py +304 -0
- annoscript/runtime.py +861 -0
- annoscript/server/__init__.py +1 -0
- annoscript/server/__main__.py +49 -0
- annoscript/server/_plan_cache.py +79 -0
- annoscript/server/api.py +146 -0
- annoscript/server/metrics.py +140 -0
- annoscript/server/migrate.py +61 -0
- annoscript/server/releases.py +143 -0
- annoscript/server/store.py +429 -0
- annoscript/server/worker.py +290 -0
- annoscript/substring_tests.py +79 -0
- annoscript/test_advisory_lengths.py +115 -0
- annoscript/test_backends.py +181 -0
- annoscript/test_client.py +349 -0
- annoscript/test_compact_batch.py +95 -0
- annoscript/test_compiler.py +444 -0
- annoscript/test_decision_adapters.py +410 -0
- annoscript/test_decision_wire.py +190 -0
- annoscript/test_diagnostics.py +97 -0
- annoscript/test_engineering_p0.py +191 -0
- annoscript/test_environment.py +65 -0
- annoscript/test_example_backends.py +251 -0
- annoscript/test_examples.py +149 -0
- annoscript/test_field_names.py +221 -0
- annoscript/test_formatter.py +153 -0
- annoscript/test_fusion_scheduling.py +109 -0
- annoscript/test_generation_policy.py +180 -0
- annoscript/test_ir.py +148 -0
- annoscript/test_jobs.py +371 -0
- annoscript/test_jobs_server.py +511 -0
- annoscript/test_llm_batch.py +428 -0
- annoscript/test_metrics.py +59 -0
- annoscript/test_metrics_server.py +261 -0
- annoscript/test_minimal_fusion.py +59 -0
- annoscript/test_nullable_generate.py +169 -0
- annoscript/test_passes.py +408 -0
- annoscript/test_plan.py +522 -0
- annoscript/test_plan_constraints.py +305 -0
- annoscript/test_plan_formatter.py +272 -0
- annoscript/test_prompt_layout.py +140 -0
- annoscript/test_qwen_backend.py +181 -0
- annoscript/test_qwen_pilot_regression.py +180 -0
- annoscript/test_release_client.py +20 -0
- annoscript/test_remote.py +308 -0
- annoscript/test_request_adapters.py +295 -0
- annoscript/test_requirements.py +228 -0
- annoscript/test_rule_dedup.py +67 -0
- annoscript/test_runtime.py +314 -0
- annoscript/test_runtime_client.py +133 -0
- annoscript/test_source_fields.py +111 -0
- annoscript/test_speculation.py +458 -0
- annoscript/test_substring_backend.py +81 -0
- annoscript/test_synthetic_example.py +61 -0
- annoscript-0.1.0.dist-info/METADATA +145 -0
- annoscript-0.1.0.dist-info/RECORD +128 -0
- annoscript-0.1.0.dist-info/WHEEL +4 -0
- annoscript-0.1.0.dist-info/licenses/LICENSE +21 -0
- annoscript_examples/__init__.py +1 -0
- annoscript_examples/fixtures.py +3 -0
- annoscript_examples/request.execution_plan.json +3017 -0
- annoscript_examples/request.json +858 -0
- annoscript_examples/request.py +37 -0
- annoscript_examples/request_preview.decision.json +23 -0
- annoscript_examples/request_preview.generation.json +20 -0
- annoscript_examples/run.py +24 -0
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Pinned decoder acceptance tests (run with XGrammar substring2, no inference)."""
|
|
2
|
+
import itertools
|
|
3
|
+
import json
|
|
4
|
+
import random
|
|
5
|
+
import unittest
|
|
6
|
+
from importlib.metadata import version
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
import xgrammar as xgr
|
|
9
|
+
from annoscript.backends._bounded_grammar import grammar
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def compiled(text):
|
|
13
|
+
return xgr.GrammarCompiler(xgr.TokenizerInfo([]),cache_enabled=False).compile_grammar(text)
|
|
14
|
+
|
|
15
|
+
def accepts(ctx,text):
|
|
16
|
+
m=xgr.GrammarMatcher(ctx,terminate_without_stop_token=True)
|
|
17
|
+
return m.accept_string(text) and m.is_terminated()
|
|
18
|
+
|
|
19
|
+
def wire(items):return json.dumps({'items':items},ensure_ascii=False,separators=(',',':'))
|
|
20
|
+
|
|
21
|
+
def call(op='extract',id='a',**args):
|
|
22
|
+
source = args.pop('source', None)
|
|
23
|
+
return SimpleNamespace(id=id,operation=op,arguments=args,extraction_source=source)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class BoundedGrammarTests(unittest.TestCase):
|
|
27
|
+
@classmethod
|
|
28
|
+
def setUpClass(cls):assert version('xgrammar')=='0.2.5+lev.substring2.3'
|
|
29
|
+
|
|
30
|
+
def test_example_locations_never_expand_current_source(self):
|
|
31
|
+
from annoscript._test_synthetic import REVIEW_INPUT, LAUNCH_INPUT
|
|
32
|
+
from annoscript.anno import Example
|
|
33
|
+
from annoscript.backends.base import normalize_call, InferenceContext
|
|
34
|
+
c = normalize_call("q", "extract", ("地点",),
|
|
35
|
+
{"source_field": "text", "max_items": 2, "max_length_per_item": 80,
|
|
36
|
+
"examples": [Example(state=REVIEW_INPUT, answer=[]),
|
|
37
|
+
Example(state=LAUNCH_INPUT, answer=["广州"])]},
|
|
38
|
+
InferenceContext(REVIEW_INPUT))
|
|
39
|
+
g = compiled(grammar([c], extract_mode="substring_chars"))
|
|
40
|
+
self.assertTrue(accepts(g, wire([])))
|
|
41
|
+
self.assertTrue(accepts(g, wire(["条目"]))) # Grammar only guarantees source, not semantics.
|
|
42
|
+
self.assertFalse(accepts(g, wire(["广州"])))
|
|
43
|
+
|
|
44
|
+
def test_macro_language_consumed_length_not_sam_longest(self):
|
|
45
|
+
rng=random.Random(20261018)
|
|
46
|
+
sources=['abbb','ababa','abcab','aaaa','', 'a']+[''.join(rng.choices('abc',k=rng.randint(1,7))) for _ in range(16)]
|
|
47
|
+
for source in sources:
|
|
48
|
+
for limit in (0,1,2,4,10):
|
|
49
|
+
for empty in (False,True):
|
|
50
|
+
args=', '.join(json.dumps(c) for c in source)
|
|
51
|
+
g=xgr.Grammar.from_ebnf('root ::= Substring('+args+(', ' if args else '')+f'allow_empty={str(empty).lower()}, max_chunks={limit})')
|
|
52
|
+
copies=[g,xgr.Grammar.from_ebnf(str(g)),xgr.Grammar.deserialize_json(g.serialize_json())]
|
|
53
|
+
expected={source[i:j] for i in range(len(source)) for j in range(i+1,min(len(source),i+limit)+1)}
|
|
54
|
+
if empty:expected.add('')
|
|
55
|
+
for clone in copies:
|
|
56
|
+
c=compiled(clone)
|
|
57
|
+
for n in range(4):
|
|
58
|
+
for parts in itertools.product('abc',repeat=n):
|
|
59
|
+
text=''.join(parts)
|
|
60
|
+
self.assertEqual(accepts(c,text),text in expected,(source,limit,empty,text))
|
|
61
|
+
for bad in ['-1','1.2','true','"2"','2147483648']:
|
|
62
|
+
with self.assertRaises(Exception):xgr.Grammar.from_ebnf('root ::= Substring("a", max_chunks='+bad+')')
|
|
63
|
+
|
|
64
|
+
def test_empty_chunks_consume_a_block_in_nullable_mode(self):
|
|
65
|
+
for empty in (True,False):
|
|
66
|
+
g=compiled('root ::= Substring("a", "", "b", allow_empty='+str(empty).lower()+', max_chunks=2)')
|
|
67
|
+
self.assertEqual(accepts(g,'ab'),not empty)
|
|
68
|
+
self.assertEqual(accepts(g,''),empty)
|
|
69
|
+
self.assertTrue(accepts(g,'a'));self.assertTrue(accepts(g,'b'))
|
|
70
|
+
|
|
71
|
+
def test_extract_json_unicode_count_and_boundaries(self):
|
|
72
|
+
for source in ['甲"乙\\丙\n😀','e\u0301/@ @','\x00\x1f\b\f\r\t','\\u4e2d','aaaa','']:
|
|
73
|
+
for maximum in (1,2,4):
|
|
74
|
+
c=compiled(grammar([call(source=source,min_items=0,max_items=2,max_length_per_item=maximum)],extract_mode='substring_chars'))
|
|
75
|
+
self.assertTrue(accepts(c,wire([])))
|
|
76
|
+
self.assertFalse(accepts(c,wire([''])))
|
|
77
|
+
for i in range(len(source)):
|
|
78
|
+
for j in range(i+1,len(source)+1):
|
|
79
|
+
self.assertEqual(accepts(c,wire([source[i:j]])),True,(source,i,j))
|
|
80
|
+
if source:
|
|
81
|
+
self.assertFalse(accepts(c,wire([source[0]]*3)))
|
|
82
|
+
for text in ['不存在','甲丙',source[::-1]]:
|
|
83
|
+
if text not in source:self.assertFalse(accepts(c,wire([text])))
|
|
84
|
+
for bad in [r'{"items":["\u00"]}',r'{"items":["\"]}','{"items":["\n"]}']:
|
|
85
|
+
self.assertFalse(accepts(c,bad))
|
|
86
|
+
|
|
87
|
+
def test_list_and_raw_generate_codepoints(self):
|
|
88
|
+
for lo,hi in ((0,0),(0,1),(1,1),(1,2),(2,3)):
|
|
89
|
+
g=grammar([call('generate_list',min_items=1,max_items=2,min_length_per_item=lo,max_length_per_item=hi)])
|
|
90
|
+
c=compiled(g)
|
|
91
|
+
for text in ('','a','甲','😀','e\u0301','"','\\','\n','甲😀中'):
|
|
92
|
+
for escaped in (False,True):
|
|
93
|
+
value=json.dumps({'items':[text]},ensure_ascii=escaped,separators=(',',':'))
|
|
94
|
+
self.assertEqual(accepts(c,value),bool(text) or lo==0,(lo,hi,value))
|
|
95
|
+
self.assertFalse(accepts(c,wire([])))
|
|
96
|
+
self.assertFalse(accepts(c,wire(['a','a','a'])))
|
|
97
|
+
raw=compiled(grammar([call('generate',min_length=lo,max_length=hi)]))
|
|
98
|
+
for text in ('','a','甲','😀','e\u0301','\n','甲😀中'):
|
|
99
|
+
self.assertEqual(accepts(raw,text),bool(text) or lo==0,(lo,hi,text))
|
|
100
|
+
zero=compiled(grammar([call('generate_list',max_items=0)]))
|
|
101
|
+
self.assertTrue(accepts(zero,wire([])));self.assertFalse(accepts(zero,wire([''])))
|
|
102
|
+
|
|
103
|
+
def test_mixed_same_source_different_limits_and_isolation(self):
|
|
104
|
+
cs=[call(id='a',source='甲乙丙',max_items=1,max_length_per_item=1),
|
|
105
|
+
call(id='b',source='甲乙丙',min_items=1,max_items=2,max_length_per_item=2),
|
|
106
|
+
call(id='c',source='丁戊',max_items=1,max_length_per_item=1),
|
|
107
|
+
call('generate','d',max_length=2),call('generate_list','e',max_items=1,max_length_per_item=1)]
|
|
108
|
+
g=grammar(cs,batch=True,extract_mode='substring_chars');self.assertEqual(g.count('Substring('),3)
|
|
109
|
+
c=compiled(g)
|
|
110
|
+
def wire_batch(a='甲',b='甲乙',cc='丁',d='😀',e=None):
|
|
111
|
+
return json.dumps({'answers':{'q1':[a],'q2':[b],'q3':[cc],'q4':d,'q5':e or ['甲']}},ensure_ascii=False,separators=(',',':'))
|
|
112
|
+
self.assertTrue(accepts(c,wire_batch()))
|
|
113
|
+
for kwargs in ({'a':'丁'},{'cc':'甲'},{'e':['甲','乙']},{'b':''}):
|
|
114
|
+
self.assertFalse(accepts(c,wire_batch(**kwargs)),kwargs)
|
|
115
|
+
|
|
116
|
+
def test_named_keys_preserve_source_and_value_constraints(self):
|
|
117
|
+
name = ' 地点"\\\n😀 '
|
|
118
|
+
cs = [call(id='internal-a', field_name=name, source='广州', min_items=1, max_items=1),
|
|
119
|
+
call('generate', 'internal-b', field_name='event_title', min_length=1, max_length=60)]
|
|
120
|
+
g = compiled(grammar(cs, batch=True, extract_mode='substring_chars'))
|
|
121
|
+
def encoded(place='广州', title='上市'):
|
|
122
|
+
# Key literals use ASCII escapes; substring values retain source bytes.
|
|
123
|
+
return ('{"answers":{' + json.dumps(name) + ':' + json.dumps([place], ensure_ascii=False)
|
|
124
|
+
+ ',"event_title":' + json.dumps(title, ensure_ascii=False) + '}}')
|
|
125
|
+
good = encoded()
|
|
126
|
+
self.assertTrue(accepts(g, good))
|
|
127
|
+
self.assertFalse(accepts(g, encoded('北京')))
|
|
128
|
+
self.assertFalse(accepts(g, encoded(title='')))
|
|
129
|
+
self.assertFalse(accepts(g, good.replace('event_title', 'q2')))
|
|
130
|
+
|
|
131
|
+
def test_nullable_generate_single_and_mixed_rule_cache_isolation(self):
|
|
132
|
+
optional = call('generate', 'a', min_length=1, max_length=24, nullable=True)
|
|
133
|
+
ordinary = call('generate', 'b', min_length=1, max_length=24)
|
|
134
|
+
single = compiled(grammar([optional]))
|
|
135
|
+
for raw in ('null', '"null"', '"中文"', '"\\\""'):
|
|
136
|
+
self.assertTrue(accepts(single, raw), raw)
|
|
137
|
+
for raw in ('[]', '1', 'true', '{}', '""', '', '未加引号'):
|
|
138
|
+
self.assertFalse(accepts(single, raw), raw)
|
|
139
|
+
for calls in ([optional, ordinary], [ordinary, optional]):
|
|
140
|
+
mixed = compiled(grammar(calls, batch=True))
|
|
141
|
+
for values in ((None, '标题'), ('null', '标题'), ('标题', None), ('标题', ''), ('标题', '标题')):
|
|
142
|
+
expected = all((v is None and c.arguments.get('nullable', False)) or
|
|
143
|
+
(isinstance(v, str) and bool(v)) for c, v in zip(calls, values))
|
|
144
|
+
raw = json.dumps({'answers': dict(zip(('q1', 'q2'), values))}, ensure_ascii=False, separators=(',', ':'))
|
|
145
|
+
self.assertEqual(accepts(mixed, raw), expected, (calls, values))
|
|
146
|
+
|
|
147
|
+
if __name__=='__main__':unittest.main()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Conservative, deterministic optimization of structured AnnoScript IR.
|
|
2
|
+
|
|
3
|
+
The input is never mutated and no source, model, or user function is executed.
|
|
4
|
+
Passes preserve ordinary execution and constraint bindings; they neither solve
|
|
5
|
+
constraints nor treat an asserted condition as a fact. Unknown effects remain.
|
|
6
|
+
Individual passes are implementation details; optimize_program is the public API.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from annoscript.ir import Program
|
|
10
|
+
from .control_flow import simplify_control_flow
|
|
11
|
+
from .dead_assignments import eliminate_dead_assignments
|
|
12
|
+
from .folding import fold_constants
|
|
13
|
+
from .propagation import propagate_values
|
|
14
|
+
from .temporaries import merge_temporaries
|
|
15
|
+
|
|
16
|
+
__all__ = ["optimize_program"]
|
|
17
|
+
|
|
18
|
+
_PASSES = (
|
|
19
|
+
propagate_values, fold_constants, simplify_control_flow,
|
|
20
|
+
eliminate_dead_assignments, merge_temporaries,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def optimize_program(program: Program) -> Program:
|
|
25
|
+
"""Optimize a Program to a fixed point without changing the IR v1 schema.
|
|
26
|
+
|
|
27
|
+
The passes only propagate atoms, fold bounded scalars, or remove structure;
|
|
28
|
+
they do not expand loops, duplicate statements, or introduce new variables.
|
|
29
|
+
JSON equality is intentional: dataclass equality conflates True, 1 and 1.0.
|
|
30
|
+
"""
|
|
31
|
+
if not isinstance(program, Program):
|
|
32
|
+
raise TypeError("optimize_program expects an ir.Program")
|
|
33
|
+
previous = program.to_json(indent=None)
|
|
34
|
+
while True:
|
|
35
|
+
for transform in _PASSES:
|
|
36
|
+
program = transform(program)
|
|
37
|
+
current = program.to_json(indent=None)
|
|
38
|
+
if current == previous:
|
|
39
|
+
return program
|
|
40
|
+
previous = current
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Shared structural analyses; none of these helpers evaluate user code.
|
|
2
|
+
|
|
3
|
+
Pure IR expressions may still raise, read mutable data, or allocate objects.
|
|
4
|
+
Safety below concerns supported IR values, excluding resource exhaustion.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from dataclasses import fields, replace
|
|
9
|
+
|
|
10
|
+
from annoscript import ir
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def map_value(value, transform):
|
|
14
|
+
if isinstance(value, ir.Expr):
|
|
15
|
+
return transform(value)
|
|
16
|
+
if isinstance(value, tuple):
|
|
17
|
+
return tuple(map_value(item, transform) for item in value)
|
|
18
|
+
return value
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def rewrite_expr(expr, transform):
|
|
22
|
+
children = {f.name: map_value(getattr(expr, f.name), lambda e: rewrite_expr(e, transform))
|
|
23
|
+
for f in fields(expr)}
|
|
24
|
+
return transform(replace(expr, **children))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def map_expressions(statement, transform):
|
|
28
|
+
"""Map immediate expression fields, never bodies or constraint expressions."""
|
|
29
|
+
if isinstance(statement, ir.Constrain):
|
|
30
|
+
return statement
|
|
31
|
+
return replace(statement, **{
|
|
32
|
+
f.name: map_value(getattr(statement, f.name), transform) for f in fields(statement)
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def expression_reads(expr):
|
|
37
|
+
result = Counter()
|
|
38
|
+
|
|
39
|
+
def visit(node):
|
|
40
|
+
if isinstance(node, ir.Var):
|
|
41
|
+
result[node.name] += 1
|
|
42
|
+
return node
|
|
43
|
+
|
|
44
|
+
rewrite_expr(expr, visit)
|
|
45
|
+
return result
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def direct_reads(statement):
|
|
49
|
+
result = Counter()
|
|
50
|
+
|
|
51
|
+
def visit(expr):
|
|
52
|
+
result.update(expression_reads(expr))
|
|
53
|
+
return expr
|
|
54
|
+
|
|
55
|
+
# Constraints participate in liveness even though transformations skip them.
|
|
56
|
+
for f in fields(statement):
|
|
57
|
+
map_value(getattr(statement, f.name), visit)
|
|
58
|
+
return result
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def target_names(target):
|
|
62
|
+
if isinstance(target, ir.NameTarget):
|
|
63
|
+
return {target.name}
|
|
64
|
+
if isinstance(target, ir.TupleTarget):
|
|
65
|
+
return set().union(*(target_names(item) for item in target.items))
|
|
66
|
+
raise TypeError(f"Unknown target: {type(target).__name__}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def definitions(statement):
|
|
70
|
+
if isinstance(statement, (ir.Assign, ir.Infer)):
|
|
71
|
+
return {statement.target}
|
|
72
|
+
if isinstance(statement, (ir.Unpack, ir.For)):
|
|
73
|
+
return target_names(statement.target)
|
|
74
|
+
return set()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def child_blocks(statement):
|
|
78
|
+
if isinstance(statement, ir.If):
|
|
79
|
+
return (statement.then_body, statement.else_body)
|
|
80
|
+
if isinstance(statement, ir.For):
|
|
81
|
+
return (statement.body,)
|
|
82
|
+
return ()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def walk_statements(body):
|
|
86
|
+
for statement in body:
|
|
87
|
+
yield statement
|
|
88
|
+
for block in child_blocks(statement):
|
|
89
|
+
yield from walk_statements(block)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def block_writes(body):
|
|
93
|
+
return set().union(*(definitions(s) for s in walk_statements(body)))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def literal_key(value):
|
|
97
|
+
# Python equality conflates bool/int/float and the signs of floating zero.
|
|
98
|
+
return (type(value), value.hex() if type(value) is float else value)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def atom_key(expr):
|
|
102
|
+
if isinstance(expr, ir.Literal):
|
|
103
|
+
return (ir.Literal, literal_key(expr.value))
|
|
104
|
+
if isinstance(expr, ir.Var):
|
|
105
|
+
return (ir.Var, expr.name)
|
|
106
|
+
raise TypeError("Expected a literal or variable")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def resolve(expr, facts):
|
|
110
|
+
seen = set()
|
|
111
|
+
while isinstance(expr, ir.Var) and expr.name in facts and expr.name not in seen:
|
|
112
|
+
seen.add(expr.name)
|
|
113
|
+
expr = facts[expr.name]
|
|
114
|
+
return expr
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def invalidate(facts, names):
|
|
118
|
+
"""Forget copies of old bindings as well as the overwritten bindings."""
|
|
119
|
+
resolved = {name: resolve(value, facts) for name, value in facts.items()}
|
|
120
|
+
return {name: value for name, value in resolved.items()
|
|
121
|
+
if name not in names and not (isinstance(value, ir.Var) and value.name in names)}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def join_facts(left, right):
|
|
125
|
+
return {name: value for name, value in left.items()
|
|
126
|
+
if name in right and atom_key(value) == atom_key(right[name])}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def join_exits(left, left_falls, right, right_falls):
|
|
130
|
+
if not left_falls:
|
|
131
|
+
return right if right_falls else {}
|
|
132
|
+
if not right_falls:
|
|
133
|
+
return left
|
|
134
|
+
return join_facts(left, right)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def bound_before(program):
|
|
138
|
+
"""Definitely bound names at each statement, including loop body entry."""
|
|
139
|
+
points = {}
|
|
140
|
+
|
|
141
|
+
def block(body, incoming):
|
|
142
|
+
bound = set(incoming)
|
|
143
|
+
falls = True
|
|
144
|
+
for statement in body:
|
|
145
|
+
points[statement.id] = frozenset(bound)
|
|
146
|
+
if isinstance(statement, ir.If):
|
|
147
|
+
a, af = block(statement.then_body, bound)
|
|
148
|
+
b, bf = block(statement.else_body, bound)
|
|
149
|
+
bound = a & b if af and bf else a if af else b if bf else set()
|
|
150
|
+
falls = af or bf
|
|
151
|
+
elif isinstance(statement, ir.For):
|
|
152
|
+
block(statement.body, bound | definitions(statement))
|
|
153
|
+
# The loop can execute zero times.
|
|
154
|
+
elif isinstance(statement, ir.Return):
|
|
155
|
+
falls = False
|
|
156
|
+
else:
|
|
157
|
+
bound |= definitions(statement)
|
|
158
|
+
if not falls:
|
|
159
|
+
break
|
|
160
|
+
return bound, falls
|
|
161
|
+
|
|
162
|
+
block(program.body, set())
|
|
163
|
+
return points
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def discardable(expr, bound):
|
|
167
|
+
"""Prove that evaluating and discarding an expression cannot fail or act."""
|
|
168
|
+
if isinstance(expr, (ir.Literal, ir.State)):
|
|
169
|
+
return True
|
|
170
|
+
if isinstance(expr, ir.Var):
|
|
171
|
+
return expr.name in bound
|
|
172
|
+
if isinstance(expr, (ir.ListExpr, ir.TupleExpr)):
|
|
173
|
+
return all(discardable(item, bound) for item in expr.items)
|
|
174
|
+
if isinstance(expr, ir.DictExpr):
|
|
175
|
+
# Only literal keys are known to be hashable without a type analysis.
|
|
176
|
+
return all(isinstance(key, ir.Literal) and discardable(value, bound)
|
|
177
|
+
for key, value in expr.entries)
|
|
178
|
+
return False
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Remove proven dead paths and redundant boolean completion guards."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
|
|
5
|
+
from annoscript import ir
|
|
6
|
+
from ._analysis import block_writes, definitions, invalidate, join_exits, resolve
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _completion_pair(first, second, facts):
|
|
10
|
+
if not (isinstance(first, ir.If) and not first.else_body and first.then_body
|
|
11
|
+
and isinstance(second, ir.If) and not second.else_body):
|
|
12
|
+
return False
|
|
13
|
+
guard = second.condition
|
|
14
|
+
if not (isinstance(guard, ir.Unary) and guard.op == "not" and isinstance(guard.value, ir.Var)):
|
|
15
|
+
return False
|
|
16
|
+
name = guard.value.name
|
|
17
|
+
initial = resolve(ir.Var(name), facts)
|
|
18
|
+
last = first.then_body[-1]
|
|
19
|
+
return (isinstance(initial, ir.Literal) and initial.value is False
|
|
20
|
+
and isinstance(last, ir.Assign) and last.target == name
|
|
21
|
+
and isinstance(last.value, ir.Literal) and last.value.value is True)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def simplify_control_flow(program: ir.Program) -> ir.Program:
|
|
25
|
+
def block(body, incoming):
|
|
26
|
+
facts = dict(incoming)
|
|
27
|
+
output = []
|
|
28
|
+
index = 0
|
|
29
|
+
falls = True
|
|
30
|
+
while index < len(body):
|
|
31
|
+
statement = body[index]
|
|
32
|
+
if index + 1 < len(body) and _completion_pair(statement, body[index + 1], facts):
|
|
33
|
+
statement = replace(statement, else_body=body[index + 1].then_body)
|
|
34
|
+
index += 1
|
|
35
|
+
index += 1
|
|
36
|
+
if isinstance(statement, ir.If):
|
|
37
|
+
condition = resolve(statement.condition, facts)
|
|
38
|
+
if isinstance(condition, ir.Literal):
|
|
39
|
+
selected = statement.then_body if condition.value else statement.else_body
|
|
40
|
+
nested, facts, falls = block(selected, facts)
|
|
41
|
+
output.extend(nested)
|
|
42
|
+
else:
|
|
43
|
+
yes, a, af = block(statement.then_body, facts)
|
|
44
|
+
no, b, bf = block(statement.else_body, facts)
|
|
45
|
+
output.append(replace(statement, then_body=yes, else_body=no))
|
|
46
|
+
facts = join_exits(a, af, b, bf)
|
|
47
|
+
falls = af or bf
|
|
48
|
+
elif isinstance(statement, ir.For):
|
|
49
|
+
facts = invalidate(facts, definitions(statement) | block_writes(statement.body))
|
|
50
|
+
nested, _, _ = block(statement.body, facts)
|
|
51
|
+
output.append(replace(statement, body=nested))
|
|
52
|
+
else:
|
|
53
|
+
output.append(statement)
|
|
54
|
+
if isinstance(statement, ir.Assign):
|
|
55
|
+
value = resolve(statement.value, facts)
|
|
56
|
+
facts = invalidate(facts, {statement.target})
|
|
57
|
+
if isinstance(value, ir.Literal):
|
|
58
|
+
facts[statement.target] = value
|
|
59
|
+
elif isinstance(statement, ir.Constrain):
|
|
60
|
+
facts.clear()
|
|
61
|
+
else:
|
|
62
|
+
facts = invalidate(facts, definitions(statement))
|
|
63
|
+
falls = not isinstance(statement, ir.Return)
|
|
64
|
+
if not falls:
|
|
65
|
+
break
|
|
66
|
+
return tuple(output), facts, falls
|
|
67
|
+
|
|
68
|
+
return replace(program, body=block(program.body, {})[0])
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Structured backward liveness, preserving evaluations that may fail or act."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
|
|
5
|
+
from annoscript import ir
|
|
6
|
+
from ._analysis import bound_before, definitions, direct_reads, discardable
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def eliminate_dead_assignments(program: ir.Program) -> ir.Program:
|
|
10
|
+
points = bound_before(program)
|
|
11
|
+
|
|
12
|
+
def block(body, outgoing):
|
|
13
|
+
live = set(outgoing)
|
|
14
|
+
output = []
|
|
15
|
+
for statement in reversed(body):
|
|
16
|
+
reads = set(direct_reads(statement))
|
|
17
|
+
if isinstance(statement, ir.Assign):
|
|
18
|
+
if statement.target not in live and discardable(statement.value, points.get(statement.id, ())):
|
|
19
|
+
continue
|
|
20
|
+
live = (live - {statement.target}) | reads
|
|
21
|
+
elif isinstance(statement, ir.If):
|
|
22
|
+
yes, a = block(statement.then_body, live)
|
|
23
|
+
no, b = block(statement.else_body, live)
|
|
24
|
+
statement = replace(statement, then_body=yes, else_body=no)
|
|
25
|
+
live = a | b | reads
|
|
26
|
+
elif isinstance(statement, ir.For):
|
|
27
|
+
# A back edge reaches another target binding or exits the loop.
|
|
28
|
+
head = set(live)
|
|
29
|
+
while True:
|
|
30
|
+
nested, before_body = block(statement.body, head)
|
|
31
|
+
updated = live | (before_body - definitions(statement))
|
|
32
|
+
if updated == head:
|
|
33
|
+
break
|
|
34
|
+
head = updated
|
|
35
|
+
statement = replace(statement, body=nested)
|
|
36
|
+
live = head | reads
|
|
37
|
+
elif isinstance(statement, ir.Return):
|
|
38
|
+
live = reads
|
|
39
|
+
else:
|
|
40
|
+
live = (live - definitions(statement)) | reads
|
|
41
|
+
output.append(statement)
|
|
42
|
+
return tuple(reversed(output)), live
|
|
43
|
+
|
|
44
|
+
return replace(program, body=block(program.body, set())[0])
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Bounded folding of a closed set of scalar operations, never Python source."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
import math
|
|
5
|
+
import operator
|
|
6
|
+
|
|
7
|
+
from annoscript import ir
|
|
8
|
+
from ._analysis import map_expressions, rewrite_expr
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
_COMPARE = {
|
|
12
|
+
"eq": operator.eq, "ne": operator.ne, "lt": operator.lt, "le": operator.le,
|
|
13
|
+
"gt": operator.gt, "ge": operator.ge, "in": lambda a, b: a in b,
|
|
14
|
+
"not_in": lambda a, b: a not in b, "is": operator.is_, "is_not": operator.is_not,
|
|
15
|
+
}
|
|
16
|
+
_MAX_STRING = 65536
|
|
17
|
+
_MAX_BITS = 4096
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _bounded(value):
|
|
21
|
+
if type(value) is str:
|
|
22
|
+
return len(value) <= _MAX_STRING
|
|
23
|
+
if type(value) is int:
|
|
24
|
+
return value.bit_length() <= _MAX_BITS
|
|
25
|
+
if type(value) is float:
|
|
26
|
+
return math.isfinite(value)
|
|
27
|
+
return value is None or type(value) is bool
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _fold(expr):
|
|
31
|
+
if isinstance(expr, ir.BoolOp):
|
|
32
|
+
values = list(expr.values)
|
|
33
|
+
while len(values) > 1 and isinstance(values[0], ir.Literal):
|
|
34
|
+
first = values.pop(0)
|
|
35
|
+
if (expr.op == "and" and not first.value) or (expr.op == "or" and first.value):
|
|
36
|
+
return first
|
|
37
|
+
return values[0] if len(values) == 1 else replace(expr, values=tuple(values))
|
|
38
|
+
|
|
39
|
+
def literal(node):
|
|
40
|
+
if not isinstance(node, ir.Literal) or not _bounded(node.value):
|
|
41
|
+
raise _NotConstant
|
|
42
|
+
return node.value
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
if isinstance(expr, ir.Unary):
|
|
46
|
+
value = {"not": operator.not_, "positive": operator.pos, "negative": operator.neg}[expr.op](literal(expr.value))
|
|
47
|
+
elif isinstance(expr, ir.Binary):
|
|
48
|
+
a, b = literal(expr.left), literal(expr.right)
|
|
49
|
+
if isinstance(a, str) and isinstance(b, str) and len(a) + len(b) > _MAX_STRING:
|
|
50
|
+
return expr
|
|
51
|
+
value = operator.add(a, b)
|
|
52
|
+
elif isinstance(expr, ir.Compare):
|
|
53
|
+
left = literal(expr.left)
|
|
54
|
+
value = True
|
|
55
|
+
for op, item in zip(expr.ops, expr.comparators):
|
|
56
|
+
right = literal(item)
|
|
57
|
+
# Identity is meaningful only for the IR's singleton values.
|
|
58
|
+
if op in ("is", "is_not") and not any(v is None or type(v) is bool for v in (left, right)):
|
|
59
|
+
return expr
|
|
60
|
+
if not _COMPARE[op](left, right):
|
|
61
|
+
value = False
|
|
62
|
+
break
|
|
63
|
+
left = right
|
|
64
|
+
elif isinstance(expr, ir.FormatValue):
|
|
65
|
+
if expr.spec is not None and literal(expr.spec) != "":
|
|
66
|
+
return expr
|
|
67
|
+
value = literal(expr.value)
|
|
68
|
+
if expr.conversion != "none":
|
|
69
|
+
value = {"s": str, "r": repr, "a": ascii}[expr.conversion](value)
|
|
70
|
+
value = format(value, "")
|
|
71
|
+
elif isinstance(expr, ir.FormatString):
|
|
72
|
+
parts = [literal(part) for part in expr.parts]
|
|
73
|
+
if not all(type(part) is str for part in parts) or sum(map(len, parts)) > _MAX_STRING:
|
|
74
|
+
return expr
|
|
75
|
+
value = "".join(parts)
|
|
76
|
+
elif isinstance(expr, ir.Intrinsic) and expr.name in ("len", "int", "str.strip"):
|
|
77
|
+
args = [literal(arg) for arg in expr.args]
|
|
78
|
+
kwargs = {name: literal(arg) for name, arg in expr.kwargs}
|
|
79
|
+
value = {"len": len, "int": int, "str.strip": str.strip}[expr.name](*args, **kwargs)
|
|
80
|
+
else:
|
|
81
|
+
return expr
|
|
82
|
+
return ir.Literal(value) if _bounded(value) else expr
|
|
83
|
+
except (_NotConstant, TypeError, ValueError, OverflowError):
|
|
84
|
+
# A failing expression must still fail at its original execution point.
|
|
85
|
+
return expr
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class _NotConstant(Exception):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def fold_constants(program: ir.Program) -> ir.Program:
|
|
93
|
+
def block(body):
|
|
94
|
+
output = []
|
|
95
|
+
for statement in body:
|
|
96
|
+
statement = map_expressions(statement, lambda e: rewrite_expr(e, _fold))
|
|
97
|
+
if isinstance(statement, ir.If):
|
|
98
|
+
statement = replace(statement, then_body=block(statement.then_body), else_body=block(statement.else_body))
|
|
99
|
+
elif isinstance(statement, ir.For):
|
|
100
|
+
statement = replace(statement, body=block(statement.body))
|
|
101
|
+
output.append(statement)
|
|
102
|
+
return tuple(output)
|
|
103
|
+
|
|
104
|
+
return replace(program, body=block(program.body))
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Forward propagation of scalar constants and copies of existing values."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import replace
|
|
4
|
+
|
|
5
|
+
from annoscript import ir
|
|
6
|
+
from ._analysis import (
|
|
7
|
+
block_writes, bound_before, definitions, invalidate, join_exits,
|
|
8
|
+
map_expressions, resolve, rewrite_expr,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def propagate_values(program: ir.Program) -> ir.Program:
|
|
13
|
+
points = bound_before(program)
|
|
14
|
+
|
|
15
|
+
def block(body, incoming):
|
|
16
|
+
facts = dict(incoming)
|
|
17
|
+
output = []
|
|
18
|
+
falls = True
|
|
19
|
+
for original in body:
|
|
20
|
+
statement = map_expressions(original, lambda e: rewrite_expr(e, lambda v: resolve(v, facts)))
|
|
21
|
+
if isinstance(statement, ir.If):
|
|
22
|
+
yes, a, af = block(statement.then_body, facts)
|
|
23
|
+
no, b, bf = block(statement.else_body, facts)
|
|
24
|
+
facts = join_exits(a, af, b, bf)
|
|
25
|
+
statement = replace(statement, then_body=yes, else_body=no)
|
|
26
|
+
falls = af or bf
|
|
27
|
+
elif isinstance(statement, ir.For):
|
|
28
|
+
writes = definitions(statement) | block_writes(statement.body)
|
|
29
|
+
facts = invalidate(facts, writes)
|
|
30
|
+
nested, _, _ = block(statement.body, facts)
|
|
31
|
+
statement = replace(statement, body=nested)
|
|
32
|
+
elif isinstance(statement, ir.Assign):
|
|
33
|
+
value = statement.value
|
|
34
|
+
if not (isinstance(value, ir.Var) and value.name == statement.target):
|
|
35
|
+
facts = invalidate(facts, {statement.target})
|
|
36
|
+
if isinstance(value, ir.Literal) or (
|
|
37
|
+
isinstance(value, ir.Var) and value.name in points.get(statement.id, ())
|
|
38
|
+
):
|
|
39
|
+
facts[statement.target] = value
|
|
40
|
+
elif isinstance(statement, ir.Constrain):
|
|
41
|
+
facts.clear()
|
|
42
|
+
elif isinstance(statement, ir.Return):
|
|
43
|
+
falls = False
|
|
44
|
+
else:
|
|
45
|
+
facts = invalidate(facts, definitions(statement))
|
|
46
|
+
output.append(statement)
|
|
47
|
+
if not falls:
|
|
48
|
+
# Unreachable code is removed by the control-flow pass.
|
|
49
|
+
output.extend(body[len(output):])
|
|
50
|
+
break
|
|
51
|
+
return tuple(output), facts, falls
|
|
52
|
+
|
|
53
|
+
return replace(program, body=block(program.body, {})[0])
|