architecture-extractor 0.27.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ae/__init__.py +3 -0
- ae/cli.py +90 -0
- ae/presenter/__init__.py +3 -0
- ae/presenter/api.py +157 -0
- ae/presenter/efficientnet_hierarchy.py +50 -0
- ae/presenter/flow.py +212 -0
- ae/presenter/flow_layout.py +171 -0
- ae/presenter/health.py +73 -0
- ae/presenter/instance_flow.py +135 -0
- ae/presenter/instance_hierarchy.py +188 -0
- ae/presenter/layout.py +155 -0
- ae/presenter/mobilenetv3_hierarchy.py +81 -0
- ae/presenter/projection.py +240 -0
- ae/presenter/reader.py +41 -0
- ae/presenter/render.py +185 -0
- ae/presenter/semantic.py +129 -0
- ae/presenter/themes/__init__.py +4 -0
- ae/presenter/themes/scientific_latex.py +49 -0
- ae/presenter/vgg16_hierarchy.py +53 -0
- ae/presenter/web_viewer.py +281 -0
- ae/sae/__init__.py +4 -0
- ae/sae/analysis/__init__.py +0 -0
- ae/sae/analysis/analyzer.py +366 -0
- ae/sae/analysis/composite_shapes.py +320 -0
- ae/sae/analysis/configuration.py +79 -0
- ae/sae/analysis/container_specialization.py +341 -0
- ae/sae/analysis/dataflow.py +258 -0
- ae/sae/analysis/dense_instance_forward.py +205 -0
- ae/sae/analysis/efficientnet_b0_instance_forward.py +276 -0
- ae/sae/analysis/factory_specialization.py +298 -0
- ae/sae/analysis/forward_flow.py +275 -0
- ae/sae/analysis/instance_forward.py +518 -0
- ae/sae/analysis/interprocedural.py +191 -0
- ae/sae/analysis/mobilenetv3_instance_forward.py +324 -0
- ae/sae/analysis/nested_shapes.py +87 -0
- ae/sae/analysis/pytorch_static.py +154 -0
- ae/sae/analysis/scopes.py +49 -0
- ae/sae/analysis/symbolic_shapes.py +152 -0
- ae/sae/analysis/unet_instance_forward.py +358 -0
- ae/sae/analysis/vgg16_instance_forward.py +168 -0
- ae/sae/analysis/vit_instance_forward.py +330 -0
- ae/sae/api.py +266 -0
- ae/sae/indexing/__init__.py +0 -0
- ae/sae/indexing/inherited_forward.py +127 -0
- ae/sae/indexing/model_index.py +370 -0
- ae/sae/indexing/nested_templates.py +103 -0
- ae/sae/inventory/__init__.py +0 -0
- ae/sae/inventory/walker.py +86 -0
- ae/sae/ir/__init__.py +0 -0
- ae/sae/ir/canonical.py +19 -0
- ae/sae/ir/models.py +34 -0
- ae/sae/ir/validator.py +63 -0
- ae/sae/linking/__init__.py +0 -0
- ae/sae/linking/project.py +380 -0
- ae/sae/output/__init__.py +0 -0
- ae/sae/output/bundle.py +21 -0
- ae/sae/parsing/__init__.py +0 -0
- ae/sae/parsing/python_parser.py +35 -0
- ae/sae/parsing/source_spans.py +54 -0
- ae/sae/settings.py +18 -0
- ae/verification/__init__.py +1 -0
- ae/verification/fx_correspondence.py +142 -0
- ae/verification/instance_correspondence.py +235 -0
- ae/verification/runtime_reference.py +186 -0
- architecture_extractor-0.27.0.dist-info/METADATA +87 -0
- architecture_extractor-0.27.0.dist-info/RECORD +70 -0
- architecture_extractor-0.27.0.dist-info/WHEEL +5 -0
- architecture_extractor-0.27.0.dist-info/entry_points.txt +2 -0
- architecture_extractor-0.27.0.dist-info/licenses/LICENSE +21 -0
- architecture_extractor-0.27.0.dist-info/top_level.txt +1 -0
ae/__init__.py
ADDED
ae/cli.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Command-line entry point for the static extractor."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from .sae.api import extract, inspect_bundle, validate_bundle
|
|
9
|
+
from .presenter import present
|
|
10
|
+
from .presenter.web_viewer import build_viewer
|
|
11
|
+
from .verification.runtime_reference import runtime_audit
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
15
|
+
parser = argparse.ArgumentParser(prog="ae", description="AE: static source-grounded model architecture extraction")
|
|
16
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
17
|
+
extraction = commands.add_parser("extract", help="Extract from source without importing or executing it")
|
|
18
|
+
extraction.add_argument("source", help="Source repository directory")
|
|
19
|
+
extraction.add_argument("--output", "-o", required=True, help="Output IR bundle directory")
|
|
20
|
+
extraction.add_argument("--entrypoint", help="Optional class/factory identifier (metadata hint, not executed)")
|
|
21
|
+
extraction.add_argument("--input-shape", help="Concrete input dimensions, e.g. 1,3,64,64")
|
|
22
|
+
extraction.add_argument("--attention-mode", choices=["fused", "unfused"],
|
|
23
|
+
help="Explicit ViT-Tiny attention implementation branch")
|
|
24
|
+
extraction.add_argument("--max-files", type=int, default=2000)
|
|
25
|
+
extraction.add_argument("--max-file-bytes", type=int, default=2_000_000)
|
|
26
|
+
for command in ("validate", "inspect"):
|
|
27
|
+
p = commands.add_parser(command)
|
|
28
|
+
p.add_argument("bundle", help="IR bundle directory or architecture.ir.json")
|
|
29
|
+
presenter = commands.add_parser("present", help="Draw a static architecture from a validated SAE bundle")
|
|
30
|
+
presenter.add_argument("bundle", help="Existing SAE extraction bundle directory")
|
|
31
|
+
presenter.add_argument("--output", "-o", required=True, help="Target diagram directory")
|
|
32
|
+
presenter.add_argument("--view", choices=["structure", "flow", "nested", "instance"], default="structure")
|
|
33
|
+
presenter.add_argument("--level", choices=["overview", "block", "operation"], default="block")
|
|
34
|
+
presenter.add_argument("--mode", choices=["exploration", "publication"], default="exploration")
|
|
35
|
+
presenter.add_argument("--format", choices=["svg", "pdf", "tikz", "all"], default="svg")
|
|
36
|
+
presenter.add_argument("--focus", help="Focus a module in flow/nested or a proven instance stage such as blocks.0")
|
|
37
|
+
presenter.add_argument("--theme", choices=["scientific-latex", "legacy"], default="scientific-latex")
|
|
38
|
+
presenter.add_argument("--allow-partial", action="store_true", help="Allow clearly labeled partial publication export")
|
|
39
|
+
viewer = commands.add_parser('viewer', help='Build a self-contained offline HTML viewer from a validated bundle')
|
|
40
|
+
viewer.add_argument('bundle', help='Existing SAE extraction bundle directory')
|
|
41
|
+
viewer.add_argument('--output', '-o', required=True, help='Output directory containing index.html')
|
|
42
|
+
viewer.add_argument('--view', choices=['auto','instance','nested','flow','structure'], default='auto')
|
|
43
|
+
viewer.add_argument('--title', default='Architecture Extractor')
|
|
44
|
+
runtime = commands.add_parser('runtime-audit', help='OPT-IN: execute trusted PyTorch model as independent FX/runtime reference')
|
|
45
|
+
runtime.add_argument('source', help='Trusted Python source directory. Executed: review first.')
|
|
46
|
+
runtime.add_argument('--factory', required=True, help='module:zero_arg_factory returning torch.nn.Module')
|
|
47
|
+
runtime.add_argument('--input-shape', required=True, action='append', help='Repeated positional synthetic input shape, e.g. 1,3,64,64')
|
|
48
|
+
runtime.add_argument('--input-dtype', action='append', help='Repeated input dtype: float32,int64,etc; defaults float32')
|
|
49
|
+
runtime.add_argument('--output', '-o', required=True, help='Reference JSON output directory')
|
|
50
|
+
runtime.add_argument('--allow-execute', action='store_true', help='Explicit approval to import/execute target model; NOT a sandbox')
|
|
51
|
+
runtime.add_argument('--max-nodes', type=int, default=20000)
|
|
52
|
+
return parser
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def main(argv: list[str] | None = None) -> int:
|
|
56
|
+
args = build_parser().parse_args(argv)
|
|
57
|
+
try:
|
|
58
|
+
if args.command == "extract":
|
|
59
|
+
result = extract(args.source, args.output, entrypoint=args.entrypoint,
|
|
60
|
+
max_files=args.max_files, max_file_bytes=args.max_file_bytes,
|
|
61
|
+
input_shape=tuple(map(int, args.input_shape.split(","))) if args.input_shape else None,
|
|
62
|
+
attention_mode=args.attention_mode)
|
|
63
|
+
elif args.command == "validate":
|
|
64
|
+
result = validate_bundle(args.bundle)
|
|
65
|
+
elif args.command == 'runtime-audit':
|
|
66
|
+
shapes = [tuple(int(x) for x in item.split(',')) for item in args.input_shape]
|
|
67
|
+
result = runtime_audit(source_root=args.source, factory=args.factory,
|
|
68
|
+
input_shapes=shapes, input_dtypes=args.input_dtype,
|
|
69
|
+
output=args.output, allow_execute=args.allow_execute,
|
|
70
|
+
max_nodes=args.max_nodes)
|
|
71
|
+
elif args.command == 'viewer':
|
|
72
|
+
result = build_viewer(args.bundle, args.output, view=args.view, title=args.title)
|
|
73
|
+
elif args.command == "present":
|
|
74
|
+
result = present(args.bundle, args.output, level=args.level, mode=args.mode,
|
|
75
|
+
format=args.format, allow_partial=args.allow_partial, view=args.view, focus=args.focus, theme=args.theme)
|
|
76
|
+
else:
|
|
77
|
+
result = inspect_bundle(args.bundle)
|
|
78
|
+
print(json.dumps(result, sort_keys=True, indent=2, ensure_ascii=False))
|
|
79
|
+
if args.command == 'viewer':
|
|
80
|
+
return 0 if result['rendered_levels'] else 2
|
|
81
|
+
if args.command == 'runtime-audit':
|
|
82
|
+
return 0 if result['status'] == 'traced' else 2
|
|
83
|
+
return 0 if args.command != "present" or result["render_allowed"] else 2
|
|
84
|
+
except (ValueError, OSError, json.JSONDecodeError) as exc:
|
|
85
|
+
print(f"ae: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
86
|
+
return 1
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
raise SystemExit(main())
|
ae/presenter/__init__.py
ADDED
ae/presenter/api.py
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""One-stage Presenter API: verified bundle -> provenance-backed scientific outputs."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from ae.sae.ir.canonical import canonical_bytes, sha256
|
|
9
|
+
from .reader import load_bundle
|
|
10
|
+
from .projection import project
|
|
11
|
+
from .flow import project_flow, project_nested_flow
|
|
12
|
+
from .instance_flow import project_instance_forward, make_instance_scene
|
|
13
|
+
from .flow_layout import make_flow_scene
|
|
14
|
+
from .health import evaluate
|
|
15
|
+
from .layout import make_scene
|
|
16
|
+
from .render import svg, pdf, tikz
|
|
17
|
+
from .themes import select_theme
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
FORMATS={'svg': ('svg',), 'pdf': ('pdf',), 'tikz': ('tex',),
|
|
21
|
+
'all': ('svg','pdf','tex')}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def present(bundle_path: str | Path, output: str | Path, *, level: str = 'block',
|
|
25
|
+
mode: str = 'exploration', format: str = 'svg',
|
|
26
|
+
allow_partial: bool = False, view: str = 'structure',
|
|
27
|
+
focus: str | None = None, theme: str = 'scientific-latex') -> dict[str, Any]:
|
|
28
|
+
"""Do not render on invalid bundles or without sufficient evidence.
|
|
29
|
+
|
|
30
|
+
Always write the health and manifest JSON; diagrams are absent if blocked
|
|
31
|
+
or withheld by publication policy.
|
|
32
|
+
"""
|
|
33
|
+
style = select_theme(theme)
|
|
34
|
+
if view not in ('structure','flow','nested','instance'):
|
|
35
|
+
raise ValueError(f'Invalid view: {view}')
|
|
36
|
+
if focus and view not in ('flow', 'nested', 'instance'):
|
|
37
|
+
raise ValueError('--focus currently supports flow, nested and instance views only')
|
|
38
|
+
if level not in ('overview','block','operation'):
|
|
39
|
+
raise ValueError(f'Invalid level: {level}')
|
|
40
|
+
if mode not in ('exploration','publication'):
|
|
41
|
+
raise ValueError(f'Invalid mode: {mode}')
|
|
42
|
+
if format not in FORMATS:
|
|
43
|
+
raise ValueError(f'Invalid format: {format}')
|
|
44
|
+
if allow_partial and mode != 'publication':
|
|
45
|
+
raise ValueError('--allow-partial applies to publication mode only')
|
|
46
|
+
output_dir=Path(output)
|
|
47
|
+
output_dir.mkdir(parents=True,exist_ok=True)
|
|
48
|
+
# No stale figures should survive a failed or subsequently blocked invocation.
|
|
49
|
+
for ext in ('svg','pdf','tex'):
|
|
50
|
+
(output_dir/f'architecture.{ext}').unlink(missing_ok=True)
|
|
51
|
+
bundle=None; graph=None; problem=None
|
|
52
|
+
try:
|
|
53
|
+
bundle=load_bundle(bundle_path)
|
|
54
|
+
graph=(project_instance_forward(bundle,level,focus=focus) if view=='instance' else
|
|
55
|
+
project_flow(bundle,level,focus=focus) if view=='flow' else
|
|
56
|
+
project_nested_flow(bundle,level,focus=focus) if view=='nested' else
|
|
57
|
+
project(bundle,level))
|
|
58
|
+
except (OSError,ValueError,KeyError,TypeError,json.JSONDecodeError) as exc:
|
|
59
|
+
problem=f'{type(exc).__name__}: {exc}'
|
|
60
|
+
health=evaluate(bundle,graph,mode=mode,level=level,input_error=problem)
|
|
61
|
+
if view=='instance':
|
|
62
|
+
health['scope']='Bounded source-proven forward for explicit input shape; external execution check is separate'
|
|
63
|
+
if view in ('flow', 'nested'):
|
|
64
|
+
health['scope']='Source-backed static forward dependencies, supported call summaries and conditional type-level shapes; runtime behavior unverified'
|
|
65
|
+
health['view']=view
|
|
66
|
+
health['focus']=focus
|
|
67
|
+
allowed=health['status'] != 'blocked' and (mode=='exploration' or
|
|
68
|
+
health['status']=='complete' or allow_partial)
|
|
69
|
+
health['render_allowed']=allowed
|
|
70
|
+
health['publication_gate']='withheld_partial' if health['status']=='partial' and mode=='publication' and not allow_partial else 'ok'
|
|
71
|
+
if bundle is not None:
|
|
72
|
+
input_hashes=bundle.metadata.get('artifact_sha256',{})
|
|
73
|
+
else:
|
|
74
|
+
input_hashes={}
|
|
75
|
+
manifest={'schema_version':'1.0','style_version':style['name'],
|
|
76
|
+
'mode':mode,'level':level,'view':view,'focus':focus,'entrypoint':bundle.ir.get('entrypoint') if bundle else None,
|
|
77
|
+
'input_artifact_hashes':input_hashes,'status':health['status'],
|
|
78
|
+
'rendered':allowed,'visual_elements':{},'visual_links':{},
|
|
79
|
+
'omitted_indexed_ir_ids':[],'output_sha256':{}}
|
|
80
|
+
if graph is not None:
|
|
81
|
+
# Source-backed parent-child membership is used for drill-down;
|
|
82
|
+
# membership never asserts concrete runtime tensor dependencies.
|
|
83
|
+
elements = {n['id']: n for n in graph['nodes']}
|
|
84
|
+
memberships = {nid: set(n['ir_node_ids']) for nid, n in elements.items()}
|
|
85
|
+
for n in graph['nodes']:
|
|
86
|
+
parent = n.get('parent_id')
|
|
87
|
+
visited = {n['id']}
|
|
88
|
+
while parent in elements and parent not in visited:
|
|
89
|
+
memberships[parent].update(n['ir_node_ids'])
|
|
90
|
+
visited.add(parent)
|
|
91
|
+
parent = elements[parent].get('parent_id')
|
|
92
|
+
if view == 'structure' and level != 'operation' and bundle is not None:
|
|
93
|
+
# A grouped source node can carry exact declared-operation
|
|
94
|
+
# membership from the *full* source-only Operation projection.
|
|
95
|
+
# Stable visual IDs identify the same constructor path at
|
|
96
|
+
# every zoom level; this is NOT runtime topology or execution.
|
|
97
|
+
full = project(bundle, 'operation')
|
|
98
|
+
full_nodes = {n['id']: n for n in full['nodes']}
|
|
99
|
+
full_members = {nid: set(n['ir_node_ids']) for nid, n in full_nodes.items()}
|
|
100
|
+
for deep in full['nodes']:
|
|
101
|
+
parent = deep.get('parent_id')
|
|
102
|
+
visited = {deep['id']}
|
|
103
|
+
while parent in full_nodes and parent not in visited:
|
|
104
|
+
full_members[parent].update(deep['ir_node_ids'])
|
|
105
|
+
visited.add(parent)
|
|
106
|
+
parent = full_nodes[parent].get('parent_id')
|
|
107
|
+
for vid in memberships:
|
|
108
|
+
if vid in full_members:
|
|
109
|
+
memberships[vid].update(full_members[vid])
|
|
110
|
+
if view=='instance':
|
|
111
|
+
manifest['input_shape']=graph.get('input_shape')
|
|
112
|
+
manifest['instance_artifact']='instance-forward.json'
|
|
113
|
+
manifest['collapsed_instance_edge_ids']=graph.get('collapsed_instance_edge_ids', [])
|
|
114
|
+
manifest['omitted_instance_node_ids']=graph.get('omitted_instance_node_ids', [])
|
|
115
|
+
manifest['omitted_instance_edge_ids']=graph.get('omitted_instance_edge_ids', [])
|
|
116
|
+
manifest['visual_elements']={n['id']:{'kind':n['kind'],
|
|
117
|
+
'label':n['label'], 'shape':n.get('shape'),
|
|
118
|
+
'evidence':n.get('evidence','source-backed'),
|
|
119
|
+
'member_ir_node_ids': sorted(memberships[n['id']]),
|
|
120
|
+
'ir_node_ids':n['ir_node_ids'],'ir_edge_ids':n['ir_edge_ids'],
|
|
121
|
+
'source_refs':n['source_refs'],
|
|
122
|
+
**({'semantic_category':n['semantic_category']}
|
|
123
|
+
if 'semantic_category' in n else {})}
|
|
124
|
+
for n in graph['nodes']}
|
|
125
|
+
manifest['visual_links']={e['id']:{'relation_kind':e['relation_kind'],
|
|
126
|
+
'ir_edge_ids':e['ir_edge_ids'],'source_refs':e['source_refs'],
|
|
127
|
+
**({'instance_edge_id':e['instance_edge_id']} if 'instance_edge_id' in e else {})}
|
|
128
|
+
for e in graph['links']}
|
|
129
|
+
if bundle:
|
|
130
|
+
indexed=set()
|
|
131
|
+
for m in bundle.model_index.get('modules',[]):
|
|
132
|
+
if m['definition_id'] not in {x for n in graph['nodes'] for x in n['ir_node_ids']}:
|
|
133
|
+
continue
|
|
134
|
+
indexed.update(m.get('forward_relation_ids',[]))
|
|
135
|
+
indexed.update(c['call_site_id'] for c in m['forward_calls'])
|
|
136
|
+
indexed.update(op['operation_site_id'] for op in m['forward_operations'])
|
|
137
|
+
mapped={z for n in graph['nodes'] for z in n['ir_node_ids']}
|
|
138
|
+
mapped|={z for e in graph['links'] for z in e['ir_edge_ids']}
|
|
139
|
+
manifest['omitted_indexed_ir_ids']=[{'id':i,'reason':'hidden_by_selected_view_or_unsupported_projection'}
|
|
140
|
+
for i in sorted(indexed-mapped)]
|
|
141
|
+
if allowed:
|
|
142
|
+
scene=(make_instance_scene if view=='instance' else make_flow_scene if view in ('flow','nested') else make_scene)(
|
|
143
|
+
graph, status=health['status'],
|
|
144
|
+
entrypoint=(f"{health['entrypoint']} / {focus}"
|
|
145
|
+
if focus and focus != (health['entrypoint'] or '').split(':')[-1]
|
|
146
|
+
else health['entrypoint']))
|
|
147
|
+
scene['style'] = style
|
|
148
|
+
exporters={'svg':svg,'pdf':pdf,'tex':tikz}
|
|
149
|
+
for extension in FORMATS[format]:
|
|
150
|
+
content=exporters[extension](scene)
|
|
151
|
+
(output_dir/f'architecture.{extension}').write_bytes(content)
|
|
152
|
+
manifest['output_sha256'][f'architecture.{extension}']=sha256(content)
|
|
153
|
+
(output_dir/'presentation-health.json').write_bytes(canonical_bytes(health))
|
|
154
|
+
(output_dir/'presentation-manifest.json').write_bytes(canonical_bytes(manifest))
|
|
155
|
+
return {'status':health['status'],'render_allowed':allowed,'mode':mode,'level':level,'view':view,'focus':focus,'theme':theme,
|
|
156
|
+
'formats':list(FORMATS[format]) if allowed else [],
|
|
157
|
+
'issues':len(health['issues']),'output':str(output_dir)}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Evidence-preserving two-level presentation for audited EfficientNet-B0."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
from .instance_hierarchy import _focus,_summary,_unsupported
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def project_efficientnet_hierarchy(graph:dict, *, level:str,focus:str|None,
|
|
7
|
+
source_scope:str)->dict:
|
|
8
|
+
if source_scope!="pinned_torchvision_0_25_0_efficientnet_b0":
|
|
9
|
+
return _unsupported(level,"Requires pinned TorchVision 0.25.0 EfficientNet-B0")
|
|
10
|
+
stages=[];current="input"
|
|
11
|
+
for n in graph["nodes"]:
|
|
12
|
+
target=n["target"]
|
|
13
|
+
if n["op"]=="placeholder":
|
|
14
|
+
current="input"
|
|
15
|
+
elif target.startswith("features."):
|
|
16
|
+
pieces=target.split(".")
|
|
17
|
+
if len(pieces)>2 and pieces[1].isdigit():
|
|
18
|
+
index=int(pieces[1])
|
|
19
|
+
if not 0<=index<=8:
|
|
20
|
+
return _unsupported(level,"Unexpected EfficientNet feature target")
|
|
21
|
+
current=f"features.{index}"
|
|
22
|
+
elif target=="avgpool" or target=="flatten" or target.startswith("classifier."):
|
|
23
|
+
current="classifier"
|
|
24
|
+
elif n["op"]=="output":
|
|
25
|
+
current="output"
|
|
26
|
+
# Functional SE, StochasticDepth, and residual add remain within their
|
|
27
|
+
# previously source-proven feature stage.
|
|
28
|
+
stages.append(current)
|
|
29
|
+
expected={"input","classifier","output"}|{f"features.{i}" for i in range(9)}
|
|
30
|
+
if set(stages)!=expected:
|
|
31
|
+
return _unsupported(level,"Incomplete source-evidenced EfficientNet stages")
|
|
32
|
+
if focus:
|
|
33
|
+
if level!="operation":
|
|
34
|
+
return _unsupported(level,"Only operation focus is supported")
|
|
35
|
+
return _focus(graph,stages,focus)
|
|
36
|
+
if level=="operation":return graph
|
|
37
|
+
if level not in ("overview","block"):
|
|
38
|
+
return _unsupported(level,"Unknown EfficientNet display level")
|
|
39
|
+
transformed=["backbone" if level=="overview" and s in {
|
|
40
|
+
f"features.{i}" for i in range(1,8)} else s for s in stages]
|
|
41
|
+
labels={"input":"RGB Input","features.0":"Stem Conv-BN-SiLU",
|
|
42
|
+
"backbone":"EfficientNet-B0 MBConv Backbone",
|
|
43
|
+
"features.8":"Final Conv-BN-SiLU",
|
|
44
|
+
"classifier":"Global Pool / Classifier","output":"Output",
|
|
45
|
+
**{f"features.{i}":f"MBConv Stage {i}" for i in range(1,8)}}
|
|
46
|
+
result=_summary(graph,transformed,level,model_key="torchvision-efficientnet-b0",
|
|
47
|
+
labels_override=labels,fallback_prefix="EfficientNet ")
|
|
48
|
+
if len(result["nodes"])!=(6 if level=="overview" else 12):
|
|
49
|
+
return _unsupported(level,"Unexpected EfficientNet grouping")
|
|
50
|
+
return result
|
ae/presenter/flow.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""Project SAE's *static value-dependency* evidence into a scientific flow view.
|
|
2
|
+
|
|
3
|
+
No source parsing, execution, or inferred runtime tensor-flow is performed here.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from collections import defaultdict
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from ae.sae.ir.canonical import stable_id
|
|
11
|
+
from .health import issue
|
|
12
|
+
from .reader import Bundle
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
FLOW_EDGES = frozenset({'STATIC_VALUE_DEPENDENCY', 'STATIC_FORWARD_RETURN', 'STATIC_CONDITIONAL_DEPENDENCY', 'STATIC_GUARD_DEPENDENCY', 'STATIC_GUARDED_CALL', 'STATIC_COLLECTION_ELEMENT', 'STATIC_COLLECTION_MEMBER', 'STATIC_COLLECTION_ELEMENT_BINDING'})
|
|
16
|
+
FLOW_VALUES = frozenset({'forward_input', 'forward_output', 'forward_unpack', 'forward_choice', 'forward_guard', 'forward_augassign', 'forward_collection', 'forward_collection_element'})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _caption(call: dict, components: dict[str, dict]) -> str:
|
|
20
|
+
callee = call['callee_expression']
|
|
21
|
+
constructions = call.get('module_construction_site_ids', [])
|
|
22
|
+
if len(constructions) == 1 and constructions[0] in components:
|
|
23
|
+
comp = components[constructions[0]]
|
|
24
|
+
label = comp.get('owner_expression', callee).removeprefix('self.')
|
|
25
|
+
return f"{label}: {(comp.get('declared_type') or 'Module').split('.')[-1]}"
|
|
26
|
+
if callee.startswith('self.'):
|
|
27
|
+
return callee[5:]
|
|
28
|
+
return callee.replace('torch.nn.functional.', 'F.').replace('torch.', '')
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def project_flow(bundle: Bundle, level: str = 'block', focus: str | None = None) -> dict[str, Any]:
|
|
32
|
+
if level not in ('overview', 'block', 'operation'):
|
|
33
|
+
raise ValueError(f'Unsupported flow level: {level}')
|
|
34
|
+
index = bundle.model_index
|
|
35
|
+
graph: dict[str, Any] = {'nodes': [], 'links': [], 'issues': [], 'level': level, 'view': 'flow', 'focus': focus}
|
|
36
|
+
if index.get('status') != 'indexed':
|
|
37
|
+
return graph
|
|
38
|
+
modules = index.get('modules', [])
|
|
39
|
+
if level == 'overview':
|
|
40
|
+
modules = [m for m in modules if m['definition_id'] == index.get('root_definition_id')]
|
|
41
|
+
else:
|
|
42
|
+
modules = sorted(modules, key=lambda m: (m['definition_id'] != index.get('root_definition_id'), m['qualname']))
|
|
43
|
+
|
|
44
|
+
if focus is not None:
|
|
45
|
+
selected = [m for m in modules if m['qualname'] == focus or m['qualname'].split('.')[-1] == focus]
|
|
46
|
+
if len(selected) != 1:
|
|
47
|
+
graph['issues'].append(issue('FOCUS_UNRESOLVED',
|
|
48
|
+
f'Expected exactly one module matching {focus!r}; found {len(selected)}', severity='error'))
|
|
49
|
+
return graph
|
|
50
|
+
modules = selected
|
|
51
|
+
|
|
52
|
+
flow_by_forward: dict[str, list[dict]] = defaultdict(list)
|
|
53
|
+
for n in bundle.ir['nodes']:
|
|
54
|
+
if n['kind'] in FLOW_VALUES:
|
|
55
|
+
flow_by_forward[n['attributes'].get('forward_id')].append(n)
|
|
56
|
+
flow_edges = [e for e in bundle.ir['edges'] if e['relation'] in FLOW_EDGES]
|
|
57
|
+
shape_facts = {n['attributes']['value_id']: n for n in bundle.ir['nodes']
|
|
58
|
+
if n['kind'] == 'symbolic_shape_fact'}
|
|
59
|
+
shape_edges = {e['source']:e['id'] for e in bundle.ir['edges']
|
|
60
|
+
if e['relation'] == 'HAS_SYMBOLIC_SHAPE'}
|
|
61
|
+
guarded = {e['target'] for e in bundle.ir['edges'] if e['relation'] == 'STATIC_GUARDED_CALL'}
|
|
62
|
+
global_mapped: dict[str, str] = {}
|
|
63
|
+
mapped_lane: dict[str, str] = {}
|
|
64
|
+
for module in modules:
|
|
65
|
+
fid = module.get('forward_method_id')
|
|
66
|
+
if not fid:
|
|
67
|
+
graph['issues'].append(issue('NO_FORWARD', f"No unique forward method for {module['qualname']}",
|
|
68
|
+
ids=[module['definition_id']], refs=module['source_refs']))
|
|
69
|
+
continue
|
|
70
|
+
definition = module['definition_id']
|
|
71
|
+
root_id = stable_id('flow-lane/v1', definition)
|
|
72
|
+
graph['nodes'].append({'id': root_id, 'kind': 'flow-module', 'parent_id': None,
|
|
73
|
+
'label': module['qualname'].split('.')[-1], 'details': module['qualname'],
|
|
74
|
+
'is_root': definition == index.get('root_definition_id'),
|
|
75
|
+
'ir_node_ids': [definition], 'ir_edge_ids': [],
|
|
76
|
+
'source_refs': module['source_refs'], 'evidence': 'explicit'})
|
|
77
|
+
comps = {c['construction_site_id']: c for c in module['components']}
|
|
78
|
+
steps: list[tuple[str, str, str, str]] = []
|
|
79
|
+
# tuple: kind, label, source IR ID, resolution
|
|
80
|
+
for value in flow_by_forward.get(fid, []):
|
|
81
|
+
steps.append((value['kind'], value['attributes']['name'], value['id'], value['resolution']))
|
|
82
|
+
for call in module.get('forward_calls', []):
|
|
83
|
+
sid = call['call_site_id']
|
|
84
|
+
unknown = call['callee_expression'].startswith('self.') and not (
|
|
85
|
+
call.get('module_construction_site_ids') or call.get('local_definition_ids'))
|
|
86
|
+
if unknown:
|
|
87
|
+
graph['issues'].append(issue('UNRESOLVED_FORWARD_CALL',
|
|
88
|
+
f"Unresolved forward call: {call['callee_expression']}", ids=[sid], refs=call['source_refs']))
|
|
89
|
+
steps.append(('unresolved-call' if unknown else 'flow-call', _caption(call, comps), sid,
|
|
90
|
+
'unresolved' if unknown else 'conditional' if sid in guarded else 'resolved'))
|
|
91
|
+
flow_members = {e['source'] for e in flow_edges} | {e['target'] for e in flow_edges}
|
|
92
|
+
for op in module.get('forward_operations', []):
|
|
93
|
+
if op['operation_site_id'] in flow_members:
|
|
94
|
+
steps.append(('flow-operation', op['operator'], op['operation_site_id'], 'resolved'))
|
|
95
|
+
|
|
96
|
+
# A source-backed visual step always corresponds to an existing SAE IR node.
|
|
97
|
+
mapped: dict[str, str] = {}
|
|
98
|
+
for kind, label, source_id, state in sorted(steps, key=lambda x: (
|
|
99
|
+
bundle.nodes[x[2]]['source_refs'][0]['start_byte'], x[2])):
|
|
100
|
+
if kind == 'forward_unpack' and level == 'block':
|
|
101
|
+
# Unpacking is kept in the flow diagram even in block mode: it
|
|
102
|
+
# carries the outgoing edges needed to show fan-out accurately.
|
|
103
|
+
pass
|
|
104
|
+
vid = stable_id('flow-node/v1', definition, source_id)
|
|
105
|
+
if source_id in mapped:
|
|
106
|
+
continue
|
|
107
|
+
mapped[source_id] = vid
|
|
108
|
+
global_mapped[source_id] = vid
|
|
109
|
+
mapped_lane[source_id] = root_id
|
|
110
|
+
source = bundle.nodes[source_id]
|
|
111
|
+
fact = shape_facts.get(source_id)
|
|
112
|
+
shape = '[' + ', '.join(fact['attributes']['dimensions']) + ']' if fact else None
|
|
113
|
+
graph['nodes'].append({'id': vid, 'kind': kind, 'parent_id': root_id,
|
|
114
|
+
'label': label, 'details': source.get('attributes', {}).get('name', ''),
|
|
115
|
+
'shape': shape,
|
|
116
|
+
'ir_node_ids': [source_id] + ([fact['id']] if fact else []),
|
|
117
|
+
'ir_edge_ids': [shape_edges[source_id]] if fact and source_id in shape_edges else [],
|
|
118
|
+
'source_refs': source['source_refs'],
|
|
119
|
+
'evidence': 'conditional' if state == 'conditional' else
|
|
120
|
+
'unresolved' if state == 'unresolved' else 'explicit'})
|
|
121
|
+
for edge in flow_edges:
|
|
122
|
+
if edge['source'] not in mapped or edge['target'] not in mapped:
|
|
123
|
+
continue
|
|
124
|
+
visual_id = stable_id('flow-link/v1', definition, edge['id'])
|
|
125
|
+
graph['links'].append({'id': visual_id, 'source': mapped[edge['source']],
|
|
126
|
+
'target': mapped[edge['target']],
|
|
127
|
+
'relation_kind': edge['relation'], 'ir_edge_ids': [edge['id']],
|
|
128
|
+
'source_refs': edge['source_refs'], 'style': 'control' if edge['relation']=='STATIC_GUARDED_CALL' else 'conditional-value' if edge['relation'] in ('STATIC_CONDITIONAL_DEPENDENCY','STATIC_GUARD_DEPENDENCY') else 'static-value'})
|
|
129
|
+
for record in bundle.ir['unresolved']:
|
|
130
|
+
if record.get('kind') == 'forward_flow_incomplete' and record.get('function_id') == fid:
|
|
131
|
+
graph['issues'].append(issue('STATIC_FLOW_INCOMPLETE', record.get('reason', 'Unknown flow'),
|
|
132
|
+
refs=record.get('source_refs', [])))
|
|
133
|
+
for record in bundle.ir['unresolved']:
|
|
134
|
+
if record.get('function_id') != fid:
|
|
135
|
+
continue
|
|
136
|
+
if record.get('kind') in ('interprocedural_flow_incomplete', 'symbolic_shape_conflict'):
|
|
137
|
+
graph['issues'].append(issue(record['kind'].upper(), record.get('reason', 'Unresolved semantic relationship'),
|
|
138
|
+
ids=[record['site_id']] if record.get('site_id') else [],
|
|
139
|
+
refs=record.get('source_refs', [])))
|
|
140
|
+
if not any(n['kind'] == 'forward_output' and n['parent_id'] == root_id for n in graph['nodes']):
|
|
141
|
+
graph['issues'].append(issue('STATIC_FLOW_INCOMPLETE',
|
|
142
|
+
f"No straight-line return represented in {module['qualname']}",
|
|
143
|
+
ids=[fid], refs=bundle.nodes[fid]['source_refs']))
|
|
144
|
+
if not any(e['source'] in mapped and e['target'] in mapped for e in flow_edges):
|
|
145
|
+
graph['issues'].append(issue('STATIC_FLOW_INCOMPLETE',
|
|
146
|
+
f"No source-backed value dependency in {module['qualname']}",
|
|
147
|
+
ids=[fid], refs=bundle.nodes[fid]['source_refs']))
|
|
148
|
+
|
|
149
|
+
# Only show actual SAE links between displayed module lanes. This does
|
|
150
|
+
# not imply runtime tensor flow, and Focus deliberately omits other lanes.
|
|
151
|
+
for edge in flow_edges:
|
|
152
|
+
if (edge['relation'] != 'STATIC_COLLECTION_ELEMENT_BINDING'
|
|
153
|
+
or edge['source'] not in global_mapped
|
|
154
|
+
or edge['target'] not in global_mapped
|
|
155
|
+
or mapped_lane[edge['source']] == mapped_lane[edge['target']]):
|
|
156
|
+
continue
|
|
157
|
+
graph['links'].append({
|
|
158
|
+
'id': stable_id('flow-cross-module/v1', edge['id']),
|
|
159
|
+
'source': global_mapped[edge['source']],
|
|
160
|
+
'target': global_mapped[edge['target']],
|
|
161
|
+
'relation_kind': edge['relation'], 'ir_edge_ids': [edge['id']],
|
|
162
|
+
'source_refs': edge['source_refs'], 'style': 'interprocedural'})
|
|
163
|
+
if not graph['nodes']:
|
|
164
|
+
graph['issues'].append(issue('NO_STATIC_FLOW_EVIDENCE' , 'No usable static forward flow. Re-extract with a compatible SAE.', severity='error'))
|
|
165
|
+
graph['nodes'].sort(key=lambda n: (n['parent_id'] or '', n['source_refs'][0]['path'],
|
|
166
|
+
n['source_refs'][0]['start_byte'], n['id']))
|
|
167
|
+
graph['links'].sort(key=lambda e: e['id'])
|
|
168
|
+
graph['issues'].sort(key=lambda x: (x['code'], x['message']))
|
|
169
|
+
return graph
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def project_nested_flow(bundle: Bundle, level: str = 'block', focus: str | None = None
|
|
173
|
+
) -> dict[str, Any]:
|
|
174
|
+
"""Add source-backed nested call references to a static flow diagram."""
|
|
175
|
+
graph = project_flow(bundle, level, focus)
|
|
176
|
+
graph['view'] = 'nested'
|
|
177
|
+
mapped: dict[str, str] = {}
|
|
178
|
+
lane: dict[str, str] = {}
|
|
179
|
+
for node in graph['nodes']:
|
|
180
|
+
if node.get('parent_id') is None:
|
|
181
|
+
continue
|
|
182
|
+
for ident in node['ir_node_ids']:
|
|
183
|
+
mapped[ident] = node['id']
|
|
184
|
+
lane[ident] = node['parent_id']
|
|
185
|
+
inputs = {}
|
|
186
|
+
for n in bundle.ir['nodes']:
|
|
187
|
+
if n['kind'] == 'forward_input':
|
|
188
|
+
inputs.setdefault(n['attributes']['forward_id'], []).append(n['id'])
|
|
189
|
+
invocations = {(e['source'], e['target']): e for e in bundle.ir['edges']
|
|
190
|
+
if e['relation'] == 'STATIC_NESTED_FORWARD_INVOCATION'}
|
|
191
|
+
for module in bundle.model_index.get('modules', []):
|
|
192
|
+
if focus is not None and not any(n.get('ir_node_ids') == [module['definition_id']]
|
|
193
|
+
for n in graph['nodes']):
|
|
194
|
+
continue
|
|
195
|
+
for record in module.get('nested_forward_calls', []):
|
|
196
|
+
call_id = record['call_site_id']
|
|
197
|
+
forward_id = record['target_forward_method_id']
|
|
198
|
+
members = inputs.get(forward_id, [])
|
|
199
|
+
if call_id not in mapped or len(members) != 1 or members[0] not in mapped:
|
|
200
|
+
continue
|
|
201
|
+
if lane[call_id] == lane[members[0]]:
|
|
202
|
+
continue
|
|
203
|
+
edge = invocations.get((call_id, forward_id))
|
|
204
|
+
if edge is None:
|
|
205
|
+
continue
|
|
206
|
+
graph['links'].append({
|
|
207
|
+
'id': stable_id('nested-forward-visual/v1', edge['id']),
|
|
208
|
+
'source': mapped[call_id], 'target': mapped[members[0]],
|
|
209
|
+
'relation_kind': edge['relation'], 'ir_edge_ids': [edge['id']],
|
|
210
|
+
'source_refs': edge['source_refs'], 'style': 'interprocedural'})
|
|
211
|
+
graph['links'].sort(key=lambda e: e['id'])
|
|
212
|
+
return graph
|