accelforge 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. accelforge/__init__.py +21 -0
  2. accelforge/_accelerated_imports.py +16 -0
  3. accelforge/_deprecate/_simanneal/evalmapping.py +271 -0
  4. accelforge/_deprecate/_simanneal/mapspaceglobals.py +298 -0
  5. accelforge/_deprecate/_simanneal/simanneal.py +666 -0
  6. accelforge/_deprecate/_simanneal/tracking.py +105 -0
  7. accelforge/_deprecate/_simanneal/wrappers.py +218 -0
  8. accelforge/_deprecate/_simanneal2/__init__.py +7 -0
  9. accelforge/_deprecate/_simanneal2/simanneal.py +493 -0
  10. accelforge/_deprecate/_simanneal2/tracking.py +116 -0
  11. accelforge/_deprecate/compatibility_util.py +181 -0
  12. accelforge/_deprecate/layerdeduplication/__init__.py +2 -0
  13. accelforge/_deprecate/layerdeduplication/group_similar_einsums.py +160 -0
  14. accelforge/_deprecate/layerdeduplication/grouped_einsums.py +84 -0
  15. accelforge/_deprecate/mapping_filter_tags/__init__.py +2 -0
  16. accelforge/_deprecate/mapping_filter_tags/ffmt.py +212 -0
  17. accelforge/_deprecate/mapping_filter_tags/onesplit.py +24 -0
  18. accelforge/_deprecate/mapping_filter_tags/util.py +24 -0
  19. accelforge/_deprecate/tags.py +69 -0
  20. accelforge/_deprecate/viz/__init__.py +0 -0
  21. accelforge/_deprecate/viz/interactive.py +159 -0
  22. accelforge/_deprecate/viz/reservationtree.py +307 -0
  23. accelforge/_deprecate/viz/ski_slope.py +88 -0
  24. accelforge/_version.py +15 -0
  25. accelforge/examples.py +39 -0
  26. accelforge/frontend/__init__.py +10 -0
  27. accelforge/frontend/_binding.py +129 -0
  28. accelforge/frontend/_workload_isl/__init__.py +2 -0
  29. accelforge/frontend/_workload_isl/_isl.py +149 -0
  30. accelforge/frontend/_workload_isl/_symbolic.py +141 -0
  31. accelforge/frontend/arch copy.py +1544 -0
  32. accelforge/frontend/arch.py +1642 -0
  33. accelforge/frontend/config.py +63 -0
  34. accelforge/frontend/mapper/__init__.py +5 -0
  35. accelforge/frontend/mapper/ffm.py +126 -0
  36. accelforge/frontend/mapper/mapper.py +7 -0
  37. accelforge/frontend/mapper/metrics.py +30 -0
  38. accelforge/frontend/mapping/__init__.py +1 -0
  39. accelforge/frontend/mapping/mapping.py +1736 -0
  40. accelforge/frontend/model.py +14 -0
  41. accelforge/frontend/renames.py +150 -0
  42. accelforge/frontend/spec copy.py +230 -0
  43. accelforge/frontend/spec.py +301 -0
  44. accelforge/frontend/variables.py +12 -0
  45. accelforge/frontend/workload.py +952 -0
  46. accelforge/mapper/FFM/__init__.py +9 -0
  47. accelforge/mapper/FFM/_join_pmappings/__init__.py +0 -0
  48. accelforge/mapper/FFM/_join_pmappings/compatibility.py +653 -0
  49. accelforge/mapper/FFM/_join_pmappings/compress_pmappings.py +140 -0
  50. accelforge/mapper/FFM/_join_pmappings/join_pmappings.py +703 -0
  51. accelforge/mapper/FFM/_join_pmappings/pmapping_dataframe.py +901 -0
  52. accelforge/mapper/FFM/_join_pmappings/pmapping_group.py +337 -0
  53. accelforge/mapper/FFM/_make_pmappings/contraints/__init__.py +0 -0
  54. accelforge/mapper/FFM/_make_pmappings/contraints/constraints.py +360 -0
  55. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/__init__.py +1 -0
  56. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/make_loops.py +373 -0
  57. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/make_pmapping_templates.py +463 -0
  58. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/make_reservations.py +95 -0
  59. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/make_storage_order.py +382 -0
  60. accelforge/mapper/FFM/_make_pmappings/make_pmapping_templates/make_storages.py +155 -0
  61. accelforge/mapper/FFM/_make_pmappings/make_pmappings.py +411 -0
  62. accelforge/mapper/FFM/_make_pmappings/make_pmappings_from_templates/__init__.py +1 -0
  63. accelforge/mapper/FFM/_make_pmappings/make_pmappings_from_templates/make_pmappings_from_templates.py +407 -0
  64. accelforge/mapper/FFM/_make_pmappings/make_pmappings_from_templates/make_tile_shapes.py +1681 -0
  65. accelforge/mapper/FFM/_make_pmappings/make_pmappings_from_templates/run_model.py +170 -0
  66. accelforge/mapper/FFM/_make_pmappings/make_pmappings_from_templates/symbol_relations.py +174 -0
  67. accelforge/mapper/FFM/_make_pmappings/pmapper_job.py +282 -0
  68. accelforge/mapper/FFM/_pareto_df/df_convention.py +273 -0
  69. accelforge/mapper/FFM/_pareto_df/pareto copy.py +836 -0
  70. accelforge/mapper/FFM/_pareto_df/pareto.py +508 -0
  71. accelforge/mapper/FFM/data.py +61 -0
  72. accelforge/mapper/FFM/main copy.py +236 -0
  73. accelforge/mapper/FFM/main.py +208 -0
  74. accelforge/mapper/FFM/mappings.py +510 -0
  75. accelforge/mapper/FFM/pmappings.py +310 -0
  76. accelforge/mapper/__init__.py +4 -0
  77. accelforge/mapper.py +0 -0
  78. accelforge/model/__init__.py +1 -0
  79. accelforge/model/_looptree/__init__.py +0 -0
  80. accelforge/model/_looptree/accesses.py +335 -0
  81. accelforge/model/_looptree/capacity/__init__.py +1 -0
  82. accelforge/model/_looptree/capacity/aggregators.py +36 -0
  83. accelforge/model/_looptree/capacity/capacity.py +47 -0
  84. accelforge/model/_looptree/energy.py +150 -0
  85. accelforge/model/_looptree/equivalent_ranks.py +29 -0
  86. accelforge/model/_looptree/latency/__init__.py +1 -0
  87. accelforge/model/_looptree/latency/latency.py +98 -0
  88. accelforge/model/_looptree/latency/memory.py +120 -0
  89. accelforge/model/_looptree/latency/processors.py +92 -0
  90. accelforge/model/_looptree/mapping_utilities.py +71 -0
  91. accelforge/model/_looptree/reuse/__init__.py +4 -0
  92. accelforge/model/_looptree/reuse/isl/__init__.py +1 -0
  93. accelforge/model/_looptree/reuse/isl/des.py +59 -0
  94. accelforge/model/_looptree/reuse/isl/isl_functions.py +374 -0
  95. accelforge/model/_looptree/reuse/isl/mapping_to_isl/__init__.py +4 -0
  96. accelforge/model/_looptree/reuse/isl/mapping_to_isl/analyze_mapping.py +297 -0
  97. accelforge/model/_looptree/reuse/isl/mapping_to_isl/skews_from_mapping.py +236 -0
  98. accelforge/model/_looptree/reuse/isl/mapping_to_isl/tiling.py +685 -0
  99. accelforge/model/_looptree/reuse/isl/mapping_to_isl/types.py +188 -0
  100. accelforge/model/_looptree/reuse/isl/spatial.py +260 -0
  101. accelforge/model/_looptree/reuse/isl/temporal.py +182 -0
  102. accelforge/model/_looptree/reuse/symbolic/__init__.py +1 -0
  103. accelforge/model/_looptree/reuse/symbolic/symbolic copy 2.py +1346 -0
  104. accelforge/model/_looptree/reuse/symbolic/symbolic copy.py +1408 -0
  105. accelforge/model/_looptree/reuse/symbolic/symbolic.py +1396 -0
  106. accelforge/model/_looptree/run.py +122 -0
  107. accelforge/model/_looptree/types.py +26 -0
  108. accelforge/model/_looptree/visualization/__init__.py +0 -0
  109. accelforge/model/_looptree/visualization/occupancy.py +11 -0
  110. accelforge/model/main.py +222 -0
  111. accelforge/plotting/__init__.py +2 -0
  112. accelforge/plotting/mappings.py +219 -0
  113. accelforge/plotting/specs.py +57 -0
  114. accelforge/util/__init__.py +4 -0
  115. accelforge/util/_base_analysis_types.py +24 -0
  116. accelforge/util/_basetypes.py +1089 -0
  117. accelforge/util/_frozenset.py +36 -0
  118. accelforge/util/_isl.py +29 -0
  119. accelforge/util/_itertools.py +14 -0
  120. accelforge/util/_mathfuncs.py +57 -0
  121. accelforge/util/_parse_expressions.py +339 -0
  122. accelforge/util/_picklecache.py +32 -0
  123. accelforge/util/_setexpressions.py +268 -0
  124. accelforge/util/_sympy/__init__.py +0 -0
  125. accelforge/util/_sympy/broadcast_max.py +18 -0
  126. accelforge/util/_visualization.py +112 -0
  127. accelforge/util/_yaml.py +579 -0
  128. accelforge/util/parallel.py +193 -0
  129. accelforge-0.0.1.dist-info/METADATA +64 -0
  130. accelforge-0.0.1.dist-info/RECORD +258 -0
  131. accelforge-0.0.1.dist-info/WHEEL +5 -0
  132. accelforge-0.0.1.dist-info/licenses/LICENSE +19 -0
  133. accelforge-0.0.1.dist-info/top_level.txt +5 -0
  134. docs/_build/html/_sources/fastfusion.frontend.mapper.rst.txt +37 -0
  135. docs/_build/html/_sources/fastfusion.frontend.rst.txt +70 -0
  136. docs/_build/html/_sources/fastfusion.frontend.workload.rst.txt +21 -0
  137. docs/_build/html/_sources/fastfusion.mapper.FFM.rst.txt +37 -0
  138. docs/_build/html/_sources/fastfusion.mapper.rst.txt +18 -0
  139. docs/_build/html/_sources/fastfusion.rst.txt +20 -0
  140. docs/_build/html/_sources/fastfusion.util.rst.txt +21 -0
  141. docs/_build/html/_sources/index.rst.txt +87 -0
  142. docs/_build/html/_sources/modules.rst.txt +7 -0
  143. docs/_build/html/_sources/notes/citation.rst.txt +45 -0
  144. docs/_build/html/_sources/notes/definitions.rst.txt +43 -0
  145. docs/_build/html/_sources/notes/faqs.rst.txt +39 -0
  146. docs/_build/html/_sources/notes/modeling/accelerator_energy_latency.rst.txt +72 -0
  147. docs/_build/html/_sources/notes/modeling/component_energy_area.rst.txt +96 -0
  148. docs/_build/html/_sources/notes/modeling/mapping.rst.txt +100 -0
  149. docs/_build/html/_sources/notes/modeling.rst.txt +33 -0
  150. docs/_build/html/_sources/notes/parsing/arithmetic_parsing.rst.txt +136 -0
  151. docs/_build/html/_sources/notes/parsing/setexpressions.rst.txt +63 -0
  152. docs/_build/html/_sources/notes/parsing/yaml_parsing.rst.txt +176 -0
  153. docs/_build/html/_sources/notes/quickstart_and_installation.rst.txt +9 -0
  154. docs/_build/html/_sources/notes/spec/architecture.rst.txt +133 -0
  155. docs/_build/html/_sources/notes/spec/mapping.rst.txt +12 -0
  156. docs/_build/html/_sources/notes/spec/workload.rst.txt +83 -0
  157. docs/_build/html/_sources/notes/spec.rst.txt +36 -0
  158. docs/source/_ext/include_attrs.py +213 -0
  159. docs/source/_ext/include_docstring.py +364 -0
  160. docs/source/_ext/include_functions.py +154 -0
  161. docs/source/_ext/include_notebook.py +131 -0
  162. docs/source/_ext/include_yaml.py +119 -0
  163. docs/source/_ext/inherited_attributes.py +222 -0
  164. docs/source/_ext/paths.py +4 -0
  165. docs/source/conf.py +79 -0
  166. examples/arches/compute_in_memory/_include.yaml +74 -0
  167. examples/arches/compute_in_memory/_include_functions.py +229 -0
  168. examples/arches/compute_in_memory/_load_spec.py +57 -0
  169. examples/arches/compute_in_memory/components/c2c_multiplier.py +181 -0
  170. examples/arches/compute_in_memory/components/dac_c2c_r2r.py +605 -0
  171. examples/arches/compute_in_memory/components/misc.py +195 -0
  172. examples/arches/compute_in_memory/components/util/bit_functions.py +51 -0
  173. examples/arches/compute_in_memory/components/zero_comparator.py +92 -0
  174. examples/arches/compute_in_memory/isaac.yaml +233 -0
  175. examples/arches/compute_in_memory/memory_cells/ecram_demo.yaml +63 -0
  176. examples/arches/compute_in_memory/memory_cells/rram_example.yaml +63 -0
  177. examples/arches/compute_in_memory/memory_cells/rram_isaac_isca_2016.yaml +64 -0
  178. examples/arches/compute_in_memory/memory_cells/rram_neurosim_default.yaml +63 -0
  179. examples/arches/compute_in_memory/memory_cells/rram_raella_isca_2023.yaml +70 -0
  180. examples/arches/compute_in_memory/memory_cells/rram_wan_nature_2022.yaml +63 -0
  181. examples/arches/compute_in_memory/memory_cells/sram_colonnade_jssc_2021.yaml +63 -0
  182. examples/arches/compute_in_memory/memory_cells/sram_example.yaml +63 -0
  183. examples/arches/compute_in_memory/memory_cells/sram_jia_jssc_2020.yaml +63 -0
  184. examples/arches/compute_in_memory/memory_cells/sram_sinangil_jssc_2021.yaml +63 -0
  185. examples/arches/compute_in_memory/memory_cells/sram_wang_vlsi_2022.yaml +63 -0
  186. examples/arches/compute_in_memory/wang_vlsi_2022.yaml +289 -0
  187. examples/arches/eyeriss.yaml +68 -0
  188. examples/arches/fanout_variations/at_glb.yaml +31 -0
  189. examples/arches/fanout_variations/at_glb_with_fanout_node.yaml +34 -0
  190. examples/arches/fanout_variations/at_mac.yaml +31 -0
  191. examples/arches/fanout_variations/at_mac_with_constraints.yaml +38 -0
  192. examples/arches/fanout_variations/at_mac_with_fanout_node.yaml +34 -0
  193. examples/arches/nvdla.yaml +47 -0
  194. examples/arches/simple.yaml +28 -0
  195. examples/arches/tpu_v4i.yaml +67 -0
  196. examples/mappings/unfused_matmuls_to_simple.yaml +33 -0
  197. examples/misc/component_annotated.yaml +33 -0
  198. examples/workloads/gpt3_6.7B.yaml +124 -0
  199. examples/workloads/matmuls.yaml +20 -0
  200. examples/workloads/mobilenet_28.yaml +81 -0
  201. examples/workloads/mobilenet_various_separate.yaml +106 -0
  202. examples/workloads/three_matmuls_annotated.yaml +59 -0
  203. notebooks/.ipynb_checkpoints/fastfusion_arch_study_michael-checkpoint.ipynb +359 -0
  204. notebooks/compute_in_memory/_scripts.py +339 -0
  205. notebooks/compute_in_memory/isaac.guide.ipynb +270 -0
  206. notebooks/compute_in_memory/wang_vlsi_2022.ipynb +602 -0
  207. notebooks/paths.py +4 -0
  208. notebooks/tutorials/.ipynb_checkpoints/1_FFM-checkpoint.ipynb +3110 -0
  209. notebooks/tutorials/FFM.ipynb +3498 -0
  210. notebooks/tutorials/_include.py +48 -0
  211. notebooks/tutorials/component_energy_area.ipynb +363 -0
  212. tests/Q_mapping.yaml +38 -0
  213. tests/__init__.py +0 -0
  214. tests/conv.mapping.yaml +27 -0
  215. tests/conv.workload.yaml +13 -0
  216. tests/conv_sym.mapping.yaml +43 -0
  217. tests/copy.mapping.yaml +35 -0
  218. tests/copy.workload.yaml +15 -0
  219. tests/distribuffers/__init__.py +0 -0
  220. tests/distribuffers/multicast/test_cases.yaml +482 -0
  221. tests/distribuffers/spec/binding/valid_bindings.yaml +97 -0
  222. tests/distribuffers/spec/distributed.yaml +100 -0
  223. tests/distribuffers/spec/logical_arch.yaml +32 -0
  224. tests/distribuffers/spec/physical_arch.yaml +69 -0
  225. tests/distribuffers/test_binding.py +48 -0
  226. tests/frontend/__init__.py +0 -0
  227. tests/frontend/test_mapping_viz.py +52 -0
  228. tests/mapper/__init__.py +0 -0
  229. tests/mapper/configs/conv1d/conv1d.mapping.yaml +31 -0
  230. tests/mapper/configs/conv1d/conv1d.workload.yaml +11 -0
  231. tests/mapper/configs/two_conv1d/two_conv1d.expected.yaml +38 -0
  232. tests/mapper/configs/two_conv1d/two_conv1d.mapping.yaml +54 -0
  233. tests/mapper/configs/two_conv1d/two_conv1d.workload.yaml +19 -0
  234. tests/mapper/test_mapping_to_isl.py +90 -0
  235. tests/mapper/test_spatial_reuse_analysis.py +67 -0
  236. tests/mapper/test_temporal_reuse_analysis.py +56 -0
  237. tests/mapper/util.py +58 -0
  238. tests/matmul.mapping.yaml +29 -0
  239. tests/matmul.workload.yaml +12 -0
  240. tests/matmul_spatial.mapping.yaml +44 -0
  241. tests/mha.renames.yaml +65 -0
  242. tests/mha.workload.yaml +67 -0
  243. tests/mha.yaml +59 -0
  244. tests/mha_full.workload.yaml +67 -0
  245. tests/mobilenet.workload.yaml +35 -0
  246. tests/mobilenet_long.workload.yaml +64 -0
  247. tests/pmappingcache.py +24 -0
  248. tests/processing_stage.arch.yaml +40 -0
  249. tests/snowcat.arch.yaml +36 -0
  250. tests/test_ffm_join_pmappings.py +106 -0
  251. tests/test_ffm_make_pmappings.py +82 -0
  252. tests/test_ffm_make_tile_shapes.py +49 -0
  253. tests/test_mapper.py +100 -0
  254. tests/test_model.py +37 -0
  255. tests/test_plotting.py +72 -0
  256. tests/test_processing_stage.py +46 -0
  257. tests/test_symbolic_model.py +248 -0
  258. tests/test_workload.py +141 -0
@@ -0,0 +1,154 @@
1
+ from docutils import nodes
2
+ from docutils.parsers.rst import Directive
3
+ import importlib
4
+ import inspect
5
+
6
+
7
+ class IncludeFunctions(Directive):
8
+ required_arguments = 1 # fully-qualified name
9
+
10
+ def run(self):
11
+ fqname = self.arguments[0]
12
+ parts = fqname.split(".")
13
+
14
+ # --- progressively import the longest valid module ---
15
+ module = None
16
+ for i in range(len(parts), 0, -1):
17
+ try:
18
+ module = importlib.import_module(".".join(parts[:i]))
19
+ rest = parts[i:]
20
+ break
21
+ except ImportError:
22
+ continue
23
+
24
+ if module is None:
25
+ return []
26
+
27
+ obj = module
28
+ for part in rest:
29
+ if hasattr(obj, part):
30
+ obj = getattr(obj, part)
31
+ else:
32
+ return []
33
+
34
+ # --- Collect all functions ---
35
+ functions = {} # {func_name: {'signature': ..., 'doc': ...}}
36
+
37
+ # Get all members
38
+ for name, member in inspect.getmembers(obj):
39
+ # Skip underscore-prefixed and special functions
40
+ if self._should_skip_function(name):
41
+ continue
42
+
43
+ # Check if it's a function or method
44
+ if inspect.isfunction(member) or inspect.ismethod(member):
45
+ doc = self._extract_summary_doc(member)
46
+ functions[name] = {
47
+ 'signature': self._get_signature(member),
48
+ 'doc': doc
49
+ }
50
+
51
+ # --- Build bullet list ---
52
+ if not functions:
53
+ return []
54
+
55
+ bullet_list = nodes.bullet_list()
56
+ for func_name in sorted(functions.keys()):
57
+ func_info = functions[func_name]
58
+ list_item = nodes.list_item()
59
+ para = nodes.paragraph()
60
+
61
+ # Function name as :py:func: role for clickable links
62
+ from sphinx.addnodes import pending_xref
63
+ refnode = pending_xref(
64
+ '',
65
+ refdomain='py',
66
+ reftype='func',
67
+ reftarget=fqname + '.' + func_name,
68
+ refwarn=True
69
+ )
70
+ # refnode += nodes.literal('', f"{func_name}{func_info['signature']}", classes=['xref', 'py', 'py-func'])
71
+ refnode += nodes.literal('', f"{func_name}()", classes=['xref', 'py', 'py-func'])
72
+ para += refnode
73
+
74
+ # Docstring
75
+ if func_info['doc']:
76
+ para += nodes.Text(f": {func_info['doc']}")
77
+
78
+ list_item += para
79
+ bullet_list += list_item
80
+
81
+ return [bullet_list]
82
+
83
+ def _should_skip_function(self, func_name):
84
+ """Check if a function should be skipped."""
85
+ return (
86
+ func_name.startswith('_') or
87
+ func_name.startswith('__')
88
+ )
89
+
90
+ def _get_signature(self, func):
91
+ """Get the function signature as a string."""
92
+ try:
93
+ sig = inspect.signature(func)
94
+ return str(sig)
95
+ except (ValueError, TypeError):
96
+ return "()"
97
+
98
+ def _extract_summary_doc(self, func):
99
+ """Extract the summary part of the docstring, stopping at section headers."""
100
+ # Get raw docstring
101
+ doc = getattr(func, "__doc__", None)
102
+ if not doc:
103
+ return None
104
+
105
+ # Clean and process the docstring
106
+ doc = inspect.cleandoc(doc)
107
+
108
+ section_headers = [
109
+ "Parameters", "Returns", "Postcondition", "Raises",
110
+ "Yields", "Examples", "Notes", "See Also", "Attributes",
111
+ "Methods", "References", "Warnings", "Args", "Return",
112
+ "Keyword Arguments", "Other Parameters"
113
+ ]
114
+
115
+ lines = doc.split('\n')
116
+ summary_lines = []
117
+
118
+ for line in lines:
119
+ stripped = line.strip()
120
+
121
+ # Stop at empty line followed by section header pattern
122
+ if not stripped:
123
+ # Check if we've already collected some content
124
+ if summary_lines:
125
+ # Peek ahead to see if next non-empty line is a section
126
+ continue
127
+
128
+ # Check if this line is a section header
129
+ if stripped in section_headers:
130
+ break
131
+
132
+ # Check if this line ends with a colon and might be a section header
133
+ if stripped.endswith(':'):
134
+ header_candidate = stripped[:-1].strip()
135
+ if header_candidate in section_headers:
136
+ break
137
+
138
+ # Check for common docstring section patterns (underlined headers)
139
+ if stripped and all(c in '-=~' for c in stripped):
140
+ break
141
+
142
+ summary_lines.append(line)
143
+
144
+ # Join lines and clean up
145
+ summary = '\n'.join(summary_lines).strip()
146
+
147
+ # Replace multiple newlines/spaces with single space for better formatting
148
+ summary = ' '.join(summary.split())
149
+
150
+ return summary if summary else None
151
+
152
+
153
+ def setup(app):
154
+ app.add_directive("include-functions", IncludeFunctions)
@@ -0,0 +1,131 @@
1
+ from docutils import nodes
2
+ from docutils.parsers.rst import Directive, directives
3
+ from docutils.statemachine import StringList
4
+ import json
5
+ import os
6
+ import sys
7
+ from paths import PROJECT_ROOT
8
+
9
+ class IncludeNotebook(Directive):
10
+ """
11
+ Directive to include marked sections from Jupyter notebooks.
12
+
13
+ Usage:
14
+ .. include-notebook:: path/to/notebook.ipynb
15
+ :name: example_mac
16
+ :language: python
17
+ """
18
+
19
+ required_arguments = 1 # path to notebook
20
+ option_spec = {
21
+ "name": directives.unchanged_required,
22
+ "language": directives.unchanged,
23
+ }
24
+
25
+ def run(self):
26
+ env = self.state.document.settings.env
27
+ notebook_path = self.arguments[0]
28
+ marker_name = self.options.get("name")
29
+ language = self.options.get("language", "python")
30
+
31
+ notebook_path = os.path.normpath(
32
+ os.path.join(PROJECT_ROOT, notebook_path)
33
+ )
34
+
35
+ # Resolve relative paths
36
+ if not os.path.isabs(notebook_path):
37
+ source_dir = os.path.dirname(self.state.document.current_source)
38
+ notebook_path = os.path.normpath(os.path.join(source_dir, notebook_path))
39
+
40
+ if not os.path.exists(notebook_path):
41
+ error = self.state_machine.reporter.error(
42
+ f"Notebook file not found: {notebook_path}",
43
+ nodes.literal_block("", ""),
44
+ line=self.lineno,
45
+ )
46
+ return [error]
47
+
48
+ # Read and parse the notebook
49
+ try:
50
+ with open(notebook_path, "r", encoding="utf-8") as f:
51
+ notebook = json.load(f)
52
+ except Exception as e:
53
+ error = self.state_machine.reporter.error(
54
+ f"Error reading notebook: {e}",
55
+ nodes.literal_block("", ""),
56
+ line=self.lineno,
57
+ )
58
+ return [error]
59
+
60
+ # Extract all marked sections
61
+ content_sections = self._extract_marked_sections(notebook, marker_name)
62
+
63
+ if not content_sections:
64
+ warning = self.state_machine.reporter.warning(
65
+ f'Marker "# < DOC_INCLUDE_MARKER > {marker_name}" not found in {notebook_path}',
66
+ nodes.literal_block("", ""),
67
+ line=self.lineno,
68
+ )
69
+ return [warning]
70
+
71
+ # Concatenate all sections with double newline separator
72
+ content = "\n\n".join(content_sections)
73
+
74
+ # Remove newlines at the beginning and end
75
+ content = content.strip()
76
+
77
+ # Create a code block node
78
+ code_block = nodes.literal_block(content, content)
79
+ code_block["language"] = language
80
+ return [code_block]
81
+
82
+ def _extract_marked_sections(self, notebook, marker_name):
83
+ """
84
+ Extract content between DOC_INCLUDE_MARKER and DOC_INCLUDE_END (or end of cell).
85
+ Returns a list of all matching sections.
86
+ """
87
+ start_marker = f"# < DOC_INCLUDE_MARKER > {marker_name}"
88
+ end_marker = "# < DOC_INCLUDE_END >"
89
+ sections = []
90
+
91
+ for cell_idx, cell in enumerate(notebook.get("cells", [])):
92
+ if cell.get("cell_type") != "code":
93
+ continue
94
+
95
+ source = cell.get("source", [])
96
+
97
+ # Handle both string and list formats
98
+ if isinstance(source, str):
99
+ lines = source.split("\n")
100
+ else:
101
+ lines = source
102
+
103
+ # Look for all start markers in this cell
104
+ for i, line in enumerate(lines):
105
+ # Remove trailing newlines for comparison
106
+ line_stripped = line.rstrip("\n")
107
+
108
+ if start_marker in line_stripped:
109
+ # Found the start marker, collect lines until end marker or end of cell
110
+ content_lines = []
111
+ for j in range(i + 1, len(lines)):
112
+ if end_marker in lines[j]:
113
+ break
114
+ content_lines.append(lines[j].rstrip("\n"))
115
+
116
+ # Join lines and strip trailing whitespace
117
+ content = "\n".join(content_lines).strip()
118
+ if content: # Only add non-empty sections
119
+ sections.append(content)
120
+
121
+ return sections
122
+
123
+
124
+ def setup(app):
125
+ app.add_directive("include-notebook", IncludeNotebook)
126
+
127
+ return {
128
+ "version": "0.1",
129
+ "parallel_read_safe": True,
130
+ "parallel_write_safe": True,
131
+ }
@@ -0,0 +1,119 @@
1
+ from docutils import nodes
2
+ from docutils.parsers.rst import Directive, directives
3
+ from docutils.statemachine import StringList
4
+ import re
5
+
6
+ from paths import PROJECT_ROOT
7
+ import os
8
+
9
+
10
+ def grab_from_yaml_file(
11
+ yaml_file: str,
12
+ startfrom: str | None = None,
13
+ same_indent: bool = True,
14
+ include_lines_before: int = 0,
15
+ ) -> str:
16
+ """
17
+ Grab a section from a YAML file.
18
+
19
+ Args:
20
+ yaml_file:
21
+ The path to the YAML file.
22
+ startfrom:
23
+ The string to start from. If None, return the entire file.
24
+ same_indent:
25
+ Whether to include lines with the same indentation as the startfrom line, or
26
+ only lines with >= indentation.
27
+ include_lines_before:
28
+ The number of lines to include before the startfrom line.
29
+
30
+ Returns:
31
+ The section of the YAML file as a string.
32
+ """
33
+ with open(yaml_file, "r") as f:
34
+ contents = f.readlines()
35
+ start, end = 0, len(contents)
36
+ n_whitespace = 0
37
+
38
+ if startfrom is not None:
39
+ for i, line in enumerate(contents):
40
+ if re.findall(r"\b\s*" + startfrom + r"\b", line):
41
+ start = i
42
+ n_whitespace = len(re.findall(r"^\s*", line)[0])
43
+ break
44
+ else:
45
+ raise ValueError(f"{startfrom} not found in {yaml_file}")
46
+ for i, line in enumerate(contents[start + 1 :]):
47
+ if not line.strip():
48
+ continue
49
+ ws = len(re.findall(r"^\s*", line)[0])
50
+ if ws < n_whitespace or (not same_indent and ws == n_whitespace):
51
+ end = start + i + 1
52
+ break
53
+
54
+ contents = [c[n_whitespace:] for c in contents[start - include_lines_before : end]]
55
+ return "".join(contents)
56
+
57
+
58
+ class IncludeYaml(Directive):
59
+ """
60
+ Directive to include content from a YAML file with optional filtering.
61
+
62
+ Usage:
63
+ .. include-yaml:: path/to/file.yaml
64
+ :startfrom: section_name
65
+ :same-indent:
66
+ :include-lines-before: 2
67
+ """
68
+
69
+ required_arguments = 1 # The YAML file path
70
+ optional_arguments = 0
71
+ option_spec = {
72
+ 'startfrom': directives.unchanged,
73
+ 'same-indent': directives.flag,
74
+ 'include-lines-before': directives.nonnegative_int,
75
+ }
76
+ has_content = False
77
+
78
+ def run(self):
79
+ yaml_file = os.path.join(PROJECT_ROOT, self.arguments[0])
80
+ startfrom = self.options.get('startfrom', None)
81
+ same_indent = 'same-indent' in self.options
82
+ include_lines_before = self.options.get('include-lines-before', 0)
83
+
84
+ try:
85
+ content = grab_from_yaml_file(
86
+ yaml_file=yaml_file,
87
+ startfrom=startfrom,
88
+ same_indent=same_indent,
89
+ include_lines_before=include_lines_before,
90
+ )
91
+ except Exception as e:
92
+ error = self.state_machine.reporter.error(
93
+ f'Error reading YAML file "{yaml_file}": {str(e)}',
94
+ nodes.literal_block('', ''),
95
+ line=self.lineno
96
+ )
97
+ return [error]
98
+
99
+ # Create a literal block with YAML syntax highlighting
100
+ literal = nodes.literal_block(content, content)
101
+ literal['language'] = 'yaml'
102
+
103
+ return [literal]
104
+
105
+
106
+ def setup(app):
107
+ """
108
+ Setup function for the Sphinx extension.
109
+
110
+ Add this to your conf.py:
111
+ extensions = ['path.to.this.module']
112
+ """
113
+ app.add_directive('include-yaml', IncludeYaml)
114
+
115
+ return {
116
+ 'version': '0.1',
117
+ 'parallel_read_safe': True,
118
+ 'parallel_write_safe': True,
119
+ }
@@ -0,0 +1,222 @@
1
+ from sphinx.ext.autodoc import ClassDocumenter, INSTANCEATTR, ObjectMember
2
+ from sphinx.util.inspect import safe_getattr
3
+ import inspect
4
+ from pydantic import BaseModel
5
+
6
+
7
+ # Default ignore lists
8
+ DEFAULT_IGNORE = [
9
+ 'object',
10
+ 'ABC',
11
+ 'BaseModel',
12
+ 'Enum',
13
+ 'Flag',
14
+ 'IntEnum',
15
+ 'IntFlag',
16
+ 'Generic',
17
+ 'object',
18
+ 'ABC',
19
+ 'BaseModel',
20
+ 'ParsableModel',
21
+ 'ParsableList',
22
+ 'ParsableDict',
23
+ 'ParseExtras',
24
+ 'NonParsableModel',
25
+ '_FromYAMLAble',
26
+ 'ParsesTo',
27
+ 'Enum',
28
+ 'Flag',
29
+ 'IntEnum',
30
+ 'IntFlag',
31
+ 'Generic',
32
+ 'model_validate',
33
+ 'model_validate_json',
34
+ 'model_validate_strings',
35
+ 'parse_file',
36
+ 'parse_obj',
37
+ 'parse_raw',
38
+ 'schema',
39
+ 'schema_json',
40
+ 'model_dump',
41
+ 'model_dump_json',
42
+ 'model_copy',
43
+ 'construct',
44
+ 'copy',
45
+ 'dict',
46
+ 'json',
47
+ 'update_forward_refs',
48
+ 'model_post_init',
49
+ 'model_config',
50
+ 'model_fields',
51
+ 'model_fields_set',
52
+ 'model_extra',
53
+ 'model_private',
54
+ 'model_rebuild',
55
+ 'get_fields',
56
+ 'get_validator',
57
+ 'all_fields_default',
58
+ 'model_dump_non_none',
59
+ ] + list(BaseModel.__dict__.keys())
60
+
61
+ class InheritedAttributesClassDocumenter(ClassDocumenter):
62
+ """Enhanced ClassDocumenter that includes inherited attributes."""
63
+
64
+ priority = ClassDocumenter.priority + 1
65
+
66
+ def get_object_members(self, want_all):
67
+ """Override to include inherited members."""
68
+ # Get the normal members first
69
+ members_check_module, members = super().get_object_members(want_all)
70
+
71
+ # Get configuration
72
+ config = self.env.config
73
+ ignore = getattr(config, 'inherited_attributes_ignore', []) + DEFAULT_IGNORE
74
+
75
+ # Collect ALL attributes defined directly on this class (not inherited)
76
+ own_members = set()
77
+ if hasattr(self.object, '__dict__'):
78
+ own_members.update(self.object.__dict__.keys())
79
+ if hasattr(self.object, 'model_fields'):
80
+ own_members.update(self.object.model_fields.keys())
81
+ if hasattr(self.object, '__fields__'):
82
+ own_members.update(self.object.__fields__.keys())
83
+
84
+ # Filter out members from ignored classes that were already included
85
+ # BUT keep them if they're overridden in the current class
86
+ filtered_members = []
87
+ for member in members:
88
+ if hasattr(member, '__getitem__') and len(member) >= 4:
89
+ name = member[0]
90
+ member_class = member[3] if len(member) > 3 else None
91
+
92
+ # Keep if it's defined directly on this class
93
+ if name in own_members:
94
+ filtered_members.append(member)
95
+ continue
96
+
97
+ # Skip if from an ignored class (and not overridden)
98
+ if member_class and self._should_ignore_class(member_class, ignore):
99
+ continue
100
+
101
+ # Skip ignored functions/attributes (only if not overridden)
102
+ if name in ignore:
103
+ continue
104
+
105
+ filtered_members.append(member)
106
+ else:
107
+ filtered_members.append(member)
108
+
109
+ members = filtered_members
110
+
111
+ # Get MRO
112
+ try:
113
+ mro = inspect.getmro(self.object)
114
+ except (AttributeError, TypeError):
115
+ return members_check_module, members
116
+
117
+ # Track seen names - includes both members from super() and own_members
118
+ # ObjectMember is a namedtuple, access by index
119
+ seen_names = set()
120
+ for member in members:
121
+ if hasattr(member, '__getitem__'):
122
+ seen_names.add(member[0])
123
+ # Add all own members to seen_names to prevent inheritance
124
+ seen_names.update(own_members)
125
+
126
+ # Collect inherited members
127
+ inherited = []
128
+
129
+ for parent_class in mro[1:]: # Skip self
130
+ # Check if we should stop FIRST, before processing any members
131
+ if self._should_ignore_class(parent_class, ignore):
132
+ break
133
+
134
+ # Get members from parent
135
+ for name in dir(parent_class):
136
+ # Skip underscore-prefixed
137
+ if name.startswith('_'):
138
+ continue
139
+
140
+ # Skip if already seen (including own_members)
141
+ if name in seen_names:
142
+ continue
143
+
144
+ # Skip ignored
145
+ if name in ignore:
146
+ continue
147
+
148
+ try:
149
+ obj = safe_getattr(parent_class, name)
150
+
151
+ # Skip callables in ignore list
152
+ if callable(obj) and name in ignore:
153
+ continue
154
+
155
+ # Create ObjectMember
156
+ inherited.append(ObjectMember(name, obj))
157
+ seen_names.add(name)
158
+
159
+ except (AttributeError, TypeError):
160
+ continue
161
+
162
+ # Check Pydantic fields
163
+ if hasattr(parent_class, 'model_fields'):
164
+ for field_name in parent_class.model_fields:
165
+ if field_name.startswith('_'):
166
+ continue
167
+ # Skip if already seen (including own_members)
168
+ if field_name in seen_names:
169
+ continue
170
+ if field_name in ignore:
171
+ continue
172
+
173
+ inherited.append(ObjectMember(field_name, INSTANCEATTR))
174
+ seen_names.add(field_name)
175
+
176
+ elif hasattr(parent_class, '__fields__'):
177
+ for field_name in parent_class.__fields__:
178
+ if field_name.startswith('_'):
179
+ continue
180
+ # Skip if already seen (including own_members)
181
+ if field_name in seen_names:
182
+ continue
183
+ if field_name in ignore:
184
+ continue
185
+
186
+ inherited.append(ObjectMember(field_name, INSTANCEATTR))
187
+ seen_names.add(field_name)
188
+
189
+ # Combine
190
+ all_members = list(members) + inherited
191
+
192
+ return members_check_module, all_members
193
+
194
+ def _should_ignore_class(self, cls, ignore_classes):
195
+ """Check if a class should be ignored."""
196
+ name = getattr(cls, '__name__', '')
197
+ module = getattr(cls, '__module__', '')
198
+
199
+ # Ignore if not part of accelforge package
200
+ if module and not module.startswith('accelforge'):
201
+ return True
202
+
203
+ # Check against ignore list using just the class name
204
+ for ignore_pattern in ignore_classes:
205
+ if name == ignore_pattern:
206
+ return True
207
+
208
+ return False
209
+
210
+ def setup(app):
211
+ """Setup the extension."""
212
+ # Only add config value if it doesn't exist
213
+ if not hasattr(app.config, 'inherited_attributes_ignore'):
214
+ app.add_config_value('inherited_attributes_ignore', [], 'env')
215
+
216
+ app.add_autodocumenter(InheritedAttributesClassDocumenter, override=True)
217
+
218
+ return {
219
+ 'version': '0.1',
220
+ 'parallel_read_safe': True,
221
+ 'parallel_write_safe': True,
222
+ }
@@ -0,0 +1,4 @@
1
+ import os
2
+ from pathlib import Path
3
+
4
+ PROJECT_ROOT = os.path.abspath(Path(__file__).parent.parent.parent.parent)