mapFolding 0.8.6__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapFolding/__init__.py +60 -13
- mapFolding/basecamp.py +32 -17
- mapFolding/beDRY.py +3 -3
- mapFolding/oeis.py +83 -2
- mapFolding/someAssemblyRequired/__init__.py +48 -27
- mapFolding/someAssemblyRequired/_theTypes.py +11 -15
- mapFolding/someAssemblyRequired/_tool_Make.py +35 -8
- mapFolding/someAssemblyRequired/_tool_Then.py +59 -25
- mapFolding/someAssemblyRequired/_toolboxAntecedents.py +151 -276
- mapFolding/someAssemblyRequired/_toolboxContainers.py +133 -48
- mapFolding/someAssemblyRequired/_toolboxPython.py +165 -44
- mapFolding/someAssemblyRequired/synthesizeNumbaJob.py +101 -18
- mapFolding/someAssemblyRequired/toolboxNumba.py +83 -48
- mapFolding/someAssemblyRequired/transformationTools.py +220 -138
- mapFolding/theSSOT.py +147 -54
- mapFolding/toolboxFilesystem.py +1 -1
- mapfolding-0.9.0.dist-info/METADATA +177 -0
- mapfolding-0.9.0.dist-info/RECORD +46 -0
- tests/__init__.py +44 -0
- tests/conftest.py +75 -7
- tests/test_computations.py +90 -9
- tests/test_filesystem.py +32 -33
- tests/test_other.py +0 -1
- tests/test_tasks.py +1 -1
- mapFolding/someAssemblyRequired/newInliner.py +0 -22
- mapfolding-0.8.6.dist-info/METADATA +0 -190
- mapfolding-0.8.6.dist-info/RECORD +0 -47
- {mapfolding-0.8.6.dist-info → mapfolding-0.9.0.dist-info}/WHEEL +0 -0
- {mapfolding-0.8.6.dist-info → mapfolding-0.9.0.dist-info}/entry_points.txt +0 -0
- {mapfolding-0.8.6.dist-info → mapfolding-0.9.0.dist-info}/licenses/LICENSE +0 -0
- {mapfolding-0.8.6.dist-info → mapfolding-0.9.0.dist-info}/top_level.txt +0 -0
|
@@ -1,8 +1,27 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""
|
|
2
|
+
Job-specific Numba Code Generation for Map Folding Calculations
|
|
3
|
+
|
|
4
|
+
This module specializes in generating highly-optimized, single-purpose Numba modules
|
|
5
|
+
for specific map folding calculation jobs. Unlike the general-purpose transformation
|
|
6
|
+
in toolboxNumba.py, this module creates standalone Python modules optimized for a
|
|
7
|
+
single map shape with statically-encoded parameters.
|
|
8
|
+
|
|
9
|
+
The code generation pipeline focuses on:
|
|
10
|
+
|
|
11
|
+
1. Converting function parameters to initialized variables with concrete values.
|
|
12
|
+
2. Replacing dynamic computations with statically-known values.
|
|
13
|
+
3. Eliminating unused code paths and variables.
|
|
14
|
+
4. Adding progress tracking for long-running calculations.
|
|
15
|
+
5. Applying appropriate Numba optimizations for the specific calculation.
|
|
16
|
+
|
|
17
|
+
This creates extremely fast, specialized implementations that can be run directly
|
|
18
|
+
as Python scripts or further compiled into standalone executables.
|
|
19
|
+
"""
|
|
20
|
+
|
|
2
21
|
from mapFolding.toolboxFilesystem import getPathFilenameFoldsTotal
|
|
3
|
-
from mapFolding.someAssemblyRequired import ast_Identifier,
|
|
22
|
+
from mapFolding.someAssemblyRequired import ast_Identifier, ifThis, Make, NodeChanger, Then, IngredientsFunction, IngredientsModule, LedgerOfImports
|
|
4
23
|
from mapFolding.someAssemblyRequired.toolboxNumba import RecipeJob, SpicesJobNumba, decorateCallableWithNumba
|
|
5
|
-
from mapFolding.someAssemblyRequired.transformationTools import
|
|
24
|
+
from mapFolding.someAssemblyRequired.transformationTools import extractFunctionDef, write_astModule
|
|
6
25
|
from mapFolding.someAssemblyRequired.transformationTools import makeInitializedComputationState
|
|
7
26
|
from mapFolding.theSSOT import The, raiseIfNoneGitHubIssueNumber3
|
|
8
27
|
from mapFolding.oeis import getFoldsTotalKnown
|
|
@@ -10,6 +29,7 @@ from typing import cast
|
|
|
10
29
|
from Z0Z_tools import autoDecodingRLE
|
|
11
30
|
from pathlib import PurePosixPath
|
|
12
31
|
import ast
|
|
32
|
+
"""Synthesize one file to compute `foldsTotal` of `mapShape`."""
|
|
13
33
|
|
|
14
34
|
list_IdentifiersNotUsedAllHARDCODED = ['concurrencyLimit', 'foldsTotal', 'mapShape',]
|
|
15
35
|
list_IdentifiersNotUsedParallelSequentialHARDCODED = ['indexLeaf']
|
|
@@ -22,7 +42,30 @@ list_IdentifiersStaticValuesHARDCODED = ['dimensionsTotal', 'leavesTotal',]
|
|
|
22
42
|
list_IdentifiersNotUsedHARDCODED = list_IdentifiersStaticValuesHARDCODED + list_IdentifiersReplacedHARDCODED + list_IdentifiersNotUsedAllHARDCODED + list_IdentifiersNotUsedParallelSequentialHARDCODED + list_IdentifiersNotUsedSequentialHARDCODED
|
|
23
43
|
|
|
24
44
|
def addLauncherNumbaProgress(ingredientsModule: IngredientsModule, ingredientsFunction: IngredientsFunction, job: RecipeJob, spices: SpicesJobNumba) -> tuple[IngredientsModule, IngredientsFunction]:
|
|
45
|
+
"""
|
|
46
|
+
Add progress tracking capabilities to a Numba-optimized function.
|
|
47
|
+
|
|
48
|
+
This function modifies both the module and the function to integrate Numba-compatible
|
|
49
|
+
progress tracking for long-running calculations. It performs several key transformations:
|
|
50
|
+
|
|
51
|
+
1. Adds a progress bar parameter to the function signature
|
|
52
|
+
2. Replaces counting increments with progress bar updates
|
|
53
|
+
3. Creates a launcher section that displays and updates progress
|
|
54
|
+
4. Configures file output to save results upon completion
|
|
55
|
+
|
|
56
|
+
The progress tracking is particularly important for map folding calculations
|
|
57
|
+
which can take hours or days to complete, providing visual feedback and
|
|
58
|
+
estimated completion times.
|
|
25
59
|
|
|
60
|
+
Parameters:
|
|
61
|
+
ingredientsModule: The module where the function is defined.
|
|
62
|
+
ingredientsFunction: The function to modify with progress tracking.
|
|
63
|
+
job: Configuration specifying shape details and output paths.
|
|
64
|
+
spices: Configuration specifying progress bar details.
|
|
65
|
+
|
|
66
|
+
Returns:
|
|
67
|
+
A tuple containing the modified module and function with progress tracking.
|
|
68
|
+
"""
|
|
26
69
|
linesLaunch: str = f"""
|
|
27
70
|
if __name__ == '__main__':
|
|
28
71
|
with ProgressBar(total={job.foldsTotalEstimated}, update_interval=2) as statusUpdate:
|
|
@@ -43,7 +86,7 @@ if __name__ == '__main__':
|
|
|
43
86
|
|
|
44
87
|
findThis = ifThis.isAugAssign_targetIs(ifThis.isName_Identifier(job.shatteredDataclass.countingVariableName.id))
|
|
45
88
|
doThat = Then.replaceWith(Make.Expr(Make.Call(Make.Attribute(Make.Name(spices.numbaProgressBarIdentifier),'update'),[Make.Constant(1)])))
|
|
46
|
-
countWithProgressBar = NodeChanger(findThis, doThat)
|
|
89
|
+
countWithProgressBar = NodeChanger(findThis, doThat)
|
|
47
90
|
countWithProgressBar.visit(ingredientsFunction.astFunctionDef)
|
|
48
91
|
|
|
49
92
|
ingredientsModule.appendLauncher(ast.parse(linesLaunch))
|
|
@@ -51,12 +94,35 @@ if __name__ == '__main__':
|
|
|
51
94
|
return ingredientsModule, ingredientsFunction
|
|
52
95
|
|
|
53
96
|
def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: IngredientsFunction, job: RecipeJob) -> IngredientsFunction:
|
|
97
|
+
"""
|
|
98
|
+
Convert function parameters into initialized variables with concrete values.
|
|
99
|
+
|
|
100
|
+
This function implements a critical transformation that converts function parameters
|
|
101
|
+
into statically initialized variables in the function body. This enables several
|
|
102
|
+
optimizations:
|
|
103
|
+
|
|
104
|
+
1. Eliminating parameter passing overhead.
|
|
105
|
+
2. Embedding concrete values directly in the code.
|
|
106
|
+
3. Allowing Numba to optimize based on known value characteristics.
|
|
107
|
+
4. Simplifying function signatures for specialized use cases.
|
|
108
|
+
|
|
109
|
+
The function handles different data types (scalars, arrays, custom types) appropriately,
|
|
110
|
+
replacing abstract parameter references with concrete values from the computation state.
|
|
111
|
+
It also removes unused parameters and variables to eliminate dead code.
|
|
112
|
+
|
|
113
|
+
Parameters:
|
|
114
|
+
ingredientsFunction: The function to transform.
|
|
115
|
+
job: Recipe containing concrete values for parameters and field metadata.
|
|
116
|
+
|
|
117
|
+
Returns:
|
|
118
|
+
The modified function with parameters converted to initialized variables.
|
|
119
|
+
"""
|
|
54
120
|
ingredientsFunction.imports.update(job.shatteredDataclass.ledger)
|
|
55
121
|
|
|
56
122
|
list_IdentifiersNotUsed = list_IdentifiersNotUsedHARDCODED
|
|
57
123
|
|
|
58
|
-
|
|
59
|
-
for ast_arg in
|
|
124
|
+
list_argCuzMyBrainRefusesToThink = ingredientsFunction.astFunctionDef.args.args + ingredientsFunction.astFunctionDef.args.posonlyargs + ingredientsFunction.astFunctionDef.args.kwonlyargs
|
|
125
|
+
for ast_arg in list_argCuzMyBrainRefusesToThink:
|
|
60
126
|
if ast_arg.arg in job.shatteredDataclass.field2AnnAssign:
|
|
61
127
|
if ast_arg.arg in list_IdentifiersNotUsed:
|
|
62
128
|
pass
|
|
@@ -64,11 +130,11 @@ def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: Ingre
|
|
|
64
130
|
ImaAnnAssign, elementConstructor = job.shatteredDataclass.Z0Z_field2AnnAssign[ast_arg.arg]
|
|
65
131
|
match elementConstructor:
|
|
66
132
|
case 'scalar':
|
|
67
|
-
ImaAnnAssign.value.args[0].value = int(job.state.__dict__[ast_arg.arg])
|
|
133
|
+
ImaAnnAssign.value.args[0].value = int(job.state.__dict__[ast_arg.arg]) # type: ignore
|
|
68
134
|
case 'array':
|
|
69
135
|
dataAsStrRLE: str = autoDecodingRLE(job.state.__dict__[ast_arg.arg], addSpaces=True)
|
|
70
136
|
dataAs_astExpr: ast.expr = cast(ast.Expr, ast.parse(dataAsStrRLE).body[0]).value
|
|
71
|
-
ImaAnnAssign.value.args = [dataAs_astExpr]
|
|
137
|
+
ImaAnnAssign.value.args = [dataAs_astExpr] # type: ignore
|
|
72
138
|
case _:
|
|
73
139
|
list_exprDOTannotation: list[ast.expr] = []
|
|
74
140
|
list_exprDOTvalue: list[ast.expr] = []
|
|
@@ -87,14 +153,36 @@ def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: Ingre
|
|
|
87
153
|
ast.fix_missing_locations(ingredientsFunction.astFunctionDef)
|
|
88
154
|
return ingredientsFunction
|
|
89
155
|
|
|
90
|
-
def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba):
|
|
91
|
-
|
|
156
|
+
def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba) -> None:
|
|
157
|
+
"""
|
|
158
|
+
Generate a highly-optimized, single-purpose Numba module for a specific map shape.
|
|
159
|
+
|
|
160
|
+
This function implements the complete transformation pipeline for creating a
|
|
161
|
+
standalone, specialized implementation for calculating map folding solutions for
|
|
162
|
+
a specific shape. The process includes:
|
|
163
|
+
|
|
164
|
+
1. Extracting the counting function from the source module
|
|
165
|
+
2. Removing unused code paths based on static analysis
|
|
166
|
+
3. Replacing dynamic variables with concrete values
|
|
167
|
+
4. Converting parameters to initialized variables
|
|
168
|
+
5. Adding progress tracking if requested
|
|
169
|
+
6. Applying Numba optimizations and type specifications
|
|
170
|
+
7. Writing the final module to the filesystem
|
|
171
|
+
|
|
172
|
+
The resulting Python module is both human-readable and extraordinarily efficient,
|
|
173
|
+
with all shape-specific optimizations statically encoded. This creates specialized
|
|
174
|
+
implementations that can be orders of magnitude faster than general-purpose code.
|
|
175
|
+
|
|
176
|
+
Parameters:
|
|
177
|
+
job: Configuration specifying the target shape, paths, and computation state.
|
|
178
|
+
spices: Configuration specifying Numba and progress tracking options.
|
|
179
|
+
"""
|
|
92
180
|
astFunctionDef = extractFunctionDef(job.source_astModule, job.countCallable)
|
|
93
181
|
if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
|
|
94
182
|
ingredientsCount: IngredientsFunction = IngredientsFunction(astFunctionDef, LedgerOfImports())
|
|
95
183
|
|
|
96
184
|
# Change the return so you can dynamically determine which variables are not used
|
|
97
|
-
removeReturnStatement = NodeChanger(
|
|
185
|
+
removeReturnStatement = NodeChanger(lambda node: isinstance(node, ast.Return), Then.removeIt) # type: ignore
|
|
98
186
|
removeReturnStatement.visit(ingredientsCount.astFunctionDef)
|
|
99
187
|
ingredientsCount.astFunctionDef.returns = Make.Constant(value=None)
|
|
100
188
|
|
|
@@ -109,7 +197,7 @@ def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba):
|
|
|
109
197
|
for identifier in list_IdentifiersStaticValues:
|
|
110
198
|
findThis = ifThis.isName_Identifier(identifier)
|
|
111
199
|
doThat = Then.replaceWith(Make.Constant(int(job.state.__dict__[identifier])))
|
|
112
|
-
NodeChanger(findThis, doThat).visit(ingredientsCount.astFunctionDef)
|
|
200
|
+
NodeChanger(findThis, doThat).visit(ingredientsCount.astFunctionDef)
|
|
113
201
|
|
|
114
202
|
# This launcher eliminates the use of one identifier, so run it now and you can dynamically determine which variables are not used
|
|
115
203
|
ingredientsModule = IngredientsModule()
|
|
@@ -152,16 +240,11 @@ def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba):
|
|
|
152
240
|
Z0Z_type_name = 'uint8'
|
|
153
241
|
ingredientsCount.imports.addImportFrom_asStr(Z0Z_module, Z0Z_type_name, Z0Z_asname)
|
|
154
242
|
|
|
155
|
-
from numpy import int16 as Array1DLeavesTotal, int16 as Array1DElephino, int16 as Array3D
|
|
156
|
-
|
|
157
243
|
ingredientsCount.astFunctionDef.decorator_list = [] # TODO low-priority, handle this more elegantly
|
|
158
244
|
# TODO when I add the function signature in numba style back to the decorator, the logic needs to handle `ProgressBarType:`
|
|
159
245
|
ingredientsCount = decorateCallableWithNumba(ingredientsCount, spices.parametersNumba)
|
|
160
246
|
|
|
161
247
|
ingredientsModule.appendIngredientsFunction(ingredientsCount)
|
|
162
|
-
|
|
163
|
-
# add imports, make str, remove unused imports
|
|
164
|
-
# put on disk
|
|
165
248
|
write_astModule(ingredientsModule, job.pathFilenameModule, job.packageIdentifier)
|
|
166
249
|
|
|
167
250
|
"""
|
|
@@ -186,7 +269,7 @@ def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba):
|
|
|
186
269
|
"""
|
|
187
270
|
|
|
188
271
|
if __name__ == '__main__':
|
|
189
|
-
mapShape = (
|
|
272
|
+
mapShape = (2,4)
|
|
190
273
|
state = makeInitializedComputationState(mapShape)
|
|
191
274
|
foldsTotalEstimated = getFoldsTotalKnown(state.mapShape) // state.leavesTotal
|
|
192
275
|
pathModule = PurePosixPath(The.pathPackage, 'jobs')
|
|
@@ -1,46 +1,42 @@
|
|
|
1
1
|
"""
|
|
2
|
-
Numba-specific
|
|
2
|
+
Numba-specific Tools for Generating Optimized Code
|
|
3
3
|
|
|
4
|
-
This module provides specialized tools
|
|
5
|
-
|
|
4
|
+
This module provides specialized tools for transforming standard Python code into
|
|
5
|
+
Numba-accelerated implementations. It implements a comprehensive transformation
|
|
6
|
+
assembly-line that:
|
|
6
7
|
|
|
7
|
-
1.
|
|
8
|
-
2.
|
|
9
|
-
3.
|
|
10
|
-
4.
|
|
8
|
+
1. Converts dataclass-based algorithm implementations into Numba-compatible versions.
|
|
9
|
+
2. Applies appropriate Numba decorators with optimized configuration settings.
|
|
10
|
+
3. Restructures code to work within Numba's constraints.
|
|
11
|
+
4. Manages type information for optimized compilation.
|
|
11
12
|
|
|
12
|
-
The
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
of generic code transformation tools in the package, allowing similar approaches to be
|
|
16
|
-
applied to other acceleration technologies.
|
|
17
|
-
|
|
18
|
-
This module works in conjunction with transformation tools to convert the general-purpose
|
|
19
|
-
algorithm implementation into a highly-optimized Numba version.
|
|
13
|
+
The module bridges the gap between readable, maintainable Python code and
|
|
14
|
+
highly-optimized numerical computing implementations, enabling significant
|
|
15
|
+
performance improvements while preserving code semantics and correctness.
|
|
20
16
|
"""
|
|
21
17
|
|
|
22
18
|
from collections.abc import Callable, Sequence
|
|
23
|
-
from mapFolding.
|
|
24
|
-
from mapFolding.someAssemblyRequired import
|
|
25
|
-
from mapFolding.someAssemblyRequired.transformationTools import Z0Z_inlineThisFunctionWithTheseValues, Z0Z_lameFindReplace, Z0Z_makeDictionaryReplacementStatements, astModuleToIngredientsFunction, shatter_dataclassesDOTdataclass, write_astModule
|
|
19
|
+
from mapFolding.someAssemblyRequired import grab, IngredientsModule, LedgerOfImports, Make, NodeChanger, NodeTourist, RecipeSynthesizeFlow, Then, ast_Identifier, ifThis, parsePathFilename2astModule, str_nameDOTname, IngredientsFunction, ShatteredDataclass
|
|
20
|
+
from mapFolding.someAssemblyRequired.transformationTools import inlineFunctionDef, Z0Z_lameFindReplace, astModuleToIngredientsFunction, shatter_dataclassesDOTdataclass, write_astModule
|
|
26
21
|
from mapFolding.theSSOT import ComputationState, DatatypeFoldsTotal as TheDatatypeFoldsTotal, DatatypeElephino as TheDatatypeElephino, DatatypeLeavesTotal as TheDatatypeLeavesTotal
|
|
27
|
-
|
|
22
|
+
from mapFolding.toolboxFilesystem import getPathFilenameFoldsTotal, getPathRootJobDEFAULT
|
|
28
23
|
from numba.core.compiler import CompilerBase as numbaCompilerBase
|
|
29
24
|
from pathlib import Path, PurePosixPath
|
|
30
|
-
from typing import Any, cast, Final, TYPE_CHECKING, TypeAlias
|
|
25
|
+
from typing import Any, cast, Final, TYPE_CHECKING, TypeAlias, TypeGuard
|
|
31
26
|
import ast
|
|
32
27
|
import dataclasses
|
|
33
28
|
|
|
34
29
|
try:
|
|
35
30
|
from typing import NotRequired
|
|
36
31
|
except Exception:
|
|
37
|
-
from typing_extensions import NotRequired
|
|
32
|
+
from typing_extensions import NotRequired # pyright: ignore[reportShadowedImports]
|
|
38
33
|
|
|
39
34
|
if TYPE_CHECKING:
|
|
40
35
|
from typing import TypedDict
|
|
41
36
|
else:
|
|
42
37
|
TypedDict = dict[str,Any]
|
|
43
38
|
|
|
39
|
+
# Consolidate settings classes through inheritance https://github.com/hunterhogan/mapFolding/issues/15
|
|
44
40
|
theNumbaFlow: RecipeSynthesizeFlow = RecipeSynthesizeFlow()
|
|
45
41
|
|
|
46
42
|
class ParametersNumba(TypedDict):
|
|
@@ -91,7 +87,7 @@ def decorateCallableWithNumba(ingredientsFunction: IngredientsFunction, paramete
|
|
|
91
87
|
warnings.warn(f"Removed decorator {ast.unparse(decoratorItem)} from {astCallable.name}")
|
|
92
88
|
return astCallable
|
|
93
89
|
|
|
94
|
-
def makeSpecialSignatureForNumba(signatureElement: ast.arg) -> ast.Subscript | ast.Name | None: #
|
|
90
|
+
def makeSpecialSignatureForNumba(signatureElement: ast.arg) -> ast.Subscript | ast.Name | None: # pyright: ignore[reportUnusedFunction]
|
|
95
91
|
if isinstance(signatureElement.annotation, ast.Subscript) and isinstance(signatureElement.annotation.slice, ast.Tuple):
|
|
96
92
|
annotationShape: ast.expr = signatureElement.annotation.slice.elts[0]
|
|
97
93
|
if isinstance(annotationShape, ast.Subscript) and isinstance(annotationShape.slice, ast.Tuple):
|
|
@@ -152,18 +148,20 @@ def decorateCallableWithNumba(ingredientsFunction: IngredientsFunction, paramete
|
|
|
152
148
|
ingredientsFunction.astFunctionDef.decorator_list = [astDecorator]
|
|
153
149
|
return ingredientsFunction
|
|
154
150
|
|
|
151
|
+
# Consolidate settings classes through inheritance https://github.com/hunterhogan/mapFolding/issues/15
|
|
155
152
|
@dataclasses.dataclass
|
|
156
153
|
class SpicesJobNumba:
|
|
157
154
|
useNumbaProgressBar: bool = True
|
|
158
155
|
numbaProgressBarIdentifier: ast_Identifier = 'ProgressBarGroupsOfFolds'
|
|
159
156
|
parametersNumba = parametersNumbaDefault
|
|
160
157
|
|
|
158
|
+
# Consolidate settings classes through inheritance https://github.com/hunterhogan/mapFolding/issues/15
|
|
161
159
|
@dataclasses.dataclass
|
|
162
160
|
class RecipeJob:
|
|
163
161
|
state: ComputationState
|
|
164
162
|
# TODO create function to calculate `foldsTotalEstimated`
|
|
165
163
|
foldsTotalEstimated: int = 0
|
|
166
|
-
shatteredDataclass: ShatteredDataclass = dataclasses.field(default=None, init=True) #
|
|
164
|
+
shatteredDataclass: ShatteredDataclass = dataclasses.field(default=None, init=True) # pyright: ignore[reportAssignmentType]
|
|
167
165
|
|
|
168
166
|
# ========================================
|
|
169
167
|
# Source
|
|
@@ -183,15 +181,14 @@ class RecipeJob:
|
|
|
183
181
|
pathModule: PurePosixPath | None = PurePosixPath(getPathRootJobDEFAULT())
|
|
184
182
|
""" `pathModule` will override `pathPackage` and `logicalPathRoot`."""
|
|
185
183
|
fileExtension: str = theNumbaFlow.fileExtension
|
|
186
|
-
pathFilenameFoldsTotal: PurePosixPath = dataclasses.field(default=None, init=True) #
|
|
184
|
+
pathFilenameFoldsTotal: PurePosixPath = dataclasses.field(default=None, init=True) # pyright: ignore[reportAssignmentType]
|
|
187
185
|
|
|
188
186
|
# ========================================
|
|
189
187
|
# Logical identifiers (as opposed to physical identifiers)
|
|
190
|
-
# ========================================
|
|
191
188
|
packageIdentifier: ast_Identifier | None = None
|
|
192
189
|
logicalPathRoot: str_nameDOTname | None = None
|
|
193
190
|
""" `logicalPathRoot` likely corresponds to a physical filesystem directory."""
|
|
194
|
-
moduleIdentifier: ast_Identifier = dataclasses.field(default=None, init=True) #
|
|
191
|
+
moduleIdentifier: ast_Identifier = dataclasses.field(default=None, init=True) # pyright: ignore[reportAssignmentType]
|
|
195
192
|
countCallable: ast_Identifier = sourceCountCallable
|
|
196
193
|
dataclassIdentifier: ast_Identifier | None = sourceDataclassIdentifier
|
|
197
194
|
dataclassInstance: ast_Identifier | None = sourceDataclassInstance
|
|
@@ -231,13 +228,13 @@ class RecipeJob:
|
|
|
231
228
|
def __post_init__(self):
|
|
232
229
|
pathFilenameFoldsTotal = PurePosixPath(getPathFilenameFoldsTotal(self.state.mapShape))
|
|
233
230
|
|
|
234
|
-
if self.moduleIdentifier is None: #
|
|
231
|
+
if self.moduleIdentifier is None: # pyright: ignore[reportUnnecessaryComparison]
|
|
235
232
|
self.moduleIdentifier = pathFilenameFoldsTotal.stem
|
|
236
233
|
|
|
237
|
-
if self.pathFilenameFoldsTotal is None: #
|
|
234
|
+
if self.pathFilenameFoldsTotal is None: # pyright: ignore[reportUnnecessaryComparison]
|
|
238
235
|
self.pathFilenameFoldsTotal = pathFilenameFoldsTotal
|
|
239
236
|
|
|
240
|
-
if self.shatteredDataclass is None and self.logicalPathModuleDataclass and self.dataclassIdentifier and self.dataclassInstance: #
|
|
237
|
+
if self.shatteredDataclass is None and self.logicalPathModuleDataclass and self.dataclassIdentifier and self.dataclassInstance: # pyright: ignore[reportUnnecessaryComparison]
|
|
241
238
|
self.shatteredDataclass = shatter_dataclassesDOTdataclass(self.logicalPathModuleDataclass, self.dataclassIdentifier, self.dataclassInstance)
|
|
242
239
|
|
|
243
240
|
# ========================================
|
|
@@ -253,7 +250,39 @@ class RecipeJob:
|
|
|
253
250
|
concurrencyManagerNamespace: ast_Identifier = sourceConcurrencyManagerNamespace
|
|
254
251
|
concurrencyManagerIdentifier: ast_Identifier = sourceConcurrencyManagerIdentifier
|
|
255
252
|
|
|
253
|
+
class be:
|
|
254
|
+
@staticmethod
|
|
255
|
+
def Call(node: ast.AST) -> TypeGuard[ast.Call]:
|
|
256
|
+
return isinstance(node, ast.Call)
|
|
257
|
+
@staticmethod
|
|
258
|
+
def Return(node: ast.AST) -> TypeGuard[ast.Return]:
|
|
259
|
+
return isinstance(node, ast.Return)
|
|
260
|
+
|
|
256
261
|
def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
262
|
+
"""
|
|
263
|
+
Transform standard Python algorithm code into optimized Numba implementations.
|
|
264
|
+
|
|
265
|
+
This function implements the complete transformation pipeline that converts
|
|
266
|
+
a conventional Python implementation into a high-performance Numba-accelerated
|
|
267
|
+
version. The process includes:
|
|
268
|
+
|
|
269
|
+
1. Extracting core algorithm functions from the source module
|
|
270
|
+
2. Inlining function calls to create self-contained implementations
|
|
271
|
+
3. Transforming dataclass access patterns for Numba compatibility
|
|
272
|
+
4. Applying appropriate Numba decorators with optimization settings
|
|
273
|
+
5. Generating a unified module with sequential and parallel implementations
|
|
274
|
+
6. Writing the transformed code to the filesystem with properly managed imports
|
|
275
|
+
|
|
276
|
+
The transformation preserves the logical structure and semantics of the original
|
|
277
|
+
implementation while making it compatible with Numba's constraints and
|
|
278
|
+
optimization capabilities. This creates a bridge between the general-purpose
|
|
279
|
+
implementation and the highly-optimized version needed for production use.
|
|
280
|
+
|
|
281
|
+
Parameters:
|
|
282
|
+
numbaFlow: Configuration object that specifies all aspects of the
|
|
283
|
+
transformation process, including source and target locations,
|
|
284
|
+
function and variable names, and output paths.
|
|
285
|
+
"""
|
|
257
286
|
# TODO a tool to automatically remove unused variables from the ArgumentsSpecification (return, and returns) _might_ be nice.
|
|
258
287
|
# Figure out dynamic flow control to synthesized modules https://github.com/hunterhogan/mapFolding/issues/4
|
|
259
288
|
|
|
@@ -265,14 +294,14 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
265
294
|
]
|
|
266
295
|
|
|
267
296
|
# Inline functions ========================================================
|
|
268
|
-
dictionaryReplacementStatements = Z0Z_makeDictionaryReplacementStatements(numbaFlow.source_astModule)
|
|
269
297
|
# NOTE Replacements statements are based on the identifiers in the _source_, so operate on the source identifiers.
|
|
270
|
-
ingredientsInitialize.astFunctionDef =
|
|
271
|
-
ingredientsParallel.astFunctionDef =
|
|
272
|
-
ingredientsSequential.astFunctionDef =
|
|
298
|
+
ingredientsInitialize.astFunctionDef = inlineFunctionDef(numbaFlow.sourceCallableInitialize, numbaFlow.source_astModule)
|
|
299
|
+
ingredientsParallel.astFunctionDef = inlineFunctionDef(numbaFlow.sourceCallableParallel, numbaFlow.source_astModule)
|
|
300
|
+
ingredientsSequential.astFunctionDef = inlineFunctionDef(numbaFlow.sourceCallableSequential, numbaFlow.source_astModule)
|
|
273
301
|
|
|
274
302
|
# assignRecipeIdentifiersToCallable. =============================
|
|
275
|
-
#
|
|
303
|
+
# Consolidate settings classes through inheritance https://github.com/hunterhogan/mapFolding/issues/15
|
|
304
|
+
# How can I use dataclass settings as the SSOT for specific actions? https://github.com/hunterhogan/mapFolding/issues/16
|
|
276
305
|
# NOTE reminder: you are updating these `ast.Name` here (and not in a more general search) because this is a
|
|
277
306
|
# narrow search for `ast.Call` so you won't accidentally replace unrelated `ast.Name`.
|
|
278
307
|
listFindReplace = [(numbaFlow.sourceCallableDispatcher, numbaFlow.callableDispatcher),
|
|
@@ -281,7 +310,7 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
281
310
|
(numbaFlow.sourceCallableSequential, numbaFlow.callableSequential),]
|
|
282
311
|
for ingredients in listAllIngredientsFunctions:
|
|
283
312
|
for source_Identifier, recipe_Identifier in listFindReplace:
|
|
284
|
-
updateCallName = NodeChanger(ifThis.isCall_Identifier(source_Identifier),
|
|
313
|
+
updateCallName = NodeChanger(ifThis.isCall_Identifier(source_Identifier), grab.funcAttribute(Then.replaceWith(Make.Name(recipe_Identifier))))
|
|
285
314
|
updateCallName.visit(ingredients.astFunctionDef)
|
|
286
315
|
|
|
287
316
|
ingredientsDispatcher.astFunctionDef.name = numbaFlow.callableDispatcher
|
|
@@ -295,13 +324,13 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
295
324
|
(numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.concurrencyManagerNamespace),]
|
|
296
325
|
for ingredients in listAllIngredientsFunctions:
|
|
297
326
|
for source_Identifier, recipe_Identifier in listFindReplace:
|
|
298
|
-
updateName = NodeChanger(ifThis.isName_Identifier(source_Identifier),
|
|
299
|
-
update_arg = NodeChanger(ifThis.isArgument_Identifier(source_Identifier),
|
|
327
|
+
updateName = NodeChanger(ifThis.isName_Identifier(source_Identifier) , grab.idAttribute(Then.replaceWith(recipe_Identifier)))
|
|
328
|
+
update_arg = NodeChanger(ifThis.isArgument_Identifier(source_Identifier), grab.argAttribute(Then.replaceWith(recipe_Identifier)))
|
|
300
329
|
updateName.visit(ingredients.astFunctionDef)
|
|
301
330
|
update_arg.visit(ingredients.astFunctionDef)
|
|
302
331
|
|
|
303
332
|
updateConcurrencyManager = NodeChanger(ifThis.isCallAttributeNamespace_Identifier(numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.sourceConcurrencyManagerIdentifier)
|
|
304
|
-
,
|
|
333
|
+
, grab.funcAttribute(Then.replaceWith(Make.Attribute(Make.Name(numbaFlow.concurrencyManagerNamespace), numbaFlow.concurrencyManagerIdentifier))))
|
|
305
334
|
updateConcurrencyManager.visit(ingredientsDispatcher.astFunctionDef)
|
|
306
335
|
|
|
307
336
|
# shatter Dataclass =======================================================
|
|
@@ -310,25 +339,24 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
310
339
|
shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, instance_Identifier)
|
|
311
340
|
ingredientsDispatcher.imports.update(shatteredDataclass.ledger)
|
|
312
341
|
|
|
342
|
+
# How can I use dataclass settings as the SSOT for specific actions? https://github.com/hunterhogan/mapFolding/issues/16
|
|
313
343
|
# Change callable parameters and Call to the callable at the same time ====
|
|
314
|
-
# TODO How can I use ast and/or other tools to ensure that when I change a callable, I also change the statements that call the callable?
|
|
315
|
-
# Asked differently, how do I integrate separate statements into a "subroutine", and that subroutine is "atomic/indivisible"?
|
|
316
344
|
# sequentialCallable =========================================================
|
|
317
345
|
ingredientsSequential.astFunctionDef.args = Make.argumentsSpecification(args=shatteredDataclass.list_argAnnotated4ArgumentsSpecification)
|
|
318
346
|
astCallSequentialCallable = Make.Call(Make.Name(numbaFlow.callableSequential), shatteredDataclass.listName4Parameters)
|
|
319
347
|
changeReturnSequentialCallable = NodeChanger(be.Return, Then.replaceWith(Make.Return(shatteredDataclass.fragments4AssignmentOrParameters)))
|
|
320
348
|
ingredientsSequential.astFunctionDef.returns = shatteredDataclass.signatureReturnAnnotation
|
|
321
|
-
replaceAssignSequentialCallable = NodeChanger(ifThis.
|
|
349
|
+
replaceAssignSequentialCallable = NodeChanger(ifThis.isAssignAndValueIs(ifThis.isCall_Identifier(numbaFlow.callableSequential)), Then.replaceWith(Make.Assign(listTargets=[shatteredDataclass.fragments4AssignmentOrParameters], value=astCallSequentialCallable)))
|
|
322
350
|
|
|
323
|
-
unpack4sequentialCallable = NodeChanger(ifThis.
|
|
324
|
-
repack4sequentialCallable = NodeChanger(ifThis.
|
|
351
|
+
unpack4sequentialCallable = NodeChanger(ifThis.isAssignAndValueIs(ifThis.isCall_Identifier(numbaFlow.callableSequential)), Then.insertThisAbove(shatteredDataclass.listUnpack))
|
|
352
|
+
repack4sequentialCallable = NodeChanger(ifThis.isAssignAndValueIs(ifThis.isCall_Identifier(numbaFlow.callableSequential)), Then.insertThisBelow([shatteredDataclass.repack]))
|
|
325
353
|
|
|
326
354
|
changeReturnSequentialCallable.visit(ingredientsSequential.astFunctionDef)
|
|
327
355
|
replaceAssignSequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
|
|
328
356
|
unpack4sequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
|
|
329
357
|
repack4sequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
|
|
330
358
|
|
|
331
|
-
ingredientsSequential.astFunctionDef = Z0Z_lameFindReplace(ingredientsSequential.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name)
|
|
359
|
+
ingredientsSequential.astFunctionDef = Z0Z_lameFindReplace(ingredientsSequential.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name)
|
|
332
360
|
|
|
333
361
|
# parallelCallable =========================================================
|
|
334
362
|
ingredientsParallel.astFunctionDef.args = Make.argumentsSpecification(args=shatteredDataclass.list_argAnnotated4ArgumentsSpecification)
|
|
@@ -336,20 +364,20 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
336
364
|
|
|
337
365
|
# NOTE I am dissatisfied with this logic for many reasons, including that it requires separate NodeCollector and NodeReplacer instances.
|
|
338
366
|
astCallConcurrencyResult: list[ast.Call] = []
|
|
339
|
-
get_astCallConcurrencyResult
|
|
367
|
+
get_astCallConcurrencyResult = NodeTourist(ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(getTheOtherRecord_damn)), getIt(astCallConcurrencyResult))
|
|
340
368
|
get_astCallConcurrencyResult.visit(ingredientsDispatcher.astFunctionDef)
|
|
341
|
-
replaceAssignParallelCallable = NodeChanger(ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(getTheOtherRecord_damn)),
|
|
369
|
+
replaceAssignParallelCallable = NodeChanger(ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(getTheOtherRecord_damn)), grab.valueAttribute(Then.replaceWith(astCallConcurrencyResult[0])))
|
|
342
370
|
replaceAssignParallelCallable.visit(ingredientsDispatcher.astFunctionDef)
|
|
343
371
|
changeReturnParallelCallable = NodeChanger(be.Return, Then.replaceWith(Make.Return(shatteredDataclass.countingVariableName)))
|
|
344
372
|
ingredientsParallel.astFunctionDef.returns = shatteredDataclass.countingVariableAnnotation
|
|
345
373
|
|
|
346
|
-
unpack4parallelCallable = NodeChanger(ifThis.
|
|
374
|
+
unpack4parallelCallable = NodeChanger(ifThis.isAssignAndValueIs(ifThis.isCallAttributeNamespace_Identifier(numbaFlow.concurrencyManagerNamespace, numbaFlow.concurrencyManagerIdentifier)), Then.insertThisAbove(shatteredDataclass.listUnpack))
|
|
347
375
|
|
|
348
376
|
unpack4parallelCallable.visit(ingredientsDispatcher.astFunctionDef)
|
|
349
377
|
replaceCall2concurrencyManager.visit(ingredientsDispatcher.astFunctionDef)
|
|
350
378
|
changeReturnParallelCallable.visit(ingredientsParallel.astFunctionDef)
|
|
351
379
|
|
|
352
|
-
ingredientsParallel.astFunctionDef = Z0Z_lameFindReplace(ingredientsParallel.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name)
|
|
380
|
+
ingredientsParallel.astFunctionDef = Z0Z_lameFindReplace(ingredientsParallel.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name)
|
|
353
381
|
|
|
354
382
|
# numba decorators =========================================
|
|
355
383
|
ingredientsParallel = decorateCallableWithNumba(ingredientsParallel)
|
|
@@ -357,8 +385,15 @@ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
|
|
|
357
385
|
|
|
358
386
|
# Module-level transformations ===========================================================
|
|
359
387
|
ingredientsModuleNumbaUnified = IngredientsModule(ingredientsFunction=listAllIngredientsFunctions, imports=LedgerOfImports(numbaFlow.source_astModule))
|
|
388
|
+
ingredientsModuleNumbaUnified.removeImportFromModule('numpy')
|
|
360
389
|
|
|
361
390
|
write_astModule(ingredientsModuleNumbaUnified, numbaFlow.pathFilenameDispatcher, numbaFlow.packageIdentifier)
|
|
362
391
|
|
|
392
|
+
def getIt(astCallConcurrencyResult: list[ast.Call]) -> Callable[[ast.AST], ast.AST]:
|
|
393
|
+
def workhorse(node: ast.AST) -> ast.AST:
|
|
394
|
+
NodeTourist(be.Call, Then.appendTo(astCallConcurrencyResult)).visit(node)
|
|
395
|
+
return node
|
|
396
|
+
return workhorse
|
|
397
|
+
|
|
363
398
|
if __name__ == '__main__':
|
|
364
399
|
makeNumbaFlow(theNumbaFlow)
|