mapFolding 0.7.1__py3-none-any.whl → 0.8.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mapFolding/__init__.py +33 -4
- mapFolding/basecamp.py +14 -0
- mapFolding/beDRY.py +93 -82
- mapFolding/filesystem.py +124 -90
- mapFolding/noHomeYet.py +14 -2
- mapFolding/oeis.py +18 -3
- mapFolding/reference/flattened.py +46 -45
- mapFolding/reference/hunterNumba.py +4 -4
- mapFolding/reference/irvineJavaPort.py +1 -1
- mapFolding/reference/lunnanNumpy.py +3 -4
- mapFolding/reference/lunnanWhile.py +5 -7
- mapFolding/reference/rotatedEntryPoint.py +2 -3
- mapFolding/someAssemblyRequired/__init__.py +33 -3
- mapFolding/someAssemblyRequired/getLLVMforNoReason.py +32 -15
- mapFolding/someAssemblyRequired/ingredientsNumba.py +108 -2
- mapFolding/someAssemblyRequired/synthesizeNumbaFlow.py +196 -0
- mapFolding/someAssemblyRequired/{synthesizeNumbaJob.py → synthesizeNumbaJobVESTIGIAL.py} +19 -23
- mapFolding/someAssemblyRequired/transformDataStructures.py +162 -0
- mapFolding/someAssemblyRequired/transformationTools.py +607 -252
- mapFolding/syntheticModules/numbaCount_doTheNeedful.py +197 -12
- mapFolding/theDao.py +37 -16
- mapFolding/theSSOT.py +47 -44
- {mapfolding-0.7.1.dist-info → mapfolding-0.8.1.dist-info}/METADATA +51 -46
- mapfolding-0.8.1.dist-info/RECORD +39 -0
- {mapfolding-0.7.1.dist-info → mapfolding-0.8.1.dist-info}/WHEEL +1 -1
- tests/conftest.py +2 -3
- tests/test_filesystem.py +0 -2
- tests/test_other.py +2 -3
- tests/test_tasks.py +0 -4
- mapFolding/reference/lunnan.py +0 -153
- mapFolding/someAssemblyRequired/Z0Z_workbench.py +0 -33
- mapFolding/someAssemblyRequired/synthesizeCountingFunctions.py +0 -7
- mapFolding/someAssemblyRequired/synthesizeDataConverters.py +0 -135
- mapFolding/someAssemblyRequired/synthesizeNumba.py +0 -91
- mapFolding/someAssemblyRequired/synthesizeNumbaModules.py +0 -91
- mapFolding/someAssemblyRequired/whatWillBe.py +0 -357
- mapFolding/syntheticModules/dataNamespaceFlattened.py +0 -30
- mapFolding/syntheticModules/multiprocessingCount_doTheNeedful.py +0 -216
- mapFolding/syntheticModules/numbaCount.py +0 -90
- mapFolding/syntheticModules/numbaCountExample.py +0 -158
- mapFolding/syntheticModules/numbaCountSequential.py +0 -111
- mapFolding/syntheticModules/numba_doTheNeedful.py +0 -12
- mapFolding/syntheticModules/numba_doTheNeedfulExample.py +0 -13
- mapfolding-0.7.1.dist-info/RECORD +0 -51
- /mapFolding/{syntheticModules → reference}/__init__.py +0 -0
- {mapfolding-0.7.1.dist-info → mapfolding-0.8.1.dist-info}/entry_points.txt +0 -0
- {mapfolding-0.7.1.dist-info → mapfolding-0.8.1.dist-info/licenses}/LICENSE +0 -0
- {mapfolding-0.7.1.dist-info → mapfolding-0.8.1.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Orchestrator for generating Numba-optimized versions of the map folding algorithm.
|
|
3
|
+
|
|
4
|
+
This module transforms the pure Python implementation of the map folding algorithm
|
|
5
|
+
into a highly-optimized Numba implementation. It serves as the high-level coordinator
|
|
6
|
+
for the code transformation process, orchestrating the following steps:
|
|
7
|
+
|
|
8
|
+
1. Extracting the core algorithm functions from the source implementation
|
|
9
|
+
2. Transforming function signatures and state handling for Numba compatibility
|
|
10
|
+
3. Converting state-based operations to direct primitive operations
|
|
11
|
+
4. Applying Numba decorators with appropriate optimization parameters
|
|
12
|
+
5. Managing imports and dependencies for the generated code
|
|
13
|
+
6. Assembling and writing the transformed implementation
|
|
14
|
+
|
|
15
|
+
The transformation process preserves the algorithm's logic while dramatically improving
|
|
16
|
+
performance by leveraging Numba's just-in-time compilation capabilities. This module
|
|
17
|
+
depends on the abstract transformation tools, dataclass handling utilities, and
|
|
18
|
+
Numba-specific optimization configurations from other modules in the package.
|
|
19
|
+
|
|
20
|
+
The primary entry point is the makeNumbaFlow function, which can be executed directly
|
|
21
|
+
to generate a fresh optimized implementation.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from mapFolding.someAssemblyRequired import (
|
|
25
|
+
extractFunctionDef,
|
|
26
|
+
ifThis,
|
|
27
|
+
IngredientsFunction,
|
|
28
|
+
IngredientsModule,
|
|
29
|
+
LedgerOfImports,
|
|
30
|
+
Make,
|
|
31
|
+
makeDictionaryReplacementStatements,
|
|
32
|
+
NodeCollector,
|
|
33
|
+
NodeReplacer,
|
|
34
|
+
RecipeSynthesizeFlow,
|
|
35
|
+
Then,
|
|
36
|
+
write_astModule,
|
|
37
|
+
Z0Z_replaceMatchingASTnodes,
|
|
38
|
+
inlineThisFunctionWithTheseValues,
|
|
39
|
+
)
|
|
40
|
+
from mapFolding.someAssemblyRequired.ingredientsNumba import decorateCallableWithNumba
|
|
41
|
+
from mapFolding.someAssemblyRequired.transformDataStructures import shatter_dataclassesDOTdataclass
|
|
42
|
+
from mapFolding.theSSOT import raiseIfNoneGitHubIssueNumber3
|
|
43
|
+
import ast
|
|
44
|
+
|
|
45
|
+
def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow = RecipeSynthesizeFlow()) -> None:
|
|
46
|
+
"""
|
|
47
|
+
Think about a better organization of this function.
|
|
48
|
+
|
|
49
|
+
Currently, transform `Callable` in order:
|
|
50
|
+
sourceDispatcherCallable
|
|
51
|
+
sourceInitializeCallable
|
|
52
|
+
sourceParallelCallable
|
|
53
|
+
sourceSequentialCallable
|
|
54
|
+
|
|
55
|
+
But, it should be organized around each transformation. So, when the parameters of `sourceSequentialCallable`
|
|
56
|
+
are transformed, for example, the statement in `sourceDispatcherCallable` that calls `sourceSequentialCallable` should be
|
|
57
|
+
transformed at the same time: literally in the same function-or-NodeReplacer-or-subroutine. That would help
|
|
58
|
+
avoid bugs.
|
|
59
|
+
|
|
60
|
+
Furthermore, if the above example transformation requires unpacking the dataclass, for example, then the unpacking
|
|
61
|
+
would be automatically triggered. I have no idea how that would happen, but the transformations are highly predictable,
|
|
62
|
+
so using a programming language to construct if-this-then-that cascades shouldn't be a problem, you know?
|
|
63
|
+
|
|
64
|
+
# TODO a tool to automatically remove unused variables from the ArgumentsSpecification (return, and returns) _might_ be nice.
|
|
65
|
+
"""
|
|
66
|
+
dictionaryReplacementStatements = makeDictionaryReplacementStatements(numbaFlow.source_astModule)
|
|
67
|
+
# TODO remember that `sequentialCallable` and `sourceSequentialCallable` are two different values.
|
|
68
|
+
# Figure out dynamic flow control to synthesized modules https://github.com/hunterhogan/mapFolding/issues/4
|
|
69
|
+
|
|
70
|
+
# ===========================================================
|
|
71
|
+
sourcePython = numbaFlow.sourceDispatcherCallable
|
|
72
|
+
astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
|
|
73
|
+
if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
|
|
74
|
+
ingredientsDispatcher = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
|
|
75
|
+
|
|
76
|
+
# sourceParallelCallable
|
|
77
|
+
shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, numbaFlow.sourceDataclassInstanceTaskDistribution)
|
|
78
|
+
ingredientsDispatcher.imports.update(shatteredDataclass.ledgerDataclassANDFragments)
|
|
79
|
+
|
|
80
|
+
# TODO remove hardcoding
|
|
81
|
+
namespaceHARDCODED = 'concurrencyManager'
|
|
82
|
+
identifierHARDCODED = 'submit'
|
|
83
|
+
sourceNamespace = namespaceHARDCODED
|
|
84
|
+
sourceIdentifier = identifierHARDCODED
|
|
85
|
+
NodeReplacer(
|
|
86
|
+
findThis = ifThis.isAssignAndValueIsCallNamespace_Identifier(sourceNamespace, sourceIdentifier)
|
|
87
|
+
, doThat = Then.insertThisAbove(shatteredDataclass.listAnnAssign4DataclassUnpack)
|
|
88
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
89
|
+
NodeReplacer(
|
|
90
|
+
findThis = ifThis.isCallNamespace_Identifier(sourceNamespace, sourceIdentifier)
|
|
91
|
+
, doThat = Then.replaceWith(Make.astCall(Make.astAttribute(Make.astName(sourceNamespace), sourceIdentifier)
|
|
92
|
+
, listArguments=[Make.astName(numbaFlow.parallelCallable)] + shatteredDataclass.listNameDataclassFragments4Parameters))
|
|
93
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
94
|
+
|
|
95
|
+
CapturedAssign: list[ast.AST] = []
|
|
96
|
+
CapturedCall: list[ast.Call] = []
|
|
97
|
+
findThis = ifThis.isCall
|
|
98
|
+
doThat = [Then.appendTo(CapturedCall)]
|
|
99
|
+
capture = NodeCollector(findThis, doThat)
|
|
100
|
+
|
|
101
|
+
NodeCollector(
|
|
102
|
+
findThis = ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(numbaFlow.sourceDataclassInstance))
|
|
103
|
+
, doThat = [Then.appendTo(CapturedAssign)
|
|
104
|
+
, lambda node: capture.visit(node)]
|
|
105
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
106
|
+
|
|
107
|
+
newAssign = CapturedAssign[0]
|
|
108
|
+
NodeReplacer(
|
|
109
|
+
findThis = lambda node: ifThis.isSubscript(node) and ifThis.isAttribute(node.value) and ifThis.isCall(node.value.value)
|
|
110
|
+
, doThat = Then.replaceWith(CapturedCall[0])
|
|
111
|
+
).visit(newAssign)
|
|
112
|
+
|
|
113
|
+
NodeReplacer(
|
|
114
|
+
findThis = ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(numbaFlow.sourceDataclassInstance))
|
|
115
|
+
, doThat = Then.replaceWith(newAssign)
|
|
116
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
117
|
+
|
|
118
|
+
# sourceSequentialCallable
|
|
119
|
+
shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, numbaFlow.sourceDataclassInstance)
|
|
120
|
+
|
|
121
|
+
ingredientsDispatcher.imports.update(shatteredDataclass.ledgerDataclassANDFragments)
|
|
122
|
+
|
|
123
|
+
NodeReplacer(
|
|
124
|
+
findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
|
|
125
|
+
, doThat = Then.insertThisAbove(shatteredDataclass.listAnnAssign4DataclassUnpack)
|
|
126
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
127
|
+
NodeReplacer(
|
|
128
|
+
findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
|
|
129
|
+
, doThat = Then.insertThisBelow([shatteredDataclass.astAssignDataclassRepack])
|
|
130
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
131
|
+
NodeReplacer(
|
|
132
|
+
findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
|
|
133
|
+
, doThat = Then.replaceWith(Make.astAssign(listTargets=[shatteredDataclass.astTuple4AssignTargetsToFragments], value=Make.astCall(Make.astName(numbaFlow.sequentialCallable), shatteredDataclass.listNameDataclassFragments4Parameters)))
|
|
134
|
+
).visit(ingredientsDispatcher.astFunctionDef)
|
|
135
|
+
|
|
136
|
+
# ===========================================================
|
|
137
|
+
sourcePython = numbaFlow.sourceInitializeCallable
|
|
138
|
+
astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
|
|
139
|
+
if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
|
|
140
|
+
astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
|
|
141
|
+
ingredientsInitialize = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
|
|
142
|
+
|
|
143
|
+
# ===========================================================
|
|
144
|
+
sourcePython = numbaFlow.sourceParallelCallable
|
|
145
|
+
astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
|
|
146
|
+
if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
|
|
147
|
+
astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
|
|
148
|
+
ingredientsParallel = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
|
|
149
|
+
ingredientsParallel.astFunctionDef.name = numbaFlow.parallelCallable
|
|
150
|
+
ingredientsParallel.astFunctionDef.args = Make.astArgumentsSpecification(args=shatteredDataclass.list_ast_argAnnotated4ArgumentsSpecification)
|
|
151
|
+
NodeReplacer(
|
|
152
|
+
findThis = ifThis.isReturn
|
|
153
|
+
, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
|
|
154
|
+
).visit(ingredientsParallel.astFunctionDef)
|
|
155
|
+
|
|
156
|
+
NodeReplacer(
|
|
157
|
+
findThis = ifThis.isReturn
|
|
158
|
+
, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.countingVariableName))
|
|
159
|
+
).visit(ingredientsParallel.astFunctionDef)
|
|
160
|
+
ingredientsParallel.astFunctionDef.returns = shatteredDataclass.countingVariableAnnotation
|
|
161
|
+
replacementMap = {statement.value: statement.target for statement in shatteredDataclass.listAnnAssign4DataclassUnpack}
|
|
162
|
+
ingredientsParallel.astFunctionDef = Z0Z_replaceMatchingASTnodes(ingredientsParallel.astFunctionDef, replacementMap) # type: ignore
|
|
163
|
+
ingredientsParallel = decorateCallableWithNumba(ingredientsParallel)
|
|
164
|
+
|
|
165
|
+
# ===========================================================
|
|
166
|
+
sourcePython = numbaFlow.sourceSequentialCallable
|
|
167
|
+
astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
|
|
168
|
+
if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
|
|
169
|
+
astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
|
|
170
|
+
ingredientsSequential = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
|
|
171
|
+
ingredientsSequential.astFunctionDef.name = numbaFlow.sequentialCallable
|
|
172
|
+
ingredientsSequential.astFunctionDef.args = Make.astArgumentsSpecification(args=shatteredDataclass.list_ast_argAnnotated4ArgumentsSpecification)
|
|
173
|
+
NodeReplacer(
|
|
174
|
+
findThis = ifThis.isReturn
|
|
175
|
+
, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
|
|
176
|
+
).visit(ingredientsSequential.astFunctionDef)
|
|
177
|
+
NodeReplacer(
|
|
178
|
+
findThis = ifThis.isReturn
|
|
179
|
+
, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
|
|
180
|
+
).visit(ingredientsSequential.astFunctionDef)
|
|
181
|
+
ingredientsSequential.astFunctionDef.returns = shatteredDataclass.astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns
|
|
182
|
+
replacementMap = {statement.value: statement.target for statement in shatteredDataclass.listAnnAssign4DataclassUnpack}
|
|
183
|
+
ingredientsSequential.astFunctionDef = Z0Z_replaceMatchingASTnodes(ingredientsSequential.astFunctionDef, replacementMap) # type: ignore
|
|
184
|
+
ingredientsSequential = decorateCallableWithNumba(ingredientsSequential)
|
|
185
|
+
|
|
186
|
+
# ===========================================================
|
|
187
|
+
ingredientsModuleNumbaUnified = IngredientsModule(
|
|
188
|
+
ingredientsFunction=[ingredientsInitialize,
|
|
189
|
+
ingredientsParallel,
|
|
190
|
+
ingredientsSequential,
|
|
191
|
+
ingredientsDispatcher], imports=LedgerOfImports(numbaFlow.source_astModule))
|
|
192
|
+
|
|
193
|
+
write_astModule(ingredientsModuleNumbaUnified, numbaFlow.pathFilenameDispatcher, numbaFlow.packageName)
|
|
194
|
+
|
|
195
|
+
if __name__ == '__main__':
|
|
196
|
+
makeNumbaFlow()
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
from collections.abc import Sequence
|
|
3
3
|
from typing import Any, cast, TYPE_CHECKING
|
|
4
4
|
from mapFolding.filesystem import getFilenameFoldsTotal, getPathFilenameFoldsTotal
|
|
5
|
-
from mapFolding.someAssemblyRequired import ( ifThis, Make, NodeReplacer, Then,
|
|
6
|
-
from mapFolding.theSSOT import ( ComputationState,
|
|
5
|
+
from mapFolding.someAssemblyRequired import ( ifThis, LedgerOfImports, Make, NodeReplacer, Then, )
|
|
6
|
+
from mapFolding.theSSOT import ( ComputationState, raiseIfNoneGitHubIssueNumber3, getPathJobRootDEFAULT, )
|
|
7
7
|
from os import PathLike
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from types import ModuleType
|
|
@@ -15,9 +15,9 @@ import copy
|
|
|
15
15
|
import inspect
|
|
16
16
|
import numpy
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
|
-
from mapFolding.someAssemblyRequired.
|
|
19
|
-
from mapFolding.someAssemblyRequired.
|
|
20
|
-
from mapFolding.someAssemblyRequired.
|
|
18
|
+
from mapFolding.someAssemblyRequired.transformDataStructures import makeStateJobOUTDATED
|
|
19
|
+
from mapFolding.someAssemblyRequired.ingredientsNumba import thisIsNumbaDotJit, decorateCallableWithNumba
|
|
20
|
+
from mapFolding.someAssemblyRequired.ingredientsNumba import ParametersNumba, parametersNumbaDEFAULT
|
|
21
21
|
|
|
22
22
|
def Z0Z_gamma(FunctionDefTarget: ast.FunctionDef, astAssignee: ast.Name, statement: ast.Assign | ast.stmt, identifier: str, arrayTarget: numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.integer[Any]]], allImports: LedgerOfImports) -> tuple[ast.FunctionDef, LedgerOfImports]:
|
|
23
23
|
arrayType = type(arrayTarget)
|
|
@@ -53,7 +53,7 @@ def insertArrayIn_body(FunctionDefTarget: ast.FunctionDef, identifier: str, arra
|
|
|
53
53
|
|
|
54
54
|
def findAndReplaceTrackArrayIn_body(FunctionDefTarget: ast.FunctionDef, identifier: str, arrayTarget: numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.integer[Any]]], allImports: LedgerOfImports) -> tuple[ast.FunctionDef, LedgerOfImports]:
|
|
55
55
|
for statement in FunctionDefTarget.body.copy():
|
|
56
|
-
if
|
|
56
|
+
if True:
|
|
57
57
|
indexAsStr: str = ast.unparse(statement.value.slice) # type: ignore
|
|
58
58
|
arraySlice: numpy.ndarray[Any, numpy.dtype[numpy.integer[Any]]] = arrayTarget[eval(indexAsStr)]
|
|
59
59
|
astAssignee: ast.Name = cast(ast.Name, statement.targets[0]) # type: ignore
|
|
@@ -62,10 +62,10 @@ def findAndReplaceTrackArrayIn_body(FunctionDefTarget: ast.FunctionDef, identifi
|
|
|
62
62
|
|
|
63
63
|
def findAndReplaceArraySubscriptIn_body(FunctionDefTarget: ast.FunctionDef, identifier: str, arrayTarget: numpy.ndarray[tuple[int, ...], numpy.dtype[numpy.integer[Any]]], allImports: LedgerOfImports) -> tuple[ast.FunctionDef, LedgerOfImports]:
|
|
64
64
|
# parameter: I define moduleConstructor
|
|
65
|
-
moduleConstructor =
|
|
65
|
+
moduleConstructor = 'numba'
|
|
66
66
|
|
|
67
67
|
for statement in FunctionDefTarget.body.copy():
|
|
68
|
-
if
|
|
68
|
+
if True:
|
|
69
69
|
indexAsStr: str = ast.unparse(statement.value.slice) # type: ignore
|
|
70
70
|
arraySlice: numpy.ndarray[Any, numpy.dtype[numpy.integer[Any]]] = arrayTarget[eval(indexAsStr)]
|
|
71
71
|
astAssignee: ast.Name = cast(ast.Name, statement.targets[0]) # type: ignore
|
|
@@ -83,7 +83,7 @@ def findAndReplaceArraySubscriptIn_body(FunctionDefTarget: ast.FunctionDef, iden
|
|
|
83
83
|
def removeAssignmentFrom_body(FunctionDefTarget: ast.FunctionDef, identifier: str) -> ast.FunctionDef:
|
|
84
84
|
FunctionDefSherpa: ast.AST | Sequence[ast.AST] | None = NodeReplacer(ifThis.isAnyAssignmentTo(identifier), Then.removeThis).visit(FunctionDefTarget)
|
|
85
85
|
if not FunctionDefSherpa:
|
|
86
|
-
raise
|
|
86
|
+
raise raiseIfNoneGitHubIssueNumber3("Dude, where's my function?")
|
|
87
87
|
else:
|
|
88
88
|
FunctionDefTarget = cast(ast.FunctionDef, FunctionDefSherpa)
|
|
89
89
|
ast.fix_missing_locations(FunctionDefTarget)
|
|
@@ -91,7 +91,7 @@ def removeAssignmentFrom_body(FunctionDefTarget: ast.FunctionDef, identifier: st
|
|
|
91
91
|
|
|
92
92
|
def findAndReplaceAnnAssignIn_body(FunctionDefTarget: ast.FunctionDef, allImports: LedgerOfImports) -> tuple[ast.FunctionDef, LedgerOfImports]:
|
|
93
93
|
"""Unlike most of the other functions, this is generic: it tries to turn an annotation into a construction call."""
|
|
94
|
-
moduleConstructor: str =
|
|
94
|
+
moduleConstructor: str = 'numba'
|
|
95
95
|
for stmt in FunctionDefTarget.body.copy():
|
|
96
96
|
if isinstance(stmt, ast.AnnAssign):
|
|
97
97
|
if isinstance(stmt.target, ast.Name) and isinstance(stmt.value, ast.Constant):
|
|
@@ -141,7 +141,7 @@ def insertReturnStatementIn_body(FunctionDefTarget: ast.FunctionDef, arrayTarget
|
|
|
141
141
|
|
|
142
142
|
datatype: str = 'Z0Z_identifierCountFolds'
|
|
143
143
|
FunctionDefTarget.returns = ast.Name(id=datatype, ctx=ast.Load())
|
|
144
|
-
datatypeModuleScalar: str =
|
|
144
|
+
datatypeModuleScalar: str = 'numba'
|
|
145
145
|
allImports.addImportFromStr(datatypeModuleScalar, datatype)
|
|
146
146
|
|
|
147
147
|
FunctionDefTarget.body.append(returnStatement)
|
|
@@ -290,20 +290,20 @@ def writeJobNumba(mapShape: Sequence[int], algorithmSource: ModuleType, callable
|
|
|
290
290
|
"""
|
|
291
291
|
|
|
292
292
|
# NOTE get the raw ingredients: data and the algorithm
|
|
293
|
-
stateJob =
|
|
293
|
+
stateJob = makeStateJobOUTDATED(mapShape, writeJob=False, **keywordArguments)
|
|
294
294
|
pythonSource: str = inspect.getsource(algorithmSource)
|
|
295
295
|
astModule: ast.Module = ast.parse(pythonSource)
|
|
296
296
|
setFunctionDef: set[ast.FunctionDef] = {statement for statement in astModule.body if isinstance(statement, ast.FunctionDef)}
|
|
297
297
|
|
|
298
298
|
if not callableTarget:
|
|
299
299
|
if len(setFunctionDef) == 1:
|
|
300
|
-
FunctionDefTarget
|
|
300
|
+
FunctionDefTarget = setFunctionDef.pop()
|
|
301
301
|
callableTarget = FunctionDefTarget.name
|
|
302
302
|
else:
|
|
303
303
|
raise ValueError(f"I did not receive a `callableTarget` and {algorithmSource.__name__=} has more than one callable: {setFunctionDef}. Please select one.")
|
|
304
304
|
else:
|
|
305
305
|
listFunctionDefTarget: list[ast.FunctionDef] = [statement for statement in setFunctionDef if statement.name == callableTarget]
|
|
306
|
-
FunctionDefTarget = listFunctionDefTarget[0] if listFunctionDefTarget else None
|
|
306
|
+
FunctionDefTarget = listFunctionDefTarget[0] if listFunctionDefTarget else None # type: ignore
|
|
307
307
|
if not FunctionDefTarget: raise ValueError(f"I received `{callableTarget=}` and {algorithmSource.__name__=}, but I could not find that function in that source.")
|
|
308
308
|
|
|
309
309
|
# NOTE `allImports` is a complementary container to `FunctionDefTarget`; the `FunctionDefTarget` cannot track its own imports very well.
|
|
@@ -323,7 +323,7 @@ def writeJobNumba(mapShape: Sequence[int], algorithmSource: ModuleType, callable
|
|
|
323
323
|
FunctionDefTarget.args.args.remove(pirateScowl)
|
|
324
324
|
|
|
325
325
|
identifierCounter = 'Z0Z_identifierCountFolds'
|
|
326
|
-
astExprIncrementCounter = ast.Expr(value = Make.astCall(Make.nameDOTname(identifierCounter, 'update'),
|
|
326
|
+
astExprIncrementCounter = ast.Expr(value = Make.astCall(Make.nameDOTname(identifierCounter, 'update'), listArguments=[ast.Constant(value=1)], list_astKeywords=[]))
|
|
327
327
|
FunctionDefTarget= cast(ast.FunctionDef, NodeReplacer(ifThis.isAugAssignTo(identifierCounter), Then.replaceWith(astExprIncrementCounter)).visit(FunctionDefTarget))
|
|
328
328
|
ast.fix_missing_locations(FunctionDefTarget)
|
|
329
329
|
|
|
@@ -344,7 +344,7 @@ def writeJobNumba(mapShape: Sequence[int], algorithmSource: ModuleType, callable
|
|
|
344
344
|
|
|
345
345
|
# TODO create function for assigning value to `totalEstimated`
|
|
346
346
|
totalEstimated: int = Z0Z_totalEstimated
|
|
347
|
-
astLauncher
|
|
347
|
+
astLauncher = makeLauncherTqdmJobNumba(FunctionDefTarget.name, pathFilenameFoldsTotal, totalEstimated, stateJob.foldGroups[-1])
|
|
348
348
|
|
|
349
349
|
allImports.addImportFromStr('numba_progress', 'ProgressBar')
|
|
350
350
|
allImports.addImportFromStr('numba_progress', 'ProgressBarType')
|
|
@@ -361,10 +361,9 @@ def writeJobNumba(mapShape: Sequence[int], algorithmSource: ModuleType, callable
|
|
|
361
361
|
|
|
362
362
|
FunctionDefTarget, allImports = findAndReplaceAnnAssignIn_body(FunctionDefTarget, allImports)
|
|
363
363
|
# NOTE add the perfect decorator
|
|
364
|
-
FunctionDefTarget, allImports = decorateCallableWithNumba(FunctionDefTarget, allImports, parametersNumba)
|
|
365
364
|
if thisIsNumbaDotJit(FunctionDefTarget.decorator_list[0]):
|
|
366
365
|
astCall: ast.Call = cast(ast.Call, FunctionDefTarget.decorator_list[0])
|
|
367
|
-
astCall.func = ast.Name(id=
|
|
366
|
+
astCall.func = ast.Name(id='jit', ctx=ast.Load())
|
|
368
367
|
FunctionDefTarget.decorator_list[0] = astCall
|
|
369
368
|
|
|
370
369
|
# NOTE add imports, make str, remove unused imports
|
|
@@ -400,8 +399,8 @@ if __name__ == '__main__':
|
|
|
400
399
|
}
|
|
401
400
|
|
|
402
401
|
totalEstimated: int = dictionaryEstimates.get(tuple(mapShape), 10**8)
|
|
403
|
-
from mapFolding.syntheticModules import
|
|
404
|
-
algorithmSource: ModuleType =
|
|
402
|
+
from mapFolding.syntheticModules import numbaCount_doTheNeedful
|
|
403
|
+
algorithmSource: ModuleType = numbaCount_doTheNeedful
|
|
405
404
|
|
|
406
405
|
callableTarget = 'countSequential'
|
|
407
406
|
|
|
@@ -411,7 +410,4 @@ if __name__ == '__main__':
|
|
|
411
410
|
|
|
412
411
|
pathFilenameWriteJob = None
|
|
413
412
|
|
|
414
|
-
Z0Z_setDatatypeModuleScalar('numba')
|
|
415
|
-
Z0Z_setDecoratorCallable('jit')
|
|
416
|
-
|
|
417
413
|
writeJobNumba(mapShape, algorithmSource, callableTarget, parametersNumba, pathFilenameWriteJob, Z0Z_totalEstimated=totalEstimated)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Utilities for transforming complex data structures in Python code generation.
|
|
3
|
+
|
|
4
|
+
This module provides specialized tools for working with structured data types during
|
|
5
|
+
the code transformation process, with a particular focus on handling dataclasses. It
|
|
6
|
+
implements functionality that enables:
|
|
7
|
+
|
|
8
|
+
1. Decomposing dataclasses into individual fields for efficient processing
|
|
9
|
+
2. Creating optimized parameter passing for transformed functions
|
|
10
|
+
3. Converting between different representations of data structures
|
|
11
|
+
4. Serializing and deserializing computation state objects
|
|
12
|
+
|
|
13
|
+
The core functionality revolves around the "shattering" process that breaks down
|
|
14
|
+
a dataclass into its constituent components, making each field individually accessible
|
|
15
|
+
for code generation and optimization purposes. This dataclass handling is critical for
|
|
16
|
+
transforming algorithms that operate on unified state objects into optimized implementations
|
|
17
|
+
that work with primitive types directly.
|
|
18
|
+
|
|
19
|
+
While developed for transforming map folding computation state objects, the utilities are
|
|
20
|
+
designed to be applicable to various data structure transformation scenarios.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from collections.abc import Sequence
|
|
24
|
+
from importlib import import_module
|
|
25
|
+
from inspect import getsource as inspect_getsource
|
|
26
|
+
from mapFolding.beDRY import outfitCountFolds, validateListDimensions
|
|
27
|
+
from mapFolding.filesystem import getPathFilenameFoldsTotal
|
|
28
|
+
from mapFolding.someAssemblyRequired import (
|
|
29
|
+
ast_Identifier,
|
|
30
|
+
extractClassDef,
|
|
31
|
+
ifThis,
|
|
32
|
+
LedgerOfImports,
|
|
33
|
+
Make,
|
|
34
|
+
NodeCollector,
|
|
35
|
+
strDotStrCuzPyStoopid,
|
|
36
|
+
Then,
|
|
37
|
+
Z0Z_executeActionUnlessDescendantMatches,
|
|
38
|
+
)
|
|
39
|
+
from mapFolding.theSSOT import ComputationState, getSourceAlgorithm
|
|
40
|
+
from pathlib import Path
|
|
41
|
+
from types import ModuleType
|
|
42
|
+
from typing import Any, Literal, overload
|
|
43
|
+
import ast
|
|
44
|
+
import dataclasses
|
|
45
|
+
import pickle
|
|
46
|
+
|
|
47
|
+
# Would `LibCST` be better than `ast` in some cases? https://github.com/hunterhogan/mapFolding/issues/7
|
|
48
|
+
|
|
49
|
+
countingIdentifierHARDCODED = 'groupsOfFolds'
|
|
50
|
+
|
|
51
|
+
@dataclasses.dataclass
|
|
52
|
+
class ShatteredDataclass:
|
|
53
|
+
astAssignDataclassRepack: ast.Assign
|
|
54
|
+
astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns: ast.Subscript
|
|
55
|
+
astTuple4AssignTargetsToFragments: ast.Tuple
|
|
56
|
+
countingVariableAnnotation: ast.expr
|
|
57
|
+
countingVariableName: ast.Name
|
|
58
|
+
ledgerDataclassANDFragments: LedgerOfImports = dataclasses.field(default_factory=LedgerOfImports)
|
|
59
|
+
list_ast_argAnnotated4ArgumentsSpecification: list[ast.arg] = dataclasses.field(default_factory=list)
|
|
60
|
+
list_keyword4DataclassInitialization: list[ast.keyword] = dataclasses.field(default_factory=list)
|
|
61
|
+
listAnnAssign4DataclassUnpack: list[ast.AnnAssign] = dataclasses.field(default_factory=list)
|
|
62
|
+
listAnnotations: list[ast.expr] = dataclasses.field(default_factory=list)
|
|
63
|
+
listNameDataclassFragments4Parameters: list[ast.Name] = dataclasses.field(default_factory=list)
|
|
64
|
+
|
|
65
|
+
def shatter_dataclassesDOTdataclass(logicalPathModule: strDotStrCuzPyStoopid, dataclass_Identifier: ast_Identifier, instance_Identifier: ast_Identifier) -> ShatteredDataclass:
|
|
66
|
+
"""
|
|
67
|
+
Parameters:
|
|
68
|
+
logicalPathModule: gimme string cuz python is stoopid
|
|
69
|
+
dataclass_Identifier: The identifier of the dataclass to be dismantled.
|
|
70
|
+
instance_Identifier: In the synthesized module/function/scope, the identifier that will be used for the instance.
|
|
71
|
+
"""
|
|
72
|
+
module: ast.Module = ast.parse(inspect_getsource(import_module(logicalPathModule)))
|
|
73
|
+
astName_dataclassesDOTdataclass = Make.astName(dataclass_Identifier)
|
|
74
|
+
|
|
75
|
+
dataclass = extractClassDef(dataclass_Identifier, module)
|
|
76
|
+
if not isinstance(dataclass, ast.ClassDef):
|
|
77
|
+
raise ValueError(f"I could not find {dataclass_Identifier=} in {logicalPathModule=}.")
|
|
78
|
+
|
|
79
|
+
ledgerDataclassANDFragments = LedgerOfImports()
|
|
80
|
+
list_ast_argAnnotated4ArgumentsSpecification: list[ast.arg] = []
|
|
81
|
+
list_keyword4DataclassInitialization: list[ast.keyword] = []
|
|
82
|
+
listAnnAssign4DataclassUnpack: list[ast.AnnAssign] = []
|
|
83
|
+
listAnnotations: list[ast.expr] = []
|
|
84
|
+
listNameDataclassFragments4Parameters: list[ast.Name] = []
|
|
85
|
+
|
|
86
|
+
# TODO get the value from `groupsOfFolds: DatatypeFoldsTotal = dataclasses.field(default=DatatypeFoldsTotal(0), metadata={'theCountingIdentifier': True})`
|
|
87
|
+
countingVariable = countingIdentifierHARDCODED
|
|
88
|
+
|
|
89
|
+
addToLedgerPredicate = ifThis.isAnnAssignAndAnnotationIsName
|
|
90
|
+
addToLedgerAction = Then.Z0Z_ledger(logicalPathModule, ledgerDataclassANDFragments)
|
|
91
|
+
addToLedger = NodeCollector(addToLedgerPredicate, [addToLedgerAction])
|
|
92
|
+
|
|
93
|
+
exclusionPredicate = ifThis.is_keyword_IdentifierEqualsConstantValue('init', False)
|
|
94
|
+
appendKeywordAction = Then.Z0Z_appendKeywordMirroredTo(list_keyword4DataclassInitialization)
|
|
95
|
+
filteredAppendKeywordAction = Z0Z_executeActionUnlessDescendantMatches(exclusionPredicate, appendKeywordAction) # type: ignore
|
|
96
|
+
|
|
97
|
+
NodeCollector(
|
|
98
|
+
ifThis.isAnnAssignAndTargetIsName,
|
|
99
|
+
[Then.Z0Z_appendAnnAssignOf_nameDOTnameTo(instance_Identifier, listAnnAssign4DataclassUnpack)
|
|
100
|
+
, Then.append_targetTo(listNameDataclassFragments4Parameters) # type: ignore
|
|
101
|
+
, lambda node: addToLedger.visit(node)
|
|
102
|
+
, filteredAppendKeywordAction
|
|
103
|
+
, lambda node: list_ast_argAnnotated4ArgumentsSpecification.append(Make.ast_arg(node.target.id, node.annotation)) # type: ignore
|
|
104
|
+
, lambda node: listAnnotations.append(node.annotation) # type: ignore
|
|
105
|
+
]
|
|
106
|
+
).visit(dataclass)
|
|
107
|
+
|
|
108
|
+
shatteredDataclass = ShatteredDataclass(
|
|
109
|
+
astAssignDataclassRepack = Make.astAssign(listTargets=[Make.astName(instance_Identifier)], value=Make.astCall(astName_dataclassesDOTdataclass, list_astKeywords=list_keyword4DataclassInitialization))
|
|
110
|
+
, astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns = Make.astSubscript(Make.astName('tuple'), Make.astTuple(listAnnotations))
|
|
111
|
+
, astTuple4AssignTargetsToFragments = Make.astTuple(listNameDataclassFragments4Parameters, ast.Store())
|
|
112
|
+
, countingVariableAnnotation = next(ast_arg.annotation for ast_arg in list_ast_argAnnotated4ArgumentsSpecification if ast_arg.arg == countingVariable) or Make.astName('Any')
|
|
113
|
+
, countingVariableName = Make.astName(countingVariable)
|
|
114
|
+
, ledgerDataclassANDFragments = ledgerDataclassANDFragments
|
|
115
|
+
, list_ast_argAnnotated4ArgumentsSpecification = list_ast_argAnnotated4ArgumentsSpecification
|
|
116
|
+
, list_keyword4DataclassInitialization = list_keyword4DataclassInitialization
|
|
117
|
+
, listAnnAssign4DataclassUnpack = listAnnAssign4DataclassUnpack
|
|
118
|
+
, listAnnotations = listAnnotations
|
|
119
|
+
, listNameDataclassFragments4Parameters = listNameDataclassFragments4Parameters
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
shatteredDataclass.ledgerDataclassANDFragments.addImportFromStr(logicalPathModule, dataclass_Identifier)
|
|
123
|
+
return shatteredDataclass
|
|
124
|
+
|
|
125
|
+
@overload
|
|
126
|
+
def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: Literal[True], **keywordArguments: Any) -> Path: ...
|
|
127
|
+
@overload
|
|
128
|
+
def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: Literal[False], **keywordArguments: Any) -> ComputationState: ...
|
|
129
|
+
def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: bool = True, **keywordArguments: Any) -> ComputationState | Path:
|
|
130
|
+
"""
|
|
131
|
+
Creates a computation state job for map folding calculations and optionally saves it to disk.
|
|
132
|
+
|
|
133
|
+
This function initializes a computation state for map folding calculations based on the given dimensions,
|
|
134
|
+
sets up the initial counting configuration, and can optionally save the state to a pickle file.
|
|
135
|
+
|
|
136
|
+
Parameters:
|
|
137
|
+
listDimensions: List of integers representing the dimensions of the map to be folded.
|
|
138
|
+
writeJob (True): Whether to save the state to disk.
|
|
139
|
+
**keywordArguments: Additional keyword arguments to pass to the computation state initialization.
|
|
140
|
+
|
|
141
|
+
Returns:
|
|
142
|
+
stateUniversal|pathFilenameJob: The computation state for the map folding calculations, or
|
|
143
|
+
the path to the saved state file if writeJob is True.
|
|
144
|
+
"""
|
|
145
|
+
mapShape = validateListDimensions(listDimensions)
|
|
146
|
+
stateUniversal: ComputationState = outfitCountFolds(mapShape, **keywordArguments)
|
|
147
|
+
|
|
148
|
+
moduleSource: ModuleType = getSourceAlgorithm()
|
|
149
|
+
# TODO `countInitialize` is hardcoded
|
|
150
|
+
stateUniversal = moduleSource.countInitialize(stateUniversal)
|
|
151
|
+
|
|
152
|
+
if not writeJob:
|
|
153
|
+
return stateUniversal
|
|
154
|
+
|
|
155
|
+
pathFilenameChopChop = getPathFilenameFoldsTotal(stateUniversal.mapShape, None)
|
|
156
|
+
suffix = pathFilenameChopChop.suffix
|
|
157
|
+
pathJob = Path(str(pathFilenameChopChop)[0:-len(suffix)])
|
|
158
|
+
pathJob.mkdir(parents=True, exist_ok=True)
|
|
159
|
+
pathFilenameJob = pathJob / 'stateJob.pkl'
|
|
160
|
+
|
|
161
|
+
pathFilenameJob.write_bytes(pickle.dumps(stateUniversal))
|
|
162
|
+
return pathFilenameJob
|