mapFolding 0.8.4__py3-none-any.whl → 0.8.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. mapFolding/__init__.py +4 -1
  2. mapFolding/basecamp.py +3 -3
  3. mapFolding/beDRY.py +241 -68
  4. mapFolding/oeis.py +3 -3
  5. mapFolding/reference/hunterNumba.py +1 -1
  6. mapFolding/someAssemblyRequired/__init__.py +16 -15
  7. mapFolding/someAssemblyRequired/_theTypes.py +31 -13
  8. mapFolding/someAssemblyRequired/_tool_Make.py +8 -1
  9. mapFolding/someAssemblyRequired/_tool_Then.py +12 -5
  10. mapFolding/someAssemblyRequired/_toolboxAntecedents.py +131 -99
  11. mapFolding/someAssemblyRequired/_toolboxContainers.py +35 -7
  12. mapFolding/someAssemblyRequired/_toolboxPython.py +17 -31
  13. mapFolding/someAssemblyRequired/getLLVMforNoReason.py +2 -2
  14. mapFolding/someAssemblyRequired/newInliner.py +22 -0
  15. mapFolding/someAssemblyRequired/synthesizeNumbaJob.py +24 -113
  16. mapFolding/someAssemblyRequired/toolboxNumba.py +358 -0
  17. mapFolding/someAssemblyRequired/transformationTools.py +253 -40
  18. mapFolding/theSSOT.py +30 -32
  19. mapFolding/{filesystem.py → toolboxFilesystem.py} +90 -25
  20. {mapfolding-0.8.4.dist-info → mapfolding-0.8.5.dist-info}/METADATA +3 -2
  21. mapfolding-0.8.5.dist-info/RECORD +48 -0
  22. tests/conftest.py +30 -31
  23. tests/test_computations.py +7 -6
  24. tests/test_filesystem.py +2 -2
  25. tests/test_other.py +2 -2
  26. tests/test_tasks.py +2 -2
  27. mapFolding/someAssemblyRequired/ingredientsNumba.py +0 -199
  28. mapFolding/someAssemblyRequired/synthesizeNumbaFlow.py +0 -156
  29. mapFolding/someAssemblyRequired/transformDataStructures.py +0 -235
  30. mapfolding-0.8.4.dist-info/RECORD +0 -49
  31. {mapfolding-0.8.4.dist-info → mapfolding-0.8.5.dist-info}/WHEEL +0 -0
  32. {mapfolding-0.8.4.dist-info → mapfolding-0.8.5.dist-info}/entry_points.txt +0 -0
  33. {mapfolding-0.8.4.dist-info → mapfolding-0.8.5.dist-info}/licenses/LICENSE +0 -0
  34. {mapfolding-0.8.4.dist-info → mapfolding-0.8.5.dist-info}/top_level.txt +0 -0
@@ -1,16 +1,14 @@
1
1
  """Synthesize one file to compute `foldsTotal` of `mapShape`."""
2
- from mapFolding.someAssemblyRequired import ast_Identifier, be, ifThis, Make, NodeChanger, NodeTourist, parsePathFilename2astModule, str_nameDOTname, Then, write_astModule, 又
3
- from mapFolding.someAssemblyRequired.ingredientsNumba import decorateCallableWithNumba, ParametersNumba, parametersNumbaDefault
4
- from mapFolding.someAssemblyRequired.synthesizeNumbaFlow import theNumbaFlow
5
- from mapFolding.someAssemblyRequired.transformDataStructures import makeInitializedComputationState, shatter_dataclassesDOTdataclass, ShatteredDataclass
6
- from mapFolding.someAssemblyRequired._toolboxContainers import astModuleToIngredientsFunction, IngredientsFunction, IngredientsModule, LedgerOfImports
7
- from mapFolding.filesystem import getFilenameFoldsTotal, getPathFilenameFoldsTotal, getPathRootJobDEFAULT
8
- from mapFolding.theSSOT import ComputationState, The
9
- from pathlib import Path, PurePosixPath
2
+ from mapFolding.toolboxFilesystem import getPathFilenameFoldsTotal
3
+ from mapFolding.someAssemblyRequired import ast_Identifier, be, ifThis, Make, NodeChanger, Then, IngredientsFunction, IngredientsModule
4
+ from mapFolding.someAssemblyRequired.toolboxNumba import RecipeJob, SpicesJobNumba, decorateCallableWithNumba
5
+ from mapFolding.someAssemblyRequired.transformationTools import astModuleToIngredientsFunction, write_astModule
6
+ from mapFolding.someAssemblyRequired.transformationTools import makeInitializedComputationState
7
+ from mapFolding.theSSOT import The
10
8
  from typing import cast
11
9
  from Z0Z_tools import autoDecodingRLE
10
+ from pathlib import PurePosixPath
12
11
  import ast
13
- import dataclasses
14
12
 
15
13
  list_IdentifiersNotUsedAllHARDCODED = ['concurrencyLimit', 'foldsTotal', 'mapShape',]
16
14
  list_IdentifiersNotUsedParallelSequentialHARDCODED = ['indexLeaf']
@@ -22,98 +20,7 @@ list_IdentifiersStaticValuesHARDCODED = ['dimensionsTotal', 'leavesTotal',]
22
20
 
23
21
  list_IdentifiersNotUsedHARDCODED = list_IdentifiersStaticValuesHARDCODED + list_IdentifiersReplacedHARDCODED + list_IdentifiersNotUsedAllHARDCODED + list_IdentifiersNotUsedParallelSequentialHARDCODED + list_IdentifiersNotUsedSequentialHARDCODED
24
22
 
25
- @dataclasses.dataclass
26
- class Z0Z_RecipeJob:
27
- state: ComputationState
28
- # TODO create function to calculate `foldsTotalEstimated`
29
- foldsTotalEstimated: int = 0
30
- useNumbaProgressBar: bool = True
31
- numbaProgressBarIdentifier: ast_Identifier = 'ProgressBarGroupsOfFolds'
32
- shatteredDataclass: ShatteredDataclass = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
33
-
34
- # ========================================
35
- # Source
36
- source_astModule = parsePathFilename2astModule(theNumbaFlow.pathFilenameSequential)
37
- sourceCountCallable: ast_Identifier = theNumbaFlow.callableSequential
38
-
39
- sourceLogicalPathModuleDataclass: str_nameDOTname = theNumbaFlow.logicalPathModuleDataclass
40
- sourceDataclassIdentifier: ast_Identifier = theNumbaFlow.dataclassIdentifier
41
- sourceDataclassInstance: ast_Identifier = theNumbaFlow.dataclassInstance
42
-
43
- sourcePathPackage: PurePosixPath | None = theNumbaFlow.pathPackage
44
- sourcePackageIdentifier: ast_Identifier | None = theNumbaFlow.packageIdentifier
45
-
46
- # ========================================
47
- # Filesystem (names of physical objects)
48
- pathPackage: PurePosixPath | None = None
49
- pathModule: PurePosixPath | None = PurePosixPath(getPathRootJobDEFAULT())
50
- """ `pathModule` will override `pathPackage` and `logicalPathRoot`."""
51
- fileExtension: str = theNumbaFlow.fileExtension
52
- pathFilenameFoldsTotal: PurePosixPath = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
53
-
54
- # ========================================
55
- # Logical identifiers (as opposed to physical identifiers)
56
- # ========================================
57
- packageIdentifier: ast_Identifier | None = None
58
- logicalPathRoot: str_nameDOTname | None = None
59
- """ `logicalPathRoot` likely corresponds to a physical filesystem directory."""
60
- moduleIdentifier: ast_Identifier = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
61
- countCallable: ast_Identifier = sourceCountCallable
62
- dataclassIdentifier: ast_Identifier | None = sourceDataclassIdentifier
63
- dataclassInstance: ast_Identifier | None = sourceDataclassInstance
64
- logicalPathModuleDataclass: str_nameDOTname | None = sourceLogicalPathModuleDataclass
65
-
66
- def _makePathFilename(self,
67
- pathRoot: PurePosixPath | None = None,
68
- logicalPathINFIX: str_nameDOTname | None = None,
69
- filenameStem: str | None = None,
70
- fileExtension: str | None = None,
71
- ) -> PurePosixPath:
72
- if pathRoot is None:
73
- pathRoot = self.pathPackage or PurePosixPath(Path.cwd())
74
- if logicalPathINFIX:
75
- whyIsThisStillAThing: list[str] = logicalPathINFIX.split('.')
76
- pathRoot = pathRoot.joinpath(*whyIsThisStillAThing)
77
- if filenameStem is None:
78
- filenameStem = self.moduleIdentifier
79
- if fileExtension is None:
80
- fileExtension = self.fileExtension
81
- filename: str = filenameStem + fileExtension
82
- return pathRoot.joinpath(filename)
83
-
84
- @property
85
- def pathFilenameModule(self) -> PurePosixPath:
86
- if self.pathModule is None:
87
- return self._makePathFilename()
88
- else:
89
- return self._makePathFilename(pathRoot=self.pathModule, logicalPathINFIX=None)
90
-
91
- def __post_init__(self):
92
- pathFilenameFoldsTotal = PurePosixPath(getPathFilenameFoldsTotal(self.state.mapShape))
93
-
94
- if self.moduleIdentifier is None:
95
- self.moduleIdentifier = pathFilenameFoldsTotal.stem
96
-
97
- if self.pathFilenameFoldsTotal is None:
98
- self.pathFilenameFoldsTotal = pathFilenameFoldsTotal
99
-
100
- if self.shatteredDataclass is None and self.logicalPathModuleDataclass and self.dataclassIdentifier and self.dataclassInstance:
101
- self.shatteredDataclass = shatter_dataclassesDOTdataclass(self.logicalPathModuleDataclass, self.dataclassIdentifier, self.dataclassInstance)
102
-
103
- # ========================================
104
- # Fields you probably don't need =================================
105
- # Dispatcher =================================
106
- sourceDispatcherCallable: ast_Identifier = theNumbaFlow.callableDispatcher
107
- dispatcherCallable: ast_Identifier = sourceDispatcherCallable
108
- # Parallel counting =================================
109
- sourceDataclassInstanceTaskDistribution: ast_Identifier = theNumbaFlow.dataclassInstanceTaskDistribution
110
- sourceConcurrencyManagerNamespace: ast_Identifier = theNumbaFlow.concurrencyManagerNamespace
111
- sourceConcurrencyManagerIdentifier: ast_Identifier = theNumbaFlow.concurrencyManagerIdentifier
112
- dataclassInstanceTaskDistribution: ast_Identifier = sourceDataclassInstanceTaskDistribution
113
- concurrencyManagerNamespace: ast_Identifier = sourceConcurrencyManagerNamespace
114
- concurrencyManagerIdentifier: ast_Identifier = sourceConcurrencyManagerIdentifier
115
-
116
- def addLauncherNumbaProgress(ingredientsModule: IngredientsModule, ingredientsFunction: IngredientsFunction, job: Z0Z_RecipeJob) -> IngredientsModule:
23
+ def addLauncherNumbaProgress(ingredientsModule: IngredientsModule, ingredientsFunction: IngredientsFunction, job: RecipeJob, spices: SpicesJobNumba) -> tuple[IngredientsModule, IngredientsFunction]:
117
24
 
118
25
  linesLaunch: str = f"""
119
26
  if __name__ == '__main__':
@@ -130,19 +37,19 @@ if __name__ == '__main__':
130
37
  ingredientsModule.imports.addImportFrom_asStr('numba_progress', numba_progressPythonClass)
131
38
  ingredientsModule.imports.addImportFrom_asStr('numba_progress', numba_progressNumbaType)
132
39
 
133
- ast_argNumbaProgress = ast.arg(arg=job.numbaProgressBarIdentifier, annotation=ast.Name(id=numba_progressPythonClass, ctx=ast.Load()))
40
+ ast_argNumbaProgress = ast.arg(arg=spices.numbaProgressBarIdentifier, annotation=ast.Name(id=numba_progressPythonClass, ctx=ast.Load()))
134
41
  ingredientsFunction.astFunctionDef.args.args.append(ast_argNumbaProgress)
135
42
 
136
43
  findThis = ifThis.isAugAssign_targetIs(ifThis.isName_Identifier(job.shatteredDataclass.countingVariableName.id))
137
- doThat = Then.replaceWith(Make.Expr(Make.Call(Make.Attribute(Make.Name(job.numbaProgressBarIdentifier),'update'),[Make.Constant(1)])))
44
+ doThat = Then.replaceWith(Make.Expr(Make.Call(Make.Attribute(Make.Name(spices.numbaProgressBarIdentifier),'update'),[Make.Constant(1)])))
138
45
  countWithProgressBar = NodeChanger(findThis, doThat)
139
46
  countWithProgressBar.visit(ingredientsFunction.astFunctionDef)
140
47
 
141
48
  ingredientsModule.appendLauncher(ast.parse(linesLaunch))
142
49
 
143
- return ingredientsModule
50
+ return ingredientsModule, ingredientsFunction
144
51
 
145
- def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: IngredientsFunction, job: Z0Z_RecipeJob) -> IngredientsFunction:
52
+ def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: IngredientsFunction, job: RecipeJob) -> IngredientsFunction:
146
53
  ingredientsFunction.imports.update(job.shatteredDataclass.ledger)
147
54
 
148
55
  list_IdentifiersNotUsed = list_IdentifiersNotUsedHARDCODED
@@ -180,7 +87,7 @@ def move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsFunction: Ingre
180
87
  ast.fix_missing_locations(ingredientsFunction.astFunctionDef)
181
88
  return ingredientsFunction
182
89
 
183
- def makeJobNumba(job: Z0Z_RecipeJob, parametersNumba: ParametersNumba = parametersNumbaDefault):
90
+ def makeJobNumba(job: RecipeJob, spices: SpicesJobNumba):
184
91
  # get the raw ingredients: data and the algorithm
185
92
  ingredientsCount: IngredientsFunction = astModuleToIngredientsFunction(job.source_astModule, job.countCallable)
186
93
 
@@ -203,15 +110,16 @@ def makeJobNumba(job: Z0Z_RecipeJob, parametersNumba: ParametersNumba = paramete
203
110
  NodeChanger(findThis, doThat).visit(ingredientsCount.astFunctionDef)
204
111
 
205
112
  # This launcher eliminates the use of one identifier, so run it now and you can dynamically determine which variables are not used
206
- ingredientsModule = IngredientsModule()
207
- ingredientsModule = addLauncherNumbaProgress(ingredientsModule, ingredientsCount, job)
208
- parametersNumba['nogil'] = True
113
+ if spices.useNumbaProgressBar:
114
+ ingredientsModule = IngredientsModule()
115
+ ingredientsModule, ingredientsCount = addLauncherNumbaProgress(ingredientsModule, ingredientsCount, job, spices)
116
+ spices.parametersNumba['nogil'] = True
209
117
 
210
118
  ingredientsCount = move_arg2FunctionDefDOTbodyAndAssignInitialValues(ingredientsCount, job)
211
119
 
212
120
  ingredientsCount.astFunctionDef.decorator_list = [] # TODO low-priority, handle this more elegantly
213
121
  # TODO when I add the function signature in numba style back to the decorator, the logic needs to handle `ProgressBarType:`
214
- ingredientsCount = decorateCallableWithNumba(ingredientsCount, parametersNumba)
122
+ ingredientsCount = decorateCallableWithNumba(ingredientsCount, spices.parametersNumba)
215
123
 
216
124
  ingredientsModule.appendIngredientsFunction(ingredientsCount)
217
125
 
@@ -241,7 +149,10 @@ def makeJobNumba(job: Z0Z_RecipeJob, parametersNumba: ParametersNumba = paramete
241
149
  """
242
150
 
243
151
  if __name__ == '__main__':
244
- mapShape = (6,6)
152
+ mapShape = (3,4)
245
153
  state = makeInitializedComputationState(mapShape)
246
- aJob = Z0Z_RecipeJob(state)
247
- makeJobNumba(aJob)
154
+ pathModule = PurePosixPath(The.pathPackage, 'jobs')
155
+ pathFilenameFoldsTotal = PurePosixPath(getPathFilenameFoldsTotal(state.mapShape, pathModule))
156
+ aJob = RecipeJob(state, pathModule=pathModule, pathFilenameFoldsTotal=pathFilenameFoldsTotal)
157
+ spices = SpicesJobNumba()
158
+ makeJobNumba(aJob, spices)
@@ -0,0 +1,358 @@
1
+ """
2
+ Numba-specific ingredients for optimized code generation.
3
+
4
+ This module provides specialized tools, constants, and types specifically designed
5
+ for transforming Python code into Numba-accelerated implementations. It implements:
6
+
7
+ 1. A range of Numba jit decorator configurations for different optimization scenarios
8
+ 2. Functions to identify and manipulate Numba decorators in abstract syntax trees
9
+ 3. Utilities for applying appropriate Numba typing to transformed code
10
+ 4. Parameter management for Numba compilation options
11
+
12
+ The configurations range from conservative options that prioritize compatibility and
13
+ error detection to aggressive optimizations that maximize performance at the cost of
14
+ flexibility. While this module specifically targets Numba, its design follows the pattern
15
+ of generic code transformation tools in the package, allowing similar approaches to be
16
+ applied to other acceleration technologies.
17
+
18
+ This module works in conjunction with transformation tools to convert the general-purpose
19
+ algorithm implementation into a highly-optimized Numba version.
20
+ """
21
+
22
+ from collections.abc import Callable, Sequence
23
+ from mapFolding.toolboxFilesystem import getPathFilenameFoldsTotal, getPathRootJobDEFAULT
24
+ from mapFolding.someAssemblyRequired import IngredientsModule, LedgerOfImports, Make, NodeChanger, NodeTourist, RecipeSynthesizeFlow, Then, ast_Identifier, be, ifThis, parsePathFilename2astModule, str_nameDOTname, IngredientsFunction, ShatteredDataclass
25
+ from mapFolding.someAssemblyRequired.transformationTools import Z0Z_inlineThisFunctionWithTheseValues, Z0Z_lameFindReplace, Z0Z_makeDictionaryReplacementStatements, astModuleToIngredientsFunction, shatter_dataclassesDOTdataclass, write_astModule
26
+ from mapFolding.theSSOT import ComputationState
27
+ from numba.core.compiler import CompilerBase as numbaCompilerBase
28
+ from pathlib import Path, PurePosixPath
29
+ from typing import Any, cast, Final, TYPE_CHECKING
30
+ import ast
31
+ import dataclasses
32
+
33
+ try:
34
+ from typing import NotRequired
35
+ except Exception:
36
+ from typing_extensions import NotRequired
37
+
38
+ if TYPE_CHECKING:
39
+ from typing import TypedDict
40
+ else:
41
+ TypedDict = dict[str,Any]
42
+
43
+ theNumbaFlow: RecipeSynthesizeFlow = RecipeSynthesizeFlow()
44
+
45
+ class ParametersNumba(TypedDict):
46
+ _dbg_extend_lifetimes: NotRequired[bool]
47
+ _dbg_optnone: NotRequired[bool]
48
+ _nrt: NotRequired[bool]
49
+ boundscheck: NotRequired[bool]
50
+ cache: bool
51
+ debug: NotRequired[bool]
52
+ error_model: str
53
+ fastmath: bool
54
+ forceinline: bool
55
+ forceobj: NotRequired[bool]
56
+ inline: str
57
+ locals: NotRequired[dict[str, Any]]
58
+ looplift: bool
59
+ no_cfunc_wrapper: bool
60
+ no_cpython_wrapper: bool
61
+ no_rewrites: NotRequired[bool]
62
+ nogil: NotRequired[bool]
63
+ nopython: bool
64
+ parallel: bool
65
+ pipeline_class: NotRequired[type[numbaCompilerBase]]
66
+ signature_or_function: NotRequired[Any | Callable[..., Any] | str | tuple[Any, ...]]
67
+ target: NotRequired[str]
68
+
69
+ parametersNumbaFailEarly: Final[ParametersNumba] = { '_nrt': True, 'boundscheck': True, 'cache': True, 'error_model': 'python', 'fastmath': False, 'forceinline': True, 'inline': 'always', 'looplift': False, 'no_cfunc_wrapper': False, 'no_cpython_wrapper': False, 'nopython': True, 'parallel': False, }
70
+ """For a production function: speed is irrelevant, error discovery is paramount, must be compatible with anything downstream."""
71
+ parametersNumbaDefault: Final[ParametersNumba] = { '_nrt': True, 'boundscheck': False, 'cache': True, 'error_model': 'numpy', 'fastmath': True, 'forceinline': True, 'inline': 'always', 'looplift': False, 'no_cfunc_wrapper': False, 'no_cpython_wrapper': False, 'nopython': True, 'parallel': False, }
72
+ """Middle of the road: fast, lean, but will talk to non-jitted functions."""
73
+ parametersNumbaParallelDEFAULT: Final[ParametersNumba] = { **parametersNumbaDefault, '_nrt': True, 'parallel': True, }
74
+ """Middle of the road: fast, lean, but will talk to non-jitted functions."""
75
+ parametersNumbaSuperJit: Final[ParametersNumba] = { **parametersNumbaDefault, 'no_cfunc_wrapper': True, 'no_cpython_wrapper': True, }
76
+ """Speed, no helmet, no talking to non-jitted functions."""
77
+ parametersNumbaSuperJitParallel: Final[ParametersNumba] = { **parametersNumbaSuperJit, '_nrt': True, 'parallel': True, }
78
+ """Speed, no helmet, concurrency, no talking to non-jitted functions."""
79
+ parametersNumbaMinimum: Final[ParametersNumba] = { '_nrt': True, 'boundscheck': True, 'cache': True, 'error_model': 'numpy', 'fastmath': True, 'forceinline': False, 'inline': 'always', 'looplift': False, 'no_cfunc_wrapper': False, 'no_cpython_wrapper': False, 'nopython': False, 'forceobj': True, 'parallel': False, }
80
+
81
+ Z0Z_numbaDataTypeModule = 'numba'
82
+ Z0Z_decoratorCallable = 'jit'
83
+
84
+ def decorateCallableWithNumba(ingredientsFunction: IngredientsFunction, parametersNumba: ParametersNumba | None = None) -> IngredientsFunction:
85
+ def Z0Z_UnhandledDecorators(astCallable: ast.FunctionDef) -> ast.FunctionDef:
86
+ # TODO: more explicit handling of decorators. I'm able to ignore this because I know `algorithmSource` doesn't have any decorators.
87
+ for decoratorItem in astCallable.decorator_list.copy():
88
+ import warnings
89
+ astCallable.decorator_list.remove(decoratorItem)
90
+ warnings.warn(f"Removed decorator {ast.unparse(decoratorItem)} from {astCallable.name}")
91
+ return astCallable
92
+
93
+ def makeSpecialSignatureForNumba(signatureElement: ast.arg) -> ast.Subscript | ast.Name | None: # type: ignore
94
+ if isinstance(signatureElement.annotation, ast.Subscript) and isinstance(signatureElement.annotation.slice, ast.Tuple):
95
+ annotationShape: ast.expr = signatureElement.annotation.slice.elts[0]
96
+ if isinstance(annotationShape, ast.Subscript) and isinstance(annotationShape.slice, ast.Tuple):
97
+ shapeAsListSlices: list[ast.Slice] = [ast.Slice() for _axis in range(len(annotationShape.slice.elts))]
98
+ shapeAsListSlices[-1] = ast.Slice(step=ast.Constant(value=1))
99
+ shapeAST: ast.Slice | ast.Tuple = ast.Tuple(elts=list(shapeAsListSlices), ctx=ast.Load())
100
+ else:
101
+ shapeAST = ast.Slice(step=ast.Constant(value=1))
102
+
103
+ annotationDtype: ast.expr = signatureElement.annotation.slice.elts[1]
104
+ if (isinstance(annotationDtype, ast.Subscript) and isinstance(annotationDtype.slice, ast.Attribute)):
105
+ datatypeAST = annotationDtype.slice.attr
106
+ else:
107
+ datatypeAST = None
108
+
109
+ ndarrayName = signatureElement.arg
110
+ Z0Z_hacky_dtype: str = ndarrayName
111
+ datatype_attr = datatypeAST or Z0Z_hacky_dtype
112
+ ingredientsFunction.imports.addImportFrom_asStr(datatypeModuleDecorator, datatype_attr)
113
+ datatypeNumba = ast.Name(id=datatype_attr, ctx=ast.Load())
114
+
115
+ return ast.Subscript(value=datatypeNumba, slice=shapeAST, ctx=ast.Load())
116
+
117
+ elif isinstance(signatureElement.annotation, ast.Name):
118
+ return signatureElement.annotation
119
+ return None
120
+
121
+ datatypeModuleDecorator: str = Z0Z_numbaDataTypeModule
122
+ list_argsDecorator: Sequence[ast.expr] = []
123
+
124
+ list_arg4signature_or_function: list[ast.expr] = []
125
+ for parameter in ingredientsFunction.astFunctionDef.args.args:
126
+ # Efficient translation of Python scalar types to Numba types https://github.com/hunterhogan/mapFolding/issues/8
127
+ # For now, let Numba infer them.
128
+ continue
129
+ # signatureElement: ast.Subscript | ast.Name | None = makeSpecialSignatureForNumba(parameter)
130
+ # if signatureElement:
131
+ # list_arg4signature_or_function.append(signatureElement)
132
+
133
+ if ingredientsFunction.astFunctionDef.returns and isinstance(ingredientsFunction.astFunctionDef.returns, ast.Name):
134
+ theReturn: ast.Name = ingredientsFunction.astFunctionDef.returns
135
+ list_argsDecorator = [cast(ast.expr, ast.Call(func=ast.Name(id=theReturn.id, ctx=ast.Load())
136
+ , args=list_arg4signature_or_function if list_arg4signature_or_function else [], keywords=[] ) )]
137
+ elif list_arg4signature_or_function:
138
+ list_argsDecorator = [cast(ast.expr, ast.Tuple(elts=list_arg4signature_or_function, ctx=ast.Load()))]
139
+
140
+ ingredientsFunction.astFunctionDef = Z0Z_UnhandledDecorators(ingredientsFunction.astFunctionDef)
141
+ if parametersNumba is None:
142
+ parametersNumba = parametersNumbaDefault
143
+ listDecoratorKeywords: list[ast.keyword] = [Make.keyword(parameterName, Make.Constant(parameterValue)) for parameterName, parameterValue in parametersNumba.items()]
144
+
145
+ decoratorModule: str = Z0Z_numbaDataTypeModule
146
+ decoratorCallable: str = Z0Z_decoratorCallable
147
+ ingredientsFunction.imports.addImportFrom_asStr(decoratorModule, decoratorCallable)
148
+ # Leave this line in so that global edits will change it.
149
+ astDecorator: ast.Call = Make.Call(Make.Name(decoratorCallable), list_argsDecorator, listDecoratorKeywords)
150
+ astDecorator: ast.Call = Make.Call(Make.Name(decoratorCallable), list_astKeywords=listDecoratorKeywords)
151
+
152
+ ingredientsFunction.astFunctionDef.decorator_list = [astDecorator]
153
+ return ingredientsFunction
154
+
155
+ @dataclasses.dataclass
156
+ class SpicesJobNumba:
157
+ useNumbaProgressBar: bool = True
158
+ numbaProgressBarIdentifier: ast_Identifier = 'ProgressBarGroupsOfFolds'
159
+ parametersNumba = parametersNumbaDefault
160
+
161
+ @dataclasses.dataclass
162
+ class RecipeJob:
163
+ state: ComputationState
164
+ # TODO create function to calculate `foldsTotalEstimated`
165
+ foldsTotalEstimated: int = 0
166
+ shatteredDataclass: ShatteredDataclass = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
167
+
168
+ # ========================================
169
+ # Source
170
+ source_astModule = parsePathFilename2astModule(theNumbaFlow.pathFilenameSequential)
171
+ sourceCountCallable: ast_Identifier = theNumbaFlow.callableSequential
172
+
173
+ sourceLogicalPathModuleDataclass: str_nameDOTname = theNumbaFlow.logicalPathModuleDataclass
174
+ sourceDataclassIdentifier: ast_Identifier = theNumbaFlow.dataclassIdentifier
175
+ sourceDataclassInstance: ast_Identifier = theNumbaFlow.dataclassInstance
176
+
177
+ sourcePathPackage: PurePosixPath | None = theNumbaFlow.pathPackage
178
+ sourcePackageIdentifier: ast_Identifier | None = theNumbaFlow.packageIdentifier
179
+
180
+ # ========================================
181
+ # Filesystem (names of physical objects)
182
+ pathPackage: PurePosixPath | None = None
183
+ pathModule: PurePosixPath | None = PurePosixPath(getPathRootJobDEFAULT())
184
+ """ `pathModule` will override `pathPackage` and `logicalPathRoot`."""
185
+ fileExtension: str = theNumbaFlow.fileExtension
186
+ pathFilenameFoldsTotal: PurePosixPath = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
187
+
188
+ # ========================================
189
+ # Logical identifiers (as opposed to physical identifiers)
190
+ # ========================================
191
+ packageIdentifier: ast_Identifier | None = None
192
+ logicalPathRoot: str_nameDOTname | None = None
193
+ """ `logicalPathRoot` likely corresponds to a physical filesystem directory."""
194
+ moduleIdentifier: ast_Identifier = dataclasses.field(default=None, init=True) # type: ignore[assignment, reportAssignmentType]
195
+ countCallable: ast_Identifier = sourceCountCallable
196
+ dataclassIdentifier: ast_Identifier | None = sourceDataclassIdentifier
197
+ dataclassInstance: ast_Identifier | None = sourceDataclassInstance
198
+ logicalPathModuleDataclass: str_nameDOTname | None = sourceLogicalPathModuleDataclass
199
+
200
+ def _makePathFilename(self,
201
+ pathRoot: PurePosixPath | None = None,
202
+ logicalPathINFIX: str_nameDOTname | None = None,
203
+ filenameStem: str | None = None,
204
+ fileExtension: str | None = None,
205
+ ) -> PurePosixPath:
206
+ if pathRoot is None:
207
+ pathRoot = self.pathPackage or PurePosixPath(Path.cwd())
208
+ if logicalPathINFIX:
209
+ whyIsThisStillAThing: list[str] = logicalPathINFIX.split('.')
210
+ pathRoot = pathRoot.joinpath(*whyIsThisStillAThing)
211
+ if filenameStem is None:
212
+ filenameStem = self.moduleIdentifier
213
+ if fileExtension is None:
214
+ fileExtension = self.fileExtension
215
+ filename: str = filenameStem + fileExtension
216
+ return pathRoot.joinpath(filename)
217
+
218
+ @property
219
+ def pathFilenameModule(self) -> PurePosixPath:
220
+ if self.pathModule is None:
221
+ return self._makePathFilename()
222
+ else:
223
+ return self._makePathFilename(pathRoot=self.pathModule, logicalPathINFIX=None)
224
+
225
+ def __post_init__(self):
226
+ pathFilenameFoldsTotal = PurePosixPath(getPathFilenameFoldsTotal(self.state.mapShape))
227
+
228
+ if self.moduleIdentifier is None:
229
+ self.moduleIdentifier = pathFilenameFoldsTotal.stem
230
+
231
+ if self.pathFilenameFoldsTotal is None:
232
+ self.pathFilenameFoldsTotal = pathFilenameFoldsTotal
233
+
234
+ if self.shatteredDataclass is None and self.logicalPathModuleDataclass and self.dataclassIdentifier and self.dataclassInstance:
235
+ self.shatteredDataclass = shatter_dataclassesDOTdataclass(self.logicalPathModuleDataclass, self.dataclassIdentifier, self.dataclassInstance)
236
+
237
+ # ========================================
238
+ # Fields you probably don't need =================================
239
+ # Dispatcher =================================
240
+ sourceDispatcherCallable: ast_Identifier = theNumbaFlow.callableDispatcher
241
+ dispatcherCallable: ast_Identifier = sourceDispatcherCallable
242
+ # Parallel counting =================================
243
+ sourceDataclassInstanceTaskDistribution: ast_Identifier = theNumbaFlow.dataclassInstanceTaskDistribution
244
+ sourceConcurrencyManagerNamespace: ast_Identifier = theNumbaFlow.concurrencyManagerNamespace
245
+ sourceConcurrencyManagerIdentifier: ast_Identifier = theNumbaFlow.concurrencyManagerIdentifier
246
+ dataclassInstanceTaskDistribution: ast_Identifier = sourceDataclassInstanceTaskDistribution
247
+ concurrencyManagerNamespace: ast_Identifier = sourceConcurrencyManagerNamespace
248
+ concurrencyManagerIdentifier: ast_Identifier = sourceConcurrencyManagerIdentifier
249
+
250
+ def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow) -> None:
251
+ # TODO a tool to automatically remove unused variables from the ArgumentsSpecification (return, and returns) _might_ be nice.
252
+ # Figure out dynamic flow control to synthesized modules https://github.com/hunterhogan/mapFolding/issues/4
253
+
254
+ listAllIngredientsFunctions = [
255
+ (ingredientsInitialize := astModuleToIngredientsFunction(numbaFlow.source_astModule, numbaFlow.sourceCallableInitialize)),
256
+ (ingredientsParallel := astModuleToIngredientsFunction(numbaFlow.source_astModule, numbaFlow.sourceCallableParallel)),
257
+ (ingredientsSequential := astModuleToIngredientsFunction(numbaFlow.source_astModule, numbaFlow.sourceCallableSequential)),
258
+ (ingredientsDispatcher := astModuleToIngredientsFunction(numbaFlow.source_astModule, numbaFlow.sourceCallableDispatcher)),
259
+ ]
260
+
261
+ # Inline functions ========================================================
262
+ dictionaryReplacementStatements = Z0Z_makeDictionaryReplacementStatements(numbaFlow.source_astModule)
263
+ # NOTE Replacements statements are based on the identifiers in the _source_, so operate on the source identifiers.
264
+ ingredientsInitialize.astFunctionDef = Z0Z_inlineThisFunctionWithTheseValues(ingredientsInitialize.astFunctionDef, dictionaryReplacementStatements)
265
+ ingredientsParallel.astFunctionDef = Z0Z_inlineThisFunctionWithTheseValues(ingredientsParallel.astFunctionDef, dictionaryReplacementStatements)
266
+ ingredientsSequential.astFunctionDef = Z0Z_inlineThisFunctionWithTheseValues(ingredientsSequential.astFunctionDef, dictionaryReplacementStatements)
267
+
268
+ # assignRecipeIdentifiersToCallable. =============================
269
+ # TODO How can I use `RecipeSynthesizeFlow` as the SSOT for the pairs of items that may need to be replaced?
270
+ # NOTE reminder: you are updating these `ast.Name` here (and not in a more general search) because this is a
271
+ # narrow search for `ast.Call` so you won't accidentally replace unrelated `ast.Name`.
272
+ listFindReplace = [(numbaFlow.sourceCallableDispatcher, numbaFlow.callableDispatcher),
273
+ (numbaFlow.sourceCallableInitialize, numbaFlow.callableInitialize),
274
+ (numbaFlow.sourceCallableParallel, numbaFlow.callableParallel),
275
+ (numbaFlow.sourceCallableSequential, numbaFlow.callableSequential),]
276
+ for ingredients in listAllIngredientsFunctions:
277
+ for source_Identifier, recipe_Identifier in listFindReplace:
278
+ updateCallName = NodeChanger(ifThis.isCall_Identifier(source_Identifier), Then.DOTfunc(Then.replaceWith(Make.Name(recipe_Identifier))))
279
+ updateCallName.visit(ingredients.astFunctionDef)
280
+
281
+ ingredientsDispatcher.astFunctionDef.name = numbaFlow.callableDispatcher
282
+ ingredientsInitialize.astFunctionDef.name = numbaFlow.callableInitialize
283
+ ingredientsParallel.astFunctionDef.name = numbaFlow.callableParallel
284
+ ingredientsSequential.astFunctionDef.name = numbaFlow.callableSequential
285
+
286
+ # Assign identifiers per the recipe. ==============================
287
+ listFindReplace = [(numbaFlow.sourceDataclassInstance, numbaFlow.dataclassInstance),
288
+ (numbaFlow.sourceDataclassInstanceTaskDistribution, numbaFlow.dataclassInstanceTaskDistribution),
289
+ (numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.concurrencyManagerNamespace),]
290
+ for ingredients in listAllIngredientsFunctions:
291
+ for source_Identifier, recipe_Identifier in listFindReplace:
292
+ updateName = NodeChanger(ifThis.isName_Identifier(source_Identifier), Then.DOTid(Then.replaceWith(recipe_Identifier)))
293
+ update_arg = NodeChanger(ifThis.isArgument_Identifier(source_Identifier), Then.DOTarg(Then.replaceWith(recipe_Identifier)))
294
+ updateName.visit(ingredients.astFunctionDef)
295
+ update_arg.visit(ingredients.astFunctionDef)
296
+
297
+ updateConcurrencyManager = NodeChanger(ifThis.isCallAttributeNamespace_Identifier(numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.sourceConcurrencyManagerIdentifier)
298
+ , Then.DOTfunc(Then.replaceWith(Make.Attribute(Make.Name(numbaFlow.concurrencyManagerNamespace), numbaFlow.concurrencyManagerIdentifier))))
299
+ updateConcurrencyManager.visit(ingredientsDispatcher.astFunctionDef)
300
+
301
+ # shatter Dataclass =======================================================
302
+ instance_Identifier = numbaFlow.dataclassInstance
303
+ getTheOtherRecord_damn = numbaFlow.dataclassInstanceTaskDistribution
304
+ shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, instance_Identifier)
305
+ ingredientsDispatcher.imports.update(shatteredDataclass.ledger)
306
+
307
+ # Change callable parameters and Call to the callable at the same time ====
308
+ # TODO How can I use ast and/or other tools to ensure that when I change a callable, I also change the statements that call the callable?
309
+ # Asked differently, how do I integrate separate statements into a "subroutine", and that subroutine is "atomic/indivisible"?
310
+ # sequentialCallable =========================================================
311
+ ingredientsSequential.astFunctionDef.args = Make.argumentsSpecification(args=shatteredDataclass.list_argAnnotated4ArgumentsSpecification)
312
+ astCallSequentialCallable = Make.Call(Make.Name(numbaFlow.callableSequential), shatteredDataclass.listName4Parameters)
313
+ changeReturnSequentialCallable = NodeChanger(be.Return, Then.replaceWith(Make.Return(shatteredDataclass.fragments4AssignmentOrParameters)))
314
+ ingredientsSequential.astFunctionDef.returns = shatteredDataclass.signatureReturnAnnotation
315
+ replaceAssignSequentialCallable = NodeChanger(ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.callableSequential), Then.replaceWith(Make.Assign(listTargets=[shatteredDataclass.fragments4AssignmentOrParameters], value=astCallSequentialCallable)))
316
+
317
+ unpack4sequentialCallable = NodeChanger(ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.callableSequential), Then.insertThisAbove(shatteredDataclass.listUnpack))
318
+ repack4sequentialCallable = NodeChanger(ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.callableSequential), Then.insertThisBelow([shatteredDataclass.repack]))
319
+
320
+ changeReturnSequentialCallable.visit(ingredientsSequential.astFunctionDef)
321
+ replaceAssignSequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
322
+ unpack4sequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
323
+ repack4sequentialCallable.visit(ingredientsDispatcher.astFunctionDef)
324
+
325
+ ingredientsSequential.astFunctionDef = Z0Z_lameFindReplace(ingredientsSequential.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name) # type: ignore
326
+
327
+ # parallelCallable =========================================================
328
+ ingredientsParallel.astFunctionDef.args = Make.argumentsSpecification(args=shatteredDataclass.list_argAnnotated4ArgumentsSpecification)
329
+ replaceCall2concurrencyManager = NodeChanger(ifThis.isCallAttributeNamespace_Identifier(numbaFlow.concurrencyManagerNamespace, numbaFlow.concurrencyManagerIdentifier), Then.replaceWith(Make.Call(Make.Attribute(Make.Name(numbaFlow.concurrencyManagerNamespace), numbaFlow.concurrencyManagerIdentifier), listArguments=[Make.Name(numbaFlow.callableParallel)] + shatteredDataclass.listName4Parameters)))
330
+
331
+ # NOTE I am dissatisfied with this logic for many reasons, including that it requires separate NodeCollector and NodeReplacer instances.
332
+ astCallConcurrencyResult: list[ast.Call] = []
333
+ get_astCallConcurrencyResult: NodeTourist = NodeTourist(ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(getTheOtherRecord_damn)), lambda node: NodeTourist(be.Call, Then.appendTo(astCallConcurrencyResult)).visit(node)) # pyright: ignore[reportUnknownArgumentType, reportUnknownLambdaType]
334
+ get_astCallConcurrencyResult.visit(ingredientsDispatcher.astFunctionDef)
335
+ replaceAssignParallelCallable = NodeChanger(ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(getTheOtherRecord_damn)), Then.DOTvalue(Then.replaceWith(astCallConcurrencyResult[0])))
336
+ replaceAssignParallelCallable.visit(ingredientsDispatcher.astFunctionDef)
337
+ changeReturnParallelCallable = NodeChanger(be.Return, Then.replaceWith(Make.Return(shatteredDataclass.countingVariableName)))
338
+ ingredientsParallel.astFunctionDef.returns = shatteredDataclass.countingVariableAnnotation
339
+
340
+ unpack4parallelCallable = NodeChanger(ifThis.isAssignAndValueIsCallAttributeNamespace_Identifier(numbaFlow.concurrencyManagerNamespace, numbaFlow.concurrencyManagerIdentifier), Then.insertThisAbove(shatteredDataclass.listUnpack))
341
+
342
+ unpack4parallelCallable.visit(ingredientsDispatcher.astFunctionDef)
343
+ replaceCall2concurrencyManager.visit(ingredientsDispatcher.astFunctionDef)
344
+ changeReturnParallelCallable.visit(ingredientsParallel.astFunctionDef)
345
+
346
+ ingredientsParallel.astFunctionDef = Z0Z_lameFindReplace(ingredientsParallel.astFunctionDef, shatteredDataclass.map_stateDOTfield2Name) # type: ignore
347
+
348
+ # numba decorators =========================================
349
+ ingredientsParallel = decorateCallableWithNumba(ingredientsParallel)
350
+ ingredientsSequential = decorateCallableWithNumba(ingredientsSequential)
351
+
352
+ # Module-level transformations ===========================================================
353
+ ingredientsModuleNumbaUnified = IngredientsModule(ingredientsFunction=listAllIngredientsFunctions, imports=LedgerOfImports(numbaFlow.source_astModule))
354
+
355
+ write_astModule(ingredientsModuleNumbaUnified, numbaFlow.pathFilenameDispatcher, numbaFlow.packageIdentifier)
356
+
357
+ if __name__ == '__main__':
358
+ makeNumbaFlow(theNumbaFlow)