mapFolding 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,805 @@
1
+ from mapFolding import (
2
+ computationState,
3
+ EnumIndices,
4
+ getAlgorithmSource,
5
+ getFilenameFoldsTotal,
6
+ getPathFilenameFoldsTotal,
7
+ getPathJobRootDEFAULT,
8
+ getPathSyntheticModules,
9
+ hackSSOTdatatype,
10
+ indexMy,
11
+ indexTrack,
12
+ moduleOfSyntheticModules,
13
+ myPackageNameIs,
14
+ ParametersNumba,
15
+ parametersNumbaDEFAULT,
16
+ parametersNumbaFailEarly,
17
+ parametersNumbaSuperJit,
18
+ parametersNumbaSuperJitParallel,
19
+ setDatatypeElephino,
20
+ setDatatypeFoldsTotal,
21
+ setDatatypeLeavesTotal,
22
+ setDatatypeModule,
23
+ )
24
+ from collections import namedtuple
25
+ from mapFolding.someAssemblyRequired import makeStateJob
26
+ from types import ModuleType
27
+ from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple, Type, Union, cast
28
+ import ast
29
+ import collections
30
+ import inspect
31
+ import autoflake
32
+ import more_itertools
33
+ import numba
34
+ import numpy
35
+ import os
36
+ import pathlib
37
+ import python_minifier
38
+
39
+ youOughtaKnow = namedtuple('youOughtaKnow', ['callableSynthesized', 'pathFilenameForMe', 'astForCompetentProgrammers'])
40
+
41
+ class UniversalImportTracker:
42
+ def __init__(self):
43
+ self.dictionaryImportFrom = collections.defaultdict(set)
44
+ self.setImport = set()
45
+
46
+ def addAst(self, astImport_: Union[ast.Import, ast.ImportFrom]) -> None:
47
+ if isinstance(astImport_, ast.Import):
48
+ for alias in astImport_.names:
49
+ self.setImport.add(alias.name)
50
+ elif isinstance(astImport_, ast.ImportFrom):
51
+ self.dictionaryImportFrom[astImport_.module].update(alias.name for alias in astImport_.names)
52
+
53
+ def addImportFromStr(self, module: str, name: str) -> None:
54
+ self.dictionaryImportFrom[module].add(name)
55
+
56
+ def addImportStr(self, name: str) -> None:
57
+ self.setImport.add(name)
58
+
59
+ def makeListAst(self) -> List[Union[ast.ImportFrom, ast.Import]]:
60
+ listAstImportFrom = [ast.ImportFrom(module=module, names=[ast.alias(name=name, asname=None)], level=0) for module, names in self.dictionaryImportFrom.items() for name in names]
61
+ listAstImport = [ast.Import(names=[ast.alias(name=name, asname=None)]) for name in self.setImport]
62
+ return listAstImportFrom + listAstImport
63
+ class NodeReplacer(ast.NodeTransformer):
64
+ """Base class for configurable node replacement"""
65
+ def __init__(self, findMe, nodeReplacementBuilder):
66
+ self.findMe = findMe
67
+ self.nodeReplacementBuilder = nodeReplacementBuilder
68
+
69
+ def visit(self, node: ast.AST) -> ast.AST:
70
+ if self.findMe(node):
71
+ return self.nodeReplacementBuilder(node)
72
+ return super().visit(node)
73
+
74
+ class ArgumentProcessor:
75
+ """Unified argument processing using transformation rules"""
76
+ def __init__(self, rules: List[Tuple[Callable[[ast.arg], bool], Callable]]):
77
+ self.rules = rules # (predicate, transformation)
78
+
79
+ def process(self, FunctionDef: ast.FunctionDef) -> ast.FunctionDef:
80
+ for arg in FunctionDef.args.args.copy():
81
+ for predicate, transform in self.rules:
82
+ if predicate(arg):
83
+ FunctionDef = transform(FunctionDef, arg)
84
+ return FunctionDef
85
+
86
+ def Z0Z_UnhandledDecorators(astCallable: ast.FunctionDef) -> ast.FunctionDef:
87
+ # TODO: more explicit handling of decorators. I'm able to ignore this because I know `algorithmSource` doesn't have any decorators.
88
+ for decoratorItem in astCallable.decorator_list.copy():
89
+ import warnings
90
+ astCallable.decorator_list.remove(decoratorItem)
91
+ warnings.warn(f"Removed decorator {ast.unparse(decoratorItem)} from {astCallable.name}")
92
+ return astCallable
93
+
94
+ class RecursiveInliner(ast.NodeTransformer):
95
+ """
96
+ Class RecursiveInliner:
97
+ A custom AST NodeTransformer designed to recursively inline function calls from a given dictionary
98
+ of function definitions into the AST. Once a particular function has been inlined, it is marked
99
+ as completed to avoid repeated inlining. This transformation modifies the AST in-place by substituting
100
+ eligible function calls with the body of their corresponding function.
101
+ Attributes:
102
+ dictionaryFunctions (Dict[str, ast.FunctionDef]):
103
+ A mapping of function name to its AST definition, used as a source for inlining.
104
+ callablesCompleted (Set[str]):
105
+ A set to track function names that have already been inlined to prevent multiple expansions.
106
+ Methods:
107
+ inlineFunctionBody(callableTargetName: str) -> Optional[ast.FunctionDef]:
108
+ Retrieves the AST definition for a given function name from dictionaryFunctions
109
+ and recursively inlines any function calls within it. Returns the function definition
110
+ that was inlined or None if the function was already processed.
111
+ visit_Call(callNode: ast.Call) -> ast.AST:
112
+ Inspects calls within the AST. If a function call matches one in dictionaryFunctions,
113
+ it is replaced by the inlined body. If the last statement in the inlined body is a return
114
+ or an expression, that value or expression is substituted; otherwise, a constant is returned.
115
+ visit_Expr(node: ast.Expr) -> Union[ast.AST, List[ast.AST]]:
116
+ Handles expression nodes in the AST. If the expression is a function call from
117
+ dictionaryFunctions, its statements are expanded in place, effectively inlining
118
+ the called function's statements into the surrounding context.
119
+ """
120
+ def __init__(self, dictionaryFunctions: Dict[str, ast.FunctionDef]):
121
+ self.dictionaryFunctions = dictionaryFunctions
122
+ self.callablesCompleted: Set[str] = set()
123
+
124
+ def inlineFunctionBody(self, callableTargetName: str) -> Optional[ast.FunctionDef]:
125
+ if (callableTargetName in self.callablesCompleted):
126
+ return None
127
+
128
+ self.callablesCompleted.add(callableTargetName)
129
+ inlineDefinition = self.dictionaryFunctions[callableTargetName]
130
+ for astNode in ast.walk(inlineDefinition):
131
+ self.visit(astNode)
132
+ return inlineDefinition
133
+
134
+ def visit_Call(self, node: ast.Call) -> ast.AST:
135
+ callNodeVisited = self.generic_visit(node)
136
+ if (isinstance(callNodeVisited, ast.Call) and isinstance(callNodeVisited.func, ast.Name) and callNodeVisited.func.id in self.dictionaryFunctions):
137
+ inlineDefinition = self.inlineFunctionBody(callNodeVisited.func.id)
138
+ if (inlineDefinition and inlineDefinition.body):
139
+ statementTerminating = inlineDefinition.body[-1]
140
+ if (isinstance(statementTerminating, ast.Return) and statementTerminating.value is not None):
141
+ return self.visit(statementTerminating.value)
142
+ elif (isinstance(statementTerminating, ast.Expr) and statementTerminating.value is not None):
143
+ return self.visit(statementTerminating.value)
144
+ return ast.Constant(value=None)
145
+ return callNodeVisited
146
+
147
+ def visit_Expr(self, node: ast.Expr) -> Union[ast.AST, List[ast.AST]]:
148
+ if (isinstance(node.value, ast.Call)):
149
+ if (isinstance(node.value.func, ast.Name) and node.value.func.id in self.dictionaryFunctions):
150
+ inlineDefinition = self.inlineFunctionBody(node.value.func.id)
151
+ if (inlineDefinition):
152
+ return [self.visit(stmt) for stmt in inlineDefinition.body]
153
+ return self.generic_visit(node)
154
+
155
+ class UnpackArrayAccesses(ast.NodeTransformer):
156
+ """
157
+ A class that transforms array accesses using enum indices into local variables.
158
+
159
+ This AST transformer identifies array accesses using enum indices and replaces them
160
+ with local variables, adding initialization statements at the start of functions.
161
+
162
+ Parameters:
163
+ enumIndexClass (Type[EnumIndices]): The enum class used for array indexing
164
+ arrayName (str): The name of the array being accessed
165
+
166
+ Attributes:
167
+ enumIndexClass (Type[EnumIndices]): Stored enum class for index lookups
168
+ arrayName (str): Name of the array being transformed
169
+ substitutions (dict): Tracks variable substitutions and their original nodes
170
+
171
+ The transformer handles two main cases:
172
+ 1. Scalar array access - array[EnumIndices.MEMBER]
173
+ 2. Array slice access - array[EnumIndices.MEMBER, other_indices...]
174
+ For each identified access pattern, it:
175
+ 1. Creates a local variable named after the enum member
176
+ 2. Adds initialization code at function start
177
+ 3. Replaces original array access with the local variable
178
+ """
179
+
180
+ def __init__(self, enumIndexClass: Type[EnumIndices], arrayName: str):
181
+ self.enumIndexClass = enumIndexClass
182
+ self.arrayName = arrayName
183
+ self.substitutions = {}
184
+
185
+ def extract_member_name(self, node: ast.AST) -> Optional[str]:
186
+ """Recursively extract enum member name from any node in the AST."""
187
+ if isinstance(node, ast.Attribute) and node.attr == 'value':
188
+ innerAttribute = node.value
189
+ while isinstance(innerAttribute, ast.Attribute):
190
+ if (isinstance(innerAttribute.value, ast.Name) and innerAttribute.value.id == self.enumIndexClass.__name__):
191
+ return innerAttribute.attr
192
+ innerAttribute = innerAttribute.value
193
+ return None
194
+
195
+ def transform_slice_element(self, node: ast.AST) -> ast.AST:
196
+ """Transform any enum references within a slice element."""
197
+ if isinstance(node, ast.Subscript):
198
+ if isinstance(node.slice, ast.Attribute):
199
+ member_name = self.extract_member_name(node.slice)
200
+ if member_name:
201
+ return ast.Name(id=member_name, ctx=node.ctx)
202
+ elif isinstance(node, ast.Tuple):
203
+ # Handle tuple slices by transforming each element
204
+ return ast.Tuple(elts=cast(List[ast.expr], [self.transform_slice_element(elt) for elt in node.elts]), ctx=node.ctx)
205
+ elif isinstance(node, ast.Attribute):
206
+ member_name = self.extract_member_name(node)
207
+ if member_name:
208
+ return ast.Name(id=member_name, ctx=ast.Load())
209
+ return node
210
+
211
+ def visit_Subscript(self, node: ast.Subscript) -> ast.AST:
212
+ # Recursively visit any nested subscripts in value or slice
213
+ node.value = self.visit(node.value)
214
+ node.slice = self.visit(node.slice)
215
+ # If node.value is not our arrayName, just return node
216
+ if not (isinstance(node.value, ast.Name) and node.value.id == self.arrayName):
217
+ return node
218
+
219
+ # Handle scalar array access
220
+ if isinstance(node.slice, ast.Attribute):
221
+ memberName = self.extract_member_name(node.slice)
222
+ if memberName:
223
+ self.substitutions[memberName] = ('scalar', node)
224
+ return ast.Name(id=memberName, ctx=ast.Load())
225
+
226
+ # Handle array slice access
227
+ if isinstance(node.slice, ast.Tuple) and node.slice.elts:
228
+ firstElement = node.slice.elts[0]
229
+ memberName = self.extract_member_name(firstElement)
230
+ sliceRemainder = [self.visit(elem) for elem in node.slice.elts[1:]]
231
+ if memberName:
232
+ self.substitutions[memberName] = ('array', node)
233
+ if len(sliceRemainder) == 0:
234
+ return ast.Name(id=memberName, ctx=ast.Load())
235
+ return ast.Subscript(value=ast.Name(id=memberName, ctx=ast.Load()), slice=ast.Tuple(elts=sliceRemainder, ctx=ast.Load()) if len(sliceRemainder) > 1 else sliceRemainder[0], ctx=ast.Load())
236
+
237
+ # If single-element tuple, unwrap
238
+ if isinstance(node.slice, ast.Tuple) and len(node.slice.elts) == 1:
239
+ node.slice = node.slice.elts[0]
240
+
241
+ return node
242
+
243
+ def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef:
244
+ node = cast(ast.FunctionDef, self.generic_visit(node))
245
+
246
+ initializations = []
247
+ for name, (kind, original_node) in self.substitutions.items():
248
+ if kind == 'scalar':
249
+ initializations.append(ast.Assign(targets=[ast.Name(id=name, ctx=ast.Store())], value=original_node))
250
+ else: # array
251
+ initializations.append(
252
+ ast.Assign(
253
+ targets=[ast.Name(id=name, ctx=ast.Store())],
254
+ value=ast.Subscript(value=ast.Name(id=self.arrayName, ctx=ast.Load()),
255
+ slice=ast.Attribute(value=ast.Attribute(
256
+ value=ast.Name(id=self.enumIndexClass.__name__, ctx=ast.Load()),
257
+ attr=name, ctx=ast.Load()), attr='value', ctx=ast.Load()), ctx=ast.Load())))
258
+
259
+ node.body = initializations + node.body
260
+ return node
261
+
262
+ def decorateCallableWithNumba(FunctionDefTarget: ast.FunctionDef, parametersNumba: Optional[ParametersNumba]=None) -> ast.FunctionDef:
263
+ def makeNumbaParameterSignatureElement(signatureElement: ast.arg):
264
+ if isinstance(signatureElement.annotation, ast.Subscript) and isinstance(signatureElement.annotation.slice, ast.Tuple):
265
+ annotationShape = signatureElement.annotation.slice.elts[0]
266
+ if isinstance(annotationShape, ast.Subscript) and isinstance(annotationShape.slice, ast.Tuple):
267
+ shapeAsListSlices: Sequence[ast.expr] = [ast.Slice() for axis in range(len(annotationShape.slice.elts))]
268
+ shapeAsListSlices[-1] = ast.Slice(step=ast.Constant(value=1))
269
+ shapeAST = ast.Tuple(elts=list(shapeAsListSlices), ctx=ast.Load())
270
+ else:
271
+ shapeAST = ast.Slice(step=ast.Constant(value=1))
272
+
273
+ annotationDtype = signatureElement.annotation.slice.elts[1]
274
+ if (isinstance(annotationDtype, ast.Subscript) and isinstance(annotationDtype.slice, ast.Attribute)):
275
+ datatypeAST = annotationDtype.slice.attr
276
+ else:
277
+ datatypeAST = None
278
+
279
+ ndarrayName = signatureElement.arg
280
+ Z0Z_hacky_dtype = hackSSOTdatatype(ndarrayName)
281
+ datatype_attr = datatypeAST or Z0Z_hacky_dtype
282
+ allImports.addImportFromStr(datatypeModuleDecorator, datatype_attr)
283
+ datatypeNumba = ast.Name(id=datatype_attr, ctx=ast.Load())
284
+
285
+ return ast.Subscript(value=datatypeNumba, slice=shapeAST, ctx=ast.Load())
286
+
287
+ FunctionDefTarget = Z0Z_UnhandledDecorators(FunctionDefTarget)
288
+
289
+ listNumbaParameterSignature: Sequence[ast.expr] = []
290
+ for parameter in FunctionDefTarget.args.args:
291
+ signatureElement = makeNumbaParameterSignatureElement(parameter)
292
+ if (signatureElement):
293
+ listNumbaParameterSignature.append(signatureElement)
294
+
295
+ astTupleSignatureParameters = ast.Tuple(elts=listNumbaParameterSignature, ctx=ast.Load())
296
+
297
+ # TODO if `astCallable` has a return, the return needs to be added to `astArgsNumbaSignature` in the appropriate place
298
+ # The return, when placed in the args, is treated as a `Call`. This is logical because numba is converting to machine code.
299
+ # , args=[Call(func=Name(id='int64', ctx=Load()))]
300
+ ast_argsSignature = astTupleSignatureParameters
301
+
302
+ ImaReturn = next((node for node in FunctionDefTarget.body if isinstance(node, ast.Return)), None)
303
+ # Return(value=Name(id='groupsOfFolds', ctx=Load()))]
304
+ if ImaReturn is not None and isinstance(ImaReturn.value, ast.Name):
305
+ my_idIf_I_wereA_astCall_func_astName_idParameter = ImaReturn.value.id
306
+ ast_argsSignature = ast.Call(
307
+ func=ast.Name(id=my_idIf_I_wereA_astCall_func_astName_idParameter, ctx=ast.Load()),
308
+ args=[astTupleSignatureParameters],
309
+ keywords=[]
310
+ )
311
+ else:
312
+ ast_argsSignature = astTupleSignatureParameters
313
+
314
+ if parametersNumba is None:
315
+ parametersNumba = parametersNumbaDEFAULT
316
+
317
+ listKeywordsNumbaSignature = [ast.keyword(arg=parameterName, value=ast.Constant(value=parameterValue)) for parameterName, parameterValue in parametersNumba.items()]
318
+ allImports.addImportFromStr(datatypeModuleDecorator, 'jit')
319
+ astDecoratorNumba = ast.Call(func=ast.Name(id='jit', ctx=ast.Load()), args=[ast_argsSignature], keywords=listKeywordsNumbaSignature)
320
+
321
+ FunctionDefTarget.decorator_list = [astDecoratorNumba]
322
+ return FunctionDefTarget
323
+
324
+ def inlineOneCallable(pythonSource: str, callableTarget: str) -> str | None:
325
+ astModule: ast.Module = ast.parse(pythonSource, type_comments=True)
326
+
327
+ for statement in astModule.body:
328
+ if isinstance(statement, (ast.Import, ast.ImportFrom)):
329
+ allImports.addAst(statement)
330
+
331
+ dictionaryFunctionDef = {statement.name: statement for statement in astModule.body if isinstance(statement, ast.FunctionDef)}
332
+ callableInlinerWorkhorse = RecursiveInliner(dictionaryFunctionDef)
333
+ FunctionDefTarget = callableInlinerWorkhorse.inlineFunctionBody(callableTarget)
334
+
335
+ if FunctionDefTarget:
336
+ ast.fix_missing_locations(FunctionDefTarget)
337
+ parametersNumba = None
338
+ match callableTarget:
339
+ case 'countParallel':
340
+ parametersNumba = parametersNumbaSuperJitParallel
341
+ case 'countSequential':
342
+ parametersNumba = parametersNumbaSuperJit
343
+ case 'countInitialize':
344
+ parametersNumba = parametersNumbaDEFAULT
345
+
346
+ FunctionDefTarget = decorateCallableWithNumba(FunctionDefTarget, parametersNumba)
347
+
348
+ if callableTarget == 'countSequential':
349
+ unpackerMy = UnpackArrayAccesses(indexMy, 'my')
350
+ FunctionDefTarget = cast(ast.FunctionDef, unpackerMy.visit(FunctionDefTarget))
351
+ ast.fix_missing_locations(FunctionDefTarget)
352
+
353
+ unpackerTrack = UnpackArrayAccesses(indexTrack, 'track')
354
+ FunctionDefTarget = cast(ast.FunctionDef, unpackerTrack.visit(FunctionDefTarget))
355
+ ast.fix_missing_locations(FunctionDefTarget)
356
+
357
+ moduleAST = ast.Module(body=cast(List[ast.stmt], allImports.makeListAst() + [FunctionDefTarget]), type_ignores=[])
358
+ ast.fix_missing_locations(moduleAST)
359
+ moduleSource = ast.unparse(moduleAST)
360
+ return moduleSource
361
+
362
+ def makeDispatcherNumba(pythonSource: str, callableTarget: str, listStuffYouOughtaKnow: List[youOughtaKnow]) -> str:
363
+ astSource = ast.parse(pythonSource)
364
+
365
+ for statement in astSource.body:
366
+ if isinstance(statement, (ast.Import, ast.ImportFrom)):
367
+ allImports.addAst(statement)
368
+
369
+ for stuff in listStuffYouOughtaKnow:
370
+ statement = stuff.astForCompetentProgrammers
371
+ if isinstance(statement, (ast.Import, ast.ImportFrom)):
372
+ allImports.addAst(statement)
373
+
374
+ FunctionDefTarget = next((node for node in astSource.body if isinstance(node, ast.FunctionDef) and node.name == callableTarget), None)
375
+
376
+ if not FunctionDefTarget:
377
+ raise ValueError(f"Could not find function {callableTarget} in source code")
378
+
379
+ # Zero-out the decorator list
380
+ FunctionDefTarget = Z0Z_UnhandledDecorators(FunctionDefTarget)
381
+
382
+ FunctionDefTarget = decorateCallableWithNumba(FunctionDefTarget, parametersNumbaFailEarly)
383
+
384
+ astModule = ast.Module( body=cast(List[ast.stmt], allImports.makeListAst()
385
+ + [FunctionDefTarget]), type_ignores=[])
386
+
387
+ ast.fix_missing_locations(astModule)
388
+ return ast.unparse(astModule)
389
+
390
+ def makeNumbaOptimizedFlow(listCallablesInline: List[str], callableDispatcher: Optional[str] = None, algorithmSource: Optional[ModuleType] = None) -> None:
391
+ if not algorithmSource:
392
+ algorithmSource = getAlgorithmSource()
393
+
394
+ formatModuleNameDEFAULT = "numba_{callableTarget}"
395
+
396
+ # When I am a more competent programmer, I will make getPathFilenameWrite dependent on makeAstImport or vice versa,
397
+ # so the name of the physical file doesn't get out of whack with the name of the logical module.
398
+ def getPathFilenameWrite(callableTarget: str
399
+ , pathWrite: Optional[pathlib.Path] = None
400
+ , formatFilenameWrite: Optional[str] = None
401
+ ) -> pathlib.Path:
402
+ if not pathWrite:
403
+ pathWrite = getPathSyntheticModules()
404
+ if not formatFilenameWrite:
405
+ formatFilenameWrite = formatModuleNameDEFAULT + '.py'
406
+
407
+ pathFilename = pathWrite / formatFilenameWrite.format(callableTarget=callableTarget)
408
+ return pathFilename
409
+
410
+ def makeAstImport(callableTarget: str
411
+ , packageName: Optional[str] = None
412
+ , subPackageName: Optional[str] = None
413
+ , moduleName: Optional[str] = None
414
+ , astNodeLogicalPathThingy: Optional[ast.AST] = None
415
+ ) -> ast.ImportFrom:
416
+ """Creates import AST node for synthetic modules."""
417
+ if astNodeLogicalPathThingy is None:
418
+ if packageName is None:
419
+ packageName = myPackageNameIs
420
+ if subPackageName is None:
421
+ subPackageName = moduleOfSyntheticModules
422
+ if moduleName is None:
423
+ moduleName = formatModuleNameDEFAULT.format(callableTarget=callableTarget)
424
+ module=f'{packageName}.{subPackageName}.{moduleName}'
425
+ else:
426
+ module = str(astNodeLogicalPathThingy)
427
+ return ast.ImportFrom(
428
+ module=module,
429
+ names=[ast.alias(name=callableTarget, asname=None)],
430
+ level=0
431
+ )
432
+
433
+ listStuffYouOughtaKnow: List[youOughtaKnow] = []
434
+
435
+ global allImports
436
+ for callableTarget in listCallablesInline:
437
+ allImports = UniversalImportTracker()
438
+ pythonSource = inspect.getsource(algorithmSource)
439
+ pythonSource = inlineOneCallable(pythonSource, callableTarget)
440
+ if not pythonSource:
441
+ raise Exception("Pylance, OMG! The sky is falling!")
442
+
443
+ pathFilename = getPathFilenameWrite(callableTarget)
444
+
445
+ listStuffYouOughtaKnow.append(youOughtaKnow(
446
+ callableSynthesized=callableTarget,
447
+ pathFilenameForMe=pathFilename,
448
+ astForCompetentProgrammers=makeAstImport(callableTarget)
449
+ ))
450
+ pythonSource = autoflake.fix_code(pythonSource, ['mapFolding', 'numba', 'numpy'])
451
+ pathFilename.write_text(pythonSource)
452
+
453
+ # Generate dispatcher if requested
454
+ if callableDispatcher:
455
+ allImports = UniversalImportTracker()
456
+ pythonSource = inspect.getsource(algorithmSource)
457
+ pythonSource = makeDispatcherNumba(pythonSource, callableDispatcher, listStuffYouOughtaKnow)
458
+ if not pythonSource:
459
+ raise Exception("Pylance, OMG! The sky is falling!")
460
+
461
+ pathFilename = getPathFilenameWrite(callableDispatcher)
462
+
463
+ listStuffYouOughtaKnow.append(youOughtaKnow(
464
+ callableSynthesized=callableDispatcher,
465
+ pathFilenameForMe=pathFilename,
466
+ astForCompetentProgrammers=makeAstImport(callableDispatcher)
467
+ ))
468
+ pythonSource = autoflake.fix_code(pythonSource, ['mapFolding', 'numba', 'numpy'])
469
+ pathFilename.write_text(pythonSource)
470
+
471
+ def makeStrRLEcompacted(arrayTarget: numpy.ndarray, identifierName: Optional[str]=None) -> str:
472
+ """Converts a NumPy array into a compressed string representation using run-length encoding (RLE).
473
+
474
+ This function takes a NumPy array and converts it into an optimized string representation by:
475
+ 1. Compressing consecutive sequences of numbers into range objects
476
+ 2. Minimizing repeated zeros using array multiplication syntax
477
+ 3. Converting the result into a valid Python array initialization statement
478
+
479
+ Parameters:
480
+ arrayTarget (numpy.ndarray): The input NumPy array to be converted
481
+ identifierName (str): The variable name to use in the output string
482
+
483
+ Returns:
484
+ str: A string containing Python code that recreates the input array in compressed form.
485
+ Format: "{identifierName} = numpy.array({compressed_data}, dtype=numpy.{dtype})"
486
+ """
487
+
488
+ def compressRangesNDArrayNoFlatten(arraySlice):
489
+ if isinstance(arraySlice, numpy.ndarray) and arraySlice.ndim > 1:
490
+ return [compressRangesNDArrayNoFlatten(arraySlice[index]) for index in range(arraySlice.shape[0])]
491
+ elif isinstance(arraySlice, numpy.ndarray) and arraySlice.ndim == 1:
492
+ listWithRanges = []
493
+ for group in more_itertools.consecutive_groups(arraySlice.tolist()):
494
+ ImaSerious = list(group)
495
+ if len(ImaSerious) <= 4:
496
+ listWithRanges += ImaSerious
497
+ else:
498
+ ImaRange = [range(ImaSerious[0], ImaSerious[-1] + 1)]
499
+ listWithRanges += ImaRange
500
+ return listWithRanges
501
+ return arraySlice
502
+
503
+ arrayAsNestedLists = compressRangesNDArrayNoFlatten(arrayTarget)
504
+
505
+ stringMinimized = python_minifier.minify(str(arrayAsNestedLists))
506
+ commaZeroMaximum = arrayTarget.shape[-1] - 1
507
+ stringMinimized = stringMinimized.replace('[0' + ',0'*commaZeroMaximum + ']', '[0]*'+str(commaZeroMaximum+1))
508
+ for countZeros in range(commaZeroMaximum, 2, -1):
509
+ stringMinimized = stringMinimized.replace(',0'*countZeros + ']', ']+[0]*'+str(countZeros))
510
+
511
+ stringMinimized = stringMinimized.replace('range', '*range')
512
+
513
+ if identifierName:
514
+ return f"{identifierName} = array({stringMinimized}, dtype={arrayTarget.dtype})"
515
+ return stringMinimized
516
+
517
+ def moveArrayTo_body(FunctionDefTarget: ast.FunctionDef, astArg: ast.arg, argData: numpy.ndarray) -> ast.FunctionDef:
518
+ arrayType = type(argData)
519
+ moduleConstructor = arrayType.__module__
520
+ constructorName = arrayType.__name__
521
+ # NOTE hack
522
+ constructorName = constructorName.replace('ndarray', 'array')
523
+ argData_dtype: numpy.dtype = argData.dtype
524
+ argData_dtypeName = argData.dtype.name
525
+
526
+ allImports.addImportFromStr(moduleConstructor, constructorName)
527
+ allImports.addImportFromStr(moduleConstructor, argData_dtypeName)
528
+
529
+ onlyDataRLE = makeStrRLEcompacted(argData)
530
+ astStatement = cast(ast.Expr, ast.parse(onlyDataRLE).body[0])
531
+ dataAst = astStatement.value
532
+
533
+ arrayCall = ast.Call(
534
+ func=ast.Name(id=constructorName, ctx=ast.Load())
535
+ , args=[dataAst]
536
+ , keywords=[ast.keyword(arg='dtype' , value=ast.Name(id=argData_dtypeName , ctx=ast.Load()) ) ] )
537
+
538
+ assignment = ast.Assign( targets=[ast.Name(id=astArg.arg, ctx=ast.Store())], value=arrayCall )
539
+ FunctionDefTarget.body.insert(0, assignment)
540
+ FunctionDefTarget.args.args.remove(astArg)
541
+
542
+ return FunctionDefTarget
543
+
544
+ def evaluateArrayIn_body(FunctionDefTarget: ast.FunctionDef, astArg: ast.arg, argData: numpy.ndarray) -> ast.FunctionDef:
545
+ arrayType = type(argData)
546
+ moduleConstructor = arrayType.__module__
547
+ constructorName = arrayType.__name__
548
+ # NOTE hack
549
+ constructorName = constructorName.replace('ndarray', 'array')
550
+ allImports.addImportFromStr(moduleConstructor, constructorName)
551
+
552
+ for stmt in FunctionDefTarget.body.copy():
553
+ if isinstance(stmt, ast.Assign):
554
+ if isinstance(stmt.targets[0], ast.Name) and isinstance(stmt.value, ast.Subscript):
555
+ astAssignee: ast.Name = stmt.targets[0]
556
+ argData_dtypeName = hackSSOTdatatype(astAssignee.id)
557
+ allImports.addImportFromStr(moduleConstructor, argData_dtypeName)
558
+ astSubscript: ast.Subscript = stmt.value
559
+ if isinstance(astSubscript.value, ast.Name) and astSubscript.value.id == astArg.arg and isinstance(astSubscript.slice, ast.Attribute):
560
+ indexAs_astAttribute: ast.Attribute = astSubscript.slice
561
+ indexAsStr = ast.unparse(indexAs_astAttribute)
562
+ argDataSlice = argData[eval(indexAsStr)]
563
+
564
+ onlyDataRLE = makeStrRLEcompacted(argDataSlice)
565
+ astStatement = cast(ast.Expr, ast.parse(onlyDataRLE).body[0])
566
+ dataAst = astStatement.value
567
+
568
+ arrayCall = ast.Call(
569
+ func=ast.Name(id=constructorName, ctx=ast.Load()) , args=[dataAst]
570
+ , keywords=[ast.keyword(arg='dtype', value=ast.Name(id=argData_dtypeName, ctx=ast.Load()) ) ] )
571
+
572
+ assignment = ast.Assign( targets=[astAssignee], value=arrayCall )
573
+ FunctionDefTarget.body.insert(0, assignment)
574
+ FunctionDefTarget.body.remove(stmt)
575
+
576
+ FunctionDefTarget.args.args.remove(astArg)
577
+ return FunctionDefTarget
578
+
579
+ def evaluate_argIn_body(FunctionDefTarget: ast.FunctionDef, astArg: ast.arg, argData: numpy.ndarray, Z0Z_listChaff: List[str]) -> ast.FunctionDef:
580
+ moduleConstructor = datatypeModuleScalar
581
+ for stmt in FunctionDefTarget.body.copy():
582
+ if isinstance(stmt, ast.Assign):
583
+ if isinstance(stmt.targets[0], ast.Name) and isinstance(stmt.value, ast.Subscript):
584
+ astAssignee: ast.Name = stmt.targets[0]
585
+ argData_dtypeName = hackSSOTdatatype(astAssignee.id)
586
+ allImports.addImportFromStr(moduleConstructor, argData_dtypeName)
587
+ astSubscript: ast.Subscript = stmt.value
588
+ if isinstance(astSubscript.value, ast.Name) and astSubscript.value.id == astArg.arg and isinstance(astSubscript.slice, ast.Attribute):
589
+ indexAs_astAttribute: ast.Attribute = astSubscript.slice
590
+ indexAsStr = ast.unparse(indexAs_astAttribute)
591
+ argDataSlice: int = argData[eval(indexAsStr)].item()
592
+ astCall = ast.Call(func=ast.Name(id=argData_dtypeName, ctx=ast.Load()) , args=[ast.Constant(value=argDataSlice)], keywords=[])
593
+ assignment = ast.Assign(targets=[astAssignee], value=astCall)
594
+ if astAssignee.id not in Z0Z_listChaff:
595
+ FunctionDefTarget.body.insert(0, assignment)
596
+ FunctionDefTarget.body.remove(stmt)
597
+ FunctionDefTarget.args.args.remove(astArg)
598
+ return FunctionDefTarget
599
+
600
+ def evaluateAnnAssignIn_body(FunctionDefTarget: ast.FunctionDef) -> ast.FunctionDef:
601
+ moduleConstructor = datatypeModuleScalar
602
+ for stmt in FunctionDefTarget.body.copy():
603
+ if isinstance(stmt, ast.AnnAssign):
604
+ if isinstance(stmt.target, ast.Name) and isinstance(stmt.value, ast.Constant):
605
+ astAssignee: ast.Name = stmt.target
606
+ argData_dtypeName = hackSSOTdatatype(astAssignee.id)
607
+ allImports.addImportFromStr(moduleConstructor, argData_dtypeName)
608
+ astCall = ast.Call(func=ast.Name(id=argData_dtypeName, ctx=ast.Load()) , args=[stmt.value], keywords=[])
609
+ assignment = ast.Assign(targets=[astAssignee], value=astCall)
610
+ FunctionDefTarget.body.insert(0, assignment)
611
+ FunctionDefTarget.body.remove(stmt)
612
+ return FunctionDefTarget
613
+
614
+ def removeIdentifierFrom_body(FunctionDefTarget: ast.FunctionDef, astArg: ast.arg) -> ast.FunctionDef:
615
+ for stmt in FunctionDefTarget.body.copy():
616
+ if isinstance(stmt, ast.Assign):
617
+ if isinstance(stmt.targets[0], ast.Subscript) and isinstance(stmt.targets[0].value, ast.Name):
618
+ if stmt.targets[0].value.id == astArg.arg:
619
+ FunctionDefTarget.body.remove(stmt)
620
+ FunctionDefTarget.args.args.remove(astArg)
621
+ return FunctionDefTarget
622
+
623
+ def astObjectToAstConstant(FunctionDefTarget: ast.FunctionDef, object: str, value: int) -> ast.FunctionDef:
624
+ """
625
+ Replaces nodes in astFunction matching the AST of the string `object`
626
+ with a constant node holding the provided value.
627
+ """
628
+ targetExpression = ast.parse(object, mode='eval').body
629
+ targetDump = ast.dump(targetExpression, annotate_fields=False)
630
+
631
+ def findNode(node: ast.AST) -> bool:
632
+ return ast.dump(node, annotate_fields=False) == targetDump
633
+
634
+ def replaceWithConstant(node: ast.AST) -> ast.AST:
635
+ return ast.copy_location(ast.Constant(value=value), node)
636
+
637
+ transformer = NodeReplacer(findNode, replaceWithConstant)
638
+ newFunction = cast(ast.FunctionDef, transformer.visit(FunctionDefTarget))
639
+ ast.fix_missing_locations(newFunction)
640
+ return newFunction
641
+
642
+ def astNameToAstConstant(FunctionDefTarget: ast.FunctionDef, name: str, value: int) -> ast.FunctionDef:
643
+ def findName(node: ast.AST) -> bool:
644
+ return isinstance(node, ast.Name) and node.id == name
645
+
646
+ def replaceWithConstant(node: ast.AST) -> ast.AST:
647
+ return ast.copy_location(ast.Constant(value=value), node)
648
+
649
+ return cast(ast.FunctionDef, NodeReplacer(findName, replaceWithConstant).visit(FunctionDefTarget))
650
+
651
+ def makeDecorator(FunctionDefTarget: ast.FunctionDef, parametersNumba: Optional[ParametersNumba]=None) -> ast.FunctionDef:
652
+ # Use NodeReplacer to handle decorator cleanup
653
+ def isNumbaJitDecorator(node: ast.AST) -> bool:
654
+ return ((isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
655
+ and getattr(node.func.value, "id", None) == "numba" and node.func.attr == "jit")
656
+ or
657
+ (isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "jit"))
658
+
659
+ def extractNumbaParams(node: ast.Call) -> None:
660
+ nonlocal parametersNumba
661
+ if parametersNumba is None:
662
+ parametersNumbaExtracted: Dict[str, Any] = {}
663
+ for keywordItem in node.keywords:
664
+ if isinstance(keywordItem.value, ast.Constant) and keywordItem.arg is not None:
665
+ parametersNumbaExtracted[keywordItem.arg] = keywordItem.value.value
666
+ if parametersNumbaExtracted:
667
+ parametersNumba = ParametersNumba(parametersNumbaExtracted) # type: ignore
668
+ return None
669
+
670
+ # Remove existing numba decorators
671
+ decoratorCleaner = NodeReplacer(isNumbaJitDecorator, extractNumbaParams)
672
+ FunctionDefTarget = cast(ast.FunctionDef, decoratorCleaner.visit(FunctionDefTarget))
673
+
674
+ FunctionDefTarget = Z0Z_UnhandledDecorators(FunctionDefTarget)
675
+ allImports.addImportFromStr('numba', 'jit')
676
+ FunctionDefTarget = decorateCallableWithNumba(FunctionDefTarget, parametersNumba)
677
+
678
+ # Convert @numba.jit to @jit
679
+ def isNumbaJitCall(node: ast.AST) -> bool:
680
+ return (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == "jit")
681
+
682
+ def convertToPlainJit(node: ast.Call) -> ast.Call:
683
+ node.func = ast.Name(id="jit", ctx=ast.Load())
684
+ return node
685
+
686
+ jitSimplifier = NodeReplacer(isNumbaJitCall, convertToPlainJit)
687
+ return cast(ast.FunctionDef, jitSimplifier.visit(FunctionDefTarget))
688
+
689
+ def makeLauncher(callableTarget: str) -> ast.Module:
690
+ linesLaunch = f"""
691
+ if __name__ == '__main__':
692
+ import time
693
+ timeStart = time.perf_counter()
694
+ {callableTarget}()
695
+ print(time.perf_counter() - timeStart)
696
+ """
697
+ astLaunch = ast.parse(linesLaunch)
698
+ return astLaunch
699
+
700
+ def make_writeFoldsTotal(stateJob: computationState, pathFilenameFoldsTotal: pathlib.Path) -> ast.Module:
701
+ allImports.addImportFromStr('numba', 'objmode')
702
+ linesWriteFoldsTotal = f"""
703
+ groupsOfFolds *= {str(stateJob['foldGroups'][-1])}
704
+ print(groupsOfFolds)
705
+ with objmode():
706
+ open('{pathFilenameFoldsTotal.as_posix()}', 'w').write(str(groupsOfFolds))
707
+ return groupsOfFolds
708
+ """
709
+ return ast.parse(linesWriteFoldsTotal)
710
+
711
+ def writeJobNumba(listDimensions: Sequence[int], callableTarget: str, algorithmSource: ModuleType, parametersNumba: Optional[ParametersNumba]=None, pathFilenameWriteJob: Optional[Union[str, os.PathLike[str]]] = None, **keywordArguments: Optional[Any]) -> pathlib.Path:
712
+ """
713
+ Parameters:
714
+ **keywordArguments: most especially for `computationDivisions` if you want to make a parallel job. Also `CPUlimit`.
715
+ """
716
+ stateJob = makeStateJob(listDimensions, writeJob=False, **keywordArguments)
717
+ pythonSource = inspect.getsource(algorithmSource)
718
+ astModule = ast.parse(pythonSource)
719
+
720
+ for statement in astModule.body:
721
+ if isinstance(statement, (ast.Import, ast.ImportFrom)):
722
+ allImports.addAst(statement)
723
+
724
+ FunctionDefTarget = next((node for node in astModule.body if isinstance(node, ast.FunctionDef) and node.name == callableTarget), None)
725
+
726
+ if not FunctionDefTarget:
727
+ raise ValueError(f"Could not find function {callableTarget} in source code.")
728
+
729
+ # Define argument processing rules
730
+ argumentRules = [
731
+ (lambda arg: arg.arg in ['connectionGraph', 'gapsWhere'],
732
+ lambda node, arg: moveArrayTo_body(node, arg, stateJob[arg.arg])),
733
+
734
+ (lambda arg: arg.arg in ['track'],
735
+ lambda node, arg: evaluateArrayIn_body(node, arg, stateJob[arg.arg])),
736
+
737
+ (lambda arg: arg.arg in ['my'],
738
+ lambda node, arg: evaluate_argIn_body(node, arg, stateJob[arg.arg], ['taskIndex', 'dimensionsTotal'])),
739
+
740
+ (lambda arg: arg.arg in ['foldGroups'],
741
+ lambda node, arg: removeIdentifierFrom_body(node, arg))
742
+ ]
743
+
744
+ # Process arguments using ArgumentProcessor
745
+ argumentProcessor = ArgumentProcessor(argumentRules)
746
+ FunctionDefTarget = argumentProcessor.process(FunctionDefTarget)
747
+
748
+ FunctionDefTarget = evaluateAnnAssignIn_body(FunctionDefTarget)
749
+ FunctionDefTarget = astNameToAstConstant(FunctionDefTarget, 'dimensionsTotal', int(stateJob['my'][indexMy.dimensionsTotal]))
750
+ FunctionDefTarget = astObjectToAstConstant(FunctionDefTarget, 'foldGroups[-1]', int(stateJob['foldGroups'][-1]))
751
+
752
+ FunctionDefTarget = makeDecorator(FunctionDefTarget, parametersNumba)
753
+
754
+ pathFilenameFoldsTotal = getPathFilenameFoldsTotal(stateJob['mapShape'])
755
+ astWriteFoldsTotal = make_writeFoldsTotal(stateJob, pathFilenameFoldsTotal)
756
+ FunctionDefTarget.body += astWriteFoldsTotal.body
757
+
758
+ astLauncher = makeLauncher(FunctionDefTarget.name)
759
+
760
+ astImports = allImports.makeListAst()
761
+
762
+ astModule = ast.Module(
763
+ body=cast(List[ast.stmt]
764
+ , astImports
765
+ + [FunctionDefTarget]
766
+ + [astLauncher])
767
+ , type_ignores=[]
768
+ )
769
+ ast.fix_missing_locations(astModule)
770
+
771
+ pythonSource = ast.unparse(astModule)
772
+ pythonSource = autoflake.fix_code(pythonSource, ['mapFolding', 'numba', 'numpy'])
773
+
774
+ if pathFilenameWriteJob is None:
775
+ filename = getFilenameFoldsTotal(stateJob['mapShape'])
776
+ pathRoot = getPathJobRootDEFAULT()
777
+ pathFilenameWriteJob = pathlib.Path(pathRoot, pathlib.Path(filename).stem, pathlib.Path(filename).with_suffix('.py'))
778
+ else:
779
+ pathFilenameWriteJob = pathlib.Path(pathFilenameWriteJob)
780
+ pathFilenameWriteJob.parent.mkdir(parents=True, exist_ok=True)
781
+
782
+ pathFilenameWriteJob.write_text(pythonSource)
783
+ return pathFilenameWriteJob
784
+
785
+ if __name__ == '__main__':
786
+ setDatatypeModule('numpy', sourGrapes=True)
787
+ setDatatypeFoldsTotal('int64', sourGrapes=True)
788
+ setDatatypeElephino('uint8', sourGrapes=True)
789
+ setDatatypeLeavesTotal('uint8', sourGrapes=True)
790
+ listCallablesInline: List[str] = ['countInitialize', 'countParallel', 'countSequential']
791
+ datatypeModuleScalar = 'numba'
792
+ datatypeModuleDecorator = 'numba'
793
+ callableDispatcher = 'doTheNeedful'
794
+ makeNumbaOptimizedFlow(listCallablesInline, callableDispatcher)
795
+
796
+ listDimensions = [5,5]
797
+ setDatatypeFoldsTotal('int64', sourGrapes=True)
798
+ setDatatypeElephino('uint8', sourGrapes=True)
799
+ setDatatypeLeavesTotal('uint8', sourGrapes=True)
800
+ from mapFolding.syntheticModules import numba_countSequential
801
+ algorithmSource: ModuleType = numba_countSequential
802
+ datatypeModuleScalar = 'numba'
803
+ datatypeModuleDecorator = 'numba'
804
+ allImports = UniversalImportTracker()
805
+ pathFilenameModule = writeJobNumba(listDimensions, 'countSequential', algorithmSource, parametersNumbaDEFAULT)