mapFolding 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

mapFolding/someAssemblyRequired/synthesizeNumbaFlow.py

@@ -0,0 +1,193 @@
+"""
+Orchestrator for generating Numba-optimized versions of the map folding algorithm.
+
+This module transforms the pure Python implementation of the map folding algorithm
+into a highly-optimized Numba implementation. It serves as the high-level coordinator
+for the code transformation process, orchestrating the following steps:
+
+1. Extracting the core algorithm functions from the source implementation
+2. Transforming function signatures and state handling for Numba compatibility
+3. Converting state-based operations to direct primitive operations
+4. Applying Numba decorators with appropriate optimization parameters
+5. Managing imports and dependencies for the generated code
+6. Assembling and writing the transformed implementation
+
+The transformation process preserves the algorithm's logic while dramatically improving
+performance by leveraging Numba's just-in-time compilation capabilities. This module
+depends on the abstract transformation tools, dataclass handling utilities, and
+Numba-specific optimization configurations from other modules in the package.
+
+The primary entry point is the makeNumbaFlow function, which can be executed directly
+to generate a fresh optimized implementation.
+"""
+
+from mapFolding.someAssemblyRequired import (
+	extractFunctionDef,
+	ifThis,
+	IngredientsFunction,
+	IngredientsModule,
+	LedgerOfImports,
+	Make,
+	makeDictionaryReplacementStatements,
+	NodeCollector,
+	NodeReplacer,
+	RecipeSynthesizeFlow,
+	Then,
+	write_astModule,
+	Z0Z_replaceMatchingASTnodes,
+	inlineThisFunctionWithTheseValues,
+)
+from mapFolding.someAssemblyRequired.ingredientsNumba import decorateCallableWithNumba
+from mapFolding.someAssemblyRequired.transformDataStructures import shatter_dataclassesDOTdataclass
+from mapFolding.theSSOT import raiseIfNoneGitHubIssueNumber3
+import ast
+
+def makeNumbaFlow(numbaFlow: RecipeSynthesizeFlow = RecipeSynthesizeFlow()) -> None:
+	"""
+	Think about a better organization of this function.
+
+	Currently, transform `Callable` in order:
+		sourceDispatcherCallable
+		sourceInitializeCallable
+		sourceParallelCallable
+		sourceSequentialCallable
+
+	But, it should be organized around each transformation. So, when the parameters of `sourceSequentialCallable`
+	are transformed, for example, the statement in `sourceDispatcherCallable` that calls `sourceSequentialCallable` should be
+	transformed at the same time: literally in the same function-or-NodeReplacer-or-subroutine. That would help
+	avoid bugs.
+
+	Furthermore, if the above example transformation requires unpacking the dataclass, for example, then the unpacking
+	would be automatically triggered. I have no idea how that would happen, but the transformations are highly predictable,
+	so using a programming language to construct if-this-then-that cascades shouldn't be a problem, you know?
+
+	# TODO a tool to automatically remove unused variables from the ArgumentsSpecification (return, and returns) _might_ be nice.
+	"""
+	dictionaryReplacementStatements = makeDictionaryReplacementStatements(numbaFlow.source_astModule)
+	# TODO remember that `sequentialCallable` and `sourceSequentialCallable` are two different values.
+	# Figure out dynamic flow control to synthesized modules https://github.com/hunterhogan/mapFolding/issues/4
+
+	# ===========================================================
+	sourcePython = numbaFlow.sourceDispatcherCallable
+	astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
+	if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
+	ingredientsDispatcher = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
+
+	# sourceParallelCallable
+	shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, numbaFlow.sourceDataclassInstanceTaskDistribution)
+	ingredientsDispatcher.imports.update(shatteredDataclass.ledgerDataclassANDFragments)
+
+	NodeReplacer(
+		findThis = ifThis.isAssignAndValueIsCallNamespace_Identifier(numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.sourceConcurrencyManagerIdentifier)
+		, doThat = Then.insertThisAbove(shatteredDataclass.listAnnAssign4DataclassUnpack)
+			).visit(ingredientsDispatcher.astFunctionDef)
+	NodeReplacer(
+		findThis = ifThis.isCallNamespace_Identifier(numbaFlow.sourceConcurrencyManagerNamespace, numbaFlow.sourceConcurrencyManagerIdentifier)
+		, doThat = Then.replaceWith(Make.astCall(Make.astAttribute(Make.astName(numbaFlow.sourceConcurrencyManagerNamespace), numbaFlow.sourceConcurrencyManagerIdentifier)
+									, listArguments=[Make.astName(numbaFlow.parallelCallable)] + shatteredDataclass.listNameDataclassFragments4Parameters))
+			).visit(ingredientsDispatcher.astFunctionDef)
+
+	CapturedAssign: list[ast.AST] = []
+	CapturedCall: list[ast.Call] = []
+	findThis = ifThis.isCall
+	doThat = [Then.appendTo(CapturedCall)]
+	capture = NodeCollector(findThis, doThat)
+
+	NodeCollector(
+		findThis = ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(numbaFlow.sourceDataclassInstance))
+		, doThat = [Then.appendTo(CapturedAssign)
+					, lambda node: capture.visit(node)]
+			).visit(ingredientsDispatcher.astFunctionDef)
+
+	newAssign = CapturedAssign[0]
+	NodeReplacer(
+		findThis = lambda node: ifThis.isSubscript(node) and ifThis.isAttribute(node.value) and ifThis.isCall(node.value.value)
+		, doThat = Then.replaceWith(CapturedCall[0])
+			).visit(newAssign)
+
+	NodeReplacer(
+		findThis = ifThis.isAssignAndTargets0Is(ifThis.isSubscript_Identifier(numbaFlow.sourceDataclassInstance))
+		, doThat = Then.replaceWith(newAssign)
+			).visit(ingredientsDispatcher.astFunctionDef)
+
+	# sourceSequentialCallable
+	shatteredDataclass = shatter_dataclassesDOTdataclass(numbaFlow.logicalPathModuleDataclass, numbaFlow.sourceDataclassIdentifier, numbaFlow.sourceDataclassInstance)
+
+	ingredientsDispatcher.imports.update(shatteredDataclass.ledgerDataclassANDFragments)
+
+	NodeReplacer(
+		findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
+		, doThat = Then.insertThisAbove(shatteredDataclass.listAnnAssign4DataclassUnpack)
+			).visit(ingredientsDispatcher.astFunctionDef)
+	NodeReplacer(
+		findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
+		, doThat = Then.insertThisBelow([shatteredDataclass.astAssignDataclassRepack])
+			).visit(ingredientsDispatcher.astFunctionDef)
+	NodeReplacer(
+		findThis = ifThis.isAssignAndValueIsCall_Identifier(numbaFlow.sourceSequentialCallable)
+		, doThat = Then.replaceWith(Make.astAssign(listTargets=[shatteredDataclass.astTuple4AssignTargetsToFragments], value=Make.astCall(Make.astName(numbaFlow.sequentialCallable), shatteredDataclass.listNameDataclassFragments4Parameters)))
+			).visit(ingredientsDispatcher.astFunctionDef)
+
+	ingredientsDispatcher.astFunctionDef.name = numbaFlow.dispatcherCallable
+
+	# ===========================================================
+	sourcePython = numbaFlow.sourceInitializeCallable
+	astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
+	if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
+	astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
+	ingredientsInitialize = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
+
+	# ===========================================================
+	sourcePython = numbaFlow.sourceParallelCallable
+	astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
+	if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
+	astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
+	ingredientsParallel = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
+	ingredientsParallel.astFunctionDef.name = numbaFlow.parallelCallable
+	ingredientsParallel.astFunctionDef.args = Make.astArgumentsSpecification(args=shatteredDataclass.list_ast_argAnnotated4ArgumentsSpecification)
+	NodeReplacer(
+		findThis = ifThis.isReturn
+		, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
+			).visit(ingredientsParallel.astFunctionDef)
+
+	NodeReplacer(
+		findThis = ifThis.isReturn
+		, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.countingVariableName))
+			).visit(ingredientsParallel.astFunctionDef)
+	ingredientsParallel.astFunctionDef.returns = shatteredDataclass.countingVariableAnnotation
+	replacementMap = {statement.value: statement.target for statement in shatteredDataclass.listAnnAssign4DataclassUnpack}
+	ingredientsParallel.astFunctionDef = Z0Z_replaceMatchingASTnodes(ingredientsParallel.astFunctionDef, replacementMap) # type: ignore
+	ingredientsParallel = decorateCallableWithNumba(ingredientsParallel)
+
+	# ===========================================================
+	sourcePython = numbaFlow.sourceSequentialCallable
+	astFunctionDef = extractFunctionDef(sourcePython, numbaFlow.source_astModule)
+	if not astFunctionDef: raise raiseIfNoneGitHubIssueNumber3
+	astFunctionDef = inlineThisFunctionWithTheseValues(astFunctionDef, dictionaryReplacementStatements)
+	ingredientsSequential = IngredientsFunction(astFunctionDef, LedgerOfImports(numbaFlow.source_astModule))
+	ingredientsSequential.astFunctionDef.name = numbaFlow.sequentialCallable
+	ingredientsSequential.astFunctionDef.args = Make.astArgumentsSpecification(args=shatteredDataclass.list_ast_argAnnotated4ArgumentsSpecification)
+	NodeReplacer(
+		findThis = ifThis.isReturn
+		, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
+			).visit(ingredientsSequential.astFunctionDef)
+	NodeReplacer(
+		findThis = ifThis.isReturn
+		, doThat = Then.replaceWith(Make.astReturn(shatteredDataclass.astTuple4AssignTargetsToFragments))
+			).visit(ingredientsSequential.astFunctionDef)
+	ingredientsSequential.astFunctionDef.returns = shatteredDataclass.astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns
+	replacementMap = {statement.value: statement.target for statement in shatteredDataclass.listAnnAssign4DataclassUnpack}
+	ingredientsSequential.astFunctionDef = Z0Z_replaceMatchingASTnodes(ingredientsSequential.astFunctionDef, replacementMap) # type: ignore
+	ingredientsSequential = decorateCallableWithNumba(ingredientsSequential)
+
+	# ===========================================================
+	ingredientsModuleNumbaUnified = IngredientsModule(
+		ingredientsFunction=[ingredientsInitialize,
+							ingredientsParallel,
+							ingredientsSequential,
+							ingredientsDispatcher], imports=LedgerOfImports(numbaFlow.source_astModule))
+
+	write_astModule(ingredientsModuleNumbaUnified, numbaFlow.pathFilenameDispatcher, numbaFlow.packageName)
+
+if __name__ == '__main__':
+	makeNumbaFlow()

mapFolding/someAssemblyRequired/synthesizeNumbaJobVESTIGIAL.py

@@ -2,8 +2,7 @@
 from collections.abc import Sequence
 from typing import Any, cast, TYPE_CHECKING
 from mapFolding.filesystem import getFilenameFoldsTotal, getPathFilenameFoldsTotal
-from mapFolding.someAssemblyRequired import ( ifThis, Make, NodeReplacer, Then, )
-from mapFolding.someAssemblyRequired.transformationTools import LedgerOfImports
+from mapFolding.someAssemblyRequired import ( ifThis, LedgerOfImports, Make, NodeReplacer, Then, )
 from mapFolding.theSSOT import ( ComputationState, raiseIfNoneGitHubIssueNumber3, getPathJobRootDEFAULT, )
 from os import PathLike
 from pathlib import Path
@@ -16,7 +15,7 @@ import copy
 import inspect
 import numpy
 if TYPE_CHECKING:
-	from mapFolding.someAssemblyRequired.synthesizeDataConverters import makeStateJob
+	from mapFolding.someAssemblyRequired.transformDataStructures import makeStateJobOUTDATED
 	from mapFolding.someAssemblyRequired.ingredientsNumba import thisIsNumbaDotJit, decorateCallableWithNumba
 	from mapFolding.someAssemblyRequired.ingredientsNumba import ParametersNumba, parametersNumbaDEFAULT
 
@@ -291,7 +290,7 @@ def writeJobNumba(mapShape: Sequence[int], algorithmSource: ModuleType, callable
 	"""
 
 	# NOTE get the raw ingredients: data and the algorithm
-	stateJob = makeStateJob(mapShape, writeJob=False, **keywordArguments)
+	stateJob = makeStateJobOUTDATED(mapShape, writeJob=False, **keywordArguments)
 	pythonSource: str = inspect.getsource(algorithmSource)
 	astModule: ast.Module = ast.parse(pythonSource)
 	setFunctionDef: set[ast.FunctionDef] = {statement for statement in astModule.body if isinstance(statement, ast.FunctionDef)}

mapFolding/someAssemblyRequired/transformDataStructures.py

@@ -0,0 +1,168 @@
+"""
+Utilities for transforming complex data structures in Python code generation.
+
+This module provides specialized tools for working with structured data types during
+the code transformation process, with a particular focus on handling dataclasses. It
+implements functionality that enables:
+
+1. Decomposing dataclasses into individual fields for efficient processing
+2. Creating optimized parameter passing for transformed functions
+3. Converting between different representations of data structures
+4. Serializing and deserializing computation state objects
+
+The core functionality revolves around the "shattering" process that breaks down
+a dataclass into its constituent components, making each field individually accessible
+for code generation and optimization purposes. This dataclass handling is critical for
+transforming algorithms that operate on unified state objects into optimized implementations
+that work with primitive types directly.
+
+While developed for transforming map folding computation state objects, the utilities are
+designed to be applicable to various data structure transformation scenarios.
+"""
+
+from collections.abc import Sequence
+from importlib import import_module as importlib_import_module
+from inspect import getsource as inspect_getsource
+from mapFolding.beDRY import outfitCountFolds, validateListDimensions
+from mapFolding.filesystem import getPathFilenameFoldsTotal
+from mapFolding.someAssemblyRequired import (
+	ast_Identifier,
+	extractClassDef,
+	ifThis,
+	LedgerOfImports,
+	Make,
+	NodeCollector,
+	strDotStrCuzPyStoopid,
+	Then,
+	Z0Z_executeActionUnlessDescendantMatches,
+)
+from mapFolding.theSSOT import ComputationState, The
+from pathlib import Path
+from types import ModuleType
+from typing import Any, Literal, overload
+import ast
+import dataclasses
+import pickle
+
+# Would `LibCST` be better than `ast` in some cases? https://github.com/hunterhogan/mapFolding/issues/7
+
+countingIdentifierHARDCODED = 'groupsOfFolds'
+
+@dataclasses.dataclass
+class ShatteredDataclass:
+	astAssignDataclassRepack: ast.Assign
+	astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns: ast.Subscript
+	astTuple4AssignTargetsToFragments: ast.Tuple
+	countingVariableAnnotation: ast.expr
+	countingVariableName: ast.Name
+	ledgerDataclassANDFragments: LedgerOfImports = dataclasses.field(default_factory=LedgerOfImports)
+	list_ast_argAnnotated4ArgumentsSpecification: list[ast.arg] = dataclasses.field(default_factory=list)
+	list_keyword4DataclassInitialization: list[ast.keyword] = dataclasses.field(default_factory=list)
+	listAnnAssign4DataclassUnpack: list[ast.AnnAssign] = dataclasses.field(default_factory=list)
+	listAnnotations: list[ast.expr] = dataclasses.field(default_factory=list)
+	listNameDataclassFragments4Parameters: list[ast.Name] = dataclasses.field(default_factory=list)
+
+def shatter_dataclassesDOTdataclass(logicalPathModule: strDotStrCuzPyStoopid, dataclass_Identifier: ast_Identifier, instance_Identifier: ast_Identifier) -> ShatteredDataclass:
+	"""
+	Parameters:
+		logicalPathModule: gimme string cuz python is stoopid
+		dataclass_Identifier: The identifier of the dataclass to be dismantled.
+		instance_Identifier: In the synthesized module/function/scope, the identifier that will be used for the instance.
+	"""
+	# TODO learn whether dataclasses.make_dataclass would be useful to transform the target dataclass into the `ShatteredDataclass`
+
+	module: ast.Module = ast.parse(inspect_getsource(importlib_import_module(logicalPathModule)))
+	astName_dataclassesDOTdataclass = Make.astName(dataclass_Identifier)
+
+	dataclass = extractClassDef(dataclass_Identifier, module)
+	if not isinstance(dataclass, ast.ClassDef):
+		raise ValueError(f"I could not find {dataclass_Identifier=} in {logicalPathModule=}.")
+
+	ledgerDataclassANDFragments = LedgerOfImports()
+	list_ast_argAnnotated4ArgumentsSpecification: list[ast.arg] = []
+	list_keyword4DataclassInitialization: list[ast.keyword] = []
+	listAnnAssign4DataclassUnpack: list[ast.AnnAssign] = []
+	listAnnotations: list[ast.expr] = []
+	listNameDataclassFragments4Parameters: list[ast.Name] = []
+
+	# TODO get the value from `groupsOfFolds: DatatypeFoldsTotal = dataclasses.field(default=DatatypeFoldsTotal(0), metadata={'theCountingIdentifier': True})`
+	countingVariable = countingIdentifierHARDCODED
+
+	addToLedgerPredicate = ifThis.isAnnAssignAndAnnotationIsName
+	addToLedgerAction = Then.Z0Z_ledger(logicalPathModule, ledgerDataclassANDFragments)
+	addToLedger = NodeCollector(addToLedgerPredicate, [addToLedgerAction])
+
+	exclusionPredicate = ifThis.is_keyword_IdentifierEqualsConstantValue('init', False)
+	appendKeywordAction = Then.Z0Z_appendKeywordMirroredTo(list_keyword4DataclassInitialization)
+	filteredAppendKeywordAction = Z0Z_executeActionUnlessDescendantMatches(exclusionPredicate, appendKeywordAction) # type: ignore
+
+	NodeCollector(
+		ifThis.isAnnAssignAndTargetIsName,
+			[Then.Z0Z_appendAnnAssignOf_nameDOTnameTo(instance_Identifier, listAnnAssign4DataclassUnpack)
+			, Then.append_targetTo(listNameDataclassFragments4Parameters) # type: ignore
+			, lambda node: addToLedger.visit(node)
+			, filteredAppendKeywordAction
+			, lambda node: list_ast_argAnnotated4ArgumentsSpecification.append(Make.ast_arg(node.target.id, node.annotation)) # type: ignore
+			, lambda node: listAnnotations.append(node.annotation) # type: ignore
+			]
+		).visit(dataclass)
+
+	shatteredDataclass = ShatteredDataclass(
+	astAssignDataclassRepack = Make.astAssign(listTargets=[Make.astName(instance_Identifier)], value=Make.astCall(astName_dataclassesDOTdataclass, list_astKeywords=list_keyword4DataclassInitialization))
+	, astSubscriptPrimitiveTupleAnnotations4FunctionDef_returns = Make.astSubscript(Make.astName('tuple'), Make.astTuple(listAnnotations))
+	, astTuple4AssignTargetsToFragments = Make.astTuple(listNameDataclassFragments4Parameters, ast.Store())
+	, countingVariableAnnotation = next(ast_arg.annotation for ast_arg in list_ast_argAnnotated4ArgumentsSpecification if ast_arg.arg == countingVariable) or Make.astName('Any')
+	, countingVariableName = Make.astName(countingVariable)
+	, ledgerDataclassANDFragments = ledgerDataclassANDFragments
+	, list_ast_argAnnotated4ArgumentsSpecification = list_ast_argAnnotated4ArgumentsSpecification
+	, list_keyword4DataclassInitialization = list_keyword4DataclassInitialization
+	, listAnnAssign4DataclassUnpack = listAnnAssign4DataclassUnpack
+	, listAnnotations = listAnnotations
+	, listNameDataclassFragments4Parameters = listNameDataclassFragments4Parameters
+	)
+
+	shatteredDataclass.ledgerDataclassANDFragments.addImportFromStr(logicalPathModule, dataclass_Identifier)
+	return shatteredDataclass
+
+def getSourceAlgorithmVESTIGIAL() -> ModuleType:
+	moduleImported: ModuleType = importlib_import_module(The.logicalPathModuleSourceAlgorithm)
+	return moduleImported
+
+@overload
+def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: Literal[True], **keywordArguments: Any) -> Path: ...
+@overload
+def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: Literal[False], **keywordArguments: Any) -> ComputationState: ...
+def makeStateJobOUTDATED(listDimensions: Sequence[int], *, writeJob: bool = True, **keywordArguments: Any) -> ComputationState | Path:
+	"""
+	Creates a computation state job for map folding calculations and optionally saves it to disk.
+
+	This function initializes a computation state for map folding calculations based on the given dimensions,
+	sets up the initial counting configuration, and can optionally save the state to a pickle file.
+
+	Parameters:
+		listDimensions: List of integers representing the dimensions of the map to be folded.
+		writeJob (True): Whether to save the state to disk.
+		**keywordArguments: Additional keyword arguments to pass to the computation state initialization.
+
+	Returns:
+		stateUniversal|pathFilenameJob: The computation state for the map folding calculations, or
+			the path to the saved state file if writeJob is True.
+	"""
+	mapShape = validateListDimensions(listDimensions)
+	stateUniversal: ComputationState = outfitCountFolds(mapShape, **keywordArguments)
+
+	moduleSource: ModuleType = getSourceAlgorithmVESTIGIAL()
+	# TODO `countInitialize` is hardcoded
+	stateUniversal = moduleSource.countInitialize(stateUniversal)
+
+	if not writeJob:
+		return stateUniversal
+
+	pathFilenameChopChop = getPathFilenameFoldsTotal(stateUniversal.mapShape, None)
+	suffix = pathFilenameChopChop.suffix
+	pathJob = Path(str(pathFilenameChopChop)[0:-len(suffix)])
+	pathJob.mkdir(parents=True, exist_ok=True)
+	pathFilenameJob = pathJob / 'stateJob.pkl'
+
+	pathFilenameJob.write_bytes(pickle.dumps(stateUniversal))
+	return pathFilenameJob