astToolkit 0.1.3__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- asttoolkit-0.2.0/PKG-INFO +358 -0
- asttoolkit-0.2.0/README.md +315 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/__init__.py +4 -8
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_astTypes.py +14 -1
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolBe.py +1 -9
- asttoolkit-0.2.0/astToolkit/_toolClassIsAndAttribute.py +713 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolDOT.py +1 -1
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolGrab.py +2 -2
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolIfThis.py +0 -16
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolMake.py +1 -1
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolkitNodeVisitor.py +8 -8
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_types.py +2 -1
- asttoolkit-0.2.0/astToolkit/_typesSpecial.py +16 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/transformationTools.py +5 -3
- asttoolkit-0.2.0/astToolkit.egg-info/PKG-INFO +358 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit.egg-info/SOURCES.txt +1 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/pyproject.toml +4 -4
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/Z0Z_hardcoded.py +3 -1
- asttoolkit-0.2.0/toolFactory/Z0Z_makeAstTools.py +42 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/__init__.py +3 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/astFactory.py +155 -62
- asttoolkit-0.2.0/toolFactory/astFactory_annex.py +88 -0
- asttoolkit-0.1.3/PKG-INFO +0 -221
- asttoolkit-0.1.3/README.md +0 -178
- asttoolkit-0.1.3/astToolkit/_typesSpecial.py +0 -33
- asttoolkit-0.1.3/astToolkit.egg-info/PKG-INFO +0 -221
- asttoolkit-0.1.3/toolFactory/Z0Z_makeAstTools.py +0 -32
- asttoolkit-0.1.3/toolFactory/astFactory_annex.py +0 -62
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/LICENSE +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolThen.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolkitAST.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/_toolkitContainers.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/py.typed +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit/theSSOT.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit.egg-info/dependency_links.txt +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit.egg-info/requires.txt +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/astToolkit.egg-info/top_level.txt +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/setup.cfg +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/tests/test_basic.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/docstrings.py +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/py.typed +0 -0
- {asttoolkit-0.1.3 → asttoolkit-0.2.0}/toolFactory/startHere.py +0 -0
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: astToolkit
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A powerfully composable, type-safe toolkit for Python abstract syntax tree (AST) manipulation, analysis, transformation, and code generation with a layered architecture designed for building sophisticated code processing assembly-lines.
|
|
5
|
+
Author-email: Hunter Hogan <HunterHogan@pm.me>
|
|
6
|
+
License: CC-BY-NC-4.0
|
|
7
|
+
Project-URL: Donate, https://www.patreon.com/integrated
|
|
8
|
+
Project-URL: Homepage, https://github.com/hunterhogan/astToolkit
|
|
9
|
+
Project-URL: Issues, https://github.com/hunterhogan/astToolkit/issues
|
|
10
|
+
Project-URL: Repository, https://github.com/hunterhogan/astToolkit.git
|
|
11
|
+
Keywords: ast,abstract syntax tree,code generation,code transformation,ast-analysis,ast-manipulation,ast-traversal,ast-visitor,code-analysis,code-refactoring,static-analysis,python-ast,metaprogramming,visitor-pattern,typed-ast,type-safe,composable-apis,function-inlining,source-code-generation,ast-predicate
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Natural Language :: English
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
26
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
27
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Requires-Dist: autoflake
|
|
32
|
+
Requires-Dist: python_minifier
|
|
33
|
+
Requires-Dist: tomli
|
|
34
|
+
Requires-Dist: Z0Z_tools
|
|
35
|
+
Provides-Extra: testing
|
|
36
|
+
Requires-Dist: mypy; extra == "testing"
|
|
37
|
+
Requires-Dist: pytest; extra == "testing"
|
|
38
|
+
Requires-Dist: pytest-cov; extra == "testing"
|
|
39
|
+
Requires-Dist: pytest-xdist; extra == "testing"
|
|
40
|
+
Requires-Dist: pyupgrade; extra == "testing"
|
|
41
|
+
Requires-Dist: trove_classifiers; extra == "testing"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
# astToolkit
|
|
45
|
+
|
|
46
|
+
## Do You Want This Package?
|
|
47
|
+
|
|
48
|
+
astToolkit provides a powerfully composable system for manipulating Python Abstract Syntax Trees. Use it when:
|
|
49
|
+
|
|
50
|
+
- You need to programmatically analyze, transform, or generate Python code.
|
|
51
|
+
- You want type-safe operations that help prevent AST manipulation errors.
|
|
52
|
+
- You prefer working with a consistent, fluent API rather than raw AST nodes.
|
|
53
|
+
- You desire the ability to compose complex AST transformations from simple, reusable parts.
|
|
54
|
+
|
|
55
|
+
Don't use it for simple text-based code manipulation—use regex or string operations instead.
|
|
56
|
+
|
|
57
|
+
## Architecture
|
|
58
|
+
|
|
59
|
+
astToolkit implements a layered architecture designed for composability and type safety:
|
|
60
|
+
|
|
61
|
+
1. **Core "Atomic" Classes** - The foundation of the system:
|
|
62
|
+
- `Be`: Type guards that return `TypeGuard[ast.NodeType]` for safe type narrowing.
|
|
63
|
+
- `DOT`: Read-only accessors that retrieve node attributes with proper typing.
|
|
64
|
+
- `Grab`: Transformation functions that modify specific attributes while preserving node structure.
|
|
65
|
+
- `Make`: Factory methods that create properly configured AST nodes with consistent interfaces.
|
|
66
|
+
|
|
67
|
+
2. **Traversal and Transformation** - Built on the visitor pattern:
|
|
68
|
+
- `NodeTourist`: Extends `ast.NodeVisitor` to extract information from nodes that match the antecedent (sometimes called "predicate").
|
|
69
|
+
- `NodeChanger`: Extends `ast.NodeTransformer` to selectively transform nodes that match antecedents.
|
|
70
|
+
|
|
71
|
+
3. **Composable APIs** - The antecedent-action pattern:
|
|
72
|
+
- `ClassIsAndAttribute`: A powerful antecedent constructor: it confirms the class type of `node`, then applies whatever condition check you want to an attribute of `node`. As long as you listen to your type checker, you won't accidentally pair an attribute to a class that doesn't have that attribute. Furthermore, your IDE's hover type hints will tell you which classes are valid for the attribute you are checking.
|
|
73
|
+
- `IfThis`: Generates predicate functions that identify nodes based on structure, content, or relationships.
|
|
74
|
+
- `Then`: Creates action functions that specify what to do with matched nodes (extract, replace, modify).
|
|
75
|
+
|
|
76
|
+
4. **Higher-level Tools** - Built from the core components:
|
|
77
|
+
- `_toolkitAST.py`: Functions for common operations like extracting function definitions or importing modules.
|
|
78
|
+
- `transformationTools.py`: Advanced utilities like function inlining and code generation.
|
|
79
|
+
- `IngredientsFunction` and `IngredientsModule`: Containers for holding AST components and their dependencies.
|
|
80
|
+
|
|
81
|
+
5. **Type System** - Over 120 specialized types for AST components:
|
|
82
|
+
- Custom type annotations for AST node attributes.
|
|
83
|
+
- Union types that accurately model Python's AST structure.
|
|
84
|
+
- Type guards that enable static type checkers to understand dynamic type narrowing.
|
|
85
|
+
|
|
86
|
+
### Easy-to-use Tools for Annoying Tasks
|
|
87
|
+
|
|
88
|
+
- extractClassDef
|
|
89
|
+
- extractFunctionDef
|
|
90
|
+
- parseLogicalPath2astModule
|
|
91
|
+
- parsePathFilename2astModule
|
|
92
|
+
|
|
93
|
+
### Easy-to-use Tools for More Complicated Tasks
|
|
94
|
+
|
|
95
|
+
- removeUnusedParameters
|
|
96
|
+
- write_astModule
|
|
97
|
+
|
|
98
|
+
### The `toolFactory`
|
|
99
|
+
|
|
100
|
+
Hypothetically, you could customize every aspect of the classes `Be`, `DOT`, `GRAB`, and `Make` and more than 100 `TypeAlias` in the toolFactory directory/package.
|
|
101
|
+
|
|
102
|
+
## Usage
|
|
103
|
+
|
|
104
|
+
astToolkit provides a comprehensive set of tools for AST manipulation, organized in a layered architecture for composability and type safety. The following examples demonstrate how to use these tools in real-world scenarios.
|
|
105
|
+
|
|
106
|
+
### Core Pattern: Layered AST Manipulation
|
|
107
|
+
|
|
108
|
+
The astToolkit approach follows a layered pattern:
|
|
109
|
+
|
|
110
|
+
1. **Create/Access/Check** - Use `Make`, `DOT`, and `Be` to work with AST nodes
|
|
111
|
+
2. **Locate** - Use `IfThis` predicates to identify nodes of interest
|
|
112
|
+
3. **Transform** - Use `NodeChanger` and `Then` to modify nodes
|
|
113
|
+
4. **Extract** - Use `NodeTourist` to collect information from the AST
|
|
114
|
+
|
|
115
|
+
### Example 1: Extracting Information from AST
|
|
116
|
+
|
|
117
|
+
This example shows how to extract information from a function's parameters:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from astToolkit import Be, DOT, NodeTourist, Then
|
|
121
|
+
import ast
|
|
122
|
+
|
|
123
|
+
# Parse some Python code into an AST
|
|
124
|
+
code = """
|
|
125
|
+
def process_data(state: DataClass):
|
|
126
|
+
result = state.value * 2
|
|
127
|
+
return result
|
|
128
|
+
"""
|
|
129
|
+
tree = ast.parse(code)
|
|
130
|
+
|
|
131
|
+
# Extract the parameter name from the function
|
|
132
|
+
function_def = tree.body[0]
|
|
133
|
+
param_name = NodeTourist(
|
|
134
|
+
Be.arg, # Look for function parameters
|
|
135
|
+
Then.extractIt(DOT.arg) # Extract the parameter name
|
|
136
|
+
).captureLastMatch(function_def)
|
|
137
|
+
|
|
138
|
+
print(f"Function parameter name: {param_name}") # Outputs: state
|
|
139
|
+
|
|
140
|
+
# Extract the parameter's type annotation
|
|
141
|
+
annotation = NodeTourist(
|
|
142
|
+
Be.arg, # Look for function parameters
|
|
143
|
+
Then.extractIt(DOT.annotation) # Extract the type annotation
|
|
144
|
+
).captureLastMatch(function_def)
|
|
145
|
+
|
|
146
|
+
if annotation and Be.Name(annotation):
|
|
147
|
+
annotation_name = DOT.id(annotation)
|
|
148
|
+
print(f"Parameter type: {annotation_name}") # Outputs: DataClass
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Example 2: Transforming AST Nodes
|
|
152
|
+
|
|
153
|
+
This example demonstrates how to transform a specific node in the AST:
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from astToolkit import Be, IfThis, Make, NodeChanger, Then
|
|
157
|
+
import ast
|
|
158
|
+
|
|
159
|
+
# Parse some Python code into an AST
|
|
160
|
+
code = """
|
|
161
|
+
def double(x):
|
|
162
|
+
return x * 2
|
|
163
|
+
"""
|
|
164
|
+
tree = ast.parse(code)
|
|
165
|
+
|
|
166
|
+
# Define a predicate to find the multiplication operation
|
|
167
|
+
find_mult = Be.Mult
|
|
168
|
+
|
|
169
|
+
# Define a transformation to change multiplication to addition
|
|
170
|
+
change_to_add = Then.replaceWith(ast.Add())
|
|
171
|
+
|
|
172
|
+
# Apply the transformation
|
|
173
|
+
NodeChanger(find_mult, change_to_add).visit(tree)
|
|
174
|
+
|
|
175
|
+
# Now the code is equivalent to:
|
|
176
|
+
# def double(x):
|
|
177
|
+
# return x + x
|
|
178
|
+
print(ast.unparse(tree))
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### Example 3: Advanced AST Transformation with Custom Predicates
|
|
182
|
+
|
|
183
|
+
This example shows a more complex transformation inspired by the mapFolding package:
|
|
184
|
+
|
|
185
|
+
```python
|
|
186
|
+
from astToolkit import ast_Identifier, Be, DOT, Grab, IfThis as astToolkit_IfThis, Make, NodeChanger, Then
|
|
187
|
+
import ast
|
|
188
|
+
|
|
189
|
+
# Define custom predicates by extending IfThis
|
|
190
|
+
class IfThis(astToolkit_IfThis):
|
|
191
|
+
@staticmethod
|
|
192
|
+
def isAttributeNamespace_IdentifierGreaterThan0(
|
|
193
|
+
namespace: ast_Identifier,
|
|
194
|
+
identifier: ast_Identifier
|
|
195
|
+
) -> Callable[[ast.AST], TypeGuard[ast.Compare] | bool]:
|
|
196
|
+
|
|
197
|
+
return lambda node: (
|
|
198
|
+
Be.Compare(node)
|
|
199
|
+
and IfThis.isAttributeNamespace_Identifier(namespace, identifier)(DOT.left(node))
|
|
200
|
+
and Be.Gt(node.ops[0])
|
|
201
|
+
and IfThis.isConstant_value(0)(node.comparators[0]))
|
|
202
|
+
|
|
203
|
+
@staticmethod
|
|
204
|
+
def isWhileAttributeNamespace_IdentifierGreaterThan0(
|
|
205
|
+
namespace: ast_Identifier,
|
|
206
|
+
identifier: ast_Identifier
|
|
207
|
+
) -> Callable[[ast.AST], TypeGuard[ast.While] | bool]:
|
|
208
|
+
|
|
209
|
+
return lambda node: (
|
|
210
|
+
Be.While(node)
|
|
211
|
+
and IfThis.isAttributeNamespace_IdentifierGreaterThan0(namespace, identifier)(DOT.test(node)))
|
|
212
|
+
|
|
213
|
+
# Parse some code
|
|
214
|
+
code = """
|
|
215
|
+
while claude.counter > 0:
|
|
216
|
+
result += counter
|
|
217
|
+
counter -= 1
|
|
218
|
+
"""
|
|
219
|
+
tree = ast.parse(code)
|
|
220
|
+
|
|
221
|
+
# Find the while loop with our custom predicate
|
|
222
|
+
find_while_loop = IfThis.isWhileAttributeNamespace_IdentifierGreaterThan0("claude", "counter")
|
|
223
|
+
|
|
224
|
+
# Replace counter > 0 with counter > 1
|
|
225
|
+
change_condition = Grab.testAttribute(
|
|
226
|
+
Grab.comparatorsAttribute(
|
|
227
|
+
Then.replaceWith([Make.Constant(1)])
|
|
228
|
+
)
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
# Apply the transformation
|
|
232
|
+
NodeChanger(find_while_loop, change_condition).visit(tree)
|
|
233
|
+
|
|
234
|
+
print(ast.unparse(tree))
|
|
235
|
+
# Now outputs:
|
|
236
|
+
# while counter > 1:
|
|
237
|
+
# result += counter
|
|
238
|
+
# counter -= 1
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
### Example 4: Building Code Generation Systems
|
|
242
|
+
|
|
243
|
+
The following example shows how to set up a foundation for code generation and transformation systems:
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
from astToolkit import (
|
|
247
|
+
Be, DOT, IngredientsFunction, IngredientsModule, LedgerOfImports,
|
|
248
|
+
Make, NodeTourist, Then, parseLogicalPath2astModule, write_astModule
|
|
249
|
+
)
|
|
250
|
+
import ast
|
|
251
|
+
|
|
252
|
+
# Parse a module to extract a function
|
|
253
|
+
module_ast = parseLogicalPath2astModule("my_package.source_module")
|
|
254
|
+
|
|
255
|
+
# Extract a function and track its imports
|
|
256
|
+
function_name = "target_function"
|
|
257
|
+
function_def = NodeTourist(
|
|
258
|
+
IfThis.isFunctionDef_Identifier(function_name),
|
|
259
|
+
Then.extractIt
|
|
260
|
+
).captureLastMatch(module_ast)
|
|
261
|
+
|
|
262
|
+
if function_def:
|
|
263
|
+
# Create a self-contained function with tracked imports
|
|
264
|
+
ingredients = IngredientsFunction(
|
|
265
|
+
function_def,
|
|
266
|
+
LedgerOfImports(module_ast)
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
# Rename the function
|
|
270
|
+
ingredients.astFunctionDef.name = "optimized_" + function_name
|
|
271
|
+
|
|
272
|
+
# Add a decorator
|
|
273
|
+
decorator = Make.Call(
|
|
274
|
+
Make.Name("jit"),
|
|
275
|
+
[],
|
|
276
|
+
[Make.keyword("cache", Make.Constant(True))]
|
|
277
|
+
)
|
|
278
|
+
ingredients.astFunctionDef.decorator_list.append(decorator)
|
|
279
|
+
|
|
280
|
+
# Add required import
|
|
281
|
+
ingredients.imports.addImportFrom_asStr("numba", "jit")
|
|
282
|
+
|
|
283
|
+
# Create a module and write it to disk
|
|
284
|
+
module = IngredientsModule(ingredients)
|
|
285
|
+
write_astModule(module, "path/to/generated_code.py", "my_package")
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
### Example 5: Extending Core Classes
|
|
289
|
+
|
|
290
|
+
To create specialized patterns for your codebase, extend the core classes:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from astToolkit import ast_Identifier, Be, IfThis as astToolkit_IfThis
|
|
294
|
+
from collections.abc import Callable
|
|
295
|
+
from typing import TypeGuard
|
|
296
|
+
import ast
|
|
297
|
+
|
|
298
|
+
class IfThis(astToolkit_IfThis):
|
|
299
|
+
@staticmethod
|
|
300
|
+
def isAttributeNamespace_IdentifierGreaterThan0(
|
|
301
|
+
namespace: ast_Identifier,
|
|
302
|
+
identifier: ast_Identifier
|
|
303
|
+
) -> Callable[[ast.AST], TypeGuard[ast.Compare] | bool]:
|
|
304
|
+
"""Find comparisons like 'state.counter > 0'"""
|
|
305
|
+
return lambda node: (
|
|
306
|
+
Be.Compare(node)
|
|
307
|
+
and IfThis.isAttributeNamespace_Identifier(namespace, identifier)(node.left)
|
|
308
|
+
and Be.Gt(node.ops[0])
|
|
309
|
+
and IfThis.isConstant_value(0)(node.comparators[0])
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
@staticmethod
|
|
313
|
+
def isWhileAttributeNamespace_IdentifierGreaterThan0(
|
|
314
|
+
namespace: ast_Identifier,
|
|
315
|
+
identifier: ast_Identifier
|
|
316
|
+
) -> Callable[[ast.AST], TypeGuard[ast.While] | bool]:
|
|
317
|
+
"""Find while loops like 'while state.counter > 0:'"""
|
|
318
|
+
return lambda node: (
|
|
319
|
+
Be.While(node)
|
|
320
|
+
and IfThis.isAttributeNamespace_IdentifierGreaterThan0(namespace, identifier)(node.test)
|
|
321
|
+
)
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
### Real-world Application: Code Transformation Assembly-line
|
|
325
|
+
|
|
326
|
+
In the [mapFolding](https://github.com/hunterhogan/mapFolding) project, astToolkit is used to build a complete transformation assembly-line that:
|
|
327
|
+
|
|
328
|
+
1. Extracts algorithms from source modules
|
|
329
|
+
2. Transforms them into optimized variants
|
|
330
|
+
3. Applies numerical computing decorators
|
|
331
|
+
4. Handles dataclass management and type systems
|
|
332
|
+
5. Generates complete modules with proper imports
|
|
333
|
+
|
|
334
|
+
This pattern enables the separation of readable algorithm implementations from their high-performance variants while ensuring they remain functionally equivalent.
|
|
335
|
+
|
|
336
|
+
For deeper examples, see the [mapFolding/someAssemblyRequired](https://github.com/hunterhogan/mapFolding/tree/main/mapFolding/someAssemblyRequired/) directory.
|
|
337
|
+
|
|
338
|
+
## Installation
|
|
339
|
+
|
|
340
|
+
```bash
|
|
341
|
+
pip install astToolkit
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
## My Recovery
|
|
345
|
+
|
|
346
|
+
[](https://HunterThinks.com/support)
|
|
347
|
+
[](https://www.youtube.com/@HunterHogan)
|
|
348
|
+
|
|
349
|
+
## How to code
|
|
350
|
+
|
|
351
|
+
Coding One Step at a Time:
|
|
352
|
+
|
|
353
|
+
0. WRITE CODE.
|
|
354
|
+
1. Don't write stupid code that's hard to revise.
|
|
355
|
+
2. Write good code.
|
|
356
|
+
3. When revising, write better code.
|
|
357
|
+
|
|
358
|
+
[](https://creativecommons.org/licenses/by-nc/4.0/)
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# astToolkit
|
|
2
|
+
|
|
3
|
+
## Do You Want This Package?
|
|
4
|
+
|
|
5
|
+
astToolkit provides a powerfully composable system for manipulating Python Abstract Syntax Trees. Use it when:
|
|
6
|
+
|
|
7
|
+
- You need to programmatically analyze, transform, or generate Python code.
|
|
8
|
+
- You want type-safe operations that help prevent AST manipulation errors.
|
|
9
|
+
- You prefer working with a consistent, fluent API rather than raw AST nodes.
|
|
10
|
+
- You desire the ability to compose complex AST transformations from simple, reusable parts.
|
|
11
|
+
|
|
12
|
+
Don't use it for simple text-based code manipulation—use regex or string operations instead.
|
|
13
|
+
|
|
14
|
+
## Architecture
|
|
15
|
+
|
|
16
|
+
astToolkit implements a layered architecture designed for composability and type safety:
|
|
17
|
+
|
|
18
|
+
1. **Core "Atomic" Classes** - The foundation of the system:
|
|
19
|
+
- `Be`: Type guards that return `TypeGuard[ast.NodeType]` for safe type narrowing.
|
|
20
|
+
- `DOT`: Read-only accessors that retrieve node attributes with proper typing.
|
|
21
|
+
- `Grab`: Transformation functions that modify specific attributes while preserving node structure.
|
|
22
|
+
- `Make`: Factory methods that create properly configured AST nodes with consistent interfaces.
|
|
23
|
+
|
|
24
|
+
2. **Traversal and Transformation** - Built on the visitor pattern:
|
|
25
|
+
- `NodeTourist`: Extends `ast.NodeVisitor` to extract information from nodes that match the antecedent (sometimes called "predicate").
|
|
26
|
+
- `NodeChanger`: Extends `ast.NodeTransformer` to selectively transform nodes that match antecedents.
|
|
27
|
+
|
|
28
|
+
3. **Composable APIs** - The antecedent-action pattern:
|
|
29
|
+
- `ClassIsAndAttribute`: A powerful antecedent constructor: it confirms the class type of `node`, then applies whatever condition check you want to an attribute of `node`. As long as you listen to your type checker, you won't accidentally pair an attribute to a class that doesn't have that attribute. Furthermore, your IDE's hover type hints will tell you which classes are valid for the attribute you are checking.
|
|
30
|
+
- `IfThis`: Generates predicate functions that identify nodes based on structure, content, or relationships.
|
|
31
|
+
- `Then`: Creates action functions that specify what to do with matched nodes (extract, replace, modify).
|
|
32
|
+
|
|
33
|
+
4. **Higher-level Tools** - Built from the core components:
|
|
34
|
+
- `_toolkitAST.py`: Functions for common operations like extracting function definitions or importing modules.
|
|
35
|
+
- `transformationTools.py`: Advanced utilities like function inlining and code generation.
|
|
36
|
+
- `IngredientsFunction` and `IngredientsModule`: Containers for holding AST components and their dependencies.
|
|
37
|
+
|
|
38
|
+
5. **Type System** - Over 120 specialized types for AST components:
|
|
39
|
+
- Custom type annotations for AST node attributes.
|
|
40
|
+
- Union types that accurately model Python's AST structure.
|
|
41
|
+
- Type guards that enable static type checkers to understand dynamic type narrowing.
|
|
42
|
+
|
|
43
|
+
### Easy-to-use Tools for Annoying Tasks
|
|
44
|
+
|
|
45
|
+
- extractClassDef
|
|
46
|
+
- extractFunctionDef
|
|
47
|
+
- parseLogicalPath2astModule
|
|
48
|
+
- parsePathFilename2astModule
|
|
49
|
+
|
|
50
|
+
### Easy-to-use Tools for More Complicated Tasks
|
|
51
|
+
|
|
52
|
+
- removeUnusedParameters
|
|
53
|
+
- write_astModule
|
|
54
|
+
|
|
55
|
+
### The `toolFactory`
|
|
56
|
+
|
|
57
|
+
Hypothetically, you could customize every aspect of the classes `Be`, `DOT`, `GRAB`, and `Make` and more than 100 `TypeAlias` in the toolFactory directory/package.
|
|
58
|
+
|
|
59
|
+
## Usage
|
|
60
|
+
|
|
61
|
+
astToolkit provides a comprehensive set of tools for AST manipulation, organized in a layered architecture for composability and type safety. The following examples demonstrate how to use these tools in real-world scenarios.
|
|
62
|
+
|
|
63
|
+
### Core Pattern: Layered AST Manipulation
|
|
64
|
+
|
|
65
|
+
The astToolkit approach follows a layered pattern:
|
|
66
|
+
|
|
67
|
+
1. **Create/Access/Check** - Use `Make`, `DOT`, and `Be` to work with AST nodes
|
|
68
|
+
2. **Locate** - Use `IfThis` predicates to identify nodes of interest
|
|
69
|
+
3. **Transform** - Use `NodeChanger` and `Then` to modify nodes
|
|
70
|
+
4. **Extract** - Use `NodeTourist` to collect information from the AST
|
|
71
|
+
|
|
72
|
+
### Example 1: Extracting Information from AST
|
|
73
|
+
|
|
74
|
+
This example shows how to extract information from a function's parameters:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from astToolkit import Be, DOT, NodeTourist, Then
|
|
78
|
+
import ast
|
|
79
|
+
|
|
80
|
+
# Parse some Python code into an AST
|
|
81
|
+
code = """
|
|
82
|
+
def process_data(state: DataClass):
|
|
83
|
+
result = state.value * 2
|
|
84
|
+
return result
|
|
85
|
+
"""
|
|
86
|
+
tree = ast.parse(code)
|
|
87
|
+
|
|
88
|
+
# Extract the parameter name from the function
|
|
89
|
+
function_def = tree.body[0]
|
|
90
|
+
param_name = NodeTourist(
|
|
91
|
+
Be.arg, # Look for function parameters
|
|
92
|
+
Then.extractIt(DOT.arg) # Extract the parameter name
|
|
93
|
+
).captureLastMatch(function_def)
|
|
94
|
+
|
|
95
|
+
print(f"Function parameter name: {param_name}") # Outputs: state
|
|
96
|
+
|
|
97
|
+
# Extract the parameter's type annotation
|
|
98
|
+
annotation = NodeTourist(
|
|
99
|
+
Be.arg, # Look for function parameters
|
|
100
|
+
Then.extractIt(DOT.annotation) # Extract the type annotation
|
|
101
|
+
).captureLastMatch(function_def)
|
|
102
|
+
|
|
103
|
+
if annotation and Be.Name(annotation):
|
|
104
|
+
annotation_name = DOT.id(annotation)
|
|
105
|
+
print(f"Parameter type: {annotation_name}") # Outputs: DataClass
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### Example 2: Transforming AST Nodes
|
|
109
|
+
|
|
110
|
+
This example demonstrates how to transform a specific node in the AST:
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from astToolkit import Be, IfThis, Make, NodeChanger, Then
|
|
114
|
+
import ast
|
|
115
|
+
|
|
116
|
+
# Parse some Python code into an AST
|
|
117
|
+
code = """
|
|
118
|
+
def double(x):
|
|
119
|
+
return x * 2
|
|
120
|
+
"""
|
|
121
|
+
tree = ast.parse(code)
|
|
122
|
+
|
|
123
|
+
# Define a predicate to find the multiplication operation
|
|
124
|
+
find_mult = Be.Mult
|
|
125
|
+
|
|
126
|
+
# Define a transformation to change multiplication to addition
|
|
127
|
+
change_to_add = Then.replaceWith(ast.Add())
|
|
128
|
+
|
|
129
|
+
# Apply the transformation
|
|
130
|
+
NodeChanger(find_mult, change_to_add).visit(tree)
|
|
131
|
+
|
|
132
|
+
# Now the code is equivalent to:
|
|
133
|
+
# def double(x):
|
|
134
|
+
# return x + x
|
|
135
|
+
print(ast.unparse(tree))
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Example 3: Advanced AST Transformation with Custom Predicates
|
|
139
|
+
|
|
140
|
+
This example shows a more complex transformation inspired by the mapFolding package:
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from astToolkit import ast_Identifier, Be, DOT, Grab, IfThis as astToolkit_IfThis, Make, NodeChanger, Then
|
|
144
|
+
import ast
|
|
145
|
+
|
|
146
|
+
# Define custom predicates by extending IfThis
|
|
147
|
+
class IfThis(astToolkit_IfThis):
|
|
148
|
+
@staticmethod
|
|
149
|
+
def isAttributeNamespace_IdentifierGreaterThan0(
|
|
150
|
+
namespace: ast_Identifier,
|
|
151
|
+
identifier: ast_Identifier
|
|
152
|
+
) -> Callable[[ast.AST], TypeGuard[ast.Compare] | bool]:
|
|
153
|
+
|
|
154
|
+
return lambda node: (
|
|
155
|
+
Be.Compare(node)
|
|
156
|
+
and IfThis.isAttributeNamespace_Identifier(namespace, identifier)(DOT.left(node))
|
|
157
|
+
and Be.Gt(node.ops[0])
|
|
158
|
+
and IfThis.isConstant_value(0)(node.comparators[0]))
|
|
159
|
+
|
|
160
|
+
@staticmethod
|
|
161
|
+
def isWhileAttributeNamespace_IdentifierGreaterThan0(
|
|
162
|
+
namespace: ast_Identifier,
|
|
163
|
+
identifier: ast_Identifier
|
|
164
|
+
) -> Callable[[ast.AST], TypeGuard[ast.While] | bool]:
|
|
165
|
+
|
|
166
|
+
return lambda node: (
|
|
167
|
+
Be.While(node)
|
|
168
|
+
and IfThis.isAttributeNamespace_IdentifierGreaterThan0(namespace, identifier)(DOT.test(node)))
|
|
169
|
+
|
|
170
|
+
# Parse some code
|
|
171
|
+
code = """
|
|
172
|
+
while claude.counter > 0:
|
|
173
|
+
result += counter
|
|
174
|
+
counter -= 1
|
|
175
|
+
"""
|
|
176
|
+
tree = ast.parse(code)
|
|
177
|
+
|
|
178
|
+
# Find the while loop with our custom predicate
|
|
179
|
+
find_while_loop = IfThis.isWhileAttributeNamespace_IdentifierGreaterThan0("claude", "counter")
|
|
180
|
+
|
|
181
|
+
# Replace counter > 0 with counter > 1
|
|
182
|
+
change_condition = Grab.testAttribute(
|
|
183
|
+
Grab.comparatorsAttribute(
|
|
184
|
+
Then.replaceWith([Make.Constant(1)])
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Apply the transformation
|
|
189
|
+
NodeChanger(find_while_loop, change_condition).visit(tree)
|
|
190
|
+
|
|
191
|
+
print(ast.unparse(tree))
|
|
192
|
+
# Now outputs:
|
|
193
|
+
# while counter > 1:
|
|
194
|
+
# result += counter
|
|
195
|
+
# counter -= 1
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
### Example 4: Building Code Generation Systems
|
|
199
|
+
|
|
200
|
+
The following example shows how to set up a foundation for code generation and transformation systems:
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
from astToolkit import (
|
|
204
|
+
Be, DOT, IngredientsFunction, IngredientsModule, LedgerOfImports,
|
|
205
|
+
Make, NodeTourist, Then, parseLogicalPath2astModule, write_astModule
|
|
206
|
+
)
|
|
207
|
+
import ast
|
|
208
|
+
|
|
209
|
+
# Parse a module to extract a function
|
|
210
|
+
module_ast = parseLogicalPath2astModule("my_package.source_module")
|
|
211
|
+
|
|
212
|
+
# Extract a function and track its imports
|
|
213
|
+
function_name = "target_function"
|
|
214
|
+
function_def = NodeTourist(
|
|
215
|
+
IfThis.isFunctionDef_Identifier(function_name),
|
|
216
|
+
Then.extractIt
|
|
217
|
+
).captureLastMatch(module_ast)
|
|
218
|
+
|
|
219
|
+
if function_def:
|
|
220
|
+
# Create a self-contained function with tracked imports
|
|
221
|
+
ingredients = IngredientsFunction(
|
|
222
|
+
function_def,
|
|
223
|
+
LedgerOfImports(module_ast)
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
# Rename the function
|
|
227
|
+
ingredients.astFunctionDef.name = "optimized_" + function_name
|
|
228
|
+
|
|
229
|
+
# Add a decorator
|
|
230
|
+
decorator = Make.Call(
|
|
231
|
+
Make.Name("jit"),
|
|
232
|
+
[],
|
|
233
|
+
[Make.keyword("cache", Make.Constant(True))]
|
|
234
|
+
)
|
|
235
|
+
ingredients.astFunctionDef.decorator_list.append(decorator)
|
|
236
|
+
|
|
237
|
+
# Add required import
|
|
238
|
+
ingredients.imports.addImportFrom_asStr("numba", "jit")
|
|
239
|
+
|
|
240
|
+
# Create a module and write it to disk
|
|
241
|
+
module = IngredientsModule(ingredients)
|
|
242
|
+
write_astModule(module, "path/to/generated_code.py", "my_package")
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
### Example 5: Extending Core Classes
|
|
246
|
+
|
|
247
|
+
To create specialized patterns for your codebase, extend the core classes:
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
from astToolkit import ast_Identifier, Be, IfThis as astToolkit_IfThis
|
|
251
|
+
from collections.abc import Callable
|
|
252
|
+
from typing import TypeGuard
|
|
253
|
+
import ast
|
|
254
|
+
|
|
255
|
+
class IfThis(astToolkit_IfThis):
|
|
256
|
+
@staticmethod
|
|
257
|
+
def isAttributeNamespace_IdentifierGreaterThan0(
|
|
258
|
+
namespace: ast_Identifier,
|
|
259
|
+
identifier: ast_Identifier
|
|
260
|
+
) -> Callable[[ast.AST], TypeGuard[ast.Compare] | bool]:
|
|
261
|
+
"""Find comparisons like 'state.counter > 0'"""
|
|
262
|
+
return lambda node: (
|
|
263
|
+
Be.Compare(node)
|
|
264
|
+
and IfThis.isAttributeNamespace_Identifier(namespace, identifier)(node.left)
|
|
265
|
+
and Be.Gt(node.ops[0])
|
|
266
|
+
and IfThis.isConstant_value(0)(node.comparators[0])
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
@staticmethod
|
|
270
|
+
def isWhileAttributeNamespace_IdentifierGreaterThan0(
|
|
271
|
+
namespace: ast_Identifier,
|
|
272
|
+
identifier: ast_Identifier
|
|
273
|
+
) -> Callable[[ast.AST], TypeGuard[ast.While] | bool]:
|
|
274
|
+
"""Find while loops like 'while state.counter > 0:'"""
|
|
275
|
+
return lambda node: (
|
|
276
|
+
Be.While(node)
|
|
277
|
+
and IfThis.isAttributeNamespace_IdentifierGreaterThan0(namespace, identifier)(node.test)
|
|
278
|
+
)
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
### Real-world Application: Code Transformation Assembly-line
|
|
282
|
+
|
|
283
|
+
In the [mapFolding](https://github.com/hunterhogan/mapFolding) project, astToolkit is used to build a complete transformation assembly-line that:
|
|
284
|
+
|
|
285
|
+
1. Extracts algorithms from source modules
|
|
286
|
+
2. Transforms them into optimized variants
|
|
287
|
+
3. Applies numerical computing decorators
|
|
288
|
+
4. Handles dataclass management and type systems
|
|
289
|
+
5. Generates complete modules with proper imports
|
|
290
|
+
|
|
291
|
+
This pattern enables the separation of readable algorithm implementations from their high-performance variants while ensuring they remain functionally equivalent.
|
|
292
|
+
|
|
293
|
+
For deeper examples, see the [mapFolding/someAssemblyRequired](https://github.com/hunterhogan/mapFolding/tree/main/mapFolding/someAssemblyRequired/) directory.
|
|
294
|
+
|
|
295
|
+
## Installation
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
pip install astToolkit
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
## My Recovery
|
|
302
|
+
|
|
303
|
+
[](https://HunterThinks.com/support)
|
|
304
|
+
[](https://www.youtube.com/@HunterHogan)
|
|
305
|
+
|
|
306
|
+
## How to code
|
|
307
|
+
|
|
308
|
+
Coding One Step at a Time:
|
|
309
|
+
|
|
310
|
+
0. WRITE CODE.
|
|
311
|
+
1. Don't write stupid code that's hard to revise.
|
|
312
|
+
2. Write good code.
|
|
313
|
+
3. When revising, write better code.
|
|
314
|
+
|
|
315
|
+
[](https://creativecommons.org/licenses/by-nc/4.0/)
|