cjm-python-decompose-core 0.0.7__tar.gz → 0.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/PKG-INFO +24 -6
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/README.md +23 -5
- cjm_python_decompose_core-0.0.8/cjm_python_decompose_core/__init__.py +1 -0
- cjm_python_decompose_core-0.0.8/cjm_python_decompose_core/elements.py +480 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core/emit.py +51 -134
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core/extract.py +1 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core/parse.py +45 -2
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core.egg-info/PKG-INFO +24 -6
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core.egg-info/SOURCES.txt +2 -0
- cjm_python_decompose_core-0.0.8/tests/test_elements.py +220 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/tests/test_extract_ingest.py +1 -3
- cjm_python_decompose_core-0.0.8/tests/test_imports_emit.py +165 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/tests/test_parse.py +11 -2
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/tests/test_regions_emit.py +13 -2
- cjm_python_decompose_core-0.0.7/cjm_python_decompose_core/__init__.py +0 -1
- cjm_python_decompose_core-0.0.7/tests/test_imports_emit.py +0 -214
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/LICENSE +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core/ingest.py +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core.egg-info/dependency_links.txt +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core.egg-info/requires.txt +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/cjm_python_decompose_core.egg-info/top_level.txt +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/pyproject.toml +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/setup.cfg +0 -0
- {cjm_python_decompose_core-0.0.7 → cjm_python_decompose_core-0.0.8}/tests/test_method_attribution.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-python-decompose-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.8
|
|
4
4
|
Summary: A Python source-code decomposition core for context graphs: parses .py modules (docstrings, imports, and the symbol tree of functions/classes/methods with their calls) into provenance-carrying graph nodes and edges that co-reside with the markdown source-type. First source = the ecosystem's own code; generalizes to any Python corpus.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -24,6 +24,7 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
24
24
|
## Modules
|
|
25
25
|
|
|
26
26
|
- **`cjm_python_decompose_core.__init__`**
|
|
27
|
+
- **`cjm_python_decompose_core.elements`** — The element source unit for Python modules (design abfc2829 (1); d26e00f9; 3e155cd6).
|
|
27
28
|
- **`cjm_python_decompose_core.emit`** — Project a module BACK out of the graph (graph -> .py) — the canonical emit leg.
|
|
28
29
|
- **`cjm_python_decompose_core.extract`** — Bind parsed Python onto dev-graph-schema code nodes (the dev-domain binding).
|
|
29
30
|
- **`cjm_python_decompose_core.ingest`** — Flatten a decomposed Python corpus into graph elements (the queue-free half).
|
|
@@ -31,15 +32,30 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
31
32
|
|
|
32
33
|
## API
|
|
33
34
|
|
|
35
|
+
### `cjm_python_decompose_core.elements`
|
|
36
|
+
|
|
37
|
+
- `CodeElement` _class_ — One element of a module, its text at depth 0.
|
|
38
|
+
- `LiftedModule` _class_ — A module decomposed in the element regime: its elements (one `imports` element among them,
|
|
39
|
+
- `binding_union` _function_ — The union of a module's bindings -- the input the import block renders from.
|
|
40
|
+
- `class_members` _function_ — A class's members as 0-based inclusive line spans: the class's own text ends at `anchor`,
|
|
41
|
+
- `compose_elements` _function_ — Compose elements back into a module: members re-indented by depth, every seam derived.
|
|
42
|
+
- `compose_module` _function_ — Compose a captured module: the imports element's text is rendered from the union of every
|
|
43
|
+
- `decompose_elements` _function_ — Decompose a module into elements -- top-level regions, then class members recursively. A
|
|
44
|
+
- `element_names` _function_ — The names an element's own text uses -- what claims an import binding (abfc2829 (3): a binding
|
|
45
|
+
- `import_slot` _function_ — The import block's derived slot: after a leading docstring or comment preamble, else first --
|
|
46
|
+
- `lift_imports` _function_ — Decompose a module in the element regime: lift its leading run of imports (after the module
|
|
47
|
+
- `string_lines` _function_ — The lines a string literal continues onto -- their whitespace is the string's value.
|
|
48
|
+
- `string_reads` _function_ — The names an element reads through a STRING -- a fixture string (`usefixtures("db")`,
|
|
49
|
+
- `unresolved_names` _function_ — The names a module reads that no module-level binding (an import, an assignment, a def or class,
|
|
50
|
+
|
|
34
51
|
### `cjm_python_decompose_core.emit`
|
|
35
52
|
|
|
36
|
-
- `emit_module_from_nodes` _function_ — Reassemble a module's
|
|
37
|
-
- `module_used_bindings` _function_ — Union of every contained symbol's import bindings + the module-level ones (+ any
|
|
53
|
+
- `emit_module_from_nodes` _function_ — Reassemble a module's `.py` source from its graph nodes (the round-trip): its ordered
|
|
38
54
|
- `nodes_for_module` _function_ — Filter queried region nodes down to one module (by `module_id` property).
|
|
39
55
|
- `regions_from_nodes` _function_ — Collect a module's top-level regions from its nodes, ordered by `order_index`.
|
|
40
56
|
- `render_binding` _function_ — Render one import-binding descriptor back to its canonical import statement.
|
|
41
|
-
- `render_import_block` _function_ —
|
|
42
|
-
- `
|
|
57
|
+
- `render_import_block` _function_ — Render a module's canonical import block from its import bindings (the element regime's
|
|
58
|
+
- `sibling_modules` _function_ — The modules a bare import in `module_path` reaches as siblings: `<dir>/<name>.py` and
|
|
43
59
|
|
|
44
60
|
### `cjm_python_decompose_core.extract`
|
|
45
61
|
|
|
@@ -64,13 +80,15 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
64
80
|
- `ParsedModule` _class_ — The structural decomposition of one Python module.
|
|
65
81
|
- `ParsedSymbol` _class_ — One definition within a module (function/class/method), with its nesting.
|
|
66
82
|
- `SourceRegion` _class_ — One ordered top-level region of a module, held verbatim (symbol or text).
|
|
83
|
+
- `accessor_of` _function_ — Whether a def continues the accessor chain of the def directly before it: `@<name>.setter`,
|
|
67
84
|
- `emit_regions` _function_ — Reassemble ordered regions into canonical `.py` source — the graph owns formatting.
|
|
68
85
|
- `iter_symbols` _function_ — Flatten the symbol tree depth-first (parent before its children).
|
|
69
86
|
- `monkeypatch_assignments` _function_ — Top-level monkey-patch assignments: `Class.method = func` (the incremental-class idiom).
|
|
70
87
|
- `parse_module` _function_ — Parse Python source into module docstring + imports + the symbol tree.
|
|
71
88
|
- `parse_regions` _function_ — Decompose a module into ordered verbatim top-level regions (the round-trip substrate).
|
|
89
|
+
- `source_lines` _function_ — Split source where the tokenizer counts lines -- at \n, \r\n and \r only. `str.splitlines` also
|
|
72
90
|
|
|
73
91
|
## Dependencies
|
|
74
92
|
|
|
75
93
|
**Depends on:** `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-dev-graph-schema`
|
|
76
|
-
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`
|
|
94
|
+
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`, `cjm-substrate`
|
|
@@ -7,6 +7,7 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
7
7
|
## Modules
|
|
8
8
|
|
|
9
9
|
- **`cjm_python_decompose_core.__init__`**
|
|
10
|
+
- **`cjm_python_decompose_core.elements`** — The element source unit for Python modules (design abfc2829 (1); d26e00f9; 3e155cd6).
|
|
10
11
|
- **`cjm_python_decompose_core.emit`** — Project a module BACK out of the graph (graph -> .py) — the canonical emit leg.
|
|
11
12
|
- **`cjm_python_decompose_core.extract`** — Bind parsed Python onto dev-graph-schema code nodes (the dev-domain binding).
|
|
12
13
|
- **`cjm_python_decompose_core.ingest`** — Flatten a decomposed Python corpus into graph elements (the queue-free half).
|
|
@@ -14,15 +15,30 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
14
15
|
|
|
15
16
|
## API
|
|
16
17
|
|
|
18
|
+
### `cjm_python_decompose_core.elements`
|
|
19
|
+
|
|
20
|
+
- `CodeElement` _class_ — One element of a module, its text at depth 0.
|
|
21
|
+
- `LiftedModule` _class_ — A module decomposed in the element regime: its elements (one `imports` element among them,
|
|
22
|
+
- `binding_union` _function_ — The union of a module's bindings -- the input the import block renders from.
|
|
23
|
+
- `class_members` _function_ — A class's members as 0-based inclusive line spans: the class's own text ends at `anchor`,
|
|
24
|
+
- `compose_elements` _function_ — Compose elements back into a module: members re-indented by depth, every seam derived.
|
|
25
|
+
- `compose_module` _function_ — Compose a captured module: the imports element's text is rendered from the union of every
|
|
26
|
+
- `decompose_elements` _function_ — Decompose a module into elements -- top-level regions, then class members recursively. A
|
|
27
|
+
- `element_names` _function_ — The names an element's own text uses -- what claims an import binding (abfc2829 (3): a binding
|
|
28
|
+
- `import_slot` _function_ — The import block's derived slot: after a leading docstring or comment preamble, else first --
|
|
29
|
+
- `lift_imports` _function_ — Decompose a module in the element regime: lift its leading run of imports (after the module
|
|
30
|
+
- `string_lines` _function_ — The lines a string literal continues onto -- their whitespace is the string's value.
|
|
31
|
+
- `string_reads` _function_ — The names an element reads through a STRING -- a fixture string (`usefixtures("db")`,
|
|
32
|
+
- `unresolved_names` _function_ — The names a module reads that no module-level binding (an import, an assignment, a def or class,
|
|
33
|
+
|
|
17
34
|
### `cjm_python_decompose_core.emit`
|
|
18
35
|
|
|
19
|
-
- `emit_module_from_nodes` _function_ — Reassemble a module's
|
|
20
|
-
- `module_used_bindings` _function_ — Union of every contained symbol's import bindings + the module-level ones (+ any
|
|
36
|
+
- `emit_module_from_nodes` _function_ — Reassemble a module's `.py` source from its graph nodes (the round-trip): its ordered
|
|
21
37
|
- `nodes_for_module` _function_ — Filter queried region nodes down to one module (by `module_id` property).
|
|
22
38
|
- `regions_from_nodes` _function_ — Collect a module's top-level regions from its nodes, ordered by `order_index`.
|
|
23
39
|
- `render_binding` _function_ — Render one import-binding descriptor back to its canonical import statement.
|
|
24
|
-
- `render_import_block` _function_ —
|
|
25
|
-
- `
|
|
40
|
+
- `render_import_block` _function_ — Render a module's canonical import block from its import bindings (the element regime's
|
|
41
|
+
- `sibling_modules` _function_ — The modules a bare import in `module_path` reaches as siblings: `<dir>/<name>.py` and
|
|
26
42
|
|
|
27
43
|
### `cjm_python_decompose_core.extract`
|
|
28
44
|
|
|
@@ -47,13 +63,15 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
47
63
|
- `ParsedModule` _class_ — The structural decomposition of one Python module.
|
|
48
64
|
- `ParsedSymbol` _class_ — One definition within a module (function/class/method), with its nesting.
|
|
49
65
|
- `SourceRegion` _class_ — One ordered top-level region of a module, held verbatim (symbol or text).
|
|
66
|
+
- `accessor_of` _function_ — Whether a def continues the accessor chain of the def directly before it: `@<name>.setter`,
|
|
50
67
|
- `emit_regions` _function_ — Reassemble ordered regions into canonical `.py` source — the graph owns formatting.
|
|
51
68
|
- `iter_symbols` _function_ — Flatten the symbol tree depth-first (parent before its children).
|
|
52
69
|
- `monkeypatch_assignments` _function_ — Top-level monkey-patch assignments: `Class.method = func` (the incremental-class idiom).
|
|
53
70
|
- `parse_module` _function_ — Parse Python source into module docstring + imports + the symbol tree.
|
|
54
71
|
- `parse_regions` _function_ — Decompose a module into ordered verbatim top-level regions (the round-trip substrate).
|
|
72
|
+
- `source_lines` _function_ — Split source where the tokenizer counts lines -- at \n, \r\n and \r only. `str.splitlines` also
|
|
55
73
|
|
|
56
74
|
## Dependencies
|
|
57
75
|
|
|
58
76
|
**Depends on:** `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-dev-graph-schema`
|
|
59
|
-
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`
|
|
77
|
+
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`, `cjm-substrate`
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.8"
|
|
@@ -0,0 +1,480 @@
|
|
|
1
|
+
"""The element source unit for Python modules (design abfc2829 (1); d26e00f9; 3e155cd6).
|
|
2
|
+
|
|
3
|
+
A module decomposes into ELEMENTS: its top-level regions (`parse.parse_regions`), and inside a
|
|
4
|
+
class -- recursively -- its members. A def or a nested class is one element; a run of class-level
|
|
5
|
+
statements and the free comment blocks among them is one text element; a comment indented deeper
|
|
6
|
+
than the member level that trails a body belongs to that member. A class element holds only its
|
|
7
|
+
OWN text (decorators, leading comments, header, docstring); its members are its children.
|
|
8
|
+
|
|
9
|
+
Each element's text is held at depth 0: indentation is derived from depth (one level = 4 spaces,
|
|
10
|
+
the corpus's only unit) as a heading's level is, so a member's lines are dedented on decompose and
|
|
11
|
+
re-indented on compose. A line that continues a multi-line string literal is never touched -- its
|
|
12
|
+
whitespace is the string's value. Seams are derived too: at the top level `parse.emit_regions`'
|
|
13
|
+
(two blank lines around a def or class, one between text regions); inside a class none under a
|
|
14
|
+
bare header, one after a docstring, none between two consecutive one-line defs, one otherwise.
|
|
15
|
+
|
|
16
|
+
`compose_elements(decompose_elements(text)) == text` for every canonical module: the round trip is
|
|
17
|
+
the oracle the element capture is accepted by. Pure and schema-free, like `parse`.
|
|
18
|
+
|
|
19
|
+
THE IMPORT BLOCK (abfc2829 (2)-(3), ruling 21b8fcca, design detail 6d35c07d (3)): a captured module's
|
|
20
|
+
leading run of imports leaves the text -- `lift_imports` gives each binding to every element whose own
|
|
21
|
+
text names it (a reference or a parameter: a pytest fixture is requested by parameter name) and puts
|
|
22
|
+
the bindings no element claims on ONE `imports` element; `compose_module` renders the union of every
|
|
23
|
+
element's bindings as that element's text at its derived slot (after a leading docstring or comment
|
|
24
|
+
preamble, else first). The statements from the first non-import onward stay verbatim, imports among
|
|
25
|
+
them included: an import after a statement may depend on it (a sys.path change, an environment
|
|
26
|
+
setting), so it is never hoisted. `lift_imports(compose_module(m)) == m` is the element regime's
|
|
27
|
+
invariant, as the round trip is the snapshot regime's.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
import ast
|
|
31
|
+
import builtins
|
|
32
|
+
import io
|
|
33
|
+
import symtable
|
|
34
|
+
import tokenize
|
|
35
|
+
from dataclasses import dataclass, field
|
|
36
|
+
from typing import List, Optional, Tuple
|
|
37
|
+
|
|
38
|
+
from cjm_python_decompose_core.emit import _looks_like_docstring, render_import_block
|
|
39
|
+
from cjm_python_decompose_core.parse import (_import_bindings, accessor_of, emit_regions,
|
|
40
|
+
parse_regions, SourceRegion)
|
|
41
|
+
|
|
42
|
+
INDENT = 4 # One depth level: the corpus's only member indentation (492 of 492 classes)
|
|
43
|
+
IMPORTS = "imports" # The kind of a module's one import-block element (its text derived from bindings)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass
|
|
47
|
+
class CodeElement:
|
|
48
|
+
"""One element of a module, its text at depth 0."""
|
|
49
|
+
|
|
50
|
+
kind: str # "symbol" (a def / class) | "text" (statements and comments) | "imports" (the derived import block)
|
|
51
|
+
text: str # The element's OWN text, dedented to depth 0 (no trailing newline); an imports element's is derived
|
|
52
|
+
depth: int = 0 # 0 = top level; a class member sits one deeper than its class
|
|
53
|
+
parent: Optional[int] = None # Index of the enclosing class element in the list (None at top level)
|
|
54
|
+
qualname: str = "" # Symbols: dotted through enclosing classes ("Cls.method")
|
|
55
|
+
symbol_kind: str = "" # Symbols: "function" | "class"
|
|
56
|
+
bindings: list = field(default_factory=list) # The import-binding descriptors the element owns (`lift_imports`)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def string_lines(
|
|
60
|
+
text: str, # Python source
|
|
61
|
+
) -> set: # 0-based indexes of the lines that continue a multi-line string literal
|
|
62
|
+
"""The lines a string literal continues onto -- their whitespace is the string's value."""
|
|
63
|
+
inside, starts = set(), []
|
|
64
|
+
for tok in tokenize.generate_tokens(io.StringIO(text).readline):
|
|
65
|
+
if tok.type == tokenize.FSTRING_START:
|
|
66
|
+
starts.append(tok.start[0])
|
|
67
|
+
elif tok.type == tokenize.FSTRING_END:
|
|
68
|
+
inside.update(range(starts.pop(), tok.end[0]))
|
|
69
|
+
elif tok.type == tokenize.STRING:
|
|
70
|
+
inside.update(range(tok.start[0], tok.end[0]))
|
|
71
|
+
return inside
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _indent(line: str) -> int:
|
|
75
|
+
return len(line) - len(line.lstrip())
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _is_comment(line: str) -> bool:
|
|
79
|
+
return line.lstrip().startswith("#")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def class_members(
|
|
83
|
+
lines: list, # The source's lines
|
|
84
|
+
node: ast.ClassDef, # A class in that source
|
|
85
|
+
) -> Optional[Tuple[int, bool, list]]: # (anchor, doc, [(first, last, one_line_def, member node | None)])
|
|
86
|
+
"""A class's members as 0-based inclusive line spans: the class's own text ends at `anchor`,
|
|
87
|
+
`doc` tells whether it holds a docstring, a text member carries None. None when the class has
|
|
88
|
+
no member below its own text (an empty body, or a body on the header line)."""
|
|
89
|
+
body = node.body
|
|
90
|
+
doc = (isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant)
|
|
91
|
+
and isinstance(body[0].value.value, str))
|
|
92
|
+
members = body[1:] if doc else body
|
|
93
|
+
if not members:
|
|
94
|
+
return None
|
|
95
|
+
first = members[0].decorator_list[0] if getattr(members[0], "decorator_list", None) else members[0]
|
|
96
|
+
if doc:
|
|
97
|
+
anchor = body[0].end_lineno - 1
|
|
98
|
+
else:
|
|
99
|
+
anchor = next((j for j in range(first.lineno - 2, node.lineno - 2, -1)
|
|
100
|
+
if lines[j].strip() and not _is_comment(lines[j])), None)
|
|
101
|
+
if anchor is None or anchor >= first.lineno - 1:
|
|
102
|
+
return None
|
|
103
|
+
level = members[0].col_offset
|
|
104
|
+
|
|
105
|
+
def opens(s, floor): # the statement's first line, its attached member-level comments included
|
|
106
|
+
ln = (s.decorator_list[0].lineno if getattr(s, "decorator_list", None) else s.lineno) - 1
|
|
107
|
+
while ln - 1 > floor and _is_comment(lines[ln - 1]) and _indent(lines[ln - 1]) == level:
|
|
108
|
+
ln -= 1
|
|
109
|
+
return ln
|
|
110
|
+
|
|
111
|
+
defs = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
|
|
112
|
+
units, floor, prev = [], anchor, None
|
|
113
|
+
for s in members:
|
|
114
|
+
if accessor_of(s, prev): # a property's next accessor extends its unit: one attribute, one element
|
|
115
|
+
units[-1][1], units[-1][3] = s.end_lineno - 1, False
|
|
116
|
+
else:
|
|
117
|
+
start = opens(s, floor)
|
|
118
|
+
is_def = isinstance(s, defs)
|
|
119
|
+
units.append([start, s.end_lineno - 1, is_def, is_def and start == s.end_lineno - 1,
|
|
120
|
+
s if is_def else None])
|
|
121
|
+
floor, prev = s.end_lineno - 1, s
|
|
122
|
+
pieces, prev_end = [], anchor
|
|
123
|
+
for i, u in enumerate(units):
|
|
124
|
+
gap = range(prev_end + 1, u[0])
|
|
125
|
+
if i: # a deeper comment trailing the previous body belongs to it
|
|
126
|
+
last = units[i - 1]
|
|
127
|
+
for j in gap:
|
|
128
|
+
if not lines[j].strip():
|
|
129
|
+
continue
|
|
130
|
+
if _indent(lines[j]) > level:
|
|
131
|
+
last[1], last[3] = j, False
|
|
132
|
+
continue
|
|
133
|
+
break
|
|
134
|
+
gap = range(last[1] + 1, u[0])
|
|
135
|
+
free = [j for j in gap if lines[j].strip()]
|
|
136
|
+
if free:
|
|
137
|
+
pieces.append([free[0], free[-1], False, False, None])
|
|
138
|
+
pieces.append(u)
|
|
139
|
+
prev_end = u[1]
|
|
140
|
+
merged = []
|
|
141
|
+
for p in pieces: # a run of statements and free comments is one text element
|
|
142
|
+
if not p[2] and merged and not merged[-1][2]:
|
|
143
|
+
merged[-1][1] = p[1]
|
|
144
|
+
else:
|
|
145
|
+
merged.append(list(p))
|
|
146
|
+
return anchor, doc, [(a, b, one, n) for a, b, _, one, n in merged]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _shift(
|
|
150
|
+
text: str, # Source at some depth
|
|
151
|
+
by: int, # Columns to add (> 0) or remove (< 0) on each line outside a string literal
|
|
152
|
+
) -> str:
|
|
153
|
+
"""Re-indent every non-blank line that does not continue a string literal."""
|
|
154
|
+
keep = string_lines(text)
|
|
155
|
+
out = []
|
|
156
|
+
for i, ln in enumerate(text.split("\n")):
|
|
157
|
+
if i in keep or not ln.strip():
|
|
158
|
+
out.append(ln)
|
|
159
|
+
elif by > 0:
|
|
160
|
+
out.append(" " * by + ln)
|
|
161
|
+
else:
|
|
162
|
+
if ln[:-by].strip():
|
|
163
|
+
raise ValueError(f"line {i + 1} sits shallower than its member level: {ln!r}")
|
|
164
|
+
out.append(ln[-by:])
|
|
165
|
+
return "\n".join(out)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _split_symbol(
|
|
169
|
+
text: str, # A def / class region or member at depth 0 (leading comments + decorators included)
|
|
170
|
+
depth: int, # Its depth
|
|
171
|
+
parent: Optional[int], # Its enclosing class element
|
|
172
|
+
prefix: str, # The enclosing classes' dotted qualname ("" at top level)
|
|
173
|
+
out: list, # The element list being built (pre-order)
|
|
174
|
+
) -> None:
|
|
175
|
+
tree = ast.parse(text)
|
|
176
|
+
node = tree.body[-1]
|
|
177
|
+
qualname = f"{prefix}{node.name}"
|
|
178
|
+
kind = "class" if isinstance(node, ast.ClassDef) else "function"
|
|
179
|
+
lines = text.split("\n")
|
|
180
|
+
found = class_members(lines, node) if kind == "class" else None
|
|
181
|
+
if not found:
|
|
182
|
+
out.append(CodeElement("symbol", text, depth, parent, qualname, kind))
|
|
183
|
+
return
|
|
184
|
+
anchor, _doc, members = found
|
|
185
|
+
me = len(out)
|
|
186
|
+
out.append(CodeElement("symbol", "\n".join(lines[:anchor + 1]), depth, parent, qualname, kind))
|
|
187
|
+
for a, b, _one, member in members:
|
|
188
|
+
body = _shift("\n".join(lines[a:b + 1]), -INDENT)
|
|
189
|
+
if member is None:
|
|
190
|
+
out.append(CodeElement("text", body, depth + 1, me))
|
|
191
|
+
else:
|
|
192
|
+
_split_symbol(body, depth + 1, me, f"{qualname}.", out)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def decompose_elements(
|
|
196
|
+
text: str, # A module's source
|
|
197
|
+
) -> List[CodeElement]: # Its elements in source order (pre-order: a class precedes its members)
|
|
198
|
+
"""Decompose a module into elements -- top-level regions, then class members recursively. A
|
|
199
|
+
symbol element is one qualname: a property's accessor chain is one element (`accessor_of`), and
|
|
200
|
+
any other two definitions of one qualname (an overload, a redefinition) raise ValueError -- one
|
|
201
|
+
id per element, so the writers refuse the text rather than guess (design detail ed20aa51 (2))."""
|
|
202
|
+
out: List[CodeElement] = []
|
|
203
|
+
for r in parse_regions(text):
|
|
204
|
+
if r.kind == "symbol":
|
|
205
|
+
_split_symbol(r.text, 0, None, "", out)
|
|
206
|
+
else:
|
|
207
|
+
out.append(CodeElement("text", r.text))
|
|
208
|
+
seen: set = set()
|
|
209
|
+
for e in out:
|
|
210
|
+
if e.kind == "symbol":
|
|
211
|
+
if e.qualname in seen:
|
|
212
|
+
raise ValueError(f"two definitions of {e.qualname} -- one element per qualname "
|
|
213
|
+
"(only a property's accessor chain shares one)")
|
|
214
|
+
seen.add(e.qualname)
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _has_docstring(
|
|
219
|
+
own: str, # A class element's own text (header, possibly a docstring)
|
|
220
|
+
) -> bool:
|
|
221
|
+
"""Whether a class's own text ends in its docstring (derived, never stored)."""
|
|
222
|
+
body = ast.parse(own + "\n" + " " * INDENT + "pass").body[-1].body
|
|
223
|
+
return (isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant)
|
|
224
|
+
and isinstance(body[0].value.value, str))
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _compose(
|
|
228
|
+
elements: List[CodeElement], # The module's elements
|
|
229
|
+
i: int, # The element to compose
|
|
230
|
+
children: dict, # parent index -> child indexes, in order
|
|
231
|
+
) -> str: # The element with its members, at depth 0
|
|
232
|
+
e = elements[i]
|
|
233
|
+
kids = children.get(i)
|
|
234
|
+
if not kids:
|
|
235
|
+
return e.text
|
|
236
|
+
parts = [e.text]
|
|
237
|
+
if _has_docstring(e.text):
|
|
238
|
+
parts.append("")
|
|
239
|
+
composed = [_compose(elements, k, children) for k in kids]
|
|
240
|
+
one_line = [elements[k].kind == "symbol" and "\n" not in c for k, c in zip(kids, composed)]
|
|
241
|
+
for n, c in enumerate(composed):
|
|
242
|
+
if n and not (one_line[n] and one_line[n - 1]): # a class with members is never one line
|
|
243
|
+
parts.append("")
|
|
244
|
+
parts.append(_shift(c, INDENT))
|
|
245
|
+
return "\n".join(parts)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def compose_elements(
|
|
249
|
+
elements: List[CodeElement], # A module's elements (pre-order, parents by index)
|
|
250
|
+
) -> str: # The module's canonical source
|
|
251
|
+
"""Compose elements back into a module: members re-indented by depth, every seam derived."""
|
|
252
|
+
children: dict = {}
|
|
253
|
+
for i, e in enumerate(elements):
|
|
254
|
+
if e.parent is not None:
|
|
255
|
+
children.setdefault(e.parent, []).append(i)
|
|
256
|
+
regions = [SourceRegion(kind="symbol" if e.kind == "symbol" else "text",
|
|
257
|
+
text=_compose(elements, i, children), start_line=0, end_line=0)
|
|
258
|
+
for i, e in enumerate(elements)
|
|
259
|
+
if e.parent is None and not (e.kind == IMPORTS and not e.text)]
|
|
260
|
+
return emit_regions(regions)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def import_slot(
|
|
264
|
+
elements: List[CodeElement], # A module's elements WITHOUT its imports element (pre-order)
|
|
265
|
+
) -> int: # The list index the imports element takes
|
|
266
|
+
"""The import block's derived slot: after a leading docstring or comment preamble, else first --
|
|
267
|
+
`emit_module_from_nodes`' rule, so a captured module keeps the block where the snapshot regime put it."""
|
|
268
|
+
first = next((i for i, e in enumerate(elements) if e.parent is None), None)
|
|
269
|
+
if first is not None and elements[first].kind == "text" and _looks_like_docstring(elements[first].text):
|
|
270
|
+
return first + 1
|
|
271
|
+
return 0
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _parse_own(
|
|
275
|
+
text: str, # An element's own text at depth 0
|
|
276
|
+
) -> ast.Module:
|
|
277
|
+
"""Parse an element's own text; a class's own text (header and docstring, its members apart)
|
|
278
|
+
is given a body first."""
|
|
279
|
+
try:
|
|
280
|
+
return ast.parse(text)
|
|
281
|
+
except SyntaxError:
|
|
282
|
+
return ast.parse(text + "\n" + " " * INDENT + "pass")
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def element_names(
|
|
286
|
+
text: str, # An element's own text at depth 0
|
|
287
|
+
) -> set: # Every name it loads, stores or declares as a parameter
|
|
288
|
+
"""The names an element's own text uses -- what claims an import binding (abfc2829 (3): a binding
|
|
289
|
+
goes to the elements whose refs or parameters name it). A parameter counts: pytest requests a
|
|
290
|
+
fixture by parameter name. A string (`usefixtures("db")`) names nothing; its import stays unclaimed."""
|
|
291
|
+
out = set()
|
|
292
|
+
for n in ast.walk(_parse_own(text)):
|
|
293
|
+
if isinstance(n, ast.Name):
|
|
294
|
+
out.add(n.id)
|
|
295
|
+
elif isinstance(n, ast.arg):
|
|
296
|
+
out.add(n.arg)
|
|
297
|
+
elif isinstance(n, (ast.Global, ast.Nonlocal)):
|
|
298
|
+
out.update(n.names)
|
|
299
|
+
return out
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
# Calls whose string arguments name a pytest fixture: the fixture channel a ref walk cannot see.
|
|
303
|
+
_FIXTURE_CALLS = frozenset({"usefixtures", "getfixturevalue"})
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _string_names(
|
|
307
|
+
node: Optional[ast.AST], # A string constant read where a name is meant (or None)
|
|
308
|
+
) -> set: # The names it holds
|
|
309
|
+
"""A string that stands for a name or an expression: a fixture name, an `__all__` entry, a
|
|
310
|
+
forward-reference annotation ("Path", "Optional[Path]")."""
|
|
311
|
+
if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):
|
|
312
|
+
return set()
|
|
313
|
+
try:
|
|
314
|
+
expr = ast.parse(node.value.strip(), mode="eval")
|
|
315
|
+
except SyntaxError:
|
|
316
|
+
return set()
|
|
317
|
+
return {n.id for n in ast.walk(expr) if isinstance(n, ast.Name)}
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def string_reads(
|
|
321
|
+
text: str, # An element's own text at depth 0
|
|
322
|
+
) -> set: # The names its strings stand for
|
|
323
|
+
"""The names an element reads through a STRING -- a fixture string (`usefixtures("db")`,
|
|
324
|
+
`getfixturevalue("db")`), an `__all__` entry (a re-export), a string annotation. They claim no
|
|
325
|
+
binding (`element_names` decides ownership, which the element records hold); they keep a binding
|
|
326
|
+
alive: a binding its element stops reading is dropped only when no element reads its name by any
|
|
327
|
+
channel (design detail 25c1fadc, ruling D)."""
|
|
328
|
+
out = set()
|
|
329
|
+
for n in ast.walk(_parse_own(text)):
|
|
330
|
+
if isinstance(n, ast.arg):
|
|
331
|
+
out |= _string_names(n.annotation)
|
|
332
|
+
elif isinstance(n, ast.Call):
|
|
333
|
+
f = n.func
|
|
334
|
+
if (f.attr if isinstance(f, ast.Attribute) else getattr(f, "id", "")) in _FIXTURE_CALLS:
|
|
335
|
+
for a in n.args:
|
|
336
|
+
out |= _string_names(a)
|
|
337
|
+
elif isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
338
|
+
out |= _string_names(n.returns)
|
|
339
|
+
elif isinstance(n, ast.AnnAssign):
|
|
340
|
+
out |= _string_names(n.annotation)
|
|
341
|
+
elif isinstance(n, (ast.Assign, ast.AugAssign)):
|
|
342
|
+
targets = n.targets if isinstance(n, ast.Assign) else [n.target]
|
|
343
|
+
if any(isinstance(t, ast.Name) and t.id == "__all__" for t in targets) and \
|
|
344
|
+
isinstance(n.value, (ast.List, ast.Tuple)):
|
|
345
|
+
for e in n.value.elts:
|
|
346
|
+
out |= _string_names(e)
|
|
347
|
+
return out
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def binding_union(
|
|
351
|
+
elements: List[CodeElement], # A module's elements
|
|
352
|
+
) -> list: # Every binding they own, deduplicated, in element order
|
|
353
|
+
"""The union of a module's bindings -- the input the import block renders from."""
|
|
354
|
+
out, seen = [], set()
|
|
355
|
+
for e in elements:
|
|
356
|
+
for b in e.bindings:
|
|
357
|
+
k = (b.get("kind"), b.get("level", 0), b.get("module", ""), b.get("imported", ""),
|
|
358
|
+
b.get("alias", ""))
|
|
359
|
+
if k not in seen:
|
|
360
|
+
seen.add(k)
|
|
361
|
+
out.append(b)
|
|
362
|
+
return out
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def compose_module(
|
|
366
|
+
elements: List[CodeElement], # A captured module's elements, its imports element among them
|
|
367
|
+
local: frozenset = frozenset(), # The module's sibling module names (`emit.sibling_modules`)
|
|
368
|
+
) -> str: # The module's source
|
|
369
|
+
"""Compose a captured module: the imports element's text is rendered from the union of every
|
|
370
|
+
element's bindings, then the elements compose as `compose_elements`."""
|
|
371
|
+
block = render_import_block(binding_union(elements), local)
|
|
372
|
+
return compose_elements([CodeElement(e.kind, block, e.depth, e.parent, e.qualname, e.symbol_kind,
|
|
373
|
+
e.bindings) if e.kind == IMPORTS else e for e in elements])
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
# Names every module holds without binding them: the interpreter sets them on the module object.
|
|
377
|
+
MODULE_NAMES = frozenset({"__annotations__", "__builtins__", "__cached__", "__dict__", "__doc__", "__file__",
|
|
378
|
+
"__loader__", "__name__", "__package__", "__path__", "__spec__"})
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def unresolved_names(
|
|
382
|
+
text: str, # A module's source
|
|
383
|
+
) -> Optional[set]: # The names it reads that nothing binds; None when a star import makes that unknowable
|
|
384
|
+
"""The names a module reads that no module-level binding (an import, an assignment, a def or class,
|
|
385
|
+
a `global` declaration that assigns), builtin or module attribute covers -- in any scope: a name a
|
|
386
|
+
function, class body or comprehension reads from the module (`symtable`'s implicit globals) and a
|
|
387
|
+
name the module body itself reads. The binding check of abfc2829 (2): such a name raises NameError
|
|
388
|
+
when the code runs. A `from x import *` binds names no parse can see, so the check cannot run."""
|
|
389
|
+
if any(isinstance(n, ast.ImportFrom) and any(a.name == "*" for a in n.names) for n in ast.walk(ast.parse(text))):
|
|
390
|
+
return None
|
|
391
|
+
top = symtable.symtable(text, "<module>", "exec")
|
|
392
|
+
syms = top.get_symbols()
|
|
393
|
+
bound = {s.get_name() for s in syms if s.is_assigned() or s.is_imported() or s.is_declared_global()}
|
|
394
|
+
reads: List[str] = [s.get_name() for s in syms if s.is_referenced()]
|
|
395
|
+
tables = list(top.get_children())
|
|
396
|
+
while tables:
|
|
397
|
+
t = tables.pop()
|
|
398
|
+
tables.extend(t.get_children())
|
|
399
|
+
for s in t.get_symbols():
|
|
400
|
+
if s.is_declared_global() and s.is_assigned():
|
|
401
|
+
bound.add(s.get_name())
|
|
402
|
+
elif s.is_global() and not s.is_declared_global() and s.is_referenced():
|
|
403
|
+
reads.append(s.get_name())
|
|
404
|
+
known = bound | set(dir(builtins)) | MODULE_NAMES
|
|
405
|
+
return {n for n in reads if n not in known}
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
@dataclass
|
|
409
|
+
class LiftedModule:
|
|
410
|
+
"""A module decomposed in the element regime: its elements (one `imports` element among them,
|
|
411
|
+
its text empty -- derived at compose) and, per top-level text element, the region it came from."""
|
|
412
|
+
|
|
413
|
+
elements: List[CodeElement] # Pre-order; the imports element at `import_slot`
|
|
414
|
+
origins: List[Optional[int]] # Per element: the index (in `parse_regions(text)`) of the top-level text region it holds content of; None for symbols, members and the imports element
|
|
415
|
+
imports_origin: Optional[int] # The first import-holding region the lift empties (the imports element takes its identity), else None
|
|
416
|
+
emptied: List[int] # Import-holding regions the lift empties, `imports_origin` included
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def lift_imports(
|
|
420
|
+
text: str, # A module's source (canonical)
|
|
421
|
+
) -> LiftedModule:
|
|
422
|
+
"""Decompose a module in the element regime: lift its leading run of imports (after the module
|
|
423
|
+
docstring) into bindings, give each binding to every element whose own text names it, and put
|
|
424
|
+
the rest on the imports element. Statements from the first non-import onward stay verbatim."""
|
|
425
|
+
tree = ast.parse(text)
|
|
426
|
+
body = tree.body
|
|
427
|
+
i = 1 if (body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant)
|
|
428
|
+
and isinstance(body[0].value.value, str)) else 0
|
|
429
|
+
run = []
|
|
430
|
+
while i < len(body) and isinstance(body[i], (ast.Import, ast.ImportFrom)):
|
|
431
|
+
run.append(body[i])
|
|
432
|
+
i += 1
|
|
433
|
+
lines = text.split("\n")
|
|
434
|
+
lifted = {ln for s in run for ln in range(s.lineno - 1, s.end_lineno)}
|
|
435
|
+
kept = [n for n in range(len(lines)) if n not in lifted]
|
|
436
|
+
table = _import_bindings(ast.Module(body=run, type_ignores=[]))
|
|
437
|
+
before = [r for r in parse_regions(text)]
|
|
438
|
+
region_of = {} # original 0-based line -> original region index
|
|
439
|
+
for k, r in enumerate(before):
|
|
440
|
+
for ln in range(r.start_line - 1, r.end_line):
|
|
441
|
+
region_of[ln] = k
|
|
442
|
+
# A comment preamble above the import run (a coding cookie, a license header) stays above it: it
|
|
443
|
+
# decomposes on its own, so it never becomes the leading comment of the def the lift leaves next
|
|
444
|
+
# to it, and the import block's slot follows it (`import_slot`).
|
|
445
|
+
first = min(lifted) if lifted else None
|
|
446
|
+
cut = sum(1 for n in kept if first is not None and n < first)
|
|
447
|
+
if not any(lines[n].lstrip().startswith("#") for n in kept[:cut]):
|
|
448
|
+
cut = 0
|
|
449
|
+
elements: List[CodeElement] = []
|
|
450
|
+
origins: List[Optional[int]] = []
|
|
451
|
+
for part in ((kept[:cut], kept[cut:]) if cut else (kept,)):
|
|
452
|
+
part_text = "\n".join(lines[n] for n in part)
|
|
453
|
+
base = len(elements)
|
|
454
|
+
part_elements = decompose_elements(part_text)
|
|
455
|
+
top = iter(parse_regions(part_text))
|
|
456
|
+
for e in part_elements:
|
|
457
|
+
if e.parent is None:
|
|
458
|
+
r = next(top)
|
|
459
|
+
origins.append(region_of.get(part[r.start_line - 1]) if e.kind == "text" else None)
|
|
460
|
+
else:
|
|
461
|
+
e.parent += base
|
|
462
|
+
origins.append(None)
|
|
463
|
+
elements.append(e)
|
|
464
|
+
holding = sorted({region_of[ln] for ln in lifted})
|
|
465
|
+
keeping = {o for o in origins if o is not None}
|
|
466
|
+
emptied = [k for k in holding if k not in keeping]
|
|
467
|
+
claimed = set()
|
|
468
|
+
for e in elements:
|
|
469
|
+
names = element_names(e.text)
|
|
470
|
+
e.bindings = [b for name, bs in table.items() if name in names for b in bs]
|
|
471
|
+
claimed.update(name for name in table if name in names)
|
|
472
|
+
unclaimed = [b for name, bs in table.items() if name not in claimed for b in bs]
|
|
473
|
+
slot = import_slot(elements)
|
|
474
|
+
imports = CodeElement(IMPORTS, "", bindings=unclaimed)
|
|
475
|
+
shifted = [CodeElement(e.kind, e.text, e.depth,
|
|
476
|
+
None if e.parent is None else e.parent + (e.parent >= slot),
|
|
477
|
+
e.qualname, e.symbol_kind, e.bindings) for e in elements]
|
|
478
|
+
shifted.insert(slot, imports)
|
|
479
|
+
origins.insert(slot, None)
|
|
480
|
+
return LiftedModule(shifted, origins, emptied[0] if emptied else None, emptied)
|