cjm-python-decompose-core 0.0.6__tar.gz → 0.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/PKG-INFO +3 -74
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/README.md +1 -72
- cjm_python_decompose_core-0.0.7/cjm_python_decompose_core/__init__.py +1 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core/emit.py +21 -2
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core/extract.py +22 -5
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core.egg-info/PKG-INFO +3 -74
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core.egg-info/requires.txt +1 -1
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/pyproject.toml +1 -1
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_extract_ingest.py +33 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_imports_emit.py +31 -0
- cjm_python_decompose_core-0.0.6/cjm_python_decompose_core/__init__.py +0 -1
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/LICENSE +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core/ingest.py +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core/parse.py +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core.egg-info/SOURCES.txt +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core.egg-info/dependency_links.txt +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/cjm_python_decompose_core.egg-info/top_level.txt +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/setup.cfg +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_method_attribution.py +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_parse.py +0 -0
- {cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_regions_emit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-python-decompose-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.7
|
|
4
4
|
Summary: A Python source-code decomposition core for context graphs: parses .py modules (docstrings, imports, and the symbol tree of functions/classes/methods with their calls) into provenance-carrying graph nodes and edges that co-reside with the markdown source-type. First source = the ecosystem's own code; generalizes to any Python corpus.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
10
10
|
Requires-Python: >=3.12
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: cjm-dev-graph-schema>=0.0.
|
|
13
|
+
Requires-Dist: cjm-dev-graph-schema>=0.0.28
|
|
14
14
|
Requires-Dist: cjm-context-graph-layer>=0.0.7
|
|
15
15
|
Requires-Dist: cjm-context-graph-primitives>=0.0.12
|
|
16
16
|
Dynamic: license-file
|
|
@@ -28,12 +28,6 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
28
28
|
- **`cjm_python_decompose_core.extract`** — Bind parsed Python onto dev-graph-schema code nodes (the dev-domain binding).
|
|
29
29
|
- **`cjm_python_decompose_core.ingest`** — Flatten a decomposed Python corpus into graph elements (the queue-free half).
|
|
30
30
|
- **`cjm_python_decompose_core.parse`** — Schema-free Python parsing (stdlib `ast`).
|
|
31
|
-
- **`tests.test_extract_ingest`** — Extract (parse -> schema nodes) + ingest (corpus -> graph elements) binding.
|
|
32
|
-
- **`tests.test_imports_emit`** — Imports-as-projection: deriving a module's canonical import block from bindings.
|
|
33
|
-
- **`tests.test_method_attribution`** — First-param capture + monkey-patch-assignment detection (the cross-cell method idioms).
|
|
34
|
-
- **`tests.test_parse`** — Schema-free Python parsing: docstrings, imports, the symbol tree, and calls.
|
|
35
|
-
- **`tests.test_regions_emit`** — Verbatim region decomposition + canonical emit (the authoring / round-trip substrate).
|
|
36
|
-
- **`tests_manual.imports_projection`** — Imports-as-projection faithfulness sweep over the real arc libs (corpus harness).
|
|
37
31
|
|
|
38
32
|
## API
|
|
39
33
|
|
|
@@ -76,72 +70,7 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
76
70
|
- `parse_module` _function_ — Parse Python source into module docstring + imports + the symbol tree.
|
|
77
71
|
- `parse_regions` _function_ — Decompose a module into ordered verbatim top-level regions (the round-trip substrate).
|
|
78
72
|
|
|
79
|
-
### `tests.test_extract_ingest`
|
|
80
|
-
|
|
81
|
-
- `test_corpus_imports_and_calls_resolve_intra_corpus` _function_
|
|
82
|
-
- `test_decompose_text_builds_module_and_symbols` _function_
|
|
83
|
-
- `test_external_imports_not_minted` _function_
|
|
84
|
-
- `test_import_map_and_call_map` _function_
|
|
85
|
-
- `test_ingest_is_idempotent_in_ids` _function_
|
|
86
|
-
- `test_local_edges_about_repo_and_defines_nesting` _function_
|
|
87
|
-
- `test_module_path_and_import_name` _function_
|
|
88
|
-
- `test_resolve_import_relative_and_absolute` _function_
|
|
89
|
-
|
|
90
|
-
### `tests.test_imports_emit`
|
|
91
|
-
|
|
92
|
-
- `test_coexisting_submodule_imports_both_survive_round_trip` _function_ — import urllib.request + import urllib.error both bind `urllib` and both are
|
|
93
|
-
- `test_derived_block_is_faithful_and_prunes_dead_imports` _function_
|
|
94
|
-
- `test_docstring_lines_starting_with_from_survive_derive` _function_
|
|
95
|
-
- `test_emit_module_default_is_verbatim_not_derived` _function_
|
|
96
|
-
- `test_emit_module_derive_imports_prunes_dead_and_keeps_bodies` _function_
|
|
97
|
-
- `test_parameter_default_reference_is_not_pruned` _function_
|
|
98
|
-
- `test_rebinding_imports_still_supersede` _function_ — Aliased / from-imports rebinding a name follow Python last-binding-wins —
|
|
99
|
-
- `test_render_binding_forms` _function_
|
|
100
|
-
- `test_render_import_block_orders_dedups_and_groups` _function_
|
|
101
|
-
- `test_render_import_block_wraps_long_from_lines_in_aligned_parens` _function_
|
|
102
|
-
- `test_same_module_imported_twice_dedupes` _function_ — A literally duplicated plain import collapses to one statement.
|
|
103
|
-
|
|
104
|
-
### `tests.test_method_attribution`
|
|
105
|
-
|
|
106
|
-
- `test_first_param_and_annotation_captured` _function_
|
|
107
|
-
- `test_first_param_forward_ref_annotation` _function_
|
|
108
|
-
- `test_first_param_unannotated_and_classes_have_none` _function_
|
|
109
|
-
- `test_monkeypatch_assignments_detected` _function_
|
|
110
|
-
- `test_monkeypatch_is_structural_filtering_is_downstream` _function_
|
|
111
|
-
- `test_no_arg_function_has_empty_first_param` _function_
|
|
112
|
-
|
|
113
|
-
### `tests.test_parse`
|
|
114
|
-
|
|
115
|
-
- `parsed` _function_
|
|
116
|
-
- `test_class_methods_are_children_with_method_kind` _function_
|
|
117
|
-
- `test_empty_and_docstringless_module` _function_
|
|
118
|
-
- `test_function_calls_are_direct_only` _function_
|
|
119
|
-
- `test_import_bindings_per_symbol_and_module_level` _function_
|
|
120
|
-
- `test_imports_collected_with_relative_and_submodule_forms` _function_
|
|
121
|
-
- `test_iter_symbols_flattens_parents_before_children` _function_
|
|
122
|
-
- `test_local_import_not_hoisted_to_bindings` _function_
|
|
123
|
-
- `test_method_calls` _function_
|
|
124
|
-
- `test_module_docstring_first_line` _function_
|
|
125
|
-
- `test_nested_function_is_its_own_symbol_with_own_calls` _function_
|
|
126
|
-
- `test_refs_are_superset_of_calls_capturing_bases_and_annotations` _function_
|
|
127
|
-
- `test_syntax_error_propagates` _function_
|
|
128
|
-
- `test_top_level_symbols` _function_
|
|
129
|
-
|
|
130
|
-
### `tests.test_regions_emit`
|
|
131
|
-
|
|
132
|
-
- `test_codetext_regions_cover_the_non_def_source` _function_
|
|
133
|
-
- `test_decompose_attaches_body_and_order_to_top_level_symbols` _function_
|
|
134
|
-
- `test_decompose_emit_round_trip_from_nodes` _function_
|
|
135
|
-
- `test_emit_is_ast_equal_and_seam_canonical_on_messy_input` _function_
|
|
136
|
-
- `test_emit_regions_round_trips_pep8_byte_exact` _function_
|
|
137
|
-
- `test_parse_regions_orders_symbols_and_text` _function_
|
|
138
|
-
- `test_symbol_body_is_verbatim_with_decorators_and_leading_comments` _function_
|
|
139
|
-
|
|
140
|
-
### `tests_manual.imports_projection`
|
|
141
|
-
|
|
142
|
-
- `sweep` _function_ — Decompose every arc-lib module and tally false prunes + genuine dead-import prunes.
|
|
143
|
-
|
|
144
73
|
## Dependencies
|
|
145
74
|
|
|
146
|
-
**Depends on:** `cjm-dev-graph-schema`
|
|
75
|
+
**Depends on:** `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-dev-graph-schema`
|
|
147
76
|
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`
|
|
@@ -11,12 +11,6 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
11
11
|
- **`cjm_python_decompose_core.extract`** — Bind parsed Python onto dev-graph-schema code nodes (the dev-domain binding).
|
|
12
12
|
- **`cjm_python_decompose_core.ingest`** — Flatten a decomposed Python corpus into graph elements (the queue-free half).
|
|
13
13
|
- **`cjm_python_decompose_core.parse`** — Schema-free Python parsing (stdlib `ast`).
|
|
14
|
-
- **`tests.test_extract_ingest`** — Extract (parse -> schema nodes) + ingest (corpus -> graph elements) binding.
|
|
15
|
-
- **`tests.test_imports_emit`** — Imports-as-projection: deriving a module's canonical import block from bindings.
|
|
16
|
-
- **`tests.test_method_attribution`** — First-param capture + monkey-patch-assignment detection (the cross-cell method idioms).
|
|
17
|
-
- **`tests.test_parse`** — Schema-free Python parsing: docstrings, imports, the symbol tree, and calls.
|
|
18
|
-
- **`tests.test_regions_emit`** — Verbatim region decomposition + canonical emit (the authoring / round-trip substrate).
|
|
19
|
-
- **`tests_manual.imports_projection`** — Imports-as-projection faithfulness sweep over the real arc libs (corpus harness).
|
|
20
14
|
|
|
21
15
|
## API
|
|
22
16
|
|
|
@@ -59,72 +53,7 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
59
53
|
- `parse_module` _function_ — Parse Python source into module docstring + imports + the symbol tree.
|
|
60
54
|
- `parse_regions` _function_ — Decompose a module into ordered verbatim top-level regions (the round-trip substrate).
|
|
61
55
|
|
|
62
|
-
### `tests.test_extract_ingest`
|
|
63
|
-
|
|
64
|
-
- `test_corpus_imports_and_calls_resolve_intra_corpus` _function_
|
|
65
|
-
- `test_decompose_text_builds_module_and_symbols` _function_
|
|
66
|
-
- `test_external_imports_not_minted` _function_
|
|
67
|
-
- `test_import_map_and_call_map` _function_
|
|
68
|
-
- `test_ingest_is_idempotent_in_ids` _function_
|
|
69
|
-
- `test_local_edges_about_repo_and_defines_nesting` _function_
|
|
70
|
-
- `test_module_path_and_import_name` _function_
|
|
71
|
-
- `test_resolve_import_relative_and_absolute` _function_
|
|
72
|
-
|
|
73
|
-
### `tests.test_imports_emit`
|
|
74
|
-
|
|
75
|
-
- `test_coexisting_submodule_imports_both_survive_round_trip` _function_ — import urllib.request + import urllib.error both bind `urllib` and both are
|
|
76
|
-
- `test_derived_block_is_faithful_and_prunes_dead_imports` _function_
|
|
77
|
-
- `test_docstring_lines_starting_with_from_survive_derive` _function_
|
|
78
|
-
- `test_emit_module_default_is_verbatim_not_derived` _function_
|
|
79
|
-
- `test_emit_module_derive_imports_prunes_dead_and_keeps_bodies` _function_
|
|
80
|
-
- `test_parameter_default_reference_is_not_pruned` _function_
|
|
81
|
-
- `test_rebinding_imports_still_supersede` _function_ — Aliased / from-imports rebinding a name follow Python last-binding-wins —
|
|
82
|
-
- `test_render_binding_forms` _function_
|
|
83
|
-
- `test_render_import_block_orders_dedups_and_groups` _function_
|
|
84
|
-
- `test_render_import_block_wraps_long_from_lines_in_aligned_parens` _function_
|
|
85
|
-
- `test_same_module_imported_twice_dedupes` _function_ — A literally duplicated plain import collapses to one statement.
|
|
86
|
-
|
|
87
|
-
### `tests.test_method_attribution`
|
|
88
|
-
|
|
89
|
-
- `test_first_param_and_annotation_captured` _function_
|
|
90
|
-
- `test_first_param_forward_ref_annotation` _function_
|
|
91
|
-
- `test_first_param_unannotated_and_classes_have_none` _function_
|
|
92
|
-
- `test_monkeypatch_assignments_detected` _function_
|
|
93
|
-
- `test_monkeypatch_is_structural_filtering_is_downstream` _function_
|
|
94
|
-
- `test_no_arg_function_has_empty_first_param` _function_
|
|
95
|
-
|
|
96
|
-
### `tests.test_parse`
|
|
97
|
-
|
|
98
|
-
- `parsed` _function_
|
|
99
|
-
- `test_class_methods_are_children_with_method_kind` _function_
|
|
100
|
-
- `test_empty_and_docstringless_module` _function_
|
|
101
|
-
- `test_function_calls_are_direct_only` _function_
|
|
102
|
-
- `test_import_bindings_per_symbol_and_module_level` _function_
|
|
103
|
-
- `test_imports_collected_with_relative_and_submodule_forms` _function_
|
|
104
|
-
- `test_iter_symbols_flattens_parents_before_children` _function_
|
|
105
|
-
- `test_local_import_not_hoisted_to_bindings` _function_
|
|
106
|
-
- `test_method_calls` _function_
|
|
107
|
-
- `test_module_docstring_first_line` _function_
|
|
108
|
-
- `test_nested_function_is_its_own_symbol_with_own_calls` _function_
|
|
109
|
-
- `test_refs_are_superset_of_calls_capturing_bases_and_annotations` _function_
|
|
110
|
-
- `test_syntax_error_propagates` _function_
|
|
111
|
-
- `test_top_level_symbols` _function_
|
|
112
|
-
|
|
113
|
-
### `tests.test_regions_emit`
|
|
114
|
-
|
|
115
|
-
- `test_codetext_regions_cover_the_non_def_source` _function_
|
|
116
|
-
- `test_decompose_attaches_body_and_order_to_top_level_symbols` _function_
|
|
117
|
-
- `test_decompose_emit_round_trip_from_nodes` _function_
|
|
118
|
-
- `test_emit_is_ast_equal_and_seam_canonical_on_messy_input` _function_
|
|
119
|
-
- `test_emit_regions_round_trips_pep8_byte_exact` _function_
|
|
120
|
-
- `test_parse_regions_orders_symbols_and_text` _function_
|
|
121
|
-
- `test_symbol_body_is_verbatim_with_decorators_and_leading_comments` _function_
|
|
122
|
-
|
|
123
|
-
### `tests_manual.imports_projection`
|
|
124
|
-
|
|
125
|
-
- `sweep` _function_ — Decompose every arc-lib module and tally false prunes + genuine dead-import prunes.
|
|
126
|
-
|
|
127
56
|
## Dependencies
|
|
128
57
|
|
|
129
|
-
**Depends on:** `cjm-dev-graph-schema`
|
|
58
|
+
**Depends on:** `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-dev-graph-schema`
|
|
130
59
|
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.7"
|
|
@@ -241,8 +241,27 @@ def _label_of(node: Any) -> str:
|
|
|
241
241
|
|
|
242
242
|
|
|
243
243
|
def _looks_like_docstring(text: str) -> bool:
|
|
244
|
-
"""Whether a text region
|
|
245
|
-
|
|
244
|
+
"""Whether a text region is the module PREAMBLE the derived import block must follow.
|
|
245
|
+
|
|
246
|
+
True when the region opens with a string literal (the module docstring), when comment
|
|
247
|
+
lines — a shebang, an encoding cookie, a license header — lead into one, or when the
|
|
248
|
+
region is comments only. AST-located like `_strip_import_lines`: the first STATEMENT
|
|
249
|
+
decides, so `#!/usr/bin/env python3` above a docstring no longer reads as "not a
|
|
250
|
+
docstring" (the cascade_manifests.py capture landed the import block at line 1, above
|
|
251
|
+
the shebang AND the docstring — the shebang stopped working and the docstring became
|
|
252
|
+
a bare expression; housekeeping sitting 2026-09-23). A region `ast.parse` rejects
|
|
253
|
+
falls back to the same test with comment lines removed."""
|
|
254
|
+
try:
|
|
255
|
+
tree = ast.parse(text)
|
|
256
|
+
except SyntaxError:
|
|
257
|
+
stripped = "\n".join(ln for ln in text.splitlines()
|
|
258
|
+
if not ln.lstrip().startswith("#")).lstrip()
|
|
259
|
+
return (not stripped) or stripped.startswith(('"""', "'''", '"', "'"))
|
|
260
|
+
if not tree.body:
|
|
261
|
+
return bool(text.strip()) # comments only (a shebang / header with no statements)
|
|
262
|
+
first = tree.body[0]
|
|
263
|
+
return (isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant)
|
|
264
|
+
and isinstance(first.value.value, str))
|
|
246
265
|
|
|
247
266
|
|
|
248
267
|
_IMPORT_LINE = re.compile(r"\s*(from|import)\s")
|
|
@@ -77,21 +77,31 @@ def _build_symbols(
|
|
|
77
77
|
parsed: ParsedModule, # Its parse
|
|
78
78
|
content_hash: str, # The module file's content hash (symbols share their file's source)
|
|
79
79
|
path: str, # The file path (provenance locator)
|
|
80
|
+
symbol_identity: Any = None, # Optional identity hook: qualname -> birth fields ({birth_repo_key, birth_module_path, birth_qualname, generation}) or None = born at this address
|
|
80
81
|
) -> Tuple[List[CodeSymbolNode], List[Dict[str, Any]]]: # (flat symbol nodes, DEFINES edges)
|
|
81
82
|
"""Flatten the parse tree into symbol nodes + structural DEFINES edges.
|
|
82
83
|
|
|
83
84
|
DEFINES runs module->top-level symbol and parent-symbol->nested-symbol, so the
|
|
84
|
-
nesting (class -> its methods) is preserved as edges, not just qualname text.
|
|
85
|
+
nesting (class -> its methods) is preserved as edges, not just qualname text.
|
|
86
|
+
|
|
87
|
+
`symbol_identity` is the container-independent identity hook (36f649d3): asked per
|
|
88
|
+
qualname, it names the BIRTH address of a symbol that was renamed or re-homed since
|
|
89
|
+
it was born (a keep-identity op in the source journal), so the node keeps the id it
|
|
90
|
+
was born with and every journaled edge onto it survives the move; None means the
|
|
91
|
+
symbol is born at this address (the plain derivation). It runs BEFORE the DEFINES
|
|
92
|
+
edges are built, because those edges name the ids."""
|
|
85
93
|
symbols: List[CodeSymbolNode] = []
|
|
86
94
|
edges: List[Dict[str, Any]] = []
|
|
87
95
|
|
|
88
96
|
def make(ps) -> CodeSymbolNode:
|
|
97
|
+
birth = (symbol_identity(ps.qualname) or {}) if symbol_identity else {}
|
|
89
98
|
node = CodeSymbolNode(
|
|
90
99
|
module_id=module.id, qualname=ps.qualname, symbol_kind=ps.kind,
|
|
91
100
|
path=path, content_hash=content_hash, lineno=ps.lineno,
|
|
92
101
|
docstring=ps.docstring, calls=list(ps.calls), refs=list(ps.refs),
|
|
93
102
|
import_bindings=list(ps.import_bindings),
|
|
94
103
|
properties={"decorators": list(ps.decorators)} if ps.decorators else {},
|
|
104
|
+
**birth,
|
|
95
105
|
)
|
|
96
106
|
symbols.append(node)
|
|
97
107
|
children = [make(c) for c in ps.children]
|
|
@@ -111,6 +121,7 @@ def decompose_text(
|
|
|
111
121
|
text: str, # The module source text
|
|
112
122
|
content_hash: Optional[str] = None, # Precomputed file hash (else hashed over the UTF-8 text)
|
|
113
123
|
import_name: Optional[str] = None, # Override the derived dotted import name
|
|
124
|
+
symbol_identity: Any = None, # Optional identity hook (qualname -> birth fields; see `_build_symbols`) — container-independent ids (36f649d3)
|
|
114
125
|
) -> DecomposedModule: # The decomposed module
|
|
115
126
|
"""Parse + bind in one step from in-memory source text."""
|
|
116
127
|
ch = content_hash if content_hash is not None else SourceRef.compute_hash(text.encode("utf-8"))
|
|
@@ -121,7 +132,7 @@ def decompose_text(
|
|
|
121
132
|
docstring=parsed.docstring, imports=list(parsed.imports),
|
|
122
133
|
import_bindings=list(parsed.module_used_bindings),
|
|
123
134
|
)
|
|
124
|
-
symbols, defines = _build_symbols(module, parsed, ch, path)
|
|
135
|
+
symbols, defines = _build_symbols(module, parsed, ch, path, symbol_identity=symbol_identity)
|
|
125
136
|
|
|
126
137
|
# Verbatim-region overlay (the authoring / round-trip substrate): attach each
|
|
127
138
|
# top-level symbol's VERBATIM body + order, mint CodeText nodes for the non-def
|
|
@@ -155,12 +166,14 @@ def decompose_file(
|
|
|
155
166
|
repo_key: str, # The repo's durable conceptual slug
|
|
156
167
|
path: str, # Path to the .py file
|
|
157
168
|
repo_root: str, # Repo root (for the repo-relative module path)
|
|
169
|
+
symbol_identity_for: Any = None, # Optional factory: module_path -> the module's identity hook (see `decompose_text`)
|
|
158
170
|
) -> DecomposedModule: # The decomposed module
|
|
159
171
|
"""Read a `.py` file and decompose it (hash over the raw file bytes)."""
|
|
160
172
|
raw = Path(path).read_bytes()
|
|
161
173
|
mp = module_path_for(path, repo_root)
|
|
162
174
|
return decompose_text(repo_key, mp, str(path), raw.decode("utf-8"),
|
|
163
|
-
content_hash=SourceRef.compute_hash(raw)
|
|
175
|
+
content_hash=SourceRef.compute_hash(raw),
|
|
176
|
+
symbol_identity=symbol_identity_for(mp) if symbol_identity_for else None)
|
|
164
177
|
|
|
165
178
|
|
|
166
179
|
def iter_py_files(
|
|
@@ -178,12 +191,14 @@ def decompose_paths(
|
|
|
178
191
|
repo_key: str, # The repo's durable conceptual slug
|
|
179
192
|
paths: Iterable[str], # The .py files to decompose
|
|
180
193
|
repo_root: str, # Repo root (for repo-relative module paths)
|
|
194
|
+
symbol_identity_for: Any = None, # Optional factory: module_path -> the module's identity hook (see `decompose_text`)
|
|
181
195
|
) -> List[DecomposedModule]: # One DecomposedModule per parseable file
|
|
182
196
|
"""Decompose an explicit set of files; unparseable files are skipped (recorded by the caller)."""
|
|
183
197
|
out: List[DecomposedModule] = []
|
|
184
198
|
for path in paths:
|
|
185
199
|
try:
|
|
186
|
-
out.append(decompose_file(repo_key, path, repo_root
|
|
200
|
+
out.append(decompose_file(repo_key, path, repo_root,
|
|
201
|
+
symbol_identity_for=symbol_identity_for))
|
|
187
202
|
except SyntaxError:
|
|
188
203
|
continue
|
|
189
204
|
return out
|
|
@@ -193,10 +208,12 @@ def decompose_package(
|
|
|
193
208
|
repo_key: str, # The repo's durable conceptual slug
|
|
194
209
|
package_dir: str, # The importable package directory (e.g. ".../cjm_dev_graph_schema")
|
|
195
210
|
repo_root: Optional[str] = None, # Repo root for relative module paths (default = package_dir's parent)
|
|
211
|
+
symbol_identity_for: Any = None, # Optional factory: module_path -> the module's identity hook (see `decompose_text`)
|
|
196
212
|
) -> List[DecomposedModule]: # The decomposed package modules
|
|
197
213
|
"""Decompose every `.py` under a package dir (the lib's own importable source).
|
|
198
214
|
|
|
199
215
|
`repo_root` defaults to the package dir's parent, so module paths read like
|
|
200
216
|
`cjm_dev_graph_schema/nodes.py` (the importable form)."""
|
|
201
217
|
root = repo_root if repo_root is not None else str(Path(package_dir).parent)
|
|
202
|
-
return decompose_paths(repo_key, iter_py_files(package_dir), root
|
|
218
|
+
return decompose_paths(repo_key, iter_py_files(package_dir), root,
|
|
219
|
+
symbol_identity_for=symbol_identity_for)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cjm-python-decompose-core
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.7
|
|
4
4
|
Summary: A Python source-code decomposition core for context graphs: parses .py modules (docstrings, imports, and the symbol tree of functions/classes/methods with their calls) into provenance-carrying graph nodes and edges that co-reside with the markdown source-type. First source = the ecosystem's own code; generalizes to any Python corpus.
|
|
5
5
|
Author-email: "Christian J. Mills" <9126128+cj-mills@users.noreply.github.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
10
10
|
Requires-Python: >=3.12
|
|
11
11
|
Description-Content-Type: text/markdown
|
|
12
12
|
License-File: LICENSE
|
|
13
|
-
Requires-Dist: cjm-dev-graph-schema>=0.0.
|
|
13
|
+
Requires-Dist: cjm-dev-graph-schema>=0.0.28
|
|
14
14
|
Requires-Dist: cjm-context-graph-layer>=0.0.7
|
|
15
15
|
Requires-Dist: cjm-context-graph-primitives>=0.0.12
|
|
16
16
|
Dynamic: license-file
|
|
@@ -28,12 +28,6 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
28
28
|
- **`cjm_python_decompose_core.extract`** — Bind parsed Python onto dev-graph-schema code nodes (the dev-domain binding).
|
|
29
29
|
- **`cjm_python_decompose_core.ingest`** — Flatten a decomposed Python corpus into graph elements (the queue-free half).
|
|
30
30
|
- **`cjm_python_decompose_core.parse`** — Schema-free Python parsing (stdlib `ast`).
|
|
31
|
-
- **`tests.test_extract_ingest`** — Extract (parse -> schema nodes) + ingest (corpus -> graph elements) binding.
|
|
32
|
-
- **`tests.test_imports_emit`** — Imports-as-projection: deriving a module's canonical import block from bindings.
|
|
33
|
-
- **`tests.test_method_attribution`** — First-param capture + monkey-patch-assignment detection (the cross-cell method idioms).
|
|
34
|
-
- **`tests.test_parse`** — Schema-free Python parsing: docstrings, imports, the symbol tree, and calls.
|
|
35
|
-
- **`tests.test_regions_emit`** — Verbatim region decomposition + canonical emit (the authoring / round-trip substrate).
|
|
36
|
-
- **`tests_manual.imports_projection`** — Imports-as-projection faithfulness sweep over the real arc libs (corpus harness).
|
|
37
31
|
|
|
38
32
|
## API
|
|
39
33
|
|
|
@@ -76,72 +70,7 @@ A Python source-code decomposition core for context graphs: parses .py modules (
|
|
|
76
70
|
- `parse_module` _function_ — Parse Python source into module docstring + imports + the symbol tree.
|
|
77
71
|
- `parse_regions` _function_ — Decompose a module into ordered verbatim top-level regions (the round-trip substrate).
|
|
78
72
|
|
|
79
|
-
### `tests.test_extract_ingest`
|
|
80
|
-
|
|
81
|
-
- `test_corpus_imports_and_calls_resolve_intra_corpus` _function_
|
|
82
|
-
- `test_decompose_text_builds_module_and_symbols` _function_
|
|
83
|
-
- `test_external_imports_not_minted` _function_
|
|
84
|
-
- `test_import_map_and_call_map` _function_
|
|
85
|
-
- `test_ingest_is_idempotent_in_ids` _function_
|
|
86
|
-
- `test_local_edges_about_repo_and_defines_nesting` _function_
|
|
87
|
-
- `test_module_path_and_import_name` _function_
|
|
88
|
-
- `test_resolve_import_relative_and_absolute` _function_
|
|
89
|
-
|
|
90
|
-
### `tests.test_imports_emit`
|
|
91
|
-
|
|
92
|
-
- `test_coexisting_submodule_imports_both_survive_round_trip` _function_ — import urllib.request + import urllib.error both bind `urllib` and both are
|
|
93
|
-
- `test_derived_block_is_faithful_and_prunes_dead_imports` _function_
|
|
94
|
-
- `test_docstring_lines_starting_with_from_survive_derive` _function_
|
|
95
|
-
- `test_emit_module_default_is_verbatim_not_derived` _function_
|
|
96
|
-
- `test_emit_module_derive_imports_prunes_dead_and_keeps_bodies` _function_
|
|
97
|
-
- `test_parameter_default_reference_is_not_pruned` _function_
|
|
98
|
-
- `test_rebinding_imports_still_supersede` _function_ — Aliased / from-imports rebinding a name follow Python last-binding-wins —
|
|
99
|
-
- `test_render_binding_forms` _function_
|
|
100
|
-
- `test_render_import_block_orders_dedups_and_groups` _function_
|
|
101
|
-
- `test_render_import_block_wraps_long_from_lines_in_aligned_parens` _function_
|
|
102
|
-
- `test_same_module_imported_twice_dedupes` _function_ — A literally duplicated plain import collapses to one statement.
|
|
103
|
-
|
|
104
|
-
### `tests.test_method_attribution`
|
|
105
|
-
|
|
106
|
-
- `test_first_param_and_annotation_captured` _function_
|
|
107
|
-
- `test_first_param_forward_ref_annotation` _function_
|
|
108
|
-
- `test_first_param_unannotated_and_classes_have_none` _function_
|
|
109
|
-
- `test_monkeypatch_assignments_detected` _function_
|
|
110
|
-
- `test_monkeypatch_is_structural_filtering_is_downstream` _function_
|
|
111
|
-
- `test_no_arg_function_has_empty_first_param` _function_
|
|
112
|
-
|
|
113
|
-
### `tests.test_parse`
|
|
114
|
-
|
|
115
|
-
- `parsed` _function_
|
|
116
|
-
- `test_class_methods_are_children_with_method_kind` _function_
|
|
117
|
-
- `test_empty_and_docstringless_module` _function_
|
|
118
|
-
- `test_function_calls_are_direct_only` _function_
|
|
119
|
-
- `test_import_bindings_per_symbol_and_module_level` _function_
|
|
120
|
-
- `test_imports_collected_with_relative_and_submodule_forms` _function_
|
|
121
|
-
- `test_iter_symbols_flattens_parents_before_children` _function_
|
|
122
|
-
- `test_local_import_not_hoisted_to_bindings` _function_
|
|
123
|
-
- `test_method_calls` _function_
|
|
124
|
-
- `test_module_docstring_first_line` _function_
|
|
125
|
-
- `test_nested_function_is_its_own_symbol_with_own_calls` _function_
|
|
126
|
-
- `test_refs_are_superset_of_calls_capturing_bases_and_annotations` _function_
|
|
127
|
-
- `test_syntax_error_propagates` _function_
|
|
128
|
-
- `test_top_level_symbols` _function_
|
|
129
|
-
|
|
130
|
-
### `tests.test_regions_emit`
|
|
131
|
-
|
|
132
|
-
- `test_codetext_regions_cover_the_non_def_source` _function_
|
|
133
|
-
- `test_decompose_attaches_body_and_order_to_top_level_symbols` _function_
|
|
134
|
-
- `test_decompose_emit_round_trip_from_nodes` _function_
|
|
135
|
-
- `test_emit_is_ast_equal_and_seam_canonical_on_messy_input` _function_
|
|
136
|
-
- `test_emit_regions_round_trips_pep8_byte_exact` _function_
|
|
137
|
-
- `test_parse_regions_orders_symbols_and_text` _function_
|
|
138
|
-
- `test_symbol_body_is_verbatim_with_decorators_and_leading_comments` _function_
|
|
139
|
-
|
|
140
|
-
### `tests_manual.imports_projection`
|
|
141
|
-
|
|
142
|
-
- `sweep` _function_ — Decompose every arc-lib module and tally false prunes + genuine dead-import prunes.
|
|
143
|
-
|
|
144
73
|
## Dependencies
|
|
145
74
|
|
|
146
|
-
**Depends on:** `cjm-dev-graph-schema`
|
|
75
|
+
**Depends on:** `cjm-context-graph-layer`, `cjm-context-graph-primitives`, `cjm-dev-graph-schema`
|
|
147
76
|
**Used by:** `cjm-context-graph-projection`, `cjm-notebook-decompose-core`
|
|
@@ -14,7 +14,7 @@ classifiers = [
|
|
|
14
14
|
"Programming Language :: Python :: 3",
|
|
15
15
|
"Programming Language :: Python :: 3 :: Only",
|
|
16
16
|
]
|
|
17
|
-
dependencies = ['cjm-dev-graph-schema>=0.0.
|
|
17
|
+
dependencies = ['cjm-dev-graph-schema>=0.0.28', 'cjm-context-graph-layer>=0.0.7', 'cjm-context-graph-primitives>=0.0.12']
|
|
18
18
|
|
|
19
19
|
[project.urls]
|
|
20
20
|
Repository = "https://github.com/cj-mills/cjm-python-decompose-core"
|
{cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_extract_ingest.py
RENAMED
|
@@ -134,3 +134,36 @@ def test_external_imports_not_minted():
|
|
|
134
134
|
d = decompose_text("demo-repo", "demo/c.py", "/abs/c.py", src, import_name="demo.c")
|
|
135
135
|
_, edges = corpus_graph_elements([d])
|
|
136
136
|
assert [e for e in edges if e["relation_type"] == DevRelations.IMPORTS] == []
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_symbol_identity_hook_keeps_the_birth_id_and_covers_nested_symbols():
|
|
140
|
+
# The container-independent identity hook (36f649d3): a class re-homed from a.py keeps
|
|
141
|
+
# a.py's id in b.py, its methods derive from the same birth address, and a symbol the
|
|
142
|
+
# hook does not name is simply born here.
|
|
143
|
+
born = decompose_text("cjm-demo", "demo/a.py", "/tmp/a.py",
|
|
144
|
+
"class Thing:\n def m(self):\n return 1\n")
|
|
145
|
+
born_ids = {s.qualname: s.id for s in born.symbols}
|
|
146
|
+
|
|
147
|
+
def hook(qualname):
|
|
148
|
+
if qualname.split(".")[0] == "Thing":
|
|
149
|
+
return {"birth_repo_key": "cjm-demo", "birth_module_path": "demo/a.py",
|
|
150
|
+
"birth_qualname": "Thing" + qualname[len("Thing"):], "generation": 0}
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
moved = decompose_text("cjm-demo", "demo/b.py", "/tmp/b.py",
|
|
154
|
+
"class Thing:\n def m(self):\n return 1\n\n\n"
|
|
155
|
+
"def fresh():\n return 2\n",
|
|
156
|
+
symbol_identity=hook)
|
|
157
|
+
ids = {s.qualname: s.id for s in moved.symbols}
|
|
158
|
+
assert ids["Thing"] == born_ids["Thing"] and ids["Thing.m"] == born_ids["Thing.m"]
|
|
159
|
+
assert ids["fresh"] not in born_ids.values()
|
|
160
|
+
assert moved.module.id != born.module.id
|
|
161
|
+
# The DEFINES edges name the kept ids (the hook ran BEFORE the edges were built).
|
|
162
|
+
defines = {(e["source_id"], e["target_id"]) for e in moved.local_edges
|
|
163
|
+
if e["relation_type"] == DevRelations.DEFINES}
|
|
164
|
+
assert (moved.module.id, ids["Thing"]) in defines and (ids["Thing"], ids["Thing.m"]) in defines
|
|
165
|
+
# The birth fields ride the wire; a symbol born here carries none.
|
|
166
|
+
thing = next(s.to_graph_node() for s in moved.symbols if s.qualname == "Thing")
|
|
167
|
+
assert thing["properties"]["birth_module_path"] == "demo/a.py"
|
|
168
|
+
fresh = next(s.to_graph_node() for s in moved.symbols if s.qualname == "fresh")
|
|
169
|
+
assert "birth_module_path" not in fresh["properties"]
|
{cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_imports_emit.py
RENAMED
|
@@ -119,6 +119,37 @@ def test_docstring_lines_starting_with_from_survive_derive():
|
|
|
119
119
|
compile(emitted, "m.py", "exec")
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
def test_derived_import_block_lands_below_a_shebang_and_docstring():
|
|
123
|
+
# A shebang (or any comment header) ABOVE the module docstring is still the preamble:
|
|
124
|
+
# the derived import block must land after it, never at line 1 (regression: the
|
|
125
|
+
# cascade_manifests.py capture put the imports above `#!/usr/bin/env python3` and the
|
|
126
|
+
# docstring — the shebang died and the docstring became a bare expression).
|
|
127
|
+
src = ("#!/usr/bin/env python3\n"
|
|
128
|
+
'"""A script with a shebang.\n'
|
|
129
|
+
'"""\n'
|
|
130
|
+
"\n"
|
|
131
|
+
"import os\n"
|
|
132
|
+
"\n\n"
|
|
133
|
+
"def f():\n"
|
|
134
|
+
" return os.getpid()\n")
|
|
135
|
+
d = decompose_text("pkg", "pkg/m.py", "/tmp/m.py", src)
|
|
136
|
+
nodes = [s.to_graph_node() for s in d.symbols] + [t.to_graph_node() for t in d.texts]
|
|
137
|
+
emitted = emit_module_from_nodes(nodes, module_node=d.module.to_graph_node(), derive_imports=True)
|
|
138
|
+
lines = emitted.splitlines()
|
|
139
|
+
assert lines[0] == "#!/usr/bin/env python3"
|
|
140
|
+
assert lines[1] == '"""A script with a shebang.'
|
|
141
|
+
assert lines.index("import os") > lines.index('"""', 2)
|
|
142
|
+
compile(emitted, "m.py", "exec")
|
|
143
|
+
# A comments-only header (no docstring) is a preamble too.
|
|
144
|
+
src2 = "# -*- coding: utf-8 -*-\n# license header\n\nimport os\n\n\ndef g():\n return os.sep\n"
|
|
145
|
+
d2 = decompose_text("pkg", "pkg/n.py", "/tmp/n.py", src2)
|
|
146
|
+
nodes2 = [s.to_graph_node() for s in d2.symbols] + [t.to_graph_node() for t in d2.texts]
|
|
147
|
+
emitted2 = emit_module_from_nodes(nodes2, module_node=d2.module.to_graph_node(), derive_imports=True)
|
|
148
|
+
assert emitted2.splitlines()[0] == "# -*- coding: utf-8 -*-"
|
|
149
|
+
assert "import os" in emitted2
|
|
150
|
+
compile(emitted2, "n.py", "exec")
|
|
151
|
+
|
|
152
|
+
|
|
122
153
|
def test_coexisting_submodule_imports_both_survive_round_trip():
|
|
123
154
|
"""import urllib.request + import urllib.error both bind `urllib` and both are
|
|
124
155
|
live — the emit must carry BOTH (regression: the one-descriptor-per-name table
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.6"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_method_attribution.py
RENAMED
|
File without changes
|
|
File without changes
|
{cjm_python_decompose_core-0.0.6 → cjm_python_decompose_core-0.0.7}/tests/test_regions_emit.py
RENAMED
|
File without changes
|