split-python4gpt 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- split_python4gpt/__init__.py +10 -12
- split_python4gpt/__main__.py +17 -6
- split_python4gpt/_version.py +24 -0
- split_python4gpt/minifier.py +410 -46
- split_python4gpt-1.0.5.dist-info/METADATA +330 -0
- split_python4gpt-1.0.5.dist-info/RECORD +10 -0
- {split_python4gpt-1.0.3.dist-info → split_python4gpt-1.0.5.dist-info}/WHEEL +1 -2
- split_python4gpt-1.0.5.dist-info/licenses/AUTHORS.md +4 -0
- split_python4gpt-1.0.3.dist-info/METADATA +0 -152
- split_python4gpt-1.0.3.dist-info/RECORD +0 -9
- split_python4gpt-1.0.3.dist-info/top_level.txt +0 -1
- {split_python4gpt-1.0.3.dist-info → split_python4gpt-1.0.5.dist-info}/entry_points.txt +0 -0
- {split_python4gpt-1.0.3.dist-info → split_python4gpt-1.0.5.dist-info/licenses}/LICENSE.txt +0 -0
split_python4gpt/__init__.py
CHANGED
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
|
|
1
|
+
# this_file: src/split_python4gpt/__init__.py
|
|
2
|
+
"""split-python4gpt — minify and split Python projects for LLM consumption."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
7
|
+
|
|
2
8
|
from .minifier import PyTypingMinifier
|
|
3
9
|
|
|
4
|
-
|
|
5
|
-
# TODO: Import directly (no need for conditional) when `python_requires = >= 3.8`
|
|
6
|
-
from importlib.metadata import PackageNotFoundError, version # pragma: no cover
|
|
7
|
-
else:
|
|
8
|
-
from importlib_metadata import PackageNotFoundError, version # pragma: no cover
|
|
10
|
+
__all__ = ["PyTypingMinifier", "__version__"]
|
|
9
11
|
|
|
10
12
|
try:
|
|
11
|
-
|
|
12
|
-
dist_name = __name__
|
|
13
|
-
__version__ = version(dist_name)
|
|
13
|
+
__version__: str = version("split-python4gpt")
|
|
14
14
|
except PackageNotFoundError: # pragma: no cover
|
|
15
|
-
__version__ = "
|
|
16
|
-
finally:
|
|
17
|
-
del version, PackageNotFoundError
|
|
15
|
+
__version__ = "0.0.0"
|
split_python4gpt/__main__.py
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
# this_file: src/split_python4gpt/__main__.py
|
|
3
|
+
"""CLI entry point for split-python4gpt (``mdsplit4gpt`` command)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
2
7
|
from pathlib import Path
|
|
3
8
|
|
|
4
9
|
import fire
|
|
5
|
-
|
|
10
|
+
|
|
11
|
+
from .minifier import PyLLMSplitter
|
|
6
12
|
|
|
7
13
|
|
|
8
14
|
def split_python4gpt(
|
|
@@ -14,7 +20,7 @@ def split_python4gpt(
|
|
|
14
20
|
mini_docs: bool = True,
|
|
15
21
|
mini_globs: bool = False,
|
|
16
22
|
mini_locs: bool = False,
|
|
17
|
-
mini_lits: bool =
|
|
23
|
+
mini_lits: bool = False,
|
|
18
24
|
mini_annotations: bool = True,
|
|
19
25
|
mini_asserts: bool = True,
|
|
20
26
|
mini_debug: bool = True,
|
|
@@ -51,7 +57,8 @@ def split_python4gpt(
|
|
|
51
57
|
Returns:
|
|
52
58
|
list[Path]: List of output Python files.
|
|
53
59
|
"""
|
|
54
|
-
|
|
60
|
+
splitter = PyLLMSplitter()
|
|
61
|
+
splitter.process_py(
|
|
55
62
|
py_path_or_folder=path_or_folder,
|
|
56
63
|
out_py_folder=out,
|
|
57
64
|
pyi_folder=pyis,
|
|
@@ -71,11 +78,15 @@ def split_python4gpt(
|
|
|
71
78
|
rename_globals=mini_globs,
|
|
72
79
|
rename_locals=mini_locs,
|
|
73
80
|
)
|
|
81
|
+
splitter.write_splits()
|
|
82
|
+
|
|
74
83
|
|
|
84
|
+
def cli() -> None:
|
|
85
|
+
"""Run the CLI using python-fire."""
|
|
86
|
+
import sys
|
|
75
87
|
|
|
76
|
-
|
|
77
|
-
fire.
|
|
78
|
-
fire.Fire(split_python4gpt)
|
|
88
|
+
fire.core.Display = lambda lines, out: print(*lines, file=sys.stdout)
|
|
89
|
+
fire.Fire(split_python4gpt, name="mdsplit4gpt")
|
|
79
90
|
|
|
80
91
|
|
|
81
92
|
if __name__ == "__main__":
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '1.0.5'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 5)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
split_python4gpt/minifier.py
CHANGED
|
@@ -1,38 +1,80 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# this_file: src/split_python4gpt/minifier.py
|
|
3
|
+
"""Core minification and type-inference logic for split-python4gpt."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
2
6
|
|
|
3
7
|
import contextlib
|
|
8
|
+
import logging
|
|
4
9
|
import shutil
|
|
5
10
|
import subprocess
|
|
11
|
+
from ast import ClassDef, FunctionDef, NodeTransformer, fix_missing_locations, parse
|
|
12
|
+
from ast import unparse as ast_unparse
|
|
13
|
+
from os import environ
|
|
6
14
|
from pathlib import Path
|
|
7
15
|
|
|
8
16
|
from python_minifier import minify
|
|
9
|
-
|
|
17
|
+
|
|
18
|
+
OPENAI_MODELS: dict[str, int] = {
|
|
19
|
+
"gpt-4": 8192,
|
|
20
|
+
"gpt-4-32k": 32768,
|
|
21
|
+
"gpt-4-32k-0613": 32768,
|
|
22
|
+
"gpt-4-0613": 8192,
|
|
23
|
+
"gpt-3.5-turbo": 4096,
|
|
24
|
+
"gpt-3.5-turbo-0613": 4096,
|
|
25
|
+
"gpt-3.5-turbo-16k": 16384,
|
|
26
|
+
"gpt-3.5-turbo-16k-0613": 16384,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
10
30
|
|
|
11
31
|
|
|
12
32
|
class PyTypingMinifier:
|
|
13
|
-
|
|
33
|
+
"""Minifies Python files and optionally infers types using pytype.
|
|
34
|
+
|
|
35
|
+
Attributes:
|
|
36
|
+
PY_TYPE_PY_VER: Python version string used for pytype (e.g. ``"3.10"``).
|
|
37
|
+
PY_TYPE_PY_EXE: Path to the Python executable for that version, or
|
|
38
|
+
``None`` if not found.
|
|
39
|
+
py_folder: Resolved source folder (set by :meth:`init_folders`).
|
|
40
|
+
out_py_folder: Resolved output folder.
|
|
41
|
+
pyi_folder: Folder used to store ``.pyi`` stubs generated by pytype.
|
|
42
|
+
code_folder_data: Mapping from output path to per-file metadata dict.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(self, py_ver: str = "3.10") -> None:
|
|
14
46
|
self.PY_TYPE_PY_VER = py_ver
|
|
15
|
-
self.PY_TYPE_PY_EXE = shutil.which(f"python{self.PY_TYPE_PY_VER}")
|
|
16
|
-
self.
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
self.
|
|
23
|
-
self.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
47
|
+
self.PY_TYPE_PY_EXE: str | None = shutil.which(f"python{self.PY_TYPE_PY_VER}")
|
|
48
|
+
if not self.PY_TYPE_PY_EXE:
|
|
49
|
+
logger.warning(
|
|
50
|
+
"Python executable for version %s not found. "
|
|
51
|
+
"Type inference with Pytype will likely fail.",
|
|
52
|
+
self.PY_TYPE_PY_VER,
|
|
53
|
+
)
|
|
54
|
+
self.py_folder: Path | None = None
|
|
55
|
+
self.out_py_folder: Path | None = None
|
|
56
|
+
self.pyi_folder: Path | None = None
|
|
57
|
+
self.code_folder_data: dict[Path, dict] = {}
|
|
58
|
+
|
|
59
|
+
# ------------------------------------------------------------------
|
|
60
|
+
# Folder / file initialisation helpers
|
|
61
|
+
# ------------------------------------------------------------------
|
|
29
62
|
|
|
30
63
|
def init_folders(
|
|
31
64
|
self,
|
|
32
65
|
py_folder: str | Path,
|
|
33
66
|
out_py_folder: str | Path | None = None,
|
|
34
67
|
pyi_folder: str | Path | None = None,
|
|
35
|
-
):
|
|
68
|
+
) -> None:
|
|
69
|
+
"""Resolve and create output folders.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
py_folder: Source Python folder.
|
|
73
|
+
out_py_folder: Destination folder for processed files. Defaults
|
|
74
|
+
to *py_folder* when ``None``.
|
|
75
|
+
pyi_folder: Destination for ``.pyi`` stubs. Defaults to
|
|
76
|
+
*out_py_folder* when ``None``.
|
|
77
|
+
"""
|
|
36
78
|
self.py_folder = Path(py_folder).resolve()
|
|
37
79
|
self.out_py_folder = (
|
|
38
80
|
Path(out_py_folder).resolve() if out_py_folder else self.py_folder
|
|
@@ -46,7 +88,14 @@ class PyTypingMinifier:
|
|
|
46
88
|
py_path: str | Path,
|
|
47
89
|
out_py_folder: str | Path | None = None,
|
|
48
90
|
pyi_folder: str | Path | None = None,
|
|
49
|
-
):
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Register a single Python file for processing.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
py_path: Path to the source ``.py`` file.
|
|
96
|
+
out_py_folder: Output folder override.
|
|
97
|
+
pyi_folder: Stub folder override.
|
|
98
|
+
"""
|
|
50
99
|
py_path = Path(py_path).resolve()
|
|
51
100
|
self.init_folders(py_path.parent, out_py_folder, pyi_folder)
|
|
52
101
|
out_py_path, code_data = self.init_code_data(py_path)
|
|
@@ -57,51 +106,111 @@ class PyTypingMinifier:
|
|
|
57
106
|
py_folder: str | Path,
|
|
58
107
|
out_py_folder: str | Path | None = None,
|
|
59
108
|
pyi_folder: str | Path | None = None,
|
|
60
|
-
):
|
|
109
|
+
) -> None:
|
|
110
|
+
"""Register all ``.py`` files under *py_folder* for processing.
|
|
111
|
+
|
|
112
|
+
Args:
|
|
113
|
+
py_folder: Root of the source tree to process recursively.
|
|
114
|
+
out_py_folder: Output folder override.
|
|
115
|
+
pyi_folder: Stub folder override.
|
|
116
|
+
"""
|
|
61
117
|
self.init_folders(py_folder, out_py_folder, pyi_folder)
|
|
62
|
-
for py_path in Path(self.py_folder).rglob("*.py"):
|
|
118
|
+
for py_path in Path(self.py_folder).rglob("*.py"): # type: ignore[arg-type]
|
|
63
119
|
out_py_path, code_data = self.init_code_data(py_path)
|
|
64
120
|
self.code_folder_data[out_py_path] = code_data
|
|
65
121
|
|
|
66
|
-
def init_code_data(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
122
|
+
def init_code_data(self, py_path: str | Path) -> tuple[Path, dict]:
|
|
123
|
+
"""Copy *py_path* to the output folder and build its metadata dict.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
py_path: Absolute path to a source ``.py`` file.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
A ``(out_py_path, code_data)`` tuple where *out_py_path* is the
|
|
130
|
+
destination path and *code_data* holds path/content metadata.
|
|
131
|
+
"""
|
|
70
132
|
py_path = Path(py_path).resolve()
|
|
71
|
-
rel_py_path =
|
|
72
|
-
out_py_path = Path(self.out_py_folder, rel_py_path)
|
|
133
|
+
rel_py_path = py_path.relative_to(self.py_folder) # type: ignore[arg-type]
|
|
134
|
+
out_py_path = Path(self.out_py_folder, rel_py_path) # type: ignore[arg-type]
|
|
135
|
+
rel_out_py_path = out_py_path.relative_to(self.out_py_folder) # type: ignore[arg-type]
|
|
73
136
|
out_py_path.parent.mkdir(parents=True, exist_ok=True)
|
|
74
137
|
if out_py_path != py_path:
|
|
75
138
|
shutil.copy2(py_path, out_py_path)
|
|
76
139
|
pyi_path = Path(
|
|
77
|
-
self.pyi_folder,
|
|
140
|
+
self.pyi_folder, # type: ignore[arg-type]
|
|
141
|
+
".pytype",
|
|
142
|
+
"pyi",
|
|
143
|
+
rel_out_py_path.with_suffix(".pyi").name,
|
|
78
144
|
)
|
|
79
|
-
py_code = out_py_path.read_text()
|
|
80
|
-
code_data = {
|
|
145
|
+
py_code = out_py_path.read_text(encoding="utf-8")
|
|
146
|
+
code_data: dict = {
|
|
81
147
|
"py_path": py_path,
|
|
148
|
+
"rel_path": rel_py_path,
|
|
82
149
|
"pyi_path": pyi_path,
|
|
83
150
|
"py_code": py_code,
|
|
84
151
|
}
|
|
85
152
|
return out_py_path, code_data
|
|
86
153
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
154
|
+
# ------------------------------------------------------------------
|
|
155
|
+
# Core operations
|
|
156
|
+
# ------------------------------------------------------------------
|
|
157
|
+
|
|
158
|
+
def infer_types(self, py_path: Path, pyi_path: Path, py_code: str) -> str:
|
|
159
|
+
"""Run pytype on *py_path* and merge type stubs back into the source.
|
|
160
|
+
|
|
161
|
+
If the pytype executable is unavailable or fails, a warning is logged
|
|
162
|
+
and the original *py_code* is returned unchanged.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
py_path: Absolute path to the (output) Python file to analyse.
|
|
166
|
+
pyi_path: Expected location of the ``.pyi`` stub generated by
|
|
167
|
+
pytype.
|
|
168
|
+
py_code: Current source text of the file.
|
|
169
|
+
|
|
170
|
+
Returns:
|
|
171
|
+
Source text annotated with inferred types, or *py_code* unchanged
|
|
172
|
+
on failure.
|
|
173
|
+
"""
|
|
174
|
+
if not self.PY_TYPE_PY_EXE:
|
|
175
|
+
logger.warning(
|
|
176
|
+
"Pytype failed for %s: Python %s executable not found.",
|
|
177
|
+
py_path,
|
|
178
|
+
self.PY_TYPE_PY_VER,
|
|
179
|
+
)
|
|
180
|
+
return py_code
|
|
181
|
+
try:
|
|
182
|
+
from pytype.tools.merge_pyi import merge_pyi # lazy optional import
|
|
183
|
+
|
|
91
184
|
command = [
|
|
92
185
|
self.PY_TYPE_PY_EXE,
|
|
93
186
|
"-m",
|
|
94
187
|
"pytype",
|
|
95
188
|
f"--python-version={self.PY_TYPE_PY_VER}",
|
|
96
|
-
str(py_path),
|
|
189
|
+
str(py_path.relative_to(self.pyi_folder)), # type: ignore[arg-type]
|
|
97
190
|
]
|
|
98
|
-
subprocess.run(
|
|
99
|
-
|
|
191
|
+
subprocess.run(
|
|
192
|
+
command, cwd=self.pyi_folder, check=True, capture_output=True
|
|
193
|
+
)
|
|
194
|
+
pyi_code = pyi_path.read_text(encoding="utf-8")
|
|
100
195
|
py_code = merge_pyi.merge_sources(py=py_code, pyi=pyi_code)
|
|
196
|
+
except Exception as exc:
|
|
197
|
+
logger.warning("Pytype failed for %s: %s", py_path, exc)
|
|
101
198
|
return py_code
|
|
102
199
|
|
|
103
|
-
def minify(self, py_code: str, **custom_minify_options):
|
|
104
|
-
|
|
200
|
+
def minify(self, py_code: str, **custom_minify_options: object) -> str:
|
|
201
|
+
"""Minify *py_code* using python-minifier with sensible defaults.
|
|
202
|
+
|
|
203
|
+
Keyword overrides are merged on top of the defaults, so any
|
|
204
|
+
``python_minifier.minify`` option can be passed as a keyword argument.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
py_code: Python source text to minify.
|
|
208
|
+
**custom_minify_options: Overrides for python-minifier options.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
Minified Python source text.
|
|
212
|
+
"""
|
|
213
|
+
minify_options: dict[str, object] = {
|
|
105
214
|
"combine_imports": True,
|
|
106
215
|
"convert_posargs_to_args": True,
|
|
107
216
|
"hoist_literals": False,
|
|
@@ -116,7 +225,7 @@ class PyTypingMinifier:
|
|
|
116
225
|
"rename_globals": False,
|
|
117
226
|
"rename_locals": False,
|
|
118
227
|
} | custom_minify_options
|
|
119
|
-
return minify(py_code, **minify_options)
|
|
228
|
+
return minify(py_code, **minify_options) # type: ignore[arg-type]
|
|
120
229
|
|
|
121
230
|
def process_py(
|
|
122
231
|
self,
|
|
@@ -125,8 +234,24 @@ class PyTypingMinifier:
|
|
|
125
234
|
pyi_folder: str | Path | None = None,
|
|
126
235
|
types: bool = True,
|
|
127
236
|
mini: bool = True,
|
|
128
|
-
**minify_options,
|
|
237
|
+
**minify_options: object,
|
|
129
238
|
) -> list[Path]:
|
|
239
|
+
"""Process one Python file or an entire directory tree.
|
|
240
|
+
|
|
241
|
+
Reads all relevant ``.py`` files, optionally runs type inference and/or
|
|
242
|
+
minification on each one, and writes results to *out_py_folder*.
|
|
243
|
+
|
|
244
|
+
Args:
|
|
245
|
+
py_path_or_folder: Path to a single ``.py`` file or a directory.
|
|
246
|
+
out_py_folder: Destination folder for output files.
|
|
247
|
+
pyi_folder: Folder for pytype stub files.
|
|
248
|
+
types: Whether to run pytype type inference.
|
|
249
|
+
mini: Whether to minify the output.
|
|
250
|
+
**minify_options: Extra options forwarded to :meth:`minify`.
|
|
251
|
+
|
|
252
|
+
Returns:
|
|
253
|
+
List of absolute paths to all written output files.
|
|
254
|
+
"""
|
|
130
255
|
py_path_or_folder = Path(py_path_or_folder).resolve()
|
|
131
256
|
if py_path_or_folder.is_dir():
|
|
132
257
|
self.read_py_folder(py_path_or_folder, out_py_folder, pyi_folder)
|
|
@@ -134,13 +259,252 @@ class PyTypingMinifier:
|
|
|
134
259
|
self.read_py_file(py_path_or_folder, out_py_folder, pyi_folder)
|
|
135
260
|
else:
|
|
136
261
|
return []
|
|
262
|
+
|
|
137
263
|
for out_py_path, code_data in self.code_folder_data.items():
|
|
264
|
+
original_py_code: str = code_data["py_code"]
|
|
265
|
+
py_code: str = original_py_code
|
|
266
|
+
|
|
138
267
|
if types:
|
|
139
|
-
py_code = self.infer_types(
|
|
140
|
-
|
|
141
|
-
)
|
|
268
|
+
py_code = self.infer_types(out_py_path, code_data["pyi_path"], py_code)
|
|
269
|
+
|
|
142
270
|
if mini:
|
|
143
|
-
|
|
271
|
+
try:
|
|
272
|
+
py_code = self.minify(py_code, **minify_options)
|
|
273
|
+
except Exception as exc:
|
|
274
|
+
logger.error("Minification failed for %s: %s", out_py_path, exc)
|
|
275
|
+
py_code = original_py_code # fall back to pre-minification text
|
|
276
|
+
|
|
144
277
|
code_data["py_code"] = py_code
|
|
145
|
-
out_py_path.write_text(py_code)
|
|
278
|
+
out_py_path.write_text(py_code, encoding="utf-8")
|
|
279
|
+
|
|
146
280
|
return list(self.code_folder_data.keys())
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# ---------------------------------------------------------------------------
|
|
284
|
+
# Optional LLM-powered classes (require tiktoken, simpleaichat, astor extras)
|
|
285
|
+
# ---------------------------------------------------------------------------
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class PyBodySummarizer(NodeTransformer):
|
|
289
|
+
"""AST transformer that replaces large function/class bodies with stubs.
|
|
290
|
+
|
|
291
|
+
When a function or class body exceeds the token threshold it is replaced
|
|
292
|
+
with ``...`` and (optionally) an AI-generated docstring summary.
|
|
293
|
+
|
|
294
|
+
Args:
|
|
295
|
+
py_llm_splitter: The :class:`PyLLMSplitter` instance that drives
|
|
296
|
+
summarisation and token counting.
|
|
297
|
+
"""
|
|
298
|
+
|
|
299
|
+
def __init__(self, py_llm_splitter: PyLLMSplitter) -> None:
|
|
300
|
+
self.py_llm_splitter = py_llm_splitter
|
|
301
|
+
|
|
302
|
+
def visit_FunctionDef(
|
|
303
|
+
self, node: FunctionDef, code: str | None = None
|
|
304
|
+
) -> FunctionDef: # type: ignore[override]
|
|
305
|
+
"""Replace *node*'s body with ``...`` (and an AI summary if *code* given).
|
|
306
|
+
|
|
307
|
+
Args:
|
|
308
|
+
node: The function AST node to transform.
|
|
309
|
+
code: Minified source of the function body, used to generate a
|
|
310
|
+
summary. Pass ``None`` to skip summarisation.
|
|
311
|
+
|
|
312
|
+
Returns:
|
|
313
|
+
The mutated function node.
|
|
314
|
+
"""
|
|
315
|
+
node.body = []
|
|
316
|
+
if code:
|
|
317
|
+
with contextlib.suppress(Exception):
|
|
318
|
+
doc = self.py_llm_splitter.llm_summarize(
|
|
319
|
+
self.py_llm_splitter.minify(code, remove_literal_statements=False)
|
|
320
|
+
)
|
|
321
|
+
node.body.append(parse(f'"""{doc}"""').body[0])
|
|
322
|
+
node.body.append(parse("...").body[0])
|
|
323
|
+
return node
|
|
324
|
+
|
|
325
|
+
def visit_ClassDef(self, node: ClassDef) -> ClassDef: # type: ignore[override]
|
|
326
|
+
"""Summarise oversized methods inside *node*.
|
|
327
|
+
|
|
328
|
+
Args:
|
|
329
|
+
node: The class AST node to transform.
|
|
330
|
+
|
|
331
|
+
Returns:
|
|
332
|
+
The mutated class node.
|
|
333
|
+
"""
|
|
334
|
+
for i, body_node in enumerate(node.body):
|
|
335
|
+
if isinstance(body_node, FunctionDef):
|
|
336
|
+
code = ast_unparse(body_node)
|
|
337
|
+
minified_code = self.py_llm_splitter.minify(
|
|
338
|
+
code, remove_literal_statements=False
|
|
339
|
+
)
|
|
340
|
+
size = self.py_llm_splitter.gptok_size(minified_code)
|
|
341
|
+
if size > self.py_llm_splitter.gptok_threshold:
|
|
342
|
+
node.body[i] = self.visit_FunctionDef(body_node, minified_code)
|
|
343
|
+
return node
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class PyLLMSplitter(PyTypingMinifier):
|
|
347
|
+
"""Extends :class:`PyTypingMinifier` with token-based splitting and LLM summaries.
|
|
348
|
+
|
|
349
|
+
Requires the ``llm`` extras: ``tiktoken``, ``simpleaichat``, ``astor``.
|
|
350
|
+
|
|
351
|
+
Args:
|
|
352
|
+
*args: Forwarded to :class:`PyTypingMinifier`.
|
|
353
|
+
gptok_model: OpenAI model name used for token counting and
|
|
354
|
+
summarisation.
|
|
355
|
+
gptok_limit: Maximum tokens per output split file. Defaults to the
|
|
356
|
+
context window of *gptok_model*.
|
|
357
|
+
gptok_threshold: Minimum token count before a function/class body is
|
|
358
|
+
replaced with a stub.
|
|
359
|
+
**kwargs: Forwarded to :class:`PyTypingMinifier`.
|
|
360
|
+
"""
|
|
361
|
+
|
|
362
|
+
def __init__(
|
|
363
|
+
self,
|
|
364
|
+
*args: object,
|
|
365
|
+
gptok_model: str = "gpt-3.5-turbo",
|
|
366
|
+
gptok_limit: int | None = None,
|
|
367
|
+
gptok_threshold: int = 128,
|
|
368
|
+
**kwargs: object,
|
|
369
|
+
) -> None:
|
|
370
|
+
super().__init__(*args, **kwargs) # type: ignore[arg-type]
|
|
371
|
+
self.gptok_model = gptok_model
|
|
372
|
+
self.gptok_limit: int = gptok_limit or OPENAI_MODELS.get(gptok_model, 2048)
|
|
373
|
+
self.gptok_threshold = gptok_threshold
|
|
374
|
+
self.code_summary: dict[str, dict] = {}
|
|
375
|
+
|
|
376
|
+
# tiktoken — lazy; fall back to char-count estimate if unavailable
|
|
377
|
+
self.gptoker = None
|
|
378
|
+
try:
|
|
379
|
+
import tiktoken # lazy optional import
|
|
380
|
+
|
|
381
|
+
self.gptoker = tiktoken.encoding_for_model(gptok_model)
|
|
382
|
+
except Exception as exc:
|
|
383
|
+
logger.warning("tiktoken unavailable (%s); using character estimate.", exc)
|
|
384
|
+
|
|
385
|
+
# simpleaichat — lazy; LLM summarisation disabled when unavailable
|
|
386
|
+
self.llm_summarize = None
|
|
387
|
+
try:
|
|
388
|
+
from simpleaichat import AIChat # lazy optional import
|
|
389
|
+
|
|
390
|
+
self.llm_summarize = AIChat(
|
|
391
|
+
api_key=environ.get("OPENAI_API_KEY"),
|
|
392
|
+
system=(
|
|
393
|
+
"Write an extremely short, compact description of this Python code, "
|
|
394
|
+
'starting with "This class" or "This method" or "This function"'
|
|
395
|
+
),
|
|
396
|
+
model=self.gptok_model,
|
|
397
|
+
)
|
|
398
|
+
except Exception as exc:
|
|
399
|
+
logger.warning("AIChat unavailable (%s); LLM summarisation disabled.", exc)
|
|
400
|
+
|
|
401
|
+
def gptok_size(self, text: str) -> int:
|
|
402
|
+
"""Count GPT tokens in *text* using the model's tokeniser.
|
|
403
|
+
|
|
404
|
+
Falls back to ``len(text) // 4`` (a rough GPT token estimate) when the
|
|
405
|
+
tiktoken encoder is unavailable.
|
|
406
|
+
|
|
407
|
+
Args:
|
|
408
|
+
text: Arbitrary text to measure.
|
|
409
|
+
|
|
410
|
+
Returns:
|
|
411
|
+
Number of tokens (exact when tiktoken available, estimated otherwise).
|
|
412
|
+
"""
|
|
413
|
+
if self.gptoker is not None:
|
|
414
|
+
return len(list(self.gptoker.encode(text)))
|
|
415
|
+
return len(text) // 4 # rough estimate: ~4 chars per token
|
|
416
|
+
|
|
417
|
+
def process_py_code(self, py_code: str) -> list[dict]:
|
|
418
|
+
"""Split *py_code* into token-bounded sections.
|
|
419
|
+
|
|
420
|
+
Large top-level functions and classes are replaced with stubs before
|
|
421
|
+
sectioning. Uses :func:`ast.unparse` (stdlib) to convert AST nodes
|
|
422
|
+
back to source — no third-party code-generation library required.
|
|
423
|
+
|
|
424
|
+
Args:
|
|
425
|
+
py_code: Python source text (already minified/typed).
|
|
426
|
+
|
|
427
|
+
Returns:
|
|
428
|
+
List of ``{"py": str, "gptok_size": int}`` dicts.
|
|
429
|
+
"""
|
|
430
|
+
tree = parse(py_code)
|
|
431
|
+
sections: list[dict] = []
|
|
432
|
+
body_summary = PyBodySummarizer(self)
|
|
433
|
+
|
|
434
|
+
for node in tree.body:
|
|
435
|
+
code = ast_unparse(node)
|
|
436
|
+
minified_code = self.minify(code, remove_literal_statements=False)
|
|
437
|
+
size = self.gptok_size(minified_code)
|
|
438
|
+
|
|
439
|
+
if size > self.gptok_threshold and isinstance(
|
|
440
|
+
node, (FunctionDef, ClassDef)
|
|
441
|
+
):
|
|
442
|
+
node = body_summary.visit(node)
|
|
443
|
+
fix_missing_locations(node)
|
|
444
|
+
minified_code = self.minify(
|
|
445
|
+
ast_unparse(node), remove_literal_statements=False
|
|
446
|
+
)
|
|
447
|
+
size = self.gptok_size(minified_code)
|
|
448
|
+
|
|
449
|
+
sections.append({"py": minified_code, "gptok_size": size})
|
|
450
|
+
|
|
451
|
+
return sections
|
|
452
|
+
|
|
453
|
+
def process_py(self, *args: object, **kwargs: object) -> list[Path]:
|
|
454
|
+
"""Process files and compute per-file sections for splitting.
|
|
455
|
+
|
|
456
|
+
Delegates to :meth:`PyTypingMinifier.process_py` then attaches
|
|
457
|
+
section data to each file entry in :attr:`code_summary`.
|
|
458
|
+
|
|
459
|
+
Returns:
|
|
460
|
+
List of output file paths (same as parent return value).
|
|
461
|
+
"""
|
|
462
|
+
paths = super().process_py(*args, **kwargs) # type: ignore[arg-type]
|
|
463
|
+
|
|
464
|
+
for path in paths:
|
|
465
|
+
code_data = self.code_folder_data[path]
|
|
466
|
+
sections = self.process_py_code(code_data["py_code"])
|
|
467
|
+
code_data["sections"] = sections
|
|
468
|
+
code_data["gptok_size"] = sum(sec["gptok_size"] for sec in sections)
|
|
469
|
+
self.code_summary[str(path)] = code_data
|
|
470
|
+
|
|
471
|
+
return paths
|
|
472
|
+
|
|
473
|
+
def write_splits(self) -> None:
|
|
474
|
+
"""Write token-bounded split files to ``<out_py_folder>/split4gpt/``.
|
|
475
|
+
|
|
476
|
+
Each split file contains consecutive sections from :attr:`code_summary`
|
|
477
|
+
that together do not exceed :attr:`gptok_limit` tokens. Does nothing
|
|
478
|
+
when :attr:`out_py_folder` has not been set (i.e. no files were
|
|
479
|
+
processed).
|
|
480
|
+
"""
|
|
481
|
+
if self.out_py_folder is None:
|
|
482
|
+
logger.warning(
|
|
483
|
+
"write_splits called before process_py; no output folder set."
|
|
484
|
+
)
|
|
485
|
+
return
|
|
486
|
+
splits_folder = self.out_py_folder / "split4gpt"
|
|
487
|
+
splits_folder.mkdir(parents=True, exist_ok=True)
|
|
488
|
+
|
|
489
|
+
textportions: list[str] = []
|
|
490
|
+
current_size = 0
|
|
491
|
+
current_portion: list[str] = []
|
|
492
|
+
|
|
493
|
+
for path, code_data in self.code_summary.items():
|
|
494
|
+
header = f"# File: {path}\n"
|
|
495
|
+
current_portion.append(header)
|
|
496
|
+
current_size += self.gptok_size(header)
|
|
497
|
+
|
|
498
|
+
for section in code_data["sections"]:
|
|
499
|
+
if current_size + section["gptok_size"] > self.gptok_limit:
|
|
500
|
+
textportions.append("".join(current_portion))
|
|
501
|
+
current_portion = []
|
|
502
|
+
current_size = 0
|
|
503
|
+
current_portion.append(section["py"])
|
|
504
|
+
current_size += section["gptok_size"]
|
|
505
|
+
|
|
506
|
+
if current_portion:
|
|
507
|
+
textportions.append("".join(current_portion))
|
|
508
|
+
|
|
509
|
+
for i, textportion in enumerate(textportions, start=1):
|
|
510
|
+
(splits_folder / f"split{i}.py").write_text(textportion, encoding="utf-8")
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: split-python4gpt
|
|
3
|
+
Version: 1.0.5
|
|
4
|
+
Summary: Reorganise large Python projects into minified, token-bounded files for LLM consumption.
|
|
5
|
+
Project-URL: Homepage, https://github.com/twardoch/split-python4gpt
|
|
6
|
+
Project-URL: Documentation, https://twardoch.github.io/split-python4gpt/
|
|
7
|
+
Project-URL: Source, https://github.com/twardoch/split-python4gpt
|
|
8
|
+
Project-URL: Changelog, https://github.com/twardoch/split-python4gpt/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Adam Twardoch <adam+github@twardoch.com>
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
License-File: AUTHORS.md
|
|
12
|
+
License-File: LICENSE.txt
|
|
13
|
+
Keywords: code-summarization,data-preprocessing,gpt,gpt-4,openai,openai-gpt,python,text-summarization
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Topic :: Text Processing
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: fire>=0.5.0
|
|
28
|
+
Requires-Dist: python-minifier>=2.9.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
31
|
+
Requires-Dist: psutil>=5.9; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: ruff>=0.4; extra == 'dev'
|
|
35
|
+
Provides-Extra: llm
|
|
36
|
+
Requires-Dist: simpleaichat>=0.2.0; extra == 'llm'
|
|
37
|
+
Requires-Dist: tiktoken>=0.4.0; extra == 'llm'
|
|
38
|
+
Provides-Extra: testing
|
|
39
|
+
Requires-Dist: psutil>=5.9; extra == 'testing'
|
|
40
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'testing'
|
|
41
|
+
Requires-Dist: pytest>=7.0; extra == 'testing'
|
|
42
|
+
Provides-Extra: types
|
|
43
|
+
Requires-Dist: pytype>=2023.6.16; extra == 'types'
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
|
|
46
|
+
# split-python4gpt
|
|
47
|
+
|
|
48
|
+
[](https://pypi.org/project/split-python4gpt/)
|
|
49
|
+
[](https://github.com/twardoch/split-python4gpt/blob/main/LICENSE.txt)
|
|
50
|
+
[](https://twardoch.github.io/split-python4gpt/)
|
|
51
|
+
[](https://github.com/twardoch/split-python4gpt/actions/workflows/ci.yml)
|
|
52
|
+
|
|
53
|
+
`split-python4gpt` processes and reorganizes large Python projects into minified, type-annotated, and token-limited files. This prepares codebases for analysis by Large Language Models (LLMs) like OpenAI's GPT series, allowing them to handle data in manageable chunks.
|
|
54
|
+
|
|
55
|
+
## Overview
|
|
56
|
+
|
|
57
|
+
**What is `split-python4gpt`?**
|
|
58
|
+
|
|
59
|
+
A command-line and programmatic tool that takes Python files or project directories as input and performs:
|
|
60
|
+
1. **Type Inference:** Integrates with `pytype` to add type hints
|
|
61
|
+
2. **Minification:** Uses `python-minifier` with configurable options
|
|
62
|
+
3. **Code Summarization:** Replaces large functions/classes with `...` and AI-generated summaries (requires OpenAI API key)
|
|
63
|
+
4. **Splitting for LLMs:** Breaks processed code into smaller files respecting token limits
|
|
64
|
+
|
|
65
|
+
**Who is it for?**
|
|
66
|
+
|
|
67
|
+
* **Developers** working with LLMs who need to analyze large Python codebases
|
|
68
|
+
* **Researchers** in software engineering or NLP preprocessing Python code for studies
|
|
69
|
+
* Anyone reducing Python code size while preserving structure before passing to token-sensitive systems
|
|
70
|
+
|
|
71
|
+
**Why is it useful?**
|
|
72
|
+
|
|
73
|
+
* **Manages LLM Context Limits:** Breaks code into chunks that fit finite context windows
|
|
74
|
+
* **Reduces Token Count:** Minification and summarization significantly cut tokens, reducing processing time and API costs
|
|
75
|
+
* **Improves Code Clarity:** Type hints help LLMs understand code; summaries provide context for complex blocks
|
|
76
|
+
* **Automates Preprocessing:** Handles tedious parts of the MLOps pipeline for code-based LLM tasks
|
|
77
|
+
|
|
78
|
+
## Features
|
|
79
|
+
|
|
80
|
+
* **Process Single Files or Entire Directories:** Handles individual scripts or recursively processes all `.py` files
|
|
81
|
+
* **Optional Type Inference:** Uses `pytype` to add type annotations
|
|
82
|
+
* **Comprehensive Minification:** Leverages `python-minifier` with configurable options:
|
|
83
|
+
* Remove docstrings (`mini_docs`)
|
|
84
|
+
* Rename global/local variables (`mini_globs`, `mini_locs`)
|
|
85
|
+
* Hoist literal statements (`mini_lits`)
|
|
86
|
+
* Remove type annotations (`mini_annotations`)
|
|
87
|
+
* Remove `assert` and debugging statements (`mini_asserts`, `mini_debug`)
|
|
88
|
+
* Combine imports (`mini_imports`)
|
|
89
|
+
* Remove `object` base from classes (`mini_obj`)
|
|
90
|
+
* Remove `pass` statements (`mini_pass`)
|
|
91
|
+
* Convert positional to keyword arguments (`mini_posargs`)
|
|
92
|
+
* Remove explicit `return None` (`mini_retnone`)
|
|
93
|
+
* Remove shebang (`mini_shebang`)
|
|
94
|
+
* **AI-Powered Code Summarization:** Replaces large functions/classes with `...` and summaries via OpenAI models
|
|
95
|
+
* **Token-Based Splitting:** Uses `tiktoken` to count tokens and split code into multiple files respecting limits
|
|
96
|
+
* **Configurable Output:** Specify output directories for processed files and type stubs
|
|
97
|
+
* **Preserves Relative Paths:** Maintains original project structure in output
|
|
98
|
+
|
|
99
|
+
## Installation
|
|
100
|
+
|
|
101
|
+
**Prerequisites:**
|
|
102
|
+
|
|
103
|
+
* Python 3.10 or later (3.11, 3.12, 3.13 are all supported)
|
|
104
|
+
* An OpenAI API key is required **only** for LLM summarisation (`export OPENAI_API_KEY="your_api_key_here"`)
|
|
105
|
+
* `pytype` is optional — install the `[types]` extra for type inference
|
|
106
|
+
|
|
107
|
+
### Installation
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install split-python4gpt
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Optional extras:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
# Type inference via pytype
|
|
117
|
+
pip install "split-python4gpt[types]"
|
|
118
|
+
|
|
119
|
+
# LLM summarisation (tiktoken + simpleaichat)
|
|
120
|
+
pip install "split-python4gpt[llm]"
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
#### From source
|
|
124
|
+
|
|
125
|
+
1. Install in virtual environment:
|
|
126
|
+
```bash
|
|
127
|
+
python -m venv .venv
|
|
128
|
+
source .venv/bin/activate
|
|
129
|
+
pip install -e ".[dev]"
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Installs dependencies: `fire`, `tiktoken`, `python-minifier`, `pytype`, and `simpleaichat`.
|
|
133
|
+
|
|
134
|
+
#### Option 2: Pre-built Binary
|
|
135
|
+
|
|
136
|
+
Download from [releases page](https://github.com/twardoch/split-python4gpt/releases):
|
|
137
|
+
|
|
138
|
+
- **Linux**: `mdsplit4gpt-linux-x86_64`
|
|
139
|
+
- **macOS**: `mdsplit4gpt-macos-x86_64`
|
|
140
|
+
- **Windows**: `mdsplit4gpt-windows-x86_64.exe`
|
|
141
|
+
|
|
142
|
+
Make executable and add to PATH:
|
|
143
|
+
```bash
|
|
144
|
+
chmod +x mdsplit4gpt-linux-x86_64
|
|
145
|
+
mv mdsplit4gpt-linux-x86_64 ~/.local/bin/mdsplit4gpt
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
#### Option 3: Source Installation
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
git clone https://github.com/twardoch/split-python4gpt.git
|
|
152
|
+
cd split-python4gpt
|
|
153
|
+
./scripts/install-dev.sh
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
## Usage
|
|
157
|
+
|
|
158
|
+
### Command-Line Interface
|
|
159
|
+
|
|
160
|
+
Primary command: `mdsplit4gpt`
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
mdsplit4gpt [PATH_OR_FOLDER] [OPTIONS]
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
**Key Arguments:**
|
|
167
|
+
|
|
168
|
+
* `path_or_folder`: Input Python file or folder
|
|
169
|
+
* `--out`: Output folder for processed files (defaults to input folder)
|
|
170
|
+
* `--pyis`: Directory for `.pyi` files (defaults to output folder)
|
|
171
|
+
* `--types`: Infer types using PyType (default: True)
|
|
172
|
+
* `--mini`: Minify scripts (default: True)
|
|
173
|
+
|
|
174
|
+
**Minification Options:**
|
|
175
|
+
* `--mini_docs`: Remove docstrings
|
|
176
|
+
* `--mini_globs`: Rename global names (default: False)
|
|
177
|
+
* `--mini_locs`: Rename local names (default: False)
|
|
178
|
+
* `--mini_lits`: Hoist literal statements
|
|
179
|
+
* `--mini_annotations`: Remove annotations
|
|
180
|
+
* `--mini_asserts`: Remove asserts
|
|
181
|
+
* `--mini_debug`: Remove debugging statements
|
|
182
|
+
* `--mini_imports`: Combine imports
|
|
183
|
+
* `--mini_obj`: Remove object base
|
|
184
|
+
* `--mini_pass`: Remove pass statements
|
|
185
|
+
* `--mini_posargs`: Convert positional to keyword args
|
|
186
|
+
* `--mini_retnone`: Remove explicit return None
|
|
187
|
+
* `--mini_shebang`: Remove shebang (set to False to preserve)
|
|
188
|
+
|
|
189
|
+
**Examples:**
|
|
190
|
+
|
|
191
|
+
1. Process single file with minification and type inference:
|
|
192
|
+
```bash
|
|
193
|
+
mdsplit4gpt my_script.py --out output_dir
|
|
194
|
+
```
|
|
195
|
+
Creates `output_dir/my_script.py` and `output_dir/split4gpt/split1.py`.
|
|
196
|
+
|
|
197
|
+
2. Process project without type inference, preserving docstrings:
|
|
198
|
+
```bash
|
|
199
|
+
mdsplit4gpt my_project/ --out processed_project/ --types=False --mini_docs=False
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### Programmatic Usage
|
|
203
|
+
|
|
204
|
+
```python
|
|
205
|
+
from pathlib import Path
|
|
206
|
+
from split_python4gpt import PyLLMSplitter
|
|
207
|
+
|
|
208
|
+
# Set OPENAI_API_KEY environment variable for summarization
|
|
209
|
+
# os.environ["OPENAI_API_KEY"] = "your_api_key"
|
|
210
|
+
|
|
211
|
+
splitter = PyLLMSplitter(
|
|
212
|
+
gptok_model="gpt-3.5-turbo", gptok_limit=4000, gptok_threshold=200
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
input_path = "path/to/your/python_project_or_file"
|
|
216
|
+
output_dir = "path/to/output_directory"
|
|
217
|
+
pyi_dir = "path/to/pyi_files_directory"
|
|
218
|
+
|
|
219
|
+
processed_file_paths = splitter.process_py(
|
|
220
|
+
py_path_or_folder=input_path,
|
|
221
|
+
out_py_folder=output_dir,
|
|
222
|
+
pyi_folder=pyi_dir,
|
|
223
|
+
types=True,
|
|
224
|
+
mini=True,
|
|
225
|
+
remove_literal_statements=True,
|
|
226
|
+
rename_globals=False,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
splitter.write_splits()
|
|
230
|
+
|
|
231
|
+
print(f"Processed files: {processed_file_paths}")
|
|
232
|
+
print(f"LLM splits: {Path(output_dir) / 'split4gpt'}")
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
## Technical Details
|
|
236
|
+
|
|
237
|
+
### Workflow
|
|
238
|
+
|
|
239
|
+
1. **File Discovery:** Processes single files or recursively finds all `*.py` files in folders
|
|
240
|
+
|
|
241
|
+
2. **Initialization:** Resolves and creates input/output directories; copies files if needed
|
|
242
|
+
|
|
243
|
+
3. **Per-file Processing:**
|
|
244
|
+
* **Type Inference:** Runs `pytype` subprocess, merges generated stubs back into source
|
|
245
|
+
* **Minification:** Passes code through `python-minifier` with specified options
|
|
246
|
+
|
|
247
|
+
4. **LLM Preparation:**
|
|
248
|
+
* Parses code into AST
|
|
249
|
+
* Processes top-level nodes (imports, assignments, functions, classes)
|
|
250
|
+
* For each function/class:
|
|
251
|
+
* Calculates token count with `tiktoken`
|
|
252
|
+
* If count exceeds threshold, replaces body with `...` and AI summary
|
|
253
|
+
|
|
254
|
+
5. **Splitting:**
|
|
255
|
+
* Collects all processed sections
|
|
256
|
+
* Accumulates sections into portions respecting token limits
|
|
257
|
+
* Writes `splitN.py` files in `split4gpt` subdirectory
|
|
258
|
+
|
|
259
|
+
### Output Structure
|
|
260
|
+
|
|
261
|
+
* **Processed Files:** Minified/type-annotated versions in output directory (modifies in-place if no output specified)
|
|
262
|
+
* **`.pyi` Files:** Type stubs in specified directory (defaults to output)
|
|
263
|
+
* **`split4gpt` Directory:** Contains final chunks for LLM consumption
|
|
264
|
+
|
|
265
|
+
### Core Components
|
|
266
|
+
|
|
267
|
+
* **`PyTypingMinifier`:** Handles file management, type inference, and minification
|
|
268
|
+
* **`PyBodySummarizer`:** AST transformer that summarizes large functions/classes
|
|
269
|
+
* **`PyLLMSplitter`:** Inherits from `PyTypingMinifier`, adds token counting and splitting
|
|
270
|
+
|
|
271
|
+
## Contributing
|
|
272
|
+
|
|
273
|
+
### Development Setup
|
|
274
|
+
|
|
275
|
+
1. Fork repository
|
|
276
|
+
2. Create branch: `git checkout -b feature/your-feature-name`
|
|
277
|
+
3. Set up environment:
|
|
278
|
+
```bash
|
|
279
|
+
./scripts/install-dev.sh
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
### Development Workflow
|
|
283
|
+
|
|
284
|
+
4. Make changes
|
|
285
|
+
5. Run tests:
|
|
286
|
+
```bash
|
|
287
|
+
./scripts/build-and-test.sh
|
|
288
|
+
./scripts/build-and-test.sh --with-coverage
|
|
289
|
+
./scripts/build-and-test.sh --with-performance
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
6. Code standards:
|
|
293
|
+
* Formatted with `black`
|
|
294
|
+
* Imports sorted with `isort`
|
|
295
|
+
* PEP 8 compliant
|
|
296
|
+
* Linted with `flake8`
|
|
297
|
+
* Pre-commit hooks run automatically
|
|
298
|
+
|
|
299
|
+
7. Add tests in `tests/` directory
|
|
300
|
+
8. Commit with descriptive message
|
|
301
|
+
9. Push branch: `git push origin feature/your-feature-name`
|
|
302
|
+
10. Create Pull Request
|
|
303
|
+
|
|
304
|
+
### Release Process
|
|
305
|
+
|
|
306
|
+
1. **Create release:**
|
|
307
|
+
```bash
|
|
308
|
+
./scripts/release.sh 1.2.3
|
|
309
|
+
```
|
|
310
|
+
|
|
311
|
+
2. **Automated CI/CD:**
|
|
312
|
+
* Tests run on push/PR for Linux, macOS, Windows
|
|
313
|
+
* Git tags trigger: PyPI publication, binary builds, GitHub releases
|
|
314
|
+
|
|
315
|
+
3. **Scripts:**
|
|
316
|
+
* `install-dev.sh`: Development setup
|
|
317
|
+
* `build-and-test.sh`: Testing
|
|
318
|
+
* `release.sh`: Create releases
|
|
319
|
+
* `get_version.py`: Check current version
|
|
320
|
+
* `validate_tag.py`: Validate version format
|
|
321
|
+
|
|
322
|
+
## License
|
|
323
|
+
|
|
324
|
+
Apache License 2.0. See [LICENSE.txt](LICENSE.txt).
|
|
325
|
+
|
|
326
|
+
## Authors
|
|
327
|
+
|
|
328
|
+
* Adam Twardoch ([@twardoch](https://github.com/twardoch))
|
|
329
|
+
|
|
330
|
+
Scaffolded with [PyScaffold](https://pyscaffold.org/).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
split_python4gpt/__init__.py,sha256=jI8JtkFs_eZ5mlIlFRkk_SMMagfFRrDG6FFhVxKr_FA,444
|
|
2
|
+
split_python4gpt/__main__.py,sha256=4KXOZvg9f9qwS_JUerisfr9HvxwKGER8UaUk7k-7Log,3436
|
|
3
|
+
split_python4gpt/_version.py,sha256=jH6PrO31DRJ5btw71cW8_c0F1kZAOC-szxq6RBig7a8,520
|
|
4
|
+
split_python4gpt/minifier.py,sha256=eJj4BpsM2efr75T86oWWUP3v4yGecURCnBN61qPoQXw,19700
|
|
5
|
+
split_python4gpt-1.0.5.dist-info/METADATA,sha256=8s0f62LBB5wrdy9mU-ETTc0beuYEVvm5-ecKhSY-Xtw,11682
|
|
6
|
+
split_python4gpt-1.0.5.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
7
|
+
split_python4gpt-1.0.5.dist-info/entry_points.txt,sha256=wdxVWTgbPsCKmnN_e8XeLuCjzcUfh1SQDbaizEaD0o4,62
|
|
8
|
+
split_python4gpt-1.0.5.dist-info/licenses/AUTHORS.md,sha256=JEIsUOYyigWOn1upIYbuC6Fv4rx4YKB5I4sf1qlx0yU,60
|
|
9
|
+
split_python4gpt-1.0.5.dist-info/licenses/LICENSE.txt,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
|
10
|
+
split_python4gpt-1.0.5.dist-info/RECORD,,
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: split-python4gpt
|
|
3
|
-
Version: 1.0.3
|
|
4
|
-
Summary: Python tool designed to reorganize large Python projects into minified files based on a specified token limit. This is particularly useful for processing large Python projects with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
5
|
-
Home-page: https://github.com/twardoch/split-python4gpt
|
|
6
|
-
Author: Adam Twardoch
|
|
7
|
-
Author-email: adam+github@twardoch.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Project-URL: Documentation, https://twardoch.github.io/split-python4gpt/
|
|
10
|
-
Project-URL: Source, https://github.com/twardoch/split-python4gpt
|
|
11
|
-
Keywords: python,code-summarization,openai,text-summarization,summarization,code-processing,gpt,data-preprocessing,openai-gpt,gpt-3,gpt-4,gpt-35-turbo,gpt-35-turbo-16k
|
|
12
|
-
Platform: any
|
|
13
|
-
Classifier: Development Status :: 4 - Beta
|
|
14
|
-
Classifier: Programming Language :: Python
|
|
15
|
-
Classifier: Programming Language :: Python :: 3
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
-
Classifier: Intended Audience :: Developers
|
|
18
|
-
Classifier: Intended Audience :: Science/Research
|
|
19
|
-
Classifier: Topic :: Text Processing
|
|
20
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
21
|
-
Classifier: Operating System :: OS Independent
|
|
22
|
-
Requires-Python: <3.11,>=3.10
|
|
23
|
-
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
24
|
-
License-File: LICENSE.txt
|
|
25
|
-
Requires-Dist: fire (>=0.5.0)
|
|
26
|
-
Requires-Dist: tiktoken (>=0.4.0)
|
|
27
|
-
Requires-Dist: python-minifier (>=2.9.0)
|
|
28
|
-
Requires-Dist: pytype (>=2023.6.16)
|
|
29
|
-
Provides-Extra: testing
|
|
30
|
-
Requires-Dist: setuptools ; extra == 'testing'
|
|
31
|
-
Requires-Dist: pytest ; extra == 'testing'
|
|
32
|
-
Requires-Dist: pytest-cov ; extra == 'testing'
|
|
33
|
-
|
|
34
|
-
# split_python4gpt
|
|
35
|
-
|
|
36
|
-
`split_python4gpt` is a Python tool designed to reorganize large Python projects into minified files based on a specified token limit. This is particularly useful for processing large Python projects with GPT models, as it allows the models to handle the data in manageable chunks.
|
|
37
|
-
|
|
38
|
-
_**Version 1.0.3** (2023-06-22)_
|
|
39
|
-
|
|
40
|
-
### NOT IMPLEMENTED YET
|
|
41
|
-
|
|
42
|
-
Warning: The code does not yet implement the splitting or token counting, only type inference and minification. Use at your own risk.
|
|
43
|
-
|
|
44
|
-
## Installation
|
|
45
|
-
|
|
46
|
-
You can install `split_python4gpt` via pip:
|
|
47
|
-
|
|
48
|
-
```bash
|
|
49
|
-
pip install split_python4gpt
|
|
50
|
-
```
|
|
51
|
-
|
|
52
|
-
## CLI Usage
|
|
53
|
-
|
|
54
|
-
After installation, you can use the `pysplit4gpt` or `python3.10 -m split_python4gpt` command:
|
|
55
|
-
|
|
56
|
-
```
|
|
57
|
-
python3.10 -m split_python4gpt PATH_OR_FOLDER [FLAGS]
|
|
58
|
-
|
|
59
|
-
POSITIONAL ARGUMENTS
|
|
60
|
-
PATH_OR_FOLDER
|
|
61
|
-
Type: str | pathlib.Path
|
|
62
|
-
Path to the input Python file or folder.
|
|
63
|
-
|
|
64
|
-
FLAGS
|
|
65
|
-
-o, --out=OUT
|
|
66
|
-
Type: Optional[str | pathlib...
|
|
67
|
-
Default: None
|
|
68
|
-
Output folder for the processed files. Defaults to input folder.
|
|
69
|
-
-p, --pyis=PYIS
|
|
70
|
-
Type: Optional[str | pathlib...
|
|
71
|
-
Default: None
|
|
72
|
-
Directory for storing generated .pyi files. Defaults to the output folder.
|
|
73
|
-
-t, --types=TYPES
|
|
74
|
-
Type: bool
|
|
75
|
-
Default: True
|
|
76
|
-
Infer types using PyType? Defaults to True.
|
|
77
|
-
--mini=MINI
|
|
78
|
-
Type: bool
|
|
79
|
-
Default: True
|
|
80
|
-
Minify the Python scripts? Defaults to True.
|
|
81
|
-
--mini_docs=MINI_DOCS
|
|
82
|
-
Type: bool
|
|
83
|
-
Default: True
|
|
84
|
-
Remove docstrings? Defaults to True.
|
|
85
|
-
--mini_globs=MINI_GLOBS
|
|
86
|
-
Type: bool
|
|
87
|
-
Default: False
|
|
88
|
-
Rename global names? Defaults to False.
|
|
89
|
-
--mini_locs=MINI_LOCS
|
|
90
|
-
Type: bool
|
|
91
|
-
Default: False
|
|
92
|
-
Rename local names? Defaults to False.
|
|
93
|
-
--mini_lits=MINI_LITS
|
|
94
|
-
Type: bool
|
|
95
|
-
Default: True
|
|
96
|
-
Hoist literal statements? Defaults to True.
|
|
97
|
-
--mini_annotations=MINI_ANNOTATIONS
|
|
98
|
-
Type: bool
|
|
99
|
-
Default: True
|
|
100
|
-
Remove annotations? Defaults to True.
|
|
101
|
-
--mini_asserts=MINI_ASSERTS
|
|
102
|
-
Type: bool
|
|
103
|
-
Default: True
|
|
104
|
-
Remove asserts? Defaults to True.
|
|
105
|
-
--mini_debug=MINI_DEBUG
|
|
106
|
-
Type: bool
|
|
107
|
-
Default: True
|
|
108
|
-
Remove debugging statements? Defaults to True.
|
|
109
|
-
--mini_imports=MINI_IMPORTS
|
|
110
|
-
Type: bool
|
|
111
|
-
Default: True
|
|
112
|
-
Combine imports? Defaults to True.
|
|
113
|
-
--mini_obj=MINI_OBJ
|
|
114
|
-
Type: bool
|
|
115
|
-
Default: True
|
|
116
|
-
Remove object base? Defaults to True.
|
|
117
|
-
--mini_pass=MINI_PASS
|
|
118
|
-
Type: bool
|
|
119
|
-
Default: True
|
|
120
|
-
Remove pass statements? Defaults to True.
|
|
121
|
-
--mini_posargs=MINI_POSARGS
|
|
122
|
-
Type: bool
|
|
123
|
-
Default: True
|
|
124
|
-
Convert positional to keyword args? Defaults to True.
|
|
125
|
-
--mini_retnone=MINI_RETNONE
|
|
126
|
-
Type: bool
|
|
127
|
-
Default: True
|
|
128
|
-
Remove explicit return None statements? Defaults to True.
|
|
129
|
-
--mini_shebang=MINI_SHEBANG
|
|
130
|
-
Type: bool
|
|
131
|
-
Default: True
|
|
132
|
-
Remove shebang? Defaults to True.
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
## Python usage
|
|
136
|
-
|
|
137
|
-
- **[See the API documentation](https://twardoch.github.io/split-python4gpt/API.html)** for more advanced usage
|
|
138
|
-
|
|
139
|
-
## Changelog
|
|
140
|
-
|
|
141
|
-
- v1.0.0: Initial release
|
|
142
|
-
|
|
143
|
-
## Contributing
|
|
144
|
-
|
|
145
|
-
Contributions to `split_python4gpt` are welcome! Please open an issue or submit a pull request on the [GitHub repository](https://github.com/twardoch/split-python4gpt).
|
|
146
|
-
|
|
147
|
-
## License
|
|
148
|
-
|
|
149
|
-
- Copyright (c) 2023 [Adam Twardoch](./AUTHORS.md)
|
|
150
|
-
- Written with assistance from ChatGPT
|
|
151
|
-
- Licensed under the [Apache License 2.0](./LICENSE.txt)<a id="split_python4gpt"></a>
|
|
152
|
-
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
split_python4gpt/__init__.py,sha256=F200hMu1S5x-eG-PVE-qpxVahijxKU-2g3GN2pkmRVI,616
|
|
2
|
-
split_python4gpt/__main__.py,sha256=ABi78muoBppCOW9wvz3BBHApOb97VSaYEfowdzlE-Dc,3150
|
|
3
|
-
split_python4gpt/minifier.py,sha256=KyCJbpfrLtELN_oyDRrD9i0TjfebrhLzkX7-4XAH_PI,5144
|
|
4
|
-
split_python4gpt-1.0.3.dist-info/LICENSE.txt,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
|
5
|
-
split_python4gpt-1.0.3.dist-info/METADATA,sha256=QdXkNJ3QWqaw-n2PTECR2Q9aWkSy0IC9atlWYKtTjvQ,5196
|
|
6
|
-
split_python4gpt-1.0.3.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
7
|
-
split_python4gpt-1.0.3.dist-info/entry_points.txt,sha256=wdxVWTgbPsCKmnN_e8XeLuCjzcUfh1SQDbaizEaD0o4,62
|
|
8
|
-
split_python4gpt-1.0.3.dist-info/top_level.txt,sha256=C4lm52jvftRWovJ0n725kZAGTiwxhuovvGFEUz_Kzys,17
|
|
9
|
-
split_python4gpt-1.0.3.dist-info/RECORD,,
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
split_python4gpt
|
|
File without changes
|
|
File without changes
|