rpgsync 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rpgsync/__init__.py +8 -0
- rpgsync/__main__.py +3 -0
- rpgsync/cli.py +1242 -0
- rpgsync/commands.py +920 -0
- rpgsync/compat.py +203 -0
- rpgsync/database.py +1025 -0
- rpgsync/db.py +1377 -0
- rpgsync/dbschema.py +1011 -0
- rpgsync/dsl.py +2616 -0
- rpgsync/dynparams.py +357 -0
- rpgsync/dynrpg.py +612 -0
- rpgsync/fmt.py +79 -0
- rpgsync/lcf.py +826 -0
- rpgsync/maniac.py +387 -0
- rpgsync/maptree.py +888 -0
- rpgsync/messages.py +113 -0
- rpgsync/named.py +1051 -0
- rpgsync/project.py +323 -0
- rpgsync/py.typed +0 -0
- rpgsync/pyexpr.py +1472 -0
- rpgsync/resources.py +455 -0
- rpgsync/scaffold.py +567 -0
- rpgsync/script.py +3532 -0
- rpgsync/scriptfiles.py +924 -0
- rpgsync/sync.py +1365 -0
- rpgsync/tables.py +114 -0
- rpgsync/venv.py +102 -0
- rpgsync-0.1.0.dist-info/METADATA +288 -0
- rpgsync-0.1.0.dist-info/RECORD +31 -0
- rpgsync-0.1.0.dist-info/WHEEL +4 -0
- rpgsync-0.1.0.dist-info/entry_points.txt +3 -0
rpgsync/scriptfiles.py
ADDED
|
@@ -0,0 +1,924 @@
|
|
|
1
|
+
"""A script written as one file, or as a folder of files when it is too big for one.
|
|
2
|
+
|
|
3
|
+
GitLab shows no diff of a file whose patch is over 200 KB (and collapses the ones over
|
|
4
|
+
20 KB): a script bigger than ``max_file_kb`` ([tool.rpgsync] of Scripts/pyproject.toml,
|
|
5
|
+
160 by default) is written as a Python package instead::
|
|
6
|
+
|
|
7
|
+
database/common_events/__init__.py its header, Config and instance
|
|
8
|
+
database/common_events/events_0001_0036.py class Events_0001_0036(CommonEventTable): ...
|
|
9
|
+
database/common_events/events_0037_0097.py class Events_0037_0097(CommonEventTable): ...
|
|
10
|
+
|
|
11
|
+
The class of __init__.py inherits from the class of every part, so ty and Pylance find
|
|
12
|
+
the entries where they found them before (``common_events.mr_clean``, ``variables.gold``),
|
|
13
|
+
and every ``from database.common_events import common_events`` keeps working. The
|
|
14
|
+
events of a map are at the top level of its parts (Map0195/events_0001_0040.py); a part
|
|
15
|
+
imports the events of the other parts it names.
|
|
16
|
+
|
|
17
|
+
rpgsync itself sees one text: the script as it would be in one file (the "joined" text).
|
|
18
|
+
A folder is joined when it is read, with a line map back to the files, and split again
|
|
19
|
+
when it is written. Every .py file of the folder is a part, in the order of their names.
|
|
20
|
+
What is above the class of a part, and the imports of the parts and the class statement
|
|
21
|
+
of __init__.py, are rpgsync's; the rest is yours. Writing a script into its folder keeps
|
|
22
|
+
every entry in its file and in its place: a new entry goes after the one before it (by
|
|
23
|
+
id), an emptied part is removed, and a part grown over the limit is cut in two. The
|
|
24
|
+
first split cuts the script by size, in id order, into parts named after their ids.
|
|
25
|
+
Parts are never renamed nor merged back; a part holding one entry bigger than the limit
|
|
26
|
+
stays as it is.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import ast
|
|
32
|
+
import bisect
|
|
33
|
+
import io
|
|
34
|
+
import os
|
|
35
|
+
import re
|
|
36
|
+
import tokenize
|
|
37
|
+
from collections.abc import Callable
|
|
38
|
+
|
|
39
|
+
from pydantic import BaseModel
|
|
40
|
+
|
|
41
|
+
KB = 1024
|
|
42
|
+
MAX_FILE_KB = 160 # default limit (GitLab shows no diff of a file over 200 KB)
|
|
43
|
+
FILL = 0.75 # a new part is filled up to this share of the limit: it has room to grow
|
|
44
|
+
LINE_LENGTH = 120
|
|
45
|
+
|
|
46
|
+
MARK = (
|
|
47
|
+
"# Its entries are in the other files of this folder, in the order of their names: rpgsync",
|
|
48
|
+
"# splits a script bigger than %d KB (GitLab shows no diff of a bigger file). Move them freely.",
|
|
49
|
+
)
|
|
50
|
+
_MARK_2 = re.compile(re.escape(MARK[1]).replace("%d", r"\d+"))
|
|
51
|
+
_CLASS = re.compile(r"class (\w+)\((\w+)\):\s*(?:#.*)?$")
|
|
52
|
+
_TABLE_ENTRY = re.compile(r" [A-Za-z_]\w*\s*(?::[^=]*)?=(?!=)")
|
|
53
|
+
_POSITIONAL_ID = re.compile(r"\s*\w+\s*(?::[^=]*)?=\s*\w+\(\s*(\d+)\s*[,)]")
|
|
54
|
+
_FIRST_KEYWORD_ID = re.compile(r"=\s*\w+\(\s*id=(\d+)\b") # potion = Item(id=5, ...)
|
|
55
|
+
_NEXT_LINE_ID = re.compile(r"\s+id=(\d+),") # its first argument, on the next line
|
|
56
|
+
_TOP_DEF = re.compile(r"(?:class|def) (\w+)\b")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class ChangedMeanwhile(Exception):
|
|
60
|
+
"""A file changed while rpgsync was syncing it (an edit in the editor or in the
|
|
61
|
+
script): nothing more is written, the next pass syncs both changes."""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class LayoutError(Exception):
|
|
65
|
+
"""A script folder rpgsync cannot read as one script."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, path: str, message: str, line: int | None = None):
|
|
68
|
+
self.path = path
|
|
69
|
+
self.message = message
|
|
70
|
+
self.line = line
|
|
71
|
+
super().__init__("%s%s: %s" % (path, ":%d" % line if line else "", message))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Style(BaseModel):
|
|
75
|
+
"""How the entries of a kind of script are laid out."""
|
|
76
|
+
|
|
77
|
+
wrapped: bool # in a class (common events, troops, tables), else at the top level (maps)
|
|
78
|
+
decorator: str | None = None # entries are `@decorator(id, ...)`; None: `name = Model(id=...)`
|
|
79
|
+
prefix: str # names of the part files: <prefix>_<first id>_<last id>.py
|
|
80
|
+
sep: int # blank lines between two entries
|
|
81
|
+
self_import: str | None = None # what a part imports to name its own script's instance
|
|
82
|
+
event_refs: bool = False # maps: a part imports the event classes it names from the others
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def indent(self) -> str:
|
|
86
|
+
return " " if self.wrapped else ""
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
COMMON_EVENTS = Style(
|
|
90
|
+
wrapped=True,
|
|
91
|
+
decorator="common_event",
|
|
92
|
+
prefix="events",
|
|
93
|
+
sep=1,
|
|
94
|
+
self_import="from database.common_events import common_events",
|
|
95
|
+
)
|
|
96
|
+
TROOP_EVENTS = Style(wrapped=True, decorator="troop", prefix="troops", sep=1)
|
|
97
|
+
MAP = Style(wrapped=False, decorator="event", prefix="events", sep=2, event_refs=True)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def table_style(table: str) -> Style:
|
|
101
|
+
"""A database table: `name = Model(id=...)` entries, no blank line between them."""
|
|
102
|
+
return Style(wrapped=True, prefix=table, sep=0)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class Part(BaseModel):
|
|
106
|
+
"""A part of a script folder, as read."""
|
|
107
|
+
|
|
108
|
+
path: str
|
|
109
|
+
cls: str | None # its class (None: a map's part)
|
|
110
|
+
first: int # the lines of its entries in the joined text: [first, end) (0-based)
|
|
111
|
+
end: int
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class ScriptFiles(BaseModel):
|
|
115
|
+
"""A script as read: its joined text, the bytes of each of its files (to tell
|
|
116
|
+
whether they changed since), and where each line of the text comes from."""
|
|
117
|
+
|
|
118
|
+
path: str # the script's path as one file (database/common_events.py)
|
|
119
|
+
text: str
|
|
120
|
+
raw: dict[str, bytes]
|
|
121
|
+
package: bool = False
|
|
122
|
+
parts: list[Part] = []
|
|
123
|
+
spans: list[tuple[int, str, int, int]] = [] # (joined line index, file, its line number, count)
|
|
124
|
+
|
|
125
|
+
def where(self, lineno: int | None) -> tuple[str, int | None]:
|
|
126
|
+
"""The file and line of a line of the joined text (1-based)."""
|
|
127
|
+
if lineno is None:
|
|
128
|
+
return (package_dir(self.path) if self.package else self.path, None)
|
|
129
|
+
if not self.spans:
|
|
130
|
+
return (self.path, lineno)
|
|
131
|
+
n = bisect.bisect_right(self.spans, (lineno - 1, "")) - 1
|
|
132
|
+
start, path, line, count = self.spans[max(n, 0)]
|
|
133
|
+
return (path, line + min(lineno - 1 - start, max(count - 1, 0)))
|
|
134
|
+
|
|
135
|
+
def entry_files(self, style: Style | None) -> dict[int, str]:
|
|
136
|
+
"""id -> the file of each entry, for a script split in files."""
|
|
137
|
+
lay = layout(self.text, style) if self.package and style is not None else None
|
|
138
|
+
if lay is None:
|
|
139
|
+
return {}
|
|
140
|
+
return {c.id: self.where(c.start + 1)[0] for c in lay.chunks if c.id is not None}
|
|
141
|
+
|
|
142
|
+
def at(self, lineno: int | None) -> str:
|
|
143
|
+
"""The file and line of a line of the joined text, as path:line."""
|
|
144
|
+
path, line = self.where(lineno)
|
|
145
|
+
return "%s:%d" % (path, line) if line else path
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# --------------------------------------------------------------------------
|
|
149
|
+
# Files
|
|
150
|
+
# --------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def max_file_kb(script_dir: str) -> int:
|
|
154
|
+
"""[tool.rpgsync] max_file_kb of a scripts folder's pyproject.toml: a script bigger
|
|
155
|
+
than this is split in files (0: never)."""
|
|
156
|
+
from .compat import load_config
|
|
157
|
+
|
|
158
|
+
value = load_config(script_dir).get("max_file_kb", MAX_FILE_KB)
|
|
159
|
+
return value if isinstance(value, int) and value >= 0 else MAX_FILE_KB
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def package_dir(py_path: str) -> str:
|
|
163
|
+
return py_path[:-3] if py_path.endswith(".py") else py_path
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _package_files(py_path: str) -> list[str]:
|
|
167
|
+
folder = package_dir(py_path)
|
|
168
|
+
try:
|
|
169
|
+
names = os.listdir(folder)
|
|
170
|
+
except OSError:
|
|
171
|
+
return []
|
|
172
|
+
return sorted(os.path.join(folder, n) for n in names if n.endswith(".py") and not n.startswith("."))
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def is_package(py_path: str) -> bool:
|
|
176
|
+
return bool(_package_files(py_path))
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def exists(py_path: str) -> bool:
|
|
180
|
+
return os.path.exists(py_path) or is_package(py_path)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def paths(py_path: str) -> list[str]:
|
|
184
|
+
"""The files of a script now (one file, or those of its folder), its path as one
|
|
185
|
+
file first even when missing: what to watch."""
|
|
186
|
+
return [py_path, *_package_files(py_path)]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _stat(path: str):
|
|
190
|
+
try:
|
|
191
|
+
st = os.stat(path)
|
|
192
|
+
return (st.st_mtime_ns, st.st_size)
|
|
193
|
+
except OSError:
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def stat_key(py_path: str) -> tuple:
|
|
198
|
+
"""Changes when a file of the script changes, appears or goes."""
|
|
199
|
+
return tuple((p, _stat(p)) for p in paths(py_path))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def read_bytes(path: str) -> bytes | None:
|
|
203
|
+
try:
|
|
204
|
+
with open(path, "rb") as f:
|
|
205
|
+
return f.read()
|
|
206
|
+
except FileNotFoundError:
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def write_atomic(path: str, data: bytes) -> None:
|
|
211
|
+
tmp = path + ".rpgsync-tmp"
|
|
212
|
+
with open(tmp, "wb") as f:
|
|
213
|
+
f.write(data)
|
|
214
|
+
os.replace(tmp, path)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _norm(data: bytes) -> str:
|
|
218
|
+
return data.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def unchanged(py_path: str, expect: ScriptFiles | None) -> bool:
|
|
222
|
+
"""Are the files of the script still the ones read (`expect`, None: no script)?"""
|
|
223
|
+
current = [p for p in paths(py_path) if os.path.exists(p)]
|
|
224
|
+
if expect is None:
|
|
225
|
+
return not current
|
|
226
|
+
return set(current) == set(expect.raw) and all(read_bytes(p) == expect.raw[p] for p in current)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def check_unchanged(py_path: str, expect: ScriptFiles | None) -> None:
|
|
230
|
+
if not unchanged(py_path, expect):
|
|
231
|
+
raise ChangedMeanwhile(os.path.basename(package_dir(py_path) if is_package(py_path) else py_path))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
# --------------------------------------------------------------------------
|
|
235
|
+
# Reading
|
|
236
|
+
# --------------------------------------------------------------------------
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def read(py_path: str, style: Style | None) -> ScriptFiles | None:
|
|
240
|
+
"""The script (None: there is none). Raises LayoutError for a folder that is not
|
|
241
|
+
one script, ChangedMeanwhile when its files keep changing while they are read."""
|
|
242
|
+
for _ in range(5):
|
|
243
|
+
before = stat_key(py_path)
|
|
244
|
+
found = _read(py_path, style)
|
|
245
|
+
if stat_key(py_path) == before: # a git checkout, a refactoring over several files...
|
|
246
|
+
return found
|
|
247
|
+
raise ChangedMeanwhile(os.path.basename(py_path))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def read_text(py_path: str, style: Style | None) -> str | None:
|
|
251
|
+
found = read(py_path, style)
|
|
252
|
+
return None if found is None else found.text
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _read(py_path: str, style: Style | None) -> ScriptFiles | None:
|
|
256
|
+
files = _package_files(py_path)
|
|
257
|
+
single = read_bytes(py_path)
|
|
258
|
+
if not files:
|
|
259
|
+
if single is None:
|
|
260
|
+
return None
|
|
261
|
+
return ScriptFiles(path=py_path, text=_norm(single), raw={py_path: single})
|
|
262
|
+
folder = package_dir(py_path)
|
|
263
|
+
if single is not None:
|
|
264
|
+
raise LayoutError(py_path, "the folder %s/ holds this script too: keep one of them" % os.path.basename(folder))
|
|
265
|
+
if style is None:
|
|
266
|
+
raise LayoutError(folder, "this script cannot be split: keep it in %s" % os.path.basename(py_path))
|
|
267
|
+
init = os.path.join(folder, "__init__.py")
|
|
268
|
+
if init not in files:
|
|
269
|
+
raise LayoutError(folder, "a script folder needs its __init__.py")
|
|
270
|
+
raw = {}
|
|
271
|
+
for p in files:
|
|
272
|
+
data = read_bytes(p)
|
|
273
|
+
if data is None:
|
|
274
|
+
raise ChangedMeanwhile(os.path.basename(p))
|
|
275
|
+
raw[p] = data
|
|
276
|
+
texts = {}
|
|
277
|
+
for p, data in raw.items():
|
|
278
|
+
try:
|
|
279
|
+
texts[p] = _norm(data)
|
|
280
|
+
except UnicodeDecodeError as e:
|
|
281
|
+
raise LayoutError(p, "not UTF-8 text (%s)" % e) from None
|
|
282
|
+
return join(py_path, style, texts, raw)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _owned_init_lines(lines: list[str], tree: ast.Module) -> set[int]:
|
|
286
|
+
"""Indexes of the lines of an __init__.py that are rpgsync's: the note on the parts,
|
|
287
|
+
the imports of the parts, the class statement."""
|
|
288
|
+
owned = set()
|
|
289
|
+
for stmt in tree.body:
|
|
290
|
+
if isinstance(stmt, ast.ImportFrom) and stmt.level:
|
|
291
|
+
owned.update(range(stmt.lineno - 1, (stmt.end_lineno or stmt.lineno)))
|
|
292
|
+
for n, line in enumerate(lines):
|
|
293
|
+
if line == MARK[0] or _MARK_2.fullmatch(line):
|
|
294
|
+
owned.add(n)
|
|
295
|
+
if line == MARK[0] and n and not lines[n - 1].strip():
|
|
296
|
+
owned.add(n - 1) # the blank line rpgsync puts above it
|
|
297
|
+
return owned
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _part_body(path: str, lines: list[str], style: Style) -> tuple[str | None, str | None, int]:
|
|
301
|
+
"""(class, its base, index of the first line of the entries) of a part file."""
|
|
302
|
+
if style.wrapped:
|
|
303
|
+
classes = [n for n, line in enumerate(lines) if line.startswith("class ")]
|
|
304
|
+
if len(classes) != 1:
|
|
305
|
+
raise LayoutError(path, "a part holds one class (its entries), it has %d" % len(classes))
|
|
306
|
+
c = classes[0]
|
|
307
|
+
m = _CLASS.match(lines[c])
|
|
308
|
+
if not m:
|
|
309
|
+
raise LayoutError(path, "the class of a part is written `class Name(Base):`", c + 1)
|
|
310
|
+
_check_header(path, lines[:c])
|
|
311
|
+
for n in range(c + 1, len(lines)):
|
|
312
|
+
if lines[n][:1] not in ("", " ", "\t", "#"):
|
|
313
|
+
raise LayoutError(path, "a part holds only its class: this line is outside it", n + 1)
|
|
314
|
+
return m.group(1), m.group(2), c + 1
|
|
315
|
+
end = _header_end(lines)
|
|
316
|
+
_check_header(path, lines[:end])
|
|
317
|
+
return None, None, end
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _check_header(path: str, lines: list[str]) -> None:
|
|
321
|
+
"""The lines above the entries of a part: a docstring, comments and imports only."""
|
|
322
|
+
try:
|
|
323
|
+
tree = ast.parse("\n".join(lines))
|
|
324
|
+
except SyntaxError as e:
|
|
325
|
+
raise LayoutError(path, "syntax error: %s" % e.msg, e.lineno) from None
|
|
326
|
+
for n, stmt in enumerate(tree.body):
|
|
327
|
+
if isinstance(stmt, (ast.Import, ast.ImportFrom)):
|
|
328
|
+
continue
|
|
329
|
+
if n == 0 and isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant):
|
|
330
|
+
continue
|
|
331
|
+
raise LayoutError(path, "above its entries, a part holds only its docstring and imports", stmt.lineno)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _header_end(lines: list[str]) -> int:
|
|
335
|
+
"""Index of the first line after the docstring and imports a map part starts with
|
|
336
|
+
(comments right above the first event belong to it)."""
|
|
337
|
+
n, end = 0, 0
|
|
338
|
+
while n < len(lines) and not lines[n].strip():
|
|
339
|
+
n += 1
|
|
340
|
+
if n < len(lines) and lines[n].lstrip().startswith(('"""', "'''", '"', "'")):
|
|
341
|
+
q = lines[n].lstrip()[:3] if lines[n].lstrip()[:3] in ('"""', "'''") else lines[n].lstrip()[:1]
|
|
342
|
+
rest = lines[n].lstrip()[len(q) :]
|
|
343
|
+
while q not in rest and n + 1 < len(lines):
|
|
344
|
+
n += 1
|
|
345
|
+
rest = lines[n]
|
|
346
|
+
n += 1
|
|
347
|
+
end = n
|
|
348
|
+
while n < len(lines):
|
|
349
|
+
line = lines[n]
|
|
350
|
+
if not line.strip() or line.startswith("#"):
|
|
351
|
+
n += 1
|
|
352
|
+
continue
|
|
353
|
+
if line.startswith(("from ", "import ")):
|
|
354
|
+
depth = line.count("(") - line.count(")")
|
|
355
|
+
n += 1
|
|
356
|
+
while depth > 0 and n < len(lines):
|
|
357
|
+
depth += lines[n].count("(") - lines[n].count(")")
|
|
358
|
+
n += 1
|
|
359
|
+
end = n
|
|
360
|
+
continue
|
|
361
|
+
break
|
|
362
|
+
while end < len(lines) and not lines[end].strip():
|
|
363
|
+
end += 1
|
|
364
|
+
return end
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _strip(lines: list[str]) -> list[str]:
|
|
368
|
+
a, b = 0, len(lines)
|
|
369
|
+
while a < b and not lines[a].strip():
|
|
370
|
+
a += 1
|
|
371
|
+
while b > a and not lines[b - 1].strip():
|
|
372
|
+
b -= 1
|
|
373
|
+
return lines[a:b]
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def join(py_path: str, style: Style, texts: dict[str, str], raw: dict[str, bytes] | None = None) -> ScriptFiles:
|
|
377
|
+
"""The script of a folder's files (path -> text) as one text."""
|
|
378
|
+
folder = package_dir(py_path)
|
|
379
|
+
init = os.path.join(folder, "__init__.py")
|
|
380
|
+
init_lines = texts[init].split("\n")
|
|
381
|
+
try:
|
|
382
|
+
tree = ast.parse(texts[init])
|
|
383
|
+
except SyntaxError as e:
|
|
384
|
+
raise LayoutError(init, "syntax error: %s" % e.msg, e.lineno) from None
|
|
385
|
+
owned = _owned_init_lines(init_lines, tree)
|
|
386
|
+
|
|
387
|
+
# the parts, in the order of their names
|
|
388
|
+
parts = []
|
|
389
|
+
for path in sorted(p for p in texts if p != init):
|
|
390
|
+
lines = texts[path].split("\n")
|
|
391
|
+
cls, base, start = _part_body(path, lines, style)
|
|
392
|
+
body = lines[start:]
|
|
393
|
+
a = 0
|
|
394
|
+
while a < len(body) and not body[a].strip():
|
|
395
|
+
a += 1
|
|
396
|
+
parts.append((path, cls, base, start + a + 1, _strip(body))) # (line number of its first entry line)
|
|
397
|
+
names = [cls for _, cls, _, _, _ in parts if cls]
|
|
398
|
+
if len(set(names)) != len(names):
|
|
399
|
+
raise LayoutError(folder, "two parts have the same class name")
|
|
400
|
+
bases = {base for _, _, base, _, _ in parts if base}
|
|
401
|
+
if len(bases) > 1:
|
|
402
|
+
raise LayoutError(folder, "the classes of the parts have different bases: %s" % ", ".join(sorted(bases)))
|
|
403
|
+
|
|
404
|
+
out: list[str] = []
|
|
405
|
+
spans: list[tuple[int, str, int, int]] = []
|
|
406
|
+
|
|
407
|
+
def emit(lines: list[str], path: str, line: int) -> None: # line: of lines[0] in its file (1-based)
|
|
408
|
+
if lines:
|
|
409
|
+
spans.append((len(out), path, line, len(lines)))
|
|
410
|
+
out.extend(lines)
|
|
411
|
+
|
|
412
|
+
def emit_init(a: int, b: int) -> None: # the lines [a, b) of __init__.py that are not rpgsync's
|
|
413
|
+
run = a
|
|
414
|
+
for n in range(a, b + 1):
|
|
415
|
+
if n == b or n in owned:
|
|
416
|
+
emit(init_lines[run:n], init, run + 1)
|
|
417
|
+
run = n + 1
|
|
418
|
+
|
|
419
|
+
entries = any(body for *_, body in parts)
|
|
420
|
+
result_parts = []
|
|
421
|
+
if style.wrapped:
|
|
422
|
+
classes = [stmt for stmt in tree.body if isinstance(stmt, ast.ClassDef)]
|
|
423
|
+
if len(classes) != 1:
|
|
424
|
+
raise LayoutError(init, "__init__.py holds one class (made of the parts), it has %d" % len(classes))
|
|
425
|
+
cdef = classes[0]
|
|
426
|
+
if bases:
|
|
427
|
+
base = next(iter(bases))
|
|
428
|
+
elif cdef.bases and isinstance(cdef.bases[-1], ast.Name):
|
|
429
|
+
base = cdef.bases[-1].id
|
|
430
|
+
else:
|
|
431
|
+
raise LayoutError(init, "the class must end with the base of the script's class", cdef.lineno)
|
|
432
|
+
c = cdef.lineno - 1 # the class statement: up to its first body line
|
|
433
|
+
first = cdef.body[0].lineno - 1 if cdef.body else c + 1
|
|
434
|
+
while first > c + 1 and init_lines[first - 1].strip().startswith("#"):
|
|
435
|
+
first -= 1 # comments above the first statement of the body are its
|
|
436
|
+
t = next(
|
|
437
|
+
(n for n in range(first, len(init_lines)) if init_lines[n][:1] not in ("", " ", "\t", "#")),
|
|
438
|
+
len(init_lines),
|
|
439
|
+
)
|
|
440
|
+
end = t
|
|
441
|
+
while end > first and not init_lines[end - 1].strip():
|
|
442
|
+
end -= 1
|
|
443
|
+
emit_init(0, c)
|
|
444
|
+
spans.append((len(out), init, c + 1, 1))
|
|
445
|
+
out.append("class %s(%s):" % (cdef.name, base))
|
|
446
|
+
pre = init_lines[first:end]
|
|
447
|
+
if entries and [line.strip() for line in pre] == ["pass"]:
|
|
448
|
+
pre = [] # the body of a class made of parts only
|
|
449
|
+
emit(pre, init, first + 1)
|
|
450
|
+
lead = 1 if pre else 0
|
|
451
|
+
for path, cls, _, line, body in parts:
|
|
452
|
+
if not body:
|
|
453
|
+
continue
|
|
454
|
+
gap = lead if not result_parts else style.sep
|
|
455
|
+
out.extend([""] * gap)
|
|
456
|
+
result_parts.append(Part(path=path, cls=cls, first=len(out), end=len(out) + len(body)))
|
|
457
|
+
emit(body, path, line)
|
|
458
|
+
emit_init(end, len(init_lines))
|
|
459
|
+
else:
|
|
460
|
+
end = max((n + 1 for n, line in enumerate(init_lines) if line.strip() and n not in owned), default=0)
|
|
461
|
+
emit_init(0, end)
|
|
462
|
+
for path, cls, _, line, body in parts:
|
|
463
|
+
if not body:
|
|
464
|
+
continue
|
|
465
|
+
out.extend([""] * style.sep)
|
|
466
|
+
result_parts.append(Part(path=path, cls=cls, first=len(out), end=len(out) + len(body)))
|
|
467
|
+
emit(body, path, line)
|
|
468
|
+
out.append("")
|
|
469
|
+
if raw is None:
|
|
470
|
+
raw = {p: t.encode("utf-8") for p, t in texts.items()}
|
|
471
|
+
return ScriptFiles(path=py_path, text="\n".join(out), raw=raw, package=True, parts=result_parts, spans=spans)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# --------------------------------------------------------------------------
|
|
475
|
+
# The entries of a text
|
|
476
|
+
# --------------------------------------------------------------------------
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
class Chunk(BaseModel):
|
|
480
|
+
"""An entry and the comments and blank lines above it: lines [lead, end), the
|
|
481
|
+
entry itself starting at `start`."""
|
|
482
|
+
|
|
483
|
+
lead: int
|
|
484
|
+
start: int
|
|
485
|
+
end: int
|
|
486
|
+
id: int | None
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
class Layout(BaseModel):
|
|
490
|
+
"""A joined text cut into its header, its class statement, what the class holds
|
|
491
|
+
before its entries (Config), its entries, and what follows them."""
|
|
492
|
+
|
|
493
|
+
lines: list[str]
|
|
494
|
+
header: list[str] # up to the class statement (maps: up to the first entry)
|
|
495
|
+
cls: tuple[str, str] | None # (name, base) of the class
|
|
496
|
+
pre: list[str]
|
|
497
|
+
chunks: list[Chunk]
|
|
498
|
+
tail: list[str]
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _is_start(line: str, style: Style) -> bool:
|
|
502
|
+
if style.decorator:
|
|
503
|
+
return line.startswith(style.indent + "@" + style.decorator + "(")
|
|
504
|
+
return bool(_TABLE_ENTRY.match(line))
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _entry_id(lines: list[str], start: int, style: Style) -> int | None:
|
|
508
|
+
if style.decorator:
|
|
509
|
+
m = re.match(r"\s*@" + re.escape(style.decorator) + r"\(\s*(\d+)\s*[,)]", "\n".join(lines[start : start + 3]))
|
|
510
|
+
return int(m.group(1)) if m else None
|
|
511
|
+
m = _POSITIONAL_ID.match(lines[start]) or _FIRST_KEYWORD_ID.search(lines[start])
|
|
512
|
+
if m is None and start + 1 < len(lines):
|
|
513
|
+
m = _NEXT_LINE_ID.match(lines[start + 1])
|
|
514
|
+
return int(m.group(1)) if m else None
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _leading(lines: list[str], start: int, floor: int, indent: str) -> int:
|
|
518
|
+
"""Index of the first of the blank lines and comments right above `start` (not
|
|
519
|
+
above `floor`)."""
|
|
520
|
+
k = start
|
|
521
|
+
while k - 1 >= floor:
|
|
522
|
+
line = lines[k - 1]
|
|
523
|
+
s = line.lstrip()
|
|
524
|
+
if s and not (s.startswith("#") and len(line) - len(s) <= len(indent)):
|
|
525
|
+
break
|
|
526
|
+
k -= 1
|
|
527
|
+
return k
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def layout(text: str, style: Style) -> Layout | None:
|
|
531
|
+
"""The parts of a joined text (maybe no entries); None without its class."""
|
|
532
|
+
lines = text.split("\n")
|
|
533
|
+
indent = style.indent
|
|
534
|
+
if style.wrapped:
|
|
535
|
+
c = next((n for n, line in enumerate(lines) if line.startswith("class ")), None)
|
|
536
|
+
if c is None or not _CLASS.match(lines[c]):
|
|
537
|
+
return None
|
|
538
|
+
m = _CLASS.match(lines[c])
|
|
539
|
+
t = next((n for n in range(c + 1, len(lines)) if lines[n][:1] not in ("", " ", "\t", "#")), len(lines))
|
|
540
|
+
end = t
|
|
541
|
+
while end > c + 1 and not lines[end - 1].strip():
|
|
542
|
+
end -= 1
|
|
543
|
+
floor = c + 1
|
|
544
|
+
if not style.decorator: # a table: its Config is not an entry
|
|
545
|
+
f = floor
|
|
546
|
+
while f < end and not lines[f].strip():
|
|
547
|
+
f += 1
|
|
548
|
+
if f < end and lines[f].startswith(" class Config"):
|
|
549
|
+
f += 1
|
|
550
|
+
while f < end and (not lines[f].strip() or lines[f].startswith(" ")):
|
|
551
|
+
f += 1
|
|
552
|
+
floor = f
|
|
553
|
+
elif f < end and re.match(r" size\s*=", lines[f]):
|
|
554
|
+
floor = f + 1
|
|
555
|
+
starts = [n for n in range(floor, end) if _is_start(lines[n], style)]
|
|
556
|
+
cls = (m.group(1), m.group(2)) # type: ignore[union-attr]
|
|
557
|
+
else:
|
|
558
|
+
end = len(lines)
|
|
559
|
+
while end > 0 and not lines[end - 1].strip():
|
|
560
|
+
end -= 1
|
|
561
|
+
starts = [n for n in range(end) if _is_start(lines[n], style)]
|
|
562
|
+
c, floor, cls = -1, 0, None
|
|
563
|
+
leads = [_leading(lines, s, starts[k - 1] + 1 if k else floor, indent) for k, s in enumerate(starts)]
|
|
564
|
+
chunks = [
|
|
565
|
+
Chunk(lead=leads[k], start=s, end=leads[k + 1] if k + 1 < len(starts) else end, id=_entry_id(lines, s, style))
|
|
566
|
+
for k, s in enumerate(starts)
|
|
567
|
+
]
|
|
568
|
+
first = leads[0] if leads else end
|
|
569
|
+
if style.wrapped:
|
|
570
|
+
header, pre = lines[:c], _strip(lines[c + 1 : first])
|
|
571
|
+
else:
|
|
572
|
+
header, pre = lines[:first], []
|
|
573
|
+
return Layout(lines=lines, header=header, cls=cls, pre=pre, chunks=chunks, tail=lines[end:])
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _entry_text(lay: Layout, c: Chunk) -> str:
|
|
577
|
+
"""An entry's lines without the blank lines above it."""
|
|
578
|
+
lines = lay.lines[c.lead : c.end]
|
|
579
|
+
n = 0
|
|
580
|
+
while n < len(lines) and not lines[n].strip():
|
|
581
|
+
n += 1
|
|
582
|
+
return "\n".join(lines[n:])
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def equivalent(a: str, b: str, style: Style) -> bool:
|
|
586
|
+
"""The same script, whatever the order of its entries and the blank lines above
|
|
587
|
+
them: what splitting a script into files and joining them back may change."""
|
|
588
|
+
la, lb = layout(a, style), layout(b, style)
|
|
589
|
+
if la is None or lb is None:
|
|
590
|
+
return a == b
|
|
591
|
+
|
|
592
|
+
def key(lay: Layout):
|
|
593
|
+
header = _strip(lay.header)
|
|
594
|
+
return (header, lay.cls, _strip(lay.pre), sorted(_entry_text(lay, c) for c in lay.chunks), _strip(lay.tail))
|
|
595
|
+
|
|
596
|
+
return key(la) == key(lb)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
# --------------------------------------------------------------------------
|
|
600
|
+
# Writing
|
|
601
|
+
# --------------------------------------------------------------------------
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _imports(header: list[str]) -> list[str]:
|
|
605
|
+
"""The import statements of a script's header, as written there."""
|
|
606
|
+
try:
|
|
607
|
+
tree = ast.parse("\n".join(header))
|
|
608
|
+
except SyntaxError:
|
|
609
|
+
return []
|
|
610
|
+
out = []
|
|
611
|
+
for stmt in tree.body:
|
|
612
|
+
if isinstance(stmt, (ast.Import, ast.ImportFrom)) and not (isinstance(stmt, ast.ImportFrom) and stmt.level):
|
|
613
|
+
out += header[stmt.lineno - 1 : stmt.end_lineno]
|
|
614
|
+
return out
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _label(py_path: str) -> str:
|
|
618
|
+
folder = package_dir(py_path)
|
|
619
|
+
parent = os.path.basename(os.path.dirname(folder))
|
|
620
|
+
name = os.path.basename(folder)
|
|
621
|
+
return "%s/%s" % (parent, name) if parent == "database" else name
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def _import_line(module: str, names: list[str], line_length: int) -> list[str]:
|
|
625
|
+
one = "from .%s import %s" % (module, ", ".join(names))
|
|
626
|
+
if len(one) <= line_length:
|
|
627
|
+
return [one]
|
|
628
|
+
return ["from .%s import (" % module, *(" %s," % n for n in names), ")"]
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _class_stmt(name: str, bases: list[str], line_length: int) -> list[str]:
|
|
632
|
+
one = "class %s(%s):" % (name, ", ".join(bases))
|
|
633
|
+
if len(one) <= line_length:
|
|
634
|
+
return [one]
|
|
635
|
+
return ["class %s(" % name, *(" %s," % b for b in bases), "):"]
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _names_used(src: str) -> set[str]:
|
|
639
|
+
"""The names a map part uses (not as attributes)."""
|
|
640
|
+
out = set()
|
|
641
|
+
prev = None
|
|
642
|
+
try:
|
|
643
|
+
for tok in tokenize.generate_tokens(io.StringIO(src).readline):
|
|
644
|
+
if tok.type == tokenize.NAME and not (prev is not None and prev.type == tokenize.OP and prev.string == "."):
|
|
645
|
+
out.add(tok.string)
|
|
646
|
+
if tok.type not in (tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, tokenize.INDENT, tokenize.DEDENT):
|
|
647
|
+
prev = tok
|
|
648
|
+
except (tokenize.TokenError, SyntaxError, IndentationError):
|
|
649
|
+
pass
|
|
650
|
+
return out
|
|
651
|
+
|
|
652
|
+
|
|
653
|
+
class _Plan(BaseModel):
|
|
654
|
+
module: str
|
|
655
|
+
cls: str | None
|
|
656
|
+
chunks: list[int] # indexes in the layout's chunks
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def _module_name(style: Style, ids: list[int | None], taken: set[str]) -> str:
|
|
660
|
+
known = [i for i in ids if i is not None]
|
|
661
|
+
base = "%s_%04d_%04d" % (style.prefix, min(known), max(known)) if known else "%s_part" % style.prefix
|
|
662
|
+
name, n = base, 2
|
|
663
|
+
while name in taken:
|
|
664
|
+
name, n = "%s_%d" % (base, n), n + 1
|
|
665
|
+
return name
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def _class_name(module: str) -> str:
|
|
669
|
+
return module[:1].upper() + module[1:]
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def _group(sizes: list[int], items: list[int], budget: int, overhead: int) -> list[list[int]]:
|
|
673
|
+
"""Consecutive items in groups of at most `budget` bytes (one item at least)."""
|
|
674
|
+
groups: list[list[int]] = []
|
|
675
|
+
cur: list[int] = []
|
|
676
|
+
size = overhead
|
|
677
|
+
for i in items:
|
|
678
|
+
if cur and size + sizes[i] > budget:
|
|
679
|
+
groups.append(cur)
|
|
680
|
+
cur, size = [], overhead
|
|
681
|
+
cur.append(i)
|
|
682
|
+
size += sizes[i]
|
|
683
|
+
if cur:
|
|
684
|
+
groups.append(cur)
|
|
685
|
+
return groups
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _place(lay: Layout, old: ScriptFiles, old_lay: Layout | None) -> list[_Plan]:
|
|
689
|
+
"""The chunks of `lay` in the parts of `old`: each entry where it was."""
|
|
690
|
+
plans = [_Plan(module=os.path.basename(p.path)[:-3], cls=p.cls, chunks=[]) for p in old.parts]
|
|
691
|
+
if old_lay is None or not plans:
|
|
692
|
+
return plans
|
|
693
|
+
part_of = [] # part of each old chunk
|
|
694
|
+
for c in old_lay.chunks:
|
|
695
|
+
part_of.append(next((n for n, p in enumerate(old.parts) if p.first <= c.start < p.end), 0))
|
|
696
|
+
same = len(old_lay.chunks) == len(lay.chunks) and all(
|
|
697
|
+
a.lead == b.lead and a.start == b.start and (a.id is None or a.id == b.id)
|
|
698
|
+
for a, b in zip(old_lay.chunks, lay.chunks)
|
|
699
|
+
)
|
|
700
|
+
if same: # the same entries at the same lines (ids written in, labels updated...)
|
|
701
|
+
for n in range(len(lay.chunks)):
|
|
702
|
+
plans[part_of[n]].chunks.append(n)
|
|
703
|
+
return plans
|
|
704
|
+
where: dict[int, tuple[int, int]] = {} # id -> (part, rank in the part)
|
|
705
|
+
rank = [0] * len(plans)
|
|
706
|
+
for n, c in enumerate(old_lay.chunks):
|
|
707
|
+
if c.id is not None and c.id not in where:
|
|
708
|
+
where[c.id] = (part_of[n], rank[part_of[n]])
|
|
709
|
+
rank[part_of[n]] += 1
|
|
710
|
+
placed: dict[int, int] = {} # chunk -> part
|
|
711
|
+
known = sorted(
|
|
712
|
+
(where[c.id] + (n,) for n, c in enumerate(lay.chunks) if c.id is not None and c.id in where),
|
|
713
|
+
)
|
|
714
|
+
seen_ids: set[int] = set()
|
|
715
|
+
for p, _, n in known:
|
|
716
|
+
if lay.chunks[n].id in seen_ids:
|
|
717
|
+
continue
|
|
718
|
+
seen_ids.add(lay.chunks[n].id) # type: ignore[arg-type]
|
|
719
|
+
plans[p].chunks.append(n)
|
|
720
|
+
placed[n] = p
|
|
721
|
+
for n in range(len(lay.chunks)):
|
|
722
|
+
if n in placed:
|
|
723
|
+
continue
|
|
724
|
+
if n and n - 1 in placed: # after the entry before it
|
|
725
|
+
p = placed[n - 1]
|
|
726
|
+
plans[p].chunks.insert(plans[p].chunks.index(n - 1) + 1, n)
|
|
727
|
+
else:
|
|
728
|
+
p = 0
|
|
729
|
+
plans[0].chunks.insert(0, n)
|
|
730
|
+
placed[n] = p
|
|
731
|
+
return plans
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
def plan(
|
|
735
|
+
py_path: str,
|
|
736
|
+
style: Style | None,
|
|
737
|
+
text: str,
|
|
738
|
+
old: ScriptFiles | None,
|
|
739
|
+
max_kb: int,
|
|
740
|
+
line_length: int = LINE_LENGTH,
|
|
741
|
+
) -> dict[str, str] | None:
|
|
742
|
+
"""The files to write the script as: path -> text, None for one file (py_path)."""
|
|
743
|
+
if style is None:
|
|
744
|
+
return None
|
|
745
|
+
limit = max_kb * KB
|
|
746
|
+
folder = old is not None and old.package
|
|
747
|
+
if not folder and (not max_kb or len(text.encode("utf-8")) <= limit):
|
|
748
|
+
return None
|
|
749
|
+
lay = layout(text, style)
|
|
750
|
+
if lay is None and folder:
|
|
751
|
+
raise LayoutError(package_dir(py_path), "internal error: no class to split this script into")
|
|
752
|
+
if lay is None or (not folder and len(lay.chunks) < 2):
|
|
753
|
+
return None # no entries to split it into
|
|
754
|
+
pkg = package_dir(py_path)
|
|
755
|
+
label = _label(py_path)
|
|
756
|
+
imports = _imports(lay.header)
|
|
757
|
+
head = ['"""Part of %s: see __init__.py"""' % label, ""] + imports
|
|
758
|
+
if style.self_import and style.self_import not in imports:
|
|
759
|
+
head.append(style.self_import)
|
|
760
|
+
sizes = [len("\n".join(lay.lines[c.lead : c.end]).encode("utf-8")) + 1 for c in lay.chunks]
|
|
761
|
+
overhead = len("\n".join(head).encode("utf-8")) + 60
|
|
762
|
+
budget = int(limit * FILL) if max_kb else 1 << 60
|
|
763
|
+
|
|
764
|
+
if folder:
|
|
765
|
+
assert old is not None
|
|
766
|
+
plans = _place(lay, old, layout(old.text, style))
|
|
767
|
+
if not plans:
|
|
768
|
+
plans = [
|
|
769
|
+
_Plan(module="", cls=None, chunks=g)
|
|
770
|
+
for g in _group(sizes, list(range(len(lay.chunks))), budget, overhead)
|
|
771
|
+
]
|
|
772
|
+
else:
|
|
773
|
+
plans = [
|
|
774
|
+
_Plan(module="", cls=None, chunks=g) for g in _group(sizes, list(range(len(lay.chunks))), budget, overhead)
|
|
775
|
+
]
|
|
776
|
+
# cut the parts grown over the limit
|
|
777
|
+
out_plans: list[_Plan] = []
|
|
778
|
+
for p in plans:
|
|
779
|
+
if not p.chunks:
|
|
780
|
+
continue # emptied: removed
|
|
781
|
+
total = overhead + sum(sizes[n] for n in p.chunks)
|
|
782
|
+
if max_kb and total > limit and len(p.chunks) > 1:
|
|
783
|
+
groups = _group(sizes, p.chunks, budget, overhead)
|
|
784
|
+
out_plans.append(_Plan(module=p.module, cls=p.cls, chunks=groups[0]))
|
|
785
|
+
out_plans += [_Plan(module="", cls=None, chunks=g) for g in groups[1:]]
|
|
786
|
+
else:
|
|
787
|
+
out_plans.append(p)
|
|
788
|
+
taken = {p.module for p in out_plans if p.module}
|
|
789
|
+
for p in out_plans:
|
|
790
|
+
if not p.module:
|
|
791
|
+
p.module = _module_name(style, [lay.chunks[n].id for n in p.chunks], taken | {"__init__"})
|
|
792
|
+
taken.add(p.module)
|
|
793
|
+
if style.wrapped and not p.cls:
|
|
794
|
+
p.cls = _class_name(p.module)
|
|
795
|
+
out_plans.sort(key=lambda p: p.module)
|
|
796
|
+
|
|
797
|
+
files: dict[str, str] = {}
|
|
798
|
+
bodies = []
|
|
799
|
+
for p in out_plans:
|
|
800
|
+
body: list[str] = []
|
|
801
|
+
for k, n in enumerate(p.chunks):
|
|
802
|
+
c = lay.chunks[n]
|
|
803
|
+
lines = lay.lines[c.lead : c.end]
|
|
804
|
+
if k == 0:
|
|
805
|
+
while lines and not lines[0].strip():
|
|
806
|
+
lines = lines[1:]
|
|
807
|
+
body += lines
|
|
808
|
+
bodies.append(_strip(body) if body else body)
|
|
809
|
+
refs: list[list[str]] = [[] for _ in out_plans]
|
|
810
|
+
if style.event_refs:
|
|
811
|
+
defined = [{m.group(1) for line in body if (m := _TOP_DEF.match(line))} for body in bodies]
|
|
812
|
+
owner = {name: k for k, names in enumerate(defined) for name in names}
|
|
813
|
+
for k, body in enumerate(bodies):
|
|
814
|
+
used = _names_used("\n".join(body) + "\n")
|
|
815
|
+
by_part: dict[int, list[str]] = {}
|
|
816
|
+
for name in sorted(used):
|
|
817
|
+
if owner.get(name, k) != k:
|
|
818
|
+
by_part.setdefault(owner[name], []).append(name)
|
|
819
|
+
for j in sorted(by_part):
|
|
820
|
+
refs[k] += _import_line(out_plans[j].module, by_part[j], line_length)
|
|
821
|
+
for k, p in enumerate(out_plans):
|
|
822
|
+
lines = head + refs[k] + ["", ""]
|
|
823
|
+
if style.wrapped:
|
|
824
|
+
lines.append("class %s(%s):" % (p.cls, lay.cls[1])) # type: ignore[index]
|
|
825
|
+
lines += bodies[k] or ([style.indent + "pass"] if style.wrapped else [])
|
|
826
|
+
files[os.path.join(pkg, p.module + ".py")] = "\n".join(lines) + "\n"
|
|
827
|
+
|
|
828
|
+
header = list(lay.header)
|
|
829
|
+
gap: list[str] = []
|
|
830
|
+
while header and not header[-1].strip():
|
|
831
|
+
gap.append(header.pop())
|
|
832
|
+
block = ["", MARK[0], MARK[1] % max_kb if max_kb else MARK[1] % MAX_FILE_KB]
|
|
833
|
+
if style.wrapped:
|
|
834
|
+
block += ["from .%s import %s" % (p.module, p.cls) for p in out_plans]
|
|
835
|
+
name, base = lay.cls # type: ignore[misc]
|
|
836
|
+
init = header + block + (gap or ["", ""])
|
|
837
|
+
init += _class_stmt(name, [p.cls for p in out_plans] + [base], line_length) # type: ignore[misc]
|
|
838
|
+
init += lay.pre or [style.indent + "pass"]
|
|
839
|
+
init += lay.tail
|
|
840
|
+
else:
|
|
841
|
+
block += ["from . import %s" % p.module for p in out_plans]
|
|
842
|
+
init = header + block + [""]
|
|
843
|
+
files[os.path.join(pkg, "__init__.py")] = "\n".join(init) if init[-1] == "" else "\n".join(init) + "\n"
|
|
844
|
+
return files
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
def write(
|
|
848
|
+
py_path: str,
|
|
849
|
+
style: Style | None,
|
|
850
|
+
text: str,
|
|
851
|
+
old: ScriptFiles | None,
|
|
852
|
+
max_kb: int,
|
|
853
|
+
backup: Callable[[str], None],
|
|
854
|
+
line_length: int = LINE_LENGTH,
|
|
855
|
+
) -> ScriptFiles:
|
|
856
|
+
"""Write a script (one file, or the files of its folder when it is one or the text
|
|
857
|
+
is over the limit): only the files that change, each backed up first. `old`: the
|
|
858
|
+
script as read before (ChangedMeanwhile if its files changed since). -> the script
|
|
859
|
+
as now written (its text may differ from `text` by blank lines above entries)."""
|
|
860
|
+
check_unchanged(py_path, old)
|
|
861
|
+
files = plan(py_path, style, text, old, max_kb, line_length)
|
|
862
|
+
if files is not None:
|
|
863
|
+
assert style is not None
|
|
864
|
+
written = join(py_path, style, files)
|
|
865
|
+
if not equivalent(written.text, text, style):
|
|
866
|
+
if old is not None and old.package:
|
|
867
|
+
raise LayoutError(package_dir(py_path), "internal error: rpgsync cannot split this script back")
|
|
868
|
+
files = None # write it as one file
|
|
869
|
+
if files is None:
|
|
870
|
+
data = text.encode("utf-8")
|
|
871
|
+
if read_bytes(py_path) != data:
|
|
872
|
+
os.makedirs(os.path.dirname(py_path), exist_ok=True)
|
|
873
|
+
backup(py_path)
|
|
874
|
+
write_atomic(py_path, data)
|
|
875
|
+
return ScriptFiles(path=py_path, text=text, raw={py_path: data})
|
|
876
|
+
folder = package_dir(py_path)
|
|
877
|
+
os.makedirs(folder, exist_ok=True)
|
|
878
|
+
raw: dict[str, bytes] = {}
|
|
879
|
+
init = os.path.join(folder, "__init__.py")
|
|
880
|
+
current = {p: (old.raw.get(p) if old is not None else None) for p in files}
|
|
881
|
+
# new parts first, then the changed ones, then __init__.py, then the stale ones go:
|
|
882
|
+
# each step leaves a folder that reads as an error at worst, never as fewer entries
|
|
883
|
+
order = sorted(files, key=lambda p: (p == init, current[p] is not None, p))
|
|
884
|
+
for p in order:
|
|
885
|
+
data = files[p].encode("utf-8")
|
|
886
|
+
before = current[p]
|
|
887
|
+
if before is not None and _norm(before) == files[p]:
|
|
888
|
+
raw[p] = before # the same text (maybe with other line endings): left alone
|
|
889
|
+
continue
|
|
890
|
+
if before is not None:
|
|
891
|
+
backup(p)
|
|
892
|
+
write_atomic(p, data)
|
|
893
|
+
raw[p] = data
|
|
894
|
+
for p in sorted(old.raw if old is not None else ()):
|
|
895
|
+
if p not in files and os.path.exists(p):
|
|
896
|
+
backup(p)
|
|
897
|
+
os.remove(p)
|
|
898
|
+
written.raw = raw
|
|
899
|
+
return written
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def relayout_due(found: ScriptFiles, style: Style | None, max_kb: int) -> bool:
|
|
903
|
+
"""Is a file of the script over the limit (a script to split, a part to cut)?"""
|
|
904
|
+
if style is None or not max_kb:
|
|
905
|
+
return False
|
|
906
|
+
limit = max_kb * KB
|
|
907
|
+
return any(len(data) > limit for p, data in found.raw.items() if not p.endswith("__init__.py"))
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
def entries_lost(found: ScriptFiles, style: Style | None, ids: set[int | None]) -> list[tuple[str, int, int]]:
|
|
911
|
+
"""The entries the parts of a folder show that a compile of its joined text did not
|
|
912
|
+
read (a bracket or a quote left open at the end of a part swallows the next ones):
|
|
913
|
+
[(file, line, id)]."""
|
|
914
|
+
if not found.package or style is None:
|
|
915
|
+
return []
|
|
916
|
+
lay = layout(found.text, style)
|
|
917
|
+
if lay is None:
|
|
918
|
+
return []
|
|
919
|
+
out = []
|
|
920
|
+
for c in lay.chunks:
|
|
921
|
+
if c.id is not None and c.id not in ids:
|
|
922
|
+
path, line = found.where(c.start + 1)
|
|
923
|
+
out.append((path, line or 0, c.id))
|
|
924
|
+
return out
|