rpgsync 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rpgsync/scriptfiles.py ADDED
@@ -0,0 +1,924 @@
1
+ """A script written as one file, or as a folder of files when it is too big for one.
2
+
3
+ GitLab shows no diff of a file whose patch is over 200 KB (and collapses the ones over
4
+ 20 KB): a script bigger than ``max_file_kb`` ([tool.rpgsync] of Scripts/pyproject.toml,
5
+ 160 by default) is written as a Python package instead::
6
+
7
+ database/common_events/__init__.py its header, Config and instance
8
+ database/common_events/events_0001_0036.py class Events_0001_0036(CommonEventTable): ...
9
+ database/common_events/events_0037_0097.py class Events_0037_0097(CommonEventTable): ...
10
+
11
+ The class of __init__.py inherits from the class of every part, so ty and Pylance find
12
+ the entries where they found them before (``common_events.mr_clean``, ``variables.gold``),
13
+ and every ``from database.common_events import common_events`` keeps working. The
14
+ events of a map are at the top level of its parts (Map0195/events_0001_0040.py); a part
15
+ imports the events of the other parts it names.
16
+
17
+ rpgsync itself sees one text: the script as it would be in one file (the "joined" text).
18
+ A folder is joined when it is read, with a line map back to the files, and split again
19
+ when it is written. Every .py file of the folder is a part, in the order of their names.
20
+ What is above the class of a part, and the imports of the parts and the class statement
21
+ of __init__.py, are rpgsync's; the rest is yours. Writing a script into its folder keeps
22
+ every entry in its file and in its place: a new entry goes after the one before it (by
23
+ id), an emptied part is removed, and a part grown over the limit is cut in two. The
24
+ first split cuts the script by size, in id order, into parts named after their ids.
25
+ Parts are never renamed nor merged back; a part holding one entry bigger than the limit
26
+ stays as it is.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import ast
32
+ import bisect
33
+ import io
34
+ import os
35
+ import re
36
+ import tokenize
37
+ from collections.abc import Callable
38
+
39
+ from pydantic import BaseModel
40
+
41
+ KB = 1024
42
+ MAX_FILE_KB = 160 # default limit (GitLab shows no diff of a file over 200 KB)
43
+ FILL = 0.75 # a new part is filled up to this share of the limit: it has room to grow
44
+ LINE_LENGTH = 120
45
+
46
+ MARK = (
47
+ "# Its entries are in the other files of this folder, in the order of their names: rpgsync",
48
+ "# splits a script bigger than %d KB (GitLab shows no diff of a bigger file). Move them freely.",
49
+ )
50
+ _MARK_2 = re.compile(re.escape(MARK[1]).replace("%d", r"\d+"))
51
+ _CLASS = re.compile(r"class (\w+)\((\w+)\):\s*(?:#.*)?$")
52
+ _TABLE_ENTRY = re.compile(r" [A-Za-z_]\w*\s*(?::[^=]*)?=(?!=)")
53
+ _POSITIONAL_ID = re.compile(r"\s*\w+\s*(?::[^=]*)?=\s*\w+\(\s*(\d+)\s*[,)]")
54
+ _FIRST_KEYWORD_ID = re.compile(r"=\s*\w+\(\s*id=(\d+)\b") # potion = Item(id=5, ...)
55
+ _NEXT_LINE_ID = re.compile(r"\s+id=(\d+),") # its first argument, on the next line
56
+ _TOP_DEF = re.compile(r"(?:class|def) (\w+)\b")
57
+
58
+
59
+ class ChangedMeanwhile(Exception):
60
+ """A file changed while rpgsync was syncing it (an edit in the editor or in the
61
+ script): nothing more is written, the next pass syncs both changes."""
62
+
63
+
64
+ class LayoutError(Exception):
65
+ """A script folder rpgsync cannot read as one script."""
66
+
67
+ def __init__(self, path: str, message: str, line: int | None = None):
68
+ self.path = path
69
+ self.message = message
70
+ self.line = line
71
+ super().__init__("%s%s: %s" % (path, ":%d" % line if line else "", message))
72
+
73
+
74
+ class Style(BaseModel):
75
+ """How the entries of a kind of script are laid out."""
76
+
77
+ wrapped: bool # in a class (common events, troops, tables), else at the top level (maps)
78
+ decorator: str | None = None # entries are `@decorator(id, ...)`; None: `name = Model(id=...)`
79
+ prefix: str # names of the part files: <prefix>_<first id>_<last id>.py
80
+ sep: int # blank lines between two entries
81
+ self_import: str | None = None # what a part imports to name its own script's instance
82
+ event_refs: bool = False # maps: a part imports the event classes it names from the others
83
+
84
+ @property
85
+ def indent(self) -> str:
86
+ return " " if self.wrapped else ""
87
+
88
+
89
+ COMMON_EVENTS = Style(
90
+ wrapped=True,
91
+ decorator="common_event",
92
+ prefix="events",
93
+ sep=1,
94
+ self_import="from database.common_events import common_events",
95
+ )
96
+ TROOP_EVENTS = Style(wrapped=True, decorator="troop", prefix="troops", sep=1)
97
+ MAP = Style(wrapped=False, decorator="event", prefix="events", sep=2, event_refs=True)
98
+
99
+
100
+ def table_style(table: str) -> Style:
101
+ """A database table: `name = Model(id=...)` entries, no blank line between them."""
102
+ return Style(wrapped=True, prefix=table, sep=0)
103
+
104
+
105
+ class Part(BaseModel):
106
+ """A part of a script folder, as read."""
107
+
108
+ path: str
109
+ cls: str | None # its class (None: a map's part)
110
+ first: int # the lines of its entries in the joined text: [first, end) (0-based)
111
+ end: int
112
+
113
+
114
+ class ScriptFiles(BaseModel):
115
+ """A script as read: its joined text, the bytes of each of its files (to tell
116
+ whether they changed since), and where each line of the text comes from."""
117
+
118
+ path: str # the script's path as one file (database/common_events.py)
119
+ text: str
120
+ raw: dict[str, bytes]
121
+ package: bool = False
122
+ parts: list[Part] = []
123
+ spans: list[tuple[int, str, int, int]] = [] # (joined line index, file, its line number, count)
124
+
125
+ def where(self, lineno: int | None) -> tuple[str, int | None]:
126
+ """The file and line of a line of the joined text (1-based)."""
127
+ if lineno is None:
128
+ return (package_dir(self.path) if self.package else self.path, None)
129
+ if not self.spans:
130
+ return (self.path, lineno)
131
+ n = bisect.bisect_right(self.spans, (lineno - 1, "￿")) - 1
132
+ start, path, line, count = self.spans[max(n, 0)]
133
+ return (path, line + min(lineno - 1 - start, max(count - 1, 0)))
134
+
135
+ def entry_files(self, style: Style | None) -> dict[int, str]:
136
+ """id -> the file of each entry, for a script split in files."""
137
+ lay = layout(self.text, style) if self.package and style is not None else None
138
+ if lay is None:
139
+ return {}
140
+ return {c.id: self.where(c.start + 1)[0] for c in lay.chunks if c.id is not None}
141
+
142
+ def at(self, lineno: int | None) -> str:
143
+ """The file and line of a line of the joined text, as path:line."""
144
+ path, line = self.where(lineno)
145
+ return "%s:%d" % (path, line) if line else path
146
+
147
+
148
+ # --------------------------------------------------------------------------
149
+ # Files
150
+ # --------------------------------------------------------------------------
151
+
152
+
153
+ def max_file_kb(script_dir: str) -> int:
154
+ """[tool.rpgsync] max_file_kb of a scripts folder's pyproject.toml: a script bigger
155
+ than this is split in files (0: never)."""
156
+ from .compat import load_config
157
+
158
+ value = load_config(script_dir).get("max_file_kb", MAX_FILE_KB)
159
+ return value if isinstance(value, int) and value >= 0 else MAX_FILE_KB
160
+
161
+
162
+ def package_dir(py_path: str) -> str:
163
+ return py_path[:-3] if py_path.endswith(".py") else py_path
164
+
165
+
166
+ def _package_files(py_path: str) -> list[str]:
167
+ folder = package_dir(py_path)
168
+ try:
169
+ names = os.listdir(folder)
170
+ except OSError:
171
+ return []
172
+ return sorted(os.path.join(folder, n) for n in names if n.endswith(".py") and not n.startswith("."))
173
+
174
+
175
+ def is_package(py_path: str) -> bool:
176
+ return bool(_package_files(py_path))
177
+
178
+
179
+ def exists(py_path: str) -> bool:
180
+ return os.path.exists(py_path) or is_package(py_path)
181
+
182
+
183
+ def paths(py_path: str) -> list[str]:
184
+ """The files of a script now (one file, or those of its folder), its path as one
185
+ file first even when missing: what to watch."""
186
+ return [py_path, *_package_files(py_path)]
187
+
188
+
189
+ def _stat(path: str):
190
+ try:
191
+ st = os.stat(path)
192
+ return (st.st_mtime_ns, st.st_size)
193
+ except OSError:
194
+ return None
195
+
196
+
197
+ def stat_key(py_path: str) -> tuple:
198
+ """Changes when a file of the script changes, appears or goes."""
199
+ return tuple((p, _stat(p)) for p in paths(py_path))
200
+
201
+
202
+ def read_bytes(path: str) -> bytes | None:
203
+ try:
204
+ with open(path, "rb") as f:
205
+ return f.read()
206
+ except FileNotFoundError:
207
+ return None
208
+
209
+
210
+ def write_atomic(path: str, data: bytes) -> None:
211
+ tmp = path + ".rpgsync-tmp"
212
+ with open(tmp, "wb") as f:
213
+ f.write(data)
214
+ os.replace(tmp, path)
215
+
216
+
217
+ def _norm(data: bytes) -> str:
218
+ return data.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n")
219
+
220
+
221
+ def unchanged(py_path: str, expect: ScriptFiles | None) -> bool:
222
+ """Are the files of the script still the ones read (`expect`, None: no script)?"""
223
+ current = [p for p in paths(py_path) if os.path.exists(p)]
224
+ if expect is None:
225
+ return not current
226
+ return set(current) == set(expect.raw) and all(read_bytes(p) == expect.raw[p] for p in current)
227
+
228
+
229
+ def check_unchanged(py_path: str, expect: ScriptFiles | None) -> None:
230
+ if not unchanged(py_path, expect):
231
+ raise ChangedMeanwhile(os.path.basename(package_dir(py_path) if is_package(py_path) else py_path))
232
+
233
+
234
+ # --------------------------------------------------------------------------
235
+ # Reading
236
+ # --------------------------------------------------------------------------
237
+
238
+
239
+ def read(py_path: str, style: Style | None) -> ScriptFiles | None:
240
+ """The script (None: there is none). Raises LayoutError for a folder that is not
241
+ one script, ChangedMeanwhile when its files keep changing while they are read."""
242
+ for _ in range(5):
243
+ before = stat_key(py_path)
244
+ found = _read(py_path, style)
245
+ if stat_key(py_path) == before: # a git checkout, a refactoring over several files...
246
+ return found
247
+ raise ChangedMeanwhile(os.path.basename(py_path))
248
+
249
+
250
+ def read_text(py_path: str, style: Style | None) -> str | None:
251
+ found = read(py_path, style)
252
+ return None if found is None else found.text
253
+
254
+
255
+ def _read(py_path: str, style: Style | None) -> ScriptFiles | None:
256
+ files = _package_files(py_path)
257
+ single = read_bytes(py_path)
258
+ if not files:
259
+ if single is None:
260
+ return None
261
+ return ScriptFiles(path=py_path, text=_norm(single), raw={py_path: single})
262
+ folder = package_dir(py_path)
263
+ if single is not None:
264
+ raise LayoutError(py_path, "the folder %s/ holds this script too: keep one of them" % os.path.basename(folder))
265
+ if style is None:
266
+ raise LayoutError(folder, "this script cannot be split: keep it in %s" % os.path.basename(py_path))
267
+ init = os.path.join(folder, "__init__.py")
268
+ if init not in files:
269
+ raise LayoutError(folder, "a script folder needs its __init__.py")
270
+ raw = {}
271
+ for p in files:
272
+ data = read_bytes(p)
273
+ if data is None:
274
+ raise ChangedMeanwhile(os.path.basename(p))
275
+ raw[p] = data
276
+ texts = {}
277
+ for p, data in raw.items():
278
+ try:
279
+ texts[p] = _norm(data)
280
+ except UnicodeDecodeError as e:
281
+ raise LayoutError(p, "not UTF-8 text (%s)" % e) from None
282
+ return join(py_path, style, texts, raw)
283
+
284
+
285
+ def _owned_init_lines(lines: list[str], tree: ast.Module) -> set[int]:
286
+ """Indexes of the lines of an __init__.py that are rpgsync's: the note on the parts,
287
+ the imports of the parts, the class statement."""
288
+ owned = set()
289
+ for stmt in tree.body:
290
+ if isinstance(stmt, ast.ImportFrom) and stmt.level:
291
+ owned.update(range(stmt.lineno - 1, (stmt.end_lineno or stmt.lineno)))
292
+ for n, line in enumerate(lines):
293
+ if line == MARK[0] or _MARK_2.fullmatch(line):
294
+ owned.add(n)
295
+ if line == MARK[0] and n and not lines[n - 1].strip():
296
+ owned.add(n - 1) # the blank line rpgsync puts above it
297
+ return owned
298
+
299
+
300
+ def _part_body(path: str, lines: list[str], style: Style) -> tuple[str | None, str | None, int]:
301
+ """(class, its base, index of the first line of the entries) of a part file."""
302
+ if style.wrapped:
303
+ classes = [n for n, line in enumerate(lines) if line.startswith("class ")]
304
+ if len(classes) != 1:
305
+ raise LayoutError(path, "a part holds one class (its entries), it has %d" % len(classes))
306
+ c = classes[0]
307
+ m = _CLASS.match(lines[c])
308
+ if not m:
309
+ raise LayoutError(path, "the class of a part is written `class Name(Base):`", c + 1)
310
+ _check_header(path, lines[:c])
311
+ for n in range(c + 1, len(lines)):
312
+ if lines[n][:1] not in ("", " ", "\t", "#"):
313
+ raise LayoutError(path, "a part holds only its class: this line is outside it", n + 1)
314
+ return m.group(1), m.group(2), c + 1
315
+ end = _header_end(lines)
316
+ _check_header(path, lines[:end])
317
+ return None, None, end
318
+
319
+
320
+ def _check_header(path: str, lines: list[str]) -> None:
321
+ """The lines above the entries of a part: a docstring, comments and imports only."""
322
+ try:
323
+ tree = ast.parse("\n".join(lines))
324
+ except SyntaxError as e:
325
+ raise LayoutError(path, "syntax error: %s" % e.msg, e.lineno) from None
326
+ for n, stmt in enumerate(tree.body):
327
+ if isinstance(stmt, (ast.Import, ast.ImportFrom)):
328
+ continue
329
+ if n == 0 and isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant):
330
+ continue
331
+ raise LayoutError(path, "above its entries, a part holds only its docstring and imports", stmt.lineno)
332
+
333
+
334
+ def _header_end(lines: list[str]) -> int:
335
+ """Index of the first line after the docstring and imports a map part starts with
336
+ (comments right above the first event belong to it)."""
337
+ n, end = 0, 0
338
+ while n < len(lines) and not lines[n].strip():
339
+ n += 1
340
+ if n < len(lines) and lines[n].lstrip().startswith(('"""', "'''", '"', "'")):
341
+ q = lines[n].lstrip()[:3] if lines[n].lstrip()[:3] in ('"""', "'''") else lines[n].lstrip()[:1]
342
+ rest = lines[n].lstrip()[len(q) :]
343
+ while q not in rest and n + 1 < len(lines):
344
+ n += 1
345
+ rest = lines[n]
346
+ n += 1
347
+ end = n
348
+ while n < len(lines):
349
+ line = lines[n]
350
+ if not line.strip() or line.startswith("#"):
351
+ n += 1
352
+ continue
353
+ if line.startswith(("from ", "import ")):
354
+ depth = line.count("(") - line.count(")")
355
+ n += 1
356
+ while depth > 0 and n < len(lines):
357
+ depth += lines[n].count("(") - lines[n].count(")")
358
+ n += 1
359
+ end = n
360
+ continue
361
+ break
362
+ while end < len(lines) and not lines[end].strip():
363
+ end += 1
364
+ return end
365
+
366
+
367
+ def _strip(lines: list[str]) -> list[str]:
368
+ a, b = 0, len(lines)
369
+ while a < b and not lines[a].strip():
370
+ a += 1
371
+ while b > a and not lines[b - 1].strip():
372
+ b -= 1
373
+ return lines[a:b]
374
+
375
+
376
+ def join(py_path: str, style: Style, texts: dict[str, str], raw: dict[str, bytes] | None = None) -> ScriptFiles:
377
+ """The script of a folder's files (path -> text) as one text."""
378
+ folder = package_dir(py_path)
379
+ init = os.path.join(folder, "__init__.py")
380
+ init_lines = texts[init].split("\n")
381
+ try:
382
+ tree = ast.parse(texts[init])
383
+ except SyntaxError as e:
384
+ raise LayoutError(init, "syntax error: %s" % e.msg, e.lineno) from None
385
+ owned = _owned_init_lines(init_lines, tree)
386
+
387
+ # the parts, in the order of their names
388
+ parts = []
389
+ for path in sorted(p for p in texts if p != init):
390
+ lines = texts[path].split("\n")
391
+ cls, base, start = _part_body(path, lines, style)
392
+ body = lines[start:]
393
+ a = 0
394
+ while a < len(body) and not body[a].strip():
395
+ a += 1
396
+ parts.append((path, cls, base, start + a + 1, _strip(body))) # (line number of its first entry line)
397
+ names = [cls for _, cls, _, _, _ in parts if cls]
398
+ if len(set(names)) != len(names):
399
+ raise LayoutError(folder, "two parts have the same class name")
400
+ bases = {base for _, _, base, _, _ in parts if base}
401
+ if len(bases) > 1:
402
+ raise LayoutError(folder, "the classes of the parts have different bases: %s" % ", ".join(sorted(bases)))
403
+
404
+ out: list[str] = []
405
+ spans: list[tuple[int, str, int, int]] = []
406
+
407
+ def emit(lines: list[str], path: str, line: int) -> None: # line: of lines[0] in its file (1-based)
408
+ if lines:
409
+ spans.append((len(out), path, line, len(lines)))
410
+ out.extend(lines)
411
+
412
+ def emit_init(a: int, b: int) -> None: # the lines [a, b) of __init__.py that are not rpgsync's
413
+ run = a
414
+ for n in range(a, b + 1):
415
+ if n == b or n in owned:
416
+ emit(init_lines[run:n], init, run + 1)
417
+ run = n + 1
418
+
419
+ entries = any(body for *_, body in parts)
420
+ result_parts = []
421
+ if style.wrapped:
422
+ classes = [stmt for stmt in tree.body if isinstance(stmt, ast.ClassDef)]
423
+ if len(classes) != 1:
424
+ raise LayoutError(init, "__init__.py holds one class (made of the parts), it has %d" % len(classes))
425
+ cdef = classes[0]
426
+ if bases:
427
+ base = next(iter(bases))
428
+ elif cdef.bases and isinstance(cdef.bases[-1], ast.Name):
429
+ base = cdef.bases[-1].id
430
+ else:
431
+ raise LayoutError(init, "the class must end with the base of the script's class", cdef.lineno)
432
+ c = cdef.lineno - 1 # the class statement: up to its first body line
433
+ first = cdef.body[0].lineno - 1 if cdef.body else c + 1
434
+ while first > c + 1 and init_lines[first - 1].strip().startswith("#"):
435
+ first -= 1 # comments above the first statement of the body are its
436
+ t = next(
437
+ (n for n in range(first, len(init_lines)) if init_lines[n][:1] not in ("", " ", "\t", "#")),
438
+ len(init_lines),
439
+ )
440
+ end = t
441
+ while end > first and not init_lines[end - 1].strip():
442
+ end -= 1
443
+ emit_init(0, c)
444
+ spans.append((len(out), init, c + 1, 1))
445
+ out.append("class %s(%s):" % (cdef.name, base))
446
+ pre = init_lines[first:end]
447
+ if entries and [line.strip() for line in pre] == ["pass"]:
448
+ pre = [] # the body of a class made of parts only
449
+ emit(pre, init, first + 1)
450
+ lead = 1 if pre else 0
451
+ for path, cls, _, line, body in parts:
452
+ if not body:
453
+ continue
454
+ gap = lead if not result_parts else style.sep
455
+ out.extend([""] * gap)
456
+ result_parts.append(Part(path=path, cls=cls, first=len(out), end=len(out) + len(body)))
457
+ emit(body, path, line)
458
+ emit_init(end, len(init_lines))
459
+ else:
460
+ end = max((n + 1 for n, line in enumerate(init_lines) if line.strip() and n not in owned), default=0)
461
+ emit_init(0, end)
462
+ for path, cls, _, line, body in parts:
463
+ if not body:
464
+ continue
465
+ out.extend([""] * style.sep)
466
+ result_parts.append(Part(path=path, cls=cls, first=len(out), end=len(out) + len(body)))
467
+ emit(body, path, line)
468
+ out.append("")
469
+ if raw is None:
470
+ raw = {p: t.encode("utf-8") for p, t in texts.items()}
471
+ return ScriptFiles(path=py_path, text="\n".join(out), raw=raw, package=True, parts=result_parts, spans=spans)
472
+
473
+
474
+ # --------------------------------------------------------------------------
475
+ # The entries of a text
476
+ # --------------------------------------------------------------------------
477
+
478
+
479
+ class Chunk(BaseModel):
480
+ """An entry and the comments and blank lines above it: lines [lead, end), the
481
+ entry itself starting at `start`."""
482
+
483
+ lead: int
484
+ start: int
485
+ end: int
486
+ id: int | None
487
+
488
+
489
+ class Layout(BaseModel):
490
+ """A joined text cut into its header, its class statement, what the class holds
491
+ before its entries (Config), its entries, and what follows them."""
492
+
493
+ lines: list[str]
494
+ header: list[str] # up to the class statement (maps: up to the first entry)
495
+ cls: tuple[str, str] | None # (name, base) of the class
496
+ pre: list[str]
497
+ chunks: list[Chunk]
498
+ tail: list[str]
499
+
500
+
501
+ def _is_start(line: str, style: Style) -> bool:
502
+ if style.decorator:
503
+ return line.startswith(style.indent + "@" + style.decorator + "(")
504
+ return bool(_TABLE_ENTRY.match(line))
505
+
506
+
507
+ def _entry_id(lines: list[str], start: int, style: Style) -> int | None:
508
+ if style.decorator:
509
+ m = re.match(r"\s*@" + re.escape(style.decorator) + r"\(\s*(\d+)\s*[,)]", "\n".join(lines[start : start + 3]))
510
+ return int(m.group(1)) if m else None
511
+ m = _POSITIONAL_ID.match(lines[start]) or _FIRST_KEYWORD_ID.search(lines[start])
512
+ if m is None and start + 1 < len(lines):
513
+ m = _NEXT_LINE_ID.match(lines[start + 1])
514
+ return int(m.group(1)) if m else None
515
+
516
+
517
+ def _leading(lines: list[str], start: int, floor: int, indent: str) -> int:
518
+ """Index of the first of the blank lines and comments right above `start` (not
519
+ above `floor`)."""
520
+ k = start
521
+ while k - 1 >= floor:
522
+ line = lines[k - 1]
523
+ s = line.lstrip()
524
+ if s and not (s.startswith("#") and len(line) - len(s) <= len(indent)):
525
+ break
526
+ k -= 1
527
+ return k
528
+
529
+
530
+ def layout(text: str, style: Style) -> Layout | None:
531
+ """The parts of a joined text (maybe no entries); None without its class."""
532
+ lines = text.split("\n")
533
+ indent = style.indent
534
+ if style.wrapped:
535
+ c = next((n for n, line in enumerate(lines) if line.startswith("class ")), None)
536
+ if c is None or not _CLASS.match(lines[c]):
537
+ return None
538
+ m = _CLASS.match(lines[c])
539
+ t = next((n for n in range(c + 1, len(lines)) if lines[n][:1] not in ("", " ", "\t", "#")), len(lines))
540
+ end = t
541
+ while end > c + 1 and not lines[end - 1].strip():
542
+ end -= 1
543
+ floor = c + 1
544
+ if not style.decorator: # a table: its Config is not an entry
545
+ f = floor
546
+ while f < end and not lines[f].strip():
547
+ f += 1
548
+ if f < end and lines[f].startswith(" class Config"):
549
+ f += 1
550
+ while f < end and (not lines[f].strip() or lines[f].startswith(" ")):
551
+ f += 1
552
+ floor = f
553
+ elif f < end and re.match(r" size\s*=", lines[f]):
554
+ floor = f + 1
555
+ starts = [n for n in range(floor, end) if _is_start(lines[n], style)]
556
+ cls = (m.group(1), m.group(2)) # type: ignore[union-attr]
557
+ else:
558
+ end = len(lines)
559
+ while end > 0 and not lines[end - 1].strip():
560
+ end -= 1
561
+ starts = [n for n in range(end) if _is_start(lines[n], style)]
562
+ c, floor, cls = -1, 0, None
563
+ leads = [_leading(lines, s, starts[k - 1] + 1 if k else floor, indent) for k, s in enumerate(starts)]
564
+ chunks = [
565
+ Chunk(lead=leads[k], start=s, end=leads[k + 1] if k + 1 < len(starts) else end, id=_entry_id(lines, s, style))
566
+ for k, s in enumerate(starts)
567
+ ]
568
+ first = leads[0] if leads else end
569
+ if style.wrapped:
570
+ header, pre = lines[:c], _strip(lines[c + 1 : first])
571
+ else:
572
+ header, pre = lines[:first], []
573
+ return Layout(lines=lines, header=header, cls=cls, pre=pre, chunks=chunks, tail=lines[end:])
574
+
575
+
576
+ def _entry_text(lay: Layout, c: Chunk) -> str:
577
+ """An entry's lines without the blank lines above it."""
578
+ lines = lay.lines[c.lead : c.end]
579
+ n = 0
580
+ while n < len(lines) and not lines[n].strip():
581
+ n += 1
582
+ return "\n".join(lines[n:])
583
+
584
+
585
+ def equivalent(a: str, b: str, style: Style) -> bool:
586
+ """The same script, whatever the order of its entries and the blank lines above
587
+ them: what splitting a script into files and joining them back may change."""
588
+ la, lb = layout(a, style), layout(b, style)
589
+ if la is None or lb is None:
590
+ return a == b
591
+
592
+ def key(lay: Layout):
593
+ header = _strip(lay.header)
594
+ return (header, lay.cls, _strip(lay.pre), sorted(_entry_text(lay, c) for c in lay.chunks), _strip(lay.tail))
595
+
596
+ return key(la) == key(lb)
597
+
598
+
599
+ # --------------------------------------------------------------------------
600
+ # Writing
601
+ # --------------------------------------------------------------------------
602
+
603
+
604
+ def _imports(header: list[str]) -> list[str]:
605
+ """The import statements of a script's header, as written there."""
606
+ try:
607
+ tree = ast.parse("\n".join(header))
608
+ except SyntaxError:
609
+ return []
610
+ out = []
611
+ for stmt in tree.body:
612
+ if isinstance(stmt, (ast.Import, ast.ImportFrom)) and not (isinstance(stmt, ast.ImportFrom) and stmt.level):
613
+ out += header[stmt.lineno - 1 : stmt.end_lineno]
614
+ return out
615
+
616
+
617
+ def _label(py_path: str) -> str:
618
+ folder = package_dir(py_path)
619
+ parent = os.path.basename(os.path.dirname(folder))
620
+ name = os.path.basename(folder)
621
+ return "%s/%s" % (parent, name) if parent == "database" else name
622
+
623
+
624
+ def _import_line(module: str, names: list[str], line_length: int) -> list[str]:
625
+ one = "from .%s import %s" % (module, ", ".join(names))
626
+ if len(one) <= line_length:
627
+ return [one]
628
+ return ["from .%s import (" % module, *(" %s," % n for n in names), ")"]
629
+
630
+
631
+ def _class_stmt(name: str, bases: list[str], line_length: int) -> list[str]:
632
+ one = "class %s(%s):" % (name, ", ".join(bases))
633
+ if len(one) <= line_length:
634
+ return [one]
635
+ return ["class %s(" % name, *(" %s," % b for b in bases), "):"]
636
+
637
+
638
+ def _names_used(src: str) -> set[str]:
639
+ """The names a map part uses (not as attributes)."""
640
+ out = set()
641
+ prev = None
642
+ try:
643
+ for tok in tokenize.generate_tokens(io.StringIO(src).readline):
644
+ if tok.type == tokenize.NAME and not (prev is not None and prev.type == tokenize.OP and prev.string == "."):
645
+ out.add(tok.string)
646
+ if tok.type not in (tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT, tokenize.INDENT, tokenize.DEDENT):
647
+ prev = tok
648
+ except (tokenize.TokenError, SyntaxError, IndentationError):
649
+ pass
650
+ return out
651
+
652
+
653
+ class _Plan(BaseModel):
654
+ module: str
655
+ cls: str | None
656
+ chunks: list[int] # indexes in the layout's chunks
657
+
658
+
659
+ def _module_name(style: Style, ids: list[int | None], taken: set[str]) -> str:
660
+ known = [i for i in ids if i is not None]
661
+ base = "%s_%04d_%04d" % (style.prefix, min(known), max(known)) if known else "%s_part" % style.prefix
662
+ name, n = base, 2
663
+ while name in taken:
664
+ name, n = "%s_%d" % (base, n), n + 1
665
+ return name
666
+
667
+
668
+ def _class_name(module: str) -> str:
669
+ return module[:1].upper() + module[1:]
670
+
671
+
672
+ def _group(sizes: list[int], items: list[int], budget: int, overhead: int) -> list[list[int]]:
673
+ """Consecutive items in groups of at most `budget` bytes (one item at least)."""
674
+ groups: list[list[int]] = []
675
+ cur: list[int] = []
676
+ size = overhead
677
+ for i in items:
678
+ if cur and size + sizes[i] > budget:
679
+ groups.append(cur)
680
+ cur, size = [], overhead
681
+ cur.append(i)
682
+ size += sizes[i]
683
+ if cur:
684
+ groups.append(cur)
685
+ return groups
686
+
687
+
688
+ def _place(lay: Layout, old: ScriptFiles, old_lay: Layout | None) -> list[_Plan]:
689
+ """The chunks of `lay` in the parts of `old`: each entry where it was."""
690
+ plans = [_Plan(module=os.path.basename(p.path)[:-3], cls=p.cls, chunks=[]) for p in old.parts]
691
+ if old_lay is None or not plans:
692
+ return plans
693
+ part_of = [] # part of each old chunk
694
+ for c in old_lay.chunks:
695
+ part_of.append(next((n for n, p in enumerate(old.parts) if p.first <= c.start < p.end), 0))
696
+ same = len(old_lay.chunks) == len(lay.chunks) and all(
697
+ a.lead == b.lead and a.start == b.start and (a.id is None or a.id == b.id)
698
+ for a, b in zip(old_lay.chunks, lay.chunks)
699
+ )
700
+ if same: # the same entries at the same lines (ids written in, labels updated...)
701
+ for n in range(len(lay.chunks)):
702
+ plans[part_of[n]].chunks.append(n)
703
+ return plans
704
+ where: dict[int, tuple[int, int]] = {} # id -> (part, rank in the part)
705
+ rank = [0] * len(plans)
706
+ for n, c in enumerate(old_lay.chunks):
707
+ if c.id is not None and c.id not in where:
708
+ where[c.id] = (part_of[n], rank[part_of[n]])
709
+ rank[part_of[n]] += 1
710
+ placed: dict[int, int] = {} # chunk -> part
711
+ known = sorted(
712
+ (where[c.id] + (n,) for n, c in enumerate(lay.chunks) if c.id is not None and c.id in where),
713
+ )
714
+ seen_ids: set[int] = set()
715
+ for p, _, n in known:
716
+ if lay.chunks[n].id in seen_ids:
717
+ continue
718
+ seen_ids.add(lay.chunks[n].id) # type: ignore[arg-type]
719
+ plans[p].chunks.append(n)
720
+ placed[n] = p
721
+ for n in range(len(lay.chunks)):
722
+ if n in placed:
723
+ continue
724
+ if n and n - 1 in placed: # after the entry before it
725
+ p = placed[n - 1]
726
+ plans[p].chunks.insert(plans[p].chunks.index(n - 1) + 1, n)
727
+ else:
728
+ p = 0
729
+ plans[0].chunks.insert(0, n)
730
+ placed[n] = p
731
+ return plans
732
+
733
+
734
+ def plan(
735
+ py_path: str,
736
+ style: Style | None,
737
+ text: str,
738
+ old: ScriptFiles | None,
739
+ max_kb: int,
740
+ line_length: int = LINE_LENGTH,
741
+ ) -> dict[str, str] | None:
742
+ """The files to write the script as: path -> text, None for one file (py_path)."""
743
+ if style is None:
744
+ return None
745
+ limit = max_kb * KB
746
+ folder = old is not None and old.package
747
+ if not folder and (not max_kb or len(text.encode("utf-8")) <= limit):
748
+ return None
749
+ lay = layout(text, style)
750
+ if lay is None and folder:
751
+ raise LayoutError(package_dir(py_path), "internal error: no class to split this script into")
752
+ if lay is None or (not folder and len(lay.chunks) < 2):
753
+ return None # no entries to split it into
754
+ pkg = package_dir(py_path)
755
+ label = _label(py_path)
756
+ imports = _imports(lay.header)
757
+ head = ['"""Part of %s: see __init__.py"""' % label, ""] + imports
758
+ if style.self_import and style.self_import not in imports:
759
+ head.append(style.self_import)
760
+ sizes = [len("\n".join(lay.lines[c.lead : c.end]).encode("utf-8")) + 1 for c in lay.chunks]
761
+ overhead = len("\n".join(head).encode("utf-8")) + 60
762
+ budget = int(limit * FILL) if max_kb else 1 << 60
763
+
764
+ if folder:
765
+ assert old is not None
766
+ plans = _place(lay, old, layout(old.text, style))
767
+ if not plans:
768
+ plans = [
769
+ _Plan(module="", cls=None, chunks=g)
770
+ for g in _group(sizes, list(range(len(lay.chunks))), budget, overhead)
771
+ ]
772
+ else:
773
+ plans = [
774
+ _Plan(module="", cls=None, chunks=g) for g in _group(sizes, list(range(len(lay.chunks))), budget, overhead)
775
+ ]
776
+ # cut the parts grown over the limit
777
+ out_plans: list[_Plan] = []
778
+ for p in plans:
779
+ if not p.chunks:
780
+ continue # emptied: removed
781
+ total = overhead + sum(sizes[n] for n in p.chunks)
782
+ if max_kb and total > limit and len(p.chunks) > 1:
783
+ groups = _group(sizes, p.chunks, budget, overhead)
784
+ out_plans.append(_Plan(module=p.module, cls=p.cls, chunks=groups[0]))
785
+ out_plans += [_Plan(module="", cls=None, chunks=g) for g in groups[1:]]
786
+ else:
787
+ out_plans.append(p)
788
+ taken = {p.module for p in out_plans if p.module}
789
+ for p in out_plans:
790
+ if not p.module:
791
+ p.module = _module_name(style, [lay.chunks[n].id for n in p.chunks], taken | {"__init__"})
792
+ taken.add(p.module)
793
+ if style.wrapped and not p.cls:
794
+ p.cls = _class_name(p.module)
795
+ out_plans.sort(key=lambda p: p.module)
796
+
797
+ files: dict[str, str] = {}
798
+ bodies = []
799
+ for p in out_plans:
800
+ body: list[str] = []
801
+ for k, n in enumerate(p.chunks):
802
+ c = lay.chunks[n]
803
+ lines = lay.lines[c.lead : c.end]
804
+ if k == 0:
805
+ while lines and not lines[0].strip():
806
+ lines = lines[1:]
807
+ body += lines
808
+ bodies.append(_strip(body) if body else body)
809
+ refs: list[list[str]] = [[] for _ in out_plans]
810
+ if style.event_refs:
811
+ defined = [{m.group(1) for line in body if (m := _TOP_DEF.match(line))} for body in bodies]
812
+ owner = {name: k for k, names in enumerate(defined) for name in names}
813
+ for k, body in enumerate(bodies):
814
+ used = _names_used("\n".join(body) + "\n")
815
+ by_part: dict[int, list[str]] = {}
816
+ for name in sorted(used):
817
+ if owner.get(name, k) != k:
818
+ by_part.setdefault(owner[name], []).append(name)
819
+ for j in sorted(by_part):
820
+ refs[k] += _import_line(out_plans[j].module, by_part[j], line_length)
821
+ for k, p in enumerate(out_plans):
822
+ lines = head + refs[k] + ["", ""]
823
+ if style.wrapped:
824
+ lines.append("class %s(%s):" % (p.cls, lay.cls[1])) # type: ignore[index]
825
+ lines += bodies[k] or ([style.indent + "pass"] if style.wrapped else [])
826
+ files[os.path.join(pkg, p.module + ".py")] = "\n".join(lines) + "\n"
827
+
828
+ header = list(lay.header)
829
+ gap: list[str] = []
830
+ while header and not header[-1].strip():
831
+ gap.append(header.pop())
832
+ block = ["", MARK[0], MARK[1] % max_kb if max_kb else MARK[1] % MAX_FILE_KB]
833
+ if style.wrapped:
834
+ block += ["from .%s import %s" % (p.module, p.cls) for p in out_plans]
835
+ name, base = lay.cls # type: ignore[misc]
836
+ init = header + block + (gap or ["", ""])
837
+ init += _class_stmt(name, [p.cls for p in out_plans] + [base], line_length) # type: ignore[misc]
838
+ init += lay.pre or [style.indent + "pass"]
839
+ init += lay.tail
840
+ else:
841
+ block += ["from . import %s" % p.module for p in out_plans]
842
+ init = header + block + [""]
843
+ files[os.path.join(pkg, "__init__.py")] = "\n".join(init) if init[-1] == "" else "\n".join(init) + "\n"
844
+ return files
845
+
846
+
847
+ def write(
848
+ py_path: str,
849
+ style: Style | None,
850
+ text: str,
851
+ old: ScriptFiles | None,
852
+ max_kb: int,
853
+ backup: Callable[[str], None],
854
+ line_length: int = LINE_LENGTH,
855
+ ) -> ScriptFiles:
856
+ """Write a script (one file, or the files of its folder when it is one or the text
857
+ is over the limit): only the files that change, each backed up first. `old`: the
858
+ script as read before (ChangedMeanwhile if its files changed since). -> the script
859
+ as now written (its text may differ from `text` by blank lines above entries)."""
860
+ check_unchanged(py_path, old)
861
+ files = plan(py_path, style, text, old, max_kb, line_length)
862
+ if files is not None:
863
+ assert style is not None
864
+ written = join(py_path, style, files)
865
+ if not equivalent(written.text, text, style):
866
+ if old is not None and old.package:
867
+ raise LayoutError(package_dir(py_path), "internal error: rpgsync cannot split this script back")
868
+ files = None # write it as one file
869
+ if files is None:
870
+ data = text.encode("utf-8")
871
+ if read_bytes(py_path) != data:
872
+ os.makedirs(os.path.dirname(py_path), exist_ok=True)
873
+ backup(py_path)
874
+ write_atomic(py_path, data)
875
+ return ScriptFiles(path=py_path, text=text, raw={py_path: data})
876
+ folder = package_dir(py_path)
877
+ os.makedirs(folder, exist_ok=True)
878
+ raw: dict[str, bytes] = {}
879
+ init = os.path.join(folder, "__init__.py")
880
+ current = {p: (old.raw.get(p) if old is not None else None) for p in files}
881
+ # new parts first, then the changed ones, then __init__.py, then the stale ones go:
882
+ # each step leaves a folder that reads as an error at worst, never as fewer entries
883
+ order = sorted(files, key=lambda p: (p == init, current[p] is not None, p))
884
+ for p in order:
885
+ data = files[p].encode("utf-8")
886
+ before = current[p]
887
+ if before is not None and _norm(before) == files[p]:
888
+ raw[p] = before # the same text (maybe with other line endings): left alone
889
+ continue
890
+ if before is not None:
891
+ backup(p)
892
+ write_atomic(p, data)
893
+ raw[p] = data
894
+ for p in sorted(old.raw if old is not None else ()):
895
+ if p not in files and os.path.exists(p):
896
+ backup(p)
897
+ os.remove(p)
898
+ written.raw = raw
899
+ return written
900
+
901
+
902
+ def relayout_due(found: ScriptFiles, style: Style | None, max_kb: int) -> bool:
903
+ """Is a file of the script over the limit (a script to split, a part to cut)?"""
904
+ if style is None or not max_kb:
905
+ return False
906
+ limit = max_kb * KB
907
+ return any(len(data) > limit for p, data in found.raw.items() if not p.endswith("__init__.py"))
908
+
909
+
910
+ def entries_lost(found: ScriptFiles, style: Style | None, ids: set[int | None]) -> list[tuple[str, int, int]]:
911
+ """The entries the parts of a folder show that a compile of its joined text did not
912
+ read (a bracket or a quote left open at the end of a part swallows the next ones):
913
+ [(file, line, id)]."""
914
+ if not found.package or style is None:
915
+ return []
916
+ lay = layout(found.text, style)
917
+ if lay is None:
918
+ return []
919
+ out = []
920
+ for c in lay.chunks:
921
+ if c.id is not None and c.id not in ids:
922
+ path, line = found.where(c.start + 1)
923
+ out.append((path, line or 0, c.id))
924
+ return out