lib-installer 3.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. lib_installer/__init__.py +51 -0
  2. lib_installer/__main__.py +8 -0
  3. lib_installer/arduino/__init__.py +1 -0
  4. lib_installer/arduino/arduino.py +334 -0
  5. lib_installer/arduino/properties.py +168 -0
  6. lib_installer/arduino/sketchbook.py +133 -0
  7. lib_installer/cli.py +38 -0
  8. lib_installer/core/__init__.py +8 -0
  9. lib_installer/core/blocks.py +373 -0
  10. lib_installer/core/cli.py +1432 -0
  11. lib_installer/core/console.py +132 -0
  12. lib_installer/core/download.py +781 -0
  13. lib_installer/core/files.py +35 -0
  14. lib_installer/core/installer.py +1222 -0
  15. lib_installer/core/manifest.py +1487 -0
  16. lib_installer/core/steps.py +222 -0
  17. lib_installer/core/target.py +261 -0
  18. lib_installer/core/updates.py +120 -0
  19. lib_installer/core/usercode.py +193 -0
  20. lib_installer/core/yamlreader.py +417 -0
  21. lib_installer/esp32/__init__.py +1 -0
  22. lib_installer/esp32/component.py +354 -0
  23. lib_installer/esp32/esp32.py +491 -0
  24. lib_installer/esp32/project.py +768 -0
  25. lib_installer/pio/__init__.py +1 -0
  26. lib_installer/pio/ini.py +304 -0
  27. lib_installer/pio/library_json.py +161 -0
  28. lib_installer/pio/pio.py +208 -0
  29. lib_installer/stm32/__init__.py +1 -0
  30. lib_installer/stm32/checks.py +204 -0
  31. lib_installer/stm32/ide/__init__.py +110 -0
  32. lib_installer/stm32/ide/base.py +269 -0
  33. lib_installer/stm32/ide/cmake.py +299 -0
  34. lib_installer/stm32/ide/cubeide.py +344 -0
  35. lib_installer/stm32/ide/iar.py +509 -0
  36. lib_installer/stm32/ide/keil.py +419 -0
  37. lib_installer/stm32/ide/makefile.py +376 -0
  38. lib_installer/stm32/stm32.py +101 -0
  39. lib_installer/zephyr/__init__.py +1 -0
  40. lib_installer/zephyr/app.py +648 -0
  41. lib_installer/zephyr/module.py +375 -0
  42. lib_installer/zephyr/zephyr.py +620 -0
  43. lib_installer-3.0.1.dist-info/METADATA +583 -0
  44. lib_installer-3.0.1.dist-info/RECORD +48 -0
  45. lib_installer-3.0.1.dist-info/WHEEL +5 -0
  46. lib_installer-3.0.1.dist-info/entry_points.txt +2 -0
  47. lib_installer-3.0.1.dist-info/licenses/LICENSE +201 -0
  48. lib_installer-3.0.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1222 @@
1
+ """
2
+ Copying a library into a user's project.
3
+
4
+ Three rules shape everything here.
5
+
6
+ The library's files are replaced on every install, so an update actually
7
+ updates, and a library the user broke by accident is put right.
8
+
9
+ What the user wrote between USER CODE BEGIN and USER CODE END survives that,
10
+ carried into the new file in the section of the same name, the way STM32CubeMX
11
+ keeps the user's code when it generates again. See usercode.py.
12
+
13
+ Files listed under once are copied once and then belong to the user, for the
14
+ libraries that still list any. Those predate the USER CODE sections.
15
+ """
16
+
17
+ import fnmatch
18
+ import json
19
+ import os
20
+ import re
21
+ import shutil
22
+ import stat
23
+ import sys
24
+ from datetime import datetime, timezone
25
+ from pathlib import Path
26
+
27
+ from . import target, usercode
28
+ from .files import backup
29
+ from .manifest import COMPILED, DEFINE
30
+
31
+ # What a library repository carries that a firmware project must not.
32
+ #
33
+ # test/ matters most. STM32CubeIDE compiles every .c file under the project
34
+ # tree, so a test harness left in place brings a second main() into the build
35
+ # and the link fails with an error that tells the user nothing useful.
36
+ #
37
+ # This is a fixed list rather than "everything that is not a library file", so
38
+ # that anything the user put in the folder themselves survives.
39
+ #
40
+ # LICENSE.md and NOTICE are deliberately absent. The Apache licence requires both
41
+ # to travel with the code, and the NOTICE file is what carries the author's
42
+ # attribution into the user's product.
43
+ DEFAULT_CLEANUP = [
44
+ ".git",
45
+ ".github",
46
+ ".clang-format",
47
+ ".gitignore",
48
+ "build",
49
+ "test",
50
+ "inc",
51
+ "src",
52
+ "template",
53
+ "CMakeLists.txt",
54
+ "CONTRIBUTING.md",
55
+ "installer.yml",
56
+ "library.yml",
57
+ "install.py",
58
+ ]
59
+
60
+
61
+ class InstallError(Exception):
62
+ """The install cannot proceed, with a reason worth showing the user."""
63
+
64
+
65
+ # The shape of the record. 2 adds where a library came from, the project's own
66
+ # once files, the blocks in project files, and the run steps that ran. 3 is
67
+ # lib-installer's, which also says which platform the project is.
68
+ RECORD_VERSION = 3
69
+
70
+
71
+ class Result:
72
+ """What an install actually did, so the caller can report it honestly."""
73
+
74
+ def __init__(self, name, version, destination):
75
+ self.name = name
76
+ self.version = version
77
+ self.destination = Path(destination)
78
+ self.installed = []
79
+ self.created = []
80
+ self.kept = []
81
+ self.removed = []
82
+ # An update only: (from, to) for each of the user's files carried to
83
+ # where the new version wants it, and the library's own files from the
84
+ # last install that the new version no longer ships.
85
+ self.moved = []
86
+ self.dropped = []
87
+ # An update only: (from, to, [section names]) for each library file the
88
+ # new version puts in another folder, and the USER CODE sections that
89
+ # went with it, none for a file that had none.
90
+ self.carried = []
91
+ # An update only: defines the last install of this library put in the
92
+ # project and this version no longer asks for, so the IDE files can let
93
+ # go of them. Only ever ones this library added.
94
+ self.dropped_defines = []
95
+ # The once files of a 2.0.0 manifest, in the project's folder: created
96
+ # because they were missing, or kept because they were there. Either
97
+ # way the project's own from then on.
98
+ self.yours_created = []
99
+ self.yours_kept = []
100
+ # The extras with a root of a 2.1.0 manifest, in the project's folder: the
101
+ # library's own there, written on every install.
102
+ self.placed = []
103
+ # (file, [section names]) for each file whose USER CODE sections were
104
+ # carried into the new version.
105
+ self.preserved = []
106
+ # (file, backup, reason) for each copy put aside before something in it
107
+ # was replaced or removed.
108
+ self.backups = []
109
+ # What each file written holds outside its USER CODE sections, kept in
110
+ # the record so the next install can tell a hand edit there.
111
+ self.fingerprints = {}
112
+
113
+ @property
114
+ def was_update(self):
115
+ """True when the user's work was found and kept: a file, or a section in one."""
116
+ return bool(self.kept) or bool(self.preserved)
117
+
118
+
119
+ def install_to(library, destination, project_root=None, record=True):
120
+ """
121
+ Copy a library's files into one folder.
122
+
123
+ Where each file lands inside it is the library's layout: flat puts them all
124
+ at the top, mirror keeps the repository's own folders.
125
+
126
+ When the project already has this library in the same folder, this is an
127
+ update, and the project is brought in step with the new version. The
128
+ user's files move to wherever the new version puts them, rather than being
129
+ created again beside the old ones. Files the last install wrote and the new
130
+ version no longer ships are removed. Without that, a library that moved its
131
+ code into src/ left the old copies behind, CubeIDE compiled both, and a
132
+ fresh default config sat next to the header, silently winning over the
133
+ user's own.
134
+
135
+ Args:
136
+ library: a Manifest, from manifest.load().
137
+ destination: folder to write into. Created if missing.
138
+ project_root: where the record of what was installed lives. Without
139
+ it there is no record to read or write, so no update handling.
140
+ record: write the record at the end. install_in_place() turns this off,
141
+ since it records only after its cleanup has run.
142
+
143
+ Returns:
144
+ A Result listing every file written, created, kept, moved or dropped.
145
+ """
146
+ destination = Path(destination).resolve()
147
+
148
+ if destination.exists() and not destination.is_dir():
149
+ raise InstallError(f"{destination} is a file, so it cannot hold the library.")
150
+
151
+ result = Result(library.name, library.version, destination)
152
+ previous = _previous(project_root, library.name, destination) if project_root is not None else None
153
+ # Where the project's own once files go, each checked before anything is
154
+ # written, so a manifest that names a file it must not leaves no trace.
155
+ yours = _project_once(library, project_root, destination) if library.modern else []
156
+ # The library's own files in the project's folder, checked the same way.
157
+ placed = _project_placed(library, project_root, destination, previous)
158
+
159
+ # Every code file is checked before any is written. Markers that do not
160
+ # pair are the library's mistake, and finding one half way would leave the
161
+ # project with half of each version.
162
+ for entry in library.code_files:
163
+ try:
164
+ usercode.parse((library.root / entry.source).read_bytes())
165
+ except usercode.MarkerError as error:
166
+ raise InstallError(
167
+ f"{entry.source.as_posix()} in {library.name} has USER CODE markers that do not "
168
+ f"pair, {error}. Nothing was installed."
169
+ ) from error
170
+
171
+ destination.mkdir(parents=True, exist_ok=True)
172
+
173
+ # Code belongs to the library. Always replaced, so an update takes effect,
174
+ # with the user's sections carried over.
175
+ targets = {(destination / entry.destination).resolve() for entry in library.code_files}
176
+
177
+ for entry in library.code_files:
178
+ target = destination / entry.destination
179
+ earlier = _moved_code(previous, project_root, target, targets)
180
+ _write(library.root / entry.source, target, previous, project_root, result, earlier)
181
+
182
+ # The project's own: made when missing, and never touched again, not by
183
+ # an update and not by an uninstall.
184
+ for template, target in yours:
185
+ if target.exists():
186
+ result.yours_kept.append(target)
187
+ continue
188
+
189
+ target.parent.mkdir(parents=True, exist_ok=True)
190
+ shutil.copyfile(template, target)
191
+ result.yours_created.append(target)
192
+
193
+ # A kept file belongs to the user from the moment it first lands. The
194
+ # meaning once had before 2.0.0: in the library's own folder.
195
+ for entry in ([] if library.modern else library.once):
196
+ target = destination / entry.destination
197
+ template = library.root / entry.source
198
+ earlier = _earlier_copy(previous, project_root, target)
199
+ target.parent.mkdir(parents=True, exist_ok=True)
200
+
201
+ # The user's copy from the last install, somewhere the new version no
202
+ # longer looks. It moves, rather than a fresh default being created
203
+ # beside the header, where it would be found first and quietly replace
204
+ # the user's settings. It also takes over from a copy that is still
205
+ # the untouched default: that is the template itself when installing
206
+ # in place, or what an older installer created on an update.
207
+ if earlier is not None and (not target.exists() or _same_content(target, template)):
208
+ os.replace(earlier, target)
209
+ result.moved.append((earlier, target))
210
+ result.kept.append(target)
211
+ elif target.exists():
212
+ result.kept.append(target)
213
+ else:
214
+ shutil.copyfile(template, target)
215
+ result.created.append(target)
216
+
217
+ # The licence and the NOTICE ride along, because the licence says they must,
218
+ # and anything else the manifest lists comes with them. Copied as they are:
219
+ # they hold no USER CODE of the user's, and a README that shows the markers
220
+ # in an example must not be read as having sections.
221
+ for entry in library.present_extras():
222
+ source = library.root / entry.source
223
+ target = destination / entry.destination
224
+ target.parent.mkdir(parents=True, exist_ok=True)
225
+
226
+ if source.resolve() != target.resolve():
227
+ shutil.copyfile(source, target)
228
+
229
+ result.installed.append(target)
230
+
231
+ # Extras with a to: the library's own files, such as its tools, in the
232
+ # project's folder. Written again on every install, as every extra is.
233
+ for source, found in placed:
234
+ found.parent.mkdir(parents=True, exist_ok=True)
235
+
236
+ if source.resolve() != found:
237
+ shutil.copyfile(source, found)
238
+
239
+ result.placed.append(found)
240
+
241
+ if previous is not None:
242
+ _drop_stale(previous, project_root, destination, result)
243
+ _drop_stale_placed(previous, project_root, result)
244
+ result.dropped_defines = _dropped_defines(previous, library)
245
+
246
+ if project_root is not None and record:
247
+ _record(project_root, library, result)
248
+
249
+ return result
250
+
251
+
252
+ def _project_once(library, project_root, destination):
253
+ """
254
+ Where each once file of a 2.0.0 manifest lands: (template, target) pairs.
255
+
256
+ Raises InstallError, before anything is written, for a file with nowhere to
257
+ go, or one that would be a file the installer edits or reads as the
258
+ project's own, or one inside the library's folder, which every install
259
+ replaces.
260
+ """
261
+ if not library.once:
262
+ return []
263
+
264
+ root = target.current.once_root(project_root) if project_root is not None else None
265
+
266
+ if root is None:
267
+ listed = ", ".join(entry.destination for entry in library.once)
268
+ raise InstallError(
269
+ f"{library.name} has files for the project's own folder ({listed}). "
270
+ f"{target.current.no_once_root} Nothing was installed."
271
+ )
272
+
273
+ root = Path(root).resolve()
274
+ managed = [str(p).replace("\\", "/") for p in target.current.managed_files(project_root, library)]
275
+ placed = []
276
+
277
+ for entry in library.once:
278
+ to = entry.destination
279
+ found = (root / to).resolve()
280
+
281
+ # The manifest's check already refused .. and the like, so this only
282
+ # catches a link in the project that leads out of it.
283
+ if not _within(found, root):
284
+ raise InstallError(f"{library.name} would put {to} outside the project's folder. Nothing was installed.")
285
+
286
+ if found == destination or _within(found, destination):
287
+ raise InstallError(
288
+ f"{library.name} would put {to} inside its own folder, which every install replaces. "
289
+ "A file of the library's own belongs under files, with USER CODE sections for what is the "
290
+ "user's. Nothing was installed."
291
+ )
292
+
293
+ relative = found.relative_to(root).as_posix()
294
+
295
+ for pattern in managed:
296
+ if fnmatch.fnmatch(relative.lower(), pattern.lower()):
297
+ raise InstallError(
298
+ f"{library.name} would put {to} in the project, and {relative} is a file "
299
+ f"{target.current.name} edits or reads as the project's own. Nothing was installed."
300
+ )
301
+
302
+ placed.append((library.root / entry.source, found))
303
+
304
+ return placed
305
+
306
+
307
+ def _project_placed(library, project_root, destination, previous):
308
+ """
309
+ Where each extra with a root of a 2.1.0 manifest lands, in the project's
310
+ folder: (source, target) pairs.
311
+
312
+ Checked as once files are, before anything is written. A file already there
313
+ that the last install did not put there is the user's, and is never written
314
+ over: the install stops and says so.
315
+ """
316
+ if not library.placed:
317
+ return []
318
+
319
+ root = target.current.once_root(project_root) if project_root is not None else None
320
+
321
+ if root is None:
322
+ listed = ", ".join(entry.destination for entry in library.placed)
323
+ raise InstallError(
324
+ f"{library.name} has files for the project's own folder ({listed}). "
325
+ f"{target.current.no_once_root} Nothing was installed."
326
+ )
327
+
328
+ root = Path(root).resolve()
329
+ managed = [str(p).replace("\\", "/") for p in target.current.managed_files(project_root, library)]
330
+ ours = set()
331
+
332
+ for relative in (previous or {}).get("placed") or []:
333
+ if isinstance(relative, str):
334
+ ours.add((Path(project_root) / relative).resolve())
335
+
336
+ placed = []
337
+
338
+ for entry in library.placed:
339
+ to = entry.destination
340
+ source = library.root / entry.source
341
+ found = (root / to).resolve()
342
+
343
+ if not _within(found, root):
344
+ raise InstallError(f"{library.name} would put {to} outside the project's folder. Nothing was installed.")
345
+
346
+ if found == destination or _within(found, destination):
347
+ raise InstallError(
348
+ f"{library.name} would put {to} inside its own folder. An extra for the library's own "
349
+ "folder needs no to. Nothing was installed."
350
+ )
351
+
352
+ relative = found.relative_to(root).as_posix()
353
+
354
+ for pattern in managed:
355
+ if fnmatch.fnmatch(relative.lower(), pattern.lower()):
356
+ raise InstallError(
357
+ f"{library.name} would put {to} in the project, and {relative} is a file "
358
+ f"{target.current.name} edits or reads as the project's own. Nothing was installed."
359
+ )
360
+
361
+ if found.is_file() and found not in ours and source.resolve() != found and not _same_content(found, source):
362
+ raise InstallError(
363
+ f"{library.name} would write {relative} in the project, where a file of yours already is. "
364
+ "Move it or rename it, then install again. Nothing was installed."
365
+ )
366
+
367
+ placed.append((source, found))
368
+
369
+ return placed
370
+
371
+
372
+ def install_in_place(library, cleanup=True, project_root=None):
373
+ """
374
+ Turn a downloaded repository into a usable library folder, where it sits.
375
+
376
+ Used when someone drops the whole repository into their project. The library
377
+ files move up to the folder root and the repository scaffolding is removed,
378
+ leaving a folder that holds only what the firmware needs.
379
+
380
+ Args:
381
+ library: a Manifest, from manifest.load().
382
+ cleanup: remove the repository scaffolding. Off only for testing.
383
+ project_root: where the record of the install goes. Defaults to the
384
+ folder holding the library, which is the project only when the
385
+ library sits at its top level rather than in, say, Libs/.
386
+
387
+ Returns:
388
+ A Result, with removed listing everything deleted.
389
+ """
390
+ root = Path(project_root) if project_root is not None else library.root.parent
391
+ result = install_to(library, library.root, project_root=root, record=False)
392
+
393
+ if cleanup:
394
+ _remove_scaffolding(library, result)
395
+ _remove_unchosen(library, result)
396
+
397
+ _record(root, library, result)
398
+
399
+ return result
400
+
401
+
402
+ def _write(source, target, previous, project_root, result, earlier=None):
403
+ """
404
+ Put one of the library's files in place, keeping the user's sections.
405
+
406
+ The copy already there is put aside first when replacing it would lose
407
+ something of the user's: a section the new version has no place for,
408
+ sections whose markers no longer pair, or a hand edit outside them.
409
+
410
+ earlier is where the last install put this file, when the new version puts
411
+ it in another folder. Its sections are carried into the new place, the
412
+ same as from a copy already there.
413
+ """
414
+ # A mirror layout, or an explicit "to", can put a file in a subfolder that
415
+ # does not exist yet.
416
+ target.parent.mkdir(parents=True, exist_ok=True)
417
+
418
+ if source.resolve() == target.resolve():
419
+ # Installing in place: the file already is the library's own.
420
+ data = source.read_bytes()
421
+ else:
422
+ data = source.read_bytes()
423
+
424
+ current = target if target.is_file() else earlier
425
+
426
+ if current is not None:
427
+ old = current.read_bytes()
428
+ merged = usercode.merge(data, old)
429
+ reasons = _losses(merged, old, data, current, previous, project_root)
430
+
431
+ if reasons:
432
+ result.backups.append((current, backup(current), "; ".join(reasons)))
433
+
434
+ if merged.kept:
435
+ result.preserved.append((target, merged.kept))
436
+
437
+ if current != target:
438
+ result.carried.append((current, target, merged.kept))
439
+
440
+ data = merged.data
441
+
442
+ # Left alone when nothing changed, so the build does not see a newer
443
+ # file and compile it again for nothing.
444
+ if (current != target) or (old != data):
445
+ target.write_bytes(data)
446
+ else:
447
+ target.write_bytes(data)
448
+
449
+ result.installed.append(target)
450
+ result.fingerprints[target] = usercode.fingerprint(data)
451
+
452
+
453
+ def _moved_code(previous, project_root, target, targets):
454
+ """
455
+ Where the last install put a library file that this version puts elsewhere.
456
+
457
+ Found by its name among the library's own files from the last install,
458
+ littlefs/lfs_defines.h for littlefs/src/lfs_defines.h. Only when exactly one
459
+ matches, it is still there, and this version installs nothing in its place.
460
+ Anything else is not a move, and the file starts from what the library
461
+ ships, as before.
462
+ """
463
+ if previous is None or project_root is None or target.exists():
464
+ return None
465
+
466
+ found = []
467
+
468
+ for relative in previous.get("files") or []:
469
+ old = Path(project_root) / relative
470
+
471
+ try:
472
+ resolved = old.resolve()
473
+ except OSError:
474
+ continue
475
+
476
+ if old.name == target.name and resolved not in targets and old.is_file():
477
+ found.append(old)
478
+
479
+ return found[0] if len(found) == 1 else None
480
+
481
+
482
+ def _losses(merged, old, new, target, previous, project_root):
483
+ """What replacing the copy already in the project would lose, in words."""
484
+ reasons = []
485
+
486
+ if merged.lost:
487
+ reasons.append("the new version has no USER CODE " + ", ".join(merged.lost))
488
+
489
+ if merged.broken:
490
+ # Nothing in it could be told apart from the library's own text.
491
+ reasons.append(f"its USER CODE markers do not pair ({merged.broken})")
492
+ return reasons
493
+
494
+ if previous is None or project_root is None:
495
+ # No record of what was written here, so no way to tell a hand edit.
496
+ return reasons
497
+
498
+ relative = _relative(target, project_root)
499
+ known = previous.get("fingerprints")
500
+
501
+ if isinstance(known, dict) and relative in known:
502
+ if usercode.fingerprint(old) != known[relative]:
503
+ reasons.append("it was changed outside its USER CODE sections")
504
+ elif relative in (previous.get("once") or []):
505
+ # The user's own file until now, from a version that listed it under
506
+ # once. Anything outside its sections may be theirs.
507
+ if usercode.fingerprint(old) != usercode.fingerprint(new):
508
+ reasons.append("it was yours to edit until now, and differs outside its USER CODE sections")
509
+
510
+ return reasons
511
+
512
+
513
+ def _holds_user_work(path, previous, project_root):
514
+ """Whether a file about to be removed has something of the user's in it."""
515
+ try:
516
+ data = path.read_bytes()
517
+ except OSError:
518
+ return False
519
+
520
+ try:
521
+ lines, sections = usercode.parse(data)
522
+ except usercode.MarkerError:
523
+ return True
524
+
525
+ if any(b"".join(lines[first:last]).strip() for first, last in sections.values()):
526
+ return True
527
+
528
+ known = previous.get("fingerprints")
529
+ relative = _relative(path, project_root)
530
+
531
+ return isinstance(known, dict) and relative in known and usercode.fingerprint(data) != known[relative]
532
+
533
+
534
+ def _previous(project_root, name, destination):
535
+ """
536
+ The record of this library's last install, when it went into this folder.
537
+
538
+ Installed somewhere else, it is a separate copy, and nothing of it is moved
539
+ or removed from here.
540
+ """
541
+ entry = installed_libraries(project_root).get(name)
542
+
543
+ if not isinstance(entry, dict) or not isinstance(entry.get("folder"), str):
544
+ return None
545
+
546
+ try:
547
+ same = (Path(project_root) / entry["folder"]).resolve() == Path(destination).resolve()
548
+ except OSError:
549
+ return None
550
+
551
+ return entry if same else None
552
+
553
+
554
+ def _earlier_copy(previous, project_root, target):
555
+ """
556
+ Where the last install put the user's copy of this file, if that is
557
+ somewhere else and it is still there.
558
+ """
559
+ if previous is None:
560
+ return None
561
+
562
+ for relative in previous.get("once") or []:
563
+ earlier = Path(project_root) / relative
564
+
565
+ if earlier.name == target.name and earlier.resolve() != target.resolve() and earlier.is_file():
566
+ return earlier
567
+
568
+ return None
569
+
570
+
571
+ def _same_content(path, other):
572
+ """Whether two files hold the same bytes. The same file counts, of course."""
573
+ try:
574
+ return path.resolve() == other.resolve() or path.read_bytes() == other.read_bytes()
575
+ except OSError:
576
+ return False
577
+
578
+
579
+ def _drop_stale(previous, project_root, destination, result):
580
+ """
581
+ Remove the library's own files that the last install wrote and this one did not.
582
+
583
+ Only files the record lists as the library's, and only inside its folder.
584
+ The user's files are recorded apart from those, and anything the user added
585
+ was never recorded at all, so neither can end up here.
586
+ """
587
+ current = {p.resolve() for p in result.installed + result.created + result.kept}
588
+ carried = {old.resolve() for old, _, _ in result.carried}
589
+
590
+ for relative in previous.get("files") or []:
591
+ old = Path(project_root) / relative
592
+
593
+ try:
594
+ resolved = old.resolve()
595
+ except OSError:
596
+ continue
597
+
598
+ if resolved in current or not _within(resolved, destination) or not old.is_file():
599
+ continue
600
+
601
+ # Removed all the same, since a stale copy can be compiled beside the
602
+ # new one, but not before anything of the user's in it is put aside.
603
+ # A file whose sections went to its new place holds nothing more of the
604
+ # user's. Anything it held outside them was put aside as it went.
605
+ if resolved not in carried and _holds_user_work(old, previous, project_root):
606
+ try:
607
+ result.backups.append((old, backup(old), "no longer part of the library, and it held your changes"))
608
+ except OSError:
609
+ continue
610
+
611
+ try:
612
+ _unlink(old)
613
+ except OSError:
614
+ # A file the IDE holds open stays. Better than failing the update,
615
+ # and it is not reported as removed.
616
+ continue
617
+
618
+ result.dropped.append(old)
619
+ _prune(old.parent, destination)
620
+
621
+
622
+ def _drop_stale_placed(previous, project_root, result):
623
+ """
624
+ Remove the extras with a root that the last install put in the project's
625
+ folder and this one did not, such as a tool the library no longer ships.
626
+ Only files the record lists as placed, and only inside the project.
627
+ """
628
+ root = Path(project_root).resolve()
629
+ current = {p.resolve() for p in result.placed}
630
+
631
+ for relative in previous.get("placed") or []:
632
+ if not isinstance(relative, str):
633
+ continue
634
+
635
+ old = root / relative
636
+
637
+ try:
638
+ resolved = old.resolve()
639
+ except OSError:
640
+ continue
641
+
642
+ if resolved in current or not _within(resolved, root) or not old.is_file():
643
+ continue
644
+
645
+ try:
646
+ _unlink(old)
647
+ except OSError:
648
+ continue
649
+
650
+ result.dropped.append(old)
651
+ _prune(old.parent, root)
652
+
653
+
654
+ def _dropped_defines(previous, library):
655
+ """
656
+ The defines the last install of this library asked for and this one does not.
657
+
658
+ Read from the record, which is a file in the user's project anyone can
659
+ edit, so an entry is checked as strictly as one from a manifest before it
660
+ is used to find text to take out of a project file.
661
+ """
662
+ recorded = previous.get("defines")
663
+
664
+ if not isinstance(recorded, list):
665
+ return []
666
+
667
+ return [
668
+ define for define in recorded
669
+ if isinstance(define, str) and DEFINE.match(define) and define not in library.defines
670
+ ]
671
+
672
+
673
+ def _within(path, root):
674
+ """Whether path is inside root, not root itself. 3.8 has no is_relative_to."""
675
+ try:
676
+ Path(path).relative_to(root)
677
+ except ValueError:
678
+ return False
679
+
680
+ return Path(path) != Path(root)
681
+
682
+
683
+ def _prune(folder, stop):
684
+ """Remove folders left empty, up to but not including stop."""
685
+ folder = Path(folder)
686
+
687
+ while _within(folder, stop):
688
+ try:
689
+ folder.rmdir()
690
+ except OSError:
691
+ break
692
+
693
+ folder = folder.parent
694
+
695
+
696
+ def _remove_scaffolding(library, result):
697
+ """
698
+ Delete the repository-only paths, leaving anything the user added.
699
+
700
+ What is kept is worked out from where the files actually landed, not from
701
+ their names. A mirror layout leaves them in inc/ and src/, and those folders
702
+ are on the cleanup list, so going by name alone would install four files and
703
+ then delete all four.
704
+ """
705
+ keep = set()
706
+
707
+ for path in result.installed + result.created + result.kept:
708
+ try:
709
+ landed = path.relative_to(library.root)
710
+ except ValueError:
711
+ continue
712
+
713
+ # The top level name under the install folder, which is what the
714
+ # cleanup list is written in terms of.
715
+ if landed.parts:
716
+ keep.add(landed.parts[0])
717
+
718
+ for name in DEFAULT_CLEANUP:
719
+ target = library.root / name
720
+
721
+ if not target.exists() or name in keep:
722
+ continue
723
+
724
+ try:
725
+ if target.is_dir():
726
+ _rmtree(target)
727
+ else:
728
+ _unlink(target)
729
+ except OSError as error:
730
+ # A locked file is worth mentioning, never worth failing the install.
731
+ result.removed.append(f"{name} (could not remove: {error.strerror or error})")
732
+ continue
733
+
734
+ result.removed.append(name)
735
+
736
+
737
+ def _remove_unchosen(library, result):
738
+ """
739
+ Delete the files of the options not taken, from a library installed in place.
740
+
741
+ They came with the repository, and left there, STM32CubeIDE would compile
742
+ them with everything else under the project: spif's littlefs port without
743
+ littlefs, and a build that fails on a header nobody asked for.
744
+ """
745
+ for name, option in library.options.items():
746
+ if name in library.chosen:
747
+ continue
748
+
749
+ for entry in option.code_files:
750
+ target = library.root / entry.source
751
+
752
+ if not target.is_file():
753
+ continue
754
+
755
+ try:
756
+ _unlink(target)
757
+ except OSError as error:
758
+ result.removed.append(f"{entry.source.as_posix()} (could not remove: {error.strerror or error})")
759
+ continue
760
+
761
+ result.removed.append(entry.source.as_posix())
762
+ _prune(target.parent, library.root)
763
+
764
+
765
+ def _force_writable(func, path, _error):
766
+ """
767
+ Retry a failed delete after clearing the read-only bit.
768
+
769
+ Git marks everything under .git/objects read-only, and on Windows that stops
770
+ a delete outright rather than being advisory as it is on Unix. Without this,
771
+ removing a cloned repository always fails part way through.
772
+ """
773
+ try:
774
+ os.chmod(path, stat.S_IWRITE)
775
+ func(path)
776
+ except OSError:
777
+ raise
778
+
779
+
780
+ def _rmtree(path):
781
+ """Delete a folder, including read-only files. The keyword changed in 3.12."""
782
+ if sys.version_info >= (3, 12):
783
+ shutil.rmtree(path, onexc=_force_writable)
784
+ else:
785
+ shutil.rmtree(path, onerror=_force_writable)
786
+
787
+
788
+ def _unlink(path):
789
+ """Delete a file, including a read-only one."""
790
+ try:
791
+ path.unlink()
792
+ except PermissionError:
793
+ os.chmod(path, stat.S_IWRITE)
794
+ path.unlink()
795
+
796
+
797
+ def _record(project_root, library, result):
798
+ """Note what was installed, so update and uninstall have something to work from."""
799
+ project_root = Path(project_root)
800
+ path = project_root / target.current.record_name
801
+ data = {"libraries": {}}
802
+
803
+ if path.is_file():
804
+ try:
805
+ loaded = json.loads(path.read_text(encoding="utf-8"))
806
+ if isinstance(loaded, dict) and isinstance(loaded.get("libraries"), dict):
807
+ data = loaded
808
+ except (json.JSONDecodeError, OSError):
809
+ # A damaged record should never block an install. It gets rewritten.
810
+ pass
811
+
812
+ previous = data["libraries"].get(library.name)
813
+ previous = previous if isinstance(previous, dict) else {}
814
+
815
+ # What 2.0.0 adds to a record. An older installer reads past it.
816
+ data["record"] = RECORD_VERSION
817
+
818
+ # The platform, so a later run, an uninstall above all, knows it without
819
+ # being told, and a library for another platform is refused.
820
+ if getattr(target.current, "platform", None):
821
+ data["platform"] = target.current.platform
822
+
823
+ data["libraries"][library.name] = {
824
+ "version": library.version,
825
+ "repository": library.repository,
826
+ # Where it came from: owner/name on GitHub, the ref asked for, none for
827
+ # the default branch, and the commit that was, or a zip or a folder.
828
+ "origin": dict(library.origin),
829
+ "folder": _relative(result.destination, project_root),
830
+ "files": sorted(_relative(p, project_root) for p in result.installed),
831
+ "once": sorted(_relative(p, project_root) for p in result.created + result.kept),
832
+ # A 2.0.0 manifest's once files, the project's own: never removed.
833
+ "project_files": sorted(_relative(p, project_root) for p in result.yours_created + result.yours_kept),
834
+ # A 2.1.0 manifest's extras with a root: the library's own, in the
835
+ # project's folder, replaced on an update and removed by an uninstall.
836
+ "placed": sorted(_relative(p, project_root) for p in result.placed),
837
+ # The blocks it has in project files, and the run steps that ran, kept
838
+ # from the last install until the target and the steps say again.
839
+ "blocks": previous.get("blocks") if isinstance(previous.get("blocks"), dict) else {},
840
+ "ran": previous.get("ran") if isinstance(previous.get("ran"), dict) else {},
841
+ "fingerprints": {
842
+ _relative(p, project_root): value for p, value in sorted(result.fingerprints.items())
843
+ },
844
+ "defines": list(library.defines),
845
+ # The libraries this one needs, so an uninstall can refuse to take
846
+ # one out from under another, and say which are no longer needed.
847
+ "needs": sorted({dependency.name for dependency in library.dependencies}),
848
+ # Every option offered, and whether it was taken. One the record does
849
+ # not name is new to this project, and is asked about.
850
+ "options": {name: name in library.chosen for name in library.options},
851
+ "installed_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
852
+ }
853
+
854
+ try:
855
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
856
+ except OSError:
857
+ # A read-only project root is unusual, but it must not fail a good install.
858
+ pass
859
+
860
+
861
+ def note(project_root, name, **fields):
862
+ """
863
+ Add what was learnt after the files went in to a library's record: the
864
+ blocks the target wrote, the run steps that ran. Never raises, as _record.
865
+ """
866
+ path = Path(project_root) / target.current.record_name
867
+
868
+ try:
869
+ data = json.loads(path.read_text(encoding="utf-8"))
870
+ except (OSError, json.JSONDecodeError):
871
+ return
872
+
873
+ entry = data.get("libraries", {}).get(name) if isinstance(data, dict) and isinstance(data.get("libraries"), dict) else None
874
+
875
+ if not isinstance(entry, dict):
876
+ return
877
+
878
+ entry.update(fields)
879
+
880
+ try:
881
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
882
+ except OSError:
883
+ pass
884
+
885
+
886
+ class Installed:
887
+ """
888
+ A library as the record remembers it, which is all an uninstall has.
889
+
890
+ The record is a file in the user's project that anyone can edit, so what
891
+ comes from it is checked as it is read: a path that would lead out of the
892
+ library's folder is dropped, never deleted, and a define that could not
893
+ have come from a manifest is ignored.
894
+ """
895
+
896
+ def __init__(self, project_root, name, entry):
897
+ root = Path(project_root).resolve()
898
+ self.name = name
899
+ self.version = str(entry.get("version") or "")
900
+ self.folder = entry.get("folder") if isinstance(entry.get("folder"), str) else None
901
+ self.destination = (root / self.folder).resolve() if self.folder else None
902
+ self.needs = [str(n) for n in entry.get("needs") or [] if isinstance(n, str)]
903
+ self.defines = [d for d in entry.get("defines") or [] if isinstance(d, str) and DEFINE.match(d)]
904
+ self.files = self._inside(root, entry.get("files"))
905
+ self.once = self._inside(root, entry.get("once"))
906
+ self.project_files = self._in_project(root, entry.get("project_files"))
907
+ self.placed = self._in_project(root, entry.get("placed"))
908
+ self.entry = entry
909
+
910
+ @staticmethod
911
+ def _in_project(root, listed):
912
+ """The recorded project files, a 2.0.0 manifest's once files, inside the project."""
913
+ found = []
914
+
915
+ for relative in listed if isinstance(listed, list) else []:
916
+ if not isinstance(relative, str):
917
+ continue
918
+
919
+ try:
920
+ path = (root / relative).resolve()
921
+ except OSError:
922
+ continue
923
+
924
+ if _within(path, root):
925
+ found.append(path)
926
+
927
+ return found
928
+
929
+ def _inside(self, root, listed):
930
+ """The recorded paths that are files of this library's folder."""
931
+ if self.destination is None or not isinstance(listed, list):
932
+ return []
933
+
934
+ found = []
935
+
936
+ for relative in listed:
937
+ if not isinstance(relative, str):
938
+ continue
939
+
940
+ try:
941
+ path = (root / relative).resolve()
942
+ except OSError:
943
+ continue
944
+
945
+ if _within(path, self.destination) and _within(path, root):
946
+ found.append(path)
947
+
948
+ return found
949
+
950
+ @property
951
+ def compiled(self):
952
+ """The files a compiler takes, as posix names inside the library's folder."""
953
+ return [
954
+ path.relative_to(self.destination).as_posix()
955
+ for path in self.files + self.once
956
+ if path.suffix.lower() in COMPILED
957
+ ]
958
+
959
+
960
+ class Removal:
961
+ """What an uninstall did, so the caller can report it honestly."""
962
+
963
+ def __init__(self, library):
964
+ self.library = library
965
+ self.removed = []
966
+ # (file, backup, reason) for each file of the user's put aside first.
967
+ self.backups = []
968
+ # Files in the library's folder the record does not name: the user's own.
969
+ self.left = []
970
+ # The project's own files the library made once, left where they are.
971
+ self.yours = []
972
+ self.folder_removed = False
973
+ # Outcomes of taking it out of the project's build, from the target.
974
+ self.outcomes = []
975
+ # Libraries it needed that nothing left in the project needs any more.
976
+ self.unneeded = []
977
+
978
+
979
+ def uninstall(project_root, name):
980
+ """
981
+ Take a library out of a project: out of its build, its files, and the record.
982
+
983
+ What the user wrote in its files is not lost. A file of the user's own, a
984
+ configuration file the library created or one with something written in
985
+ its USER CODE sections or changed by hand, is copied aside before it goes,
986
+ as an update does. A file the library never installed is left where it is,
987
+ and with it the folder.
988
+
989
+ Raises:
990
+ InstallError when the library is not installed, or another one needs it.
991
+ """
992
+ root = Path(project_root).resolve()
993
+ installed = installed_libraries(root)
994
+ entry = installed.get(name)
995
+
996
+ if not isinstance(entry, dict):
997
+ known = ", ".join(sorted(installed)) or "nothing"
998
+ raise InstallError(
999
+ f"{name} is not installed in this project. {target.current.record_name} has {known}."
1000
+ )
1001
+
1002
+ needed_by = sorted(
1003
+ other for other, data in installed.items()
1004
+ if other != name and isinstance(data, dict) and name in (data.get("needs") or [])
1005
+ )
1006
+
1007
+ if needed_by:
1008
+ raise InstallError(
1009
+ f"{', '.join(needed_by)} {'needs' if len(needed_by) == 1 else 'need'} {name}, so it stays. "
1010
+ f"Uninstall {' and '.join(needed_by)} first, or name them all on one line."
1011
+ )
1012
+
1013
+ library = Installed(root, name, entry)
1014
+ removal = Removal(library)
1015
+
1016
+ # Defines another library still asks for stay in the project.
1017
+ others = {d for other, data in installed.items() if other != name and isinstance(data, dict)
1018
+ for d in data.get("defines") or []}
1019
+ library.defines = [d for d in library.defines if d not in others]
1020
+
1021
+ # Out of the build first: a project file still naming a file that is gone
1022
+ # fails the build, where one naming a file still there does not.
1023
+ removal.outcomes = target.current.uninstall(root, library)
1024
+
1025
+ for path in library.files + library.once:
1026
+ if not path.is_file():
1027
+ continue
1028
+
1029
+ reason = "it was yours to edit" if path in library.once else _what_is_yours(path, entry, root)
1030
+
1031
+ try:
1032
+ if reason:
1033
+ removal.backups.append((path, backup(path), reason))
1034
+
1035
+ _unlink(path)
1036
+ except OSError:
1037
+ continue
1038
+
1039
+ removal.removed.append(path)
1040
+ _prune(path.parent, library.destination)
1041
+
1042
+ # The library's own files in the project's folder go with it, as those in
1043
+ # its own folder do: they were written again on every install.
1044
+ for path in library.placed:
1045
+ if not path.is_file():
1046
+ continue
1047
+
1048
+ try:
1049
+ _unlink(path)
1050
+ except OSError:
1051
+ continue
1052
+
1053
+ removal.removed.append(path)
1054
+ _prune(path.parent, root)
1055
+
1056
+ # Never removed: they were the project's from the moment they landed.
1057
+ removal.yours = [path for path in library.project_files if path.is_file()]
1058
+
1059
+ if library.destination is not None and library.destination.is_dir():
1060
+ removal.left = sorted(p for p in library.destination.rglob("*") if p.is_file())
1061
+
1062
+ if not removal.left:
1063
+ try:
1064
+ library.destination.rmdir()
1065
+ removal.folder_removed = True
1066
+ except OSError:
1067
+ pass
1068
+
1069
+ _forget(root, name)
1070
+
1071
+ still = {need for other, data in installed.items() if other != name and isinstance(data, dict)
1072
+ for need in data.get("needs") or []}
1073
+ removal.unneeded = [need for need in library.needs if need in installed and need not in still]
1074
+
1075
+ return removal
1076
+
1077
+
1078
+ def _what_is_yours(path, entry, project_root):
1079
+ """
1080
+ Why a library file is saved before an uninstall removes it, or None.
1081
+
1082
+ Only the library's code is asked. An extra, a README for one, is copied as
1083
+ it is and has no fingerprint in the record, and one that shows USER CODE
1084
+ lines as an example must not be taken for the user's work.
1085
+ """
1086
+ known = entry.get("fingerprints")
1087
+ relative = _relative(path, project_root)
1088
+
1089
+ if not isinstance(known, dict) or relative not in known:
1090
+ return None
1091
+
1092
+ try:
1093
+ data = path.read_bytes()
1094
+ lines, sections = usercode.parse(data)
1095
+ except usercode.MarkerError:
1096
+ return "its USER CODE markers do not pair, so what is yours in it cannot be told apart"
1097
+ except OSError:
1098
+ return None
1099
+
1100
+ if usercode.fingerprint(data) != known[relative]:
1101
+ return "it was changed outside its USER CODE sections"
1102
+
1103
+ if any(b"".join(lines[first:last]).strip() for first, last in sections.values()):
1104
+ return "it has something between its USER CODE lines, such as your settings"
1105
+
1106
+ return None
1107
+
1108
+
1109
+ def _forget(project_root, name):
1110
+ """Take one library out of the record of what was installed."""
1111
+ path = Path(project_root) / target.current.record_name
1112
+
1113
+ try:
1114
+ data = json.loads(path.read_text(encoding="utf-8"))
1115
+ except (OSError, json.JSONDecodeError):
1116
+ return
1117
+
1118
+ if isinstance(data, dict) and isinstance(data.get("libraries"), dict):
1119
+ data["libraries"].pop(name, None)
1120
+
1121
+ try:
1122
+ path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
1123
+ except OSError:
1124
+ pass
1125
+
1126
+
1127
+ def _relative(path, root):
1128
+ """Path relative to root where possible, absolute where not, always posix style."""
1129
+ try:
1130
+ return Path(path).relative_to(root).as_posix()
1131
+ except ValueError:
1132
+ return Path(path).as_posix()
1133
+
1134
+
1135
+ def installed_libraries(project_root):
1136
+ """Read back what this tool has installed into a project. Empty when nothing has."""
1137
+ path = Path(project_root) / target.current.record_name
1138
+
1139
+ if not path.is_file():
1140
+ return {}
1141
+
1142
+ try:
1143
+ data = json.loads(path.read_text(encoding="utf-8"))
1144
+ except (json.JSONDecodeError, OSError):
1145
+ return {}
1146
+
1147
+ libraries = data.get("libraries")
1148
+
1149
+ return libraries if isinstance(libraries, dict) else {}
1150
+
1151
+
1152
+ def recorded_platform(folder):
1153
+ """The platform a folder's lib-installer record names, or None."""
1154
+ path = Path(folder) / target.Platform.record_name
1155
+
1156
+ try:
1157
+ data = json.loads(path.read_text(encoding="utf-8"))
1158
+ except (json.JSONDecodeError, OSError, ValueError):
1159
+ return None
1160
+
1161
+ return data.get("platform") if isinstance(data, dict) else None
1162
+
1163
+
1164
+ def migrate(project_root):
1165
+ """
1166
+ Move a project an installer before 3.0.0 installed into over to
1167
+ lib-installer, the first time lib-installer runs in it: the old record,
1168
+ such as .zephyr-installer.json, becomes .lib-installer.json with the
1169
+ platform in it, and the markers of its blocks in the project's files name
1170
+ lib-installer. What is inside each block is not touched, so the record's
1171
+ fingerprints of them still hold. Each file is saved first, as a .bak, and
1172
+ keeps its line endings.
1173
+
1174
+ Returns the files changed, the new record last, and nothing when there
1175
+ was nothing to move.
1176
+ """
1177
+ current = target.current
1178
+ legacy = getattr(current, "legacy", None)
1179
+
1180
+ if not legacy:
1181
+ return []
1182
+
1183
+ root = Path(project_root)
1184
+ old = root / current.legacy_record
1185
+ new = root / current.record_name
1186
+
1187
+ if not old.is_file() or new.exists():
1188
+ return []
1189
+
1190
+ # Only the markers this platform's installer wrote: # >>> zephyr-installer: spif >>>,
1191
+ # or ; <<< pio-installer: x <<< in platformio.ini.
1192
+ marker = re.compile(rf"(?m)^([ \t]*[#;][ \t]*)(>>>|<<<) {re.escape(legacy)}: ")
1193
+ changed = []
1194
+
1195
+ for path in current.legacy_files(root):
1196
+ path = Path(path)
1197
+
1198
+ try:
1199
+ text = path.read_bytes().decode("utf-8")
1200
+ except (OSError, UnicodeDecodeError):
1201
+ continue
1202
+
1203
+ updated = marker.sub(lambda found: f"{found.group(1)}{found.group(2)} {current.name}: ", text)
1204
+
1205
+ if updated != text:
1206
+ backup(path)
1207
+ path.write_bytes(updated.encode("utf-8"))
1208
+ changed.append(path)
1209
+
1210
+ try:
1211
+ data = json.loads(old.read_text(encoding="utf-8"))
1212
+ except (json.JSONDecodeError, OSError, ValueError):
1213
+ data = {}
1214
+
1215
+ data = data if isinstance(data, dict) else {}
1216
+ data["platform"] = current.platform
1217
+ backup(old)
1218
+ new.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
1219
+ old.unlink()
1220
+ changed.append(new)
1221
+
1222
+ return changed