codetrail-cli 1.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. codetrail/__init__.py +3 -0
  2. codetrail/__main__.py +5 -0
  3. codetrail/agents/__init__.py +4 -0
  4. codetrail/agents/convert.py +92 -0
  5. codetrail/agents/export.py +284 -0
  6. codetrail/agents/instructions.py +42 -0
  7. codetrail/agents/protocol.py +111 -0
  8. codetrail/agents/tools.py +185 -0
  9. codetrail/assistant/__init__.py +161 -0
  10. codetrail/assistant/claude_code.py +330 -0
  11. codetrail/assistant/codex.py +260 -0
  12. codetrail/assistant/environment.py +66 -0
  13. codetrail/assistant/estimate.py +225 -0
  14. codetrail/assistant/fake.py +74 -0
  15. codetrail/assistant/guard.py +80 -0
  16. codetrail/assistant/guard_hook.py +58 -0
  17. codetrail/assistant/local.py +245 -0
  18. codetrail/assistant/local_tools.py +177 -0
  19. codetrail/assistant/prompts.py +404 -0
  20. codetrail/assistant/routing.py +86 -0
  21. codetrail/assistant/runner.py +120 -0
  22. codetrail/assistant/status.py +186 -0
  23. codetrail/assistant/usage.py +93 -0
  24. codetrail/bridge.py +292 -0
  25. codetrail/cli.py +407 -0
  26. codetrail/config.py +485 -0
  27. codetrail/database/__init__.py +60 -0
  28. codetrail/database/migrations/0001_facts.sql +34 -0
  29. codetrail/database/migrations/0002_settings.sql +5 -0
  30. codetrail/database/migrations/0003_learning.sql +34 -0
  31. codetrail/database/migrations/0004_assistant_usage.sql +24 -0
  32. codetrail/database/migrations/0005_page_failures.sql +7 -0
  33. codetrail/database/migrations/0006_metric_values.sql +9 -0
  34. codetrail/errors.py +5 -0
  35. codetrail/extract/__init__.py +249 -0
  36. codetrail/extract/adr.py +98 -0
  37. codetrail/extract/github_actions.py +140 -0
  38. codetrail/extract/openapi.py +101 -0
  39. codetrail/extract/python.py +221 -0
  40. codetrail/extract/swift.py +204 -0
  41. codetrail/extract/terraform.py +166 -0
  42. codetrail/extract/typescript.py +464 -0
  43. codetrail/facts/__init__.py +112 -0
  44. codetrail/facts/store.py +209 -0
  45. codetrail/generate/__init__.py +1 -0
  46. codetrail/generate/failures.py +31 -0
  47. codetrail/generate/outline.py +159 -0
  48. codetrail/generate/revise.py +72 -0
  49. codetrail/generate/run.py +530 -0
  50. codetrail/generate/scope.py +135 -0
  51. codetrail/generate/validate.py +218 -0
  52. codetrail/guide/__init__.py +147 -0
  53. codetrail/learn/__init__.py +143 -0
  54. codetrail/learn/routes.py +139 -0
  55. codetrail/locales/ar/LC_MESSAGES/codetrail.mo +0 -0
  56. codetrail/locales/ar/LC_MESSAGES/codetrail.po +2236 -0
  57. codetrail/locales/codetrail.pot +1942 -0
  58. codetrail/lock.py +114 -0
  59. codetrail/metrics/__init__.py +6 -0
  60. codetrail/metrics/activity.py +96 -0
  61. codetrail/metrics/churn.py +156 -0
  62. codetrail/metrics/commits.py +45 -0
  63. codetrail/metrics/coverage.py +114 -0
  64. codetrail/metrics/decisions.py +77 -0
  65. codetrail/metrics/history.py +47 -0
  66. codetrail/metrics/inventory.py +63 -0
  67. codetrail/metrics/rationale.py +74 -0
  68. codetrail/metrics/releases.py +26 -0
  69. codetrail/metrics/report.py +111 -0
  70. codetrail/metrics/repository.py +128 -0
  71. codetrail/metrics/size.py +95 -0
  72. codetrail/py.typed +0 -0
  73. codetrail/remove.py +50 -0
  74. codetrail/repo/__init__.py +4 -0
  75. codetrail/repo/git.py +56 -0
  76. codetrail/repo/gitleaks.toml +6 -0
  77. codetrail/repo/history.py +262 -0
  78. codetrail/repo/mirror.py +137 -0
  79. codetrail/repo/refresh.py +41 -0
  80. codetrail/repo/rules.py +83 -0
  81. codetrail/repo/secrets.py +145 -0
  82. codetrail/repo/signal.py +65 -0
  83. codetrail/repo/source.py +166 -0
  84. codetrail/search.py +178 -0
  85. codetrail/server.py +45 -0
  86. codetrail/system.py +484 -0
  87. codetrail/update.py +276 -0
  88. codetrail/web/__init__.py +1 -0
  89. codetrail/web/app.py +897 -0
  90. codetrail/web/diagrams.py +414 -0
  91. codetrail/web/documentation.py +146 -0
  92. codetrail/web/i18n.py +44 -0
  93. codetrail/web/navigation.py +127 -0
  94. codetrail/web/render.py +126 -0
  95. codetrail/web/repository_view.py +230 -0
  96. codetrail/web/security.py +171 -0
  97. codetrail/web/static/brand/favicon.svg +1 -0
  98. codetrail/web/static/brand/mark.svg +1 -0
  99. codetrail/web/static/css/base.css +192 -0
  100. codetrail/web/static/css/components.css +2254 -0
  101. codetrail/web/static/css/content.css +310 -0
  102. codetrail/web/static/css/layout.css +410 -0
  103. codetrail/web/static/css/print.css +42 -0
  104. codetrail/web/static/css/tokens.css +228 -0
  105. codetrail/web/static/js/api.js +49 -0
  106. codetrail/web/static/js/ask.js +182 -0
  107. codetrail/web/static/js/d2-worker.js +34 -0
  108. codetrail/web/static/js/d2.js +28 -0
  109. codetrail/web/static/js/diagram.js +52 -0
  110. codetrail/web/static/js/documentation.js +24 -0
  111. codetrail/web/static/js/learning.js +47 -0
  112. codetrail/web/static/js/main.js +38 -0
  113. codetrail/web/static/js/outline.js +44 -0
  114. codetrail/web/static/js/palette.js +211 -0
  115. codetrail/web/static/js/repository.js +528 -0
  116. codetrail/web/static/js/resize.js +126 -0
  117. codetrail/web/static/js/shortcuts.js +87 -0
  118. codetrail/web/static/js/sidebar.js +194 -0
  119. codetrail/web/static/js/theme-init.js +27 -0
  120. codetrail/web/static/js/theme.js +42 -0
  121. codetrail/web/static/js/update.js +107 -0
  122. codetrail/web/static/vendor/d2/LICENSE +375 -0
  123. codetrail/web/static/vendor/d2/LICENSE-elk +263 -0
  124. codetrail/web/static/vendor/d2/LICENSE-wasm_exec +27 -0
  125. codetrail/web/static/vendor/d2/VERSION +12 -0
  126. codetrail/web/static/vendor/d2/d2.wasm +0 -0
  127. codetrail/web/static/vendor/d2/elk.js +105914 -0
  128. codetrail/web/static/vendor/d2/wasm_exec.js +656 -0
  129. codetrail/web/static/vendor/inter/InterVariable-Italic.woff2 +0 -0
  130. codetrail/web/static/vendor/inter/InterVariable.woff2 +0 -0
  131. codetrail/web/static/vendor/inter/LICENSE.txt +92 -0
  132. codetrail/web/static/vendor/inter/VERSION +6 -0
  133. codetrail/web/static/vendor/noto-sans-arabic/NotoSansArabic-wght.woff2 +0 -0
  134. codetrail/web/static/vendor/noto-sans-arabic/OFL.txt +93 -0
  135. codetrail/web/static/vendor/noto-sans-arabic/VERSION +12 -0
  136. codetrail/web/target_view.py +63 -0
  137. codetrail/web/templates/answers.html +25 -0
  138. codetrail/web/templates/area.html +46 -0
  139. codetrail/web/templates/base.html +37 -0
  140. codetrail/web/templates/decisions.html +30 -0
  141. codetrail/web/templates/diagram.html +10 -0
  142. codetrail/web/templates/digests.html +24 -0
  143. codetrail/web/templates/documentation.html +247 -0
  144. codetrail/web/templates/error.html +9 -0
  145. codetrail/web/templates/estimate.html +48 -0
  146. codetrail/web/templates/fact.html +29 -0
  147. codetrail/web/templates/home.html +196 -0
  148. codetrail/web/templates/layout/ask_panel.html +27 -0
  149. codetrail/web/templates/layout/header.html +32 -0
  150. codetrail/web/templates/layout/icons.html +27 -0
  151. codetrail/web/templates/layout/outline.html +4 -0
  152. codetrail/web/templates/layout/palette.html +27 -0
  153. codetrail/web/templates/layout/shortcuts.html +27 -0
  154. codetrail/web/templates/layout/sidebar.html +79 -0
  155. codetrail/web/templates/layout/update_dialog.html +18 -0
  156. codetrail/web/templates/macros.html +13 -0
  157. codetrail/web/templates/page.html +93 -0
  158. codetrail/web/templates/progress.html +46 -0
  159. codetrail/web/templates/repository.html +369 -0
  160. codetrail/web/templates/search.html +31 -0
  161. codetrail/web/templates/source.html +15 -0
  162. codetrail/web/templates/system.html +43 -0
  163. codetrail/web/templates/system_arrows.html +17 -0
  164. codetrail/web/templates/system_legend.html +9 -0
  165. codetrail/web/templates/update_log.html +30 -0
  166. codetrail/web/trend.py +22 -0
  167. codetrail_cli-1.4.1.dist-info/METADATA +372 -0
  168. codetrail_cli-1.4.1.dist-info/RECORD +171 -0
  169. codetrail_cli-1.4.1.dist-info/WHEEL +4 -0
  170. codetrail_cli-1.4.1.dist-info/entry_points.txt +3 -0
  171. codetrail_cli-1.4.1.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,63 @@
1
+ """Facts by kind, and the dependencies that arrived and left, from the validity ranges (design section 19.1).
2
+
3
+ Codetrail's history of a repository starts with its first snapshot: packages present then were there from the start,
4
+ not arrivals.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass
10
+
11
+ from codetrail.facts import EntityKind, Snapshot
12
+ from codetrail.facts.store import FactStore
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class KindCount:
17
+ kind: str
18
+ count: int
19
+ change: int | None # since the previous snapshot
20
+ trend: list[int] # over the last snapshots, oldest first
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Change:
25
+ package: str
26
+ snapshot: Snapshot # the snapshot that first saw it, or first no longer saw it
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Inventory:
31
+ since: Snapshot | None # the first snapshot: where Codetrail's history of the repository starts
32
+ kinds: list[KindCount] # the most facts first
33
+ arrived: list[Change] # newest first
34
+ left: list[Change] # newest first
35
+
36
+
37
+ def measure_inventory(store: FactStore, trend_updates: int) -> Inventory:
38
+ snapshots = store.snapshots()
39
+ if not snapshots:
40
+ return Inventory(None, [], [], [])
41
+ recent = snapshots[-trend_updates:]
42
+ counts = store.entity_counts([snapshot.id for snapshot in recent])
43
+ latest = counts[recent[-1].id]
44
+ previous = counts[recent[-2].id] if len(recent) > 1 else None
45
+ kinds = [
46
+ KindCount(kind, count, None if previous is None else count - previous.get(kind, 0),
47
+ [counts[snapshot.id].get(kind, 0) for snapshot in recent])
48
+ for kind, count in sorted(latest.items(), key=lambda item: (-item[1], item[0]))
49
+ ] # fmt: skip
50
+ by_id = {snapshot.id: snapshot for snapshot in snapshots}
51
+ arrived, left = [], []
52
+ for package, first, last in store.entity_spans(EntityKind.PACKAGE):
53
+ if first > snapshots[0].id:
54
+ arrived.append(Change(package, by_id[first]))
55
+ if last is not None:
56
+ after = next((snapshot for snapshot in snapshots if snapshot.id > last), None)
57
+ if after is not None:
58
+ left.append(Change(package, after))
59
+ return Inventory(snapshots[0], kinds, sorted(arrived, key=_newest_first), sorted(left, key=_newest_first))
60
+
61
+
62
+ def _newest_first(change: Change) -> tuple[int, str]:
63
+ return -change.snapshot.id, change.package
@@ -0,0 +1,74 @@
1
+ """The documented share of the guide's rationale, by source type and by page (design section 18.1).
2
+
3
+ Only area and concept pages count: digests accumulate and are mostly about commits, and saved answers follow the
4
+ reader's questions rather than the repository.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Iterable
10
+ from dataclasses import dataclass
11
+
12
+ from pathspec import GitIgnoreSpec
13
+
14
+ from codetrail.generate.validate import parse_rationale
15
+ from codetrail.guide import Page
16
+
17
+ GUIDE_KINDS = ("area", "concept")
18
+ SOURCE_KINDS = ("adr", "docs", "code", "commit")
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class InferredBlock:
23
+ page: Page
24
+ first_line: str
25
+
26
+
27
+ @dataclass(frozen=True)
28
+ class PageRationale:
29
+ page: Page
30
+ documented: int
31
+ inferred: int
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class RationaleMetric:
36
+ documented: int
37
+ inferred: int
38
+ by_source: dict[str, int]
39
+ pages: list[PageRationale] # most inferred first, then by title
40
+ inferred_blocks: list[InferredBlock]
41
+
42
+
43
+ def guide_pages(pages: Iterable[Page]) -> list[Page]:
44
+ return [page for page in pages if page.kind in GUIDE_KINDS]
45
+
46
+
47
+ def citation_source(citation: str, adr: GitIgnoreSpec, documents: GitIgnoreSpec) -> str:
48
+ """Where a documented block's quote comes from: a commit, an ADR, another document, or code (a comment)."""
49
+ if citation.startswith("commit:"):
50
+ return "commit"
51
+ path = citation.split("#", 1)[0]
52
+ if adr.match_file(path):
53
+ return "adr"
54
+ return "docs" if documents.match_file(path) else "code"
55
+
56
+
57
+ def measure_rationale(pages: Iterable[Page], adr: GitIgnoreSpec, documents: GitIgnoreSpec) -> RationaleMetric:
58
+ by_source = dict.fromkeys(SOURCE_KINDS, 0)
59
+ rows: list[PageRationale] = []
60
+ inferred_blocks: list[InferredBlock] = []
61
+ for page in guide_pages(pages):
62
+ documented = inferred = 0
63
+ for block in parse_rationale(page.body):
64
+ if block.kind == "documented":
65
+ documented += 1
66
+ by_source[citation_source(block.citation or "", adr, documents)] += 1
67
+ else:
68
+ inferred += 1
69
+ inferred_blocks.append(InferredBlock(page, next(iter(block.text.splitlines()), "")))
70
+ rows.append(PageRationale(page, documented, inferred))
71
+ rows.sort(key=lambda row: (-row.inferred, row.page.title.casefold(), row.page.id))
72
+ inferred_blocks.sort(key=lambda block: block.page.id) # stable: a page's blocks stay in order
73
+ return RationaleMetric(sum(row.documented for row in rows), sum(row.inferred for row in rows), by_source, rows,
74
+ inferred_blocks) # fmt: skip
@@ -0,0 +1,26 @@
1
+ """Releases: the target's tags that point into the branch's history (design section 19.1)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import statistics
6
+ from collections.abc import Sequence
7
+ from dataclasses import dataclass, field
8
+ from itertools import pairwise
9
+
10
+ from codetrail.repo.history import Tag
11
+
12
+ DAY_SECONDS = 86_400
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Releases:
17
+ tags: list[Tag] = field(default_factory=list) # newest first
18
+ since_latest: int | None = None # commits after the latest release
19
+ median_days: int | None = None # between consecutive releases
20
+
21
+
22
+ def measure_releases(tags: Sequence[Tag], since_latest: int | None) -> Releases:
23
+ newest_first = sorted(tags, key=lambda tag: (tag.date, tag.name), reverse=True)
24
+ gaps = [newer.date - older.date for newer, older in pairwise(newest_first)]
25
+ median = round(statistics.median(gaps) / DAY_SECONDS) if gaps else None
26
+ return Releases(newest_first, since_latest, median)
@@ -0,0 +1,111 @@
1
+ """The documentation metrics' report: every metric, read from the target's data (design section 18.2).
2
+
3
+ Documents are read from `source/` through the allowed-files reader, so excluded and secret files are never read; commits
4
+ come from the filtered history, with flagged messages withheld. If the history can't be read, the commit metric is
5
+ missing and the rest of the report stands.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from dataclasses import dataclass
12
+ from datetime import date
13
+ from enum import StrEnum
14
+
15
+ from pathspec import GitIgnoreSpec
16
+
17
+ from codetrail.config import GlobalConfig, Paths, load_target
18
+ from codetrail.errors import CodetrailError
19
+ from codetrail.facts import EntityKind
20
+ from codetrail.facts.store import FactStore
21
+ from codetrail.guide import GuideRepository
22
+ from codetrail.metrics.commits import CommitMetric, measure_commits
23
+ from codetrail.metrics.coverage import ExplainedMetric, MentionMetric, measure_explained, measure_mentions
24
+ from codetrail.metrics.decisions import DecisionMetric, measure_decisions
25
+ from codetrail.metrics.rationale import RationaleMetric, measure_rationale
26
+ from codetrail.repo.history import latest_commits
27
+ from codetrail.repo.secrets import SecretScanner
28
+ from codetrail.repo.source import SourceManifest, allowed_reader
29
+
30
+ logger = logging.getLogger(__name__)
31
+
32
+
33
+ class Metric(StrEnum):
34
+ DOCUMENTED_SHARE = "documented_share"
35
+ COMMIT_WHY_SHARE = "commit_why_share"
36
+ ADR_ATTENTION = "adr_attention"
37
+ MENTIONED_SHARE = "mentioned_share"
38
+ EXPLAINED_SHARE = "explained_share"
39
+ # The repository statistics recorded for their trend (design section 19.4)
40
+ COMMITS = "commits"
41
+ CODE_LINES = "code_lines"
42
+ SOURCE_FILES = "source_files"
43
+ TEST_SHARE = "test_share"
44
+
45
+
46
+ @dataclass(frozen=True)
47
+ class Value:
48
+ numerator: int
49
+ denominator: int | None # None for a count
50
+
51
+ @property
52
+ def share(self) -> float | None:
53
+ """The share, or None for a count or when there was nothing to measure."""
54
+ return self.numerator / self.denominator if self.denominator else None
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ class MetricsReport:
59
+ rationale: RationaleMetric
60
+ commits: CommitMetric | None # None when the history couldn't be read
61
+ decisions: DecisionMetric
62
+ mentions: MentionMetric
63
+ explained: ExplainedMetric
64
+
65
+ def values(self) -> dict[Metric, Value]:
66
+ rationale, mentions = self.rationale, self.mentions
67
+ values = {
68
+ Metric.DOCUMENTED_SHARE: Value(rationale.documented, rationale.documented + rationale.inferred),
69
+ Metric.ADR_ATTENTION: Value(self.decisions.attention, None),
70
+ Metric.MENTIONED_SHARE: Value(len(mentions.mentioned), len(mentions.mentioned) + len(mentions.unmentioned)),
71
+ Metric.EXPLAINED_SHARE: Value(self.explained.explained, self.explained.total),
72
+ }
73
+ if self.commits is not None:
74
+ values[Metric.COMMIT_WHY_SHARE] = Value(self.commits.explains_why, self.commits.measured)
75
+ return {metric: values[metric] for metric in Metric if metric in values}
76
+
77
+
78
+ def build_report(paths: Paths, name: str, settings: GlobalConfig, store: FactStore, today: date) -> MetricsReport:
79
+ target = load_target(paths, name)
80
+ data = paths.target_data(name)
81
+ guide = GuideRepository(data / "guide")
82
+ pages = guide.pages() if guide.root.exists() else []
83
+ manifest = SourceManifest.load(data / "source.json")
84
+ files = manifest.files if manifest else {}
85
+ document_globs = GitIgnoreSpec.from_lines(target.metrics.document_globs)
86
+ read = allowed_reader(data / "source", files, settings.extract.max_file_bytes)
87
+ documents = {}
88
+ for path in files:
89
+ if document_globs.match_file(path):
90
+ content = read(path)
91
+ # An image under docs/ matches the globs but isn't text: its bytes would cost time and fake mentions.
92
+ # A NUL byte marks it binary, as the code size counts it (metrics/size.py).
93
+ if content is not None and b"\0" not in content:
94
+ documents[path] = content.decode("utf-8", "replace")
95
+ commits = None
96
+ if manifest is not None:
97
+ try:
98
+ found = latest_commits(data / "mirror.git", manifest.commit, settings.metrics.commit_window,
99
+ SecretScanner(settings.tools)) # fmt: skip
100
+ commits = measure_commits(found, target.metrics.why, settings.metrics.max_message_chars)
101
+ except (CodetrailError, OSError) as error:
102
+ logger.warning("The commit metric isn't available: %s", error)
103
+ entities = store.entities()
104
+ decisions = [entity for entity in entities if entity.kind == EntityKind.DECISION]
105
+ return MetricsReport(
106
+ measure_rationale(pages, GitIgnoreSpec.from_lines(target.adr.paths), document_globs),
107
+ commits,
108
+ measure_decisions(decisions, pages, today, settings.metrics.proposed_adr_days),
109
+ measure_mentions(entities, documents),
110
+ measure_explained(entities, pages),
111
+ )
@@ -0,0 +1,128 @@
1
+ """The repository statistics' report: history, releases, code size and facts, read from the target's data (19.3).
2
+
3
+ Files are read from `source/` through the allowed-files reader, so excluded and secret files are never read; commit
4
+ subjects and tags pass gitleaks first. If the history or the tags can't be read, those parts are missing and the rest of
5
+ the report stands.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ from dataclasses import dataclass, replace
12
+ from datetime import UTC, date, datetime, timedelta
13
+ from pathlib import Path
14
+
15
+ from pathspec import GitIgnoreSpec
16
+
17
+ from codetrail.config import GlobalConfig, Paths, load_target
18
+ from codetrail.errors import CodetrailError
19
+ from codetrail.facts.store import FactStore
20
+ from codetrail.guide import GuideRepository, Page
21
+ from codetrail.metrics.activity import Activity, CommitTypes, measure_activity, measure_types
22
+ from codetrail.metrics.churn import Churn, measure_churn
23
+ from codetrail.metrics.coverage import page_scope
24
+ from codetrail.metrics.inventory import Inventory, measure_inventory
25
+ from codetrail.metrics.rationale import measure_rationale
26
+ from codetrail.metrics.releases import Releases, measure_releases
27
+ from codetrail.metrics.report import Metric, Value
28
+ from codetrail.metrics.size import CodeSize, measure_size
29
+ from codetrail.repo.history import (
30
+ changed_files,
31
+ commit_count,
32
+ commit_times,
33
+ commit_totals,
34
+ files_changed_since,
35
+ first_commit_time,
36
+ latest_commits,
37
+ read_tags,
38
+ )
39
+ from codetrail.repo.secrets import SecretScanner
40
+ from codetrail.repo.source import SourceManifest, allowed_reader
41
+
42
+ logger = logging.getLogger(__name__)
43
+ # A part that fails these ways is unavailable and the rest of the report stands. ValueError and OverflowError come
44
+ # from a date past the year 9999, which a commit or tag can claim and Python's dates can't hold.
45
+ UNREADABLE = (CodetrailError, OSError, ValueError, OverflowError)
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class RepositoryReport:
50
+ activity: Activity | None # None when the history couldn't be read
51
+ types: CommitTypes | None # None, with the releases, when the commits or tags couldn't be read
52
+ releases: Releases | None
53
+ size: CodeSize
54
+ inventory: Inventory
55
+ churn: Churn | None = None # None when the history couldn't be read
56
+
57
+ def values(self) -> dict[Metric, Value]:
58
+ """The statistics recorded per update for their trend (design section 19.4)."""
59
+ size = self.size
60
+ values = {
61
+ Metric.CODE_LINES: Value(size.code_lines, None),
62
+ Metric.SOURCE_FILES: Value(size.files, None),
63
+ Metric.TEST_SHARE: Value(size.test_lines, size.code_lines),
64
+ }
65
+ if self.activity is not None:
66
+ values[Metric.COMMITS] = Value(self.activity.commits, None)
67
+ return {metric: values[metric] for metric in Metric if metric in values}
68
+
69
+
70
+ def build_repository_report(
71
+ paths: Paths, name: str, settings: GlobalConfig, store: FactStore, today: date
72
+ ) -> RepositoryReport:
73
+ target = load_target(paths, name)
74
+ data = paths.target_data(name)
75
+ manifest = SourceManifest.load(data / "source.json")
76
+ files = manifest.files if manifest else {}
77
+ read = allowed_reader(data / "source", files, settings.extract.max_file_bytes)
78
+ documents = GitIgnoreSpec.from_lines(target.metrics.document_globs)
79
+ tests = GitIgnoreSpec.from_lines(target.metrics.test_globs)
80
+ size = measure_size(files, read, settings.metrics.languages, documents, tests)
81
+ activity = types = releases = churn = None
82
+ if manifest is not None:
83
+ mirror, end = data / "mirror.git", manifest.commit
84
+ try:
85
+ commits, merges = commit_totals(mirror, end)
86
+ times = commit_times(mirror, end, settings.metrics.history_limit)
87
+ activity = measure_activity(times, commits, merges, today, settings.metrics.activity_months)
88
+ first = first_commit_time(mirror, end) if activity.cut else None
89
+ if first is not None: # the times read stop short of the first commit
90
+ activity = replace(activity, first=datetime.fromtimestamp(first, UTC).date())
91
+ except UNREADABLE as error:
92
+ logger.warning("The history isn't available: %s", error)
93
+ try:
94
+ scanner = SecretScanner(settings.tools)
95
+ latest = latest_commits(mirror, end, settings.metrics.commit_window, scanner)
96
+ types = measure_types(latest, target.metrics.types, settings.metrics.max_message_chars)
97
+ tags = read_tags(mirror, end, scanner)
98
+ newest = max(tags, key=lambda tag: (tag.date, tag.name), default=None)
99
+ releases = measure_releases(tags, commit_count(mirror, newest.commit, end) if newest else None)
100
+ except UNREADABLE as error:
101
+ types = None
102
+ logger.warning("The commit types and releases aren't available: %s", error)
103
+ try:
104
+ changes = changed_files(mirror, end, settings.metrics.churn_window)
105
+ recent = files_changed_since(mirror, end, today - timedelta(days=settings.metrics.quiet_days))
106
+ code_lines = {file.path: file.lines for file in size.largest}
107
+ areas = _areas(data / "guide", target.adr.paths, target.metrics.document_globs)
108
+ metrics = settings.metrics
109
+ churn = measure_churn(changes, files, code_lines, recent, areas, today, metrics.folder_depth,
110
+ metrics.heatmap_tiles, metrics.churn_window) # fmt: skip
111
+ except UNREADABLE as error:
112
+ logger.warning("Where change happens isn't available: %s", error)
113
+ inventory = measure_inventory(store, settings.metrics.trend_updates)
114
+ return RepositoryReport(activity, types, releases, size, inventory, churn)
115
+
116
+
117
+ def _areas(guide_root: Path, adr_paths: list[str], document_globs: list[str]) -> list[tuple[Page, list[str], int, int]]:
118
+ """Each area page with a usable scope, with its documented and inferred rationale blocks."""
119
+ guide = GuideRepository(guide_root)
120
+ pages = guide.pages() if guide.root.exists() else []
121
+ rationale = measure_rationale(pages, GitIgnoreSpec.from_lines(adr_paths), GitIgnoreSpec.from_lines(document_globs))
122
+ counts = {row.page.id: (row.documented, row.inferred) for row in rationale.pages}
123
+ areas = []
124
+ for page in pages:
125
+ entry = page_scope(page) if page.kind == "area" else None
126
+ if entry is not None:
127
+ areas.append((page, entry.scope_paths, *counts.get(page.id, (0, 0))))
128
+ return areas
@@ -0,0 +1,95 @@
1
+ """Code size: files and lines by language, documents and tests (design section 19.1).
2
+
3
+ A file's language is looked up by its name, then its suffix, in the configured map: a dictionary lookup, never a
4
+ pattern a repository could control.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from collections.abc import Callable, Iterable, Mapping
10
+ from dataclasses import dataclass
11
+ from pathlib import PurePosixPath
12
+
13
+ from pathspec import GitIgnoreSpec
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class LanguageSize:
18
+ language: str
19
+ files: int
20
+ lines: int
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class FileSize:
25
+ path: str
26
+ language: str
27
+ lines: int
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class CodeSize:
32
+ files: int # every allowed file
33
+ languages: list[LanguageSize] # the most lines first
34
+ documents: LanguageSize
35
+ other_files: int # in no language
36
+ binary_files: int
37
+ too_large_files: int # over extract.max_file_bytes, or unreadable
38
+ code_lines: int # lines in files with a language, tests included
39
+ test_lines: int
40
+ largest: list[FileSize] # files with a language, the most lines first
41
+
42
+
43
+ def count_lines(content: bytes) -> int:
44
+ """The newline bytes, plus one for a last line without one."""
45
+ return content.count(b"\n") + (1 if content and not content.endswith(b"\n") else 0)
46
+
47
+
48
+ def language_of(path: str, languages: Mapping[str, str]) -> str | None:
49
+ name = PurePosixPath(path).name.lower()
50
+ if not name.startswith(".") and name in languages:
51
+ return languages[name]
52
+ suffix = PurePosixPath(name).suffix
53
+ return languages.get(suffix) if suffix else None
54
+
55
+
56
+ def measure_size(
57
+ paths: Iterable[str],
58
+ read: Callable[[str], bytes | None],
59
+ languages: Mapping[str, str],
60
+ documents: GitIgnoreSpec,
61
+ tests: GitIgnoreSpec,
62
+ ) -> CodeSize:
63
+ """Reads each allowed file once and sorts it: too large, binary, a document, a language or other."""
64
+ files = other = binary = too_large = document_files = document_lines = test_lines = 0
65
+ by_language: dict[str, list[int]] = {}
66
+ largest = []
67
+ for path in paths:
68
+ files += 1
69
+ content = read(path)
70
+ if content is None:
71
+ too_large += 1
72
+ continue
73
+ if b"\0" in content:
74
+ binary += 1
75
+ continue
76
+ lines = count_lines(content)
77
+ if documents.match_file(path):
78
+ document_files += 1
79
+ document_lines += lines
80
+ continue
81
+ language = language_of(path, languages)
82
+ if language is None:
83
+ other += 1
84
+ continue
85
+ counts = by_language.setdefault(language, [0, 0])
86
+ counts[0] += 1
87
+ counts[1] += lines
88
+ largest.append(FileSize(path, language, lines))
89
+ if tests.match_file(path):
90
+ test_lines += lines
91
+ sizes = [LanguageSize(language, counts[0], counts[1]) for language, counts in by_language.items()]
92
+ sizes.sort(key=lambda size: (-size.lines, -size.files, size.language))
93
+ largest.sort(key=lambda file: (-file.lines, file.path))
94
+ return CodeSize(files, sizes, LanguageSize("Documents", document_files, document_lines), other, binary,
95
+ too_large, sum(size.lines for size in sizes), test_lines, largest) # fmt: skip
codetrail/py.typed ADDED
File without changes
codetrail/remove.py ADDED
@@ -0,0 +1,50 @@
1
+ """Removing a target: everything Codetrail keeps for it, never the repository it teaches (AGENTS.md, Never do, 6)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import shutil
6
+ import tomllib
7
+ from collections.abc import Callable
8
+ from pathlib import Path
9
+
10
+ from codetrail.config import Paths, check_containment, validate_target_name
11
+ from codetrail.errors import CodetrailError
12
+ from codetrail.lock import existing_target_lock, target_removal
13
+
14
+
15
+ def remove_target(paths: Paths, name: str, confirm: Callable[[list[Path]], bool]) -> bool:
16
+ """Shows `confirm` the target's existing locations and deletes them if it agrees; returns whether it did.
17
+
18
+ The settings file goes last, and its lock is held until then: a removal that fails part way leaves a target
19
+ that can be removed again, and no command can start using the target meanwhile."""
20
+ settings = paths.target_file(validate_target_name(name))
21
+ if not settings.exists():
22
+ raise CodetrailError(f"No target named {name!r}.")
23
+ repository = _repository(settings)
24
+ if repository is not None: # the XDG variables may have moved Codetrail's folders since the target was added
25
+ check_containment(paths, repository)
26
+ data = paths.target_data(name)
27
+ locations = [data, paths.target_state(name), paths.ignore_file(name), settings]
28
+ if not confirm([location for location in locations if _exists(location)]):
29
+ return False
30
+ # Refused while serve, update or files uses the target, or an update runs.
31
+ with target_removal(paths, name), existing_target_lock(paths, name):
32
+ for location in locations:
33
+ if location.is_dir() and not location.is_symlink():
34
+ shutil.rmtree(location) # never follows a symlink inside the folder
35
+ elif _exists(location):
36
+ location.unlink() # a file, or a symlink: the link goes, never what it points to
37
+ return True
38
+
39
+
40
+ def _exists(location: Path) -> bool:
41
+ return location.is_symlink() or location.exists()
42
+
43
+
44
+ def _repository(settings: Path) -> Path | None:
45
+ """The target's repository, if its settings still say; a broken file mustn't stop the target being removed."""
46
+ try:
47
+ repository = tomllib.loads(settings.read_text(encoding="utf-8")).get("repository")
48
+ except OSError, UnicodeDecodeError, tomllib.TOMLDecodeError:
49
+ return None
50
+ return Path(repository) if isinstance(repository, str) else None
@@ -0,0 +1,4 @@
1
+ """Everything that reads a target repository: the mirror, the exclusion rules, the sources and their history.
2
+
3
+ Every other module reads a target only through this package (design section 2.2).
4
+ """
codetrail/repo/git.py ADDED
@@ -0,0 +1,56 @@
1
+ """Runs git for Codetrail: argument lists only, never a shell, never a prompt."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import subprocess
7
+ from collections.abc import Sequence
8
+ from contextlib import suppress
9
+ from pathlib import Path
10
+
11
+ from codetrail.errors import CodetrailError
12
+
13
+ # How long a git stopped early gets to remove its lock files before it's killed; it needs only a moment.
14
+ STOP_WAIT_SECONDS = 5
15
+
16
+
17
+ def run_git(
18
+ arguments: Sequence[str],
19
+ *,
20
+ git_dir: Path | None = None,
21
+ work_tree: Path | None = None,
22
+ input: bytes | None = None,
23
+ allowed_exit_codes: Sequence[int] = (0,),
24
+ ) -> bytes:
25
+ """Runs git and returns its standard output; any other exit code raises a CodetrailError with git's message."""
26
+ command = ["git"]
27
+ if git_dir is not None:
28
+ command.append(f"--git-dir={git_dir}")
29
+ if work_tree is not None:
30
+ command.append(f"--work-tree={work_tree}")
31
+ command.extend(arguments)
32
+ environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
33
+ environment["GIT_TERMINAL_PROMPT"] = "0"
34
+ # Run from the file system's root and stop repository discovery there: git never finds a repository (a
35
+ # target's, or another user's in a shared temporary folder) from where it runs.
36
+ environment["GIT_CEILING_DIRECTORIES"] = "/"
37
+ with subprocess.Popen( # noqa: S603
38
+ command, stdin=subprocess.PIPE if input is not None else None, stdout=subprocess.PIPE,
39
+ stderr=subprocess.PIPE, env=environment, cwd="/",
40
+ ) as process: # fmt: skip
41
+ try:
42
+ stdout, stderr = process.communicate(input)
43
+ except BaseException:
44
+ # Stopped early (Ctrl-C, a hangup or terminate signal): SIGTERM lets git remove its lock files, which
45
+ # SIGKILL would leave behind to fail every later command in that repository.
46
+ try:
47
+ process.terminate()
48
+ with suppress(subprocess.TimeoutExpired):
49
+ process.wait(STOP_WAIT_SECONDS)
50
+ finally:
51
+ process.kill() # does nothing once git has exited
52
+ raise
53
+ if process.returncode not in allowed_exit_codes:
54
+ message = stderr.decode("utf-8", "replace").strip() or f"exit code {process.returncode}"
55
+ raise CodetrailError(f"git {arguments[0]} failed: {message}")
56
+ return stdout
@@ -0,0 +1,6 @@
1
+ # Codetrail's gitleaks configuration (ADR 0002). It is always passed with --config, so a target's own .gitleaks.toml,
2
+ # a GITLEAKS_CONFIG setting or a .gitleaksignore can't change what is found.
3
+ title = "codetrail"
4
+
5
+ [extend]
6
+ useDefault = true