codetrail-cli 1.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codetrail/__init__.py +3 -0
- codetrail/__main__.py +5 -0
- codetrail/agents/__init__.py +4 -0
- codetrail/agents/convert.py +92 -0
- codetrail/agents/export.py +284 -0
- codetrail/agents/instructions.py +42 -0
- codetrail/agents/protocol.py +111 -0
- codetrail/agents/tools.py +185 -0
- codetrail/assistant/__init__.py +161 -0
- codetrail/assistant/claude_code.py +330 -0
- codetrail/assistant/codex.py +260 -0
- codetrail/assistant/environment.py +66 -0
- codetrail/assistant/estimate.py +225 -0
- codetrail/assistant/fake.py +74 -0
- codetrail/assistant/guard.py +80 -0
- codetrail/assistant/guard_hook.py +58 -0
- codetrail/assistant/local.py +245 -0
- codetrail/assistant/local_tools.py +177 -0
- codetrail/assistant/prompts.py +404 -0
- codetrail/assistant/routing.py +86 -0
- codetrail/assistant/runner.py +120 -0
- codetrail/assistant/status.py +186 -0
- codetrail/assistant/usage.py +93 -0
- codetrail/bridge.py +292 -0
- codetrail/cli.py +407 -0
- codetrail/config.py +485 -0
- codetrail/database/__init__.py +60 -0
- codetrail/database/migrations/0001_facts.sql +34 -0
- codetrail/database/migrations/0002_settings.sql +5 -0
- codetrail/database/migrations/0003_learning.sql +34 -0
- codetrail/database/migrations/0004_assistant_usage.sql +24 -0
- codetrail/database/migrations/0005_page_failures.sql +7 -0
- codetrail/database/migrations/0006_metric_values.sql +9 -0
- codetrail/errors.py +5 -0
- codetrail/extract/__init__.py +249 -0
- codetrail/extract/adr.py +98 -0
- codetrail/extract/github_actions.py +140 -0
- codetrail/extract/openapi.py +101 -0
- codetrail/extract/python.py +221 -0
- codetrail/extract/swift.py +204 -0
- codetrail/extract/terraform.py +166 -0
- codetrail/extract/typescript.py +464 -0
- codetrail/facts/__init__.py +112 -0
- codetrail/facts/store.py +209 -0
- codetrail/generate/__init__.py +1 -0
- codetrail/generate/failures.py +31 -0
- codetrail/generate/outline.py +159 -0
- codetrail/generate/revise.py +72 -0
- codetrail/generate/run.py +530 -0
- codetrail/generate/scope.py +135 -0
- codetrail/generate/validate.py +218 -0
- codetrail/guide/__init__.py +147 -0
- codetrail/learn/__init__.py +143 -0
- codetrail/learn/routes.py +139 -0
- codetrail/locales/ar/LC_MESSAGES/codetrail.mo +0 -0
- codetrail/locales/ar/LC_MESSAGES/codetrail.po +2236 -0
- codetrail/locales/codetrail.pot +1942 -0
- codetrail/lock.py +114 -0
- codetrail/metrics/__init__.py +6 -0
- codetrail/metrics/activity.py +96 -0
- codetrail/metrics/churn.py +156 -0
- codetrail/metrics/commits.py +45 -0
- codetrail/metrics/coverage.py +114 -0
- codetrail/metrics/decisions.py +77 -0
- codetrail/metrics/history.py +47 -0
- codetrail/metrics/inventory.py +63 -0
- codetrail/metrics/rationale.py +74 -0
- codetrail/metrics/releases.py +26 -0
- codetrail/metrics/report.py +111 -0
- codetrail/metrics/repository.py +128 -0
- codetrail/metrics/size.py +95 -0
- codetrail/py.typed +0 -0
- codetrail/remove.py +50 -0
- codetrail/repo/__init__.py +4 -0
- codetrail/repo/git.py +56 -0
- codetrail/repo/gitleaks.toml +6 -0
- codetrail/repo/history.py +262 -0
- codetrail/repo/mirror.py +137 -0
- codetrail/repo/refresh.py +41 -0
- codetrail/repo/rules.py +83 -0
- codetrail/repo/secrets.py +145 -0
- codetrail/repo/signal.py +65 -0
- codetrail/repo/source.py +166 -0
- codetrail/search.py +178 -0
- codetrail/server.py +45 -0
- codetrail/system.py +484 -0
- codetrail/update.py +276 -0
- codetrail/web/__init__.py +1 -0
- codetrail/web/app.py +897 -0
- codetrail/web/diagrams.py +414 -0
- codetrail/web/documentation.py +146 -0
- codetrail/web/i18n.py +44 -0
- codetrail/web/navigation.py +127 -0
- codetrail/web/render.py +126 -0
- codetrail/web/repository_view.py +230 -0
- codetrail/web/security.py +171 -0
- codetrail/web/static/brand/favicon.svg +1 -0
- codetrail/web/static/brand/mark.svg +1 -0
- codetrail/web/static/css/base.css +192 -0
- codetrail/web/static/css/components.css +2254 -0
- codetrail/web/static/css/content.css +310 -0
- codetrail/web/static/css/layout.css +410 -0
- codetrail/web/static/css/print.css +42 -0
- codetrail/web/static/css/tokens.css +228 -0
- codetrail/web/static/js/api.js +49 -0
- codetrail/web/static/js/ask.js +182 -0
- codetrail/web/static/js/d2-worker.js +34 -0
- codetrail/web/static/js/d2.js +28 -0
- codetrail/web/static/js/diagram.js +52 -0
- codetrail/web/static/js/documentation.js +24 -0
- codetrail/web/static/js/learning.js +47 -0
- codetrail/web/static/js/main.js +38 -0
- codetrail/web/static/js/outline.js +44 -0
- codetrail/web/static/js/palette.js +211 -0
- codetrail/web/static/js/repository.js +528 -0
- codetrail/web/static/js/resize.js +126 -0
- codetrail/web/static/js/shortcuts.js +87 -0
- codetrail/web/static/js/sidebar.js +194 -0
- codetrail/web/static/js/theme-init.js +27 -0
- codetrail/web/static/js/theme.js +42 -0
- codetrail/web/static/js/update.js +107 -0
- codetrail/web/static/vendor/d2/LICENSE +375 -0
- codetrail/web/static/vendor/d2/LICENSE-elk +263 -0
- codetrail/web/static/vendor/d2/LICENSE-wasm_exec +27 -0
- codetrail/web/static/vendor/d2/VERSION +12 -0
- codetrail/web/static/vendor/d2/d2.wasm +0 -0
- codetrail/web/static/vendor/d2/elk.js +105914 -0
- codetrail/web/static/vendor/d2/wasm_exec.js +656 -0
- codetrail/web/static/vendor/inter/InterVariable-Italic.woff2 +0 -0
- codetrail/web/static/vendor/inter/InterVariable.woff2 +0 -0
- codetrail/web/static/vendor/inter/LICENSE.txt +92 -0
- codetrail/web/static/vendor/inter/VERSION +6 -0
- codetrail/web/static/vendor/noto-sans-arabic/NotoSansArabic-wght.woff2 +0 -0
- codetrail/web/static/vendor/noto-sans-arabic/OFL.txt +93 -0
- codetrail/web/static/vendor/noto-sans-arabic/VERSION +12 -0
- codetrail/web/target_view.py +63 -0
- codetrail/web/templates/answers.html +25 -0
- codetrail/web/templates/area.html +46 -0
- codetrail/web/templates/base.html +37 -0
- codetrail/web/templates/decisions.html +30 -0
- codetrail/web/templates/diagram.html +10 -0
- codetrail/web/templates/digests.html +24 -0
- codetrail/web/templates/documentation.html +247 -0
- codetrail/web/templates/error.html +9 -0
- codetrail/web/templates/estimate.html +48 -0
- codetrail/web/templates/fact.html +29 -0
- codetrail/web/templates/home.html +196 -0
- codetrail/web/templates/layout/ask_panel.html +27 -0
- codetrail/web/templates/layout/header.html +32 -0
- codetrail/web/templates/layout/icons.html +27 -0
- codetrail/web/templates/layout/outline.html +4 -0
- codetrail/web/templates/layout/palette.html +27 -0
- codetrail/web/templates/layout/shortcuts.html +27 -0
- codetrail/web/templates/layout/sidebar.html +79 -0
- codetrail/web/templates/layout/update_dialog.html +18 -0
- codetrail/web/templates/macros.html +13 -0
- codetrail/web/templates/page.html +93 -0
- codetrail/web/templates/progress.html +46 -0
- codetrail/web/templates/repository.html +369 -0
- codetrail/web/templates/search.html +31 -0
- codetrail/web/templates/source.html +15 -0
- codetrail/web/templates/system.html +43 -0
- codetrail/web/templates/system_arrows.html +17 -0
- codetrail/web/templates/system_legend.html +9 -0
- codetrail/web/templates/update_log.html +30 -0
- codetrail/web/trend.py +22 -0
- codetrail_cli-1.4.1.dist-info/METADATA +372 -0
- codetrail_cli-1.4.1.dist-info/RECORD +171 -0
- codetrail_cli-1.4.1.dist-info/WHEEL +4 -0
- codetrail_cli-1.4.1.dist-info/entry_points.txt +3 -0
- codetrail_cli-1.4.1.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Facts by kind, and the dependencies that arrived and left, from the validity ranges (design section 19.1).
|
|
2
|
+
|
|
3
|
+
Codetrail's history of a repository starts with its first snapshot: packages present then were there from the start,
|
|
4
|
+
not arrivals.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from codetrail.facts import EntityKind, Snapshot
|
|
12
|
+
from codetrail.facts.store import FactStore
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class KindCount:
|
|
17
|
+
kind: str
|
|
18
|
+
count: int
|
|
19
|
+
change: int | None # since the previous snapshot
|
|
20
|
+
trend: list[int] # over the last snapshots, oldest first
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Change:
|
|
25
|
+
package: str
|
|
26
|
+
snapshot: Snapshot # the snapshot that first saw it, or first no longer saw it
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Inventory:
|
|
31
|
+
since: Snapshot | None # the first snapshot: where Codetrail's history of the repository starts
|
|
32
|
+
kinds: list[KindCount] # the most facts first
|
|
33
|
+
arrived: list[Change] # newest first
|
|
34
|
+
left: list[Change] # newest first
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def measure_inventory(store: FactStore, trend_updates: int) -> Inventory:
|
|
38
|
+
snapshots = store.snapshots()
|
|
39
|
+
if not snapshots:
|
|
40
|
+
return Inventory(None, [], [], [])
|
|
41
|
+
recent = snapshots[-trend_updates:]
|
|
42
|
+
counts = store.entity_counts([snapshot.id for snapshot in recent])
|
|
43
|
+
latest = counts[recent[-1].id]
|
|
44
|
+
previous = counts[recent[-2].id] if len(recent) > 1 else None
|
|
45
|
+
kinds = [
|
|
46
|
+
KindCount(kind, count, None if previous is None else count - previous.get(kind, 0),
|
|
47
|
+
[counts[snapshot.id].get(kind, 0) for snapshot in recent])
|
|
48
|
+
for kind, count in sorted(latest.items(), key=lambda item: (-item[1], item[0]))
|
|
49
|
+
] # fmt: skip
|
|
50
|
+
by_id = {snapshot.id: snapshot for snapshot in snapshots}
|
|
51
|
+
arrived, left = [], []
|
|
52
|
+
for package, first, last in store.entity_spans(EntityKind.PACKAGE):
|
|
53
|
+
if first > snapshots[0].id:
|
|
54
|
+
arrived.append(Change(package, by_id[first]))
|
|
55
|
+
if last is not None:
|
|
56
|
+
after = next((snapshot for snapshot in snapshots if snapshot.id > last), None)
|
|
57
|
+
if after is not None:
|
|
58
|
+
left.append(Change(package, after))
|
|
59
|
+
return Inventory(snapshots[0], kinds, sorted(arrived, key=_newest_first), sorted(left, key=_newest_first))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _newest_first(change: Change) -> tuple[int, str]:
|
|
63
|
+
return -change.snapshot.id, change.package
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""The documented share of the guide's rationale, by source type and by page (design section 18.1).
|
|
2
|
+
|
|
3
|
+
Only area and concept pages count: digests accumulate and are mostly about commits, and saved answers follow the
|
|
4
|
+
reader's questions rather than the repository.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Iterable
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from pathspec import GitIgnoreSpec
|
|
13
|
+
|
|
14
|
+
from codetrail.generate.validate import parse_rationale
|
|
15
|
+
from codetrail.guide import Page
|
|
16
|
+
|
|
17
|
+
GUIDE_KINDS = ("area", "concept")
|
|
18
|
+
SOURCE_KINDS = ("adr", "docs", "code", "commit")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class InferredBlock:
|
|
23
|
+
page: Page
|
|
24
|
+
first_line: str
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class PageRationale:
|
|
29
|
+
page: Page
|
|
30
|
+
documented: int
|
|
31
|
+
inferred: int
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class RationaleMetric:
|
|
36
|
+
documented: int
|
|
37
|
+
inferred: int
|
|
38
|
+
by_source: dict[str, int]
|
|
39
|
+
pages: list[PageRationale] # most inferred first, then by title
|
|
40
|
+
inferred_blocks: list[InferredBlock]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def guide_pages(pages: Iterable[Page]) -> list[Page]:
|
|
44
|
+
return [page for page in pages if page.kind in GUIDE_KINDS]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def citation_source(citation: str, adr: GitIgnoreSpec, documents: GitIgnoreSpec) -> str:
|
|
48
|
+
"""Where a documented block's quote comes from: a commit, an ADR, another document, or code (a comment)."""
|
|
49
|
+
if citation.startswith("commit:"):
|
|
50
|
+
return "commit"
|
|
51
|
+
path = citation.split("#", 1)[0]
|
|
52
|
+
if adr.match_file(path):
|
|
53
|
+
return "adr"
|
|
54
|
+
return "docs" if documents.match_file(path) else "code"
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def measure_rationale(pages: Iterable[Page], adr: GitIgnoreSpec, documents: GitIgnoreSpec) -> RationaleMetric:
|
|
58
|
+
by_source = dict.fromkeys(SOURCE_KINDS, 0)
|
|
59
|
+
rows: list[PageRationale] = []
|
|
60
|
+
inferred_blocks: list[InferredBlock] = []
|
|
61
|
+
for page in guide_pages(pages):
|
|
62
|
+
documented = inferred = 0
|
|
63
|
+
for block in parse_rationale(page.body):
|
|
64
|
+
if block.kind == "documented":
|
|
65
|
+
documented += 1
|
|
66
|
+
by_source[citation_source(block.citation or "", adr, documents)] += 1
|
|
67
|
+
else:
|
|
68
|
+
inferred += 1
|
|
69
|
+
inferred_blocks.append(InferredBlock(page, next(iter(block.text.splitlines()), "")))
|
|
70
|
+
rows.append(PageRationale(page, documented, inferred))
|
|
71
|
+
rows.sort(key=lambda row: (-row.inferred, row.page.title.casefold(), row.page.id))
|
|
72
|
+
inferred_blocks.sort(key=lambda block: block.page.id) # stable: a page's blocks stay in order
|
|
73
|
+
return RationaleMetric(sum(row.documented for row in rows), sum(row.inferred for row in rows), by_source, rows,
|
|
74
|
+
inferred_blocks) # fmt: skip
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Releases: the target's tags that point into the branch's history (design section 19.1)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import statistics
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from itertools import pairwise
|
|
9
|
+
|
|
10
|
+
from codetrail.repo.history import Tag
|
|
11
|
+
|
|
12
|
+
DAY_SECONDS = 86_400
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Releases:
|
|
17
|
+
tags: list[Tag] = field(default_factory=list) # newest first
|
|
18
|
+
since_latest: int | None = None # commits after the latest release
|
|
19
|
+
median_days: int | None = None # between consecutive releases
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def measure_releases(tags: Sequence[Tag], since_latest: int | None) -> Releases:
|
|
23
|
+
newest_first = sorted(tags, key=lambda tag: (tag.date, tag.name), reverse=True)
|
|
24
|
+
gaps = [newer.date - older.date for newer, older in pairwise(newest_first)]
|
|
25
|
+
median = round(statistics.median(gaps) / DAY_SECONDS) if gaps else None
|
|
26
|
+
return Releases(newest_first, since_latest, median)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""The documentation metrics' report: every metric, read from the target's data (design section 18.2).
|
|
2
|
+
|
|
3
|
+
Documents are read from `source/` through the allowed-files reader, so excluded and secret files are never read; commits
|
|
4
|
+
come from the filtered history, with flagged messages withheld. If the history can't be read, the commit metric is
|
|
5
|
+
missing and the rest of the report stands.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from datetime import date
|
|
13
|
+
from enum import StrEnum
|
|
14
|
+
|
|
15
|
+
from pathspec import GitIgnoreSpec
|
|
16
|
+
|
|
17
|
+
from codetrail.config import GlobalConfig, Paths, load_target
|
|
18
|
+
from codetrail.errors import CodetrailError
|
|
19
|
+
from codetrail.facts import EntityKind
|
|
20
|
+
from codetrail.facts.store import FactStore
|
|
21
|
+
from codetrail.guide import GuideRepository
|
|
22
|
+
from codetrail.metrics.commits import CommitMetric, measure_commits
|
|
23
|
+
from codetrail.metrics.coverage import ExplainedMetric, MentionMetric, measure_explained, measure_mentions
|
|
24
|
+
from codetrail.metrics.decisions import DecisionMetric, measure_decisions
|
|
25
|
+
from codetrail.metrics.rationale import RationaleMetric, measure_rationale
|
|
26
|
+
from codetrail.repo.history import latest_commits
|
|
27
|
+
from codetrail.repo.secrets import SecretScanner
|
|
28
|
+
from codetrail.repo.source import SourceManifest, allowed_reader
|
|
29
|
+
|
|
30
|
+
logger = logging.getLogger(__name__)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Metric(StrEnum):
|
|
34
|
+
DOCUMENTED_SHARE = "documented_share"
|
|
35
|
+
COMMIT_WHY_SHARE = "commit_why_share"
|
|
36
|
+
ADR_ATTENTION = "adr_attention"
|
|
37
|
+
MENTIONED_SHARE = "mentioned_share"
|
|
38
|
+
EXPLAINED_SHARE = "explained_share"
|
|
39
|
+
# The repository statistics recorded for their trend (design section 19.4)
|
|
40
|
+
COMMITS = "commits"
|
|
41
|
+
CODE_LINES = "code_lines"
|
|
42
|
+
SOURCE_FILES = "source_files"
|
|
43
|
+
TEST_SHARE = "test_share"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class Value:
|
|
48
|
+
numerator: int
|
|
49
|
+
denominator: int | None # None for a count
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def share(self) -> float | None:
|
|
53
|
+
"""The share, or None for a count or when there was nothing to measure."""
|
|
54
|
+
return self.numerator / self.denominator if self.denominator else None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class MetricsReport:
|
|
59
|
+
rationale: RationaleMetric
|
|
60
|
+
commits: CommitMetric | None # None when the history couldn't be read
|
|
61
|
+
decisions: DecisionMetric
|
|
62
|
+
mentions: MentionMetric
|
|
63
|
+
explained: ExplainedMetric
|
|
64
|
+
|
|
65
|
+
def values(self) -> dict[Metric, Value]:
|
|
66
|
+
rationale, mentions = self.rationale, self.mentions
|
|
67
|
+
values = {
|
|
68
|
+
Metric.DOCUMENTED_SHARE: Value(rationale.documented, rationale.documented + rationale.inferred),
|
|
69
|
+
Metric.ADR_ATTENTION: Value(self.decisions.attention, None),
|
|
70
|
+
Metric.MENTIONED_SHARE: Value(len(mentions.mentioned), len(mentions.mentioned) + len(mentions.unmentioned)),
|
|
71
|
+
Metric.EXPLAINED_SHARE: Value(self.explained.explained, self.explained.total),
|
|
72
|
+
}
|
|
73
|
+
if self.commits is not None:
|
|
74
|
+
values[Metric.COMMIT_WHY_SHARE] = Value(self.commits.explains_why, self.commits.measured)
|
|
75
|
+
return {metric: values[metric] for metric in Metric if metric in values}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def build_report(paths: Paths, name: str, settings: GlobalConfig, store: FactStore, today: date) -> MetricsReport:
|
|
79
|
+
target = load_target(paths, name)
|
|
80
|
+
data = paths.target_data(name)
|
|
81
|
+
guide = GuideRepository(data / "guide")
|
|
82
|
+
pages = guide.pages() if guide.root.exists() else []
|
|
83
|
+
manifest = SourceManifest.load(data / "source.json")
|
|
84
|
+
files = manifest.files if manifest else {}
|
|
85
|
+
document_globs = GitIgnoreSpec.from_lines(target.metrics.document_globs)
|
|
86
|
+
read = allowed_reader(data / "source", files, settings.extract.max_file_bytes)
|
|
87
|
+
documents = {}
|
|
88
|
+
for path in files:
|
|
89
|
+
if document_globs.match_file(path):
|
|
90
|
+
content = read(path)
|
|
91
|
+
# An image under docs/ matches the globs but isn't text: its bytes would cost time and fake mentions.
|
|
92
|
+
# A NUL byte marks it binary, as the code size counts it (metrics/size.py).
|
|
93
|
+
if content is not None and b"\0" not in content:
|
|
94
|
+
documents[path] = content.decode("utf-8", "replace")
|
|
95
|
+
commits = None
|
|
96
|
+
if manifest is not None:
|
|
97
|
+
try:
|
|
98
|
+
found = latest_commits(data / "mirror.git", manifest.commit, settings.metrics.commit_window,
|
|
99
|
+
SecretScanner(settings.tools)) # fmt: skip
|
|
100
|
+
commits = measure_commits(found, target.metrics.why, settings.metrics.max_message_chars)
|
|
101
|
+
except (CodetrailError, OSError) as error:
|
|
102
|
+
logger.warning("The commit metric isn't available: %s", error)
|
|
103
|
+
entities = store.entities()
|
|
104
|
+
decisions = [entity for entity in entities if entity.kind == EntityKind.DECISION]
|
|
105
|
+
return MetricsReport(
|
|
106
|
+
measure_rationale(pages, GitIgnoreSpec.from_lines(target.adr.paths), document_globs),
|
|
107
|
+
commits,
|
|
108
|
+
measure_decisions(decisions, pages, today, settings.metrics.proposed_adr_days),
|
|
109
|
+
measure_mentions(entities, documents),
|
|
110
|
+
measure_explained(entities, pages),
|
|
111
|
+
)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""The repository statistics' report: history, releases, code size and facts, read from the target's data (19.3).
|
|
2
|
+
|
|
3
|
+
Files are read from `source/` through the allowed-files reader, so excluded and secret files are never read; commit
|
|
4
|
+
subjects and tags pass gitleaks first. If the history or the tags can't be read, those parts are missing and the rest of
|
|
5
|
+
the report stands.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from dataclasses import dataclass, replace
|
|
12
|
+
from datetime import UTC, date, datetime, timedelta
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from pathspec import GitIgnoreSpec
|
|
16
|
+
|
|
17
|
+
from codetrail.config import GlobalConfig, Paths, load_target
|
|
18
|
+
from codetrail.errors import CodetrailError
|
|
19
|
+
from codetrail.facts.store import FactStore
|
|
20
|
+
from codetrail.guide import GuideRepository, Page
|
|
21
|
+
from codetrail.metrics.activity import Activity, CommitTypes, measure_activity, measure_types
|
|
22
|
+
from codetrail.metrics.churn import Churn, measure_churn
|
|
23
|
+
from codetrail.metrics.coverage import page_scope
|
|
24
|
+
from codetrail.metrics.inventory import Inventory, measure_inventory
|
|
25
|
+
from codetrail.metrics.rationale import measure_rationale
|
|
26
|
+
from codetrail.metrics.releases import Releases, measure_releases
|
|
27
|
+
from codetrail.metrics.report import Metric, Value
|
|
28
|
+
from codetrail.metrics.size import CodeSize, measure_size
|
|
29
|
+
from codetrail.repo.history import (
|
|
30
|
+
changed_files,
|
|
31
|
+
commit_count,
|
|
32
|
+
commit_times,
|
|
33
|
+
commit_totals,
|
|
34
|
+
files_changed_since,
|
|
35
|
+
first_commit_time,
|
|
36
|
+
latest_commits,
|
|
37
|
+
read_tags,
|
|
38
|
+
)
|
|
39
|
+
from codetrail.repo.secrets import SecretScanner
|
|
40
|
+
from codetrail.repo.source import SourceManifest, allowed_reader
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger(__name__)
|
|
43
|
+
# A part that fails these ways is unavailable and the rest of the report stands. ValueError and OverflowError come
|
|
44
|
+
# from a date past the year 9999, which a commit or tag can claim and Python's dates can't hold.
|
|
45
|
+
UNREADABLE = (CodetrailError, OSError, ValueError, OverflowError)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class RepositoryReport:
|
|
50
|
+
activity: Activity | None # None when the history couldn't be read
|
|
51
|
+
types: CommitTypes | None # None, with the releases, when the commits or tags couldn't be read
|
|
52
|
+
releases: Releases | None
|
|
53
|
+
size: CodeSize
|
|
54
|
+
inventory: Inventory
|
|
55
|
+
churn: Churn | None = None # None when the history couldn't be read
|
|
56
|
+
|
|
57
|
+
def values(self) -> dict[Metric, Value]:
|
|
58
|
+
"""The statistics recorded per update for their trend (design section 19.4)."""
|
|
59
|
+
size = self.size
|
|
60
|
+
values = {
|
|
61
|
+
Metric.CODE_LINES: Value(size.code_lines, None),
|
|
62
|
+
Metric.SOURCE_FILES: Value(size.files, None),
|
|
63
|
+
Metric.TEST_SHARE: Value(size.test_lines, size.code_lines),
|
|
64
|
+
}
|
|
65
|
+
if self.activity is not None:
|
|
66
|
+
values[Metric.COMMITS] = Value(self.activity.commits, None)
|
|
67
|
+
return {metric: values[metric] for metric in Metric if metric in values}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def build_repository_report(
|
|
71
|
+
paths: Paths, name: str, settings: GlobalConfig, store: FactStore, today: date
|
|
72
|
+
) -> RepositoryReport:
|
|
73
|
+
target = load_target(paths, name)
|
|
74
|
+
data = paths.target_data(name)
|
|
75
|
+
manifest = SourceManifest.load(data / "source.json")
|
|
76
|
+
files = manifest.files if manifest else {}
|
|
77
|
+
read = allowed_reader(data / "source", files, settings.extract.max_file_bytes)
|
|
78
|
+
documents = GitIgnoreSpec.from_lines(target.metrics.document_globs)
|
|
79
|
+
tests = GitIgnoreSpec.from_lines(target.metrics.test_globs)
|
|
80
|
+
size = measure_size(files, read, settings.metrics.languages, documents, tests)
|
|
81
|
+
activity = types = releases = churn = None
|
|
82
|
+
if manifest is not None:
|
|
83
|
+
mirror, end = data / "mirror.git", manifest.commit
|
|
84
|
+
try:
|
|
85
|
+
commits, merges = commit_totals(mirror, end)
|
|
86
|
+
times = commit_times(mirror, end, settings.metrics.history_limit)
|
|
87
|
+
activity = measure_activity(times, commits, merges, today, settings.metrics.activity_months)
|
|
88
|
+
first = first_commit_time(mirror, end) if activity.cut else None
|
|
89
|
+
if first is not None: # the times read stop short of the first commit
|
|
90
|
+
activity = replace(activity, first=datetime.fromtimestamp(first, UTC).date())
|
|
91
|
+
except UNREADABLE as error:
|
|
92
|
+
logger.warning("The history isn't available: %s", error)
|
|
93
|
+
try:
|
|
94
|
+
scanner = SecretScanner(settings.tools)
|
|
95
|
+
latest = latest_commits(mirror, end, settings.metrics.commit_window, scanner)
|
|
96
|
+
types = measure_types(latest, target.metrics.types, settings.metrics.max_message_chars)
|
|
97
|
+
tags = read_tags(mirror, end, scanner)
|
|
98
|
+
newest = max(tags, key=lambda tag: (tag.date, tag.name), default=None)
|
|
99
|
+
releases = measure_releases(tags, commit_count(mirror, newest.commit, end) if newest else None)
|
|
100
|
+
except UNREADABLE as error:
|
|
101
|
+
types = None
|
|
102
|
+
logger.warning("The commit types and releases aren't available: %s", error)
|
|
103
|
+
try:
|
|
104
|
+
changes = changed_files(mirror, end, settings.metrics.churn_window)
|
|
105
|
+
recent = files_changed_since(mirror, end, today - timedelta(days=settings.metrics.quiet_days))
|
|
106
|
+
code_lines = {file.path: file.lines for file in size.largest}
|
|
107
|
+
areas = _areas(data / "guide", target.adr.paths, target.metrics.document_globs)
|
|
108
|
+
metrics = settings.metrics
|
|
109
|
+
churn = measure_churn(changes, files, code_lines, recent, areas, today, metrics.folder_depth,
|
|
110
|
+
metrics.heatmap_tiles, metrics.churn_window) # fmt: skip
|
|
111
|
+
except UNREADABLE as error:
|
|
112
|
+
logger.warning("Where change happens isn't available: %s", error)
|
|
113
|
+
inventory = measure_inventory(store, settings.metrics.trend_updates)
|
|
114
|
+
return RepositoryReport(activity, types, releases, size, inventory, churn)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _areas(guide_root: Path, adr_paths: list[str], document_globs: list[str]) -> list[tuple[Page, list[str], int, int]]:
|
|
118
|
+
"""Each area page with a usable scope, with its documented and inferred rationale blocks."""
|
|
119
|
+
guide = GuideRepository(guide_root)
|
|
120
|
+
pages = guide.pages() if guide.root.exists() else []
|
|
121
|
+
rationale = measure_rationale(pages, GitIgnoreSpec.from_lines(adr_paths), GitIgnoreSpec.from_lines(document_globs))
|
|
122
|
+
counts = {row.page.id: (row.documented, row.inferred) for row in rationale.pages}
|
|
123
|
+
areas = []
|
|
124
|
+
for page in pages:
|
|
125
|
+
entry = page_scope(page) if page.kind == "area" else None
|
|
126
|
+
if entry is not None:
|
|
127
|
+
areas.append((page, entry.scope_paths, *counts.get(page.id, (0, 0))))
|
|
128
|
+
return areas
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Code size: files and lines by language, documents and tests (design section 19.1).
|
|
2
|
+
|
|
3
|
+
A file's language is looked up by its name, then its suffix, in the configured map: a dictionary lookup, never a
|
|
4
|
+
pattern a repository could control.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import PurePosixPath
|
|
12
|
+
|
|
13
|
+
from pathspec import GitIgnoreSpec
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class LanguageSize:
|
|
18
|
+
language: str
|
|
19
|
+
files: int
|
|
20
|
+
lines: int
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class FileSize:
|
|
25
|
+
path: str
|
|
26
|
+
language: str
|
|
27
|
+
lines: int
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class CodeSize:
|
|
32
|
+
files: int # every allowed file
|
|
33
|
+
languages: list[LanguageSize] # the most lines first
|
|
34
|
+
documents: LanguageSize
|
|
35
|
+
other_files: int # in no language
|
|
36
|
+
binary_files: int
|
|
37
|
+
too_large_files: int # over extract.max_file_bytes, or unreadable
|
|
38
|
+
code_lines: int # lines in files with a language, tests included
|
|
39
|
+
test_lines: int
|
|
40
|
+
largest: list[FileSize] # files with a language, the most lines first
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def count_lines(content: bytes) -> int:
|
|
44
|
+
"""The newline bytes, plus one for a last line without one."""
|
|
45
|
+
return content.count(b"\n") + (1 if content and not content.endswith(b"\n") else 0)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def language_of(path: str, languages: Mapping[str, str]) -> str | None:
|
|
49
|
+
name = PurePosixPath(path).name.lower()
|
|
50
|
+
if not name.startswith(".") and name in languages:
|
|
51
|
+
return languages[name]
|
|
52
|
+
suffix = PurePosixPath(name).suffix
|
|
53
|
+
return languages.get(suffix) if suffix else None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def measure_size(
|
|
57
|
+
paths: Iterable[str],
|
|
58
|
+
read: Callable[[str], bytes | None],
|
|
59
|
+
languages: Mapping[str, str],
|
|
60
|
+
documents: GitIgnoreSpec,
|
|
61
|
+
tests: GitIgnoreSpec,
|
|
62
|
+
) -> CodeSize:
|
|
63
|
+
"""Reads each allowed file once and sorts it: too large, binary, a document, a language or other."""
|
|
64
|
+
files = other = binary = too_large = document_files = document_lines = test_lines = 0
|
|
65
|
+
by_language: dict[str, list[int]] = {}
|
|
66
|
+
largest = []
|
|
67
|
+
for path in paths:
|
|
68
|
+
files += 1
|
|
69
|
+
content = read(path)
|
|
70
|
+
if content is None:
|
|
71
|
+
too_large += 1
|
|
72
|
+
continue
|
|
73
|
+
if b"\0" in content:
|
|
74
|
+
binary += 1
|
|
75
|
+
continue
|
|
76
|
+
lines = count_lines(content)
|
|
77
|
+
if documents.match_file(path):
|
|
78
|
+
document_files += 1
|
|
79
|
+
document_lines += lines
|
|
80
|
+
continue
|
|
81
|
+
language = language_of(path, languages)
|
|
82
|
+
if language is None:
|
|
83
|
+
other += 1
|
|
84
|
+
continue
|
|
85
|
+
counts = by_language.setdefault(language, [0, 0])
|
|
86
|
+
counts[0] += 1
|
|
87
|
+
counts[1] += lines
|
|
88
|
+
largest.append(FileSize(path, language, lines))
|
|
89
|
+
if tests.match_file(path):
|
|
90
|
+
test_lines += lines
|
|
91
|
+
sizes = [LanguageSize(language, counts[0], counts[1]) for language, counts in by_language.items()]
|
|
92
|
+
sizes.sort(key=lambda size: (-size.lines, -size.files, size.language))
|
|
93
|
+
largest.sort(key=lambda file: (-file.lines, file.path))
|
|
94
|
+
return CodeSize(files, sizes, LanguageSize("Documents", document_files, document_lines), other, binary,
|
|
95
|
+
too_large, sum(size.lines for size in sizes), test_lines, largest) # fmt: skip
|
codetrail/py.typed
ADDED
|
File without changes
|
codetrail/remove.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Removing a target: everything Codetrail keeps for it, never the repository it teaches (AGENTS.md, Never do, 6)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import shutil
|
|
6
|
+
import tomllib
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from codetrail.config import Paths, check_containment, validate_target_name
|
|
11
|
+
from codetrail.errors import CodetrailError
|
|
12
|
+
from codetrail.lock import existing_target_lock, target_removal
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def remove_target(paths: Paths, name: str, confirm: Callable[[list[Path]], bool]) -> bool:
|
|
16
|
+
"""Shows `confirm` the target's existing locations and deletes them if it agrees; returns whether it did.
|
|
17
|
+
|
|
18
|
+
The settings file goes last, and its lock is held until then: a removal that fails part way leaves a target
|
|
19
|
+
that can be removed again, and no command can start using the target meanwhile."""
|
|
20
|
+
settings = paths.target_file(validate_target_name(name))
|
|
21
|
+
if not settings.exists():
|
|
22
|
+
raise CodetrailError(f"No target named {name!r}.")
|
|
23
|
+
repository = _repository(settings)
|
|
24
|
+
if repository is not None: # the XDG variables may have moved Codetrail's folders since the target was added
|
|
25
|
+
check_containment(paths, repository)
|
|
26
|
+
data = paths.target_data(name)
|
|
27
|
+
locations = [data, paths.target_state(name), paths.ignore_file(name), settings]
|
|
28
|
+
if not confirm([location for location in locations if _exists(location)]):
|
|
29
|
+
return False
|
|
30
|
+
# Refused while serve, update or files uses the target, or an update runs.
|
|
31
|
+
with target_removal(paths, name), existing_target_lock(paths, name):
|
|
32
|
+
for location in locations:
|
|
33
|
+
if location.is_dir() and not location.is_symlink():
|
|
34
|
+
shutil.rmtree(location) # never follows a symlink inside the folder
|
|
35
|
+
elif _exists(location):
|
|
36
|
+
location.unlink() # a file, or a symlink: the link goes, never what it points to
|
|
37
|
+
return True
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _exists(location: Path) -> bool:
|
|
41
|
+
return location.is_symlink() or location.exists()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _repository(settings: Path) -> Path | None:
|
|
45
|
+
"""The target's repository, if its settings still say; a broken file mustn't stop the target being removed."""
|
|
46
|
+
try:
|
|
47
|
+
repository = tomllib.loads(settings.read_text(encoding="utf-8")).get("repository")
|
|
48
|
+
except OSError, UnicodeDecodeError, tomllib.TOMLDecodeError:
|
|
49
|
+
return None
|
|
50
|
+
return Path(repository) if isinstance(repository, str) else None
|
codetrail/repo/git.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Runs git for Codetrail: argument lists only, never a shell, never a prompt."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from contextlib import suppress
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from codetrail.errors import CodetrailError
|
|
12
|
+
|
|
13
|
+
# How long a git stopped early gets to remove its lock files before it's killed; it needs only a moment.
|
|
14
|
+
STOP_WAIT_SECONDS = 5
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def run_git(
|
|
18
|
+
arguments: Sequence[str],
|
|
19
|
+
*,
|
|
20
|
+
git_dir: Path | None = None,
|
|
21
|
+
work_tree: Path | None = None,
|
|
22
|
+
input: bytes | None = None,
|
|
23
|
+
allowed_exit_codes: Sequence[int] = (0,),
|
|
24
|
+
) -> bytes:
|
|
25
|
+
"""Runs git and returns its standard output; any other exit code raises a CodetrailError with git's message."""
|
|
26
|
+
command = ["git"]
|
|
27
|
+
if git_dir is not None:
|
|
28
|
+
command.append(f"--git-dir={git_dir}")
|
|
29
|
+
if work_tree is not None:
|
|
30
|
+
command.append(f"--work-tree={work_tree}")
|
|
31
|
+
command.extend(arguments)
|
|
32
|
+
environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")}
|
|
33
|
+
environment["GIT_TERMINAL_PROMPT"] = "0"
|
|
34
|
+
# Run from the file system's root and stop repository discovery there: git never finds a repository (a
|
|
35
|
+
# target's, or another user's in a shared temporary folder) from where it runs.
|
|
36
|
+
environment["GIT_CEILING_DIRECTORIES"] = "/"
|
|
37
|
+
with subprocess.Popen( # noqa: S603
|
|
38
|
+
command, stdin=subprocess.PIPE if input is not None else None, stdout=subprocess.PIPE,
|
|
39
|
+
stderr=subprocess.PIPE, env=environment, cwd="/",
|
|
40
|
+
) as process: # fmt: skip
|
|
41
|
+
try:
|
|
42
|
+
stdout, stderr = process.communicate(input)
|
|
43
|
+
except BaseException:
|
|
44
|
+
# Stopped early (Ctrl-C, a hangup or terminate signal): SIGTERM lets git remove its lock files, which
|
|
45
|
+
# SIGKILL would leave behind to fail every later command in that repository.
|
|
46
|
+
try:
|
|
47
|
+
process.terminate()
|
|
48
|
+
with suppress(subprocess.TimeoutExpired):
|
|
49
|
+
process.wait(STOP_WAIT_SECONDS)
|
|
50
|
+
finally:
|
|
51
|
+
process.kill() # does nothing once git has exited
|
|
52
|
+
raise
|
|
53
|
+
if process.returncode not in allowed_exit_codes:
|
|
54
|
+
message = stderr.decode("utf-8", "replace").strip() or f"exit code {process.returncode}"
|
|
55
|
+
raise CodetrailError(f"git {arguments[0]} failed: {message}")
|
|
56
|
+
return stdout
|