git2report 0.1.0__py3-none-any.whl

git2report/eval_tools.py

@@ -0,0 +1,123 @@
+"""Evaluation helpers: make a sheet to fill in by hand, then score it.
+
+    git2report-eval sheet --repo ./proj --since 45d --n 25 --out eval.csv
+    (fill in the empty columns in Excel: manual summary + 1-5 scores)
+    git2report-eval score eval.csv
+
+Per sampled commit the sheet holds: the TOOL sentence (our pipeline) and a BASELINE sentence
+(what you get by pasting the raw commit to an AI with no pipeline; offline mode: the raw commit message)."""
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Optional
+
+import typer
+
+from git2report.chunker import chunk_module
+from git2report.clustering import cluster
+from git2report.config import Config
+from git2report.extractor import extract
+from git2report.filtering import filter_commits
+from git2report.models import GitReportError
+from git2report.summarizer import SummaryRequest, get_summarizer
+from git2report.timeutil import parse_since
+
+app = typer.Typer(add_completion=False, help="Build and score the manual-vs-tool evaluation sheet.")
+
+COLUMNS = ["hash", "message", "module", "tool_sentence", "tool_mode", "baseline_sentence",
+           "manual_summary", "tool_accuracy_1to5", "tool_readability_1to5", "tool_unsupported_claims",
+           "baseline_accuracy_1to5", "baseline_readability_1to5", "baseline_unsupported_claims", "notes"]
+
+
+def pick_evenly(items: list, n: int) -> list:
+    """Deterministic, evenly spread sample (so the sheet can be rebuilt identically)."""
+    if n >= len(items):
+        return list(items)
+    step = len(items) / n
+    return [items[int(i * step)] for i in range(n)]
+
+
+def baseline_sentence(commit, summarizer) -> str:
+    if hasattr(summarizer, "ask"):
+        diff = "\n".join(f.diff for f in commit.files)[:1500]
+        prompt = f"Explain this git commit to a non-technical person.\nMessage: {commit.subject}\nChanges:\n{diff}"
+        try:
+            return " ".join(summarizer.ask(prompt).split())
+        except Exception as exc:
+            return f"[AI unavailable: {type(exc).__name__}] {commit.subject}"
+    return commit.subject                                   # offline baseline = the raw commit message
+
+
+@app.command()
+def sheet(
+    repo: Path = typer.Option(Path("."), "--repo", "-r"),
+    since: str = typer.Option("45d", "--since", "-s"),
+    n: int = typer.Option(25, "--n", help="How many commits to sample."),
+    audience: str = typer.Option("guide", "--audience", "-a"),
+    llm: str = typer.Option("offline", "--llm", help="offline or ollama"),
+    model: str = typer.Option("llama3:8b", "--model"),
+    out: Path = typer.Option(Path("eval.csv"), "--out", "-o"),
+):
+    cfg = Config()
+    try:
+        commits = extract(repo, parse_since(since))
+        kept, _ = filter_commits(commits, cfg)
+        if not kept:
+            raise GitReportError("No usable commits in that window; try a longer --since.")
+        cluster(kept, cfg)
+    except GitReportError as exc:
+        typer.secho(f"Error: {exc}", fg=typer.colors.RED, err=True)
+        raise typer.Exit(1)
+    summarizer = get_summarizer(llm, model)
+    rows = []
+    for c in pick_evenly(kept, n):
+        res = summarizer.summarize(SummaryRequest(c.module, [c], chunk_module([c], cfg).text, audience))
+        rows.append({"hash": c.hash, "message": c.subject, "module": c.module, "tool_sentence": res.text,
+                     "tool_mode": res.mode, "baseline_sentence": baseline_sentence(c, summarizer)})
+    with open(out, "w", newline="", encoding="utf-8-sig") as f:         # utf-8-sig so Excel opens it correctly
+        w = csv.DictWriter(f, fieldnames=COLUMNS)
+        w.writeheader()
+        for r in rows:
+            w.writerow({k: r.get(k, "") for k in COLUMNS})
+    typer.echo(f"Wrote {out} with {len(rows)} commits. Fill in the empty columns, then run: git2report-eval score {out}")
+
+
+def _nums(rows: list[dict], col: str) -> list[float]:
+    out = []
+    for r in rows:
+        try:
+            out.append(float((r.get(col) or "").strip()))
+        except ValueError:
+            continue
+    return out
+
+
+def _avg(xs: list[float]) -> str:
+    return f"{sum(xs) / len(xs):.2f}" if xs else "n/a"
+
+
+def summarize_scores(rows: list[dict]) -> str:
+    lines = ["| Measure | Tool (our pipeline) | Baseline |", "|---|---|---|"]
+    for label, t, b in (("Accuracy (1-5, higher is better)", "tool_accuracy_1to5", "baseline_accuracy_1to5"),
+                        ("Readability (1-5, higher is better)", "tool_readability_1to5", "baseline_readability_1to5")):
+        tv, bv = _nums(rows, t), _nums(rows, b)
+        lines.append(f"| {label} | {_avg(tv)} (n={len(tv)}) | {_avg(bv)} (n={len(bv)}) |")
+    tu, bu = _nums(rows, "tool_unsupported_claims"), _nums(rows, "baseline_unsupported_claims")
+    lines.append(f"| Unsupported claims (total, lower is better) | {int(sum(tu))} over {len(tu)} rows | {int(sum(bu))} over {len(bu)} rows |")
+    return "\n".join(lines)
+
+
+@app.command()
+def score(sheet_path: Path = typer.Argument(..., help="The filled-in CSV"),
+          out: Optional[Path] = typer.Option(None, "--out", "-o", help="Also save the table as Markdown")):
+    with open(sheet_path, newline="", encoding="utf-8-sig") as f:
+        rows = list(csv.DictReader(f))
+    table = summarize_scores(rows)
+    typer.echo(f"{len(rows)} rows read\n\n{table}")
+    if out:
+        out.write_text(table + "\n", encoding="utf-8")
+
+
+if __name__ == "__main__":
+    app()

git2report/extractor.py

@@ -0,0 +1,37 @@
+"""STEP 1: read commits from a Git repository into plain records."""
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+from pydriller import Repository
+
+from git2report.models import CommitRecord, FileChange, GitReportError
+
+
+def extract(repo_path: Path, since: datetime) -> list[CommitRecord]:
+    repo_path = Path(repo_path)
+    if not (repo_path / ".git").exists():
+        raise GitReportError(f"'{repo_path}' is not a Git repository (no .git folder found).")
+    records: list[CommitRecord] = []
+    try:
+        for c in Repository(str(repo_path), since=since).traverse_commits():
+            records.append(CommitRecord(
+                hash=c.hash[:7],
+                message=c.msg.strip(),
+                date=c.committer_date.date().isoformat(),
+                author=c.author.name,
+                is_merge=c.merge,
+                files=[
+                    FileChange(
+                        path=f.new_path or f.old_path or "",
+                        added=f.added_lines,
+                        deleted=f.deleted_lines,
+                        diff=f.diff or "",
+                    )
+                    for f in c.modified_files
+                ],
+            ))
+    except Exception as exc:                                    # PyDriller/GitPython errors
+        raise GitReportError(f"Could not read the repository: {exc}") from exc
+    return records

git2report/filtering.py

@@ -0,0 +1,34 @@
+"""STEP 2: drop commits a reader does not care about (merges, typos, unfinished work)."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from git2report.config import Config
+from git2report.models import CommitRecord
+
+
+@dataclass
+class Decision:
+    hash: str
+    subject: str
+    kept: bool
+    reason: str
+
+
+def decide(commit: CommitRecord, cfg: Config) -> Decision:
+    subject = commit.subject
+    if commit.is_merge or subject.lower().startswith("merge "):
+        return Decision(commit.hash, subject, False, "merge commit")
+    words = "|".join(re.escape(w) for w in cfg.noise_words)
+    if re.search(rf"\b({words})\b", subject.lower()):
+        if commit.lines >= cfg.keep_if_lines:
+            return Decision(commit.hash, subject, True, "noise word, but big change: kept")
+        return Decision(commit.hash, subject, False, "noise word and small change")
+    return Decision(commit.hash, subject, True, "ok")
+
+
+def filter_commits(commits: list[CommitRecord], cfg: Config) -> tuple[list[CommitRecord], list[Decision]]:
+    decisions = [decide(c, cfg) for c in commits]
+    kept = [c for c, d in zip(commits, decisions) if d.kept]
+    return kept, decisions

git2report/metrics.py

@@ -0,0 +1,111 @@
+"""STEP 4: numbers, counted by code. The AI never produces or touches these."""
+from __future__ import annotations
+
+from git2report.config import Config
+from git2report.models import CommitRecord
+from git2report.rules import commit_type, is_junk, is_test
+
+
+def compute_metrics(commits: list[CommitRecord], groups: dict[str, list[CommitRecord]], cfg: Config,
+                    total_commits_seen: int | None = None) -> dict:
+    from pathlib import Path
+
+    files = [f for c in commits for f in c.files if not is_junk(f.path, cfg.junk)]
+    test_lines = sum(f.added + f.deleted for f in files if is_test(f.path))
+    code_lines = sum(f.added + f.deleted for f in files if not is_test(f.path))
+    lines_added = sum(f.added for f in files)
+    lines_deleted = sum(f.deleted for f in files)
+
+    by_type: dict[str, int] = {}
+    for c in commits:
+        key = commit_type(c.subject) or "other"
+        by_type[key] = by_type.get(key, 0) + 1
+    top = max(groups, key=lambda g: (len(groups[g]), g)) if groups else ""
+
+    author_stats: dict[str, dict] = {}
+    for c in commits:
+        a = c.author or "Unknown"
+        if a not in author_stats:
+            author_stats[a] = {"name": a, "commits": 0, "added": 0, "deleted": 0, "modules": {}}
+        author_stats[a]["commits"] += 1
+        for f in c.files:
+            if not is_junk(f.path, cfg.junk):
+                author_stats[a]["added"] += f.added
+                author_stats[a]["deleted"] += f.deleted
+        if c.module:
+            author_stats[a]["modules"][c.module] = author_stats[a]["modules"].get(c.module, 0) + 1
+
+    contributors = []
+    for a, s in author_stats.items():
+        top_mod = max(s["modules"], key=s["modules"].get) if s["modules"] else top
+        contributors.append({
+            "name": a,
+            "commits": s["commits"],
+            "added": s["added"],
+            "deleted": s["deleted"],
+            "lines_changed": s["added"] + s["deleted"],
+            "top_module": top_mod,
+        })
+    contributors.sort(key=lambda x: (x["commits"], x["lines_changed"]), reverse=True)
+
+    total_lines = max(test_lines + code_lines, 1)
+    mod_lines: dict[str, int] = {}
+    for mod, cs in groups.items():
+        mod_files = [f for c in cs for f in c.files if not is_junk(f.path, cfg.junk)]
+        mod_lines[mod] = sum(f.added + f.deleted for f in mod_files)
+
+    module_distribution = {
+        mod: round((lines / total_lines) * 100, 1)
+        for mod, lines in sorted(mod_lines.items(), key=lambda kv: kv[1], reverse=True)
+    }
+
+    ext_stats: dict[str, dict] = {}
+    TECH_MAP = {
+        ".jsx": "React (JSX)", ".tsx": "React (TSX)", ".js": "JavaScript", ".ts": "TypeScript",
+        ".py": "Python", ".html": "HTML", ".css": "CSS", ".scss": "SCSS", ".json": "JSON / Config",
+        ".md": "Documentation", ".sql": "SQL / Database", ".yml": "YAML / CI", ".yaml": "YAML / CI",
+        ".go": "Go", ".rs": "Rust", ".java": "Java", ".sh": "Shell Script"
+    }
+    for f in files:
+        ext = Path(f.path).suffix.lower() or "other"
+        name = TECH_MAP.get(ext, ext.upper() if ext.startswith(".") else "Other")
+        if name not in ext_stats:
+            ext_stats[name] = {"tech": name, "count": 0, "lines": 0}
+        ext_stats[name]["count"] += 1
+        ext_stats[name]["lines"] += (f.added + f.deleted)
+    
+    tech_stack = []
+    for t, s in sorted(ext_stats.items(), key=lambda x: x[1]["lines"], reverse=True)[:5]:
+        pct = round((s["lines"] / total_lines) * 100, 1)
+        if pct > 0:
+            tech_stack.append({"tech": t, "percentage": pct, "lines": s["lines"]})
+
+    # Codebase Hotspots: files touched by most commits
+    file_freq: dict[str, dict] = {}
+    for c in commits:
+        for f in c.files:
+            if not is_junk(f.path, cfg.junk) and f.path:
+                if f.path not in file_freq:
+                    file_freq[f.path] = {"path": f.path, "commits": 0, "lines": 0}
+                file_freq[f.path]["commits"] += 1
+                file_freq[f.path]["lines"] += (f.added + f.deleted)
+    hotspots = sorted(file_freq.values(), key=lambda x: (x["commits"], x["lines"]), reverse=True)[:5]
+
+    return {
+        "commits": len(commits),
+        "commits_seen_before_filter": total_commits_seen if total_commits_seen is not None else len(commits),
+        "authors": len({c.author for c in commits}),
+        "lines_changed": test_lines + code_lines,                       # junk files excluded
+        "lines_changed_incl_junk": sum(c.lines for c in commits),
+        "lines_added": lines_added,
+        "lines_deleted": lines_deleted,
+        "net_lines": lines_added - lines_deleted,
+        "top_module": top,
+        "modules": len(groups),
+        "test_to_code_ratio": round(test_lines / max(code_lines, 1), 2),
+        "commits_by_type": dict(sorted(by_type.items())),
+        "contributors": contributors,
+        "module_distribution": module_distribution,
+        "tech_stack": tech_stack,
+        "hotspots": hotspots,
+    }

git2report/models.py

@@ -0,0 +1,45 @@
+"""Plain data containers shared by every pipeline step."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+class GitReportError(Exception):
+    """A problem the user can fix (bad path, no commits, ...). Shown without a traceback."""
+
+
+@dataclass
+class FileChange:
+    path: str
+    added: int
+    deleted: int
+    diff: str = ""
+
+
+@dataclass
+class CommitRecord:
+    hash: str
+    message: str
+    date: str
+    author: str
+    is_merge: bool
+    files: list[FileChange] = field(default_factory=list)
+    module: str = ""
+    module_reason: str = ""
+
+    @property
+    def subject(self) -> str:
+        text = self.message.strip()
+        return text.splitlines()[0] if text else ""
+
+    @property
+    def lines(self) -> int:
+        return sum(f.added + f.deleted for f in self.files)
+
+
+@dataclass
+class BranchInfo:
+    name: str
+    last_date: str
+    ahead: int          # commits on the branch that are not yet in the current branch
+    subject: str

git2report/pipeline.py

@@ -0,0 +1,188 @@
+"""Glue: run the 7 steps in order and (optionally) save every step's result as a file."""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from pathlib import Path
+from typing import Callable
+
+from git2report.branches import open_branches
+from git2report.chunker import chunk_module
+from git2report.clustering import cluster
+from git2report.config import Config
+from git2report.extractor import extract
+from git2report.filtering import filter_commits
+from git2report.metrics import compute_metrics
+from git2report.models import GitReportError
+from git2report.report import CategorizedCommit, ReportData, Section
+from git2report.rules import commit_type, is_junk
+from git2report.summarizer import SummaryRequest
+
+
+def get_remote_url(repo: Path) -> str | None:
+    import subprocess
+    try:
+        r = subprocess.run(["git", "config", "--get", "remote.origin.url"], cwd=repo, capture_output=True, text=True)
+        url = r.stdout.strip()
+        if not url:
+            return None
+        if url.startswith("git@github.com:"):
+            url = "https://github.com/" + url[len("git@github.com:"):]
+        elif url.startswith("git@gitlab.com:"):
+            url = "https://gitlab.com/" + url[len("git@gitlab.com:"):]
+        if url.endswith(".git"):
+            url = url[:-4]
+        return url if url.startswith("http") else None
+    except Exception:
+        return None
+
+
+@dataclass
+class Options:
+    repo: Path
+    since: datetime
+    audience: str = "guide"
+    artifacts: Path | None = None        # if set, every step's output is saved here
+    planned: list[str] = field(default_factory=list)     # goals written by the developer (never invented)
+    include_branches: bool = True
+
+
+def glance_text(by_type: dict[str, int]) -> str:
+    """'3 features, 2 fixes, 1 improvement': counted from commit types, never written by the AI."""
+    feats, fixes = by_type.get("feat", 0), by_type.get("fix", 0)
+    imps = by_type.get("perf", 0) + by_type.get("refactor", 0)
+    rest = sum(by_type.values()) - feats - fixes - imps
+    parts = []
+    for n, one, many in ((feats, "feature", "features"), (fixes, "fix", "fixes"),
+                         (imps, "improvement", "improvements"), (rest, "other change", "other changes")):
+        if n:
+            parts.append(f"{n} {one if n == 1 else many}")
+    return ", ".join(parts) or "no categorised changes"
+
+
+def run_pipeline(opts: Options, summarizer, cfg: Config | None = None,
+                 log: Callable[[str], None] = lambda _m: None) -> ReportData:
+    cfg = cfg or Config()
+
+    log("Step 1/7  extracting commits")
+    commits = extract(opts.repo, opts.since)
+    if not commits:
+        raise GitReportError("No commits found in that time window. Try a longer --since (e.g. 14d).")
+    log(f"          {len(commits)} commits found")
+
+    log("Step 2/7  filtering noise")
+    kept, decisions = filter_commits(commits, cfg)
+    if not kept:
+        raise GitReportError("Every commit in the window was filtered out as noise (merges, typos, wip).")
+    log(f"          {len(commits)} -> {len(kept)} commits")
+
+    log("Step 3/7  clustering by module")
+    groups = cluster(kept, cfg)
+    log(f"          {len(groups)} modules: {', '.join(groups)}")
+
+    log("Step 4/7  counting metrics")
+    metrics = compute_metrics(kept, groups, cfg, total_commits_seen=len(commits))
+
+    sections: list[Section] = []
+    chunks, prompts, summaries = {}, {}, {}
+    for i, (module, cs) in enumerate(groups.items(), 1):
+        log(f"Step 5-6  {module} ({i}/{len(groups)}): chunking + summarising")
+        ch = chunk_module(cs, cfg)
+        req = SummaryRequest(module, cs, ch.text, opts.audience)
+        result = summarizer.summarize(req)
+        hashes = [c.hash for c in cs]
+        cat_commits = [
+            CategorizedCommit(
+                hash=c.hash,
+                subject=c.subject,
+                author=c.author,
+                date=c.date,
+                type=commit_type(c.subject) or "other",
+                added=sum(f.added for f in c.files if not is_junk(f.path, cfg.junk)),
+                deleted=sum(f.deleted for f in c.files if not is_junk(f.path, cfg.junk)),
+            )
+            for c in cs
+        ]
+        sections.append(Section(module, result.text, hashes, result.mode, result.note, commits=cat_commits))
+        chunks[module], prompts[module] = ch, req.prompt
+        summaries[module] = {"summary": result.text, "mode": result.mode, "flagged": result.flagged,
+                             "note": result.note, "commit_hashes": hashes}
+
+    log("Step 7/7  building the report")
+    modes = {s.mode for s in sections}
+    n_off = sum(1 for s in sections if s.mode == "offline")
+    base = getattr(summarizer, "label", "unknown")
+    if modes == {"ai"} or not hasattr(summarizer, "model"):
+        mode_label = base
+    elif modes == {"offline"} and hasattr(summarizer, "model"):
+        mode_label = f"{base}; the AI step failed for every module, so offline wording was used"
+    else:
+        mode_label = f"{base}; offline wording used for {n_off} module(s)"
+    in_progress = open_branches(opts.repo, opts.since) if opts.include_branches else []
+    remote_url = get_remote_url(opts.repo)
+    exec_summary = make_executive_summary(Path(opts.repo).resolve().name, metrics, sections, summarizer)
+    data = ReportData(
+        project=Path(opts.repo).resolve().name,
+        start=opts.since.date().isoformat(), end=datetime.now().date().isoformat(),
+        audience=opts.audience, metrics=metrics, sections=sections, mode_label=mode_label,
+        glance_text=glance_text(metrics["commits_by_type"]), in_progress=in_progress, planned=opts.planned,
+        remote_url=remote_url, executive_summary=exec_summary,
+    )
+    if opts.artifacts:
+        _save_artifacts(opts.artifacts, commits, decisions, groups, metrics, chunks, prompts, summaries)
+    return data
+
+
+def make_executive_summary(project: str, metrics: dict, sections: list[Section], summarizer) -> str:
+    if hasattr(summarizer, "ask") and getattr(summarizer, "mode", "") == "ai":
+        from git2report.cache import compute_cache_key, default_cache
+        all_hashes = [h for s in sections for h in s.hashes]
+        cache_key = compute_cache_key("exec_summary", project, all_hashes, getattr(summarizer, "model", "ai"), str(metrics.get("lines_changed", 0)))
+        cached = default_cache.get(cache_key)
+        if cached:
+            return cached["summary"]
+
+        mod_summaries = "\n".join(f"- {s.module}: {s.text}" for s in sections[:5])
+        prompt = (
+            f"You are a technology leader summarizing software progress for project '{project}'.\n"
+            f"Key metrics: {metrics['commits']} commits, {metrics['lines_changed']} lines changed, primary focus on {metrics['top_module']}.\n"
+            f"Module deliverables:\n{mod_summaries}\n\n"
+            "Task: Write 2 clear, inspiring, plain-language sentences summarizing the major business progress and impact for non-technical leadership.\n"
+            "Rules: Do not write file names or numbers. Use plain, positive business English.\n"
+            'Return ONLY JSON: {"summary": "..."}'
+        )
+        try:
+            raw = summarizer.ask(prompt)
+            from git2report.summarizer import parse_model_json
+            text = parse_model_json(raw)
+            if text:
+                default_cache.set(cache_key, text, "ai")
+                return text
+        except Exception:
+            pass
+    commits = metrics["commits"]
+    modules = metrics["modules"]
+    top = metrics["top_module"]
+    added, deleted = metrics.get("lines_added", 0), metrics.get("lines_deleted", 0)
+    return (
+        f"During this delivery cycle, engineering completed {commits} verified commits across {modules} module(s), "
+        f"expanding the codebase with +{added} lines and pruning -{deleted} lines. "
+        f"Development concentrated primarily on {top}, advancing key capabilities while reinforcing overall system stability."
+    )
+
+
+def _save_artifacts(root: Path, commits, decisions, groups, metrics, chunks, prompts, summaries) -> None:
+    root = Path(root)
+    (root / "chunks").mkdir(parents=True, exist_ok=True)
+    (root / "prompts").mkdir(parents=True, exist_ok=True)
+    dump = lambda name, obj: (root / name).write_text(json.dumps(obj, indent=2), encoding="utf-8")
+    dump("commits.json", [asdict(c) for c in commits])
+    dump("filter_decisions.json", [asdict(d) for d in decisions])
+    dump("clusters.json", {m: [c.hash for c in cs] for m, cs in groups.items()})
+    dump("metrics.json", metrics)
+    dump("summaries.json", summaries)
+    for m, ch in chunks.items():
+        safe = "".join(ch_ if ch_.isalnum() or ch_ in "-_" else "_" for ch_ in m)
+        (root / "chunks" / f"{safe}.txt").write_text(ch.text, encoding="utf-8")
+        (root / "prompts" / f"{safe}.txt").write_text(prompts[m], encoding="utf-8")