codeyear 0.2.14__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
codeyear/scanner.py ADDED
@@ -0,0 +1,491 @@
1
+ """Find local git repositories and extract one year of commit data.
2
+
3
+ Everything runs against the local ``git`` binary; nothing is uploaded.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import os
9
+ import re
10
+ import subprocess
11
+ from dataclasses import dataclass, field
12
+ from datetime import datetime
13
+ from pathlib import Path
14
+
15
+ from .languages import language_for, normalize_path
16
+
17
+ SKIP_DIRS: set[str] = {
18
+ ".git", ".svn", ".hg",
19
+ "node_modules", "bower_components", "dist", "build", "out", "output",
20
+ "target", "vendor", "third_party", "third-party", "coverage", "htmlcov",
21
+ ".next", ".nuxt", ".svelte-kit", ".parcel-cache", ".turbo",
22
+ "__pycache__", ".venv", "venv", "env", "site-packages",
23
+ "pods", "deriveddata", ".gradle", ".m2", "obj", "bin",
24
+ "$recycle.bin", "system volume information",
25
+ "codeyear-output", ".testrepo",
26
+ }
27
+
28
+ # A single repo with an enormous history can make one ``git log`` very slow.
29
+ # This caps output per repo; it is far above a normal personal year.
30
+ MAX_COMMITS_PER_REPO = 50_000
31
+ _NO_WINDOW = subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0
32
+
33
+
34
+ @dataclass
35
+ class FileChange:
36
+ path: str
37
+ lang: str
38
+ lines: int
39
+ added: int = 0
40
+ deleted: int = 0
41
+
42
+
43
+ @dataclass
44
+ class Commit:
45
+ repo: str
46
+ hash: str
47
+ dt: datetime
48
+ subject: str
49
+ added: int
50
+ deleted: int
51
+ files: int
52
+ author_email: str = ""
53
+ author_name: str = ""
54
+ changes: list[FileChange] = field(default_factory=list)
55
+
56
+
57
+ @dataclass
58
+ class RepoResult:
59
+ path: Path
60
+ name: str
61
+ commits: list[Commit] = field(default_factory=list)
62
+ warning: str | None = None
63
+ remote: str | None = None
64
+ readme_line: str | None = None
65
+ root: str | None = None
66
+ root_dt: datetime | None = None
67
+ total_commits: int = 0
68
+ tags_in_year: int = 0
69
+
70
+
71
+ def find_repos(root: Path, max_depth: int = 6) -> list[Path]:
72
+ """Return git repositories under *root* (or *root* itself if it is one)."""
73
+ root = root.resolve()
74
+ if not root.exists():
75
+ raise FileNotFoundError(f"路径不存在: {root}")
76
+ if not root.is_dir():
77
+ raise NotADirectoryError(f"不是文件夹: {root}")
78
+
79
+ if (root / ".git").exists():
80
+ return [root]
81
+
82
+ repos: list[Path] = []
83
+ base_parts = len(root.parts)
84
+
85
+ def _on_error(_err: OSError) -> None: # pragma: no cover - permission noise
86
+ return None
87
+
88
+ for current, dirs, _files in os.walk(root, onerror=_on_error):
89
+ cur = Path(current)
90
+ depth = len(cur.parts) - base_parts
91
+ if depth > max_depth:
92
+ dirs[:] = []
93
+ continue
94
+
95
+ dirs[:] = sorted(
96
+ d for d in dirs
97
+ if d.lower() not in SKIP_DIRS and not d.startswith(".")
98
+ )
99
+
100
+ if (cur / ".git").exists():
101
+ repos.append(cur)
102
+ # Do not descend into a repo: submodules would double count.
103
+ dirs[:] = []
104
+
105
+ return repos
106
+
107
+
108
+ def _repo_name(path: Path, root: Path) -> str:
109
+ try:
110
+ rel = path.relative_to(root)
111
+ except ValueError:
112
+ return path.name
113
+ return rel.as_posix() if str(rel) != "." else path.name
114
+
115
+
116
+ def _parse_int(value: str) -> int:
117
+ if value == "-" or not value:
118
+ return 0
119
+ try:
120
+ return int(value)
121
+ except ValueError:
122
+ return 0
123
+
124
+
125
+ def _build_log_cmd(
126
+ repo_path: Path,
127
+ year: int,
128
+ author: str | None,
129
+ *,
130
+ since_year: int | None = None,
131
+ since_dt: datetime | None = None,
132
+ until_dt: datetime | None = None,
133
+ trust_repo: bool,
134
+ use_mailmap: bool = True,
135
+ all_branches: bool = False,
136
+ ) -> list[str]:
137
+ start_year = since_year if since_year is not None else year
138
+ since_s = (
139
+ since_dt.strftime("%Y-%m-%dT%H:%M:%S")
140
+ if since_dt is not None
141
+ else f"{start_year}-01-01T00:00:00"
142
+ )
143
+ until_s = (
144
+ until_dt.strftime("%Y-%m-%dT%H:%M:%S")
145
+ if until_dt is not None
146
+ else f"{year + 1}-01-01T00:00:00"
147
+ )
148
+ cmd = ["git", "-c", "core.quotepath=false"]
149
+ if trust_repo:
150
+ # Some repos are copies/backups owned by another Windows account.
151
+ # Git refuses to read them unless the exact path is trusted. Scope the
152
+ # trust to this one repo instead of changing the user's global config.
153
+ cmd += ["-c", f"safe.directory={repo_path}"]
154
+ mailmap = use_mailmap and (repo_path / ".mailmap").exists()
155
+ name_fmt = "%aN" if mailmap else "%an"
156
+ email_fmt = "%aE" if mailmap else "%ae"
157
+ cmd += [
158
+ "-C", str(repo_path),
159
+ "log",
160
+ f"--since={since_s}",
161
+ f"--until={until_s}",
162
+ "--date=format:%Y-%m-%dT%H:%M:%S",
163
+ f"--pretty=format:%x1e%H%x1f%ad%x1f{email_fmt}%x1f{name_fmt}%x1f%s",
164
+ "--numstat",
165
+ "--no-renames",
166
+ "--no-merges",
167
+ f"--max-count={MAX_COMMITS_PER_REPO}",
168
+ ]
169
+ if all_branches:
170
+ cmd.append("--all")
171
+ if author:
172
+ cmd.append(f"--author={author}")
173
+ if mailmap:
174
+ # Git's canonical identity mapping: old emails/names collapse into the
175
+ # current one without requiring --alias on every run.
176
+ cmd.append("--use-mailmap")
177
+ return cmd
178
+
179
+
180
+ def _readme_line(repo_path: Path) -> str | None:
181
+ """First meaningful line of the repo's README — the author's own words."""
182
+ for name in ("README.md", "readme.md", "Readme.md", "README", "README.txt", "README.rst"):
183
+ path = repo_path / name
184
+ if not path.exists():
185
+ continue
186
+ try:
187
+ text = path.read_text(encoding="utf-8", errors="replace")
188
+ except OSError:
189
+ continue
190
+ for raw in text.splitlines():
191
+ line = raw.strip()
192
+ if not line or line.startswith("<") or line.startswith("!"):
193
+ continue
194
+ line = re.sub(r"[\\#>*`_\[\]()]","", line).strip()
195
+ line = re.sub(r"\s+", " ", line)
196
+ if len(line) >= 6:
197
+ return line[:110]
198
+ return None
199
+
200
+
201
+ _ORIGIN_SECTION_RE = re.compile(
202
+ r'^\[remote\s+"origin"\]\s*$([\s\S]*?)(?=^\[|\Z)', re.M
203
+ )
204
+ _ORIGIN_URL_RE = re.compile(r"^\s*url\s*=\s*(.+)$", re.M)
205
+
206
+
207
+ def _git_remote(repo_path: Path) -> str | None:
208
+ git_cfg = repo_path / ".git" / "config"
209
+ if git_cfg.is_file():
210
+ try:
211
+ cfg_text = git_cfg.read_text(encoding="utf-8", errors="replace")
212
+ sec = _ORIGIN_SECTION_RE.search(cfg_text)
213
+ if sec:
214
+ m = _ORIGIN_URL_RE.search(sec.group(1))
215
+ if m:
216
+ url = m.group(1).strip()
217
+ if url:
218
+ return url
219
+ else:
220
+ return None
221
+ except OSError:
222
+ pass
223
+ try:
224
+ proc = subprocess.run(
225
+ ["git", "-c", f"safe.directory={repo_path.as_posix()}",
226
+ "-C", str(repo_path), "remote", "get-url", "origin"],
227
+ capture_output=True, creationflags=_NO_WINDOW,
228
+ text=True, encoding="utf-8",
229
+ errors="replace", timeout=10,
230
+ )
231
+ except (OSError, subprocess.TimeoutExpired):
232
+ return None
233
+ url = (proc.stdout or "").strip()
234
+ return url or None
235
+
236
+
237
+ def _git_root(repo_path: Path) -> str | None:
238
+ """First root commit — the shared ancestor of copies of the same project."""
239
+ try:
240
+ proc = subprocess.run(
241
+ ["git", "-c", f"safe.directory={repo_path.as_posix()}",
242
+ "-C", str(repo_path), "rev-list", "--max-parents=0", "HEAD"],
243
+ capture_output=True, creationflags=_NO_WINDOW,
244
+ text=True, encoding="utf-8",
245
+ errors="replace", timeout=10,
246
+ )
247
+ except (OSError, subprocess.TimeoutExpired):
248
+ return None
249
+ roots = proc.stdout.strip().splitlines()
250
+ return roots[0].strip() if roots else None
251
+
252
+
253
+ def _git_text(repo_path: Path, args: list[str], timeout: int = 30) -> str:
254
+ """Run a small read-only git command; failures degrade to empty output."""
255
+ try:
256
+ proc = subprocess.run(
257
+ ["git", "-c", f"safe.directory={repo_path.as_posix()}",
258
+ "-C", str(repo_path), *args],
259
+ capture_output=True, creationflags=_NO_WINDOW,
260
+ text=True, encoding="utf-8",
261
+ errors="replace", timeout=timeout,
262
+ )
263
+ except (OSError, subprocess.TimeoutExpired):
264
+ return ""
265
+ return (proc.stdout or "").strip() if proc.returncode == 0 else ""
266
+
267
+
268
+ def _git_root_date(repo_path: Path, root_hash: str) -> datetime | None:
269
+ value = _git_text(
270
+ repo_path,
271
+ ["show", "-s", "--format=%ad", "--date=format:%Y-%m-%dT%H:%M:%S", root_hash],
272
+ )
273
+ try:
274
+ return datetime.strptime(value, "%Y-%m-%dT%H:%M:%S")
275
+ except ValueError:
276
+ return None
277
+
278
+
279
+ def _git_total_commits(repo_path: Path) -> int:
280
+ """All commits on HEAD — repository scope, not the filtered year scope."""
281
+ value = _git_text(repo_path, ["rev-list", "--count", "HEAD"])
282
+ try:
283
+ return int(value)
284
+ except ValueError:
285
+ return 0
286
+
287
+
288
+ def _git_root_info(repo_path: Path) -> tuple[str | None, datetime | None]:
289
+ """Fetch both root commit hash and its author date in one git call."""
290
+ value = _git_text(
291
+ repo_path,
292
+ [
293
+ "rev-list", "--max-parents=0",
294
+ "--format=%ad", "--date=format:%Y-%m-%dT%H:%M:%S", "HEAD",
295
+ ],
296
+ )
297
+ if not value:
298
+ return None, None
299
+ lines = [line.strip() for line in value.splitlines() if line.strip()]
300
+ root_hash: str | None = None
301
+ root_dt: datetime | None = None
302
+ for idx, line in enumerate(lines):
303
+ if line.startswith("commit "):
304
+ candidate = line[7:].strip()
305
+ if candidate:
306
+ root_hash = candidate
307
+ if idx + 1 < len(lines):
308
+ try:
309
+ root_dt = datetime.strptime(
310
+ lines[idx + 1], "%Y-%m-%dT%H:%M:%S"
311
+ )
312
+ except ValueError:
313
+ root_dt = None
314
+ break
315
+ return root_hash, root_dt
316
+
317
+
318
+ def _has_any_tags(repo_path: Path) -> bool:
319
+ git_dir = repo_path / ".git"
320
+ if not git_dir.is_dir():
321
+ return True
322
+ tags_dir = git_dir / "refs" / "tags"
323
+ try:
324
+ if tags_dir.is_dir() and any(tags_dir.iterdir()):
325
+ return True
326
+ except OSError:
327
+ return True
328
+ packed = git_dir / "packed-refs"
329
+ if packed.is_file():
330
+ try:
331
+ return "refs/tags/" in packed.read_text(
332
+ encoding="utf-8", errors="replace"
333
+ )
334
+ except OSError:
335
+ return True
336
+ return False
337
+
338
+
339
+ def _git_tags_in_year(repo_path: Path, year: int) -> int:
340
+ if not _has_any_tags(repo_path):
341
+ return 0
342
+ value = _git_text(repo_path, ["for-each-ref", "--format=%(creatordate:short)", "refs/tags"])
343
+ prefix = f"{year}-"
344
+ return sum(1 for line in value.splitlines() if line.strip().startswith(prefix))
345
+
346
+
347
+ def scan_repo(
348
+ repo_path: Path,
349
+ root: Path,
350
+ year: int,
351
+ *,
352
+ since_year: int | None = None,
353
+ since_dt: datetime | None = None,
354
+ until_dt: datetime | None = None,
355
+ author: str | None = None,
356
+ timeout: int = 300,
357
+ use_mailmap: bool = True,
358
+ all_branches: bool = False,
359
+ ) -> RepoResult:
360
+ start_year = since_year if since_year is not None else year
361
+ name = _repo_name(repo_path, root)
362
+ result = RepoResult(path=repo_path, name=name)
363
+
364
+ trust_repo = False
365
+ while True:
366
+ cmd = _build_log_cmd(
367
+ repo_path, year, author,
368
+ since_year=since_year,
369
+ since_dt=since_dt,
370
+ until_dt=until_dt,
371
+ trust_repo=trust_repo, use_mailmap=use_mailmap,
372
+ all_branches=all_branches,
373
+ )
374
+ try:
375
+ proc = subprocess.run(
376
+ cmd,
377
+ capture_output=True,
378
+ creationflags=_NO_WINDOW,
379
+ text=True,
380
+ encoding="utf-8",
381
+ errors="replace",
382
+ timeout=timeout,
383
+ )
384
+ except FileNotFoundError:
385
+ result.warning = "找不到 git 命令"
386
+ return result
387
+ except subprocess.TimeoutExpired:
388
+ result.warning = f"扫描超时(>{timeout}s)"
389
+ return result
390
+ except OSError as exc: # pragma: no cover - rare shell issues
391
+ result.warning = f"无法启动 git: {exc}"
392
+ return result
393
+
394
+ if proc.returncode == 0:
395
+ break
396
+
397
+ stderr = (proc.stderr or "").strip()
398
+ lower = stderr.lower()
399
+ if not trust_repo and (
400
+ "dubious ownership" in lower
401
+ or "safe.directory" in lower
402
+ or "detected dubious ownership" in lower
403
+ ):
404
+ trust_repo = True
405
+ continue
406
+ if "not a git repository" in lower:
407
+ # A stray .git file/dir inside a vendored dependency. Nothing to
408
+ # report: it is not a broken user repository.
409
+ return result
410
+
411
+ lines = stderr.splitlines()
412
+ result.warning = lines[-1] if lines else f"git exited {proc.returncode}"
413
+ return result
414
+
415
+ result.remote = _git_remote(repo_path)
416
+ result.readme_line = _readme_line(repo_path) if proc.stdout.strip() else None
417
+ result.root, result.root_dt = _git_root_info(repo_path)
418
+ result.total_commits = _git_total_commits(repo_path)
419
+ result.tags_in_year = _git_tags_in_year(repo_path, year)
420
+
421
+ if not proc.stdout.strip():
422
+ return result
423
+
424
+ for chunk in proc.stdout.split("\x1e"):
425
+ chunk = chunk.strip("\n")
426
+ if not chunk.strip():
427
+ continue
428
+
429
+ lines = chunk.splitlines()
430
+ header = lines[0].split("\x1f", 4)
431
+ if len(header) < 5:
432
+ continue
433
+
434
+ commit_hash, date_s, author_email, author_name, subject = header
435
+ try:
436
+ dt = datetime.strptime(date_s, "%Y-%m-%dT%H:%M:%S")
437
+ except ValueError:
438
+ continue
439
+ if since_dt is not None and until_dt is not None:
440
+ if not (since_dt <= dt <= until_dt):
441
+ continue
442
+ elif not (start_year <= dt.year <= year):
443
+ continue
444
+
445
+ added_total = 0
446
+ deleted_total = 0
447
+ files = 0
448
+ changes: list[FileChange] = []
449
+
450
+ for line in lines[1:]:
451
+ if not line.strip():
452
+ continue
453
+ parts = line.split("\t")
454
+ if len(parts) < 3:
455
+ continue
456
+ added = _parse_int(parts[0])
457
+ deleted = _parse_int(parts[1])
458
+ path = normalize_path("\t".join(parts[2:]))
459
+ files += 1
460
+ added_total += added
461
+ deleted_total += deleted
462
+
463
+ lang = language_for(path)
464
+ if lang is None:
465
+ continue
466
+ changes.append(
467
+ FileChange(
468
+ path=path,
469
+ lang=lang,
470
+ lines=added + deleted,
471
+ added=added,
472
+ deleted=deleted,
473
+ )
474
+ )
475
+
476
+ result.commits.append(
477
+ Commit(
478
+ repo=name,
479
+ hash=commit_hash,
480
+ dt=dt,
481
+ subject=subject.strip(),
482
+ added=added_total,
483
+ deleted=deleted_total,
484
+ files=files,
485
+ author_email=author_email.strip(),
486
+ author_name=author_name.strip(),
487
+ changes=changes,
488
+ )
489
+ )
490
+
491
+ return result