gitlogstats 1.4.1__tar.gz → 1.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/PKG-INFO +15 -2
  2. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/README.md +14 -1
  3. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/docs/SPEC.md +7 -1
  4. gitlogstats-1.5.1/docs/examples/README.md +70 -0
  5. gitlogstats-1.5.1/docs/examples/assignment-stats.sh +113 -0
  6. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/pyproject.toml +1 -1
  7. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/scripts/board/manifest.yaml +17 -0
  8. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/__main__.py +20 -0
  9. gitlogstats-1.5.1/src/gitlogstats/clones.py +85 -0
  10. gitlogstats-1.5.1/tests/test_clobber.py +252 -0
  11. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/CLAUDE.md +0 -0
  12. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/agents/developer-agent.md +0 -0
  13. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/agents/spec-reviewer.md +0 -0
  14. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/hooks/format-edited.sh +0 -0
  15. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/hooks/guard-paths.sh +0 -0
  16. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/hooks/guard-paths.test.mjs +0 -0
  17. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/hooks/verify.sh +0 -0
  18. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/settings.json +0 -0
  19. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/skills/phase-handoff/SKILL.md +0 -0
  20. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.claude/skills/stale-check/SKILL.md +0 -0
  21. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.github/workflows/build.yaml +0 -0
  22. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.gitignore +0 -0
  23. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/.prettierrc.json +0 -0
  24. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/LICENSE +0 -0
  25. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/Pipfile +0 -0
  26. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/Pipfile.lock +0 -0
  27. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/package-lock.json +0 -0
  28. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/package.json +0 -0
  29. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/repos.txt +0 -0
  30. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/requirements.txt +0 -0
  31. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/scripts/board/config.mjs +0 -0
  32. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/scripts/board/derive.mjs +0 -0
  33. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/scripts/board/derive.test.mjs +0 -0
  34. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/scripts/board/sync.mjs +0 -0
  35. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/__init__.py +0 -0
  36. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/git_logs_parser.py +0 -0
  37. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/github.py +0 -0
  38. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/github_stats.py +0 -0
  39. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/src/gitlogstats/identity.py +0 -0
  40. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fake_gh.py +0 -0
  41. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/commit_linked.json +0 -0
  42. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/commit_unlinked.json +0 -0
  43. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/contributors.json +0 -0
  44. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/contributors_page2.json +0 -0
  45. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/pulls_page1.json +0 -0
  46. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/pulls_page2.json +0 -0
  47. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/user_full.json +0 -0
  48. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/fixtures/github/user_private.json +0 -0
  49. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/test_git_logs_parser.py +0 -0
  50. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/test_github.py +0 -0
  51. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/test_identity.py +0 -0
  52. {gitlogstats-1.4.1 → gitlogstats-1.5.1}/tests/test_integration.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: gitlogstats
3
- Version: 1.4.1
3
+ Version: 1.5.1
4
4
  Summary: Track developers' contributions to one or more git repositories within a particular time range.
5
5
  Project-URL: Homepage, https://github.com/bloombar/gitlogstats
6
6
  Project-URL: Repository, https://github.com/bloombar/gitlogstats.git
@@ -726,7 +726,7 @@ The command `gitlogstats --help` shows the usage instructions:
726
726
  ```
727
727
  usage: gitlogstats [-h] (-r REPOSITORY | -rf REPOFILE) [-u USER] [-s START]
728
728
  [-e END] [-x EXCLUSIONS] [-f {csv,json,markdown}]
729
- [-b BRANCH] [-v] [-c] [--github]
729
+ [-b BRANCH] [-v] [-c] [--github] [--clobber]
730
730
 
731
731
  options:
732
732
  -h, --help show this help message and exit
@@ -757,6 +757,9 @@ options:
757
757
  and their commits made on the GitHub website, and
758
758
  match GitHub users to git authors. Needs the GitHub
759
759
  CLI (gh), signed in
760
+ --clobber Delete the repository clones this run used from the
761
+ repos/ directory after printing the results (default:
762
+ keep them for faster later runs)
760
763
  ```
761
764
 
762
765
  In case `gitlogstats` is not found as a command, even after installing, try `python -m gitlogstats` or `python3 -m gitlogstats` instead.
@@ -837,6 +840,16 @@ By default, every contributor in a repository's history is listed, with zeros fo
837
840
  gitlogstats -rf repos.txt -c
838
841
  ```
839
842
 
843
+ ### Cloned repositories
844
+
845
+ Each repository is cloned into a `repos/` directory under the directory you run the command from, and later runs update (`git pull`) the existing clone instead of cloning again, which makes them much faster. Add `--clobber` to delete the clones the run used, once the results have been printed; `repos/` itself is removed too if nothing else is left in it. Clones from other runs are kept, and nothing is deleted if the run fails before printing, so you can inspect it. With `-v`, each deleted path is listed on standard error.
846
+
847
+ ```
848
+ gitlogstats -rf repos.txt --clobber
849
+ ```
850
+
851
+ `--clobber` is unrelated to `-c/--clean`, which only hides contributors without activity.
852
+
840
853
  ### GitHub statistics
841
854
 
842
855
  Add `--github` to also report each contributor's pull requests, pushes, website commits and email addresses. Nine columns are appended to every output format:
@@ -29,7 +29,7 @@ The command `gitlogstats --help` shows the usage instructions:
29
29
  ```
30
30
  usage: gitlogstats [-h] (-r REPOSITORY | -rf REPOFILE) [-u USER] [-s START]
31
31
  [-e END] [-x EXCLUSIONS] [-f {csv,json,markdown}]
32
- [-b BRANCH] [-v] [-c] [--github]
32
+ [-b BRANCH] [-v] [-c] [--github] [--clobber]
33
33
 
34
34
  options:
35
35
  -h, --help show this help message and exit
@@ -60,6 +60,9 @@ options:
60
60
  and their commits made on the GitHub website, and
61
61
  match GitHub users to git authors. Needs the GitHub
62
62
  CLI (gh), signed in
63
+ --clobber Delete the repository clones this run used from the
64
+ repos/ directory after printing the results (default:
65
+ keep them for faster later runs)
63
66
  ```
64
67
 
65
68
  In case `gitlogstats` is not found as a command, even after installing, try `python -m gitlogstats` or `python3 -m gitlogstats` instead.
@@ -140,6 +143,16 @@ By default, every contributor in a repository's history is listed, with zeros fo
140
143
  gitlogstats -rf repos.txt -c
141
144
  ```
142
145
 
146
+ ### Cloned repositories
147
+
148
+ Each repository is cloned into a `repos/` directory under the directory you run the command from, and later runs update (`git pull`) the existing clone instead of cloning again, which makes them much faster. Add `--clobber` to delete the clones the run used, once the results have been printed; `repos/` itself is removed too if nothing else is left in it. Clones from other runs are kept, and nothing is deleted if the run fails before printing, so you can inspect it. With `-v`, each deleted path is listed on standard error.
149
+
150
+ ```
151
+ gitlogstats -rf repos.txt --clobber
152
+ ```
153
+
154
+ `--clobber` is unrelated to `-c/--clean`, which only hides contributors without activity.
155
+
143
156
  ### GitHub statistics
144
157
 
145
158
  Add `--github` to also report each contributor's pull requests, pushes, website commits and email addresses. Nine columns are appended to every output format:
@@ -29,7 +29,7 @@ This document describes what the tool does today and what is planned. Each requi
29
29
 
30
30
  #### REPO-3 Local clone cache
31
31
 
32
- Repositories are cloned into a `repos/` directory under the current working directory, in a folder named the way `git clone` names it (the last path segment, minus a trailing `.git`; dots in the name are kept). On later runs the existing clone is updated with `git pull` instead of being cloned again.
32
+ Repositories are cloned into a `repos/` directory under the current working directory, in a folder named the way `git clone` names it (the last path segment, minus a trailing `.git`; dots in the name are kept). On later runs the existing clone is updated with `git pull` instead of being cloned again. `--clobber` removes the clones after the run (REPO-5).
33
33
 
34
34
  **Status:** Done
35
35
 
@@ -39,6 +39,12 @@ Repositories are cloned into a `repos/` directory under the current working dire
39
39
 
40
40
  **Status:** Done
41
41
 
42
+ #### REPO-5 Remove clones after the run
43
+
44
+ `--clobber` deletes the clones this run used from the `repos/` directory (REPO-3) once the results have been printed, and then removes `repos/` itself if nothing else is left in it. Clones in `repos/` that this run did not use are kept. It is off by default, so clones are kept and reused by later runs. If the run fails before the results are printed, nothing is deleted, so the clones can be inspected. Only directories inside `repos/` are deleted; a symbolic link in place of a clone is removed without following it. If `repos/` is itself a symbolic link to another directory, the clones are deleted from that directory and the link is kept. A clone that cannot be fully deleted (e.g. it contains a folder the user cannot open) is reported on standard error and the remaining clones are still deleted; the exit status stays 0, since the results were printed. Read-only files that git writes inside a clone (as on Windows) are deleted too. With `-v`, each deleted path is printed to standard error. The name follows build tools such as Rake, where `clean` removes intermediate files and `clobber` also removes caches; it is unrelated to `-c/--clean` (FILT-4).
45
+
46
+ **Status:** Done
47
+
42
48
  ### 2. Contribution Statistics
43
49
 
44
50
  #### STAT-1 Commit count
@@ -0,0 +1,70 @@
1
+ # Examples
2
+
3
+ ## `assignment-stats.sh`: statistics for one assignment
4
+
5
+ Reports every student's contributions to one assignment: all the repositories in a GitHub organization whose names start with the same prefix (e.g. `web-app-exercise-`), over a date range. It works in bash on macOS, Linux, WSL and Git Bash on Windows.
6
+
7
+ ### Requirements
8
+
9
+ - Python 3
10
+ - git
11
+ - The [GitHub CLI](https://cli.github.com) (`gh`), signed in with `gh auth login`
12
+
13
+ You don't need to install `gitlogstats` yourself: the script installs or upgrades it each time it runs (see [What the script does](#what-the-script-does)).
14
+
15
+ ### Usage
16
+
17
+ ```bash
18
+ ./assignment-stats.sh ORGANIZATION_NAME REPO_PREFIX START_DATE END_DATE [gitlogstats options...]
19
+ ```
20
+
21
+ | Argument | Meaning | Example |
22
+ | ------------------- | ---------------------------------------------------------------------- | ---------------------------- |
23
+ | `ORGANIZATION_NAME` | The GitHub organization that holds the students' repositories | `software-students-fall2029` |
24
+ | `REPO_PREFIX` | The start of the repository names for this assignment | `web-app-exercise-` |
25
+ | `START_DATE` | First day of the period, `mm/dd/yyyy` | `09/01/2029` |
26
+ | `END_DATE` | Last day of the period, `mm/dd/yyyy` (included) | `10/16/2029` |
27
+ | options | Anything after the four arguments is passed to `gitlogstats` unchanged | `--github -f markdown` |
28
+
29
+ Run it from any directory. The results are printed to standard output, so redirect them to save them; progress messages go to standard error and stay on screen.
30
+
31
+ ### Examples
32
+
33
+ Save a CSV of one assignment's statistics:
34
+
35
+ ```bash
36
+ ./assignment-stats.sh software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029 > web-app-exercise.csv
37
+ ```
38
+
39
+ Include the GitHub columns (pull requests, pushes, website commits, emails) and print a Markdown table:
40
+
41
+ ```bash
42
+ ./assignment-stats.sh software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029 --github -f markdown
43
+ ```
44
+
45
+ Leave out students with no activity in the period, as JSON:
46
+
47
+ ```bash
48
+ ./assignment-stats.sh software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029 -c -f json > web-app-exercise.json
49
+ ```
50
+
51
+ To run it from elsewhere, give its path, e.g. `bash docs/examples/assignment-stats.sh ...`. On Windows, run it from Git Bash or WSL, not from Command Prompt or PowerShell.
52
+
53
+ See the main [README](../../README.md) for every `gitlogstats` option and what each column means.
54
+
55
+ ### What the script does
56
+
57
+ 1. Checks the arguments and that Python, git and `gh` are installed and `gh` is signed in. If anything is missing it stops with a message saying what.
58
+ 2. Installs or upgrades `gitlogstats` in a Python environment of its own, `~/.gitlogstats-venv`, reused by later runs. Set `GITLOGSTATS_VENV` to keep it elsewhere. A separate environment is needed because Homebrew Python and recent Debian and Ubuntu releases refuse a system-wide `pip install`.
59
+ 3. Lists the organization's repositories whose names start with `REPO_PREFIX`, into a temporary file that is deleted when the script ends, even after an error. An existing `repos.txt` in your directory is never touched. If no repository matches, it stops with an error.
60
+ 4. Runs `gitlogstats` on those repositories with `--clobber`, so the clones it makes in `./repos/` are deleted once the results are printed.
61
+
62
+ ### Troubleshooting
63
+
64
+ | Message | What to do |
65
+ | -------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- |
66
+ | `'...' is not a date in mm/dd/yyyy format` | Write dates with two-digit month and day and a four-digit year, e.g. `09/01/2029` |
67
+ | `the GitHub CLI is not signed in` | Run `gh auth login` |
68
+ | `no repositories in '...' start with '...'` | Check the organization name and prefix; the prefix is case-sensitive |
69
+ | `could not create a virtual environment` | On Debian or Ubuntu, install the venv module: `sudo apt install python3-venv` |
70
+ | `Permission denied` when running `./assignment-stats.sh` | Make it executable with `chmod +x assignment-stats.sh`, or run it with `bash assignment-stats.sh ...` |
@@ -0,0 +1,113 @@
1
+ #!/usr/bin/env bash
2
+ #
3
+ # Report every student's contributions to one assignment: all repositories in
4
+ # a GitHub organization whose names start with a given prefix, over a date
5
+ # range. Works in bash on macOS, Linux, other Unix systems, WSL and Git Bash.
6
+ #
7
+ # Usage:
8
+ # ./assignment-stats.sh ORGANIZATION_NAME REPO_PREFIX START_DATE END_DATE [gitlogstats options...]
9
+ #
10
+ # Example:
11
+ # ./assignment-stats.sh software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029 > stats.csv
12
+ # ./assignment-stats.sh software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029 --github -f markdown
13
+ #
14
+ # Dates are mm/dd/yyyy and both days are included. Any options after the four
15
+ # arguments are passed to gitlogstats unchanged (e.g. --github, -f json, -c).
16
+ # The results go to standard output; progress messages go to standard error.
17
+ #
18
+ # Requires: Python 3, git, and the GitHub CLI (gh), signed in with
19
+ # `gh auth login` (gh is used to list the organization's repositories).
20
+ # gitlogstats is installed or upgraded automatically in its own Python
21
+ # environment (~/.gitlogstats-venv, or $GITLOGSTATS_VENV).
22
+
23
+ set -euo pipefail
24
+
25
+ usage() {
26
+ echo "usage: $(basename "$0") ORGANIZATION_NAME REPO_PREFIX START_DATE END_DATE [gitlogstats options...]" >&2
27
+ echo "example: $(basename "$0") software-students-fall2029 web-app-exercise- 09/01/2029 10/16/2029" >&2
28
+ }
29
+
30
+ if [ "$#" -lt 4 ]; then
31
+ usage
32
+ exit 2
33
+ fi
34
+
35
+ # retrieve command-line arguments
36
+ ORGANIZATION_NAME=$1
37
+ REPO_PREFIX=$2
38
+ START_DATE=$3
39
+ END_DATE=$4
40
+ shift 4 # anything left is passed to gitlogstats
41
+
42
+ # check the dates before doing any work
43
+ for date in "$START_DATE" "$END_DATE"; do
44
+ if ! [[ "$date" =~ ^[0-9]{2}/[0-9]{2}/[0-9]{4}$ ]]; then
45
+ echo "error: '$date' is not a date in mm/dd/yyyy format" >&2
46
+ usage
47
+ exit 2
48
+ fi
49
+ done
50
+
51
+ # find a Python 3 interpreter: python3 on macOS and Linux, often python on Git Bash
52
+ if command -v python3 >/dev/null 2>&1; then
53
+ PYTHON=python3
54
+ elif command -v python >/dev/null 2>&1; then
55
+ PYTHON=python
56
+ else
57
+ echo "error: Python 3 is not installed" >&2
58
+ exit 1
59
+ fi
60
+
61
+ for tool in git gh; do
62
+ if ! command -v "$tool" >/dev/null 2>&1; then
63
+ echo "error: '$tool' is not installed" >&2
64
+ exit 1
65
+ fi
66
+ done
67
+
68
+ if ! gh auth status >/dev/null 2>&1; then
69
+ echo "error: the GitHub CLI is not signed in; run 'gh auth login' first" >&2
70
+ exit 1
71
+ fi
72
+
73
+ # install or upgrade gitlogstats in a virtual environment of its own: many
74
+ # systems (Homebrew Python, recent Debian and Ubuntu) refuse a system-wide
75
+ # pip install. The environment is kept and reused by later runs; set
76
+ # GITLOGSTATS_VENV to put it somewhere else.
77
+ VENV=${GITLOGSTATS_VENV:-"$HOME/.gitlogstats-venv"}
78
+ if [ ! -d "$VENV" ]; then
79
+ echo "Creating a Python environment for gitlogstats in $VENV..." >&2
80
+ if ! "$PYTHON" -m venv "$VENV"; then
81
+ echo "error: could not create a virtual environment (on Debian or Ubuntu, install python3-venv)" >&2
82
+ exit 1
83
+ fi
84
+ fi
85
+ # the environment's Python is in bin/ on macOS, Linux and WSL, Scripts/ on Windows
86
+ if [ -x "$VENV/bin/python" ]; then
87
+ VENV_PYTHON="$VENV/bin/python"
88
+ else
89
+ VENV_PYTHON="$VENV/Scripts/python.exe"
90
+ fi
91
+ echo "Installing the latest gitlogstats..." >&2
92
+ "$VENV_PYTHON" -m pip install --quiet --disable-pip-version-check --upgrade gitlogstats >&2
93
+
94
+ # list the assignment's repositories in a temporary file, so an existing
95
+ # repos.txt in this directory is never overwritten
96
+ REPOS_FILE=$(mktemp "${TMPDIR:-/tmp}/gitlogstats-repos.XXXXXX")
97
+ trap 'rm -f "$REPOS_FILE"' EXIT
98
+
99
+ # the prefix is read by gh's built-in jq from an environment variable, so it
100
+ # needs no quoting and jq does not have to be installed
101
+ REPO_PREFIX="$REPO_PREFIX" gh repo list "$ORGANIZATION_NAME" --limit 1000 \
102
+ --json name,url \
103
+ --jq '.[] | select(.name | startswith(env.REPO_PREFIX)) | .url' >"$REPOS_FILE"
104
+
105
+ count=$(grep -c . "$REPOS_FILE" || true)
106
+ if [ "$count" -eq 0 ]; then
107
+ echo "error: no repositories in '$ORGANIZATION_NAME' start with '$REPO_PREFIX'" >&2
108
+ exit 1
109
+ fi
110
+ echo "Analyzing $count repositories from $START_DATE to $END_DATE..." >&2
111
+
112
+ # --clobber deletes the clones once the results are printed
113
+ "$VENV_PYTHON" -m gitlogstats -rf "$REPOS_FILE" -s "$START_DATE" -e "$END_DATE" --clobber "$@"
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "gitlogstats"
7
7
  description = "Track developers' contributions to one or more git repositories within a particular time range."
8
- version = "1.4.1"
8
+ version = "1.5.1"
9
9
  authors = [
10
10
  { name="Tobody Tothing", email="nobody.nothing@nowhere.com" },
11
11
  ]
@@ -194,6 +194,7 @@
194
194
  directory, in a folder named the way git clone names it (the last path
195
195
  segment, minus a trailing .git; dots in the name are kept). On later runs
196
196
  the existing clone is updated with git pull instead of being cloned again.
197
+ --clobber removes the clones after the run (REPO-5).
197
198
  section: §1 Repository Selection
198
199
  family: REPO
199
200
  phase: 1
@@ -209,6 +210,22 @@
209
210
  family: REPO
210
211
  phase: 1
211
212
  status: Done
213
+ - id: REPO-5
214
+ title: Remove clones after the run
215
+ full: --clobber deletes the clones this run used from the repos/ directory
216
+ (REPO-3) once the results have been printed, and then removes repos/ itself
217
+ if nothing else is left in it. Clones in repos/ that this run did not use
218
+ are kept. It is off by default, so clones are kept and reused by later runs.
219
+ If the run fails before the results are printed, nothing is deleted, so the
220
+ clones can be inspected. Only directories inside repos/ are deleted; a
221
+ symbolic link in place of a clone is removed without following it. If repos/
222
+ is itself a symbolic link to another directory, the clones are deleted from
223
+ that directory and the link is kept. A clone that cannot be fully deleted
224
+ (e.g. it contains a folder…
225
+ section: §1 Repository Selection
226
+ family: REPO
227
+ phase: 1
228
+ status: Done
212
229
  - id: STAT-1
213
230
  title: Commit count
214
231
  full: "For each contributor, the number of non-merge commits they authored in
@@ -5,6 +5,7 @@ import datetime
5
5
  import re
6
6
  import sys
7
7
  from . import GitLogsParser
8
+ from .clones import remove_clones
8
9
  from .git_logs_parser import FIELDNAMES
9
10
  from .github import (
10
11
  ACTIVITY_FIELDNAMES,
@@ -155,6 +156,14 @@ def main():
155
156
  default=False,
156
157
  action="store_true",
157
158
  )
159
+ parser.add_argument(
160
+ "--clobber",
161
+ help="Delete the repository clones this run used from the repos/ "
162
+ "directory after printing the results (default: keep them for faster "
163
+ "later runs)",
164
+ default=False,
165
+ action="store_true",
166
+ )
158
167
  args = parser.parse_args()
159
168
 
160
169
  if datetime.datetime.strptime(args.start, "%m/%d/%Y") > datetime.datetime.strptime(
@@ -179,6 +188,7 @@ def main():
179
188
  with open(args.repofile, "r", encoding="utf8") as f:
180
189
  repository_urls = GitLogsParser.read_repository_urls(f.read())
181
190
 
191
+ start_dir = os.getcwd() # REPO-5: where the command was started from
182
192
  # navigate into the directory where repos will be stored
183
193
  repos_dir = os.path.join(
184
194
  os.getcwd(), "repos"
@@ -190,6 +200,7 @@ def main():
190
200
  # loop through each git repository url, accumulating all results
191
201
  all_results = []
192
202
  last_parser = None
203
+ used_clones = [] # REPO-5: the clone directories this run cloned or pulled
193
204
  for repo_url in repository_urls:
194
205
  os.chdir(repos_dir) # start from the parent directory of all repos
195
206
 
@@ -221,6 +232,8 @@ def main():
221
232
  ["git", "pull"], capture_output=True, check=True
222
233
  ) # get the latest code from github
223
234
 
235
+ used_clones.append(repo_dir)
236
+
224
237
  # with --github, -u and -c are applied after GitHub users are combined
225
238
  last_parser = GitLogsParser(
226
239
  repo=repo_dir,
@@ -250,6 +263,13 @@ def main():
250
263
  # end with exactly one newline: CSV and Markdown output already has one
251
264
  print(output, end="" if output.endswith("\n") else "\n")
252
265
 
266
+ # REPO-5: only now that the results are out, delete the clones. Leave the
267
+ # clone we were last inside first, so the deletion never depends on it.
268
+ if args.clobber:
269
+ sys.stdout.flush()
270
+ os.chdir(start_dir)
271
+ remove_clones(repos_dir, used_clones, args.verbose)
272
+
253
273
 
254
274
  # if this script is being run directly...
255
275
  if __name__ == "__main__":
@@ -0,0 +1,85 @@
1
+ """Deleting the repository clones a run used (REPO-5, --clobber)."""
2
+
3
+ import os
4
+ import shutil
5
+ import stat
6
+ import sys
7
+
8
+
9
+ def _make_writable_and_retry(function, path, exc_info):
10
+ """
11
+ Error handler for shutil.rmtree: git marks some files read-only (notably
12
+ pack files on Windows), which stops them being deleted. Clear the read-only
13
+ bit on the file and its folder, then try again. Any other error is raised,
14
+ as is a failure to open or list a folder: retrying os.open or os.scandir
15
+ with only a path would itself fail with a TypeError.
16
+ """
17
+ if not isinstance(exc_info[1], PermissionError) or function not in (
18
+ os.unlink,
19
+ os.remove,
20
+ os.rmdir,
21
+ ):
22
+ raise exc_info[1]
23
+ for target in (os.path.dirname(path), path):
24
+ try:
25
+ os.chmod(target, os.stat(target).st_mode | stat.S_IRWXU)
26
+ except OSError:
27
+ pass
28
+ function(path)
29
+
30
+
31
+ def _delete(path):
32
+ """
33
+ Delete one clone. A symbolic link is unlinked, never followed.
34
+ @param path: the clone's path
35
+ """
36
+ if os.path.islink(path) or not os.path.isdir(path):
37
+ os.remove(path)
38
+ else:
39
+ shutil.rmtree(path, onerror=_make_writable_and_retry)
40
+
41
+
42
+ def remove_clones(repos_dir, clone_dirs, verbose=False):
43
+ """
44
+ Delete the given clones, then remove repos_dir if nothing else is left in it.
45
+ Only direct children of repos_dir are deleted; any other path is skipped.
46
+ A failed deletion is reported on standard error and does not stop the rest.
47
+ @param repos_dir: the directory holding the clones
48
+ @param clone_dirs: the paths of the clones to delete (duplicates are fine)
49
+ @param verbose: whether to report each deleted or skipped path on stderr
50
+ """
51
+ real_repos = os.path.realpath(repos_dir)
52
+ seen = set()
53
+ for clone_dir in clone_dirs:
54
+ path = os.path.normpath(os.path.abspath(clone_dir))
55
+ if path in seen:
56
+ continue
57
+ seen.add(path)
58
+ # the folder that holds the clone must be repos_dir itself; checking the
59
+ # parent (not the path) means a symbolic link is judged by where it sits
60
+ if os.path.realpath(os.path.dirname(path)) != real_repos:
61
+ if verbose:
62
+ print(
63
+ f"gitlogstats: not deleting {path}: not inside {repos_dir}",
64
+ file=sys.stderr,
65
+ )
66
+ continue
67
+ if not os.path.lexists(path):
68
+ continue
69
+ try:
70
+ _delete(path)
71
+ except OSError as exc:
72
+ print(
73
+ f"gitlogstats: could not delete {path}: {exc.strerror or exc}",
74
+ file=sys.stderr,
75
+ )
76
+ continue
77
+ if verbose:
78
+ print(f"gitlogstats: deleted {path}", file=sys.stderr)
79
+
80
+ try:
81
+ os.rmdir(repos_dir) # only succeeds when empty
82
+ except OSError:
83
+ return
84
+ if verbose:
85
+ print(f"gitlogstats: deleted {repos_dir}", file=sys.stderr)
@@ -0,0 +1,252 @@
1
+ """Tests for --clobber (REPO-5): deleting the clones a run used."""
2
+
3
+ import json
4
+ import os
5
+ import stat
6
+ import sys
7
+
8
+ import pytest
9
+
10
+ from gitlogstats import clones
11
+ from gitlogstats.__main__ import main
12
+ from gitlogstats.clones import remove_clones
13
+ from test_integration import (
14
+ DATES,
15
+ GITHUB_URL,
16
+ Repo,
17
+ github_world,
18
+ lines,
19
+ pr,
20
+ run_github,
21
+ )
22
+
23
+
24
+ @pytest.fixture
25
+ def remotes(tmp_path):
26
+ """Two 'remote' repositories that the command clones by path."""
27
+ one = Repo(tmp_path / "remotes" / "one")
28
+ one.write("a.txt", lines(3))
29
+ one.commit("Alice", "one: add")
30
+ two = Repo(tmp_path / "remotes" / "two.repo")
31
+ two.write("c.txt", lines(2))
32
+ two.commit("Alice", "two: add")
33
+ return one, two
34
+
35
+
36
+ @pytest.fixture
37
+ def work(tmp_path):
38
+ path = tmp_path / "work"
39
+ path.mkdir()
40
+ return path
41
+
42
+
43
+ def run(monkeypatch, capsys, work, *argv):
44
+ """Run the command in *work*; returns (stdout, stderr)."""
45
+ monkeypatch.chdir(work)
46
+ monkeypatch.setattr(
47
+ sys, "argv", ["gitlogstats", *argv, "-s", "01/01/2024"] + ["-e", "12/31/2024"]
48
+ )
49
+ main()
50
+ captured = capsys.readouterr()
51
+ return captured.out, captured.err
52
+
53
+
54
+ class TestCommand:
55
+ def test_clones_are_kept_without_the_flag(self, remotes, work, monkeypatch, capsys):
56
+ run(monkeypatch, capsys, work, "-r", remotes[0].path)
57
+ assert (work / "repos" / "one" / ".git").is_dir()
58
+
59
+ def test_clobber_prints_the_same_results_and_removes_everything(
60
+ self, remotes, work, monkeypatch, capsys
61
+ ):
62
+ kept, _ = run(monkeypatch, capsys, work, "-r", remotes[0].path, "-f", "json")
63
+ assert (work / "repos").exists()
64
+ # a fresh directory so that the second run clones again
65
+ other = work.parent / "work2"
66
+ other.mkdir()
67
+ out, err = run(
68
+ monkeypatch, capsys, other, "-r", remotes[0].path, "-f", "json", "--clobber"
69
+ )
70
+ assert out == kept and json.loads(out)[0]["commits"] == 1
71
+ assert err == ""
72
+ assert os.listdir(other) == [] # repos/ itself is gone
73
+
74
+ def test_other_clones_and_repos_dir_are_kept(
75
+ self, remotes, work, monkeypatch, capsys
76
+ ):
77
+ run(monkeypatch, capsys, work, "-r", remotes[1].path)
78
+ out, _ = run(monkeypatch, capsys, work, "-r", remotes[0].path, "--clobber")
79
+ assert "Alice,one," in out
80
+ assert sorted(os.listdir(work / "repos")) == ["two.repo"]
81
+
82
+ def test_repofile_with_two_repos_and_a_duplicate(
83
+ self, remotes, work, monkeypatch, capsys
84
+ ):
85
+ repofile = work.parent / "list.txt"
86
+ repofile.write_text(
87
+ f"{remotes[0].path}\n{remotes[1].path}\n{remotes[0].path}\n"
88
+ )
89
+ out, _ = run(monkeypatch, capsys, work, "-rf", str(repofile), "--clobber")
90
+ assert "Alice,one," in out and "Alice,two.repo," in out
91
+ assert os.listdir(work) == []
92
+
93
+ def test_failure_before_output_deletes_nothing(
94
+ self, remotes, work, monkeypatch, capsys
95
+ ):
96
+ repofile = work.parent / "list.txt"
97
+ repofile.write_text(f"{remotes[0].path}\n{work.parent / 'missing'}\n")
98
+ with pytest.raises(Exception):
99
+ run(monkeypatch, capsys, work, "-rf", str(repofile), "--clobber")
100
+ assert (work / "repos" / "one" / ".git").is_dir()
101
+ assert capsys.readouterr().out == ""
102
+
103
+ def test_verbose_lists_deleted_paths_on_stderr_only(
104
+ self, remotes, work, monkeypatch, capsys
105
+ ):
106
+ quiet, _ = run(monkeypatch, capsys, work, "-r", remotes[0].path, "--clobber")
107
+ out, err = run(
108
+ monkeypatch, capsys, work, "-r", remotes[0].path, "--clobber", "-v"
109
+ )
110
+ assert out == quiet
111
+ repos = os.path.realpath(work / "repos")
112
+ assert f"gitlogstats: deleted {os.path.join(repos, 'one')}" in err
113
+ assert f"gitlogstats: deleted {repos}\n" in err
114
+ assert "deleted" not in out
115
+
116
+ def test_clobber_with_github(self, remotes, tmp_path, monkeypatch, capsys):
117
+ github_world(tmp_path, monkeypatch, remotes[0], prs=[], contributors=[])
118
+ out, _ = run_github(
119
+ monkeypatch,
120
+ capsys,
121
+ tmp_path,
122
+ "-r",
123
+ GITHUB_URL,
124
+ "--github",
125
+ *DATES,
126
+ "--clobber",
127
+ )
128
+ assert "Alice" in out
129
+ assert not (tmp_path / "work" / "repos").exists()
130
+
131
+ def test_nothing_to_delete_when_no_repositories(self, work, monkeypatch, capsys):
132
+ repofile = work.parent / "empty.txt"
133
+ repofile.write_text("# nothing\n")
134
+ out, _ = run(monkeypatch, capsys, work, "-rf", str(repofile), "--clobber")
135
+ assert out == "" and os.listdir(work) == []
136
+
137
+
138
+ class TestRemoveClones:
139
+ def test_read_only_files_are_deleted(self, tmp_path):
140
+ repos = tmp_path / "repos"
141
+ clone = repos / "x"
142
+ (clone / "objects").mkdir(parents=True)
143
+ for name in ("a", "objects/pack"):
144
+ (clone / name).write_text("data")
145
+ os.chmod(clone / name, stat.S_IREAD)
146
+ os.chmod(clone / "objects", stat.S_IREAD | stat.S_IEXEC)
147
+ remove_clones(str(repos), [str(clone)])
148
+ assert not repos.exists()
149
+
150
+ def test_symlink_is_unlinked_and_target_untouched(self, tmp_path):
151
+ target = tmp_path / "precious"
152
+ target.mkdir()
153
+ (target / "keep.txt").write_text("keep")
154
+ repos = tmp_path / "repos"
155
+ repos.mkdir()
156
+ os.symlink(target, repos / "link")
157
+ remove_clones(str(repos), [str(repos / "link")])
158
+ assert not repos.exists()
159
+ assert (target / "keep.txt").read_text() == "keep"
160
+
161
+ def test_paths_outside_repos_are_skipped(self, tmp_path, capsys):
162
+ repos = tmp_path / "repos"
163
+ repos.mkdir()
164
+ outside = tmp_path / "outside"
165
+ outside.mkdir()
166
+ nested = repos / "a" / "b"
167
+ nested.mkdir(parents=True)
168
+ for path in (outside, repos / ".." / "outside", nested, repos):
169
+ remove_clones(str(repos), [str(path)], verbose=True)
170
+ assert outside.is_dir() and nested.is_dir() and repos.is_dir()
171
+ assert capsys.readouterr().err.count("not deleting") == 4
172
+
173
+ def test_skipped_paths_are_silent_without_verbose(self, tmp_path, capsys):
174
+ repos = tmp_path / "repos"
175
+ repos.mkdir()
176
+ outside = tmp_path / "outside"
177
+ outside.mkdir()
178
+ remove_clones(str(repos), [str(outside)])
179
+ assert outside.is_dir() and capsys.readouterr().err == ""
180
+
181
+ def test_a_file_in_place_of_a_clone_is_removed(self, tmp_path):
182
+ repos = tmp_path / "repos"
183
+ repos.mkdir()
184
+ (repos / "f").write_text("x")
185
+ remove_clones(str(repos), [str(repos / "f")])
186
+ assert not repos.exists()
187
+
188
+ def test_deletion_error_is_reported_and_the_rest_continues(
189
+ self, tmp_path, monkeypatch, capsys
190
+ ):
191
+ repos = tmp_path / "repos"
192
+ for name in ("bad", "good"):
193
+ (repos / name).mkdir(parents=True)
194
+
195
+ real = clones._delete
196
+
197
+ def flaky(path):
198
+ if path.endswith("bad"):
199
+ raise PermissionError(13, "Permission denied")
200
+ real(path)
201
+
202
+ monkeypatch.setattr(clones, "_delete", flaky)
203
+ remove_clones(str(repos), [str(repos / "bad"), str(repos / "good")])
204
+ err = capsys.readouterr().err
205
+ assert (
206
+ f"gitlogstats: could not delete {repos / 'bad'}: Permission denied" in err
207
+ )
208
+ assert os.listdir(repos) == ["bad"]
209
+
210
+ @pytest.mark.skipif(
211
+ sys.platform == "win32" or os.geteuid() == 0,
212
+ reason="needs POSIX permissions that apply to the current user",
213
+ )
214
+ def test_unreadable_folder_is_reported_and_the_rest_continues(
215
+ self, tmp_path, capsys
216
+ ):
217
+ # a folder that cannot be opened makes rmtree call the handler with
218
+ # os.open; retrying that must not crash after the results are printed
219
+ repos = tmp_path / "repos"
220
+ locked = repos / "one" / "notes" / "private"
221
+ locked.mkdir(parents=True)
222
+ (locked / "a.txt").write_text("x")
223
+ (repos / "two").mkdir()
224
+ locked.chmod(0)
225
+ try:
226
+ remove_clones(str(repos), [str(repos / "one"), str(repos / "two")])
227
+ finally:
228
+ locked.chmod(stat.S_IRWXU)
229
+ err = capsys.readouterr().err
230
+ assert f"gitlogstats: could not delete {repos / 'one'}" in err
231
+ assert not (repos / "two").exists()
232
+
233
+ def test_other_errors_while_deleting_are_not_retried(self, tmp_path):
234
+ with pytest.raises(FileNotFoundError):
235
+ clones._make_writable_and_retry(
236
+ os.remove, str(tmp_path / "gone"), (None, FileNotFoundError(), None)
237
+ )
238
+
239
+ def test_retry_gives_up_when_chmod_is_impossible(self, tmp_path):
240
+ # the paths do not exist, so chmod fails quietly and the retry raises
241
+ with pytest.raises(FileNotFoundError):
242
+ clones._make_writable_and_retry(
243
+ os.remove,
244
+ str(tmp_path / "no" / "gone"),
245
+ (None, PermissionError(), None),
246
+ )
247
+
248
+ def test_a_clone_that_is_already_gone_is_not_an_error(self, tmp_path, capsys):
249
+ repos = tmp_path / "repos"
250
+ repos.mkdir()
251
+ remove_clones(str(repos), [str(repos / "gone")], verbose=True)
252
+ assert capsys.readouterr().err == f"gitlogstats: deleted {repos}\n"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes