repodb 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repodb/__init__.py +9 -0
- repodb/core.py +629 -0
- repodb/py.typed +0 -0
- repodb-0.1.0.dist-info/METADATA +61 -0
- repodb-0.1.0.dist-info/RECORD +7 -0
- repodb-0.1.0.dist-info/WHEEL +4 -0
- repodb-0.1.0.dist-info/entry_points.txt +4 -0
repodb/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""repodb - keep a SQLite database of git project clone URLs, and clone from it.
|
|
2
|
+
|
|
3
|
+
See `repodb.core` for the data model and command-line interface.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from repodb.core import GitRepoDB, main, owner_of
|
|
7
|
+
|
|
8
|
+
__all__ = ["GitRepoDB", "main", "owner_of"]
|
|
9
|
+
__version__ = "0.1.0"
|
repodb/core.py
ADDED
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
"""Keep a SQLite database of git project clone URLs, and clone from it.
|
|
2
|
+
|
|
3
|
+
repodb scan ~/src # add local projects' origin URLs
|
|
4
|
+
repodb github USER # add USER's GitHub repos (needs gh)
|
|
5
|
+
repodb export --owner USER -o projects.json
|
|
6
|
+
repodb import projects.json
|
|
7
|
+
repodb clone ~/src [--owner USER | --json projects.json] [--group]
|
|
8
|
+
repodb list [--urls] [--owner USER] [--group]
|
|
9
|
+
repodb remove OWNER/NAME [NAME ...] | --owner USER --all
|
|
10
|
+
repodb info # schema, counts, hosts; read-only
|
|
11
|
+
|
|
12
|
+
Each row is ``(owner, name, url)``, keyed by ``(owner, name)``. The owner is
|
|
13
|
+
the first path component after the host in the URL (``alice`` in
|
|
14
|
+
``github.com/alice/r`` or ``git@gitlab.com:alice/r``), and every row must have
|
|
15
|
+
one, so local-path and ``file://`` remotes are not stored. Owner and name
|
|
16
|
+
compare case-insensitively.
|
|
17
|
+
|
|
18
|
+
The JSON form is ``{name: url}``, or with ``--group`` ``{owner: {name: url}}``.
|
|
19
|
+
Both forms are read back; group keys are ignored on read, since owner is
|
|
20
|
+
derived from the URL. ``clone`` recreates each project as ``DEST/<name>``, or
|
|
21
|
+
with ``--group`` ``DEST/<owner>/<name>``, and skips targets that already
|
|
22
|
+
exist. The flat forms reject a name held by two owners.
|
|
23
|
+
|
|
24
|
+
Installed as the ``repodb`` command; invoked as ``listrepos`` it runs ``list``.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import json
|
|
31
|
+
import re
|
|
32
|
+
import shutil
|
|
33
|
+
import sqlite3
|
|
34
|
+
import subprocess
|
|
35
|
+
import sys
|
|
36
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
37
|
+
from pathlib import Path
|
|
38
|
+
from typing import TypeVar
|
|
39
|
+
|
|
40
|
+
from typing_extensions import Self
|
|
41
|
+
|
|
42
|
+
DB_PATH = Path("~/.local/share/repodb/repos.sqlite").expanduser()
|
|
43
|
+
SRC_DIR = Path("~/src").expanduser()
|
|
44
|
+
|
|
45
|
+
# https://host/O/R, ssh://git@host:22/O/R, git@host:O/R.git; host needs a dot.
|
|
46
|
+
OWNER = re.compile(
|
|
47
|
+
r"^(?:[a-z][a-z0-9+.-]*://)?(?:[^@/]+@)?([\w-]+(?:\.[\w-]+)+)(?::\d+)?[:/]([^/]+)/[^/]",
|
|
48
|
+
re.IGNORECASE,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
Row = tuple[str, str, str] # (owner, name, url)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def owner_of(url: str) -> str | None:
|
|
55
|
+
"""Return the owner in a ``host/owner/repo`` *url*, or None if it has none."""
|
|
56
|
+
m = OWNER.match(url)
|
|
57
|
+
return m.group(2) if m else None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def host_of(url: str) -> str | None:
|
|
61
|
+
"""Return the lower-cased host in a ``host/owner/repo`` *url*, or None."""
|
|
62
|
+
m = OWNER.match(url)
|
|
63
|
+
return m.group(1).lower() if m else None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def primary_key(conn: sqlite3.Connection) -> tuple[str, ...]:
|
|
67
|
+
"""Return the primary-key columns of ``repos``, or () if there is no such table."""
|
|
68
|
+
return tuple(r[1] for r in conn.execute("PRAGMA table_info(repos)") if r[5])
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def info(db_path: Path, top: int = 5) -> list[str]:
|
|
72
|
+
"""Describe the database at *db_path* without writing to it.
|
|
73
|
+
|
|
74
|
+
Raises:
|
|
75
|
+
sqlite3.Error: if the file cannot be opened as a database.
|
|
76
|
+
"""
|
|
77
|
+
conn = sqlite3.connect(f"{db_path.resolve().as_uri()}?mode=ro", uri=True)
|
|
78
|
+
try:
|
|
79
|
+
pk = primary_key(conn)
|
|
80
|
+
current = pk == GitRepoDB.KEY
|
|
81
|
+
rows = (
|
|
82
|
+
conn.execute("SELECT owner, name, url FROM repos").fetchall()
|
|
83
|
+
if current
|
|
84
|
+
else []
|
|
85
|
+
)
|
|
86
|
+
finally:
|
|
87
|
+
conn.close()
|
|
88
|
+
fields = [
|
|
89
|
+
("database", f"{db_path} ({db_path.stat().st_size / 1024:.1f}KB)"),
|
|
90
|
+
(
|
|
91
|
+
"format",
|
|
92
|
+
"current, keyed by (owner, name)"
|
|
93
|
+
if current
|
|
94
|
+
else f"unsupported (primary key {', '.join(pk)})"
|
|
95
|
+
if pk
|
|
96
|
+
else "no repos table",
|
|
97
|
+
),
|
|
98
|
+
]
|
|
99
|
+
if current:
|
|
100
|
+
owners: dict[str, list[str]] = {} # lower-cased owner -> [spelling, names...]
|
|
101
|
+
hosts: dict[str, int] = {}
|
|
102
|
+
names: dict[str, list[str]] = {} # lower-cased name -> owners
|
|
103
|
+
for owner, name, url in rows:
|
|
104
|
+
owners.setdefault(owner.lower(), [owner]).append(name)
|
|
105
|
+
host = host_of(url) or "?"
|
|
106
|
+
hosts[host] = hosts.get(host, 0) + 1
|
|
107
|
+
names.setdefault(name.lower(), []).append(owner)
|
|
108
|
+
|
|
109
|
+
def counts(items: Iterable[tuple[str, int]]) -> str:
|
|
110
|
+
return ", ".join(f"{k} {n}" for k, n in items) or "-"
|
|
111
|
+
|
|
112
|
+
by_size = sorted(owners.values(), key=lambda o: (-(len(o) - 1), o[0].lower()))
|
|
113
|
+
shared = sorted((n, o) for n, o in names.items() if len(o) > 1)
|
|
114
|
+
fields += [
|
|
115
|
+
("projects", str(len(rows))),
|
|
116
|
+
("owners", str(len(owners))),
|
|
117
|
+
("hosts", counts(sorted(hosts.items(), key=lambda h: (-h[1], h[0])))),
|
|
118
|
+
("top owners", counts((o[0], len(o) - 1) for o in by_size[:top])),
|
|
119
|
+
(
|
|
120
|
+
"shared names",
|
|
121
|
+
", ".join(f"{n} ({', '.join(o)})" for n, o in shared) or "-",
|
|
122
|
+
),
|
|
123
|
+
]
|
|
124
|
+
return [f"{k + ':':<14}{v}" for k, v in fields]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
DB = TypeVar("DB", bound="GitRepoDB")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class GitRepoDB:
|
|
131
|
+
"""SQLite table of ``(owner, name, url)``, keyed by ``(owner, name)``."""
|
|
132
|
+
|
|
133
|
+
KEY = ("owner", "name")
|
|
134
|
+
SCHEMA = (
|
|
135
|
+
"CREATE TABLE IF NOT EXISTS repos ("
|
|
136
|
+
" owner TEXT NOT NULL COLLATE NOCASE CHECK (owner <> ''),"
|
|
137
|
+
" name TEXT NOT NULL COLLATE NOCASE CHECK (name <> ''),"
|
|
138
|
+
" url TEXT NOT NULL CHECK (url <> ''),"
|
|
139
|
+
" PRIMARY KEY (owner, name))"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
def __init__(self, db_path: str | Path = DB_PATH):
|
|
143
|
+
"""Open or create the database.
|
|
144
|
+
|
|
145
|
+
Raises:
|
|
146
|
+
ValueError: if an existing ``repos`` table has another primary key.
|
|
147
|
+
"""
|
|
148
|
+
self.db_path = Path(db_path)
|
|
149
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
150
|
+
self.conn = sqlite3.connect(self.db_path)
|
|
151
|
+
self.conn.execute(self.SCHEMA)
|
|
152
|
+
if (pk := primary_key(self.conn)) != self.KEY:
|
|
153
|
+
self.conn.close()
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f"{self.db_path}: unsupported schema, primary key {', '.join(pk)}"
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def __enter__(self) -> Self:
|
|
159
|
+
return self
|
|
160
|
+
|
|
161
|
+
def __exit__(self, *exc: object) -> None:
|
|
162
|
+
self.conn.close()
|
|
163
|
+
|
|
164
|
+
def add(self, projects: Iterable[tuple[str, str]]) -> None:
|
|
165
|
+
"""Insert or update ``(name, url)`` pairs, printing each one added or changed.
|
|
166
|
+
|
|
167
|
+
Raises:
|
|
168
|
+
ValueError: if a url has no owner; nothing is written.
|
|
169
|
+
"""
|
|
170
|
+
rows = []
|
|
171
|
+
for name, url in projects:
|
|
172
|
+
owner = owner_of(url)
|
|
173
|
+
if owner is None:
|
|
174
|
+
raise ValueError(f"{name!r}: no owner in url {url!r}")
|
|
175
|
+
rows.append((owner, name, url))
|
|
176
|
+
changes = []
|
|
177
|
+
with self.conn:
|
|
178
|
+
for owner, name, url in rows:
|
|
179
|
+
row = self.conn.execute(
|
|
180
|
+
"SELECT url FROM repos WHERE owner = ? AND name = ?", (owner, name)
|
|
181
|
+
).fetchone()
|
|
182
|
+
if row and row[0] == url:
|
|
183
|
+
continue
|
|
184
|
+
changes.append(
|
|
185
|
+
f"{'updated' if row else 'added'}: {owner}/{name} <- {url}"
|
|
186
|
+
)
|
|
187
|
+
self.conn.execute(
|
|
188
|
+
"INSERT INTO repos (owner, name, url) VALUES (?, ?, ?)"
|
|
189
|
+
" ON CONFLICT (owner, name) DO UPDATE SET url = excluded.url",
|
|
190
|
+
(owner, name, url),
|
|
191
|
+
)
|
|
192
|
+
for line in changes: # after the commit, so a rollback prints nothing
|
|
193
|
+
print(line)
|
|
194
|
+
|
|
195
|
+
def remove(self, specs: Iterable[str]) -> list[Row]:
|
|
196
|
+
"""Delete projects given as ``owner/name`` or a bare ``name``; return the rows.
|
|
197
|
+
|
|
198
|
+
Raises:
|
|
199
|
+
ValueError: if a spec matches no row, or a bare name matches several;
|
|
200
|
+
nothing is deleted.
|
|
201
|
+
"""
|
|
202
|
+
removed: list[Row] = []
|
|
203
|
+
with self.conn:
|
|
204
|
+
for spec in specs:
|
|
205
|
+
owner, _, name = spec.rpartition("/")
|
|
206
|
+
sql = "SELECT owner, name, url FROM repos WHERE name = ?"
|
|
207
|
+
if owner:
|
|
208
|
+
matches = self.conn.execute(
|
|
209
|
+
sql + " AND owner = ?", (name, owner)
|
|
210
|
+
).fetchall()
|
|
211
|
+
else:
|
|
212
|
+
matches = self.conn.execute(sql, (name,)).fetchall()
|
|
213
|
+
if not matches:
|
|
214
|
+
raise ValueError(f"{spec!r}: no such project")
|
|
215
|
+
if len(matches) > 1:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"{spec!r}: ambiguous, use one of "
|
|
218
|
+
+ ", ".join(f"{o}/{n}" for o, n, _ in matches)
|
|
219
|
+
)
|
|
220
|
+
self.conn.execute(
|
|
221
|
+
"DELETE FROM repos WHERE owner = ? AND name = ?", matches[0][:2]
|
|
222
|
+
)
|
|
223
|
+
removed.append(matches[0])
|
|
224
|
+
return removed
|
|
225
|
+
|
|
226
|
+
def remove_owner(self, owner: str) -> list[Row]:
|
|
227
|
+
"""Delete every project of *owner*; return the rows.
|
|
228
|
+
|
|
229
|
+
Raises:
|
|
230
|
+
ValueError: if *owner* has no projects.
|
|
231
|
+
"""
|
|
232
|
+
removed = self.rows(owner)
|
|
233
|
+
if not removed:
|
|
234
|
+
raise ValueError(f"{owner!r}: no projects for this owner")
|
|
235
|
+
with self.conn:
|
|
236
|
+
self.conn.execute("DELETE FROM repos WHERE owner = ?", (owner,))
|
|
237
|
+
return removed
|
|
238
|
+
|
|
239
|
+
def rows(self, owner: str | None = None) -> list[Row]:
|
|
240
|
+
"""Return rows sorted by owner then name, optionally only *owner*'s."""
|
|
241
|
+
sql = "SELECT owner, name, url FROM repos"
|
|
242
|
+
order = " ORDER BY owner, name"
|
|
243
|
+
if owner is None:
|
|
244
|
+
return self.conn.execute(sql + order).fetchall()
|
|
245
|
+
return self.conn.execute(sql + " WHERE owner = ?" + order, (owner,)).fetchall()
|
|
246
|
+
|
|
247
|
+
def by_owner(self, owner: str | None = None) -> dict[str, dict[str, str]]:
|
|
248
|
+
"""Return ``{owner: {name: url}}``, owners sorted."""
|
|
249
|
+
groups: dict[str, dict[str, str]] = {}
|
|
250
|
+
spelling: dict[
|
|
251
|
+
str, str
|
|
252
|
+
] = {} # one group per owner regardless of case; first spelling wins
|
|
253
|
+
for o, name, url in self.rows(owner):
|
|
254
|
+
groups.setdefault(spelling.setdefault(o.lower(), o), {})[name] = url
|
|
255
|
+
return groups
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def flatten(pairs: Iterable[tuple[str, str]]) -> dict[str, str]:
|
|
259
|
+
"""Return ``{name: url}``.
|
|
260
|
+
|
|
261
|
+
Raises:
|
|
262
|
+
ValueError: if a name appears twice, ignoring case, since both would
|
|
263
|
+
map to ``DEST/<name>`` on a case-insensitive filesystem.
|
|
264
|
+
"""
|
|
265
|
+
projects: dict[str, str] = {}
|
|
266
|
+
seen: dict[str, str] = {} # lower-cased name -> first spelling
|
|
267
|
+
for name, url in pairs:
|
|
268
|
+
if (first := seen.get(name.lower())) is not None:
|
|
269
|
+
raise ValueError(
|
|
270
|
+
f"{name!r}: held by more than one owner"
|
|
271
|
+
f"{'' if first == name else f' (as {first!r})'}; use --group"
|
|
272
|
+
)
|
|
273
|
+
seen[name.lower()] = name
|
|
274
|
+
projects[name] = url
|
|
275
|
+
return projects
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def scan(directory: Path) -> list[tuple[str, str]]:
|
|
279
|
+
"""Return ``(name, origin_url)`` for each git project directly under *directory*.
|
|
280
|
+
|
|
281
|
+
Projects without an ``origin`` remote, or whose url has no owner, are
|
|
282
|
+
reported on stderr and omitted.
|
|
283
|
+
"""
|
|
284
|
+
projects = []
|
|
285
|
+
for p in sorted(directory.iterdir()):
|
|
286
|
+
if not (p.is_dir() and (p / ".git").exists()):
|
|
287
|
+
continue
|
|
288
|
+
result = subprocess.run(
|
|
289
|
+
["git", "-C", str(p), "remote", "get-url", "origin"],
|
|
290
|
+
capture_output=True,
|
|
291
|
+
text=True,
|
|
292
|
+
check=False,
|
|
293
|
+
)
|
|
294
|
+
url = result.stdout.strip()
|
|
295
|
+
if result.returncode or not url:
|
|
296
|
+
print(f"no origin, skipping: {p.name}", file=sys.stderr)
|
|
297
|
+
elif owner_of(url) is None:
|
|
298
|
+
print(
|
|
299
|
+
f"no owner in origin url, skipping: {p.name} ({url})", file=sys.stderr
|
|
300
|
+
)
|
|
301
|
+
else:
|
|
302
|
+
projects.append((p.name, url))
|
|
303
|
+
return projects
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def github(
|
|
307
|
+
user: str, limit: int, ssh: bool, source: bool, no_archived: bool
|
|
308
|
+
) -> list[tuple[str, str]]:
|
|
309
|
+
"""Return ``(name, clone_url)`` for *user*'s GitHub repos via ``gh repo list``.
|
|
310
|
+
|
|
311
|
+
Raises:
|
|
312
|
+
subprocess.CalledProcessError: if ``gh`` fails, e.g. unknown user or no auth.
|
|
313
|
+
"""
|
|
314
|
+
field = "sshUrl" if ssh else "url"
|
|
315
|
+
cmd = ["gh", "repo", "list", user, "--limit", str(limit), "--json", f"name,{field}"]
|
|
316
|
+
if source:
|
|
317
|
+
cmd.append("--source")
|
|
318
|
+
if no_archived:
|
|
319
|
+
cmd.append("--no-archived")
|
|
320
|
+
out = subprocess.run(cmd, capture_output=True, text=True, check=True).stdout
|
|
321
|
+
return sorted((r["name"], r[field]) for r in json.loads(out))
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def write(projects: dict[str, str] | dict[str, dict[str, str]], output: Path) -> None:
|
|
325
|
+
"""Write *projects* as JSON to *output*, or stdout if it is ``-``."""
|
|
326
|
+
text = json.dumps(projects, indent=2) + "\n"
|
|
327
|
+
if str(output) == "-":
|
|
328
|
+
sys.stdout.write(text)
|
|
329
|
+
else:
|
|
330
|
+
output.write_text(text)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def valid_name(name: str) -> bool:
|
|
334
|
+
"""True if *name* is a single path component, so ``DEST/name`` stays in DEST."""
|
|
335
|
+
return name not in ("", ".", "..") and Path(name).name == name
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def read_pairs(path: Path) -> list[tuple[str, str]]:
|
|
339
|
+
"""Load and validate a flat or grouped projects file as ``(name, url)`` pairs.
|
|
340
|
+
|
|
341
|
+
A name may repeat across groups; `flatten` rejects that.
|
|
342
|
+
|
|
343
|
+
Raises:
|
|
344
|
+
ValueError: if the file is neither ``{name: url}`` nor
|
|
345
|
+
``{owner: {name: url}}``, or a name is not a single path component.
|
|
346
|
+
"""
|
|
347
|
+
data = json.loads(path.read_text())
|
|
348
|
+
if not isinstance(data, dict):
|
|
349
|
+
raise ValueError( # noqa: TRY004 - bad file content; callers catch ValueError
|
|
350
|
+
"expected a JSON object of {name: url} or {owner: {name: url}}"
|
|
351
|
+
)
|
|
352
|
+
if data and all(isinstance(v, dict) for v in data.values()):
|
|
353
|
+
items = [item for group in data.values() for item in group.items()]
|
|
354
|
+
else:
|
|
355
|
+
items = list(data.items())
|
|
356
|
+
for name, url in items:
|
|
357
|
+
if not isinstance(url, str) or not url:
|
|
358
|
+
raise ValueError(f"{name!r}: url must be a non-empty string")
|
|
359
|
+
if not valid_name(name):
|
|
360
|
+
raise ValueError(f"{name!r}: not a valid project directory name")
|
|
361
|
+
return items
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def clone(
|
|
365
|
+
rows: Iterable[tuple[str | None, str, str]], dest: Path, by_owner: bool = False
|
|
366
|
+
) -> list[str]:
|
|
367
|
+
"""Clone each ``(owner, name, url)`` into ``dest/<name>`` unless it exists.
|
|
368
|
+
|
|
369
|
+
With *by_owner*, clone into ``dest/<owner>/<name>``; a row without an
|
|
370
|
+
owner then fails. Return the failed targets.
|
|
371
|
+
"""
|
|
372
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
373
|
+
failed = []
|
|
374
|
+
for owner, name, url in rows:
|
|
375
|
+
if by_owner and owner is None:
|
|
376
|
+
print(f"no owner in url, skipping: {name} ({url})", file=sys.stderr)
|
|
377
|
+
failed.append(name)
|
|
378
|
+
continue
|
|
379
|
+
parts = [owner, name] if by_owner and owner else [name]
|
|
380
|
+
rel = Path(*parts)
|
|
381
|
+
# The database can be edited outside repodb, and an owner parsed from
|
|
382
|
+
# a url can be "..", so check every component stays inside DEST.
|
|
383
|
+
if not all(map(valid_name, parts)):
|
|
384
|
+
print(f"invalid name, skipping: {str(rel)!r}", file=sys.stderr)
|
|
385
|
+
failed.append(str(rel))
|
|
386
|
+
continue
|
|
387
|
+
target = dest / rel
|
|
388
|
+
if target.exists():
|
|
389
|
+
print(f"exists, skipping: {rel}")
|
|
390
|
+
continue
|
|
391
|
+
print(f"cloning: {rel} <- {url}")
|
|
392
|
+
# "--" stops a url beginning with "-" being read as a git option.
|
|
393
|
+
if subprocess.run(
|
|
394
|
+
["git", "clone", "--", url, str(target)], check=False
|
|
395
|
+
).returncode:
|
|
396
|
+
failed.append(str(rel))
|
|
397
|
+
return failed
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def report(failed: list[str]) -> int:
|
|
401
|
+
"""Print failed clone names to stderr; return the exit status."""
|
|
402
|
+
if failed:
|
|
403
|
+
print(f"failed: {', '.join(failed)}", file=sys.stderr)
|
|
404
|
+
return 1
|
|
405
|
+
return 0
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
# Subcommand handlers: each takes the parsed args and the parser (for
|
|
409
|
+
# parser.error) and returns the exit status.
|
|
410
|
+
|
|
411
|
+
Handler = Callable[[argparse.Namespace, argparse.ArgumentParser], int]
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def open_db(args: argparse.Namespace) -> GitRepoDB:
|
|
415
|
+
"""Open ``args.db``, exiting with status 1 if its schema is unsupported."""
|
|
416
|
+
try:
|
|
417
|
+
return GitRepoDB(args.db)
|
|
418
|
+
except ValueError as e:
|
|
419
|
+
raise SystemExit(str(e))
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def cmd_scan(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
423
|
+
if not args.directory.is_dir():
|
|
424
|
+
parser.error(f"not a directory: {args.directory}")
|
|
425
|
+
with open_db(args) as db:
|
|
426
|
+
db.add(scan(args.directory))
|
|
427
|
+
return 0
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def cmd_github(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
431
|
+
if shutil.which("gh") is None:
|
|
432
|
+
parser.error("github requires the gh CLI: https://cli.github.com")
|
|
433
|
+
try:
|
|
434
|
+
projects = github(
|
|
435
|
+
args.user, args.limit, args.ssh, args.source, args.no_archived
|
|
436
|
+
)
|
|
437
|
+
except subprocess.CalledProcessError as e:
|
|
438
|
+
print(f"gh failed: {e.stderr.strip()}", file=sys.stderr)
|
|
439
|
+
return 1
|
|
440
|
+
with open_db(args) as db:
|
|
441
|
+
db.add(projects)
|
|
442
|
+
return 0
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def cmd_import(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
446
|
+
with open_db(args) as db:
|
|
447
|
+
try:
|
|
448
|
+
db.add(read_pairs(args.json))
|
|
449
|
+
except (OSError, ValueError) as e: # json.JSONDecodeError is a ValueError
|
|
450
|
+
parser.error(f"{args.json}: {e}")
|
|
451
|
+
return 0
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def cmd_export(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
455
|
+
with open_db(args) as db:
|
|
456
|
+
if args.group:
|
|
457
|
+
write(db.by_owner(args.owner), args.output)
|
|
458
|
+
return 0
|
|
459
|
+
try:
|
|
460
|
+
projects = flatten((n, u) for _, n, u in db.rows(args.owner))
|
|
461
|
+
except ValueError as e:
|
|
462
|
+
parser.error(str(e))
|
|
463
|
+
write(projects, args.output)
|
|
464
|
+
return 0
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def cmd_list(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
468
|
+
with open_db(args) as db:
|
|
469
|
+
rows = db.rows(args.owner)
|
|
470
|
+
groups = db.by_owner(args.owner) if args.group else {}
|
|
471
|
+
if not rows:
|
|
472
|
+
print("no projects; run 'repodb scan' or 'repodb github USER'", file=sys.stderr)
|
|
473
|
+
return 1
|
|
474
|
+
if args.group:
|
|
475
|
+
for owner, projects in groups.items():
|
|
476
|
+
print(owner)
|
|
477
|
+
for name, url in projects.items():
|
|
478
|
+
print(f" {url if args.urls else name}")
|
|
479
|
+
else:
|
|
480
|
+
print("\n".join(url if args.urls else name for _, name, url in rows))
|
|
481
|
+
return 0
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def cmd_remove(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
485
|
+
# One flag must not delete many rows, so --owner needs --all as well.
|
|
486
|
+
if bool(args.specs) == bool(args.owner):
|
|
487
|
+
parser.error("give SPEC ... or --owner OWNER --all, not both or neither")
|
|
488
|
+
if bool(args.owner) != args.all:
|
|
489
|
+
parser.error("--owner and --all go together")
|
|
490
|
+
with open_db(args) as db:
|
|
491
|
+
try:
|
|
492
|
+
removed = (
|
|
493
|
+
db.remove_owner(args.owner) if args.owner else db.remove(args.specs)
|
|
494
|
+
)
|
|
495
|
+
except ValueError as e:
|
|
496
|
+
print(f"{e}; nothing removed", file=sys.stderr)
|
|
497
|
+
return 1
|
|
498
|
+
for o, n, u in removed:
|
|
499
|
+
print(f"removed: {o}/{n} <- {u}")
|
|
500
|
+
return 0
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def cmd_info(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
504
|
+
if not args.db.is_file():
|
|
505
|
+
print(f"no database at {args.db}", file=sys.stderr)
|
|
506
|
+
return 1
|
|
507
|
+
try:
|
|
508
|
+
print("\n".join(info(args.db)))
|
|
509
|
+
except sqlite3.Error as e:
|
|
510
|
+
print(f"{args.db}: {e}", file=sys.stderr)
|
|
511
|
+
return 1
|
|
512
|
+
return 0
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def cmd_clone(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
|
|
516
|
+
if args.json:
|
|
517
|
+
if args.owner:
|
|
518
|
+
parser.error("--owner filters the database; it does not apply with --json")
|
|
519
|
+
try:
|
|
520
|
+
pairs = read_pairs(args.json)
|
|
521
|
+
except (OSError, ValueError) as e:
|
|
522
|
+
parser.error(f"{args.json}: {e}")
|
|
523
|
+
rows: list[tuple[str | None, str, str]] = [
|
|
524
|
+
(owner_of(u), n, u) for n, u in pairs
|
|
525
|
+
]
|
|
526
|
+
else:
|
|
527
|
+
with open_db(args) as db:
|
|
528
|
+
rows = list(db.rows(args.owner))
|
|
529
|
+
if not args.group:
|
|
530
|
+
try:
|
|
531
|
+
flatten((n, u) for _, n, u in rows)
|
|
532
|
+
except ValueError as e:
|
|
533
|
+
parser.error(str(e))
|
|
534
|
+
return report(clone(rows, args.dest, args.group))
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
538
|
+
parser = argparse.ArgumentParser(
|
|
539
|
+
prog="repodb",
|
|
540
|
+
description="Keep a database of git project URLs, and clone from it.",
|
|
541
|
+
)
|
|
542
|
+
parser.add_argument(
|
|
543
|
+
"--db", type=Path, default=DB_PATH, help=f"database file (default: {DB_PATH})"
|
|
544
|
+
)
|
|
545
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
546
|
+
|
|
547
|
+
def add(name: str, handler: Handler, help: str) -> argparse.ArgumentParser:
|
|
548
|
+
p = sub.add_parser(name, help=help)
|
|
549
|
+
p.set_defaults(handler=handler)
|
|
550
|
+
return p
|
|
551
|
+
|
|
552
|
+
p = add("scan", cmd_scan, "add origin URLs of projects in DIR")
|
|
553
|
+
p.add_argument(
|
|
554
|
+
"directory",
|
|
555
|
+
type=Path,
|
|
556
|
+
nargs="?",
|
|
557
|
+
default=SRC_DIR,
|
|
558
|
+
metavar="DIR",
|
|
559
|
+
help=f"default: {SRC_DIR}",
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
p = add("github", cmd_github, "add USER's GitHub repos (needs gh)")
|
|
563
|
+
p.add_argument("user", metavar="USER", help="GitHub user or organization")
|
|
564
|
+
p.add_argument(
|
|
565
|
+
"-L",
|
|
566
|
+
"--limit",
|
|
567
|
+
type=int,
|
|
568
|
+
default=10000,
|
|
569
|
+
help="maximum repos to list (default: 10000)",
|
|
570
|
+
)
|
|
571
|
+
p.add_argument("--ssh", action="store_true", help="use SSH clone URLs, not HTTPS")
|
|
572
|
+
p.add_argument("--source", action="store_true", help="omit forks")
|
|
573
|
+
p.add_argument("--no-archived", action="store_true", help="omit archived repos")
|
|
574
|
+
|
|
575
|
+
p = add("import", cmd_import, "add projects from a JSON file, flat or grouped")
|
|
576
|
+
p.add_argument("json", type=Path, metavar="JSON")
|
|
577
|
+
|
|
578
|
+
p = add("export", cmd_export, "write projects as {name: url} JSON")
|
|
579
|
+
p.add_argument(
|
|
580
|
+
"-o",
|
|
581
|
+
"--output",
|
|
582
|
+
type=Path,
|
|
583
|
+
default=Path("projects.json"),
|
|
584
|
+
help="output file, or - for stdout (default: projects.json)",
|
|
585
|
+
)
|
|
586
|
+
p.add_argument(
|
|
587
|
+
"-g", "--group", action="store_true", help="write {owner: {name: url}}"
|
|
588
|
+
)
|
|
589
|
+
p.add_argument("--owner", help="only this owner's projects")
|
|
590
|
+
|
|
591
|
+
p = add("list", cmd_list, "print project names")
|
|
592
|
+
p.add_argument("-u", "--urls", action="store_true", help="print URLs instead")
|
|
593
|
+
p.add_argument("-g", "--group", action="store_true", help="group by owner")
|
|
594
|
+
p.add_argument("--owner", help="only this owner's projects")
|
|
595
|
+
|
|
596
|
+
p = add(
|
|
597
|
+
"remove",
|
|
598
|
+
cmd_remove,
|
|
599
|
+
"delete projects from the database; cloned directories are kept",
|
|
600
|
+
)
|
|
601
|
+
p.add_argument(
|
|
602
|
+
"specs",
|
|
603
|
+
nargs="*",
|
|
604
|
+
metavar="SPEC",
|
|
605
|
+
help="OWNER/NAME, or NAME if only one owner has it",
|
|
606
|
+
)
|
|
607
|
+
p.add_argument("--owner", help="remove every project of this owner; needs --all")
|
|
608
|
+
p.add_argument("--all", action="store_true", help="confirm --owner")
|
|
609
|
+
|
|
610
|
+
add("info", cmd_info, "describe the database; read-only")
|
|
611
|
+
|
|
612
|
+
p = add("clone", cmd_clone, "clone projects into DEST")
|
|
613
|
+
p.add_argument("dest", type=Path, metavar="DEST")
|
|
614
|
+
p.add_argument(
|
|
615
|
+
"--json",
|
|
616
|
+
type=Path,
|
|
617
|
+
metavar="FILE",
|
|
618
|
+
help="clone from this JSON file instead of the database",
|
|
619
|
+
)
|
|
620
|
+
p.add_argument(
|
|
621
|
+
"-g", "--group", action="store_true", help="clone into DEST/OWNER/NAME"
|
|
622
|
+
)
|
|
623
|
+
p.add_argument("--owner", help="only this owner's projects")
|
|
624
|
+
|
|
625
|
+
if Path(sys.argv[0]).stem == "listrepos" and argv is None:
|
|
626
|
+
argv = ["list"]
|
|
627
|
+
args = parser.parse_args(argv)
|
|
628
|
+
handler: Handler = args.handler
|
|
629
|
+
return handler(args, parser)
|
repodb/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: repodb
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Keep a SQLite database of git project clone URLs, and clone from it
|
|
5
|
+
Keywords: git,github,sqlite,clone,cli
|
|
6
|
+
Author: Shakeeb Alireza
|
|
7
|
+
Author-email: Shakeeb Alireza <shakfu@users.noreply.github.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# repodb
|
|
23
|
+
|
|
24
|
+
Keep a SQLite database of git project clone URLs, and clone from it.
|
|
25
|
+
|
|
26
|
+
Each project is stored as `(owner, name, url)`, keyed by `(owner, name)`. The owner is the first path component after the host (`alice` in `github.com/alice/r` or `git@gitlab.com:alice/r`), for any host.
|
|
27
|
+
|
|
28
|
+
## Install
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv tool install git+https://github.com/shakfu/repodb
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
This installs `repodb`, and `listrepos` as an alias for `repodb list`.
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
repodb scan ~/src # add origin URLs of local projects
|
|
40
|
+
repodb github USER [--source] [--ssh] # add USER's GitHub repos (needs gh)
|
|
41
|
+
repodb list [-g] [-u] [--owner USER] # names, grouped by owner, or URLs
|
|
42
|
+
repodb export [-g] [--owner USER] -o projects.json
|
|
43
|
+
repodb import projects.json
|
|
44
|
+
repodb clone DEST [-g] [--owner USER | --json projects.json]
|
|
45
|
+
repodb remove OWNER/NAME [NAME ...] | --owner USER --all
|
|
46
|
+
repodb info # format, counts, hosts; read-only
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
- The JSON form is `{name: url}`, or with `-g` `{owner: {name: url}}`. Both round-trip byte for byte.
|
|
50
|
+
|
|
51
|
+
- `clone` writes `DEST/<name>`, or with `-g` `DEST/<owner>/<name>`, and skips targets that exist.
|
|
52
|
+
|
|
53
|
+
- The flat forms refuse a name held by two owners, ignoring case; use `-g`.
|
|
54
|
+
|
|
55
|
+
- The database defaults to `~/.local/share/repodb/repos.sqlite`; override with `--db`.
|
|
56
|
+
|
|
57
|
+
## Development
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
make qa # ruff lint and format check, mypy, pytest
|
|
61
|
+
```
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
repodb/__init__.py,sha256=1qFUB1NWL3GqaQued-IhlgSYVmba5A9_vA2S-Ch93gc,269
|
|
2
|
+
repodb/core.py,sha256=yOU1bR8arTfHrlUCLBIRNsVn-xkFmo5phS1lf1WTNAQ,23070
|
|
3
|
+
repodb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
repodb-0.1.0.dist-info/WHEEL,sha256=e4_1dyBeezi8ZjfxrZ3bnVOxFDa3ksqVqH0jTHkUZ3k,81
|
|
5
|
+
repodb-0.1.0.dist-info/entry_points.txt,sha256=PE2Pc9n7hvmyWyJkaSUzkQIqM9L8hwaYCL9k7T4g9Mo,74
|
|
6
|
+
repodb-0.1.0.dist-info/METADATA,sha256=ZpwXHAQQ-rFxt9mdWjaW1npx65zJFh82wUB77soC7rc,2263
|
|
7
|
+
repodb-0.1.0.dist-info/RECORD,,
|