ghir 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghir/__init__.py +48 -0
- ghir/__main__.py +76 -0
- ghir/cache.py +70 -0
- ghir/config.py +69 -0
- ghir/extract.py +63 -0
- ghir/fetch.py +240 -0
- ghir/local.py +59 -0
- ghir/search.py +202 -0
- ghir-0.0.2.dist-info/METADATA +83 -0
- ghir-0.0.2.dist-info/RECORD +13 -0
- ghir-0.0.2.dist-info/WHEEL +4 -0
- ghir-0.0.2.dist-info/entry_points.txt +2 -0
- ghir-0.0.2.dist-info/licenses/LICENSE +21 -0
ghir/__init__.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Search your GitHub repos, cloned or not, as an ``ir`` corpus.
|
|
2
|
+
|
|
3
|
+
Four seams, each a keyword argument with a working default:
|
|
4
|
+
|
|
5
|
+
- **locations** (:func:`refresh`): owner logins, ``"@me"`` (you and your orgs),
|
|
6
|
+
or ``"search:<github query>"``.
|
|
7
|
+
- **extraction** (``extract=``): what text a repo contributes; default
|
|
8
|
+
description + topics + README head (:mod:`ghir.extract`).
|
|
9
|
+
- **ranking** (``rank=``): ``"ir"`` (default, hybrid dense + BM25) or ``"grep"``.
|
|
10
|
+
- **cache** (``store=``): a ``MutableMapping``; :func:`refresh` merges into it,
|
|
11
|
+
:func:`search` only reads it (offline).
|
|
12
|
+
|
|
13
|
+
>>> import ghir # doctest: +SKIP
|
|
14
|
+
>>> ghir.refresh() # doctest: +SKIP
|
|
15
|
+
>>> for hit in ghir.search("flashcards spaced repetition"): # doctest: +SKIP
|
|
16
|
+
... print(hit.id, hit.local_path)
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from .cache import refresh, repo_store
|
|
20
|
+
from .config import settings
|
|
21
|
+
from .extract import extractors
|
|
22
|
+
from .fetch import fetch_repos
|
|
23
|
+
from .local import local_clones
|
|
24
|
+
from .search import (
|
|
25
|
+
RepoHit,
|
|
26
|
+
grep_rank,
|
|
27
|
+
ir_rank,
|
|
28
|
+
rankers,
|
|
29
|
+
records,
|
|
30
|
+
register_corpus,
|
|
31
|
+
search,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"RepoHit",
|
|
36
|
+
"extractors",
|
|
37
|
+
"fetch_repos",
|
|
38
|
+
"grep_rank",
|
|
39
|
+
"ir_rank",
|
|
40
|
+
"local_clones",
|
|
41
|
+
"rankers",
|
|
42
|
+
"records",
|
|
43
|
+
"refresh",
|
|
44
|
+
"register_corpus",
|
|
45
|
+
"repo_store",
|
|
46
|
+
"search",
|
|
47
|
+
"settings",
|
|
48
|
+
]
|
ghir/__main__.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Command line: ``ghir refresh`` (online, merges into the cache) and ``ghir search``.
|
|
2
|
+
|
|
3
|
+
.. code-block:: bash
|
|
4
|
+
|
|
5
|
+
ghir refresh # @me + config locations
|
|
6
|
+
ghir refresh i2mint "search:user:thorwhalen topic:education"
|
|
7
|
+
ghir search "flashcards spaced repetition" # offline
|
|
8
|
+
ghir search "audio tagging" --rank grep --local-only
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from .cache import refresh
|
|
18
|
+
from .search import DFLT_K, rankers, register_corpus, search
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _print_hits(hits, *, as_json: bool, local_only: bool):
|
|
22
|
+
hits = [h for h in hits if h.local_path or not local_only]
|
|
23
|
+
if as_json:
|
|
24
|
+
print(json.dumps([h.to_dict() for h in hits], indent=2))
|
|
25
|
+
return
|
|
26
|
+
for h in hits:
|
|
27
|
+
where = h.local_path or "(not cloned)"
|
|
28
|
+
flags = "".join(
|
|
29
|
+
f" [{f}]"
|
|
30
|
+
for f, on in (("fork", h.is_fork), ("archived", h.is_archived))
|
|
31
|
+
if on
|
|
32
|
+
)
|
|
33
|
+
print(f"{h.score:6.3f} {h.id}{flags} {where}")
|
|
34
|
+
if h.description:
|
|
35
|
+
print(f" {h.description}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main(argv=None):
|
|
39
|
+
p = argparse.ArgumentParser(prog="ghir", description=__doc__.splitlines()[0])
|
|
40
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
41
|
+
|
|
42
|
+
r = sub.add_parser("refresh", help="fetch repos from GitHub, merge into the cache")
|
|
43
|
+
r.add_argument("locations", nargs="*", help="owners, @me, or search:<query>")
|
|
44
|
+
r.add_argument("--no-register", action="store_true", help="skip ir registration")
|
|
45
|
+
|
|
46
|
+
s = sub.add_parser("search", help="rank cached repos for a query (offline)")
|
|
47
|
+
s.add_argument("query")
|
|
48
|
+
s.add_argument("-k", type=int, default=DFLT_K)
|
|
49
|
+
s.add_argument("--rank", choices=sorted(rankers), default="ir")
|
|
50
|
+
s.add_argument("--extract", default=None, help="extractor name")
|
|
51
|
+
s.add_argument("--no-forks", action="store_true")
|
|
52
|
+
s.add_argument("--no-archived", action="store_true")
|
|
53
|
+
s.add_argument("--local-only", action="store_true", help="only cloned repos")
|
|
54
|
+
s.add_argument("--json", action="store_true")
|
|
55
|
+
|
|
56
|
+
a = p.parse_args(argv)
|
|
57
|
+
if a.cmd == "refresh":
|
|
58
|
+
counts = refresh(a.locations or None)
|
|
59
|
+
if not a.no_register:
|
|
60
|
+
register_corpus()
|
|
61
|
+
print(json.dumps(counts))
|
|
62
|
+
return 0
|
|
63
|
+
hits = search(
|
|
64
|
+
a.query,
|
|
65
|
+
k=a.k,
|
|
66
|
+
rank=a.rank,
|
|
67
|
+
extract=a.extract,
|
|
68
|
+
include_forks=not a.no_forks,
|
|
69
|
+
include_archived=not a.no_archived,
|
|
70
|
+
)
|
|
71
|
+
_print_hits(hits, as_json=a.json, local_only=a.local_only)
|
|
72
|
+
return 0
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
sys.exit(main())
|
ghir/cache.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The repo-record cache: a ``MutableMapping`` from ``owner/name`` to a record.
|
|
2
|
+
|
|
3
|
+
Searching never goes online; it reads this cache. :func:`refresh` is the only
|
|
4
|
+
thing that fetches, and it *merges*: fetched records overwrite their own keys,
|
|
5
|
+
records not seen this time are kept (a repo outside today's locations, or a
|
|
6
|
+
fetch that failed half-way, does not erase what was known). Each record carries
|
|
7
|
+
``fetched_at`` so staleness is visible.
|
|
8
|
+
|
|
9
|
+
The store is a seam (``store=``): the default is one JSON file per repo under
|
|
10
|
+
:data:`ghir.config.DFLT_CACHE_DIR` (a ``dol`` store, so an S3 or database
|
|
11
|
+
mapping drops in unchanged).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Iterable, MutableMapping
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from dol import JsonFiles, mk_dirs_if_missing, wrap_kvs
|
|
21
|
+
|
|
22
|
+
from .config import settings as _settings
|
|
23
|
+
from .fetch import fetch_repos
|
|
24
|
+
|
|
25
|
+
#: Repo ids contain a "/", which becomes a directory level (owner/name.json).
|
|
26
|
+
_EXT = ".json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def repo_store(cache_dir: str | Path | None = None) -> MutableMapping:
|
|
30
|
+
"""The default cache: ``{"owner/name": record}`` over JSON files."""
|
|
31
|
+
root = Path(cache_dir or _settings()["cache_dir"])
|
|
32
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
return wrap_kvs(
|
|
34
|
+
mk_dirs_if_missing(JsonFiles(str(root))),
|
|
35
|
+
id_of_key=lambda k: k + _EXT,
|
|
36
|
+
key_of_id=lambda _id: _id[: -len(_EXT)],
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def refresh(
|
|
41
|
+
locations: Iterable[str] | None = None,
|
|
42
|
+
*,
|
|
43
|
+
store: MutableMapping | None = None,
|
|
44
|
+
exclude: Iterable[str] | None = None,
|
|
45
|
+
fetch=fetch_repos,
|
|
46
|
+
**fetch_kwargs,
|
|
47
|
+
) -> dict:
|
|
48
|
+
"""Fetch the repos of ``locations`` and merge them into the cache.
|
|
49
|
+
|
|
50
|
+
Returns counts: ``{"fetched", "added", "updated", "kept"}`` where ``kept`` is
|
|
51
|
+
the number of cached records this fetch did not see (and left alone), plus
|
|
52
|
+
``"failed"``: ``{location: error}`` for locations that could not be fetched
|
|
53
|
+
(their cached records are kept).
|
|
54
|
+
"""
|
|
55
|
+
s = _settings(locations=locations, exclude=exclude)
|
|
56
|
+
store = repo_store(s["cache_dir"]) if store is None else store
|
|
57
|
+
before = set(store)
|
|
58
|
+
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
59
|
+
seen, failed = set(), {}
|
|
60
|
+
fetch_kwargs.setdefault("on_error", lambda loc, e: failed.update({loc: str(e)}))
|
|
61
|
+
for rec in fetch(s["locations"], exclude=s["exclude"], **fetch_kwargs):
|
|
62
|
+
store[rec["id"]] = {**rec, "fetched_at": now}
|
|
63
|
+
seen.add(rec["id"])
|
|
64
|
+
return {
|
|
65
|
+
"fetched": len(seen),
|
|
66
|
+
"added": len(seen - before),
|
|
67
|
+
"updated": len(seen & before),
|
|
68
|
+
"kept": len(before - seen),
|
|
69
|
+
"failed": failed,
|
|
70
|
+
}
|
ghir/config.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Settings for ghir: where the cache lives and which repos are in scope.
|
|
2
|
+
|
|
3
|
+
All settings have working defaults and can be overridden, in increasing order of
|
|
4
|
+
precedence, by a JSON file (``~/.config/ghir/config.json``, or ``$GHIR_CONFIG``)
|
|
5
|
+
and by keyword arguments to the functions that read them. The file is the place
|
|
6
|
+
for machine-specific choices (extra owners, owners or repos never to fetch):
|
|
7
|
+
|
|
8
|
+
.. code-block:: json
|
|
9
|
+
|
|
10
|
+
{"locations": ["@me", "otosense"],
|
|
11
|
+
"exclude": ["SomeClientOrg/*", "myorg/private-notes"]}
|
|
12
|
+
|
|
13
|
+
``"@me"`` expands to the authenticated GitHub user plus every organization they
|
|
14
|
+
belong to (see :func:`ghir.fetch.my_owners`).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
from collections.abc import Mapping
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _xdg(var: str, fallback: str) -> Path:
|
|
27
|
+
return Path(os.environ.get(var) or Path.home() / fallback)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
#: Where fetched repo records are cached (one JSON file per ``owner/name``).
|
|
31
|
+
DFLT_CACHE_DIR = _xdg("XDG_DATA_HOME", ".local/share") / "ghir" / "repos"
|
|
32
|
+
|
|
33
|
+
#: The optional user settings file.
|
|
34
|
+
DFLT_CONFIG_PATH = _xdg("XDG_CONFIG_HOME", ".config") / "ghir" / "config.json"
|
|
35
|
+
|
|
36
|
+
#: The ir corpus name the default records are registered under.
|
|
37
|
+
DFLT_CORPUS_NAME = "gh_repos"
|
|
38
|
+
|
|
39
|
+
DFLT_SETTINGS: Mapping[str, Any] = {
|
|
40
|
+
# Owners (user or org logins) and/or "search:<github search query>" specs.
|
|
41
|
+
"locations": ["@me"],
|
|
42
|
+
# fnmatch patterns over "owner/name"; matching repos are never fetched.
|
|
43
|
+
"exclude": [],
|
|
44
|
+
# Name of the extractor (see ghir.extract.extractors) turning a repo into text.
|
|
45
|
+
"extract": "description_and_readme",
|
|
46
|
+
# Root directories scanned (a few levels deep) for local clones.
|
|
47
|
+
"local_roots": [os.environ.get("PP") or str(Path.home())],
|
|
48
|
+
"cache_dir": str(DFLT_CACHE_DIR),
|
|
49
|
+
"corpus_name": DFLT_CORPUS_NAME,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def settings(path: str | Path | None = None, **overrides) -> dict:
|
|
54
|
+
"""The effective settings: defaults, then the config file, then ``overrides``.
|
|
55
|
+
|
|
56
|
+
>>> s = settings(path="/nonexistent/ghir.json", exclude=["acme/*"])
|
|
57
|
+
>>> s["exclude"], s["locations"]
|
|
58
|
+
(['acme/*'], ['@me'])
|
|
59
|
+
"""
|
|
60
|
+
path = Path(path or os.environ.get("GHIR_CONFIG") or DFLT_CONFIG_PATH)
|
|
61
|
+
from_file = json.loads(path.read_text()) if path.is_file() else {}
|
|
62
|
+
unknown = set(from_file) - set(DFLT_SETTINGS)
|
|
63
|
+
if unknown:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Unknown ghir setting(s) {sorted(unknown)} in {path}; "
|
|
66
|
+
f"known: {sorted(DFLT_SETTINGS)}"
|
|
67
|
+
)
|
|
68
|
+
given = {k: v for k, v in overrides.items() if v is not None}
|
|
69
|
+
return {**DFLT_SETTINGS, **from_file, **given}
|
ghir/extract.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Extraction: what text each cached repo contributes to the search.
|
|
2
|
+
|
|
3
|
+
An extractor is a function ``record -> str``. :data:`extractors` names the
|
|
4
|
+
built-in ones; any callable works wherever an extractor is accepted. The text
|
|
5
|
+
returned is what gets indexed and ranked, so it decides what a query can match.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable, Mapping
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
Extractor = Callable[[Mapping[str, Any]], str]
|
|
14
|
+
|
|
15
|
+
#: README characters kept by the default extractor (READMEs can be huge, and
|
|
16
|
+
#: their opening says what the repo is).
|
|
17
|
+
DFLT_README_CHARS = 6000
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _head(rec: Mapping[str, Any]) -> list[str]:
|
|
21
|
+
topics = " ".join(rec.get("topics") or ())
|
|
22
|
+
return [
|
|
23
|
+
rec.get("name", ""),
|
|
24
|
+
rec.get("description", ""),
|
|
25
|
+
f"topics: {topics}" if topics else "",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def description(rec: Mapping[str, Any]) -> str:
|
|
30
|
+
"""Name, description and topics.
|
|
31
|
+
|
|
32
|
+
>>> description({"name": "learny", "description": "Learner models",
|
|
33
|
+
... "topics": ["education"]})
|
|
34
|
+
'learny\\nLearner models\\ntopics: education'
|
|
35
|
+
"""
|
|
36
|
+
return "\n".join(p for p in _head(rec) if p)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def description_and_readme(
|
|
40
|
+
rec: Mapping[str, Any], *, readme_chars: int = DFLT_README_CHARS
|
|
41
|
+
) -> str:
|
|
42
|
+
"""Name, description, topics, and the start of the README (the default)."""
|
|
43
|
+
readme = (rec.get("readme") or "")[:readme_chars]
|
|
44
|
+
return "\n".join(p for p in [*_head(rec), readme] if p)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
extractors: dict[str, Extractor] = {
|
|
48
|
+
"description": description,
|
|
49
|
+
"description_and_readme": description_and_readme,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_extractor(extract: str | Extractor) -> Extractor:
|
|
54
|
+
"""An extractor from its name in :data:`extractors`, or a callable as is."""
|
|
55
|
+
if callable(extract):
|
|
56
|
+
return extract
|
|
57
|
+
try:
|
|
58
|
+
return extractors[extract]
|
|
59
|
+
except KeyError:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"Unknown extractor {extract!r}; use a callable or one of "
|
|
62
|
+
f"{sorted(extractors)}"
|
|
63
|
+
) from None
|
ghir/fetch.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""Fetching repo records from GitHub, through the ``gh`` CLI's GraphQL access.
|
|
2
|
+
|
|
3
|
+
One GraphQL page returns up to :data:`DFLT_PAGE_SIZE` repos *with* their README
|
|
4
|
+
text, so a whole organization costs a handful of calls rather than one per repo.
|
|
5
|
+
|
|
6
|
+
The transport is a seam: every function takes ``graphql=``, a callable
|
|
7
|
+
``(query, variables) -> dict``. The default shells out to ``gh api graphql``
|
|
8
|
+
(so authentication is whatever ``gh auth login`` set up); tests pass a fake.
|
|
9
|
+
|
|
10
|
+
A *location* is one of:
|
|
11
|
+
|
|
12
|
+
- an owner login (``"i2mint"``): every repo of that user or organization;
|
|
13
|
+
- ``"@me"``: the authenticated user and every organization they belong to;
|
|
14
|
+
- ``"search:<query>"``: the repos a GitHub search returns
|
|
15
|
+
(``"search:user:thorwhalen topic:education"``).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import shutil
|
|
22
|
+
import subprocess
|
|
23
|
+
from collections.abc import Callable, Iterable, Iterator, Mapping
|
|
24
|
+
from fnmatch import fnmatch
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
GraphQL = Callable[[str, Mapping[str, Any]], dict]
|
|
28
|
+
|
|
29
|
+
DFLT_PAGE_SIZE = 25
|
|
30
|
+
#: On a gateway error (README blobs make pages slow), the page size is halved
|
|
31
|
+
#: down to this before the error is let through.
|
|
32
|
+
MIN_PAGE_SIZE = 3
|
|
33
|
+
_TRANSIENT = ("HTTP 502", "HTTP 503", "HTTP 504", "timeout", "Timeout")
|
|
34
|
+
#: README file names tried, in order, at the default branch's root.
|
|
35
|
+
README_NAMES = ("README.md", "readme.md", "README.rst", "README", "README.txt")
|
|
36
|
+
SEARCH_PREFIX = "search:"
|
|
37
|
+
ME = "@me"
|
|
38
|
+
|
|
39
|
+
_README_FIELDS = "\n".join(
|
|
40
|
+
f'readme{i}: object(expression: "HEAD:{n}") {{ ... on Blob {{ text }} }}'
|
|
41
|
+
for i, n in enumerate(README_NAMES)
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
REPO_FIELDS = f"""
|
|
45
|
+
nameWithOwner name description url homepageUrl isFork isArchived isPrivate
|
|
46
|
+
pushedAt updatedAt stargazerCount
|
|
47
|
+
primaryLanguage {{ name }}
|
|
48
|
+
repositoryTopics(first: 20) {{ nodes {{ topic {{ name }} }} }}
|
|
49
|
+
{_README_FIELDS}
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
OWNER_QUERY = f"""
|
|
53
|
+
query($owner: String!, $after: String, $n: Int!) {{
|
|
54
|
+
repositoryOwner(login: $owner) {{
|
|
55
|
+
repositories(first: $n, after: $after, ownerAffiliations: OWNER) {{
|
|
56
|
+
pageInfo {{ hasNextPage endCursor }}
|
|
57
|
+
nodes {{ {REPO_FIELDS} }}
|
|
58
|
+
}}
|
|
59
|
+
}}
|
|
60
|
+
}}
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
SEARCH_QUERY = f"""
|
|
64
|
+
query($q: String!, $after: String, $n: Int!) {{
|
|
65
|
+
search(type: REPOSITORY, query: $q, first: $n, after: $after) {{
|
|
66
|
+
pageInfo {{ hasNextPage endCursor }}
|
|
67
|
+
nodes {{ ... on Repository {{ {REPO_FIELDS} }} }}
|
|
68
|
+
}}
|
|
69
|
+
}}
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def gh_graphql(query: str, variables: Mapping[str, Any]) -> dict:
|
|
74
|
+
"""Run a GraphQL query through ``gh api graphql`` and return its ``data``."""
|
|
75
|
+
if not shutil.which("gh"):
|
|
76
|
+
raise RuntimeError(
|
|
77
|
+
"ghir fetches through the GitHub CLI, which is not on PATH. "
|
|
78
|
+
"Install it (https://cli.github.com) and run `gh auth login`."
|
|
79
|
+
)
|
|
80
|
+
args = ["gh", "api", "graphql", "-f", f"query={query}"]
|
|
81
|
+
for k, v in variables.items():
|
|
82
|
+
if v is not None:
|
|
83
|
+
args += ["-F" if isinstance(v, int) else "-f", f"{k}={v}"]
|
|
84
|
+
proc = subprocess.run(args, capture_output=True, text=True)
|
|
85
|
+
if proc.returncode != 0:
|
|
86
|
+
raise RuntimeError(f"gh api graphql failed: {proc.stderr.strip()}")
|
|
87
|
+
payload = json.loads(proc.stdout)
|
|
88
|
+
if payload.get("errors") and not payload.get("data"):
|
|
89
|
+
raise RuntimeError(f"GitHub GraphQL errors: {payload['errors']}")
|
|
90
|
+
return payload["data"]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def gh_api(path: str) -> Any:
|
|
94
|
+
"""``gh api <path>`` (REST), JSON-decoded."""
|
|
95
|
+
proc = subprocess.run(["gh", "api", path], capture_output=True, text=True)
|
|
96
|
+
if proc.returncode != 0:
|
|
97
|
+
raise RuntimeError(f"gh api {path} failed: {proc.stderr.strip()}")
|
|
98
|
+
return json.loads(proc.stdout)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def my_owners(*, rest: Callable[[str], Any] = gh_api) -> list[str]:
|
|
102
|
+
"""The authenticated user's login followed by their organizations' logins."""
|
|
103
|
+
return [rest("user")["login"]] + [o["login"] for o in rest("user/orgs")]
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def normalize_repo(node: Mapping[str, Any]) -> dict:
|
|
107
|
+
"""Flatten one GraphQL repository node into a JSON-friendly record.
|
|
108
|
+
|
|
109
|
+
>>> normalize_repo({"nameWithOwner": "a/b", "name": "b",
|
|
110
|
+
... "primaryLanguage": {"name": "Python"},
|
|
111
|
+
... "repositoryTopics": {"nodes": [{"topic": {"name": "x"}}]},
|
|
112
|
+
... "readme0": None, "readme1": {"text": "hi"}})["readme"]
|
|
113
|
+
'hi'
|
|
114
|
+
"""
|
|
115
|
+
readme = next(
|
|
116
|
+
(
|
|
117
|
+
node[f"readme{i}"]["text"]
|
|
118
|
+
for i in range(len(README_NAMES))
|
|
119
|
+
if (node.get(f"readme{i}") or {}).get("text")
|
|
120
|
+
),
|
|
121
|
+
"",
|
|
122
|
+
)
|
|
123
|
+
owner, _, name = node["nameWithOwner"].partition("/")
|
|
124
|
+
return {
|
|
125
|
+
"id": node["nameWithOwner"],
|
|
126
|
+
"owner": owner,
|
|
127
|
+
"name": name,
|
|
128
|
+
"description": node.get("description") or "",
|
|
129
|
+
"url": node.get("url") or f"https://github.com/{node['nameWithOwner']}",
|
|
130
|
+
"homepage": node.get("homepageUrl") or "",
|
|
131
|
+
"language": (node.get("primaryLanguage") or {}).get("name") or "",
|
|
132
|
+
"topics": [
|
|
133
|
+
t["topic"]["name"]
|
|
134
|
+
for t in (node.get("repositoryTopics") or {}).get("nodes", [])
|
|
135
|
+
],
|
|
136
|
+
"is_fork": bool(node.get("isFork")),
|
|
137
|
+
"is_archived": bool(node.get("isArchived")),
|
|
138
|
+
"is_private": bool(node.get("isPrivate")),
|
|
139
|
+
"pushed_at": node.get("pushedAt") or "",
|
|
140
|
+
"updated_at": node.get("updatedAt") or "",
|
|
141
|
+
"stars": node.get("stargazerCount") or 0,
|
|
142
|
+
"readme": readme,
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _query_page(query: str, variables: dict, graphql: GraphQL) -> dict:
|
|
147
|
+
n = variables["n"]
|
|
148
|
+
while True:
|
|
149
|
+
try:
|
|
150
|
+
return graphql(query, {**variables, "n": n})
|
|
151
|
+
except RuntimeError as e:
|
|
152
|
+
if n <= MIN_PAGE_SIZE or not any(s in str(e) for s in _TRANSIENT):
|
|
153
|
+
raise
|
|
154
|
+
n = max(MIN_PAGE_SIZE, n // 2)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _paginate(query: str, variables: dict, path: str, graphql: GraphQL):
|
|
158
|
+
after = None
|
|
159
|
+
while True:
|
|
160
|
+
data = _query_page(query, {**variables, "after": after}, graphql)
|
|
161
|
+
for key in path.split("."):
|
|
162
|
+
data = data[key] if data else None
|
|
163
|
+
if not data: # unknown owner, say
|
|
164
|
+
return
|
|
165
|
+
yield from (n for n in data["nodes"] if n and n.get("nameWithOwner"))
|
|
166
|
+
if not data["pageInfo"]["hasNextPage"]:
|
|
167
|
+
return
|
|
168
|
+
after = data["pageInfo"]["endCursor"]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def fetch_location(
|
|
172
|
+
location: str,
|
|
173
|
+
*,
|
|
174
|
+
graphql: GraphQL = gh_graphql,
|
|
175
|
+
page_size: int = DFLT_PAGE_SIZE,
|
|
176
|
+
) -> Iterator[dict]:
|
|
177
|
+
"""Yield normalized repo records for one owner login or ``search:`` spec."""
|
|
178
|
+
if location.startswith(SEARCH_PREFIX):
|
|
179
|
+
q = location[len(SEARCH_PREFIX) :].strip()
|
|
180
|
+
nodes = _paginate(SEARCH_QUERY, {"q": q, "n": page_size}, "search", graphql)
|
|
181
|
+
else:
|
|
182
|
+
nodes = _paginate(
|
|
183
|
+
OWNER_QUERY,
|
|
184
|
+
{"owner": location, "n": page_size},
|
|
185
|
+
"repositoryOwner.repositories",
|
|
186
|
+
graphql,
|
|
187
|
+
)
|
|
188
|
+
yield from map(normalize_repo, nodes)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def expand_locations(
|
|
192
|
+
locations: Iterable[str], *, owners: Callable[[], list[str]] = my_owners
|
|
193
|
+
) -> list[str]:
|
|
194
|
+
"""Replace ``"@me"`` by the user's owners; keep order, drop duplicates."""
|
|
195
|
+
out: list[str] = []
|
|
196
|
+
for loc in locations:
|
|
197
|
+
for item in owners() if loc == ME else [loc]:
|
|
198
|
+
if item not in out:
|
|
199
|
+
out.append(item)
|
|
200
|
+
return out
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def is_excluded(repo_id: str, exclude: Iterable[str]) -> bool:
|
|
204
|
+
"""Whether ``owner/name`` matches any fnmatch pattern of ``exclude``.
|
|
205
|
+
|
|
206
|
+
>>> is_excluded("Acme/x", ["acme/*"]), is_excluded("i2mint/x", ["acme/*"])
|
|
207
|
+
(True, False)
|
|
208
|
+
"""
|
|
209
|
+
rid = repo_id.lower()
|
|
210
|
+
return any(fnmatch(rid, p.lower()) for p in exclude)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def fetch_repos(
|
|
214
|
+
locations: Iterable[str],
|
|
215
|
+
*,
|
|
216
|
+
exclude: Iterable[str] = (),
|
|
217
|
+
graphql: GraphQL = gh_graphql,
|
|
218
|
+
owners: Callable[[], list[str]] = my_owners,
|
|
219
|
+
on_error: Callable[[str, Exception], None] | None = None,
|
|
220
|
+
) -> Iterator[dict]:
|
|
221
|
+
"""Yield repo records over all ``locations``, skipping excluded repos.
|
|
222
|
+
|
|
223
|
+
A location whose fetch fails is passed to ``on_error(location, error)`` and
|
|
224
|
+
skipped, or, when ``on_error`` is None, the error propagates.
|
|
225
|
+
|
|
226
|
+
An owner pattern in ``exclude`` (``"Acme/*"``) also stops that owner from
|
|
227
|
+
being queried at all, so nothing of it is fetched.
|
|
228
|
+
"""
|
|
229
|
+
exclude = list(exclude)
|
|
230
|
+
for loc in expand_locations(locations, owners=owners):
|
|
231
|
+
if not loc.startswith(SEARCH_PREFIX) and is_excluded(f"{loc}/*", exclude):
|
|
232
|
+
continue
|
|
233
|
+
try:
|
|
234
|
+
for rec in fetch_location(loc, graphql=graphql):
|
|
235
|
+
if not is_excluded(rec["id"], exclude):
|
|
236
|
+
yield rec
|
|
237
|
+
except RuntimeError as e:
|
|
238
|
+
if on_error is None:
|
|
239
|
+
raise
|
|
240
|
+
on_error(loc, e)
|
ghir/local.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Which GitHub repos are cloned on this machine, and where.
|
|
2
|
+
|
|
3
|
+
Scans root directories a few levels deep for git checkouts and reads each one's
|
|
4
|
+
``origin`` remote, giving ``{"owner/name": local_path}``. Plain file reads, no
|
|
5
|
+
``git`` subprocess, so a scan of a few hundred checkouts takes well under a second.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
from collections.abc import Iterable
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
#: How many directory levels below each root to look for checkouts.
|
|
16
|
+
DFLT_MAX_DEPTH = 3
|
|
17
|
+
_SKIP = {"node_modules", ".venv", "venv", "__pycache__", "site-packages", ".git"}
|
|
18
|
+
_GITHUB_REMOTE = re.compile(
|
|
19
|
+
r"url\s*=\s*\S*github\.com[:/]([^/\s]+)/([^/\s]+?)(?:\.git)?/?\s*$", re.M
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def origin_repo_id(config_text: str) -> str | None:
|
|
24
|
+
"""The ``owner/name`` of the first GitHub remote in a ``.git/config``.
|
|
25
|
+
|
|
26
|
+
>>> origin_repo_id('[remote "origin"]\\n\\turl = git@github.com:i2mint/ir.git\\n')
|
|
27
|
+
'i2mint/ir'
|
|
28
|
+
>>> origin_repo_id('url = https://github.com/thorwhalen/learny\\n')
|
|
29
|
+
'thorwhalen/learny'
|
|
30
|
+
"""
|
|
31
|
+
m = _GITHUB_REMOTE.search(config_text)
|
|
32
|
+
return f"{m.group(1)}/{m.group(2)}" if m else None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def local_clones(
|
|
36
|
+
roots: Iterable[str | Path], *, max_depth: int = DFLT_MAX_DEPTH
|
|
37
|
+
) -> dict[str, str]:
|
|
38
|
+
"""``{"owner/name" (lowercased): path}`` for git checkouts under ``roots``."""
|
|
39
|
+
found: dict[str, str] = {}
|
|
40
|
+
|
|
41
|
+
def walk(d: Path, depth: int):
|
|
42
|
+
cfg = d / ".git" / "config"
|
|
43
|
+
if cfg.is_file():
|
|
44
|
+
rid = origin_repo_id(cfg.read_text(errors="replace"))
|
|
45
|
+
if rid:
|
|
46
|
+
found.setdefault(rid.lower(), str(d))
|
|
47
|
+
if depth >= max_depth:
|
|
48
|
+
return
|
|
49
|
+
try:
|
|
50
|
+
subdirs = [e for e in os.scandir(d) if e.is_dir(follow_symlinks=False)]
|
|
51
|
+
except OSError:
|
|
52
|
+
return
|
|
53
|
+
for e in subdirs:
|
|
54
|
+
if e.name not in _SKIP and not e.name.startswith("."):
|
|
55
|
+
walk(Path(e.path), depth + 1)
|
|
56
|
+
|
|
57
|
+
for root in roots:
|
|
58
|
+
walk(Path(root).expanduser(), 0)
|
|
59
|
+
return found
|
ghir/search.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Ranking cached repos against a query, and marking which ones are cloned here.
|
|
2
|
+
|
|
3
|
+
The ranking seam is ``rank=``, a callable ``(query, docs, *, k, **kw) -> [(id,
|
|
4
|
+
score)]`` over ``docs = {repo_id: text}``. Two are built in:
|
|
5
|
+
|
|
6
|
+
- :func:`ir_rank` (default): indexes the docs as an ``ir`` corpus (incremental:
|
|
7
|
+
only changed repos are re-embedded) and runs ``ir.search``, hybrid by default.
|
|
8
|
+
- :func:`grep_rank`: no index, no model; scores a repo by how many query terms
|
|
9
|
+
its text contains. The by-hand ``rg`` over ``gh repo list``, made reusable.
|
|
10
|
+
|
|
11
|
+
Nothing here touches the network: :func:`search` reads the cache that
|
|
12
|
+
:func:`ghir.cache.refresh` fills.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from collections.abc import Callable, Iterable, Mapping, MutableMapping
|
|
19
|
+
from dataclasses import asdict, dataclass
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from .cache import repo_store
|
|
23
|
+
from .config import settings as _settings
|
|
24
|
+
from .extract import Extractor, resolve_extractor
|
|
25
|
+
from .local import local_clones
|
|
26
|
+
|
|
27
|
+
Ranker = Callable[..., list[tuple[str, float]]]
|
|
28
|
+
|
|
29
|
+
#: Record fields lifted into ir filter metadata (usable as ``filter=`` in ir).
|
|
30
|
+
METADATA_KEYS = ("owner", "language", "is_fork", "is_archived", "is_private")
|
|
31
|
+
DFLT_K = 20
|
|
32
|
+
DFLT_IR_MODE = "hybrid"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class RepoHit:
|
|
37
|
+
"""One search result: a repo, its score, and where it is cloned (if it is)."""
|
|
38
|
+
|
|
39
|
+
id: str
|
|
40
|
+
score: float
|
|
41
|
+
description: str
|
|
42
|
+
url: str
|
|
43
|
+
language: str = ""
|
|
44
|
+
local_path: str | None = None
|
|
45
|
+
is_fork: bool = False
|
|
46
|
+
is_archived: bool = False
|
|
47
|
+
|
|
48
|
+
def to_dict(self) -> dict:
|
|
49
|
+
return asdict(self)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def ir_records(
|
|
53
|
+
store: Mapping[str, Mapping] | None = None,
|
|
54
|
+
*,
|
|
55
|
+
extract: str | Extractor | None = None,
|
|
56
|
+
) -> list[dict]:
|
|
57
|
+
"""Cached repos as ir records: ``{"id", "text", <metadata keys>}``."""
|
|
58
|
+
s = _settings(extract=extract)
|
|
59
|
+
store = repo_store(s["cache_dir"]) if store is None else store
|
|
60
|
+
fn = resolve_extractor(s["extract"])
|
|
61
|
+
return [
|
|
62
|
+
{"id": rid, "text": fn(rec), **{k: rec.get(k) for k in METADATA_KEYS}}
|
|
63
|
+
for rid, rec in store.items()
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def records() -> list[dict]:
|
|
68
|
+
"""The default records, as a zero-argument fetcher for ir's registry.
|
|
69
|
+
|
|
70
|
+
``ir.register("gh_repos", "records", fetcher="ghir:records")`` makes the
|
|
71
|
+
cache searchable with ir's own CLI (``ir discover gh_repos "..."``).
|
|
72
|
+
"""
|
|
73
|
+
return ir_records()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _corpus_name(base: str, extract: str | Extractor) -> str:
|
|
77
|
+
if extract == _settings()["extract"] and not callable(extract):
|
|
78
|
+
return base
|
|
79
|
+
tag = extract if isinstance(extract, str) else getattr(extract, "__name__", "fn")
|
|
80
|
+
return f"{base}__{tag}"
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def ir_rank(
|
|
84
|
+
query: str,
|
|
85
|
+
docs: Mapping[str, Mapping],
|
|
86
|
+
*,
|
|
87
|
+
k: int = DFLT_K,
|
|
88
|
+
corpus_name: str,
|
|
89
|
+
mode: str = DFLT_IR_MODE,
|
|
90
|
+
embedder: Any = "default",
|
|
91
|
+
ir_store: Any = None,
|
|
92
|
+
**search_kw,
|
|
93
|
+
) -> list[tuple[str, float]]:
|
|
94
|
+
"""Rank with ``ir``: build (incrementally) then ``ir.search``.
|
|
95
|
+
|
|
96
|
+
``docs`` maps repo id to an ir record (``{"id", "text", ...metadata}``).
|
|
97
|
+
``ir_store`` is the ir ``CorpusStore`` (default: ir's local store, named
|
|
98
|
+
``corpus_name``); other keyword arguments go to ``ir.search``.
|
|
99
|
+
"""
|
|
100
|
+
import ir
|
|
101
|
+
|
|
102
|
+
source = ir.CorpusSource.from_records(
|
|
103
|
+
name=corpus_name,
|
|
104
|
+
fetcher=lambda: list(docs.values()),
|
|
105
|
+
metadata_keys=METADATA_KEYS,
|
|
106
|
+
embedder=embedder,
|
|
107
|
+
)
|
|
108
|
+
corpus = ir.build(source, store=ir_store)
|
|
109
|
+
hits = ir.search(corpus, query, k=k, mode=mode, **search_kw)
|
|
110
|
+
return [(h.artifact_id, float(h.score)) for h in hits]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _terms(query: str) -> list[str]:
|
|
114
|
+
return [t for t in re.split(r"\W+", query.lower()) if len(t) > 1]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def grep_rank(
|
|
118
|
+
query: str, docs: Mapping[str, Mapping], *, k: int = DFLT_K, **_
|
|
119
|
+
) -> list[tuple[str, float]]:
|
|
120
|
+
"""Fraction of query terms (as word prefixes) found in each repo's text.
|
|
121
|
+
|
|
122
|
+
>>> docs = {"a/x": {"text": "Spaced repetition flashcards"},
|
|
123
|
+
... "a/y": {"text": "audio tools"}}
|
|
124
|
+
>>> grep_rank("flashcard spaced", docs)
|
|
125
|
+
[('a/x', 1.0)]
|
|
126
|
+
"""
|
|
127
|
+
terms = _terms(query)
|
|
128
|
+
if not terms:
|
|
129
|
+
return []
|
|
130
|
+
pats = [re.compile(rf"\b{re.escape(t)}", re.I) for t in terms]
|
|
131
|
+
scored = [
|
|
132
|
+
(rid, sum(bool(p.search(d["text"])) for p in pats) / len(pats))
|
|
133
|
+
for rid, d in docs.items()
|
|
134
|
+
]
|
|
135
|
+
scored = [x for x in scored if x[1] > 0]
|
|
136
|
+
return sorted(scored, key=lambda x: -x[1])[:k]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
rankers: dict[str, Ranker] = {"ir": ir_rank, "grep": grep_rank}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def search(
|
|
143
|
+
query: str,
|
|
144
|
+
*,
|
|
145
|
+
k: int = DFLT_K,
|
|
146
|
+
rank: str | Ranker = "ir",
|
|
147
|
+
extract: str | Extractor | None = None,
|
|
148
|
+
store: MutableMapping | None = None,
|
|
149
|
+
local_roots: Iterable[str] | None = None,
|
|
150
|
+
include_forks: bool = True,
|
|
151
|
+
include_archived: bool = True,
|
|
152
|
+
**rank_kw,
|
|
153
|
+
) -> list[RepoHit]:
|
|
154
|
+
"""Rank cached repos for ``query``; each hit says where it is cloned, if anywhere.
|
|
155
|
+
|
|
156
|
+
Offline: reads the cache filled by :func:`ghir.cache.refresh`.
|
|
157
|
+
"""
|
|
158
|
+
s = _settings(extract=extract, local_roots=local_roots)
|
|
159
|
+
store = repo_store(s["cache_dir"]) if store is None else store
|
|
160
|
+
if not len(store):
|
|
161
|
+
raise LookupError(
|
|
162
|
+
"The ghir cache is empty; fill it first with `ghir refresh` "
|
|
163
|
+
"(or ghir.refresh())."
|
|
164
|
+
)
|
|
165
|
+
repos = {
|
|
166
|
+
rid: rec
|
|
167
|
+
for rid, rec in store.items()
|
|
168
|
+
if (include_forks or not rec.get("is_fork"))
|
|
169
|
+
and (include_archived or not rec.get("is_archived"))
|
|
170
|
+
}
|
|
171
|
+
docs = {r["id"]: r for r in ir_records(repos, extract=s["extract"])}
|
|
172
|
+
ranker = rankers[rank] if isinstance(rank, str) else rank
|
|
173
|
+
if ranker is ir_rank:
|
|
174
|
+
rank_kw.setdefault("corpus_name", _corpus_name(s["corpus_name"], s["extract"]))
|
|
175
|
+
ranked = ranker(query, docs, k=k, **rank_kw)
|
|
176
|
+
clones = local_clones(s["local_roots"])
|
|
177
|
+
return [
|
|
178
|
+
RepoHit(
|
|
179
|
+
id=rid,
|
|
180
|
+
score=score,
|
|
181
|
+
description=repos[rid].get("description", ""),
|
|
182
|
+
url=repos[rid].get("url", ""),
|
|
183
|
+
language=repos[rid].get("language", ""),
|
|
184
|
+
local_path=clones.get(rid.lower()),
|
|
185
|
+
is_fork=bool(repos[rid].get("is_fork")),
|
|
186
|
+
is_archived=bool(repos[rid].get("is_archived")),
|
|
187
|
+
)
|
|
188
|
+
for rid, score in ranked
|
|
189
|
+
if rid in repos
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def register_corpus(name: str | None = None) -> dict:
|
|
194
|
+
"""Register the cache as an ir corpus, so ``ir discover <name> ...`` works."""
|
|
195
|
+
import ir
|
|
196
|
+
|
|
197
|
+
return ir.register(
|
|
198
|
+
name or _settings()["corpus_name"],
|
|
199
|
+
"records",
|
|
200
|
+
fetcher="ghir:records",
|
|
201
|
+
metadata_keys=list(METADATA_KEYS),
|
|
202
|
+
)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ghir
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Search your GitHub repos (cloned or not) as an ir corpus: locations, extraction, cache, ranking.
|
|
5
|
+
Project-URL: Homepage, https://github.com/i2mint/ghir
|
|
6
|
+
Project-URL: Repository, https://github.com/i2mint/ghir
|
|
7
|
+
Project-URL: Documentation, https://i2mint.github.io/ghir
|
|
8
|
+
Author: Thor Whalen
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: github,information-retrieval,ir,repositories,search
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: dol
|
|
14
|
+
Requires-Dist: ir
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
18
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
19
|
+
Provides-Extra: docs
|
|
20
|
+
Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
|
|
21
|
+
Requires-Dist: sphinx>=6.0; extra == 'docs'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# ghir
|
|
25
|
+
|
|
26
|
+
Search your GitHub repos, cloned or not, across every organization you belong to. Fetch once, then search offline; each hit says where it is cloned on this machine, if anywhere.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install ghir
|
|
30
|
+
ghir refresh # you + your orgs, through the gh CLI
|
|
31
|
+
ghir search "flashcards spaced repetition" # offline, ranked by ir
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
0.032 thorwhalen/tetrachord /home/me/proj/tt/tetrachord
|
|
36
|
+
Music education with a learner model and scheduler at the architectural centre.
|
|
37
|
+
0.015 thorwhalen/rbxloop (not cloned)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Requires the [GitHub CLI](https://cli.github.com) (`gh auth login`) for `refresh` only; searching never touches the network.
|
|
41
|
+
|
|
42
|
+
## The four seams
|
|
43
|
+
|
|
44
|
+
ghir is `corpus -> extraction -> ranking`, with a cache beside it. Each is one keyword argument with a working default.
|
|
45
|
+
|
|
46
|
+
| Seam | Argument | Default | Alternatives |
|
|
47
|
+
|---|---|---|---|
|
|
48
|
+
| Locations: which repos | `refresh(locations)` | `"@me"`: you and every org you belong to | owner logins, `"search:<github search query>"` |
|
|
49
|
+
| Extraction: what text a repo contributes | `extract=` | `"description_and_readme"` (name, description, topics, README head) | `"description"`, or any `record -> str` callable |
|
|
50
|
+
| Ranking | `rank=` | `"ir"`: hybrid dense + BM25 through [ir](https://github.com/i2mint/ir), indexed incrementally | `"grep"` (query terms as word prefixes, no index, no model), or any callable |
|
|
51
|
+
| Cache | `store=` | one JSON file per repo under `~/.local/share/ghir/repos/` | any `MutableMapping` (a `dol` store: S3, a database) |
|
|
52
|
+
|
|
53
|
+
`refresh` merges: fetched repos overwrite their own entries, and cached repos it did not see this time are kept. A location that fails to fetch is reported in `"failed"` and its cached repos stay.
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import ghir
|
|
57
|
+
|
|
58
|
+
ghir.refresh(["i2mint", "search:user:thorwhalen topic:education"])
|
|
59
|
+
for hit in ghir.search("audio tagging", rank="grep", include_forks=False):
|
|
60
|
+
print(hit.id, hit.local_path or hit.url)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Settings
|
|
64
|
+
|
|
65
|
+
Machine-specific choices go in `~/.config/ghir/config.json` (or `$GHIR_CONFIG`):
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{"locations": ["@me", "otosense"],
|
|
69
|
+
"exclude": ["SomeClientOrg/*", "myorg/private-thing"]}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
An owner excluded with `owner/*` is never queried. Other keys: `extract`, `local_roots` (where clones are looked for; default `$PP` or your home), `cache_dir`, `corpus_name`.
|
|
73
|
+
|
|
74
|
+
## Using it from ir
|
|
75
|
+
|
|
76
|
+
`ghir refresh` also registers the cache as the ir corpus `gh_repos` (a `records` corpus whose fetcher is `ghir:records`), so ir's own tools see it:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
ir discover gh_repos "spaced repetition"
|
|
80
|
+
ir discover gh_repos packages "audio tagging" # federated with the local packages corpus
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Why this is a separate package and not part of ir: [docs/adr/0001-where-ghir-lives.md](docs/adr/0001-where-ghir-lives.md).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
ghir/__init__.py,sha256=vrIFXCicImzsN_bO-rgXgjPc0qhjuYbeB57B3Vew5Mw,1291
|
|
2
|
+
ghir/__main__.py,sha256=zFgL1E3xjbRzFhVL4A41P2ucNG8o0_-Y9lpbg_O2Zdw,2584
|
|
3
|
+
ghir/cache.py,sha256=P3pgNy824VuKh87Lj1JlWyck_CkalgwYKObaovJS-TI,2643
|
|
4
|
+
ghir/config.py,sha256=1pxdq_MzUhALDjRVSWelrHGTEIyU-_MPwPsOArQuGJI,2610
|
|
5
|
+
ghir/extract.py,sha256=yYMl9nB8vg6yCIoFI0aFxguxxf9JRLo9l3nUrQ3FRv4,1990
|
|
6
|
+
ghir/fetch.py,sha256=NFYZYIul7CseZPykXaeh1g-Gy69sraALQk-y-gqzKk4,8572
|
|
7
|
+
ghir/local.py,sha256=6vV2BjvzIeaH_DHZuP8svE1G3L0Wxc8OLP3KWdgSQ28,2014
|
|
8
|
+
ghir/search.py,sha256=TxNBmMQtxtFJsT2GphKxj6ryG5NRvc3LMALZiX8OCL8,6554
|
|
9
|
+
ghir-0.0.2.dist-info/METADATA,sha256=dPdZVICvWkicjLchtiRvFog5cTuzW-hnXbGBkBxjTeg,3605
|
|
10
|
+
ghir-0.0.2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
11
|
+
ghir-0.0.2.dist-info/entry_points.txt,sha256=pFOeIvpH0D90AFQCvTUR3XkYLLKYwDEbjCP8T4UA300,44
|
|
12
|
+
ghir-0.0.2.dist-info/licenses/LICENSE,sha256=CJrtBuzD6GCZucCgkywlQSgsKm8QUOXgF-QMKUU1kjo,1063
|
|
13
|
+
ghir-0.0.2.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 i2mint
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|