ghir 0.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1 @@
1
+ *.ipynb linguist-documentation
ghir-0.0.2/.gitignore ADDED
@@ -0,0 +1,119 @@
1
+ .claude/handoffs/
2
+ .claude/scratch/
3
+
4
+ # Byte-compiled / optimized / DLL files
5
+ __pycache__/
6
+ *.py[cod]
7
+ *$py.class
8
+
9
+
10
+ .DS_Store
11
+ # C extensions
12
+ *.so
13
+
14
+ # TLS certificates
15
+ ## Ignore all PEM files anywhere
16
+ *.pem
17
+ ## Also ignore any certs directory
18
+ certs/
19
+
20
+ # Distribution / packaging
21
+ .Python
22
+ build/
23
+ develop-eggs/
24
+ dist/
25
+ downloads/
26
+ eggs/
27
+ .eggs/
28
+ lib/
29
+ lib64/
30
+ parts/
31
+ sdist/
32
+ var/
33
+ wheels/
34
+ *.egg-info/
35
+ .installed.cfg
36
+ *.egg
37
+ MANIFEST
38
+ _build
39
+
40
+ # PyInstaller
41
+ # Usually these files are written by a python script from a template
42
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
43
+ *.manifest
44
+ *.spec
45
+
46
+ # Installer logs
47
+ pip-log.txt
48
+ pip-delete-this-directory.txt
49
+
50
+ # Unit test / coverage reports
51
+ htmlcov/
52
+ .tox/
53
+ .coverage
54
+ .coverage.*
55
+ .cache
56
+ nosetests.xml
57
+ coverage.xml
58
+ *.cover
59
+ .hypothesis/
60
+ .pytest_cache/
61
+
62
+ # Translations
63
+ *.mo
64
+ *.pot
65
+
66
+ # Django stuff:
67
+ *.log
68
+ local_settings.py
69
+ db.sqlite3
70
+
71
+ # Flask stuff:
72
+ instance/
73
+ .webassets-cache
74
+
75
+ # Scrapy stuff:
76
+ .scrapy
77
+
78
+ # Sphinx documentation
79
+ docs/_build/
80
+
81
+ # PyBuilder
82
+ target/
83
+
84
+ # Jupyter Notebook
85
+ .ipynb_checkpoints
86
+
87
+ # pyenv
88
+ .python-version
89
+
90
+ # celery beat schedule file
91
+ celerybeat-schedule
92
+
93
+ # SageMath parsed files
94
+ *.sage.py
95
+
96
+ # Environments
97
+ .env
98
+ .venv
99
+ env/
100
+ venv/
101
+ ENV/
102
+ env.bak/
103
+ venv.bak/
104
+
105
+ # Spyder project settings
106
+ .spyderproject
107
+ .spyproject
108
+
109
+ # Rope project settings
110
+ .ropeproject
111
+
112
+ # mkdocs documentation
113
+ /site
114
+
115
+ # mypy
116
+ .mypy_cache/
117
+
118
+ # PyCharm
119
+ .idea
ghir-0.0.2/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 i2mint
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
ghir-0.0.2/PKG-INFO ADDED
@@ -0,0 +1,83 @@
1
+ Metadata-Version: 2.5
2
+ Name: ghir
3
+ Version: 0.0.2
4
+ Summary: Search your GitHub repos (cloned or not) as an ir corpus: locations, extraction, cache, ranking.
5
+ Project-URL: Homepage, https://github.com/i2mint/ghir
6
+ Project-URL: Repository, https://github.com/i2mint/ghir
7
+ Project-URL: Documentation, https://i2mint.github.io/ghir
8
+ Author: Thor Whalen
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: github,information-retrieval,ir,repositories,search
12
+ Requires-Python: >=3.10
13
+ Requires-Dist: dol
14
+ Requires-Dist: ir
15
+ Provides-Extra: dev
16
+ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
17
+ Requires-Dist: pytest>=7.0; extra == 'dev'
18
+ Requires-Dist: ruff>=0.1.0; extra == 'dev'
19
+ Provides-Extra: docs
20
+ Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
21
+ Requires-Dist: sphinx>=6.0; extra == 'docs'
22
+ Description-Content-Type: text/markdown
23
+
24
+ # ghir
25
+
26
+ Search your GitHub repos, cloned or not, across every organization you belong to. Fetch once, then search offline; each hit says where it is cloned on this machine, if anywhere.
27
+
28
+ ```bash
29
+ pip install ghir
30
+ ghir refresh # you + your orgs, through the gh CLI
31
+ ghir search "flashcards spaced repetition" # offline, ranked by ir
32
+ ```
33
+
34
+ ```
35
+ 0.032 thorwhalen/tetrachord /home/me/proj/tt/tetrachord
36
+ Music education with a learner model and scheduler at the architectural centre.
37
+ 0.015 thorwhalen/rbxloop (not cloned)
38
+ ```
39
+
40
+ Requires the [GitHub CLI](https://cli.github.com) (`gh auth login`) for `refresh` only; searching never touches the network.
41
+
42
+ ## The four seams
43
+
44
+ ghir is `corpus -> extraction -> ranking`, with a cache beside it. Each is one keyword argument with a working default.
45
+
46
+ | Seam | Argument | Default | Alternatives |
47
+ |---|---|---|---|
48
+ | Locations: which repos | `refresh(locations)` | `"@me"`: you and every org you belong to | owner logins, `"search:<github search query>"` |
49
+ | Extraction: what text a repo contributes | `extract=` | `"description_and_readme"` (name, description, topics, README head) | `"description"`, or any `record -> str` callable |
50
+ | Ranking | `rank=` | `"ir"`: hybrid dense + BM25 through [ir](https://github.com/i2mint/ir), indexed incrementally | `"grep"` (query terms as word prefixes, no index, no model), or any callable |
51
+ | Cache | `store=` | one JSON file per repo under `~/.local/share/ghir/repos/` | any `MutableMapping` (a `dol` store: S3, a database) |
52
+
53
+ `refresh` merges: fetched repos overwrite their own entries, and cached repos it did not see this time are kept. A location that fails to fetch is reported in `"failed"` and its cached repos stay.
54
+
55
+ ```python
56
+ import ghir
57
+
58
+ ghir.refresh(["i2mint", "search:user:thorwhalen topic:education"])
59
+ for hit in ghir.search("audio tagging", rank="grep", include_forks=False):
60
+ print(hit.id, hit.local_path or hit.url)
61
+ ```
62
+
63
+ ## Settings
64
+
65
+ Machine-specific choices go in `~/.config/ghir/config.json` (or `$GHIR_CONFIG`):
66
+
67
+ ```json
68
+ {"locations": ["@me", "otosense"],
69
+ "exclude": ["SomeClientOrg/*", "myorg/private-thing"]}
70
+ ```
71
+
72
+ An owner excluded with `owner/*` is never queried. Other keys: `extract`, `local_roots` (where clones are looked for; default `$PP` or your home), `cache_dir`, `corpus_name`.
73
+
74
+ ## Using it from ir
75
+
76
+ `ghir refresh` also registers the cache as the ir corpus `gh_repos` (a `records` corpus whose fetcher is `ghir:records`), so ir's own tools see it:
77
+
78
+ ```bash
79
+ ir discover gh_repos "spaced repetition"
80
+ ir discover gh_repos packages "audio tagging" # federated with the local packages corpus
81
+ ```
82
+
83
+ Why this is a separate package and not part of ir: [docs/adr/0001-where-ghir-lives.md](docs/adr/0001-where-ghir-lives.md).
ghir-0.0.2/README.md ADDED
@@ -0,0 +1,60 @@
1
+ # ghir
2
+
3
+ Search your GitHub repos, cloned or not, across every organization you belong to. Fetch once, then search offline; each hit says where it is cloned on this machine, if anywhere.
4
+
5
+ ```bash
6
+ pip install ghir
7
+ ghir refresh # you + your orgs, through the gh CLI
8
+ ghir search "flashcards spaced repetition" # offline, ranked by ir
9
+ ```
10
+
11
+ ```
12
+ 0.032 thorwhalen/tetrachord /home/me/proj/tt/tetrachord
13
+ Music education with a learner model and scheduler at the architectural centre.
14
+ 0.015 thorwhalen/rbxloop (not cloned)
15
+ ```
16
+
17
+ Requires the [GitHub CLI](https://cli.github.com) (`gh auth login`) for `refresh` only; searching never touches the network.
18
+
19
+ ## The four seams
20
+
21
+ ghir is `corpus -> extraction -> ranking`, with a cache beside it. Each is one keyword argument with a working default.
22
+
23
+ | Seam | Argument | Default | Alternatives |
24
+ |---|---|---|---|
25
+ | Locations: which repos | `refresh(locations)` | `"@me"`: you and every org you belong to | owner logins, `"search:<github search query>"` |
26
+ | Extraction: what text a repo contributes | `extract=` | `"description_and_readme"` (name, description, topics, README head) | `"description"`, or any `record -> str` callable |
27
+ | Ranking | `rank=` | `"ir"`: hybrid dense + BM25 through [ir](https://github.com/i2mint/ir), indexed incrementally | `"grep"` (query terms as word prefixes, no index, no model), or any callable |
28
+ | Cache | `store=` | one JSON file per repo under `~/.local/share/ghir/repos/` | any `MutableMapping` (a `dol` store: S3, a database) |
29
+
30
+ `refresh` merges: fetched repos overwrite their own entries, and cached repos it did not see this time are kept. A location that fails to fetch is reported in `"failed"` and its cached repos stay.
31
+
32
+ ```python
33
+ import ghir
34
+
35
+ ghir.refresh(["i2mint", "search:user:thorwhalen topic:education"])
36
+ for hit in ghir.search("audio tagging", rank="grep", include_forks=False):
37
+ print(hit.id, hit.local_path or hit.url)
38
+ ```
39
+
40
+ ## Settings
41
+
42
+ Machine-specific choices go in `~/.config/ghir/config.json` (or `$GHIR_CONFIG`):
43
+
44
+ ```json
45
+ {"locations": ["@me", "otosense"],
46
+ "exclude": ["SomeClientOrg/*", "myorg/private-thing"]}
47
+ ```
48
+
49
+ An owner excluded with `owner/*` is never queried. Other keys: `extract`, `local_roots` (where clones are looked for; default `$PP` or your home), `cache_dir`, `corpus_name`.
50
+
51
+ ## Using it from ir
52
+
53
+ `ghir refresh` also registers the cache as the ir corpus `gh_repos` (a `records` corpus whose fetcher is `ghir:records`), so ir's own tools see it:
54
+
55
+ ```bash
56
+ ir discover gh_repos "spaced repetition"
57
+ ir discover gh_repos packages "audio tagging" # federated with the local packages corpus
58
+ ```
59
+
60
+ Why this is a separate package and not part of ir: [docs/adr/0001-where-ghir-lives.md](docs/adr/0001-where-ghir-lives.md).
@@ -0,0 +1,25 @@
1
+ # ADR 0001: GitHub repo search is an extension package on top of ir, not part of ir
2
+
3
+ Status: accepted, 2026-10-23. Tracking: i2mint/ir#118.
4
+
5
+ ## Context
6
+
7
+ The need: find something that may live in one of the user's GitHub repos, across several organizations, whether or not it is cloned locally. It was first done by hand: `gh repo list` per org to a TSV, `rg` over the descriptions, README grep over local clones, then a check of which hits were cloned. The user named the generality layers: locations (orgs, or a search query yielding repos), extraction (what each repo contributes), measuring and ranking, and caching as an orthogonal concern; above all of them, the general pattern `corpus -> extraction -> measure/filter/order/present`. They asked for the GitHub dependency to stay out of ir.
8
+
9
+ ## Decision
10
+
11
+ 1. **ir already is the general pattern.** `CorpusSource` (scope, change signal, indexing strategy) followed by build, `search`/`discover` is `corpus -> extraction -> ranking`. Its `records` kind takes a fetcher by import reference (`fetcher="module:attr"`), so a corpus owned by another package persists in ir's registry and rebuilds in a fresh process. So nothing new was needed in ir core: ghir supplies the fetcher, ir does indexing and ranking.
12
+ 2. **The GitHub-specific parts live in ghir**: the locations (`@me`, owner logins, `search:<query>`), the fetch (GraphQL through the `gh` CLI, one call per page of repos with README text, not one per repo), the cache, and the local-clone index. ir never imports ghir or `gh`.
13
+ 3. **The cache is a `MutableMapping`** (dol JSON files by default) and is **merged** on refresh, never replaced; search is offline. ir's incremental build (content-hash change signal) then re-embeds only the repos whose text changed.
14
+ 4. **Ranking is a seam with two real strategies**: ir (default, hybrid) and grep (the by-hand method, no index, no model). Extraction is a seam with two named extractors plus any callable.
15
+
16
+ ## Alternatives considered
17
+
18
+ - **Inside ir, behind an optional extra.** Rejected: ir's charter is a GitHub-agnostic substrate, and the user asked for no GitHub dependency there. A shell-out to `gh` behind an extra would still put GitHub knowledge (GraphQL queries, the README lookup, owner expansion) into ir's code and tests.
19
+ - **In hubcap** (the dol interface to GitHub). It is the natural home for GitHub-as-a-mapping, but it goes through PyGithub and a token, one REST call per repo; the bulk GraphQL fetch and the ir glue would sit oddly beside it. If hubcap grows a bulk repo-records store, ghir's `fetch` seam can take it.
20
+ - **In projreg.** projreg's gh sync and doc cache are keyed by locally registered projects; this search is precisely about repos that may not be local.
21
+
22
+ ## Consequences
23
+
24
+ - `ghir refresh` writes one entry to ir's corpus registry (`gh_repos`), so `ir discover gh_repos ...` and federated `ir discover gh_repos packages ...` work.
25
+ - A new location kind (GitLab, a list of URLs) is a new fetcher behind the same record shape; a new ranker or extractor is a function.
@@ -0,0 +1,48 @@
1
+ """Search your GitHub repos, cloned or not, as an ``ir`` corpus.
2
+
3
+ Four seams, each a keyword argument with a working default:
4
+
5
+ - **locations** (:func:`refresh`): owner logins, ``"@me"`` (you and your orgs),
6
+ or ``"search:<github query>"``.
7
+ - **extraction** (``extract=``): what text a repo contributes; default
8
+ description + topics + README head (:mod:`ghir.extract`).
9
+ - **ranking** (``rank=``): ``"ir"`` (default, hybrid dense + BM25) or ``"grep"``.
10
+ - **cache** (``store=``): a ``MutableMapping``; :func:`refresh` merges into it,
11
+ :func:`search` only reads it (offline).
12
+
13
+ >>> import ghir # doctest: +SKIP
14
+ >>> ghir.refresh() # doctest: +SKIP
15
+ >>> for hit in ghir.search("flashcards spaced repetition"): # doctest: +SKIP
16
+ ... print(hit.id, hit.local_path)
17
+ """
18
+
19
+ from .cache import refresh, repo_store
20
+ from .config import settings
21
+ from .extract import extractors
22
+ from .fetch import fetch_repos
23
+ from .local import local_clones
24
+ from .search import (
25
+ RepoHit,
26
+ grep_rank,
27
+ ir_rank,
28
+ rankers,
29
+ records,
30
+ register_corpus,
31
+ search,
32
+ )
33
+
34
+ __all__ = [
35
+ "RepoHit",
36
+ "extractors",
37
+ "fetch_repos",
38
+ "grep_rank",
39
+ "ir_rank",
40
+ "local_clones",
41
+ "rankers",
42
+ "records",
43
+ "refresh",
44
+ "register_corpus",
45
+ "repo_store",
46
+ "search",
47
+ "settings",
48
+ ]
@@ -0,0 +1,76 @@
1
+ """Command line: ``ghir refresh`` (online, merges into the cache) and ``ghir search``.
2
+
3
+ .. code-block:: bash
4
+
5
+ ghir refresh # @me + config locations
6
+ ghir refresh i2mint "search:user:thorwhalen topic:education"
7
+ ghir search "flashcards spaced repetition" # offline
8
+ ghir search "audio tagging" --rank grep --local-only
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import json
15
+ import sys
16
+
17
+ from .cache import refresh
18
+ from .search import DFLT_K, rankers, register_corpus, search
19
+
20
+
21
+ def _print_hits(hits, *, as_json: bool, local_only: bool):
22
+ hits = [h for h in hits if h.local_path or not local_only]
23
+ if as_json:
24
+ print(json.dumps([h.to_dict() for h in hits], indent=2))
25
+ return
26
+ for h in hits:
27
+ where = h.local_path or "(not cloned)"
28
+ flags = "".join(
29
+ f" [{f}]"
30
+ for f, on in (("fork", h.is_fork), ("archived", h.is_archived))
31
+ if on
32
+ )
33
+ print(f"{h.score:6.3f} {h.id}{flags} {where}")
34
+ if h.description:
35
+ print(f" {h.description}")
36
+
37
+
38
+ def main(argv=None):
39
+ p = argparse.ArgumentParser(prog="ghir", description=__doc__.splitlines()[0])
40
+ sub = p.add_subparsers(dest="cmd", required=True)
41
+
42
+ r = sub.add_parser("refresh", help="fetch repos from GitHub, merge into the cache")
43
+ r.add_argument("locations", nargs="*", help="owners, @me, or search:<query>")
44
+ r.add_argument("--no-register", action="store_true", help="skip ir registration")
45
+
46
+ s = sub.add_parser("search", help="rank cached repos for a query (offline)")
47
+ s.add_argument("query")
48
+ s.add_argument("-k", type=int, default=DFLT_K)
49
+ s.add_argument("--rank", choices=sorted(rankers), default="ir")
50
+ s.add_argument("--extract", default=None, help="extractor name")
51
+ s.add_argument("--no-forks", action="store_true")
52
+ s.add_argument("--no-archived", action="store_true")
53
+ s.add_argument("--local-only", action="store_true", help="only cloned repos")
54
+ s.add_argument("--json", action="store_true")
55
+
56
+ a = p.parse_args(argv)
57
+ if a.cmd == "refresh":
58
+ counts = refresh(a.locations or None)
59
+ if not a.no_register:
60
+ register_corpus()
61
+ print(json.dumps(counts))
62
+ return 0
63
+ hits = search(
64
+ a.query,
65
+ k=a.k,
66
+ rank=a.rank,
67
+ extract=a.extract,
68
+ include_forks=not a.no_forks,
69
+ include_archived=not a.no_archived,
70
+ )
71
+ _print_hits(hits, as_json=a.json, local_only=a.local_only)
72
+ return 0
73
+
74
+
75
+ if __name__ == "__main__":
76
+ sys.exit(main())
@@ -0,0 +1,70 @@
1
+ """The repo-record cache: a ``MutableMapping`` from ``owner/name`` to a record.
2
+
3
+ Searching never goes online; it reads this cache. :func:`refresh` is the only
4
+ thing that fetches, and it *merges*: fetched records overwrite their own keys,
5
+ records not seen this time are kept (a repo outside today's locations, or a
6
+ fetch that failed half-way, does not erase what was known). Each record carries
7
+ ``fetched_at`` so staleness is visible.
8
+
9
+ The store is a seam (``store=``): the default is one JSON file per repo under
10
+ :data:`ghir.config.DFLT_CACHE_DIR` (a ``dol`` store, so an S3 or database
11
+ mapping drops in unchanged).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from collections.abc import Iterable, MutableMapping
17
+ from datetime import datetime, timezone
18
+ from pathlib import Path
19
+
20
+ from dol import JsonFiles, mk_dirs_if_missing, wrap_kvs
21
+
22
+ from .config import settings as _settings
23
+ from .fetch import fetch_repos
24
+
25
+ #: Repo ids contain a "/", which becomes a directory level (owner/name.json).
26
+ _EXT = ".json"
27
+
28
+
29
+ def repo_store(cache_dir: str | Path | None = None) -> MutableMapping:
30
+ """The default cache: ``{"owner/name": record}`` over JSON files."""
31
+ root = Path(cache_dir or _settings()["cache_dir"])
32
+ root.mkdir(parents=True, exist_ok=True)
33
+ return wrap_kvs(
34
+ mk_dirs_if_missing(JsonFiles(str(root))),
35
+ id_of_key=lambda k: k + _EXT,
36
+ key_of_id=lambda _id: _id[: -len(_EXT)],
37
+ )
38
+
39
+
40
+ def refresh(
41
+ locations: Iterable[str] | None = None,
42
+ *,
43
+ store: MutableMapping | None = None,
44
+ exclude: Iterable[str] | None = None,
45
+ fetch=fetch_repos,
46
+ **fetch_kwargs,
47
+ ) -> dict:
48
+ """Fetch the repos of ``locations`` and merge them into the cache.
49
+
50
+ Returns counts: ``{"fetched", "added", "updated", "kept"}`` where ``kept`` is
51
+ the number of cached records this fetch did not see (and left alone), plus
52
+ ``"failed"``: ``{location: error}`` for locations that could not be fetched
53
+ (their cached records are kept).
54
+ """
55
+ s = _settings(locations=locations, exclude=exclude)
56
+ store = repo_store(s["cache_dir"]) if store is None else store
57
+ before = set(store)
58
+ now = datetime.now(timezone.utc).isoformat(timespec="seconds")
59
+ seen, failed = set(), {}
60
+ fetch_kwargs.setdefault("on_error", lambda loc, e: failed.update({loc: str(e)}))
61
+ for rec in fetch(s["locations"], exclude=s["exclude"], **fetch_kwargs):
62
+ store[rec["id"]] = {**rec, "fetched_at": now}
63
+ seen.add(rec["id"])
64
+ return {
65
+ "fetched": len(seen),
66
+ "added": len(seen - before),
67
+ "updated": len(seen & before),
68
+ "kept": len(before - seen),
69
+ "failed": failed,
70
+ }
@@ -0,0 +1,69 @@
1
+ """Settings for ghir: where the cache lives and which repos are in scope.
2
+
3
+ All settings have working defaults and can be overridden, in increasing order of
4
+ precedence, by a JSON file (``~/.config/ghir/config.json``, or ``$GHIR_CONFIG``)
5
+ and by keyword arguments to the functions that read them. The file is the place
6
+ for machine-specific choices (extra owners, owners or repos never to fetch):
7
+
8
+ .. code-block:: json
9
+
10
+ {"locations": ["@me", "otosense"],
11
+ "exclude": ["SomeClientOrg/*", "myorg/private-notes"]}
12
+
13
+ ``"@me"`` expands to the authenticated GitHub user plus every organization they
14
+ belong to (see :func:`ghir.fetch.my_owners`).
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import os
21
+ from collections.abc import Mapping
22
+ from pathlib import Path
23
+ from typing import Any
24
+
25
+
26
+ def _xdg(var: str, fallback: str) -> Path:
27
+ return Path(os.environ.get(var) or Path.home() / fallback)
28
+
29
+
30
+ #: Where fetched repo records are cached (one JSON file per ``owner/name``).
31
+ DFLT_CACHE_DIR = _xdg("XDG_DATA_HOME", ".local/share") / "ghir" / "repos"
32
+
33
+ #: The optional user settings file.
34
+ DFLT_CONFIG_PATH = _xdg("XDG_CONFIG_HOME", ".config") / "ghir" / "config.json"
35
+
36
+ #: The ir corpus name the default records are registered under.
37
+ DFLT_CORPUS_NAME = "gh_repos"
38
+
39
+ DFLT_SETTINGS: Mapping[str, Any] = {
40
+ # Owners (user or org logins) and/or "search:<github search query>" specs.
41
+ "locations": ["@me"],
42
+ # fnmatch patterns over "owner/name"; matching repos are never fetched.
43
+ "exclude": [],
44
+ # Name of the extractor (see ghir.extract.extractors) turning a repo into text.
45
+ "extract": "description_and_readme",
46
+ # Root directories scanned (a few levels deep) for local clones.
47
+ "local_roots": [os.environ.get("PP") or str(Path.home())],
48
+ "cache_dir": str(DFLT_CACHE_DIR),
49
+ "corpus_name": DFLT_CORPUS_NAME,
50
+ }
51
+
52
+
53
+ def settings(path: str | Path | None = None, **overrides) -> dict:
54
+ """The effective settings: defaults, then the config file, then ``overrides``.
55
+
56
+ >>> s = settings(path="/nonexistent/ghir.json", exclude=["acme/*"])
57
+ >>> s["exclude"], s["locations"]
58
+ (['acme/*'], ['@me'])
59
+ """
60
+ path = Path(path or os.environ.get("GHIR_CONFIG") or DFLT_CONFIG_PATH)
61
+ from_file = json.loads(path.read_text()) if path.is_file() else {}
62
+ unknown = set(from_file) - set(DFLT_SETTINGS)
63
+ if unknown:
64
+ raise ValueError(
65
+ f"Unknown ghir setting(s) {sorted(unknown)} in {path}; "
66
+ f"known: {sorted(DFLT_SETTINGS)}"
67
+ )
68
+ given = {k: v for k, v in overrides.items() if v is not None}
69
+ return {**DFLT_SETTINGS, **from_file, **given}
@@ -0,0 +1,63 @@
1
+ """Extraction: what text each cached repo contributes to the search.
2
+
3
+ An extractor is a function ``record -> str``. :data:`extractors` names the
4
+ built-in ones; any callable works wherever an extractor is accepted. The text
5
+ returned is what gets indexed and ranked, so it decides what a query can match.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable, Mapping
11
+ from typing import Any
12
+
13
+ Extractor = Callable[[Mapping[str, Any]], str]
14
+
15
+ #: README characters kept by the default extractor (READMEs can be huge, and
16
+ #: their opening says what the repo is).
17
+ DFLT_README_CHARS = 6000
18
+
19
+
20
+ def _head(rec: Mapping[str, Any]) -> list[str]:
21
+ topics = " ".join(rec.get("topics") or ())
22
+ return [
23
+ rec.get("name", ""),
24
+ rec.get("description", ""),
25
+ f"topics: {topics}" if topics else "",
26
+ ]
27
+
28
+
29
+ def description(rec: Mapping[str, Any]) -> str:
30
+ """Name, description and topics.
31
+
32
+ >>> description({"name": "learny", "description": "Learner models",
33
+ ... "topics": ["education"]})
34
+ 'learny\\nLearner models\\ntopics: education'
35
+ """
36
+ return "\n".join(p for p in _head(rec) if p)
37
+
38
+
39
+ def description_and_readme(
40
+ rec: Mapping[str, Any], *, readme_chars: int = DFLT_README_CHARS
41
+ ) -> str:
42
+ """Name, description, topics, and the start of the README (the default)."""
43
+ readme = (rec.get("readme") or "")[:readme_chars]
44
+ return "\n".join(p for p in [*_head(rec), readme] if p)
45
+
46
+
47
+ extractors: dict[str, Extractor] = {
48
+ "description": description,
49
+ "description_and_readme": description_and_readme,
50
+ }
51
+
52
+
53
+ def resolve_extractor(extract: str | Extractor) -> Extractor:
54
+ """An extractor from its name in :data:`extractors`, or a callable as is."""
55
+ if callable(extract):
56
+ return extract
57
+ try:
58
+ return extractors[extract]
59
+ except KeyError:
60
+ raise ValueError(
61
+ f"Unknown extractor {extract!r}; use a callable or one of "
62
+ f"{sorted(extractors)}"
63
+ ) from None