ghir 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ghir-0.0.2/.gitattributes +1 -0
- ghir-0.0.2/.gitignore +119 -0
- ghir-0.0.2/LICENSE +21 -0
- ghir-0.0.2/PKG-INFO +83 -0
- ghir-0.0.2/README.md +60 -0
- ghir-0.0.2/docs/adr/0001-where-ghir-lives.md +25 -0
- ghir-0.0.2/ghir/__init__.py +48 -0
- ghir-0.0.2/ghir/__main__.py +76 -0
- ghir-0.0.2/ghir/cache.py +70 -0
- ghir-0.0.2/ghir/config.py +69 -0
- ghir-0.0.2/ghir/extract.py +63 -0
- ghir-0.0.2/ghir/fetch.py +240 -0
- ghir-0.0.2/ghir/local.py +59 -0
- ghir-0.0.2/ghir/search.py +202 -0
- ghir-0.0.2/pyproject.toml +177 -0
- ghir-0.0.2/tests/test_ghir.py +167 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
*.ipynb linguist-documentation
|
ghir-0.0.2/.gitignore
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
.claude/handoffs/
|
|
2
|
+
.claude/scratch/
|
|
3
|
+
|
|
4
|
+
# Byte-compiled / optimized / DLL files
|
|
5
|
+
__pycache__/
|
|
6
|
+
*.py[cod]
|
|
7
|
+
*$py.class
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
.DS_Store
|
|
11
|
+
# C extensions
|
|
12
|
+
*.so
|
|
13
|
+
|
|
14
|
+
# TLS certificates
|
|
15
|
+
## Ignore all PEM files anywhere
|
|
16
|
+
*.pem
|
|
17
|
+
## Also ignore any certs directory
|
|
18
|
+
certs/
|
|
19
|
+
|
|
20
|
+
# Distribution / packaging
|
|
21
|
+
.Python
|
|
22
|
+
build/
|
|
23
|
+
develop-eggs/
|
|
24
|
+
dist/
|
|
25
|
+
downloads/
|
|
26
|
+
eggs/
|
|
27
|
+
.eggs/
|
|
28
|
+
lib/
|
|
29
|
+
lib64/
|
|
30
|
+
parts/
|
|
31
|
+
sdist/
|
|
32
|
+
var/
|
|
33
|
+
wheels/
|
|
34
|
+
*.egg-info/
|
|
35
|
+
.installed.cfg
|
|
36
|
+
*.egg
|
|
37
|
+
MANIFEST
|
|
38
|
+
_build
|
|
39
|
+
|
|
40
|
+
# PyInstaller
|
|
41
|
+
# Usually these files are written by a python script from a template
|
|
42
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
43
|
+
*.manifest
|
|
44
|
+
*.spec
|
|
45
|
+
|
|
46
|
+
# Installer logs
|
|
47
|
+
pip-log.txt
|
|
48
|
+
pip-delete-this-directory.txt
|
|
49
|
+
|
|
50
|
+
# Unit test / coverage reports
|
|
51
|
+
htmlcov/
|
|
52
|
+
.tox/
|
|
53
|
+
.coverage
|
|
54
|
+
.coverage.*
|
|
55
|
+
.cache
|
|
56
|
+
nosetests.xml
|
|
57
|
+
coverage.xml
|
|
58
|
+
*.cover
|
|
59
|
+
.hypothesis/
|
|
60
|
+
.pytest_cache/
|
|
61
|
+
|
|
62
|
+
# Translations
|
|
63
|
+
*.mo
|
|
64
|
+
*.pot
|
|
65
|
+
|
|
66
|
+
# Django stuff:
|
|
67
|
+
*.log
|
|
68
|
+
local_settings.py
|
|
69
|
+
db.sqlite3
|
|
70
|
+
|
|
71
|
+
# Flask stuff:
|
|
72
|
+
instance/
|
|
73
|
+
.webassets-cache
|
|
74
|
+
|
|
75
|
+
# Scrapy stuff:
|
|
76
|
+
.scrapy
|
|
77
|
+
|
|
78
|
+
# Sphinx documentation
|
|
79
|
+
docs/_build/
|
|
80
|
+
|
|
81
|
+
# PyBuilder
|
|
82
|
+
target/
|
|
83
|
+
|
|
84
|
+
# Jupyter Notebook
|
|
85
|
+
.ipynb_checkpoints
|
|
86
|
+
|
|
87
|
+
# pyenv
|
|
88
|
+
.python-version
|
|
89
|
+
|
|
90
|
+
# celery beat schedule file
|
|
91
|
+
celerybeat-schedule
|
|
92
|
+
|
|
93
|
+
# SageMath parsed files
|
|
94
|
+
*.sage.py
|
|
95
|
+
|
|
96
|
+
# Environments
|
|
97
|
+
.env
|
|
98
|
+
.venv
|
|
99
|
+
env/
|
|
100
|
+
venv/
|
|
101
|
+
ENV/
|
|
102
|
+
env.bak/
|
|
103
|
+
venv.bak/
|
|
104
|
+
|
|
105
|
+
# Spyder project settings
|
|
106
|
+
.spyderproject
|
|
107
|
+
.spyproject
|
|
108
|
+
|
|
109
|
+
# Rope project settings
|
|
110
|
+
.ropeproject
|
|
111
|
+
|
|
112
|
+
# mkdocs documentation
|
|
113
|
+
/site
|
|
114
|
+
|
|
115
|
+
# mypy
|
|
116
|
+
.mypy_cache/
|
|
117
|
+
|
|
118
|
+
# PyCharm
|
|
119
|
+
.idea
|
ghir-0.0.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 i2mint
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
ghir-0.0.2/PKG-INFO
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ghir
|
|
3
|
+
Version: 0.0.2
|
|
4
|
+
Summary: Search your GitHub repos (cloned or not) as an ir corpus: locations, extraction, cache, ranking.
|
|
5
|
+
Project-URL: Homepage, https://github.com/i2mint/ghir
|
|
6
|
+
Project-URL: Repository, https://github.com/i2mint/ghir
|
|
7
|
+
Project-URL: Documentation, https://i2mint.github.io/ghir
|
|
8
|
+
Author: Thor Whalen
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: github,information-retrieval,ir,repositories,search
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Requires-Dist: dol
|
|
14
|
+
Requires-Dist: ir
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
18
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
19
|
+
Provides-Extra: docs
|
|
20
|
+
Requires-Dist: sphinx-rtd-theme>=1.0; extra == 'docs'
|
|
21
|
+
Requires-Dist: sphinx>=6.0; extra == 'docs'
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# ghir
|
|
25
|
+
|
|
26
|
+
Search your GitHub repos, cloned or not, across every organization you belong to. Fetch once, then search offline; each hit says where it is cloned on this machine, if anywhere.
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install ghir
|
|
30
|
+
ghir refresh # you + your orgs, through the gh CLI
|
|
31
|
+
ghir search "flashcards spaced repetition" # offline, ranked by ir
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
0.032 thorwhalen/tetrachord /home/me/proj/tt/tetrachord
|
|
36
|
+
Music education with a learner model and scheduler at the architectural centre.
|
|
37
|
+
0.015 thorwhalen/rbxloop (not cloned)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Requires the [GitHub CLI](https://cli.github.com) (`gh auth login`) for `refresh` only; searching never touches the network.
|
|
41
|
+
|
|
42
|
+
## The four seams
|
|
43
|
+
|
|
44
|
+
ghir is `corpus -> extraction -> ranking`, with a cache beside it. Each is one keyword argument with a working default.
|
|
45
|
+
|
|
46
|
+
| Seam | Argument | Default | Alternatives |
|
|
47
|
+
|---|---|---|---|
|
|
48
|
+
| Locations: which repos | `refresh(locations)` | `"@me"`: you and every org you belong to | owner logins, `"search:<github search query>"` |
|
|
49
|
+
| Extraction: what text a repo contributes | `extract=` | `"description_and_readme"` (name, description, topics, README head) | `"description"`, or any `record -> str` callable |
|
|
50
|
+
| Ranking | `rank=` | `"ir"`: hybrid dense + BM25 through [ir](https://github.com/i2mint/ir), indexed incrementally | `"grep"` (query terms as word prefixes, no index, no model), or any callable |
|
|
51
|
+
| Cache | `store=` | one JSON file per repo under `~/.local/share/ghir/repos/` | any `MutableMapping` (a `dol` store: S3, a database) |
|
|
52
|
+
|
|
53
|
+
`refresh` merges: fetched repos overwrite their own entries, and cached repos it did not see this time are kept. A location that fails to fetch is reported in `"failed"` and its cached repos stay.
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
import ghir
|
|
57
|
+
|
|
58
|
+
ghir.refresh(["i2mint", "search:user:thorwhalen topic:education"])
|
|
59
|
+
for hit in ghir.search("audio tagging", rank="grep", include_forks=False):
|
|
60
|
+
print(hit.id, hit.local_path or hit.url)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Settings
|
|
64
|
+
|
|
65
|
+
Machine-specific choices go in `~/.config/ghir/config.json` (or `$GHIR_CONFIG`):
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{"locations": ["@me", "otosense"],
|
|
69
|
+
"exclude": ["SomeClientOrg/*", "myorg/private-thing"]}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
An owner excluded with `owner/*` is never queried. Other keys: `extract`, `local_roots` (where clones are looked for; default `$PP` or your home), `cache_dir`, `corpus_name`.
|
|
73
|
+
|
|
74
|
+
## Using it from ir
|
|
75
|
+
|
|
76
|
+
`ghir refresh` also registers the cache as the ir corpus `gh_repos` (a `records` corpus whose fetcher is `ghir:records`), so ir's own tools see it:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
ir discover gh_repos "spaced repetition"
|
|
80
|
+
ir discover gh_repos packages "audio tagging" # federated with the local packages corpus
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Why this is a separate package and not part of ir: [docs/adr/0001-where-ghir-lives.md](docs/adr/0001-where-ghir-lives.md).
|
ghir-0.0.2/README.md
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# ghir
|
|
2
|
+
|
|
3
|
+
Search your GitHub repos, cloned or not, across every organization you belong to. Fetch once, then search offline; each hit says where it is cloned on this machine, if anywhere.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install ghir
|
|
7
|
+
ghir refresh # you + your orgs, through the gh CLI
|
|
8
|
+
ghir search "flashcards spaced repetition" # offline, ranked by ir
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
0.032 thorwhalen/tetrachord /home/me/proj/tt/tetrachord
|
|
13
|
+
Music education with a learner model and scheduler at the architectural centre.
|
|
14
|
+
0.015 thorwhalen/rbxloop (not cloned)
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Requires the [GitHub CLI](https://cli.github.com) (`gh auth login`) for `refresh` only; searching never touches the network.
|
|
18
|
+
|
|
19
|
+
## The four seams
|
|
20
|
+
|
|
21
|
+
ghir is `corpus -> extraction -> ranking`, with a cache beside it. Each is one keyword argument with a working default.
|
|
22
|
+
|
|
23
|
+
| Seam | Argument | Default | Alternatives |
|
|
24
|
+
|---|---|---|---|
|
|
25
|
+
| Locations: which repos | `refresh(locations)` | `"@me"`: you and every org you belong to | owner logins, `"search:<github search query>"` |
|
|
26
|
+
| Extraction: what text a repo contributes | `extract=` | `"description_and_readme"` (name, description, topics, README head) | `"description"`, or any `record -> str` callable |
|
|
27
|
+
| Ranking | `rank=` | `"ir"`: hybrid dense + BM25 through [ir](https://github.com/i2mint/ir), indexed incrementally | `"grep"` (query terms as word prefixes, no index, no model), or any callable |
|
|
28
|
+
| Cache | `store=` | one JSON file per repo under `~/.local/share/ghir/repos/` | any `MutableMapping` (a `dol` store: S3, a database) |
|
|
29
|
+
|
|
30
|
+
`refresh` merges: fetched repos overwrite their own entries, and cached repos it did not see this time are kept. A location that fails to fetch is reported in `"failed"` and its cached repos stay.
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
import ghir
|
|
34
|
+
|
|
35
|
+
ghir.refresh(["i2mint", "search:user:thorwhalen topic:education"])
|
|
36
|
+
for hit in ghir.search("audio tagging", rank="grep", include_forks=False):
|
|
37
|
+
print(hit.id, hit.local_path or hit.url)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Settings
|
|
41
|
+
|
|
42
|
+
Machine-specific choices go in `~/.config/ghir/config.json` (or `$GHIR_CONFIG`):
|
|
43
|
+
|
|
44
|
+
```json
|
|
45
|
+
{"locations": ["@me", "otosense"],
|
|
46
|
+
"exclude": ["SomeClientOrg/*", "myorg/private-thing"]}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
An owner excluded with `owner/*` is never queried. Other keys: `extract`, `local_roots` (where clones are looked for; default `$PP` or your home), `cache_dir`, `corpus_name`.
|
|
50
|
+
|
|
51
|
+
## Using it from ir
|
|
52
|
+
|
|
53
|
+
`ghir refresh` also registers the cache as the ir corpus `gh_repos` (a `records` corpus whose fetcher is `ghir:records`), so ir's own tools see it:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
ir discover gh_repos "spaced repetition"
|
|
57
|
+
ir discover gh_repos packages "audio tagging" # federated with the local packages corpus
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Why this is a separate package and not part of ir: [docs/adr/0001-where-ghir-lives.md](docs/adr/0001-where-ghir-lives.md).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# ADR 0001: GitHub repo search is an extension package on top of ir, not part of ir
|
|
2
|
+
|
|
3
|
+
Status: accepted, 2026-10-23. Tracking: i2mint/ir#118.
|
|
4
|
+
|
|
5
|
+
## Context
|
|
6
|
+
|
|
7
|
+
The need: find something that may live in one of the user's GitHub repos, across several organizations, whether or not it is cloned locally. It was first done by hand: `gh repo list` per org to a TSV, `rg` over the descriptions, README grep over local clones, then a check of which hits were cloned. The user named the generality layers: locations (orgs, or a search query yielding repos), extraction (what each repo contributes), measuring and ranking, and caching as an orthogonal concern; above all of them, the general pattern `corpus -> extraction -> measure/filter/order/present`. They asked for the GitHub dependency to stay out of ir.
|
|
8
|
+
|
|
9
|
+
## Decision
|
|
10
|
+
|
|
11
|
+
1. **ir already is the general pattern.** `CorpusSource` (scope, change signal, indexing strategy) followed by build, `search`/`discover` is `corpus -> extraction -> ranking`. Its `records` kind takes a fetcher by import reference (`fetcher="module:attr"`), so a corpus owned by another package persists in ir's registry and rebuilds in a fresh process. So nothing new was needed in ir core: ghir supplies the fetcher, ir does indexing and ranking.
|
|
12
|
+
2. **The GitHub-specific parts live in ghir**: the locations (`@me`, owner logins, `search:<query>`), the fetch (GraphQL through the `gh` CLI, one call per page of repos with README text, not one per repo), the cache, and the local-clone index. ir never imports ghir or `gh`.
|
|
13
|
+
3. **The cache is a `MutableMapping`** (dol JSON files by default) and is **merged** on refresh, never replaced; search is offline. ir's incremental build (content-hash change signal) then re-embeds only the repos whose text changed.
|
|
14
|
+
4. **Ranking is a seam with two real strategies**: ir (default, hybrid) and grep (the by-hand method, no index, no model). Extraction is a seam with two named extractors plus any callable.
|
|
15
|
+
|
|
16
|
+
## Alternatives considered
|
|
17
|
+
|
|
18
|
+
- **Inside ir, behind an optional extra.** Rejected: ir's charter is a GitHub-agnostic substrate, and the user asked for no GitHub dependency there. A shell-out to `gh` behind an extra would still put GitHub knowledge (GraphQL queries, the README lookup, owner expansion) into ir's code and tests.
|
|
19
|
+
- **In hubcap** (the dol interface to GitHub). It is the natural home for GitHub-as-a-mapping, but it goes through PyGithub and a token, one REST call per repo; the bulk GraphQL fetch and the ir glue would sit oddly beside it. If hubcap grows a bulk repo-records store, ghir's `fetch` seam can take it.
|
|
20
|
+
- **In projreg.** projreg's gh sync and doc cache are keyed by locally registered projects; this search is precisely about repos that may not be local.
|
|
21
|
+
|
|
22
|
+
## Consequences
|
|
23
|
+
|
|
24
|
+
- `ghir refresh` writes one entry to ir's corpus registry (`gh_repos`), so `ir discover gh_repos ...` and federated `ir discover gh_repos packages ...` work.
|
|
25
|
+
- A new location kind (GitLab, a list of URLs) is a new fetcher behind the same record shape; a new ranker or extractor is a function.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Search your GitHub repos, cloned or not, as an ``ir`` corpus.
|
|
2
|
+
|
|
3
|
+
Four seams, each a keyword argument with a working default:
|
|
4
|
+
|
|
5
|
+
- **locations** (:func:`refresh`): owner logins, ``"@me"`` (you and your orgs),
|
|
6
|
+
or ``"search:<github query>"``.
|
|
7
|
+
- **extraction** (``extract=``): what text a repo contributes; default
|
|
8
|
+
description + topics + README head (:mod:`ghir.extract`).
|
|
9
|
+
- **ranking** (``rank=``): ``"ir"`` (default, hybrid dense + BM25) or ``"grep"``.
|
|
10
|
+
- **cache** (``store=``): a ``MutableMapping``; :func:`refresh` merges into it,
|
|
11
|
+
:func:`search` only reads it (offline).
|
|
12
|
+
|
|
13
|
+
>>> import ghir # doctest: +SKIP
|
|
14
|
+
>>> ghir.refresh() # doctest: +SKIP
|
|
15
|
+
>>> for hit in ghir.search("flashcards spaced repetition"): # doctest: +SKIP
|
|
16
|
+
... print(hit.id, hit.local_path)
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from .cache import refresh, repo_store
|
|
20
|
+
from .config import settings
|
|
21
|
+
from .extract import extractors
|
|
22
|
+
from .fetch import fetch_repos
|
|
23
|
+
from .local import local_clones
|
|
24
|
+
from .search import (
|
|
25
|
+
RepoHit,
|
|
26
|
+
grep_rank,
|
|
27
|
+
ir_rank,
|
|
28
|
+
rankers,
|
|
29
|
+
records,
|
|
30
|
+
register_corpus,
|
|
31
|
+
search,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"RepoHit",
|
|
36
|
+
"extractors",
|
|
37
|
+
"fetch_repos",
|
|
38
|
+
"grep_rank",
|
|
39
|
+
"ir_rank",
|
|
40
|
+
"local_clones",
|
|
41
|
+
"rankers",
|
|
42
|
+
"records",
|
|
43
|
+
"refresh",
|
|
44
|
+
"register_corpus",
|
|
45
|
+
"repo_store",
|
|
46
|
+
"search",
|
|
47
|
+
"settings",
|
|
48
|
+
]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Command line: ``ghir refresh`` (online, merges into the cache) and ``ghir search``.
|
|
2
|
+
|
|
3
|
+
.. code-block:: bash
|
|
4
|
+
|
|
5
|
+
ghir refresh # @me + config locations
|
|
6
|
+
ghir refresh i2mint "search:user:thorwhalen topic:education"
|
|
7
|
+
ghir search "flashcards spaced repetition" # offline
|
|
8
|
+
ghir search "audio tagging" --rank grep --local-only
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from .cache import refresh
|
|
18
|
+
from .search import DFLT_K, rankers, register_corpus, search
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _print_hits(hits, *, as_json: bool, local_only: bool):
|
|
22
|
+
hits = [h for h in hits if h.local_path or not local_only]
|
|
23
|
+
if as_json:
|
|
24
|
+
print(json.dumps([h.to_dict() for h in hits], indent=2))
|
|
25
|
+
return
|
|
26
|
+
for h in hits:
|
|
27
|
+
where = h.local_path or "(not cloned)"
|
|
28
|
+
flags = "".join(
|
|
29
|
+
f" [{f}]"
|
|
30
|
+
for f, on in (("fork", h.is_fork), ("archived", h.is_archived))
|
|
31
|
+
if on
|
|
32
|
+
)
|
|
33
|
+
print(f"{h.score:6.3f} {h.id}{flags} {where}")
|
|
34
|
+
if h.description:
|
|
35
|
+
print(f" {h.description}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main(argv=None):
|
|
39
|
+
p = argparse.ArgumentParser(prog="ghir", description=__doc__.splitlines()[0])
|
|
40
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
41
|
+
|
|
42
|
+
r = sub.add_parser("refresh", help="fetch repos from GitHub, merge into the cache")
|
|
43
|
+
r.add_argument("locations", nargs="*", help="owners, @me, or search:<query>")
|
|
44
|
+
r.add_argument("--no-register", action="store_true", help="skip ir registration")
|
|
45
|
+
|
|
46
|
+
s = sub.add_parser("search", help="rank cached repos for a query (offline)")
|
|
47
|
+
s.add_argument("query")
|
|
48
|
+
s.add_argument("-k", type=int, default=DFLT_K)
|
|
49
|
+
s.add_argument("--rank", choices=sorted(rankers), default="ir")
|
|
50
|
+
s.add_argument("--extract", default=None, help="extractor name")
|
|
51
|
+
s.add_argument("--no-forks", action="store_true")
|
|
52
|
+
s.add_argument("--no-archived", action="store_true")
|
|
53
|
+
s.add_argument("--local-only", action="store_true", help="only cloned repos")
|
|
54
|
+
s.add_argument("--json", action="store_true")
|
|
55
|
+
|
|
56
|
+
a = p.parse_args(argv)
|
|
57
|
+
if a.cmd == "refresh":
|
|
58
|
+
counts = refresh(a.locations or None)
|
|
59
|
+
if not a.no_register:
|
|
60
|
+
register_corpus()
|
|
61
|
+
print(json.dumps(counts))
|
|
62
|
+
return 0
|
|
63
|
+
hits = search(
|
|
64
|
+
a.query,
|
|
65
|
+
k=a.k,
|
|
66
|
+
rank=a.rank,
|
|
67
|
+
extract=a.extract,
|
|
68
|
+
include_forks=not a.no_forks,
|
|
69
|
+
include_archived=not a.no_archived,
|
|
70
|
+
)
|
|
71
|
+
_print_hits(hits, as_json=a.json, local_only=a.local_only)
|
|
72
|
+
return 0
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
if __name__ == "__main__":
|
|
76
|
+
sys.exit(main())
|
ghir-0.0.2/ghir/cache.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""The repo-record cache: a ``MutableMapping`` from ``owner/name`` to a record.
|
|
2
|
+
|
|
3
|
+
Searching never goes online; it reads this cache. :func:`refresh` is the only
|
|
4
|
+
thing that fetches, and it *merges*: fetched records overwrite their own keys,
|
|
5
|
+
records not seen this time are kept (a repo outside today's locations, or a
|
|
6
|
+
fetch that failed half-way, does not erase what was known). Each record carries
|
|
7
|
+
``fetched_at`` so staleness is visible.
|
|
8
|
+
|
|
9
|
+
The store is a seam (``store=``): the default is one JSON file per repo under
|
|
10
|
+
:data:`ghir.config.DFLT_CACHE_DIR` (a ``dol`` store, so an S3 or database
|
|
11
|
+
mapping drops in unchanged).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections.abc import Iterable, MutableMapping
|
|
17
|
+
from datetime import datetime, timezone
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from dol import JsonFiles, mk_dirs_if_missing, wrap_kvs
|
|
21
|
+
|
|
22
|
+
from .config import settings as _settings
|
|
23
|
+
from .fetch import fetch_repos
|
|
24
|
+
|
|
25
|
+
#: Repo ids contain a "/", which becomes a directory level (owner/name.json).
|
|
26
|
+
_EXT = ".json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def repo_store(cache_dir: str | Path | None = None) -> MutableMapping:
|
|
30
|
+
"""The default cache: ``{"owner/name": record}`` over JSON files."""
|
|
31
|
+
root = Path(cache_dir or _settings()["cache_dir"])
|
|
32
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
return wrap_kvs(
|
|
34
|
+
mk_dirs_if_missing(JsonFiles(str(root))),
|
|
35
|
+
id_of_key=lambda k: k + _EXT,
|
|
36
|
+
key_of_id=lambda _id: _id[: -len(_EXT)],
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def refresh(
|
|
41
|
+
locations: Iterable[str] | None = None,
|
|
42
|
+
*,
|
|
43
|
+
store: MutableMapping | None = None,
|
|
44
|
+
exclude: Iterable[str] | None = None,
|
|
45
|
+
fetch=fetch_repos,
|
|
46
|
+
**fetch_kwargs,
|
|
47
|
+
) -> dict:
|
|
48
|
+
"""Fetch the repos of ``locations`` and merge them into the cache.
|
|
49
|
+
|
|
50
|
+
Returns counts: ``{"fetched", "added", "updated", "kept"}`` where ``kept`` is
|
|
51
|
+
the number of cached records this fetch did not see (and left alone), plus
|
|
52
|
+
``"failed"``: ``{location: error}`` for locations that could not be fetched
|
|
53
|
+
(their cached records are kept).
|
|
54
|
+
"""
|
|
55
|
+
s = _settings(locations=locations, exclude=exclude)
|
|
56
|
+
store = repo_store(s["cache_dir"]) if store is None else store
|
|
57
|
+
before = set(store)
|
|
58
|
+
now = datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
59
|
+
seen, failed = set(), {}
|
|
60
|
+
fetch_kwargs.setdefault("on_error", lambda loc, e: failed.update({loc: str(e)}))
|
|
61
|
+
for rec in fetch(s["locations"], exclude=s["exclude"], **fetch_kwargs):
|
|
62
|
+
store[rec["id"]] = {**rec, "fetched_at": now}
|
|
63
|
+
seen.add(rec["id"])
|
|
64
|
+
return {
|
|
65
|
+
"fetched": len(seen),
|
|
66
|
+
"added": len(seen - before),
|
|
67
|
+
"updated": len(seen & before),
|
|
68
|
+
"kept": len(before - seen),
|
|
69
|
+
"failed": failed,
|
|
70
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Settings for ghir: where the cache lives and which repos are in scope.
|
|
2
|
+
|
|
3
|
+
All settings have working defaults and can be overridden, in increasing order of
|
|
4
|
+
precedence, by a JSON file (``~/.config/ghir/config.json``, or ``$GHIR_CONFIG``)
|
|
5
|
+
and by keyword arguments to the functions that read them. The file is the place
|
|
6
|
+
for machine-specific choices (extra owners, owners or repos never to fetch):
|
|
7
|
+
|
|
8
|
+
.. code-block:: json
|
|
9
|
+
|
|
10
|
+
{"locations": ["@me", "otosense"],
|
|
11
|
+
"exclude": ["SomeClientOrg/*", "myorg/private-notes"]}
|
|
12
|
+
|
|
13
|
+
``"@me"`` expands to the authenticated GitHub user plus every organization they
|
|
14
|
+
belong to (see :func:`ghir.fetch.my_owners`).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
from collections.abc import Mapping
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _xdg(var: str, fallback: str) -> Path:
|
|
27
|
+
return Path(os.environ.get(var) or Path.home() / fallback)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
#: Where fetched repo records are cached (one JSON file per ``owner/name``).
|
|
31
|
+
DFLT_CACHE_DIR = _xdg("XDG_DATA_HOME", ".local/share") / "ghir" / "repos"
|
|
32
|
+
|
|
33
|
+
#: The optional user settings file.
|
|
34
|
+
DFLT_CONFIG_PATH = _xdg("XDG_CONFIG_HOME", ".config") / "ghir" / "config.json"
|
|
35
|
+
|
|
36
|
+
#: The ir corpus name the default records are registered under.
|
|
37
|
+
DFLT_CORPUS_NAME = "gh_repos"
|
|
38
|
+
|
|
39
|
+
DFLT_SETTINGS: Mapping[str, Any] = {
|
|
40
|
+
# Owners (user or org logins) and/or "search:<github search query>" specs.
|
|
41
|
+
"locations": ["@me"],
|
|
42
|
+
# fnmatch patterns over "owner/name"; matching repos are never fetched.
|
|
43
|
+
"exclude": [],
|
|
44
|
+
# Name of the extractor (see ghir.extract.extractors) turning a repo into text.
|
|
45
|
+
"extract": "description_and_readme",
|
|
46
|
+
# Root directories scanned (a few levels deep) for local clones.
|
|
47
|
+
"local_roots": [os.environ.get("PP") or str(Path.home())],
|
|
48
|
+
"cache_dir": str(DFLT_CACHE_DIR),
|
|
49
|
+
"corpus_name": DFLT_CORPUS_NAME,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def settings(path: str | Path | None = None, **overrides) -> dict:
|
|
54
|
+
"""The effective settings: defaults, then the config file, then ``overrides``.
|
|
55
|
+
|
|
56
|
+
>>> s = settings(path="/nonexistent/ghir.json", exclude=["acme/*"])
|
|
57
|
+
>>> s["exclude"], s["locations"]
|
|
58
|
+
(['acme/*'], ['@me'])
|
|
59
|
+
"""
|
|
60
|
+
path = Path(path or os.environ.get("GHIR_CONFIG") or DFLT_CONFIG_PATH)
|
|
61
|
+
from_file = json.loads(path.read_text()) if path.is_file() else {}
|
|
62
|
+
unknown = set(from_file) - set(DFLT_SETTINGS)
|
|
63
|
+
if unknown:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"Unknown ghir setting(s) {sorted(unknown)} in {path}; "
|
|
66
|
+
f"known: {sorted(DFLT_SETTINGS)}"
|
|
67
|
+
)
|
|
68
|
+
given = {k: v for k, v in overrides.items() if v is not None}
|
|
69
|
+
return {**DFLT_SETTINGS, **from_file, **given}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Extraction: what text each cached repo contributes to the search.
|
|
2
|
+
|
|
3
|
+
An extractor is a function ``record -> str``. :data:`extractors` names the
|
|
4
|
+
built-in ones; any callable works wherever an extractor is accepted. The text
|
|
5
|
+
returned is what gets indexed and ranked, so it decides what a query can match.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable, Mapping
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
Extractor = Callable[[Mapping[str, Any]], str]
|
|
14
|
+
|
|
15
|
+
#: README characters kept by the default extractor (READMEs can be huge, and
|
|
16
|
+
#: their opening says what the repo is).
|
|
17
|
+
DFLT_README_CHARS = 6000
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _head(rec: Mapping[str, Any]) -> list[str]:
|
|
21
|
+
topics = " ".join(rec.get("topics") or ())
|
|
22
|
+
return [
|
|
23
|
+
rec.get("name", ""),
|
|
24
|
+
rec.get("description", ""),
|
|
25
|
+
f"topics: {topics}" if topics else "",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def description(rec: Mapping[str, Any]) -> str:
|
|
30
|
+
"""Name, description and topics.
|
|
31
|
+
|
|
32
|
+
>>> description({"name": "learny", "description": "Learner models",
|
|
33
|
+
... "topics": ["education"]})
|
|
34
|
+
'learny\\nLearner models\\ntopics: education'
|
|
35
|
+
"""
|
|
36
|
+
return "\n".join(p for p in _head(rec) if p)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def description_and_readme(
|
|
40
|
+
rec: Mapping[str, Any], *, readme_chars: int = DFLT_README_CHARS
|
|
41
|
+
) -> str:
|
|
42
|
+
"""Name, description, topics, and the start of the README (the default)."""
|
|
43
|
+
readme = (rec.get("readme") or "")[:readme_chars]
|
|
44
|
+
return "\n".join(p for p in [*_head(rec), readme] if p)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
extractors: dict[str, Extractor] = {
|
|
48
|
+
"description": description,
|
|
49
|
+
"description_and_readme": description_and_readme,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve_extractor(extract: str | Extractor) -> Extractor:
|
|
54
|
+
"""An extractor from its name in :data:`extractors`, or a callable as is."""
|
|
55
|
+
if callable(extract):
|
|
56
|
+
return extract
|
|
57
|
+
try:
|
|
58
|
+
return extractors[extract]
|
|
59
|
+
except KeyError:
|
|
60
|
+
raise ValueError(
|
|
61
|
+
f"Unknown extractor {extract!r}; use a callable or one of "
|
|
62
|
+
f"{sorted(extractors)}"
|
|
63
|
+
) from None
|