lupaxa-github-extractor 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lupaxa/github_extractor/__init__.py +98 -0
- lupaxa/github_extractor/__main__.py +10 -0
- lupaxa/github_extractor/cli.py +383 -0
- lupaxa/github_extractor/client.py +390 -0
- lupaxa/github_extractor/constants.py +10 -0
- lupaxa/github_extractor/convert.py +182 -0
- lupaxa/github_extractor/errors.py +76 -0
- lupaxa/github_extractor/filters.py +47 -0
- lupaxa/github_extractor/http.py +289 -0
- lupaxa/github_extractor/models.py +362 -0
- lupaxa/github_extractor/pool.py +34 -0
- lupaxa/github_extractor/resources.py +66 -0
- lupaxa/github_extractor/version.py +10 -0
- lupaxa_github_extractor-0.1.0.dist-info/METADATA +175 -0
- lupaxa_github_extractor-0.1.0.dist-info/RECORD +19 -0
- lupaxa_github_extractor-0.1.0.dist-info/WHEEL +5 -0
- lupaxa_github_extractor-0.1.0.dist-info/entry_points.txt +2 -0
- lupaxa_github_extractor-0.1.0.dist-info/licenses/LICENCE +21 -0
- lupaxa_github_extractor-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Run one function across many items on a bounded thread pool."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Sequence
|
|
6
|
+
import concurrent.futures
|
|
7
|
+
from typing import TypeVar
|
|
8
|
+
|
|
9
|
+
T = TypeVar("T")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def run_workers(items: Sequence[T], worker: Callable[[T], None], max_workers: int) -> None:
|
|
13
|
+
"""Run worker on each item, a few at a time, and cancel the rest on failure."""
|
|
14
|
+
if not items:
|
|
15
|
+
return
|
|
16
|
+
worker_count = min(max_workers, len(items))
|
|
17
|
+
executor = concurrent.futures.ThreadPoolExecutor(max_workers=worker_count)
|
|
18
|
+
item_iterator = iter(items)
|
|
19
|
+
pending = {executor.submit(worker, item) for item in (next(item_iterator, None) for _ in range(worker_count)) if item is not None}
|
|
20
|
+
try:
|
|
21
|
+
while pending:
|
|
22
|
+
done, pending = concurrent.futures.wait(pending, return_when=concurrent.futures.FIRST_COMPLETED)
|
|
23
|
+
for future in done:
|
|
24
|
+
future.result()
|
|
25
|
+
for _ in done:
|
|
26
|
+
item = next(item_iterator, None)
|
|
27
|
+
if item is not None:
|
|
28
|
+
pending.add(executor.submit(worker, item))
|
|
29
|
+
except Exception:
|
|
30
|
+
for future in pending:
|
|
31
|
+
future.cancel()
|
|
32
|
+
executor.shutdown(wait=True, cancel_futures=True)
|
|
33
|
+
raise
|
|
34
|
+
executor.shutdown(wait=True)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Optional GitHub resource names and the paths that fetch them."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
|
|
7
|
+
_REPOSITORY_ADDONS = (
|
|
8
|
+
("branches", "branches", None),
|
|
9
|
+
("contributors", "contributors", None),
|
|
10
|
+
("languages", "languages", None),
|
|
11
|
+
("releases", "releases", None),
|
|
12
|
+
("tags", "tags", None),
|
|
13
|
+
("topics", "topics", None),
|
|
14
|
+
("workflows", "actions/workflows", "workflows"),
|
|
15
|
+
("commits", "commits", None),
|
|
16
|
+
("issues", "issues", None),
|
|
17
|
+
("pulls", "pulls", None),
|
|
18
|
+
("milestones", "milestones", None),
|
|
19
|
+
("forks", "forks", None),
|
|
20
|
+
("stargazers", "stargazers", None),
|
|
21
|
+
("subscribers", "subscribers", None),
|
|
22
|
+
("collaborators", "collaborators", None),
|
|
23
|
+
("labels", "labels", None),
|
|
24
|
+
("workflow_runs", "actions/runs", "workflow_runs"),
|
|
25
|
+
("deployments", "deployments", None),
|
|
26
|
+
("environments", "environments", "environments"),
|
|
27
|
+
("contents", "contents", None),
|
|
28
|
+
)
|
|
29
|
+
REPOSITORY_RESOURCES = tuple(name for name, _suffix, _merge_key in _REPOSITORY_ADDONS)
|
|
30
|
+
ORGANISATION_RESOURCES = ("members", "teams", "events", "hooks", "audit_log")
|
|
31
|
+
_USER_EXTRA_PATHS = {
|
|
32
|
+
"emails": "/user/emails",
|
|
33
|
+
"followers": "/user/followers",
|
|
34
|
+
"following": "/user/following",
|
|
35
|
+
"gpg_keys": "/user/gpg_keys",
|
|
36
|
+
"ssh_keys": "/user/keys",
|
|
37
|
+
"repos": "/user/repos",
|
|
38
|
+
"starred": "/user/starred",
|
|
39
|
+
"subscriptions": "/user/subscriptions",
|
|
40
|
+
"notifications": "/notifications",
|
|
41
|
+
}
|
|
42
|
+
USER_RESOURCES = tuple(_USER_EXTRA_PATHS)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def normalize_resources(resources: Sequence[str], allowed: tuple[str, ...]) -> list[str]:
|
|
46
|
+
"""Keep the first copy of each allowed resource name."""
|
|
47
|
+
if isinstance(resources, str):
|
|
48
|
+
raise ValueError("resources must be a list of names")
|
|
49
|
+
chosen: list[str] = []
|
|
50
|
+
for name in resources:
|
|
51
|
+
if name not in allowed:
|
|
52
|
+
raise ValueError(f"unknown resource: {name}")
|
|
53
|
+
if name not in chosen:
|
|
54
|
+
chosen.append(name)
|
|
55
|
+
return chosen
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def repository_addons(names: Sequence[str]) -> list[tuple[str, str, str | None]]:
|
|
59
|
+
"""Turn resource names into storage key, path suffix, and merge key."""
|
|
60
|
+
paths = {name: (suffix, merge_key) for name, suffix, merge_key in _REPOSITORY_ADDONS}
|
|
61
|
+
return [(name, paths[name][0], paths[name][1]) for name in names]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def user_resource_path(name: str) -> str:
|
|
65
|
+
"""Return the API path for one authenticated-user resource."""
|
|
66
|
+
return _USER_EXTRA_PATHS[name]
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lupaxa-github-extractor
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: List GitHub organisations, repositories, and related data.
|
|
5
|
+
Author: The Lupaxa Project
|
|
6
|
+
Maintainer: The Lupaxa Project
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/lupaxa-api-extractor-toolbox/github-extractor
|
|
9
|
+
Project-URL: Repository, https://github.com/lupaxa-api-extractor-toolbox/github-extractor
|
|
10
|
+
Project-URL: Issues, https://github.com/lupaxa-api-extractor-toolbox/github-extractor/issues
|
|
11
|
+
Keywords: github,extractor,api
|
|
12
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Natural Language :: English
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
18
|
+
Classifier: Operating System :: POSIX
|
|
19
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
20
|
+
Classifier: Programming Language :: Python
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
27
|
+
Classifier: Topic :: Software Development
|
|
28
|
+
Requires-Python: >=3.11
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
License-File: LICENCE
|
|
31
|
+
Requires-Dist: requests<3.0.0,>=2.32.0
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: build<2.0.0,>=1.2.2; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy<2.0.0,>=1.17.0; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest<9.0.0,>=8.4.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov<7.0.0,>=6.2.0; extra == "dev"
|
|
37
|
+
Requires-Dist: ruff<1.0.0,>=0.12.0; extra == "dev"
|
|
38
|
+
Requires-Dist: twine<7.0.0,>=6.1.0; extra == "dev"
|
|
39
|
+
Provides-Extra: test
|
|
40
|
+
Requires-Dist: pytest>=8.0; extra == "test"
|
|
41
|
+
Requires-Dist: pytest-cov>=5.0; extra == "test"
|
|
42
|
+
Dynamic: license-file
|
|
43
|
+
|
|
44
|
+
<p align="center">
|
|
45
|
+
<a href="https://github.com/lupaxa-api-extractor-toolbox">
|
|
46
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/organisations/api-extractor-toolbox/readme-logo.png" alt="API Extractor Toolbox" />
|
|
47
|
+
</a>
|
|
48
|
+
</p>
|
|
49
|
+
|
|
50
|
+
<h1 align="center">GitHub Extractor</h1>
|
|
51
|
+
|
|
52
|
+
Read-only access to GitHub organisations, repositories, and the related data
|
|
53
|
+
you ask for. It is a Python library, `lupaxa.github_extractor`, and a
|
|
54
|
+
`github-extractor` command. It does not create, update, or delete GitHub data.
|
|
55
|
+
|
|
56
|
+
## Installation
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install lupaxa-github-extractor
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
> **Note:** The command reads the token from `--token`, then `GITHUB_TOKEN`,
|
|
63
|
+
> then `gh auth token`, and passes that string into `GitHubExtractor`. The
|
|
64
|
+
> library does not read the environment.
|
|
65
|
+
|
|
66
|
+
## Library
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from lupaxa.github_extractor import GitHubExtractor
|
|
70
|
+
|
|
71
|
+
client = GitHubExtractor(token)
|
|
72
|
+
repos = client.repositories_by_org(
|
|
73
|
+
"acme",
|
|
74
|
+
resources=("topics", "workflows", "languages", "commits", "issues"),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
for repo in repos:
|
|
78
|
+
names = [] if repo.topics is None else repo.topics.names
|
|
79
|
+
workflows = [] if repo.workflows is None else [item.name for item in repo.workflows]
|
|
80
|
+
print(repo.full_name, names, workflows, repo.languages)
|
|
81
|
+
for failure in repo.errors:
|
|
82
|
+
print(failure.addon, failure.status, failure.message)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`resources` is one list. These names can be mixed in the same call:
|
|
86
|
+
`branches`, `contributors`, `languages`, `releases`, `tags`, `topics`,
|
|
87
|
+
`workflows`, `commits`, `issues`, `pulls`, `milestones`, `forks`,
|
|
88
|
+
`stargazers`, `subscribers`, `collaborators`, `labels`, `workflow_runs`,
|
|
89
|
+
`deployments`, `environments`, and `contents`. Organisation extras include
|
|
90
|
+
`audit_log`. User extras include `notifications`. A name that was not
|
|
91
|
+
requested is `None`.
|
|
92
|
+
|
|
93
|
+
`params` sets query strings per resource name. Each key must appear in the
|
|
94
|
+
`resources` list for that call. `issues` and `pulls` default to `state=all`.
|
|
95
|
+
`notifications` defaults to `all=true`. Caller values replace defaults.
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
repo = client.repository(
|
|
99
|
+
"acme/widget",
|
|
100
|
+
resources=("topics", "workflows", "workflow_runs", "issues"),
|
|
101
|
+
params={"issues": {"state": "closed"}},
|
|
102
|
+
)
|
|
103
|
+
user = client.authenticated_user(
|
|
104
|
+
resources=("repos", "notifications"),
|
|
105
|
+
repository_resources=("topics", "commits"),
|
|
106
|
+
)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`repository(full_name, ...)` returns one repository. On `authenticated_user`,
|
|
110
|
+
`repository_resources` fetches extras on each repo in `repos`, `starred`, or
|
|
111
|
+
`subscriptions` when one of those lists is requested.
|
|
112
|
+
|
|
113
|
+
`topics` is one object with `names`. `workflows` is a list of workflow
|
|
114
|
+
objects, without `total_count`. `languages` is a dictionary of byte counts.
|
|
115
|
+
`forks` in that list replaces GitHub's integer fork count. Other fields from
|
|
116
|
+
the repository JSON stay on `repo.raw`.
|
|
117
|
+
|
|
118
|
+
A `404`, `409`, or `410` on one extra stores an empty list for that name and
|
|
119
|
+
records the failure on `errors`. The rest of the result is still returned.
|
|
120
|
+
Authentication failures, rate limits, and other `403` responses raise.
|
|
121
|
+
|
|
122
|
+
`slugs=True` stays a boolean. It returns names only and does not fetch
|
|
123
|
+
`resources`, even when that list is set.
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
names = client.repositories_by_org("acme", slugs=True)
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
The other list methods are `user_repositories` (omit `username` for the
|
|
130
|
+
authenticated user), `visible_repositories`, and `organisation_repositories`.
|
|
131
|
+
`organisations(resources=("members", "teams", "hooks"))` lists the token's
|
|
132
|
+
organisations. `authenticated_user(resources=("emails", "repos"))` returns the
|
|
133
|
+
token's user. `get("/repos/acme/widget/commits")` returns raw JSON for any
|
|
134
|
+
read-only API path. `max_pages=None` follows every page. A positive integer
|
|
135
|
+
stops after that many pages.
|
|
136
|
+
|
|
137
|
+
## Command Line
|
|
138
|
+
|
|
139
|
+
Flags are the command-line form of the same resource names. Order on the
|
|
140
|
+
command line does not matter.
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
github-extractor repo acme/widget --topics --workflows --workflow-runs
|
|
144
|
+
github-extractor repos org acme --topics --workflows --languages --commits --issues
|
|
145
|
+
github-extractor repos user --username ada --releases --tags --pulls
|
|
146
|
+
github-extractor repos visible --skip-private --include 'acme/*' --topics
|
|
147
|
+
github-extractor repos organisations --include-org 'acme*'
|
|
148
|
+
github-extractor repos organizations --include-org 'acme*'
|
|
149
|
+
github-extractor organisations --members --teams --hooks
|
|
150
|
+
github-extractor user --repos --topics --notifications
|
|
151
|
+
github-extractor repos org acme --issues --param issues.state=closed --api-root https://github.example/api/v3
|
|
152
|
+
github-extractor repos org acme --slugs
|
|
153
|
+
github-extractor get /repos/acme/widget/commits --max-pages 2
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
`--include` and `--ignore` are case-insensitive globs on `full_name`.
|
|
157
|
+
`--include-name` and `--ignore-name` match the repository name.
|
|
158
|
+
`--include-org` and `--ignore-org` limit `repos organisations` and
|
|
159
|
+
`repos organizations`. Repeat a flag to pass more than one glob. `--param`
|
|
160
|
+
sets `name.key=value` query parameters. `--api-root` overrides the API host.
|
|
161
|
+
`--max-pages` caps paging. `--timeout` and `--max-workers` default to 10
|
|
162
|
+
seconds and 8 workers.
|
|
163
|
+
|
|
164
|
+
The command prints JSON indented with two spaces. It exits `0` on success,
|
|
165
|
+
`2` for usage and a missing token, `3` for authentication, `4` for a rate
|
|
166
|
+
limit, `5` when a primary request is not found, and `1` for everything else.
|
|
167
|
+
|
|
168
|
+
## Documentation
|
|
169
|
+
|
|
170
|
+
Longer walks and the full method list are in the
|
|
171
|
+
[documentation](https://github-extractor.thelupaxaproject.org/).
|
|
172
|
+
|
|
173
|
+
<a href="https://github.com/the-lupaxa-project">
|
|
174
|
+
<img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/components/footer-for-child-orgs.svg" alt="The Lupaxa Project Footer" width="100%" />
|
|
175
|
+
</a>
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
lupaxa/github_extractor/__init__.py,sha256=ZA3Wwsj8d-dN1lKPV0TOfq4dE4c1Rj8pK3pS4aqthtU,1692
|
|
2
|
+
lupaxa/github_extractor/__main__.py,sha256=1QS1JddO-aaGpyEkGF-1Vj_pKLjM7GGV7grmtNJ0FhQ,160
|
|
3
|
+
lupaxa/github_extractor/cli.py,sha256=I25dIMeVJRmzERObHRHQqJu3mrUxPFD2GblT_n2ZtLQ,14097
|
|
4
|
+
lupaxa/github_extractor/client.py,sha256=tRU_CbQiXFXIYmkBilHbU_BUj4jON6j2oYpKJwfTkw8,19518
|
|
5
|
+
lupaxa/github_extractor/constants.py,sha256=EzoEtORyfyB-Lp-WwO4ySTTaJ7FbU67jS5AzG7WVjXM,274
|
|
6
|
+
lupaxa/github_extractor/convert.py,sha256=_INnp1OOZpkCQbldGMq2YFJjHl8QRD85X8e6LmtlIV0,6158
|
|
7
|
+
lupaxa/github_extractor/errors.py,sha256=bDMTHPzSyviTC6lPZYEyUGo3VWMUUZ5KNssspAUf6ls,2425
|
|
8
|
+
lupaxa/github_extractor/filters.py,sha256=CTBASqkCKdbqZZfkRVBzi8ep-gdyRdSM6AmE1lzDAZk,1753
|
|
9
|
+
lupaxa/github_extractor/http.py,sha256=R0WBcoqS2OXs3JoJY7bWuPJU5k2lOqlcrZZld9fE8Vs,11973
|
|
10
|
+
lupaxa/github_extractor/models.py,sha256=4nWi1GJhSVFiIBC65T02kb4c2E6VLfxAwArOAa6HTno,8252
|
|
11
|
+
lupaxa/github_extractor/pool.py,sha256=CL7XVYPN1qjNmwfKVtvaLg5L7foMvh5CnTNLuV0RNgA,1294
|
|
12
|
+
lupaxa/github_extractor/resources.py,sha256=7IhmsLS0ACa4v978mNImNT_RDJgTK6uG5NcnOtdqht8,2430
|
|
13
|
+
lupaxa/github_extractor/version.py,sha256=B2rRbKnD0xNTsOvPfSMqRD_CgNgwAQMzIcj4k0qKCM4,178
|
|
14
|
+
lupaxa_github_extractor-0.1.0.dist-info/licenses/LICENCE,sha256=RXShyrlT7e281QX54JLm3YrhOlC-TJxObadJl1gH5Es,1070
|
|
15
|
+
lupaxa_github_extractor-0.1.0.dist-info/METADATA,sha256=2e0_0KJxcrgMvBYuS78ASL3NXhJ3C15vZ1dmvkUhoIw,7456
|
|
16
|
+
lupaxa_github_extractor-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
17
|
+
lupaxa_github_extractor-0.1.0.dist-info/entry_points.txt,sha256=RI94e2e76j8_tmncaTxQu9jig3NqOZhL_hf3nCDlWO4,70
|
|
18
|
+
lupaxa_github_extractor-0.1.0.dist-info/top_level.txt,sha256=enJZ4xGyTomypf_yq2pEZRa_69hF0ulc9sWjyYQV9I8,7
|
|
19
|
+
lupaxa_github_extractor-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) The Lupaxa Project
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
lupaxa
|