lupaxa-github-extractor 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,34 @@
1
+ """Run one function across many items on a bounded thread pool."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable, Sequence
6
+ import concurrent.futures
7
+ from typing import TypeVar
8
+
9
+ T = TypeVar("T")
10
+
11
+
12
+ def run_workers(items: Sequence[T], worker: Callable[[T], None], max_workers: int) -> None:
13
+ """Run worker on each item, a few at a time, and cancel the rest on failure."""
14
+ if not items:
15
+ return
16
+ worker_count = min(max_workers, len(items))
17
+ executor = concurrent.futures.ThreadPoolExecutor(max_workers=worker_count)
18
+ item_iterator = iter(items)
19
+ pending = {executor.submit(worker, item) for item in (next(item_iterator, None) for _ in range(worker_count)) if item is not None}
20
+ try:
21
+ while pending:
22
+ done, pending = concurrent.futures.wait(pending, return_when=concurrent.futures.FIRST_COMPLETED)
23
+ for future in done:
24
+ future.result()
25
+ for _ in done:
26
+ item = next(item_iterator, None)
27
+ if item is not None:
28
+ pending.add(executor.submit(worker, item))
29
+ except Exception:
30
+ for future in pending:
31
+ future.cancel()
32
+ executor.shutdown(wait=True, cancel_futures=True)
33
+ raise
34
+ executor.shutdown(wait=True)
@@ -0,0 +1,66 @@
1
+ """Optional GitHub resource names and the paths that fetch them."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Sequence
6
+
7
+ _REPOSITORY_ADDONS = (
8
+ ("branches", "branches", None),
9
+ ("contributors", "contributors", None),
10
+ ("languages", "languages", None),
11
+ ("releases", "releases", None),
12
+ ("tags", "tags", None),
13
+ ("topics", "topics", None),
14
+ ("workflows", "actions/workflows", "workflows"),
15
+ ("commits", "commits", None),
16
+ ("issues", "issues", None),
17
+ ("pulls", "pulls", None),
18
+ ("milestones", "milestones", None),
19
+ ("forks", "forks", None),
20
+ ("stargazers", "stargazers", None),
21
+ ("subscribers", "subscribers", None),
22
+ ("collaborators", "collaborators", None),
23
+ ("labels", "labels", None),
24
+ ("workflow_runs", "actions/runs", "workflow_runs"),
25
+ ("deployments", "deployments", None),
26
+ ("environments", "environments", "environments"),
27
+ ("contents", "contents", None),
28
+ )
29
+ REPOSITORY_RESOURCES = tuple(name for name, _suffix, _merge_key in _REPOSITORY_ADDONS)
30
+ ORGANISATION_RESOURCES = ("members", "teams", "events", "hooks", "audit_log")
31
+ _USER_EXTRA_PATHS = {
32
+ "emails": "/user/emails",
33
+ "followers": "/user/followers",
34
+ "following": "/user/following",
35
+ "gpg_keys": "/user/gpg_keys",
36
+ "ssh_keys": "/user/keys",
37
+ "repos": "/user/repos",
38
+ "starred": "/user/starred",
39
+ "subscriptions": "/user/subscriptions",
40
+ "notifications": "/notifications",
41
+ }
42
+ USER_RESOURCES = tuple(_USER_EXTRA_PATHS)
43
+
44
+
45
+ def normalize_resources(resources: Sequence[str], allowed: tuple[str, ...]) -> list[str]:
46
+ """Keep the first copy of each allowed resource name."""
47
+ if isinstance(resources, str):
48
+ raise ValueError("resources must be a list of names")
49
+ chosen: list[str] = []
50
+ for name in resources:
51
+ if name not in allowed:
52
+ raise ValueError(f"unknown resource: {name}")
53
+ if name not in chosen:
54
+ chosen.append(name)
55
+ return chosen
56
+
57
+
58
+ def repository_addons(names: Sequence[str]) -> list[tuple[str, str, str | None]]:
59
+ """Turn resource names into storage key, path suffix, and merge key."""
60
+ paths = {name: (suffix, merge_key) for name, suffix, merge_key in _REPOSITORY_ADDONS}
61
+ return [(name, paths[name][0], paths[name][1]) for name in names]
62
+
63
+
64
+ def user_resource_path(name: str) -> str:
65
+ """Return the API path for one authenticated-user resource."""
66
+ return _USER_EXTRA_PATHS[name]
@@ -0,0 +1,10 @@
1
+ """Package version."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __version__ = "0.1.0"
6
+
7
+
8
+ def get_version() -> str:
9
+ """Return the package version string."""
10
+ return __version__
@@ -0,0 +1,175 @@
1
+ Metadata-Version: 2.4
2
+ Name: lupaxa-github-extractor
3
+ Version: 0.1.0
4
+ Summary: List GitHub organisations, repositories, and related data.
5
+ Author: The Lupaxa Project
6
+ Maintainer: The Lupaxa Project
7
+ License-Expression: MIT
8
+ Project-URL: Homepage, https://github.com/lupaxa-api-extractor-toolbox/github-extractor
9
+ Project-URL: Repository, https://github.com/lupaxa-api-extractor-toolbox/github-extractor
10
+ Project-URL: Issues, https://github.com/lupaxa-api-extractor-toolbox/github-extractor/issues
11
+ Keywords: github,extractor,api
12
+ Classifier: Development Status :: 2 - Pre-Alpha
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Natural Language :: English
16
+ Classifier: Operating System :: MacOS
17
+ Classifier: Operating System :: Microsoft :: Windows
18
+ Classifier: Operating System :: POSIX
19
+ Classifier: Operating System :: POSIX :: Linux
20
+ Classifier: Programming Language :: Python
21
+ Classifier: Programming Language :: Python :: 3
22
+ Classifier: Programming Language :: Python :: 3 :: Only
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Classifier: Programming Language :: Python :: 3.13
26
+ Classifier: Programming Language :: Python :: 3.14
27
+ Classifier: Topic :: Software Development
28
+ Requires-Python: >=3.11
29
+ Description-Content-Type: text/markdown
30
+ License-File: LICENCE
31
+ Requires-Dist: requests<3.0.0,>=2.32.0
32
+ Provides-Extra: dev
33
+ Requires-Dist: build<2.0.0,>=1.2.2; extra == "dev"
34
+ Requires-Dist: mypy<2.0.0,>=1.17.0; extra == "dev"
35
+ Requires-Dist: pytest<9.0.0,>=8.4.0; extra == "dev"
36
+ Requires-Dist: pytest-cov<7.0.0,>=6.2.0; extra == "dev"
37
+ Requires-Dist: ruff<1.0.0,>=0.12.0; extra == "dev"
38
+ Requires-Dist: twine<7.0.0,>=6.1.0; extra == "dev"
39
+ Provides-Extra: test
40
+ Requires-Dist: pytest>=8.0; extra == "test"
41
+ Requires-Dist: pytest-cov>=5.0; extra == "test"
42
+ Dynamic: license-file
43
+
44
+ <p align="center">
45
+ <a href="https://github.com/lupaxa-api-extractor-toolbox">
46
+ <img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/organisations/api-extractor-toolbox/readme-logo.png" alt="API Extractor Toolbox" />
47
+ </a>
48
+ </p>
49
+
50
+ <h1 align="center">GitHub Extractor</h1>
51
+
52
+ Read-only access to GitHub organisations, repositories, and the related data
53
+ you ask for. It is a Python library, `lupaxa.github_extractor`, and a
54
+ `github-extractor` command. It does not create, update, or delete GitHub data.
55
+
56
+ ## Installation
57
+
58
+ ```bash
59
+ pip install lupaxa-github-extractor
60
+ ```
61
+
62
+ > **Note:** The command reads the token from `--token`, then `GITHUB_TOKEN`,
63
+ > then `gh auth token`, and passes that string into `GitHubExtractor`. The
64
+ > library does not read the environment.
65
+
66
+ ## Library
67
+
68
+ ```python
69
+ from lupaxa.github_extractor import GitHubExtractor
70
+
71
+ client = GitHubExtractor(token)
72
+ repos = client.repositories_by_org(
73
+ "acme",
74
+ resources=("topics", "workflows", "languages", "commits", "issues"),
75
+ )
76
+
77
+ for repo in repos:
78
+ names = [] if repo.topics is None else repo.topics.names
79
+ workflows = [] if repo.workflows is None else [item.name for item in repo.workflows]
80
+ print(repo.full_name, names, workflows, repo.languages)
81
+ for failure in repo.errors:
82
+ print(failure.addon, failure.status, failure.message)
83
+ ```
84
+
85
+ `resources` is one list. These names can be mixed in the same call:
86
+ `branches`, `contributors`, `languages`, `releases`, `tags`, `topics`,
87
+ `workflows`, `commits`, `issues`, `pulls`, `milestones`, `forks`,
88
+ `stargazers`, `subscribers`, `collaborators`, `labels`, `workflow_runs`,
89
+ `deployments`, `environments`, and `contents`. Organisation extras include
90
+ `audit_log`. User extras include `notifications`. A name that was not
91
+ requested is `None`.
92
+
93
+ `params` sets query strings per resource name. Each key must appear in the
94
+ `resources` list for that call. `issues` and `pulls` default to `state=all`.
95
+ `notifications` defaults to `all=true`. Caller values replace defaults.
96
+
97
+ ```python
98
+ repo = client.repository(
99
+ "acme/widget",
100
+ resources=("topics", "workflows", "workflow_runs", "issues"),
101
+ params={"issues": {"state": "closed"}},
102
+ )
103
+ user = client.authenticated_user(
104
+ resources=("repos", "notifications"),
105
+ repository_resources=("topics", "commits"),
106
+ )
107
+ ```
108
+
109
+ `repository(full_name, ...)` returns one repository. On `authenticated_user`,
110
+ `repository_resources` fetches extras on each repo in `repos`, `starred`, or
111
+ `subscriptions` when one of those lists is requested.
112
+
113
+ `topics` is one object with `names`. `workflows` is a list of workflow
114
+ objects, without `total_count`. `languages` is a dictionary of byte counts.
115
+ `forks` in that list replaces GitHub's integer fork count. Other fields from
116
+ the repository JSON stay on `repo.raw`.
117
+
118
+ A `404`, `409`, or `410` on one extra stores an empty list for that name and
119
+ records the failure on `errors`. The rest of the result is still returned.
120
+ Authentication failures, rate limits, and other `403` responses raise.
121
+
122
+ `slugs=True` stays a boolean. It returns names only and does not fetch
123
+ `resources`, even when that list is set.
124
+
125
+ ```python
126
+ names = client.repositories_by_org("acme", slugs=True)
127
+ ```
128
+
129
+ The other list methods are `user_repositories` (omit `username` for the
130
+ authenticated user), `visible_repositories`, and `organisation_repositories`.
131
+ `organisations(resources=("members", "teams", "hooks"))` lists the token's
132
+ organisations. `authenticated_user(resources=("emails", "repos"))` returns the
133
+ token's user. `get("/repos/acme/widget/commits")` returns raw JSON for any
134
+ read-only API path. `max_pages=None` follows every page. A positive integer
135
+ stops after that many pages.
136
+
137
+ ## Command Line
138
+
139
+ Flags are the command-line form of the same resource names. Order on the
140
+ command line does not matter.
141
+
142
+ ```bash
143
+ github-extractor repo acme/widget --topics --workflows --workflow-runs
144
+ github-extractor repos org acme --topics --workflows --languages --commits --issues
145
+ github-extractor repos user --username ada --releases --tags --pulls
146
+ github-extractor repos visible --skip-private --include 'acme/*' --topics
147
+ github-extractor repos organisations --include-org 'acme*'
148
+ github-extractor repos organizations --include-org 'acme*'
149
+ github-extractor organisations --members --teams --hooks
150
+ github-extractor user --repos --topics --notifications
151
+ github-extractor repos org acme --issues --param issues.state=closed --api-root https://github.example/api/v3
152
+ github-extractor repos org acme --slugs
153
+ github-extractor get /repos/acme/widget/commits --max-pages 2
154
+ ```
155
+
156
+ `--include` and `--ignore` are case-insensitive globs on `full_name`.
157
+ `--include-name` and `--ignore-name` match the repository name.
158
+ `--include-org` and `--ignore-org` limit `repos organisations` and
159
+ `repos organizations`. Repeat a flag to pass more than one glob. `--param`
160
+ sets `name.key=value` query parameters. `--api-root` overrides the API host.
161
+ `--max-pages` caps paging. `--timeout` and `--max-workers` default to 10
162
+ seconds and 8 workers.
163
+
164
+ The command prints JSON indented with two spaces. It exits `0` on success,
165
+ `2` for usage and a missing token, `3` for authentication, `4` for a rate
166
+ limit, `5` when a primary request is not found, and `1` for everything else.
167
+
168
+ ## Documentation
169
+
170
+ Longer walks and the full method list are in the
171
+ [documentation](https://github-extractor.thelupaxaproject.org/).
172
+
173
+ <a href="https://github.com/the-lupaxa-project">
174
+ <img src="https://raw.githubusercontent.com/the-lupaxa-project/brand-assets/master/logos/components/footer-for-child-orgs.svg" alt="The Lupaxa Project Footer" width="100%" />
175
+ </a>
@@ -0,0 +1,19 @@
1
+ lupaxa/github_extractor/__init__.py,sha256=ZA3Wwsj8d-dN1lKPV0TOfq4dE4c1Rj8pK3pS4aqthtU,1692
2
+ lupaxa/github_extractor/__main__.py,sha256=1QS1JddO-aaGpyEkGF-1Vj_pKLjM7GGV7grmtNJ0FhQ,160
3
+ lupaxa/github_extractor/cli.py,sha256=I25dIMeVJRmzERObHRHQqJu3mrUxPFD2GblT_n2ZtLQ,14097
4
+ lupaxa/github_extractor/client.py,sha256=tRU_CbQiXFXIYmkBilHbU_BUj4jON6j2oYpKJwfTkw8,19518
5
+ lupaxa/github_extractor/constants.py,sha256=EzoEtORyfyB-Lp-WwO4ySTTaJ7FbU67jS5AzG7WVjXM,274
6
+ lupaxa/github_extractor/convert.py,sha256=_INnp1OOZpkCQbldGMq2YFJjHl8QRD85X8e6LmtlIV0,6158
7
+ lupaxa/github_extractor/errors.py,sha256=bDMTHPzSyviTC6lPZYEyUGo3VWMUUZ5KNssspAUf6ls,2425
8
+ lupaxa/github_extractor/filters.py,sha256=CTBASqkCKdbqZZfkRVBzi8ep-gdyRdSM6AmE1lzDAZk,1753
9
+ lupaxa/github_extractor/http.py,sha256=R0WBcoqS2OXs3JoJY7bWuPJU5k2lOqlcrZZld9fE8Vs,11973
10
+ lupaxa/github_extractor/models.py,sha256=4nWi1GJhSVFiIBC65T02kb4c2E6VLfxAwArOAa6HTno,8252
11
+ lupaxa/github_extractor/pool.py,sha256=CL7XVYPN1qjNmwfKVtvaLg5L7foMvh5CnTNLuV0RNgA,1294
12
+ lupaxa/github_extractor/resources.py,sha256=7IhmsLS0ACa4v978mNImNT_RDJgTK6uG5NcnOtdqht8,2430
13
+ lupaxa/github_extractor/version.py,sha256=B2rRbKnD0xNTsOvPfSMqRD_CgNgwAQMzIcj4k0qKCM4,178
14
+ lupaxa_github_extractor-0.1.0.dist-info/licenses/LICENCE,sha256=RXShyrlT7e281QX54JLm3YrhOlC-TJxObadJl1gH5Es,1070
15
+ lupaxa_github_extractor-0.1.0.dist-info/METADATA,sha256=2e0_0KJxcrgMvBYuS78ASL3NXhJ3C15vZ1dmvkUhoIw,7456
16
+ lupaxa_github_extractor-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
17
+ lupaxa_github_extractor-0.1.0.dist-info/entry_points.txt,sha256=RI94e2e76j8_tmncaTxQu9jig3NqOZhL_hf3nCDlWO4,70
18
+ lupaxa_github_extractor-0.1.0.dist-info/top_level.txt,sha256=enJZ4xGyTomypf_yq2pEZRa_69hF0ulc9sWjyYQV9I8,7
19
+ lupaxa_github_extractor-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ github-extractor = lupaxa.github_extractor.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) The Lupaxa Project
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.