compforge-repokit 0.15.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- compforge_repokit-0.15.0.dist-info/METADATA +114 -0
- compforge_repokit-0.15.0.dist-info/RECORD +37 -0
- compforge_repokit-0.15.0.dist-info/WHEEL +4 -0
- repokit/__init__.py +40 -0
- repokit/_git.py +177 -0
- repokit/_inspect.py +70 -0
- repokit/_language.py +23 -0
- repokit/_layout.py +187 -0
- repokit/_process.py +135 -0
- repokit/_staging.py +59 -0
- repokit/budget.py +40 -0
- repokit/dependencies.py +378 -0
- repokit/forge/__init__.py +13 -0
- repokit/forge/_http_state.py +92 -0
- repokit/forge/_page.py +34 -0
- repokit/forge/_rest.py +233 -0
- repokit/forge/creation.py +88 -0
- repokit/forge/github.py +440 -0
- repokit/forge/gitlab.py +407 -0
- repokit/forge/model.py +380 -0
- repokit/forge/observation/__init__.py +9 -0
- repokit/forge/observation/context.py +255 -0
- repokit/forge/observation/model.py +40 -0
- repokit/forge/observation/service.py +283 -0
- repokit/forge/observation/store.py +136 -0
- repokit/git.py +296 -0
- repokit/git_index.py +275 -0
- repokit/git_state.py +447 -0
- repokit/languages.json +1115 -0
- repokit/model.py +59 -0
- repokit/py.typed +0 -0
- repokit/remote.py +41 -0
- repokit/repository.py +84 -0
- repokit/snapshot.py +275 -0
- repokit/stash.py +66 -0
- repokit/tree.py +191 -0
- repokit/validation.py +508 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: compforge-repokit
|
|
3
|
+
Version: 0.15.0
|
|
4
|
+
Summary: Native repository inspection, content identity, Git and Forge operations
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: harness-common<0.7,>=0.6
|
|
7
|
+
Requires-Dist: pyyaml<7,>=6
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# RepoKit for Python
|
|
11
|
+
|
|
12
|
+
Native organization inspection for Git repositories: discover Components, language and
|
|
13
|
+
package-tool evidence, then locate the owner of a repository-relative path.
|
|
14
|
+
Python 3.11+ on macOS/Linux and Git on PATH are required. Organization analysis runs
|
|
15
|
+
in Python; Git provides version-control facts through batched commands.
|
|
16
|
+
|
|
17
|
+
Install from the repository checkout:
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
python -m pip install ./toolkit/python
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
from repokit import inspect, owner
|
|
25
|
+
|
|
26
|
+
report = inspect("/path/to/repo", timeout=5)
|
|
27
|
+
if not report.complete:
|
|
28
|
+
raise RuntimeError(report.diagnostics)
|
|
29
|
+
component = owner(report, "server/main.py")
|
|
30
|
+
if component is not None:
|
|
31
|
+
print(component.name, component.root, component.language)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`inspect(..., staged=True)` selects the index; `inspect(..., head="HEAD")` selects a
|
|
35
|
+
resolved commit. The default observes the working tree, including non-ignored untracked
|
|
36
|
+
paths. `timeout` is a whole-call budget in seconds; an optional `threading.Event` passed
|
|
37
|
+
as `cancel` requests cancellation. Git processes are terminated and reaped on timeout,
|
|
38
|
+
cancellation or output overflow. Filesystem reads and Python analysis check the budget
|
|
39
|
+
between operations. Timeout raises `TimeoutError`, cancellation raises `InterruptedError`,
|
|
40
|
+
invalid selections/configuration or limits raise `ValueError`, and filesystem/Git failures
|
|
41
|
+
propagate `OSError` / `subprocess.CalledProcessError`.
|
|
42
|
+
|
|
43
|
+
Reports and their nested collections are immutable dataclasses and tuples. Shared
|
|
44
|
+
Repository, Component, Product and Forge identities come from `harness-common`;
|
|
45
|
+
ComponentBinding adds checkout layout and tool evidence. `owner` accepts a repository-relative
|
|
46
|
+
path and returns the deepest binding or None, including for deleted paths.
|
|
47
|
+
Python properties use snake_case; these objects are not a CLI JSON protocol adapter.
|
|
48
|
+
|
|
49
|
+
Inspection reads filenames and necessary metadata, not source contents or ASTs. It never
|
|
50
|
+
runs project code or installs dependencies. `complete` concerns observed organization;
|
|
51
|
+
concurrent metadata changes return an incomplete report with diagnostics. Consumers own
|
|
52
|
+
acceptance, command selection and validation policy. Snapshot, diff and graph analysis
|
|
53
|
+
remain available through the Go toolkit / CLI.
|
|
54
|
+
|
|
55
|
+
Development: `uv sync --locked`, then `make format`, `make lint`, `make test`, `make build`.
|
|
56
|
+
Tests consume the shared `conformance/inspect` cases and generated filename probes.
|
|
57
|
+
See [the inspection contract](../../docs/repository.md).
|
|
58
|
+
|
|
59
|
+
## Repository toolkit
|
|
60
|
+
|
|
61
|
+
工具包提供仓库组织、内容身份与 Git 查询。Python 还提供 worktree、提交、推送、rebase 和
|
|
62
|
+
GitHub/GitLab 的 PR/MR 操作;调用方直接导入库,组织自己的开发流程。
|
|
63
|
+
分析接口保持只读,操作接口接收明确参数,不隐式执行验证或管理任务状态。
|
|
64
|
+
|
|
65
|
+
详见 [Git 操作](../../docs/git.md)与[PR 契约](../../docs/pr.md)。
|
|
66
|
+
|
|
67
|
+
## Repository entries
|
|
68
|
+
|
|
69
|
+
`tree` lists directories, files, Git entry kinds and known roles independently of
|
|
70
|
+
Component discovery, for working-tree, index and commit inputs. Project Manifest
|
|
71
|
+
files retain their ecosystem; Component bindings retain `manifests` as path evidence.
|
|
72
|
+
Components provide the granularity for lint, test and packaging operations. Discovery prefers
|
|
73
|
+
manifests, then Makefiles with unknown language. Without either, the repository has no
|
|
74
|
+
recognized Component; no root Component or tool-specific configuration file is created.
|
|
75
|
+
See [repository contract](../../docs/repository.md) and [snapshot v2](../../docs/snapshot.md).
|
|
76
|
+
|
|
77
|
+
### Component content identities
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from repokit import component_scopes, inspect, snapshot
|
|
81
|
+
|
|
82
|
+
layout = inspect(repository)
|
|
83
|
+
contents = snapshot(repository, scopes=component_scopes(layout))
|
|
84
|
+
for component in contents.scopes:
|
|
85
|
+
print(component.name, component.digest, component.complete)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Verify layout/content consistency in the calling operation. These scopes follow file
|
|
89
|
+
ownership, not transitive build dependencies; see [Snapshot](../../docs/snapshot.md).
|
|
90
|
+
|
|
91
|
+
## Makefile validation
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from repokit import plan_validation, execute_validation
|
|
95
|
+
|
|
96
|
+
plan = plan_validation(repo, components=("backend",), checks=("lint", "test"))
|
|
97
|
+
report = execute_validation(plan, timeout=600)
|
|
98
|
+
# Use report.complete and report.passed together, not an isolated command exit code.
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Planning never executes Make. Targets and optional LINT_FILES/TEST_FILES filtering follow
|
|
102
|
+
[the shared Makefile contract](../../docs/validation.md). Prepare dependencies and run any
|
|
103
|
+
formatter before planning. Execution freezes the environment, bounds subprocesses, and
|
|
104
|
+
checks repository contents once before and once after the round. Reports contain execution
|
|
105
|
+
facts; callers own persistence, reuse and workflow gates.
|
|
106
|
+
|
|
107
|
+
PR lifecycle queries are native Python: `repokit.forge.PullRequestObserver` owns durable observations,
|
|
108
|
+
terminal-state convergence, retries and optional periodic maintenance. `PullRequestContext` relates PRs
|
|
109
|
+
to live local branches and checkouts without choosing workflow or cleanup policy. See [PR contracts](../../docs/pr.md).
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
PR 创建和复用使用 `repokit.forge.ensure_pull_request`,无需调用方解析目标仓或默认分支。
|
|
113
|
+
Git 提交与推送使用 `repokit.git.commit` / `push`;提交范围、验证门禁和动作顺序仍由调用方负责。
|
|
114
|
+
完整契约见 [PR](../../docs/pr.md#创建与复用) 与 [Git](../../docs/git.md)。
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
repokit/__init__.py,sha256=jOrujMX5YX2Tg7K9P3CiK_JEVwQxQbF0TLcq-AHLMwQ,1949
|
|
2
|
+
repokit/_git.py,sha256=ZG0RmvyJyoEMUPDNWtZIBzRGrELV9De4zoPT1r3M23Y,7231
|
|
3
|
+
repokit/_inspect.py,sha256=DLvhhnqmRnqZ5cUz0I3XTA15fKeY0KF6nDNkruuRBgE,2365
|
|
4
|
+
repokit/_language.py,sha256=VPY-Rt4R9w7CrwR60O3KDyb8BZo6OA8mUdTtyFB_HX8,841
|
|
5
|
+
repokit/_layout.py,sha256=0Fh2o18c2URNHfzRuLXcwD9d3O4eupuMzS30c8ZGfEE,5874
|
|
6
|
+
repokit/_process.py,sha256=hcIVDko1k9wPuumhQdWJmG2nEli_rVbz4epf1FiIjPM,4929
|
|
7
|
+
repokit/_staging.py,sha256=IYrwLJGJlKAhLm6gEx9adqQp1513IN7bH3iq7f8I7QM,2061
|
|
8
|
+
repokit/budget.py,sha256=FfI9QIVzyipTkykNyUH8fpSf7JNVxtjCKfBDDdp318Y,1325
|
|
9
|
+
repokit/dependencies.py,sha256=NPKyWtBG8Rr2Do9gbegWgnJzm_WqMYF_lbaY3hElVIU,15015
|
|
10
|
+
repokit/git.py,sha256=7XdbLA2dW5PzmLlOcsKpT9dW-bXcZaRZfA18P522nVY,10349
|
|
11
|
+
repokit/git_index.py,sha256=ck1kzGtCUF1WuleMQzHWwn_B9dBPiJcNygec7p6Y3V4,9684
|
|
12
|
+
repokit/git_state.py,sha256=_oILnnWRe1hkyUMTt0tLHWN3BfcWcl7E_3HVE3uRWtQ,17506
|
|
13
|
+
repokit/languages.json,sha256=as3uFeqWw-SovOSHSYHpD3ezurHt21c8uEZWMr07wVc,22404
|
|
14
|
+
repokit/model.py,sha256=myuv9HzQrX6I8XClQhKTpT2YSh1GWrnYAV3cyJFZQ80,1794
|
|
15
|
+
repokit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
repokit/remote.py,sha256=yDb5vaQKDo07DtbJKsqaBF1DuCax5hc5JPbCQwUKV74,1344
|
|
17
|
+
repokit/repository.py,sha256=cGToy2aJQfwccFw4Ez8qkXTfGZ5WXqreI4CSNkJ_RdE,3364
|
|
18
|
+
repokit/snapshot.py,sha256=eCs5g9a9g2XSWRPDsfaNj2lt_YYeInmKe4owF4g0HXo,10187
|
|
19
|
+
repokit/stash.py,sha256=3TX58q0vaWYSZPyf0LHpn-Xjpw0QflAup9gqJh7IWQg,2065
|
|
20
|
+
repokit/tree.py,sha256=Ke0YCvBuLvyM9n51t4VEOLp32o4bXDUNmXFHoCIvOU8,6194
|
|
21
|
+
repokit/validation.py,sha256=iVitF-ZuWoLS_ZAIfZ3uFTH3h27VMgJqXxpeEs4G5Zo,17942
|
|
22
|
+
repokit/forge/__init__.py,sha256=4_Z3jJlAIK0IufxcrUiT_VP_8tuhzqAwdh9mXSeMLEM,757
|
|
23
|
+
repokit/forge/_http_state.py,sha256=X4YPRmko9txmsq2pj1T7XPGEAk0-be23xYo0ODyj7fU,3483
|
|
24
|
+
repokit/forge/_page.py,sha256=8abeVy0UKWqLIE77E-1K-3z3r9B0Ya2I3vjtCIkqpP4,1331
|
|
25
|
+
repokit/forge/_rest.py,sha256=oMithQcRNkqsCiUId0Ivw4DC9B9CFSykTvOWlQjatoY,10377
|
|
26
|
+
repokit/forge/creation.py,sha256=xCpXHKBQ8_tSicJ3nkHTpLozEzH5OCt2JrK8oUcLcc8,3056
|
|
27
|
+
repokit/forge/github.py,sha256=sbyhyjr-8pe8PK6xGvvDHZZ6-yhd6MnHS9pw0K4qG-Y,18366
|
|
28
|
+
repokit/forge/gitlab.py,sha256=pVC1dCfgZhC-EDJb4GsiJj8iJzj3rUDat9OeJWh0Gcc,17105
|
|
29
|
+
repokit/forge/model.py,sha256=ULg7WsKdngqb1XoFb7jjuxeqTWk60vC89YcOTBbPIzI,15003
|
|
30
|
+
repokit/forge/observation/__init__.py,sha256=QChAwEBoZXoxKpgI5je27MOJOkCg_3U8Qh6RI3oxkms,521
|
|
31
|
+
repokit/forge/observation/context.py,sha256=XT2Co2d_wacErWJRmREbDIgUrwbK5DJJzZ6po7lm-YU,11155
|
|
32
|
+
repokit/forge/observation/model.py,sha256=hdkvJJU4Vx5jQxg3eOT537hi0KqTUORML6-h9q2SLR0,945
|
|
33
|
+
repokit/forge/observation/service.py,sha256=UrFkiIM2G0krREe5SEyNXPrhe3PlI1tPhEDf4iXuDsQ,11997
|
|
34
|
+
repokit/forge/observation/store.py,sha256=xgrzRD2TLkKYcHo-2uoZ-IzZSnYwrC3RwcO-AP3Yhwo,4786
|
|
35
|
+
compforge_repokit-0.15.0.dist-info/METADATA,sha256=xZKJSTlqxmsgKCQt_e0IND9aYj25kfnlOd3qfmak_jg,5627
|
|
36
|
+
compforge_repokit-0.15.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
37
|
+
compforge_repokit-0.15.0.dist-info/RECORD,,
|
repokit/__init__.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Native repository organization; using Git paths and metadata."""
|
|
2
|
+
|
|
3
|
+
from harness_common import Component as Component
|
|
4
|
+
from harness_common import Forge as Forge
|
|
5
|
+
from harness_common import Product as Product
|
|
6
|
+
from harness_common import Repository as Repository
|
|
7
|
+
|
|
8
|
+
from ._inspect import inspect as inspect
|
|
9
|
+
from .budget import operation as operation
|
|
10
|
+
from .dependencies import DependencyEnvironment as DependencyEnvironment
|
|
11
|
+
from .dependencies import DependencyPreparation as DependencyPreparation
|
|
12
|
+
from .dependencies import inspect_dependencies as inspect_dependencies
|
|
13
|
+
from .dependencies import prepare_dependencies as prepare_dependencies
|
|
14
|
+
from .model import ComponentBinding as ComponentBinding
|
|
15
|
+
from .model import Diagnostic as Diagnostic
|
|
16
|
+
from .model import InspectReport as InspectReport
|
|
17
|
+
from .model import Layout as Layout
|
|
18
|
+
from .model import PackageTool as PackageTool
|
|
19
|
+
from .model import owner as owner
|
|
20
|
+
from .snapshot import ScopedSnapshot as ScopedSnapshot
|
|
21
|
+
from .snapshot import Snapshot as Snapshot
|
|
22
|
+
from .snapshot import SnapshotScope as SnapshotScope
|
|
23
|
+
from .snapshot import component_scopes as component_scopes
|
|
24
|
+
from .snapshot import snapshot as snapshot
|
|
25
|
+
from .tree import Directory as Directory
|
|
26
|
+
from .tree import File as File
|
|
27
|
+
from .tree import Manifest as Manifest
|
|
28
|
+
from .tree import TreeReport as TreeReport
|
|
29
|
+
from .tree import tree as tree
|
|
30
|
+
from .validation import CheckResult as CheckResult
|
|
31
|
+
from .validation import CheckSpec as CheckSpec
|
|
32
|
+
from .validation import ComponentValidationPlan as ComponentValidationPlan
|
|
33
|
+
from .validation import ComponentValidationResult as ComponentValidationResult
|
|
34
|
+
from .validation import ValidationPlan as ValidationPlan
|
|
35
|
+
from .validation import ValidationReport as ValidationReport
|
|
36
|
+
from .validation import execute_validation as execute_validation
|
|
37
|
+
from .validation import lint as lint
|
|
38
|
+
from .validation import plan_validation as plan_validation
|
|
39
|
+
from .validation import test as test
|
|
40
|
+
from .validation import validate as validate
|
repokit/_git.py
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Read Git catalogs and layout metadata without reading source contents."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import stat
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from ._layout import Catalog, needs_content, valid_path
|
|
8
|
+
from ._process import Budget, run
|
|
9
|
+
|
|
10
|
+
MAX_FILE_BYTES = 2 << 20
|
|
11
|
+
MAX_TOTAL_BYTES = 128 << 20
|
|
12
|
+
MAX_FILES = 10_000
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Git:
|
|
16
|
+
def __init__(self, root: Path, budget: Budget):
|
|
17
|
+
self.root = root
|
|
18
|
+
self.budget = budget
|
|
19
|
+
|
|
20
|
+
def run(
|
|
21
|
+
self,
|
|
22
|
+
args: list[str],
|
|
23
|
+
*,
|
|
24
|
+
input: bytes = b"",
|
|
25
|
+
allow_missing: bool = False,
|
|
26
|
+
max_output: int = 16 << 20,
|
|
27
|
+
) -> bytes:
|
|
28
|
+
return run(
|
|
29
|
+
self.root,
|
|
30
|
+
args,
|
|
31
|
+
self.budget,
|
|
32
|
+
input=input,
|
|
33
|
+
allow_missing=allow_missing,
|
|
34
|
+
max_output=max_output,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def gitlink_oid(self, name: str, oid: str) -> str:
|
|
38
|
+
"""Read checkout HEAD only; uninitialized entries retain the index reference."""
|
|
39
|
+
path = self.root / name
|
|
40
|
+
try:
|
|
41
|
+
(path / ".git").lstat()
|
|
42
|
+
except FileNotFoundError:
|
|
43
|
+
return oid
|
|
44
|
+
if path.resolve() != path:
|
|
45
|
+
raise ValueError(f"symlinked gitlink checkout: {name}")
|
|
46
|
+
child = Git(path, self.budget)
|
|
47
|
+
root = os.fsdecode(child.run(["rev-parse", "--show-toplevel"])).removesuffix("\n")
|
|
48
|
+
if Path(root) != path:
|
|
49
|
+
raise ValueError(f"invalid gitlink checkout: {name}")
|
|
50
|
+
return child.run(["rev-parse", "--verify", "HEAD"]).decode().strip()
|
|
51
|
+
|
|
52
|
+
def catalog(self, head: str, staged: bool) -> Catalog:
|
|
53
|
+
committed = bool(head) or staged
|
|
54
|
+
output = self.run(
|
|
55
|
+
["ls-tree", "-r", "-z", "--full-tree", head] if head else ["ls-files", "--stage", "-z"]
|
|
56
|
+
)
|
|
57
|
+
entries: dict[str, str] = {}
|
|
58
|
+
for entry in output.split(b"\0"):
|
|
59
|
+
if not entry:
|
|
60
|
+
continue
|
|
61
|
+
fields, separator, raw_name = entry.partition(b"\t")
|
|
62
|
+
parts = fields.split()
|
|
63
|
+
name = os.fsdecode(raw_name)
|
|
64
|
+
if not separator or len(parts) != 3 or not valid_path(name):
|
|
65
|
+
raise ValueError("invalid Git catalog entry")
|
|
66
|
+
mode = parts[0]
|
|
67
|
+
if not head and parts[2] != b"0":
|
|
68
|
+
raise ValueError(f"unmerged index entry: {name}")
|
|
69
|
+
if mode == b"160000":
|
|
70
|
+
continue
|
|
71
|
+
if mode == b"120000" and committed:
|
|
72
|
+
if needs_content(name):
|
|
73
|
+
raise ValueError(f"metadata {name} is a symlink")
|
|
74
|
+
continue
|
|
75
|
+
if mode not in (b"100644", b"100755", b"120000"):
|
|
76
|
+
raise ValueError(f"unsupported Git entry mode for {name}")
|
|
77
|
+
entries[name] = parts[2 if head else 1].decode("ascii")
|
|
78
|
+
if not committed:
|
|
79
|
+
for raw_name in self.run(["ls-files", "-z", "--others", "--exclude-standard"]).split(
|
|
80
|
+
b"\0"
|
|
81
|
+
):
|
|
82
|
+
name = os.fsdecode(raw_name)
|
|
83
|
+
if not name or name.endswith("/"):
|
|
84
|
+
continue
|
|
85
|
+
if not valid_path(name):
|
|
86
|
+
raise ValueError("unsafe repository path")
|
|
87
|
+
entries[name] = ""
|
|
88
|
+
if len(entries) > MAX_FILES:
|
|
89
|
+
raise ValueError("repository exceeds 10000 files")
|
|
90
|
+
files: Catalog = {}
|
|
91
|
+
selected: list[tuple[str, str]] = []
|
|
92
|
+
total = 0
|
|
93
|
+
for name in sorted(entries):
|
|
94
|
+
self.budget.remaining()
|
|
95
|
+
if committed:
|
|
96
|
+
files[name] = None
|
|
97
|
+
if needs_content(name):
|
|
98
|
+
oid = entries[name]
|
|
99
|
+
selected.append((name, oid))
|
|
100
|
+
continue
|
|
101
|
+
path = self.root / name
|
|
102
|
+
try:
|
|
103
|
+
info = path.lstat()
|
|
104
|
+
except FileNotFoundError:
|
|
105
|
+
continue
|
|
106
|
+
if not stat.S_ISREG(info.st_mode):
|
|
107
|
+
if needs_content(name):
|
|
108
|
+
raise ValueError(f"metadata {name} is not a regular file")
|
|
109
|
+
continue
|
|
110
|
+
if path.parent.resolve() != path.parent:
|
|
111
|
+
if needs_content(name):
|
|
112
|
+
raise ValueError(f"metadata {name} has a symlinked directory")
|
|
113
|
+
continue
|
|
114
|
+
files[name] = None
|
|
115
|
+
if not needs_content(name):
|
|
116
|
+
continue
|
|
117
|
+
if info.st_size > MAX_FILE_BYTES:
|
|
118
|
+
raise ValueError(f"metadata {name} exceeds {MAX_FILE_BYTES} bytes")
|
|
119
|
+
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
120
|
+
try:
|
|
121
|
+
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
|
122
|
+
raise ValueError(f"metadata {name} is not a regular file")
|
|
123
|
+
content = bytearray()
|
|
124
|
+
while len(content) <= MAX_FILE_BYTES:
|
|
125
|
+
self.budget.remaining()
|
|
126
|
+
chunk = os.read(fd, min(65536, MAX_FILE_BYTES + 1 - len(content)))
|
|
127
|
+
if not chunk:
|
|
128
|
+
break
|
|
129
|
+
content.extend(chunk)
|
|
130
|
+
if len(content) > MAX_FILE_BYTES:
|
|
131
|
+
raise ValueError(f"metadata {name} exceeds {MAX_FILE_BYTES} bytes")
|
|
132
|
+
total += len(content)
|
|
133
|
+
if total > MAX_TOTAL_BYTES:
|
|
134
|
+
raise ValueError("metadata exceeds 128 MiB")
|
|
135
|
+
files[name] = bytes(content)
|
|
136
|
+
finally:
|
|
137
|
+
os.close(fd)
|
|
138
|
+
if selected:
|
|
139
|
+
self._blobs(selected, files)
|
|
140
|
+
return files
|
|
141
|
+
|
|
142
|
+
def _blobs(self, selected: list[tuple[str, str]], files: Catalog) -> None:
|
|
143
|
+
batch = ("\n".join(oid for _, oid in selected) + "\n").encode("ascii")
|
|
144
|
+
headers = self.run(["cat-file", "--batch-check"], input=batch).splitlines()
|
|
145
|
+
if len(headers) != len(selected):
|
|
146
|
+
raise ValueError("invalid Git blob batch")
|
|
147
|
+
sizes: list[int] = []
|
|
148
|
+
total = 0
|
|
149
|
+
# Check immutable object sizes before reading any metadata contents.
|
|
150
|
+
for (name, oid), header in zip(selected, headers, strict=True):
|
|
151
|
+
fields = header.split()
|
|
152
|
+
if len(fields) != 3 or fields[:2] != [oid.encode("ascii"), b"blob"]:
|
|
153
|
+
raise ValueError("invalid Git blob")
|
|
154
|
+
size = int(fields[2])
|
|
155
|
+
if size < 0:
|
|
156
|
+
raise ValueError("invalid Git blob size")
|
|
157
|
+
if size > MAX_FILE_BYTES:
|
|
158
|
+
raise ValueError(f"metadata {name} exceeds {MAX_FILE_BYTES} bytes")
|
|
159
|
+
total += size
|
|
160
|
+
if total > MAX_TOTAL_BYTES:
|
|
161
|
+
raise ValueError("metadata exceeds 128 MiB")
|
|
162
|
+
sizes.append(size)
|
|
163
|
+
data = self.run(
|
|
164
|
+
["cat-file", "--batch"], input=batch, max_output=total + len(selected) * 128
|
|
165
|
+
)
|
|
166
|
+
offset = 0
|
|
167
|
+
for (name, oid), size in zip(selected, sizes, strict=True):
|
|
168
|
+
end = data.find(b"\n", offset)
|
|
169
|
+
if end < 0 or data[offset:end] != f"{oid} blob {size}".encode("ascii"):
|
|
170
|
+
raise ValueError("invalid Git blob header")
|
|
171
|
+
offset = end + 1
|
|
172
|
+
if data[offset + size : offset + size + 1] != b"\n":
|
|
173
|
+
raise ValueError("truncated Git blob")
|
|
174
|
+
files[name] = data[offset : offset + size]
|
|
175
|
+
offset += size + 1
|
|
176
|
+
if offset != len(data):
|
|
177
|
+
raise ValueError("unexpected Git blob bytes")
|
repokit/_inspect.py
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""One operation owns input selection, deadline and consistency observation."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from threading import Event
|
|
7
|
+
|
|
8
|
+
from ._git import Git
|
|
9
|
+
from ._layout import load
|
|
10
|
+
from ._process import Budget
|
|
11
|
+
from .model import Diagnostic, InspectReport
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def inspect(
|
|
15
|
+
repository: str | os.PathLike[str] = ".",
|
|
16
|
+
*,
|
|
17
|
+
head: str | None = None,
|
|
18
|
+
staged: bool = False,
|
|
19
|
+
timeout: float = 5.0,
|
|
20
|
+
cancel: Event | None = None,
|
|
21
|
+
) -> InspectReport:
|
|
22
|
+
"""Inspect organization using Git paths and layout metadata only.
|
|
23
|
+
|
|
24
|
+
timeout is a whole-call budget in seconds; cancel accepts a threading.Event.
|
|
25
|
+
Git processes are killed and reaped on timeout/cancellation. Filesystem reads and
|
|
26
|
+
Python projection check the budget between operations. Concurrent catalog changes
|
|
27
|
+
return complete=False; consumers decide whether to accept the observation.
|
|
28
|
+
"""
|
|
29
|
+
if head and staged:
|
|
30
|
+
raise ValueError("head and staged are mutually exclusive")
|
|
31
|
+
if not math.isfinite(timeout) or timeout <= 0:
|
|
32
|
+
raise ValueError("timeout must be positive and finite")
|
|
33
|
+
budget = Budget(timeout, cancel)
|
|
34
|
+
budget.remaining()
|
|
35
|
+
start = Path(repository).resolve(strict=True)
|
|
36
|
+
initial = Git(start, budget)
|
|
37
|
+
root = Path(
|
|
38
|
+
os.fsdecode(initial.run(["rev-parse", "--show-toplevel"])).removesuffix("\n")
|
|
39
|
+
).resolve()
|
|
40
|
+
git = Git(root, budget)
|
|
41
|
+
commit = (
|
|
42
|
+
git.run(["rev-parse", "--verify", "--end-of-options", head + "^{commit}"]).decode().strip()
|
|
43
|
+
if head
|
|
44
|
+
else ""
|
|
45
|
+
)
|
|
46
|
+
files = git.catalog(commit, staged)
|
|
47
|
+
origin = (
|
|
48
|
+
git.run(["config", "--get", "remote.origin.url"], allow_missing=True)
|
|
49
|
+
.decode("utf-8", errors="replace")
|
|
50
|
+
.rstrip("\n")
|
|
51
|
+
)
|
|
52
|
+
layout = load(files, origin)
|
|
53
|
+
complete = bool(commit) or files == git.catalog(commit, staged)
|
|
54
|
+
budget.remaining()
|
|
55
|
+
return InspectReport(
|
|
56
|
+
repository=layout.repository,
|
|
57
|
+
components=layout.components,
|
|
58
|
+
checkout=str(root),
|
|
59
|
+
input="commit" if commit else "index" if staged else "working_tree",
|
|
60
|
+
head=commit or None,
|
|
61
|
+
complete=complete,
|
|
62
|
+
diagnostics=()
|
|
63
|
+
if complete
|
|
64
|
+
else (
|
|
65
|
+
Diagnostic(
|
|
66
|
+
"inspection_changed",
|
|
67
|
+
"repository paths or layout metadata changed during inspection",
|
|
68
|
+
),
|
|
69
|
+
),
|
|
70
|
+
)
|
repokit/_language.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Filename metadata generated from pinned CodeGraph, without loading a parser."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import posixpath
|
|
5
|
+
from importlib.resources import files
|
|
6
|
+
|
|
7
|
+
_CATALOG: dict[str, dict[str, str]] = json.loads(
|
|
8
|
+
files("repokit").joinpath("languages.json").read_text(encoding="utf-8")
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def language(path: str) -> str:
|
|
13
|
+
base = posixpath.basename(path)
|
|
14
|
+
ext = ("." + base.rsplit(".", 1)[1]).lower() if "." in base else ""
|
|
15
|
+
if ext in _CATALOG["overrides"]:
|
|
16
|
+
return _CATALOG["overrides"][ext]
|
|
17
|
+
if base in _CATALOG["filenames"]:
|
|
18
|
+
return _CATALOG["filenames"][base]
|
|
19
|
+
suffixes = [base[i:] for i in range(len(base) - 1, 0, -1) if base[i] == "."][:4]
|
|
20
|
+
for suffix in reversed(suffixes):
|
|
21
|
+
if suffix in _CATALOG["registry"]:
|
|
22
|
+
return _CATALOG["registry"][suffix]
|
|
23
|
+
return _CATALOG["fallback"].get(ext, "")
|
repokit/_layout.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
"""Pure organization projection from a version's paths and necessary metadata."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import posixpath
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
|
|
7
|
+
from harness_common import Forge, Repository
|
|
8
|
+
|
|
9
|
+
from .model import ComponentBinding, Layout, PackageTool
|
|
10
|
+
from .remote import parse_remote_url
|
|
11
|
+
|
|
12
|
+
Catalog = dict[str, bytes | None]
|
|
13
|
+
_EXCLUDED = {
|
|
14
|
+
"node_modules",
|
|
15
|
+
"vendor",
|
|
16
|
+
"venv",
|
|
17
|
+
"env",
|
|
18
|
+
"dist",
|
|
19
|
+
"build",
|
|
20
|
+
"target",
|
|
21
|
+
"__pycache__",
|
|
22
|
+
"testdata",
|
|
23
|
+
}
|
|
24
|
+
_MARKERS = (
|
|
25
|
+
("python", "pyproject.toml", "setup.py"),
|
|
26
|
+
("go", "go.mod"),
|
|
27
|
+
("node", "package.json"),
|
|
28
|
+
("rust", "Cargo.toml"),
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def manifest_ecosystem(path: str) -> str:
|
|
33
|
+
return next(
|
|
34
|
+
(ecosystem for ecosystem, *names in _MARKERS if posixpath.basename(path) in names), ""
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def valid_path(name: str) -> bool:
|
|
39
|
+
return (
|
|
40
|
+
name not in ("", ".", "..")
|
|
41
|
+
and "\0" not in name
|
|
42
|
+
and "\\" not in name
|
|
43
|
+
and not name.startswith(("/", "../"))
|
|
44
|
+
and posixpath.normpath(name) == name
|
|
45
|
+
and not name.endswith("/")
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def needs_content(name: str) -> bool:
|
|
50
|
+
return posixpath.basename(name) == "package.json"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _origin(origin: str) -> Repository | None:
|
|
54
|
+
remote = parse_remote_url(origin)
|
|
55
|
+
if remote is None:
|
|
56
|
+
return None
|
|
57
|
+
forge = {"github.com": "github", "gitlab.com": "gitlab"}.get(remote.host, remote.host)
|
|
58
|
+
return Repository(Forge(forge), remote.path)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _skip(name: str) -> bool:
|
|
62
|
+
return any(part.startswith(".") or part in _EXCLUDED for part in name.split("/")[:-1])
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _detect(files: Catalog, root: str) -> str:
|
|
66
|
+
for ecosystem, *markers in _MARKERS:
|
|
67
|
+
for marker in markers:
|
|
68
|
+
name = posixpath.join(root, marker).removeprefix("./")
|
|
69
|
+
if name not in files:
|
|
70
|
+
continue
|
|
71
|
+
if ecosystem != "node":
|
|
72
|
+
return ecosystem
|
|
73
|
+
text = (files[name] or b"").decode("utf-8", errors="replace").lower()
|
|
74
|
+
tsconfig = posixpath.join(root, "tsconfig.json").removeprefix("./")
|
|
75
|
+
return (
|
|
76
|
+
"typescript"
|
|
77
|
+
if "typescript" in text or "@types/" in text or tsconfig in files
|
|
78
|
+
else "javascript"
|
|
79
|
+
)
|
|
80
|
+
return ""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _discover(files: Catalog) -> list[str]:
|
|
84
|
+
"""Find Component boundaries independently of their ancestors.
|
|
85
|
+
|
|
86
|
+
@spec Manifest and Makefile boundaries remain discoverable at every nesting depth.
|
|
87
|
+
@link component://docs/repository.md
|
|
88
|
+
"""
|
|
89
|
+
candidates: set[str] = set()
|
|
90
|
+
for name in files:
|
|
91
|
+
if _skip(name):
|
|
92
|
+
continue
|
|
93
|
+
if manifest_ecosystem(name) or posixpath.basename(name) == "Makefile":
|
|
94
|
+
candidates.add(posixpath.dirname(name) or ".")
|
|
95
|
+
return sorted(candidates)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# Match Go's JSON field semantics when reading package-manager evidence.
|
|
99
|
+
def _invalid_constant(value: str) -> object:
|
|
100
|
+
raise ValueError(f"invalid JSON constant: {value}")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _json(data: bytes) -> object:
|
|
104
|
+
# Match UTF-8 JSON in Go/TS, not Python's UTF-16 auto-detection or NaN extension.
|
|
105
|
+
return json.loads(data.decode("utf-8", errors="replace"), parse_constant=_invalid_constant)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _object(value: object) -> dict[str, object]:
|
|
109
|
+
if value is None:
|
|
110
|
+
return {}
|
|
111
|
+
if not isinstance(value, dict) or any(not isinstance(key, str) for key in value):
|
|
112
|
+
raise ValueError("expected an object")
|
|
113
|
+
return value
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _field(value: Mapping[str, object], key: str, default: object = None) -> object:
|
|
117
|
+
return next(
|
|
118
|
+
(value[name] for name in reversed(list(value)) if name.lower() == key.lower()), default
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _string(value: object) -> str:
|
|
123
|
+
if value is None:
|
|
124
|
+
return ""
|
|
125
|
+
if not isinstance(value, str):
|
|
126
|
+
raise ValueError("expected a string")
|
|
127
|
+
return value
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _tools(files: Catalog, root: str) -> tuple[PackageTool, ...]:
|
|
131
|
+
found: dict[str, PackageTool] = {}
|
|
132
|
+
|
|
133
|
+
def add(name: str, version: str, evidence: str) -> None:
|
|
134
|
+
old = found.get(name, PackageTool(name))
|
|
135
|
+
found[name] = PackageTool(name, (*old.evidence, evidence), version or old.version)
|
|
136
|
+
|
|
137
|
+
manifest = posixpath.join(root, "package.json").removeprefix("./")
|
|
138
|
+
if data := files.get(manifest):
|
|
139
|
+
try:
|
|
140
|
+
metadata = _object(_json(data))
|
|
141
|
+
manager = _string(_field(metadata, "packageManager"))
|
|
142
|
+
name, _, version = manager.partition("@")
|
|
143
|
+
if name in ("npm", "pnpm", "yarn", "bun"):
|
|
144
|
+
add(name, version, manifest + "#packageManager")
|
|
145
|
+
except (ValueError, UnicodeError):
|
|
146
|
+
pass # Malformed package.json remains a marker; lockfile evidence survives.
|
|
147
|
+
for file, tool in (
|
|
148
|
+
("go.mod", "go"),
|
|
149
|
+
("uv.lock", "uv"),
|
|
150
|
+
("poetry.lock", "poetry"),
|
|
151
|
+
("Pipfile.lock", "pipenv"),
|
|
152
|
+
("package-lock.json", "npm"),
|
|
153
|
+
("npm-shrinkwrap.json", "npm"),
|
|
154
|
+
("pnpm-lock.yaml", "pnpm"),
|
|
155
|
+
("yarn.lock", "yarn"),
|
|
156
|
+
("bun.lock", "bun"),
|
|
157
|
+
("bun.lockb", "bun"),
|
|
158
|
+
):
|
|
159
|
+
name = posixpath.join(root, file).removeprefix("./")
|
|
160
|
+
if name in files:
|
|
161
|
+
add(tool, "", name)
|
|
162
|
+
return tuple(
|
|
163
|
+
PackageTool(t.name, tuple(sorted(t.evidence)), t.version) for _, t in sorted(found.items())
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def load(files: Catalog, origin: str) -> Layout:
|
|
168
|
+
repo = _origin(origin)
|
|
169
|
+
bindings = tuple(
|
|
170
|
+
ComponentBinding(
|
|
171
|
+
repository=repo or Repository(Forge(""), ""),
|
|
172
|
+
name=root,
|
|
173
|
+
root=root,
|
|
174
|
+
products=(),
|
|
175
|
+
language=_detect(files, root) or None,
|
|
176
|
+
package_tools=_tools(files, root),
|
|
177
|
+
manifests=tuple(
|
|
178
|
+
sorted(
|
|
179
|
+
name
|
|
180
|
+
for name in files
|
|
181
|
+
if (posixpath.dirname(name) or ".") == root and manifest_ecosystem(name)
|
|
182
|
+
)
|
|
183
|
+
),
|
|
184
|
+
)
|
|
185
|
+
for root in _discover(files)
|
|
186
|
+
)
|
|
187
|
+
return Layout(repo, bindings)
|