kbforge 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kbforge-0.2.0 → kbforge-0.3.0}/.github/workflows/ci.yml +1 -1
- {kbforge-0.2.0 → kbforge-0.3.0}/.pre-commit-config.yaml +2 -2
- {kbforge-0.2.0 → kbforge-0.3.0}/CHANGELOG.md +39 -1
- {kbforge-0.2.0 → kbforge-0.3.0}/PKG-INFO +56 -6
- {kbforge-0.2.0 → kbforge-0.3.0}/README.md +55 -5
- {kbforge-0.2.0 → kbforge-0.3.0}/docs/architecture.md +31 -6
- kbforge-0.3.0/examples/README.md +10 -0
- kbforge-0.3.0/examples/github-issues-connector/README.md +119 -0
- kbforge-0.3.0/examples/github-issues-connector/kbforge_github_issues/__init__.py +158 -0
- kbforge-0.3.0/examples/github-issues-connector/pyproject.toml +19 -0
- kbforge-0.3.0/examples/github-issues-connector/tests/test_connector.py +142 -0
- kbforge-0.3.0/examples/github-issues-connector/uv.lock +334 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/pyproject.toml +1 -1
- kbforge-0.3.0/src/kbforge/__init__.py +17 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/__main__.py +69 -7
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/hookspecs.py +9 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/pipeline.py +2 -0
- kbforge-0.3.0/src/kbforge/publishers/_http.py +75 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/publishers/dry_run.py +9 -20
- kbforge-0.3.0/src/kbforge/publishers/forge.py +168 -0
- kbforge-0.3.0/src/kbforge/publishers/github.py +151 -0
- kbforge-0.3.0/src/kbforge/publishers/gitlab.py +192 -0
- kbforge-0.3.0/src/kbforge/publishers/summary.py +24 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/registry.py +4 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/conftest.py +9 -2
- kbforge-0.3.0/tests/test_cli_publisher.py +220 -0
- kbforge-0.3.0/tests/test_forge.py +282 -0
- kbforge-0.3.0/tests/test_forge_http.py +164 -0
- kbforge-0.3.0/tests/test_forge_live.py +206 -0
- kbforge-0.3.0/tests/test_github_publisher.py +361 -0
- kbforge-0.3.0/tests/test_gitlab_publisher.py +350 -0
- kbforge-0.3.0/tests/test_import.py +21 -0
- kbforge-0.3.0/tests/test_publish_failure.py +94 -0
- kbforge-0.3.0/tests/test_summary.py +28 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/uv.lock +1 -1
- kbforge-0.2.0/src/kbforge/__init__.py +0 -9
- kbforge-0.2.0/tests/test_import.py +0 -5
- {kbforge-0.2.0 → kbforge-0.3.0}/.env.example +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/.github/dependabot.yml +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/.gitignore +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/.python-version +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/LICENSE +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/docs/context/knowledge-base-design.md +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/docs/design/2026-07-18-agent-facing-artifact-contract-design.md +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/docs/design/2026-07-18-datacontract-bridge-design.md +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/docs/design/2026-07-19-agentic-ingest-design.md +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/canonical.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/connectors/__init__.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/connectors/git_commits.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/connectors/local_files.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/llm_synthesizer.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/mirror.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/models.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/publishers/__init__.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/py.typed +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/synthesize.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/src/kbforge/validate.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_canonical.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_cli.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_dry_run_publisher.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_git_commits_connector.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_ingest_models.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_llm_live.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_llm_synthesizer.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_local_files_connector.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_mirror.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_models.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_pipeline.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_strict_okf.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_synthesize.py +0 -0
- {kbforge-0.2.0 → kbforge-0.3.0}/tests/test_validate.py +0 -0
|
@@ -84,4 +84,4 @@ jobs:
|
|
|
84
84
|
enable-cache: true
|
|
85
85
|
- run: uv python install 3.13
|
|
86
86
|
- run: uv build --no-sources
|
|
87
|
-
- uses: pypa/gh-action-pypi-publish@
|
|
87
|
+
- uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
repos:
|
|
2
2
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
3
|
-
rev: v0.15.
|
|
3
|
+
rev: v0.15.22
|
|
4
4
|
hooks:
|
|
5
5
|
- id: ruff-check
|
|
6
6
|
args: [--fix]
|
|
@@ -10,6 +10,6 @@ repos:
|
|
|
10
10
|
# `uv check` under the hood, so kbforge's own dependencies resolve for ty --
|
|
11
11
|
# a local `entry: ty check` hook cannot see them.
|
|
12
12
|
- repo: https://github.com/astral-sh/ty-pre-commit
|
|
13
|
-
rev: v0.0.
|
|
13
|
+
rev: v0.0.61
|
|
14
14
|
hooks:
|
|
15
15
|
- id: ty
|
|
@@ -7,6 +7,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-07-25
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `examples/github-issues-connector/` — a complete worked example of a credentialed
|
|
15
|
+
connector (GitHub issues → OKF concepts) with token auth, pagination, and a real
|
|
16
|
+
incremental cursor, plus a walkthrough README for writing your own connector.
|
|
17
|
+
- GitHub (`--publisher github`) and GitLab (`--publisher gitlab`) publishers that
|
|
18
|
+
open or update a real pull/merge request from a `ProposedChange`. No new
|
|
19
|
+
runtime dependencies — both run on stdlib `urllib`. Tokens are read from
|
|
20
|
+
`GITHUB_TOKEN` / `GITLAB_TOKEN` (configurable via `token_env`), never the CLI.
|
|
21
|
+
Both send `Authorization: Bearer`, so either forge accepts a personal,
|
|
22
|
+
project or group access token as well as an OAuth token from `gh`/`glab`.
|
|
23
|
+
- `tests/test_forge_live.py` — an opt-in (`--run-live`) suite that publishes to
|
|
24
|
+
real GitHub and GitLab scratch repos and reads the result back through `gh`
|
|
25
|
+
and `glab`, so no assertion depends on the code that wrote the state. Covers
|
|
26
|
+
the sequence offline tests structurally cannot: publish, republish, human
|
|
27
|
+
merge, publish again.
|
|
28
|
+
- `--publisher NAME` and `--publish-set KEY=VALUE` CLI flags; `kbforge list` now
|
|
29
|
+
lists publishers.
|
|
30
|
+
- `kbforge_validate_publish_config` hookspec so publisher config is checked
|
|
31
|
+
before the pipeline runs.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- Publishers are now resolved by name. Previously the first registered plugin
|
|
36
|
+
implementing `kbforge_publish` won, which became order-dependent with more
|
|
37
|
+
than one publisher installed.
|
|
38
|
+
- The core design stance is now "zero credentialed *connectors*" — publishing is
|
|
39
|
+
delivery, not a system-of-record integration.
|
|
40
|
+
|
|
41
|
+
### Fixed
|
|
42
|
+
|
|
43
|
+
- `kbforge.__version__` reported `0.1.0` for the whole of the 0.2.0 release. It
|
|
44
|
+
is now derived from installed package metadata, so `pyproject.toml` is the
|
|
45
|
+
single source of truth and the two cannot drift again.
|
|
46
|
+
|
|
10
47
|
## [0.2.0] - 2026-07-19
|
|
11
48
|
|
|
12
49
|
### Added
|
|
@@ -60,6 +97,7 @@ production protocol.
|
|
|
60
97
|
--set KEY=VALUE ...` resolves the connector from the registry and takes YAML-typed
|
|
61
98
|
config, with no per-connector knowledge in the CLI.
|
|
62
99
|
|
|
63
|
-
[Unreleased]: https://github.com/flyersworder/kbforge/compare/v0.
|
|
100
|
+
[Unreleased]: https://github.com/flyersworder/kbforge/compare/v0.3.0...HEAD
|
|
101
|
+
[0.3.0]: https://github.com/flyersworder/kbforge/compare/v0.2.0...v0.3.0
|
|
64
102
|
[0.2.0]: https://github.com/flyersworder/kbforge/compare/v0.1.0...v0.2.0
|
|
65
103
|
[0.1.0]: https://github.com/flyersworder/kbforge/releases/tag/v0.1.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kbforge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Agent-first knowledge bases, forged from your systems of record
|
|
5
5
|
Project-URL: Homepage, https://github.com/flyersworder/kbforge
|
|
6
6
|
Project-URL: Repository, https://github.com/flyersworder/kbforge
|
|
@@ -72,12 +72,13 @@ and the [artifact-contract spec](docs/design/2026-07-18-agent-facing-artifact-co
|
|
|
72
72
|
**Alpha — a working walking skeleton.** The deterministic core runs end to end with no
|
|
73
73
|
credentials: two built-in connectors (`local_files`, `git_commits`), canonicalization
|
|
74
74
|
with a stability law, a replay-safe mirror and diff, the §4.4 validator gate, and a
|
|
75
|
-
dry-run publisher
|
|
76
|
-
cursor all exercised by the test suite.
|
|
77
|
-
|
|
78
|
-
|
|
75
|
+
dry-run publisher, plus change detection, the no-op rule, and incremental sync via a
|
|
76
|
+
real cursor, all exercised by the test suite. Two credentialed publishers, GitHub and
|
|
77
|
+
GitLab, are also available (opt-in via `--publisher`, token from an env var). Synthesis ships in
|
|
78
|
+
two forms: a deterministic stub (the default, no LLM) and an opt-in grounded LLM
|
|
79
|
+
synthesizer (`--synthesizer llm`, via the `kbforge[llm]` extra).
|
|
79
80
|
|
|
80
|
-
Not built yet: a credentialed system-of-record connector
|
|
81
|
+
Not built yet: a credentialed system-of-record connector. See
|
|
81
82
|
[`docs/architecture.md`](docs/architecture.md) for the full map.
|
|
82
83
|
|
|
83
84
|
## Quickstart
|
|
@@ -112,6 +113,34 @@ kbforge run --connector local_files --set path=./docs \
|
|
|
112
113
|
The synthesizer reaches models through a LiteLLM provider, so OpenRouter and a
|
|
113
114
|
self-hosted LiteLLM gateway share one config path.
|
|
114
115
|
|
|
116
|
+
## Publishing to GitHub or GitLab
|
|
117
|
+
|
|
118
|
+
The default publisher writes the proposal to a local directory. To open a real
|
|
119
|
+
pull request or merge request instead, select a forge publisher and give it a
|
|
120
|
+
repo. The token comes from an env var, never the CLI:
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
export GITHUB_TOKEN=... # or GITLAB_TOKEN
|
|
124
|
+
kbforge run --connector local_files --set path=./docs \
|
|
125
|
+
--publisher github --publish-set repo=acme/knowledge-base \
|
|
126
|
+
--mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Both publishers accept the same config: `repo` (required), `base` (default: the
|
|
130
|
+
repo's default branch), `base_path` (a subdirectory, default: repo root),
|
|
131
|
+
`branch` (default: `sync/<system>`), `title`, `api_base` (point it at GitHub
|
|
132
|
+
Enterprise or a self-managed GitLab), and `token_env`.
|
|
133
|
+
|
|
134
|
+
kbforge maintains **one long-lived sync branch and one open review request** per
|
|
135
|
+
source system: a later run force-updates that branch and edits the existing
|
|
136
|
+
PR/MR rather than opening a second one. Two consequences worth knowing:
|
|
137
|
+
|
|
138
|
+
- Manual commits pushed onto the sync branch are discarded by the next run.
|
|
139
|
+
- Concepts deleted from the source are not deleted from the target repo; files
|
|
140
|
+
absent from a run are inherited from the base branch.
|
|
141
|
+
|
|
142
|
+
kbforge never merges. No publisher has a merge method.
|
|
143
|
+
|
|
115
144
|
## Design stance
|
|
116
145
|
|
|
117
146
|
The core ships **zero credentialed connectors and zero CI logic.** The two built-in
|
|
@@ -126,6 +155,10 @@ repositories. The interface is the product.
|
|
|
126
155
|
myservice = "my_package:connector"
|
|
127
156
|
```
|
|
128
157
|
|
|
158
|
+
A complete worked example — a credentialed GitHub Issues connector (~160 lines) with
|
|
159
|
+
token auth, pagination, and a real incremental cursor — is in
|
|
160
|
+
[`examples/github-issues-connector/`](examples/github-issues-connector/).
|
|
161
|
+
|
|
129
162
|
The pipeline order — fetch → normalize → mirror → diff → scope → synthesize → validate →
|
|
130
163
|
publish — is deliberately **not** pluggable, and neither are the no-op rule or the
|
|
131
164
|
never-auto-merge rule. Those are the trust guarantees; making them pluggable would make
|
|
@@ -166,6 +199,23 @@ prek install # ruff + ty on every commit
|
|
|
166
199
|
uv run pytest
|
|
167
200
|
```
|
|
168
201
|
|
|
202
|
+
The default suite never touches the network. Tests that call a real external
|
|
203
|
+
service are marked `live` and skipped unless you pass `--run-live`.
|
|
204
|
+
|
|
205
|
+
The forge publishers have a live suite because their offline tests can only
|
|
206
|
+
assert what we *meant* to send — a real forge is the only thing that can say
|
|
207
|
+
the intent was right. It needs a throwaway repo on each forge and the two CLIs
|
|
208
|
+
(`gh`, `glab`) authenticated; each run writes under a fresh `live/<run-id>/`
|
|
209
|
+
prefix, so nothing accumulates and no repo is ever deleted.
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
GITHUB_TOKEN=$(gh auth token) \
|
|
213
|
+
GITLAB_TOKEN=$(glab config get token --host gitlab.com) \
|
|
214
|
+
KBFORGE_LIVE_GITHUB_REPO=you/kbforge-live-test \
|
|
215
|
+
KBFORGE_LIVE_GITLAB_REPO=you/kbforge-live-test \
|
|
216
|
+
uv run pytest tests/test_forge_live.py --run-live
|
|
217
|
+
```
|
|
218
|
+
|
|
169
219
|
## License
|
|
170
220
|
|
|
171
221
|
MIT
|
|
@@ -37,12 +37,13 @@ and the [artifact-contract spec](docs/design/2026-07-18-agent-facing-artifact-co
|
|
|
37
37
|
**Alpha — a working walking skeleton.** The deterministic core runs end to end with no
|
|
38
38
|
credentials: two built-in connectors (`local_files`, `git_commits`), canonicalization
|
|
39
39
|
with a stability law, a replay-safe mirror and diff, the §4.4 validator gate, and a
|
|
40
|
-
dry-run publisher
|
|
41
|
-
cursor all exercised by the test suite.
|
|
42
|
-
|
|
43
|
-
|
|
40
|
+
dry-run publisher, plus change detection, the no-op rule, and incremental sync via a
|
|
41
|
+
real cursor, all exercised by the test suite. Two credentialed publishers, GitHub and
|
|
42
|
+
GitLab, are also available (opt-in via `--publisher`, token from an env var). Synthesis ships in
|
|
43
|
+
two forms: a deterministic stub (the default, no LLM) and an opt-in grounded LLM
|
|
44
|
+
synthesizer (`--synthesizer llm`, via the `kbforge[llm]` extra).
|
|
44
45
|
|
|
45
|
-
Not built yet: a credentialed system-of-record connector
|
|
46
|
+
Not built yet: a credentialed system-of-record connector. See
|
|
46
47
|
[`docs/architecture.md`](docs/architecture.md) for the full map.
|
|
47
48
|
|
|
48
49
|
## Quickstart
|
|
@@ -77,6 +78,34 @@ kbforge run --connector local_files --set path=./docs \
|
|
|
77
78
|
The synthesizer reaches models through a LiteLLM provider, so OpenRouter and a
|
|
78
79
|
self-hosted LiteLLM gateway share one config path.
|
|
79
80
|
|
|
81
|
+
## Publishing to GitHub or GitLab
|
|
82
|
+
|
|
83
|
+
The default publisher writes the proposal to a local directory. To open a real
|
|
84
|
+
pull request or merge request instead, select a forge publisher and give it a
|
|
85
|
+
repo. The token comes from an env var, never the CLI:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
export GITHUB_TOKEN=... # or GITLAB_TOKEN
|
|
89
|
+
kbforge run --connector local_files --set path=./docs \
|
|
90
|
+
--publisher github --publish-set repo=acme/knowledge-base \
|
|
91
|
+
--mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Both publishers accept the same config: `repo` (required), `base` (default: the
|
|
95
|
+
repo's default branch), `base_path` (a subdirectory, default: repo root),
|
|
96
|
+
`branch` (default: `sync/<system>`), `title`, `api_base` (point it at GitHub
|
|
97
|
+
Enterprise or a self-managed GitLab), and `token_env`.
|
|
98
|
+
|
|
99
|
+
kbforge maintains **one long-lived sync branch and one open review request** per
|
|
100
|
+
source system: a later run force-updates that branch and edits the existing
|
|
101
|
+
PR/MR rather than opening a second one. Two consequences worth knowing:
|
|
102
|
+
|
|
103
|
+
- Manual commits pushed onto the sync branch are discarded by the next run.
|
|
104
|
+
- Concepts deleted from the source are not deleted from the target repo; files
|
|
105
|
+
absent from a run are inherited from the base branch.
|
|
106
|
+
|
|
107
|
+
kbforge never merges. No publisher has a merge method.
|
|
108
|
+
|
|
80
109
|
## Design stance
|
|
81
110
|
|
|
82
111
|
The core ships **zero credentialed connectors and zero CI logic.** The two built-in
|
|
@@ -91,6 +120,10 @@ repositories. The interface is the product.
|
|
|
91
120
|
myservice = "my_package:connector"
|
|
92
121
|
```
|
|
93
122
|
|
|
123
|
+
A complete worked example — a credentialed GitHub Issues connector (~160 lines) with
|
|
124
|
+
token auth, pagination, and a real incremental cursor — is in
|
|
125
|
+
[`examples/github-issues-connector/`](examples/github-issues-connector/).
|
|
126
|
+
|
|
94
127
|
The pipeline order — fetch → normalize → mirror → diff → scope → synthesize → validate →
|
|
95
128
|
publish — is deliberately **not** pluggable, and neither are the no-op rule or the
|
|
96
129
|
never-auto-merge rule. Those are the trust guarantees; making them pluggable would make
|
|
@@ -131,6 +164,23 @@ prek install # ruff + ty on every commit
|
|
|
131
164
|
uv run pytest
|
|
132
165
|
```
|
|
133
166
|
|
|
167
|
+
The default suite never touches the network. Tests that call a real external
|
|
168
|
+
service are marked `live` and skipped unless you pass `--run-live`.
|
|
169
|
+
|
|
170
|
+
The forge publishers have a live suite because their offline tests can only
|
|
171
|
+
assert what we *meant* to send — a real forge is the only thing that can say
|
|
172
|
+
the intent was right. It needs a throwaway repo on each forge and the two CLIs
|
|
173
|
+
(`gh`, `glab`) authenticated; each run writes under a fresh `live/<run-id>/`
|
|
174
|
+
prefix, so nothing accumulates and no repo is ever deleted.
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
GITHUB_TOKEN=$(gh auth token) \
|
|
178
|
+
GITLAB_TOKEN=$(glab config get token --host gitlab.com) \
|
|
179
|
+
KBFORGE_LIVE_GITHUB_REPO=you/kbforge-live-test \
|
|
180
|
+
KBFORGE_LIVE_GITLAB_REPO=you/kbforge-live-test \
|
|
181
|
+
uv run pytest tests/test_forge_live.py --run-live
|
|
182
|
+
```
|
|
183
|
+
|
|
134
184
|
## License
|
|
135
185
|
|
|
136
186
|
MIT
|
|
@@ -27,9 +27,12 @@ This library is the reference implementation of that missing half:
|
|
|
27
27
|
| **Production protocol** (connectors, canonicalization, diff, provenance, publish) | **this library** | this spec |
|
|
28
28
|
| Serving protocol | MCP | exists |
|
|
29
29
|
|
|
30
|
-
Design stance carried over from the main doc: **the core ships zero
|
|
31
|
-
|
|
32
|
-
repos. The interface is the product.
|
|
30
|
+
Design stance carried over from the main doc: **the core ships zero credentialed
|
|
31
|
+
connectors, zero CI logic.** Connectors are plugins; deployments are separate
|
|
32
|
+
repos. The interface is the product. Publishers are the exception that proves
|
|
33
|
+
it: publishing is the producer's own delivery mechanism, not an integration with
|
|
34
|
+
someone's system of record, so `github` and `gitlab` ship in core — reading their
|
|
35
|
+
credentials from the environment, never from config.
|
|
33
36
|
|
|
34
37
|
---
|
|
35
38
|
|
|
@@ -75,8 +78,8 @@ remove them. (This is the same posture as ADR-2 in the main doc.)
|
|
|
75
78
|
|
|
76
79
|
1. **Connectors** (`ConnectorSpec`, §5.1) — bring data *in* from a system of record.
|
|
77
80
|
Fully specified in this sketch; this is what we build first.
|
|
78
|
-
2. **Publishers** (`PublisherSpec`, §5.2) — push proposals *out
|
|
79
|
-
|
|
81
|
+
2. **Publishers** (`PublisherSpec`, §5.2) — push proposals *out*. Three ship in core:
|
|
82
|
+
`dry-run` (local), `github` (PR), `gitlab` (MR).
|
|
80
83
|
|
|
81
84
|
Synthesis (the LLM step) and validation are **core stages with narrow extension
|
|
82
85
|
hooks** (§5.3), not open plugin families — they carry the grounding contract and the
|
|
@@ -398,7 +401,7 @@ class ConnectorSpec:
|
|
|
398
401
|
|
|
399
402
|
```python
|
|
400
403
|
class PublisherSpec:
|
|
401
|
-
"""Where proposals go.
|
|
404
|
+
"""Where proposals go. MUST NOT merge (§5.2)."""
|
|
402
405
|
|
|
403
406
|
@hookspec
|
|
404
407
|
def kbforge_publisher_info(self) -> ConnectorInfo: ...
|
|
@@ -410,6 +413,28 @@ class PublisherSpec:
|
|
|
410
413
|
re-running a failed pipeline updates the same MR, never opens twins."""
|
|
411
414
|
```
|
|
412
415
|
|
|
416
|
+
Three publishers ship in core: `dry-run` (default; writes to a directory),
|
|
417
|
+
`github` (pull requests) and `gitlab` (merge requests). All three implement
|
|
418
|
+
`kbforge_publish` and `kbforge_validate_publish_config`; none implements a merge
|
|
419
|
+
method, which is how §5.2's never-merge rule is enforced structurally rather
|
|
420
|
+
than by convention.
|
|
421
|
+
|
|
422
|
+
The two forge publishers sit behind a `ForgeClient` protocol whose methods name
|
|
423
|
+
intentions (`put_files`, `find_open_pr`) rather than REST endpoints, because the
|
|
424
|
+
forges decompose "commit these files" incompatibly — GitLab in one call, GitHub
|
|
425
|
+
in four. They are unconditional rather than gated behind an extra: both run on
|
|
426
|
+
stdlib `urllib`, so there is no dependency for an extra to install. Deliberately
|
|
427
|
+
absent, so their absence reads as a decision rather than an omission: labels,
|
|
428
|
+
reviewers and draft PRs; pagination in `find_open_pr` (a branch has at most one
|
|
429
|
+
open PR); rate-limit backoff (a run makes fewer than ten calls); and auto-merge,
|
|
430
|
+
which is excluded permanently rather than deferred.
|
|
431
|
+
|
|
432
|
+
Their offline tests inject a fake transport, which pins the request we intended
|
|
433
|
+
to make but cannot judge whether the intent was right — a real forge caught two
|
|
434
|
+
defects the offline suite structurally could not see. Hence `tests/test_forge_live.py`
|
|
435
|
+
(`--run-live`), which asserts through `gh`/`glab` rather than through kbforge's
|
|
436
|
+
own readers.
|
|
437
|
+
|
|
413
438
|
### 5.3 Core-stage extension hooks (narrow, additive-only)
|
|
414
439
|
|
|
415
440
|
```python
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Examples
|
|
2
|
+
|
|
3
|
+
Worked examples of extending kbforge. Each is a self-contained package that installs
|
|
4
|
+
alongside kbforge and is discovered through an entry point — no changes to kbforge
|
|
5
|
+
core.
|
|
6
|
+
|
|
7
|
+
- [**github-issues-connector**](github-issues-connector/) — a complete credentialed
|
|
8
|
+
connector (~160 lines) that syncs a repository's GitHub issues into OKF concepts,
|
|
9
|
+
with token auth, pagination, and a real incremental cursor. The template for
|
|
10
|
+
writing your own connector.
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# Example: a GitHub Issues connector
|
|
2
|
+
|
|
3
|
+
A complete, credentialed kbforge connector in ~160 lines — the worked example for
|
|
4
|
+
writing your own. It syncs a repository's issues into OKF concepts, and it lives in
|
|
5
|
+
its **own package**, discovered by kbforge purely through an entry point. Nothing in
|
|
6
|
+
kbforge core changes.
|
|
7
|
+
|
|
8
|
+
This is the shape every connector follows. Copy it, rename it, swap the `fetch`
|
|
9
|
+
body for your system of record.
|
|
10
|
+
|
|
11
|
+
## How kbforge finds it
|
|
12
|
+
|
|
13
|
+
A connector is any object that implements the connector hooks, advertised under the
|
|
14
|
+
`kbforge.connectors` entry-point group in your package's `pyproject.toml`:
|
|
15
|
+
|
|
16
|
+
```toml
|
|
17
|
+
[project.entry-points."kbforge.connectors"]
|
|
18
|
+
github_issues = "kbforge_github_issues:connector"
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Once your package is installed alongside kbforge, `kbforge list` shows it and
|
|
22
|
+
`kbforge run --connector github_issues …` uses it. No fork, no registration call.
|
|
23
|
+
|
|
24
|
+
## The four hooks (the whole contract)
|
|
25
|
+
|
|
26
|
+
| hook | job |
|
|
27
|
+
|---|---|
|
|
28
|
+
| `kbforge_connector_info` | static self-description (name, version, source) |
|
|
29
|
+
| `kbforge_validate_config` | return a list of problems (`[]` = ok); **no I/O** |
|
|
30
|
+
| `kbforge_fetch(config, cursor)` | pull raw records; `cursor=None` = full backfill |
|
|
31
|
+
| `kbforge_normalize(records)` | raw records → `CanonicalDocument`s; **pure & clock-free** |
|
|
32
|
+
|
|
33
|
+
That's it. Everything downstream — canonicalization, the replay-safe mirror, diff,
|
|
34
|
+
the no-op rule, incremental scoping, synthesis (stub or LLM), the §4.4 validators,
|
|
35
|
+
and publish — you get for free. A connector only teaches kbforge how to *read* your
|
|
36
|
+
source.
|
|
37
|
+
|
|
38
|
+
## What this connector shows
|
|
39
|
+
|
|
40
|
+
**`fetch` — the real-world work.** It reads a token from the `GITHUB_TOKEN`
|
|
41
|
+
environment variable (never from config; the variable name is overridable with
|
|
42
|
+
`--set token_env=MY_TOKEN` — the value still comes only from the environment), pages
|
|
43
|
+
through the REST API, filters out
|
|
44
|
+
pull requests (GitHub's issues endpoint returns them too), and returns each issue as
|
|
45
|
+
a `RawRecord` carrying the issue JSON plus an `anchor_hint` (stable id, URL,
|
|
46
|
+
timestamp). The **cursor** is the maximum `updated_at` seen, so the next run passes
|
|
47
|
+
`since=<watermark>` and fetches only issues changed since — real incremental sync,
|
|
48
|
+
just like the built-in `git_commits` connector.
|
|
49
|
+
|
|
50
|
+
**`normalize` — deterministic by law.** It is a *pure* parse of the stored JSON into
|
|
51
|
+
a `CanonicalDocument`: no network, no clock (the timestamp came from `fetch`). kbforge
|
|
52
|
+
enforces this — `assert_stability` normalizes twice and rejects a connector whose
|
|
53
|
+
output isn't identical. One design choice worth copying: `updated_at` and the comment
|
|
54
|
+
count are deliberately kept **out** of the concept, so a comment-only edit produces an
|
|
55
|
+
identical concept and stays a no-op, while a real body/label/state change is detected.
|
|
56
|
+
|
|
57
|
+
**The rules a connector must respect:**
|
|
58
|
+
- **Deterministic, clock-free `normalize`** (the §4.3 stability law).
|
|
59
|
+
- **Retriever, not extractor** — return the source content verbatim with anchors;
|
|
60
|
+
interpretation is the synthesizer's job, not the connector's.
|
|
61
|
+
- **Credentials only from the environment**, never from config values.
|
|
62
|
+
- **Stable document identity** — a `doc_id` that survives across syncs, so change
|
|
63
|
+
detection works (here: `github_issues:owner/repo/issues/<number>`).
|
|
64
|
+
|
|
65
|
+
## Run it
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
# from this directory
|
|
69
|
+
uv sync --extra dev
|
|
70
|
+
uv run pytest # 8 offline tests (deterministic, no network)
|
|
71
|
+
|
|
72
|
+
export GITHUB_TOKEN=$(gh auth token)
|
|
73
|
+
uv run kbforge run \
|
|
74
|
+
--connector github_issues \
|
|
75
|
+
--set repo=owner/name \
|
|
76
|
+
--mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Add `--synthesizer llm --llm-set model=deepseek/deepseek-v4-flash` (with
|
|
80
|
+
`kbforge[llm]` and `OPENROUTER_API_KEY`) to turn each issue into a synthesized concept
|
|
81
|
+
instead of the verbatim stub.
|
|
82
|
+
|
|
83
|
+
That run stops at the dry-run publisher, which writes the bundle under `--out`.
|
|
84
|
+
To close the loop and open a real merge request (needs kbforge 0.3.0+; this
|
|
85
|
+
package itself only depends on 0.2.0 APIs, since publishing is a kbforge CLI
|
|
86
|
+
capability rather than something a connector links against):
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
export GITLAB_TOKEN=$(glab config get token --host gitlab.com)
|
|
90
|
+
uv run kbforge run \
|
|
91
|
+
--connector github_issues --set repo=owner/name \
|
|
92
|
+
--publisher gitlab --publish-set repo=group/knowledge-base \
|
|
93
|
+
--publish-set base_path=kb-from-issues \
|
|
94
|
+
--mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The two `repo` values are independent, and the example is deliberately
|
|
98
|
+
cross-forge to make that obvious: the connector *reads* issues from GitHub, the
|
|
99
|
+
publisher *writes* concepts to GitLab. Use `--publisher github` to target a
|
|
100
|
+
GitHub repo instead — the connector side is unchanged either way.
|
|
101
|
+
|
|
102
|
+
Both halves are credentialed, which is the one wrinkle worth knowing: this
|
|
103
|
+
connector and the `github` publisher both default to `GITHUB_TOKEN`. When the
|
|
104
|
+
source read and the knowledge-base write should carry different scopes, split
|
|
105
|
+
them — `--set token_env=SOURCE_TOKEN` for the connector, `--publish-set
|
|
106
|
+
token_env=KB_TOKEN` for the publisher.
|
|
107
|
+
|
|
108
|
+
## Deliberately out of scope
|
|
109
|
+
|
|
110
|
+
Kept simple so the shape stays legible — each of these is a natural enhancement if
|
|
111
|
+
you extend it:
|
|
112
|
+
|
|
113
|
+
- **Comments** aren't folded into the concept body (one extra API call per issue).
|
|
114
|
+
- **Deletions** aren't derived from absence (a kbforge-wide gap — needs tombstones).
|
|
115
|
+
- **HTTP errors** surface as a raw `httpx.HTTPStatusError` — a production connector
|
|
116
|
+
would distinguish 401 (bad token), 404 (repo not found), and 403 rate-limits
|
|
117
|
+
(honoring `X-RateLimit-Reset`) with clearer messages.
|
|
118
|
+
- **Page-number pagination** with `sort=updated` can, in theory, skip or duplicate an
|
|
119
|
+
issue updated by someone else mid-fetch; it self-heals on the next incremental sync.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""A credentialed kbforge connector for GitHub issues.
|
|
2
|
+
|
|
3
|
+
Lives outside kbforge core (the core ships zero credentialed connectors) and is
|
|
4
|
+
discovered purely via the `kbforge.connectors` entry-point group. Each issue
|
|
5
|
+
becomes one canonical document; the cursor is the max `updated_at` watermark, so a
|
|
6
|
+
re-run fetches only issues changed since the last sync. `updated_at` and comment
|
|
7
|
+
counts are deliberately kept OUT of the concept, so a comment-only update yields an
|
|
8
|
+
identical concept — a no-op — while a real body/label/state change is detected."""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
from collections.abc import Sequence
|
|
15
|
+
from datetime import datetime
|
|
16
|
+
|
|
17
|
+
import httpx
|
|
18
|
+
|
|
19
|
+
from kbforge.canonical import content_hash
|
|
20
|
+
from kbforge.hookspecs import hookimpl
|
|
21
|
+
from kbforge.models import (
|
|
22
|
+
CanonicalDocument,
|
|
23
|
+
ConnectorInfo,
|
|
24
|
+
Cursor,
|
|
25
|
+
FetchResult,
|
|
26
|
+
RawRecord,
|
|
27
|
+
ResourceAnchor,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
_SYSTEM = "github_issues"
|
|
31
|
+
_API = "https://api.github.com"
|
|
32
|
+
_PER_PAGE = 100
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class GitHubIssuesConnector:
|
|
36
|
+
@hookimpl
|
|
37
|
+
def kbforge_connector_info(self) -> ConnectorInfo:
|
|
38
|
+
return ConnectorInfo(
|
|
39
|
+
name=_SYSTEM,
|
|
40
|
+
version="0.1.0",
|
|
41
|
+
source_system="GitHub issues (REST API)",
|
|
42
|
+
info_types=["issue"],
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
@hookimpl
|
|
46
|
+
def kbforge_validate_config(self, config: dict) -> list[str]:
|
|
47
|
+
problems: list[str] = []
|
|
48
|
+
repo = config.get("repo", "")
|
|
49
|
+
if (
|
|
50
|
+
not isinstance(repo, str)
|
|
51
|
+
or repo.count("/") != 1
|
|
52
|
+
or not all(repo.split("/"))
|
|
53
|
+
):
|
|
54
|
+
problems.append(f"config 'repo' must be 'owner/name': {repo!r}")
|
|
55
|
+
token_env = config.get("token_env", "GITHUB_TOKEN")
|
|
56
|
+
if not os.environ.get(token_env):
|
|
57
|
+
problems.append(f"env var {token_env} is not set")
|
|
58
|
+
state = config.get("state", "all")
|
|
59
|
+
if state not in ("all", "open", "closed"):
|
|
60
|
+
problems.append(f"config 'state' must be all/open/closed: {state!r}")
|
|
61
|
+
return problems
|
|
62
|
+
|
|
63
|
+
@hookimpl
|
|
64
|
+
def kbforge_fetch(self, config: dict, cursor: Cursor | None) -> FetchResult:
|
|
65
|
+
owner, repo = config["repo"].split("/", 1)
|
|
66
|
+
token = os.environ[config.get("token_env", "GITHUB_TOKEN")]
|
|
67
|
+
state = config.get("state", "all")
|
|
68
|
+
since = cursor.payload.get("since") if cursor else None
|
|
69
|
+
|
|
70
|
+
records: list[RawRecord] = []
|
|
71
|
+
max_updated = since
|
|
72
|
+
headers = {
|
|
73
|
+
"Authorization": f"Bearer {token}",
|
|
74
|
+
"Accept": "application/vnd.github+json",
|
|
75
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
76
|
+
}
|
|
77
|
+
with httpx.Client(base_url=_API, headers=headers, timeout=30.0) as client:
|
|
78
|
+
page = 1
|
|
79
|
+
while True:
|
|
80
|
+
params: dict = {
|
|
81
|
+
"state": state,
|
|
82
|
+
"per_page": _PER_PAGE,
|
|
83
|
+
"page": page,
|
|
84
|
+
"sort": "updated",
|
|
85
|
+
"direction": "asc",
|
|
86
|
+
}
|
|
87
|
+
if since:
|
|
88
|
+
params["since"] = since
|
|
89
|
+
resp = client.get(f"/repos/{owner}/{repo}/issues", params=params)
|
|
90
|
+
resp.raise_for_status()
|
|
91
|
+
batch = resp.json()
|
|
92
|
+
if not batch:
|
|
93
|
+
break
|
|
94
|
+
for issue in batch:
|
|
95
|
+
if "pull_request" in issue: # the issues endpoint also returns PRs
|
|
96
|
+
continue
|
|
97
|
+
updated = issue["updated_at"]
|
|
98
|
+
native_id = f"{owner}/{repo}/issues/{issue['number']}"
|
|
99
|
+
records.append(
|
|
100
|
+
RawRecord(
|
|
101
|
+
anchor_hint={
|
|
102
|
+
"native_id": native_id,
|
|
103
|
+
"url": issue.get("html_url"),
|
|
104
|
+
"retrieved_at": updated,
|
|
105
|
+
},
|
|
106
|
+
media_type="application/vnd.github.issue+json",
|
|
107
|
+
payload=json.dumps(issue, sort_keys=True).encode("utf-8"),
|
|
108
|
+
)
|
|
109
|
+
)
|
|
110
|
+
if max_updated is None or updated > max_updated:
|
|
111
|
+
max_updated = updated
|
|
112
|
+
if len(batch) < _PER_PAGE:
|
|
113
|
+
break
|
|
114
|
+
page += 1
|
|
115
|
+
|
|
116
|
+
payload = {"since": max_updated} if max_updated else {}
|
|
117
|
+
return FetchResult(
|
|
118
|
+
records=records, cursor=Cursor(connector=_SYSTEM, payload=payload)
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
@hookimpl
|
|
122
|
+
def kbforge_normalize(
|
|
123
|
+
self, records: Sequence[RawRecord]
|
|
124
|
+
) -> list[CanonicalDocument]:
|
|
125
|
+
docs: list[CanonicalDocument] = []
|
|
126
|
+
for rec in records:
|
|
127
|
+
issue = json.loads(rec.payload.decode("utf-8"))
|
|
128
|
+
native_id = rec.anchor_hint["native_id"]
|
|
129
|
+
anchor = ResourceAnchor(
|
|
130
|
+
system=_SYSTEM,
|
|
131
|
+
native_id=native_id,
|
|
132
|
+
url=rec.anchor_hint.get("url"),
|
|
133
|
+
retrieved_at=datetime.fromisoformat(rec.anchor_hint["retrieved_at"]),
|
|
134
|
+
content_hash="",
|
|
135
|
+
)
|
|
136
|
+
doc = CanonicalDocument(
|
|
137
|
+
anchor=anchor,
|
|
138
|
+
doc_id=f"{_SYSTEM}:{native_id}",
|
|
139
|
+
title=str(issue.get("title") or native_id),
|
|
140
|
+
text=(issue.get("body") or "").strip(),
|
|
141
|
+
# updated_at / comment count are intentionally excluded — see module
|
|
142
|
+
# docstring: a comment-only update must stay a no-op.
|
|
143
|
+
structured={
|
|
144
|
+
"state": issue.get("state"),
|
|
145
|
+
"author": (issue.get("user") or {}).get("login"),
|
|
146
|
+
"labels": sorted(
|
|
147
|
+
label["name"] for label in issue.get("labels", [])
|
|
148
|
+
),
|
|
149
|
+
"created_at": issue.get("created_at"),
|
|
150
|
+
},
|
|
151
|
+
relations=[],
|
|
152
|
+
)
|
|
153
|
+
doc.anchor.content_hash = content_hash(doc)
|
|
154
|
+
docs.append(doc)
|
|
155
|
+
return docs
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
connector = GitHubIssuesConnector() # the entry point references this instance
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "kbforge-github-issues"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A credentialed kbforge connector: GitHub issues -> OKF concepts."
|
|
5
|
+
requires-python = ">=3.12"
|
|
6
|
+
dependencies = ["kbforge>=0.2.0", "httpx>=0.27"]
|
|
7
|
+
|
|
8
|
+
[project.optional-dependencies]
|
|
9
|
+
dev = ["pytest>=8"]
|
|
10
|
+
|
|
11
|
+
[project.entry-points."kbforge.connectors"]
|
|
12
|
+
github_issues = "kbforge_github_issues:connector"
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["hatchling"]
|
|
16
|
+
build-backend = "hatchling.build"
|
|
17
|
+
|
|
18
|
+
[tool.hatch.build.targets.wheel]
|
|
19
|
+
packages = ["kbforge_github_issues"]
|