kbforge 0.2.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {kbforge-0.2.0 → kbforge-0.4.0}/.github/workflows/ci.yml +1 -1
  2. {kbforge-0.2.0 → kbforge-0.4.0}/.pre-commit-config.yaml +2 -2
  3. kbforge-0.4.0/CHANGELOG.md +173 -0
  4. {kbforge-0.2.0 → kbforge-0.4.0}/PKG-INFO +73 -6
  5. {kbforge-0.2.0 → kbforge-0.4.0}/README.md +72 -5
  6. {kbforge-0.2.0 → kbforge-0.4.0}/docs/architecture.md +64 -6
  7. kbforge-0.4.0/examples/README.md +10 -0
  8. kbforge-0.4.0/examples/github-issues-connector/README.md +119 -0
  9. kbforge-0.4.0/examples/github-issues-connector/kbforge_github_issues/__init__.py +158 -0
  10. kbforge-0.4.0/examples/github-issues-connector/pyproject.toml +19 -0
  11. kbforge-0.4.0/examples/github-issues-connector/tests/test_connector.py +142 -0
  12. kbforge-0.4.0/examples/github-issues-connector/uv.lock +334 -0
  13. {kbforge-0.2.0 → kbforge-0.4.0}/pyproject.toml +1 -1
  14. kbforge-0.4.0/src/kbforge/__init__.py +17 -0
  15. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/__main__.py +69 -7
  16. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/hookspecs.py +9 -0
  17. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/llm_synthesizer.py +2 -2
  18. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/mirror.py +15 -0
  19. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/models.py +5 -0
  20. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/pipeline.py +70 -5
  21. kbforge-0.4.0/src/kbforge/publishers/_http.py +94 -0
  22. kbforge-0.4.0/src/kbforge/publishers/dry_run.py +45 -0
  23. kbforge-0.4.0/src/kbforge/publishers/forge.py +192 -0
  24. kbforge-0.4.0/src/kbforge/publishers/github.py +215 -0
  25. kbforge-0.4.0/src/kbforge/publishers/gitlab.py +211 -0
  26. kbforge-0.4.0/src/kbforge/publishers/summary.py +36 -0
  27. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/registry.py +4 -0
  28. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/synthesize.py +17 -2
  29. {kbforge-0.2.0 → kbforge-0.4.0}/tests/conftest.py +9 -2
  30. kbforge-0.4.0/tests/test_cli_publisher.py +219 -0
  31. kbforge-0.4.0/tests/test_dry_run_publisher.py +80 -0
  32. kbforge-0.4.0/tests/test_forge.py +343 -0
  33. kbforge-0.4.0/tests/test_forge_http.py +164 -0
  34. kbforge-0.4.0/tests/test_forge_live.py +376 -0
  35. kbforge-0.4.0/tests/test_github_publisher.py +551 -0
  36. kbforge-0.4.0/tests/test_gitlab_publisher.py +443 -0
  37. kbforge-0.4.0/tests/test_import.py +21 -0
  38. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_mirror.py +22 -1
  39. kbforge-0.4.0/tests/test_pipeline.py +375 -0
  40. kbforge-0.4.0/tests/test_publish_failure.py +94 -0
  41. kbforge-0.4.0/tests/test_summary.py +28 -0
  42. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_synthesize.py +33 -1
  43. {kbforge-0.2.0 → kbforge-0.4.0}/uv.lock +1 -1
  44. kbforge-0.2.0/CHANGELOG.md +0 -65
  45. kbforge-0.2.0/src/kbforge/__init__.py +0 -9
  46. kbforge-0.2.0/src/kbforge/publishers/dry_run.py +0 -46
  47. kbforge-0.2.0/tests/test_dry_run_publisher.py +0 -26
  48. kbforge-0.2.0/tests/test_import.py +0 -5
  49. kbforge-0.2.0/tests/test_pipeline.py +0 -168
  50. {kbforge-0.2.0 → kbforge-0.4.0}/.env.example +0 -0
  51. {kbforge-0.2.0 → kbforge-0.4.0}/.github/dependabot.yml +0 -0
  52. {kbforge-0.2.0 → kbforge-0.4.0}/.gitignore +0 -0
  53. {kbforge-0.2.0 → kbforge-0.4.0}/.python-version +0 -0
  54. {kbforge-0.2.0 → kbforge-0.4.0}/LICENSE +0 -0
  55. {kbforge-0.2.0 → kbforge-0.4.0}/docs/context/knowledge-base-design.md +0 -0
  56. {kbforge-0.2.0 → kbforge-0.4.0}/docs/design/2026-07-18-agent-facing-artifact-contract-design.md +0 -0
  57. {kbforge-0.2.0 → kbforge-0.4.0}/docs/design/2026-07-18-datacontract-bridge-design.md +0 -0
  58. {kbforge-0.2.0 → kbforge-0.4.0}/docs/design/2026-07-19-agentic-ingest-design.md +0 -0
  59. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/canonical.py +0 -0
  60. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/connectors/__init__.py +0 -0
  61. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/connectors/git_commits.py +0 -0
  62. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/connectors/local_files.py +0 -0
  63. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/publishers/__init__.py +0 -0
  64. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/py.typed +0 -0
  65. {kbforge-0.2.0 → kbforge-0.4.0}/src/kbforge/validate.py +0 -0
  66. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_canonical.py +0 -0
  67. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_cli.py +0 -0
  68. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_git_commits_connector.py +0 -0
  69. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_ingest_models.py +0 -0
  70. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_llm_live.py +0 -0
  71. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_llm_synthesizer.py +0 -0
  72. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_local_files_connector.py +0 -0
  73. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_models.py +0 -0
  74. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_strict_okf.py +0 -0
  75. {kbforge-0.2.0 → kbforge-0.4.0}/tests/test_validate.py +0 -0
@@ -84,4 +84,4 @@ jobs:
84
84
  enable-cache: true
85
85
  - run: uv python install 3.13
86
86
  - run: uv build --no-sources
87
- - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
87
+ - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1
@@ -1,6 +1,6 @@
1
1
  repos:
2
2
  - repo: https://github.com/astral-sh/ruff-pre-commit
3
- rev: v0.15.21
3
+ rev: v0.15.22
4
4
  hooks:
5
5
  - id: ruff-check
6
6
  args: [--fix]
@@ -10,6 +10,6 @@ repos:
10
10
  # `uv check` under the hood, so kbforge's own dependencies resolve for ty --
11
11
  # a local `entry: ty check` hook cannot see them.
12
12
  - repo: https://github.com/astral-sh/ty-pre-commit
13
- rev: v0.0.59
13
+ rev: v0.0.61
14
14
  hooks:
15
15
  - id: ty
@@ -0,0 +1,173 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here.
4
+
5
+ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
6
+ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
+
8
+ ## [Unreleased]
9
+
10
+ ## [0.4.0] - 2026-07-25
11
+
12
+ ### Added
13
+
14
+ - **Deletion propagation.** A concept removed at the source — via an explicit
15
+ tombstone (`CanonicalDocument.deleted=True`); absence is never inferred as
16
+ deletion — is now actually deleted from the target repo, not just described.
17
+ `ProposedChange.files_removed` carries the removal list, assigned by the
18
+ pipeline after synthesis so an LLM synthesizer cannot delete a file it
19
+ dislikes. Concepts still linking to a deleted one are pulled into scope and
20
+ re-synthesized so their now-dangling links are dropped. Both forge adapters
21
+ intersect removals with what is actually on the base tree first, since
22
+ GitLab (400) and GitHub (422) both reject deleting an absent path.
23
+
24
+ ### Fixed
25
+
26
+ - **Data loss when a review request was left open across runs.** `publish_to_forge`
27
+ reset the sync branch to the default branch on every run; since the mirror
28
+ advances after each successful publish, a run publishing while a previous
29
+ review request was still open silently rebuilt the branch and lost everything
30
+ the earlier run had put there. The base now resolves to the sync branch
31
+ itself when a request is open, so runs accumulate into one review request;
32
+ when none is open the branch still rebuilds from the default branch, so a
33
+ merged or abandoned request leaves no stale branch behind.
34
+
35
+ Note the branch self-heals but the *content* does not, and never did: the
36
+ mirror advances on every successful publish, so a request closed without
37
+ merging discards its contents permanently — those concepts are never
38
+ re-proposed, and a published-then-abandoned deletion is not even seen as a
39
+ removal by a later run. Abandon a review request by merging it, or by
40
+ resetting **both** the mirror and the connector's cursor
41
+ (`<state-dir>/cursor-<connector-name>.json`) — deleting the mirror alone
42
+ does nothing for an incremental connector, whose surviving cursor still
43
+ bounds the next fetch to records past it, so few or none come back and
44
+ nothing is re-proposed.
45
+
46
+ - **Links to still-published concepts were stripped on an incremental fetch.**
47
+ The set of resolvable link targets handed to synthesis was built from the
48
+ current fetch alone. An incremental connector's fetch need not contain a
49
+ concept that still exists, so a run carrying only a tombstone re-rendered the
50
+ referrer with *every* link removed — including links to concepts that were
51
+ still live and still published. §4.4 law 2 only fails on links that do not
52
+ resolve, never on links that went missing, so it shipped silently. That set is
53
+ now built from the mirror (the published state) unioned with the fetch, minus
54
+ the run's own tombstones.
55
+
56
+ - **A fully-filtered removal set produced a degenerate commit payload.** With no
57
+ files and every removal already absent from base, the adapters posted
58
+ `actions: []` (GitLab) or `tree: []` (GitHub). Both forges reject it — GitLab
59
+ 400 "Provide at least one action, or set allow_empty to true", GitHub 422
60
+ "Invalid tree info" — so every later run failed identically until the source
61
+ changed. Nothing is now committed when the branch already is base; when the
62
+ commit is also what creates the branch, GitLab sends `allow_empty` and GitHub
63
+ commits base's own tree.
64
+
65
+ On that create-branch path, the empty commit still opens (or updates) a
66
+ review request, so the rare case where every removal is already gone from
67
+ base is now visible rather than a hard failure: expect a request whose diff
68
+ is empty, whose body can still describe a removal that was, in fact, already
69
+ applied to the target (typically because an earlier run committed it but
70
+ died before the mirror advanced). That is strictly better than the prior
71
+ 400/422 — closing or merging it is a no-op — but it is surprising enough to
72
+ flag if you see it.
73
+
74
+ - The dry-run publisher applies `safe_join` to the paths it writes *and* the
75
+ paths it deletes, so a connector-supplied `native_id` of `../../../etc/foo`
76
+ can no longer reach outside the output directory. The forge publishers
77
+ already guarded this.
78
+
79
+ ## [0.3.0] - 2026-07-25
80
+
81
+ ### Added
82
+
83
+ - `examples/github-issues-connector/` — a complete worked example of a credentialed
84
+ connector (GitHub issues → OKF concepts) with token auth, pagination, and a real
85
+ incremental cursor, plus a walkthrough README for writing your own connector.
86
+ - GitHub (`--publisher github`) and GitLab (`--publisher gitlab`) publishers that
87
+ open or update a real pull/merge request from a `ProposedChange`. No new
88
+ runtime dependencies — both run on stdlib `urllib`. Tokens are read from
89
+ `GITHUB_TOKEN` / `GITLAB_TOKEN` (configurable via `token_env`), never the CLI.
90
+ Both send `Authorization: Bearer`, so either forge accepts a personal,
91
+ project or group access token as well as an OAuth token from `gh`/`glab`.
92
+ - `tests/test_forge_live.py` — an opt-in (`--run-live`) suite that publishes to
93
+ real GitHub and GitLab scratch repos and reads the result back through `gh`
94
+ and `glab`, so no assertion depends on the code that wrote the state. Covers
95
+ the sequence offline tests structurally cannot: publish, republish, human
96
+ merge, publish again.
97
+ - `--publisher NAME` and `--publish-set KEY=VALUE` CLI flags; `kbforge list` now
98
+ lists publishers.
99
+ - `kbforge_validate_publish_config` hookspec so publisher config is checked
100
+ before the pipeline runs.
101
+
102
+ ### Changed
103
+
104
+ - Publishers are now resolved by name. Previously the first registered plugin
105
+ implementing `kbforge_publish` won, which became order-dependent with more
106
+ than one publisher installed.
107
+ - The core design stance is now "zero credentialed *connectors*" — publishing is
108
+ delivery, not a system-of-record integration.
109
+
110
+ ### Fixed
111
+
112
+ - `kbforge.__version__` reported `0.1.0` for the whole of the 0.2.0 release. It
113
+ is now derived from installed package metadata, so `pyproject.toml` is the
114
+ single source of truth and the two cannot drift again.
115
+
116
+ ## [0.2.0] - 2026-07-19
117
+
118
+ ### Added
119
+
120
+ - Grounded **LLM synthesizer** — `--synthesizer llm`, behind the optional
121
+ `kbforge[llm]` extra. The model writes only concept prose (title, description,
122
+ body) inside a kbforge-owned structural frame: anchors, links, facets, type, and
123
+ timestamp are assembled deterministically, so the §4.4 validators gate structure
124
+ the model cannot influence. Reached through Pydantic AI's LiteLLM provider, so
125
+ OpenRouter and a self-hosted LiteLLM gateway share one config path, and the API
126
+ key comes only from an environment variable. The deterministic stub remains the
127
+ default; synthesizer selection is a generic `Synthesizer` seam injected into the
128
+ pipeline.
129
+
130
+ ## [0.1.0] - 2026-07-19
131
+
132
+ First release: a deterministic, credential-free walking skeleton of the kbforge
133
+ production protocol.
134
+
135
+ ### Added
136
+
137
+ - **Fixed pipeline** — `fetch → normalize → mirror → diff → scope → synthesize →
138
+ validate → publish`, run once by `kbforge run`. The order is not pluggable, and
139
+ neither are the two trust guarantees enforced in it: the **no-op rule** (a sync
140
+ that finds no change opens no merge request) and the **never-auto-merge rule**
141
+ (a publisher proposes; it never merges).
142
+ - **Canonicalization** with the §4.3 stability law — `normalize` is deterministic,
143
+ clock-free, and volatile-free, so identical input always yields identical content
144
+ hashes. A byte-different but content-identical re-save (CRLF flips, a BOM, a
145
+ re-export) is not a change.
146
+ - **Replay-safe mirror and read-only diff** — change is detected against a
147
+ core-owned mirror; the mirror advances only after a run fully succeeds. Absence
148
+ never implies a deletion.
149
+ - **Two built-in connectors**, both credential-free:
150
+ - `local_files` — a folder of markdown-with-frontmatter, with an additive
151
+ `ignore_globs` config and always-on defaults (`.venv`, `.git`, `node_modules`,
152
+ tool caches) so pointing at a repository root does not sweep in dependencies.
153
+ - `git_commits` — one concept per commit, with genuine incremental sync: the
154
+ cursor is the last-synced SHA, so a re-run fetches only `<last>..<ref>`.
155
+ - **§4.4 agent-facing artifact laws**, enforced as core validators at the `validate`
156
+ stage: facet well-formedness, link resolvability, anchor presence, and freshness
157
+ legibility, plus a projection↔files coherence check. Nothing non-conformant ships.
158
+ - **Stub synthesizer** — deterministic, no LLM; reshapes canonical documents into
159
+ OKF concepts and gives the validators real structure to check.
160
+ - **Dry-run publisher** — writes the proposed bundle to a local directory under a
161
+ source-named branch; never merges; idempotent.
162
+ - **Plugin system** on Pluggy with entry-point discovery: any installed package
163
+ advertising the `kbforge.connectors` or `kbforge.publishers` entry-point group is
164
+ discovered without editing kbforge.
165
+ - **CLI** — `kbforge list` shows available connectors; `kbforge run --connector NAME
166
+ --set KEY=VALUE ...` resolves the connector from the registry and takes YAML-typed
167
+ config, with no per-connector knowledge in the CLI.
168
+
169
+ [Unreleased]: https://github.com/flyersworder/kbforge/compare/v0.4.0...HEAD
170
+ [0.4.0]: https://github.com/flyersworder/kbforge/compare/v0.3.0...v0.4.0
171
+ [0.3.0]: https://github.com/flyersworder/kbforge/compare/v0.2.0...v0.3.0
172
+ [0.2.0]: https://github.com/flyersworder/kbforge/compare/v0.1.0...v0.2.0
173
+ [0.1.0]: https://github.com/flyersworder/kbforge/releases/tag/v0.1.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kbforge
3
- Version: 0.2.0
3
+ Version: 0.4.0
4
4
  Summary: Agent-first knowledge bases, forged from your systems of record
5
5
  Project-URL: Homepage, https://github.com/flyersworder/kbforge
6
6
  Project-URL: Repository, https://github.com/flyersworder/kbforge
@@ -72,12 +72,13 @@ and the [artifact-contract spec](docs/design/2026-07-18-agent-facing-artifact-co
72
72
  **Alpha — a working walking skeleton.** The deterministic core runs end to end with no
73
73
  credentials: two built-in connectors (`local_files`, `git_commits`), canonicalization
74
74
  with a stability law, a replay-safe mirror and diff, the §4.4 validator gate, and a
75
- dry-run publisher change detection, the no-op rule, and incremental sync via a real
76
- cursor all exercised by the test suite. Synthesis ships in two forms: a deterministic
77
- stub (the default, no LLM) and an opt-in grounded LLM synthesizer (`--synthesizer llm`,
78
- via the `kbforge[llm]` extra).
75
+ dry-run publisher, plus change detection, the no-op rule, and incremental sync via a
76
+ real cursor, all exercised by the test suite. Two credentialed publishers, GitHub and
77
+ GitLab, are also available (opt-in via `--publisher`, token from an env var). Synthesis ships in
78
+ two forms: a deterministic stub (the default, no LLM) and an opt-in grounded LLM
79
+ synthesizer (`--synthesizer llm`, via the `kbforge[llm]` extra).
79
80
 
80
- Not built yet: a credentialed system-of-record connector, and a GitHub-PR publisher. See
81
+ Not built yet: a credentialed system-of-record connector. See
81
82
  [`docs/architecture.md`](docs/architecture.md) for the full map.
82
83
 
83
84
  ## Quickstart
@@ -112,6 +113,51 @@ kbforge run --connector local_files --set path=./docs \
112
113
  The synthesizer reaches models through a LiteLLM provider, so OpenRouter and a
113
114
  self-hosted LiteLLM gateway share one config path.
114
115
 
116
+ ## Publishing to GitHub or GitLab
117
+
118
+ The default publisher writes the proposal to a local directory. To open a real
119
+ pull request or merge request instead, select a forge publisher and give it a
120
+ repo. The token comes from an env var, never the CLI:
121
+
122
+ ```bash
123
+ export GITHUB_TOKEN=... # or GITLAB_TOKEN
124
+ kbforge run --connector local_files --set path=./docs \
125
+ --publisher github --publish-set repo=acme/knowledge-base \
126
+ --mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
127
+ ```
128
+
129
+ Both publishers accept the same config: `repo` (required), `base` (default: the
130
+ repo's default branch), `base_path` (a subdirectory, default: repo root),
131
+ `branch` (default: `sync/<system>`), `title`, `api_base` (point it at GitHub
132
+ Enterprise or a self-managed GitLab), and `token_env`.
133
+
134
+ kbforge maintains **one long-lived sync branch and one open review request** per
135
+ source system: a later run force-updates that branch and edits the existing
136
+ PR/MR rather than opening a second one. Three consequences worth knowing:
137
+
138
+ - Concepts deleted from the source are deleted from the target repo, provided the
139
+ connector emits an explicit tombstone. Absence never implies a deletion.
140
+ - Manual commits on the sync branch are preserved **while its review request is
141
+ open** — a later run builds on the branch rather than resetting it. Once no
142
+ request is open, the next run rebuilds the branch from the default branch and
143
+ those commits are gone. A hand edit to a concept kbforge later regenerates is
144
+ overwritten by that regeneration either way.
145
+ - **Close a kbforge review request only by merging it.** The mirror advances on
146
+ every successful publish, so the concepts a request carries are never
147
+ re-proposed. Closing one unmerged discards its contents permanently: the
148
+ target repo simply never gets them, and a published-then-abandoned deletion
149
+ leaves the doc gone from the mirror, so no later run even sees it as a
150
+ removal. To undo an abandoned request, reset **both** the mirror and the
151
+ connector's cursor: delete the mirror directory and, in the state directory
152
+ (`--state`), the connector's `cursor-<connector-name>.json`. Deleting the
153
+ mirror alone does not work for an incremental connector — its cursor still
154
+ points past the abandoned content, so the next `kbforge_fetch` returns only
155
+ what changed since then, which can be little or nothing, and no re-proposal
156
+ happens at all. Only once both are gone does a re-run re-propose everything
157
+ from scratch.
158
+
159
+ kbforge never merges. No publisher has a merge method.
160
+
115
161
  ## Design stance
116
162
 
117
163
  The core ships **zero credentialed connectors and zero CI logic.** The two built-in
@@ -126,6 +172,10 @@ repositories. The interface is the product.
126
172
  myservice = "my_package:connector"
127
173
  ```
128
174
 
175
+ A complete worked example — a credentialed GitHub Issues connector (~160 lines) with
176
+ token auth, pagination, and a real incremental cursor — is in
177
+ [`examples/github-issues-connector/`](examples/github-issues-connector/).
178
+
129
179
  The pipeline order — fetch → normalize → mirror → diff → scope → synthesize → validate →
130
180
  publish — is deliberately **not** pluggable, and neither are the no-op rule or the
131
181
  never-auto-merge rule. Those are the trust guarantees; making them pluggable would make
@@ -166,6 +216,23 @@ prek install # ruff + ty on every commit
166
216
  uv run pytest
167
217
  ```
168
218
 
219
+ The default suite never touches the network. Tests that call a real external
220
+ service are marked `live` and skipped unless you pass `--run-live`.
221
+
222
+ The forge publishers have a live suite because their offline tests can only
223
+ assert what we *meant* to send — a real forge is the only thing that can say
224
+ the intent was right. It needs a throwaway repo on each forge and the two CLIs
225
+ (`gh`, `glab`) authenticated; each run writes under a fresh `live/<run-id>/`
226
+ prefix, so nothing accumulates and no repo is ever deleted.
227
+
228
+ ```bash
229
+ GITHUB_TOKEN=$(gh auth token) \
230
+ GITLAB_TOKEN=$(glab config get token --host gitlab.com) \
231
+ KBFORGE_LIVE_GITHUB_REPO=you/kbforge-live-test \
232
+ KBFORGE_LIVE_GITLAB_REPO=you/kbforge-live-test \
233
+ uv run pytest tests/test_forge_live.py --run-live
234
+ ```
235
+
169
236
  ## License
170
237
 
171
238
  MIT
@@ -37,12 +37,13 @@ and the [artifact-contract spec](docs/design/2026-07-18-agent-facing-artifact-co
37
37
  **Alpha — a working walking skeleton.** The deterministic core runs end to end with no
38
38
  credentials: two built-in connectors (`local_files`, `git_commits`), canonicalization
39
39
  with a stability law, a replay-safe mirror and diff, the §4.4 validator gate, and a
40
- dry-run publisher change detection, the no-op rule, and incremental sync via a real
41
- cursor all exercised by the test suite. Synthesis ships in two forms: a deterministic
42
- stub (the default, no LLM) and an opt-in grounded LLM synthesizer (`--synthesizer llm`,
43
- via the `kbforge[llm]` extra).
40
+ dry-run publisher, plus change detection, the no-op rule, and incremental sync via a
41
+ real cursor, all exercised by the test suite. Two credentialed publishers, GitHub and
42
+ GitLab, are also available (opt-in via `--publisher`, token from an env var). Synthesis ships in
43
+ two forms: a deterministic stub (the default, no LLM) and an opt-in grounded LLM
44
+ synthesizer (`--synthesizer llm`, via the `kbforge[llm]` extra).
44
45
 
45
- Not built yet: a credentialed system-of-record connector, and a GitHub-PR publisher. See
46
+ Not built yet: a credentialed system-of-record connector. See
46
47
  [`docs/architecture.md`](docs/architecture.md) for the full map.
47
48
 
48
49
  ## Quickstart
@@ -77,6 +78,51 @@ kbforge run --connector local_files --set path=./docs \
77
78
  The synthesizer reaches models through a LiteLLM provider, so OpenRouter and a
78
79
  self-hosted LiteLLM gateway share one config path.
79
80
 
81
+ ## Publishing to GitHub or GitLab
82
+
83
+ The default publisher writes the proposal to a local directory. To open a real
84
+ pull request or merge request instead, select a forge publisher and give it a
85
+ repo. The token comes from an env var, never the CLI:
86
+
87
+ ```bash
88
+ export GITHUB_TOKEN=... # or GITLAB_TOKEN
89
+ kbforge run --connector local_files --set path=./docs \
90
+ --publisher github --publish-set repo=acme/knowledge-base \
91
+ --mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
92
+ ```
93
+
94
+ Both publishers accept the same config: `repo` (required), `base` (default: the
95
+ repo's default branch), `base_path` (a subdirectory, default: repo root),
96
+ `branch` (default: `sync/<system>`), `title`, `api_base` (point it at GitHub
97
+ Enterprise or a self-managed GitLab), and `token_env`.
98
+
99
+ kbforge maintains **one long-lived sync branch and one open review request** per
100
+ source system: a later run force-updates that branch and edits the existing
101
+ PR/MR rather than opening a second one. Three consequences worth knowing:
102
+
103
+ - Concepts deleted from the source are deleted from the target repo, provided the
104
+ connector emits an explicit tombstone. Absence never implies a deletion.
105
+ - Manual commits on the sync branch are preserved **while its review request is
106
+ open** — a later run builds on the branch rather than resetting it. Once no
107
+ request is open, the next run rebuilds the branch from the default branch and
108
+ those commits are gone. A hand edit to a concept kbforge later regenerates is
109
+ overwritten by that regeneration either way.
110
+ - **Close a kbforge review request only by merging it.** The mirror advances on
111
+ every successful publish, so the concepts a request carries are never
112
+ re-proposed. Closing one unmerged discards its contents permanently: the
113
+ target repo simply never gets them, and a published-then-abandoned deletion
114
+ leaves the doc gone from the mirror, so no later run even sees it as a
115
+ removal. To undo an abandoned request, reset **both** the mirror and the
116
+ connector's cursor: delete the mirror directory and, in the state directory
117
+ (`--state`), the connector's `cursor-<connector-name>.json`. Deleting the
118
+ mirror alone does not work for an incremental connector — its cursor still
119
+ points past the abandoned content, so the next `kbforge_fetch` returns only
120
+ what changed since then, which can be little or nothing, and no re-proposal
121
+ happens at all. Only once both are gone does a re-run re-propose everything
122
+ from scratch.
123
+
124
+ kbforge never merges. No publisher has a merge method.
125
+
80
126
  ## Design stance
81
127
 
82
128
  The core ships **zero credentialed connectors and zero CI logic.** The two built-in
@@ -91,6 +137,10 @@ repositories. The interface is the product.
91
137
  myservice = "my_package:connector"
92
138
  ```
93
139
 
140
+ A complete worked example — a credentialed GitHub Issues connector (~160 lines) with
141
+ token auth, pagination, and a real incremental cursor — is in
142
+ [`examples/github-issues-connector/`](examples/github-issues-connector/).
143
+
94
144
  The pipeline order — fetch → normalize → mirror → diff → scope → synthesize → validate →
95
145
  publish — is deliberately **not** pluggable, and neither are the no-op rule or the
96
146
  never-auto-merge rule. Those are the trust guarantees; making them pluggable would make
@@ -131,6 +181,23 @@ prek install # ruff + ty on every commit
131
181
  uv run pytest
132
182
  ```
133
183
 
184
+ The default suite never touches the network. Tests that call a real external
185
+ service are marked `live` and skipped unless you pass `--run-live`.
186
+
187
+ The forge publishers have a live suite because their offline tests can only
188
+ assert what we *meant* to send — a real forge is the only thing that can say
189
+ the intent was right. It needs a throwaway repo on each forge and the two CLIs
190
+ (`gh`, `glab`) authenticated; each run writes under a fresh `live/<run-id>/`
191
+ prefix, so nothing accumulates and no repo is ever deleted.
192
+
193
+ ```bash
194
+ GITHUB_TOKEN=$(gh auth token) \
195
+ GITLAB_TOKEN=$(glab config get token --host gitlab.com) \
196
+ KBFORGE_LIVE_GITHUB_REPO=you/kbforge-live-test \
197
+ KBFORGE_LIVE_GITLAB_REPO=you/kbforge-live-test \
198
+ uv run pytest tests/test_forge_live.py --run-live
199
+ ```
200
+
134
201
  ## License
135
202
 
136
203
  MIT
@@ -27,9 +27,12 @@ This library is the reference implementation of that missing half:
27
27
  | **Production protocol** (connectors, canonicalization, diff, provenance, publish) | **this library** | this spec |
28
28
  | Serving protocol | MCP | exists |
29
29
 
30
- Design stance carried over from the main doc: **the core ships zero connectors,
31
- zero credentials, zero CI logic.** Connectors are plugins; deployments are separate
32
- repos. The interface is the product.
30
+ Design stance carried over from the main doc: **the core ships zero credentialed
31
+ connectors, zero CI logic.** Connectors are plugins; deployments are separate
32
+ repos. The interface is the product. Publishers are the exception that proves
33
+ it: publishing is the producer's own delivery mechanism, not an integration with
34
+ someone's system of record, so `github` and `gitlab` ship in core — reading their
35
+ credentials from the environment, never from config.
33
36
 
34
37
  ---
35
38
 
@@ -75,8 +78,8 @@ remove them. (This is the same posture as ADR-2 in the main doc.)
75
78
 
76
79
  1. **Connectors** (`ConnectorSpec`, §5.1) — bring data *in* from a system of record.
77
80
  Fully specified in this sketch; this is what we build first.
78
- 2. **Publishers** (`PublisherSpec`, §5.2) — push proposals *out* (GitLab MR, GitHub PR,
79
- local dry-run). Thin; sketched here because the pilot needs exactly one (GitLab).
81
+ 2. **Publishers** (`PublisherSpec`, §5.2) — push proposals *out*. Three ship in core:
82
+ `dry-run` (local), `github` (PR), `gitlab` (MR).
80
83
 
81
84
  Synthesis (the LLM step) and validation are **core stages with narrow extension
82
85
  hooks** (§5.3), not open plugin families — they carry the grounding contract and the
@@ -398,7 +401,7 @@ class ConnectorSpec:
398
401
 
399
402
  ```python
400
403
  class PublisherSpec:
401
- """Where proposals go. Pilot ships gitlab-mr; dry-run ships in core."""
404
+ """Where proposals go. MUST NOT merge (§5.2)."""
402
405
 
403
406
  @hookspec
404
407
  def kbforge_publisher_info(self) -> ConnectorInfo: ...
@@ -410,6 +413,61 @@ class PublisherSpec:
410
413
  re-running a failed pipeline updates the same MR, never opens twins."""
411
414
  ```
412
415
 
416
+ Three publishers ship in core: `dry-run` (default; writes to a directory),
417
+ `github` (pull requests) and `gitlab` (merge requests). All three implement
418
+ `kbforge_publish` and `kbforge_validate_publish_config`; none implements a merge
419
+ method, which is how §5.2's never-merge rule is enforced structurally rather
420
+ than by convention.
421
+
422
+ The two forge publishers sit behind a `ForgeClient` protocol whose methods name
423
+ intentions (`put_files`, `find_open_pr`) rather than REST endpoints, because the
424
+ forges decompose "commit these files" incompatibly — GitLab in one call, GitHub
425
+ in four. They are unconditional rather than gated behind an extra: both run on
426
+ stdlib `urllib`, so there is no dependency for an extra to install. Deliberately
427
+ absent, so their absence reads as a decision rather than an omission: labels,
428
+ reviewers and draft PRs; pagination in `find_open_pr` (a branch has at most one
429
+ open PR); rate-limit backoff (a run makes fewer than ten calls); and auto-merge,
430
+ which is excluded permanently rather than deferred.
431
+
432
+ Their offline tests inject a fake transport, which pins the request we intended
433
+ to make but cannot judge whether the intent was right — a real forge caught two
434
+ defects the offline suite structurally could not see. Hence `tests/test_forge_live.py`
435
+ (`--run-live`), which asserts through `gh`/`glab` rather than through kbforge's
436
+ own readers.
437
+
438
+ While a review request is open, a run sets the sync branch from the branch
439
+ itself rather than from the default branch, so successive runs accumulate into
440
+ one review request. When none is open the branch is rebuilt from the default
441
+ branch, so a merged or abandoned request leaves no stale *branch* behind.
442
+
443
+ That is a property of the branch only, and emphatically not of the content.
444
+ The mirror advances after every successful publish, so a concept carried by a
445
+ request that is closed without merging is never re-proposed: the target repo
446
+ lacks it permanently. The same holds in reverse for a deletion — once published,
447
+ the doc is gone from the mirror, so a later tombstone is not even a removal.
448
+ Closing a kbforge review request without merging therefore discards its contents
449
+ for good. Abandon a request by merging it, or by resetting **both** the mirror
450
+ and the connector's cursor (`_load_cursor`/`_save_cursor` in `pipeline.py` keep
451
+ it in the state directory, at `<state-dir>/cursor-<connector-name>.json`,
452
+ separate from the mirror). Deleting the mirror alone is not enough for an
453
+ incremental connector: the surviving cursor still bounds `kbforge_fetch` to
454
+ records past it, so the next run can fetch few or no records, `ChangeSet.is_noop`
455
+ fires, and nothing is re-proposed. Only deleting both re-proposes everything
456
+ from scratch.
457
+
458
+ Deletions travel
459
+ as `ProposedChange.files_removed`, assigned by the pipeline rather than by a
460
+ synthesizer: deletion is structure, not prose, so a model cannot delete a
461
+ concept it dislikes.
462
+
463
+ Declined as scope, deliberately rather than by omission: rebasing the sync
464
+ branch when the default branch moves under an open review request (the branch
465
+ may go stale; the forge's own merge handles it, and conflicts are unlikely
466
+ because only kbforge writes concept files); inferring a deletion from a
467
+ document's absence (tombstones stay explicit — see §4.2); and flagging
468
+ non-kbforge commits pushed onto the sync branch in the review body (the README
469
+ documents the sharp edge instead).
470
+
413
471
  ### 5.3 Core-stage extension hooks (narrow, additive-only)
414
472
 
415
473
  ```python
@@ -0,0 +1,10 @@
1
+ # Examples
2
+
3
+ Worked examples of extending kbforge. Each is a self-contained package that installs
4
+ alongside kbforge and is discovered through an entry point — no changes to kbforge
5
+ core.
6
+
7
+ - [**github-issues-connector**](github-issues-connector/) — a complete credentialed
8
+ connector (~160 lines) that syncs a repository's GitHub issues into OKF concepts,
9
+ with token auth, pagination, and a real incremental cursor. The template for
10
+ writing your own connector.
@@ -0,0 +1,119 @@
1
+ # Example: a GitHub Issues connector
2
+
3
+ A complete, credentialed kbforge connector in ~160 lines — the worked example for
4
+ writing your own. It syncs a repository's issues into OKF concepts, and it lives in
5
+ its **own package**, discovered by kbforge purely through an entry point. Nothing in
6
+ kbforge core changes.
7
+
8
+ This is the shape every connector follows. Copy it, rename it, swap the `fetch`
9
+ body for your system of record.
10
+
11
+ ## How kbforge finds it
12
+
13
+ A connector is any object that implements the connector hooks, advertised under the
14
+ `kbforge.connectors` entry-point group in your package's `pyproject.toml`:
15
+
16
+ ```toml
17
+ [project.entry-points."kbforge.connectors"]
18
+ github_issues = "kbforge_github_issues:connector"
19
+ ```
20
+
21
+ Once your package is installed alongside kbforge, `kbforge list` shows it and
22
+ `kbforge run --connector github_issues …` uses it. No fork, no registration call.
23
+
24
+ ## The four hooks (the whole contract)
25
+
26
+ | hook | job |
27
+ |---|---|
28
+ | `kbforge_connector_info` | static self-description (name, version, source) |
29
+ | `kbforge_validate_config` | return a list of problems (`[]` = ok); **no I/O** |
30
+ | `kbforge_fetch(config, cursor)` | pull raw records; `cursor=None` = full backfill |
31
+ | `kbforge_normalize(records)` | raw records → `CanonicalDocument`s; **pure & clock-free** |
32
+
33
+ That's it. Everything downstream — canonicalization, the replay-safe mirror, diff,
34
+ the no-op rule, incremental scoping, synthesis (stub or LLM), the §4.4 validators,
35
+ and publish — you get for free. A connector only teaches kbforge how to *read* your
36
+ source.
37
+
38
+ ## What this connector shows
39
+
40
+ **`fetch` — the real-world work.** It reads a token from the `GITHUB_TOKEN`
41
+ environment variable (never from config; the variable name is overridable with
42
+ `--set token_env=MY_TOKEN` — the value still comes only from the environment), pages
43
+ through the REST API, filters out
44
+ pull requests (GitHub's issues endpoint returns them too), and returns each issue as
45
+ a `RawRecord` carrying the issue JSON plus an `anchor_hint` (stable id, URL,
46
+ timestamp). The **cursor** is the maximum `updated_at` seen, so the next run passes
47
+ `since=<watermark>` and fetches only issues changed since — real incremental sync,
48
+ just like the built-in `git_commits` connector.
49
+
50
+ **`normalize` — deterministic by law.** It is a *pure* parse of the stored JSON into
51
+ a `CanonicalDocument`: no network, no clock (the timestamp came from `fetch`). kbforge
52
+ enforces this — `assert_stability` normalizes twice and rejects a connector whose
53
+ output isn't identical. One design choice worth copying: `updated_at` and the comment
54
+ count are deliberately kept **out** of the concept, so a comment-only edit produces an
55
+ identical concept and stays a no-op, while a real body/label/state change is detected.
56
+
57
+ **The rules a connector must respect:**
58
+ - **Deterministic, clock-free `normalize`** (the §4.3 stability law).
59
+ - **Retriever, not extractor** — return the source content verbatim with anchors;
60
+ interpretation is the synthesizer's job, not the connector's.
61
+ - **Credentials only from the environment**, never from config values.
62
+ - **Stable document identity** — a `doc_id` that survives across syncs, so change
63
+ detection works (here: `github_issues:owner/repo/issues/<number>`).
64
+
65
+ ## Run it
66
+
67
+ ```bash
68
+ # from this directory
69
+ uv sync --extra dev
70
+ uv run pytest # 8 offline tests (deterministic, no network)
71
+
72
+ export GITHUB_TOKEN=$(gh auth token)
73
+ uv run kbforge run \
74
+ --connector github_issues \
75
+ --set repo=owner/name \
76
+ --mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
77
+ ```
78
+
79
+ Add `--synthesizer llm --llm-set model=deepseek/deepseek-v4-flash` (with
80
+ `kbforge[llm]` and `OPENROUTER_API_KEY`) to turn each issue into a synthesized concept
81
+ instead of the verbatim stub.
82
+
83
+ That run stops at the dry-run publisher, which writes the bundle under `--out`.
84
+ To close the loop and open a real merge request (needs kbforge 0.3.0+; this
85
+ package itself only depends on 0.2.0 APIs, since publishing is a kbforge CLI
86
+ capability rather than something a connector links against):
87
+
88
+ ```bash
89
+ export GITLAB_TOKEN=$(glab config get token --host gitlab.com)
90
+ uv run kbforge run \
91
+ --connector github_issues --set repo=owner/name \
92
+ --publisher gitlab --publish-set repo=group/knowledge-base \
93
+ --publish-set base_path=kb-from-issues \
94
+ --mirror .kbforge/mirror --out .kbforge/out --state .kbforge/state
95
+ ```
96
+
97
+ The two `repo` values are independent, and the example is deliberately
98
+ cross-forge to make that obvious: the connector *reads* issues from GitHub, the
99
+ publisher *writes* concepts to GitLab. Use `--publisher github` to target a
100
+ GitHub repo instead — the connector side is unchanged either way.
101
+
102
+ Both halves are credentialed, which is the one wrinkle worth knowing: this
103
+ connector and the `github` publisher both default to `GITHUB_TOKEN`. When the
104
+ source read and the knowledge-base write should carry different scopes, split
105
+ them — `--set token_env=SOURCE_TOKEN` for the connector, `--publish-set
106
+ token_env=KB_TOKEN` for the publisher.
107
+
108
+ ## Deliberately out of scope
109
+
110
+ Kept simple so the shape stays legible — each of these is a natural enhancement if
111
+ you extend it:
112
+
113
+ - **Comments** aren't folded into the concept body (one extra API call per issue).
114
+ - **Deletions** aren't derived from absence (a kbforge-wide gap — needs tombstones).
115
+ - **HTTP errors** surface as a raw `httpx.HTTPStatusError` — a production connector
116
+ would distinguish 401 (bad token), 404 (repo not found), and 403 rate-limits
117
+ (honoring `X-RateLimit-Reset`) with clearer messages.
118
+ - **Page-number pagination** with `sort=updated` can, in theory, skip or duplicate an
119
+ issue updated by someone else mid-fetch; it self-heals on the next incremental sync.