dikw-core 0.0.2__tar.gz

dikw_core-0.0.2/.env.example

@@ -0,0 +1,38 @@
+# Copy to `.env` (gitignored) and fill in real values.
+# Loaded automatically by pytest-dotenv for tests; for CLI use
+# `uv run --env-file .env dikw …` or `set -a; source .env; set +a`.
+#
+# This file holds SECRETS ONLY. Non-secret config (endpoint URLs, model
+# names, embedding dimensions, batch sizes, display labels) lives in
+# `dikw.yml`. The live e2e test reads its provider config from
+# `tests/fixtures/live-minimax-gitee.dikw.yml` — edit that file (or your
+# own `dikw.yml`) to point at a different vendor; do not add vendor
+# prefixes to env vars here.
+
+# ---- LLM credentials (Anthropic SDK) ----
+# Consumed by the `anthropic` async SDK. When `provider.llm: anthropic`,
+# set this to your Anthropic key — or to a MiniMax key, since MiniMax
+# exposes an Anthropic-compatible endpoint via `provider.llm_base_url`.
+ANTHROPIC_API_KEY=
+
+# ---- LLM credentials (OpenAI SDK) ----
+# Consumed by the `openai` async SDK. Only needed when
+# `provider.llm: openai_compat` (OpenAI, Azure, Ollama, vLLM, DeepSeek,
+# GLM, Gemini OpenAI-compat, …). Not read by the embedding leg.
+OPENAI_API_KEY=
+
+# ---- Embedding credentials ----
+# Required for every embedding call. Deliberately independent from
+# OPENAI_API_KEY so LLM and embedding can target different vendors
+# (e.g., MiniMax LLM + Gitee AI embeddings) without cross-wiring
+# credentials. No fallback — a missing value raises a clear error.
+DIKW_EMBEDDING_API_KEY=
+
+# ---- optional: Postgres storage-contract tests ----
+# Leave unset to skip; set to a pgvector-enabled DSN to include them.
+# DIKW_TEST_POSTGRES_DSN=postgresql://dikw:dikw@localhost:5432/dikw_test
+
+# ---- optional: log level ----
+# Controls the root logger level for `dikw …` and `dikw serve`.
+# Accepts DEBUG / INFO / WARNING / ERROR / CRITICAL. Default INFO.
+# DIKW_LOG_LEVEL=INFO

dikw_core-0.0.2/.github/workflows/ci.yml

@@ -0,0 +1,113 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint-type-test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Sync dependencies
+        run: uv sync --all-extras
+
+      - name: Ruff
+        run: uv run ruff check .
+
+      - name: Mypy
+        run: uv run mypy src
+
+      - name: Pytest (fast, excludes slow + perf)
+        run: uv run pytest -v -m "not slow and not perf"
+
+  server-e2e:
+    name: Server e2e (serve-and-run)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Set up Python 3.12
+        run: uv python install 3.12
+
+      - name: Sync dependencies
+        run: uv sync --all-extras
+
+      - name: Slow tests (subprocess + bind real port)
+        # Covers ``client/serve_and_run.py`` end-to-end: spawns the real
+        # ``dikw serve`` process, polls /v1/healthz, runs the inner
+        # client command, and asserts a clean shutdown. Splitting this
+        # out from the matrix job keeps the failure mode visible (a
+        # broken subprocess lifecycle dominates the CI report instead
+        # of being one row in a pytest summary).
+        run: uv run pytest -v -m "slow"
+
+  postgres-contract:
+    name: Postgres contract tests
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_USER: dikw
+          POSTGRES_PASSWORD: dikw
+          POSTGRES_DB: dikw
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U dikw -d dikw"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DIKW_TEST_POSTGRES_DSN: postgresql://dikw:dikw@localhost:5432/dikw
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Set up Python 3.12
+        run: uv python install 3.12
+
+      - name: Sync dependencies (with postgres extra)
+        run: uv sync --all-extras
+
+      - name: Wait for Postgres
+        run: |
+          for i in {1..30}; do
+            if pg_isready -h localhost -p 5432 -U dikw; then
+              echo "postgres is ready"; exit 0
+            fi
+            sleep 1
+          done
+          echo "postgres failed to become ready" >&2
+          exit 1
+
+      - name: Run storage contract tests against Postgres
+        run: uv run pytest -v tests/test_storage_contract.py

dikw_core-0.0.2/.github/workflows/release.yml

@@ -0,0 +1,69 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+  id-token: write   # required for PyPI trusted publishing
+
+jobs:
+  build:
+    name: Build sdist + wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Set up Python 3.12
+        run: uv python install 3.12
+
+      - name: Verify tag matches pyproject version
+        run: |
+          set -e
+          tag_version="${GITHUB_REF_NAME#v}"
+          pyproject_version=$(uv run --no-project python -c \
+            "import tomllib,sys;print(tomllib.loads(open('pyproject.toml','rb').read().decode())['project']['version'])")
+          if [ "$tag_version" != "$pyproject_version" ]; then
+            echo "tag $GITHUB_REF_NAME does not match pyproject version $pyproject_version" >&2
+            exit 1
+          fi
+
+      - name: Run test gate before releasing
+        run: |
+          uv sync --all-extras
+          uv run ruff check .
+          uv run mypy src
+          uv run pytest -q
+
+      - name: Build distributions
+        run: uv build
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI (trusted publishing)
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/dikw-core
+    steps:
+      - name: Download dist artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

dikw_core-0.0.2/.gitignore

@@ -0,0 +1,31 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+dist/
+build/
+.venv/
+venv/
+*.sqlite
+*.sqlite-journal
+*.db
+.env
+.env.*
+!.env.example
+.DS_Store
+.idea/
+.vscode/
+.claude/
+# engine-managed state in example/test wikis
+**/.dikw/
+# parallel-development worktrees
+.worktrees/
+# throwaway smoke / debug tools under evals/tools — names prefixed with _
+evals/tools/_*_smoke.py
+# eval-snapshot cache (per-dataset ingested wikis; built on first eval run,
+# reused across re-runs). Each snapshot is ~30-150 MB SQLite + corpus copy.
+evals/.cache/

dikw_core-0.0.2/.python-version

@@ -0,0 +1 @@
+3.12

dikw_core-0.0.2/AGENTS.md

@@ -0,0 +1,161 @@
+# AGENTS.md
+
+> Guide for AI agents that **use** dikw-core as a knowledge backend. If you're
+> here to contribute code to the engine itself, see [CLAUDE.md](./CLAUDE.md)
+> instead.
+
+## What dikw-core is, from your point of view
+
+`dikw-core` is a Python service that turns a directory of markdown notes
+(plus assets) into a queryable knowledge engine spanning the **D**ata →
+**I**nformation → **K**nowledge → **W**isdom layers. The directory it points
+at is called a **dikw base** — `dikw.yml` lives at its root, `sources/`
+holds the raw notes, and the engine writes K-layer wiki pages and W-layer
+distilled wisdom back into the same tree as plain markdown.
+
+You talk to it over HTTP+NDJSON via a long-running `dikw serve` process.
+The Typer CLI (`dikw client …`) is one client; you can write your own
+agent loop with any HTTP library.
+
+The full server spec — auth posture, NDJSON wire format, every route — is
+in [`docs/server.md`](./docs/server.md).
+
+## Bootstrap
+
+You probably want [`INSTALL_FOR_AGENTS.md`](./INSTALL_FOR_AGENTS.md). It
+walks through install → init a base → set keys → start the server →
+first retrieve call in concrete commands.
+
+## Endpoints you will actually use
+
+The agent surface is intentionally small. The server is **manually
+started by the human operator** — if you can't reach `GET /v1/health`,
+ask the user to run `dikw serve` (don't try to start it yourself).
+
+**dikw-core does NOT do LLM answer synthesis.** It hands you ranked
+chunks + applicable wisdom + the parsed wiki tree. Composing those into
+an answer is your job — you run your own LLM with your own prompt,
+applying query rewrite / expansion / conversation context as you see
+fit. dikw-core is stateless; agents have the context dikw-core doesn't.
+
+| route | purpose | when to call |
+| --- | --- | --- |
+| `GET /v1/health` | server self-description (`base_root`, `version`, `storage_engine`, `layer_counts`, `providers`) | first call after attach — confirms the server is up and which base it's bound to |
+| `POST /v1/retrieve` | retrieval-only NDJSON (chunks + page refs, no LLM call) | knowledge access — feed the chunks into your own LLM prompt |
+| `GET /v1/base/pages` | list pages registered in the base, optional `?layer=` filter | discovering page paths to read |
+| `GET /v1/base/pages/{path}` | full page body + chunk anchors aligned to the parsed coordinate space | reading a specific page after a retrieval hit lands you on it |
+| `GET /v1/base/pages/{path}/links` | K-layer link neighbours of a page — `outgoing[]` (edges from this page) + `incoming[]` (edges to this page), optional `?direction=in\|out\|both` and `?limit=N`. Every returned edge resolves to an active document (bare URLs and deactivated dsts are filtered), so you can always feed `dst_path` / `src_path` back into `/v1/base/pages/{path}` | walking the wiki graph one hop without re-parsing `[[wikilink]]` syntax |
+| `GET /v1/base/graph` | whole-base graph in one read: `nodes[]`, `edges[]`, `unresolved[]`, `stats{}`, plus a `base_revision` content hash for cheap caching. Optional `?active=true\|false`. | global graph view (knowledge-graph UI, connectivity analysis) without N requests |
+| `GET /v1/assets/{asset_id}` | stream the raw bytes of a content-addressed asset (sha256 hex id). Immutable; `ETag` + `Cache-Control: public, max-age=31536000, immutable`. Look up the id via the `assets[]` array on `GET /v1/base/pages/{path}` | rendering images embedded in a page response, or piping a binary into another tool |
+| `POST /v1/ingest` | ingest whatever is on disk under `<base>/sources/` (loaded there by `POST /v1/import` or by the user dropping files in) | when the user adds/edits markdown and wants the index refreshed |
+| `GET /v1/status`, `POST /v1/lint`, `POST /v1/check` | counts, lint issues, provider connectivity | sanity checks the user may ask for |
+
+CLI equivalents — all `dikw client` commands default to JSON output
+suitable for piping into `jq` or an agent loop. Human-readable rendering
+requires opting in via `--format table`:
+
+```
+dikw client health                           # JSON by default
+dikw client retrieve "your question" --plain # pipe-safe final JSON (chunks + page_refs)
+dikw client pages list                       # JSON by default
+dikw client pages get sources/notes/alpha.md # JSON
+dikw client pages links wiki/Foo.md          # JSON: {outgoing, incoming}
+dikw client graph get                        # JSON: full base graph in one call
+dikw client assets get <asset_id> --output f # streams bytes to a local file; metadata to stdout JSON
+dikw client import ./local-sources           # pre-flights + imports md packages
+dikw client ingest                           # async-default: prints task-handle JSON, exits 0
+dikw client ingest --wait                    # rendered progress + report; succeeded=0 / failed=1 / cancelled=130
+dikw client tasks wait <task_id>             # block + render on an existing task_id (same exit-code map)
+dikw client tasks events <task_id>           # raw EventsPage JSON for a single cursor page
+```
+
+`retrieve` consumes the NDJSON event stream server-side and emits the
+final JSON payload (chunks + page_refs) to stdout. Pass `--plain`
+whenever you pipe, otherwise the "retrieving…" rich banner lands on
+stdout and breaks `jq`. The `--format json` and `--plain` toggles are
+orthogonal: `--format` picks the *final* shape, `--plain` suppresses
+the *intermediate* status. If you want the raw NDJSON event stream,
+talk to `POST /v1/retrieve` over HTTP directly.
+
+## A typical retrieval-augmented loop
+
+1. `GET /v1/health` — confirm the server is up and grab `base_root` so you
+   know which base the user pointed it at.
+2. `POST /v1/retrieve` with the question + a `limit`. Each chunk hit
+   carries a `path`, `layer`, `anchor`, `start_off`/`end_off`, plus
+   full chunk `text` (on both the intermediate `retrieval_done` partial
+   *and* `final.result.chunks`), plus `page_refs` listing the parent
+   pages. A streaming agent can prompt off the partial without waiting
+   for `final`.
+3. If you want full pages instead of just chunks, follow the page refs
+   with `GET /v1/base/pages/{path}` — that returns the parsed body plus
+   anchors so you can re-locate every chunk hit inside the page body.
+4. To expand context across the wiki graph, call
+   `GET /v1/base/pages/{path}/links` on hit pages. The response is
+   `{outgoing[{dst_path, link_type, line, anchor}], incoming[{src_doc_id,
+   src_path, link_type, line, anchor}]}` — each edge is a hop you can
+   immediately re-feed into `GET /v1/base/pages/{path}` without scanning
+   wiki bodies for `[[wikilinks]]`.
+5. Feed the chunks (and any neighbour pages you pulled) into your own
+   LLM prompt and produce the final answer client-side. dikw-core does
+   not own the synthesis step.
+
+## Things that will trip you up
+
+- **Server lifecycle.** `dikw serve` is started manually by the user, not
+  by the agent. If you can't connect, surface that to the user — do not
+  spawn it yourself.
+- **Auth.** Loopback (default) is open. Non-loopback hosts require a
+  `DIKW_SERVER_TOKEN` bearer. The server-bound config is reflected in
+  `GET /v1/info`'s `auth_required` field; `/v1/health` is the rich
+  bootstrap probe and intentionally omits auth state.
+- **The "base" terminology.** When the docs or CLI say "base", they mean
+  the whole bound directory (which contains `sources/`, `wiki/`,
+  `wisdom/`, `.dikw/`, `dikw.yml`). The K-layer subdirectory is still
+  called `wiki/` on disk — that's intentional, since the user opens it
+  in Obsidian. Don't confuse "the dikw base" with "the wiki/ folder
+  inside it".
+- **`/v1/base/pages/{path}` is index-driven.** Only paths registered as
+  `DocumentRecord` rows resolve. If a markdown file exists on disk but
+  hasn't been ingested, the route returns 404. Use `GET /v1/base/pages`
+  to enumerate what's actually queryable.
+- **NDJSON streaming vs cursor JSON.** `POST /v1/retrieve` streams
+  NDJSON directly on its response body (`type=retrieve_started →
+  retrieval_done → final`). The async-task ops
+  (`POST /v1/{ingest,synth,distill,eval,lint.propose,lint.apply}`)
+  instead return a JSON `TaskHandle` (`{"task_id": "..."}`); follow
+  the task by polling **`GET /v1/tasks/{task_id}/events?from_seq=N&limit=M&wait=K`**.
+  Each response is one `EventsPage` (cursor JSON, server long-poll up
+  to `wait` seconds, capped at 60s) — agents advance via the returned
+  `next_from_seq` and stop when `task_status` is terminal AND the
+  `final` event has appeared in the rendered page. The final event
+  itself carries the result/error payload. There is no SSE.
+- **CLI mirrors the wire contract.** Op commands default to async:
+  `dikw client ingest` submits + prints `{task_id, status, events_url,
+  wait_command}` and exits 0. Pass `--wait` to block + render + map
+  the final status to the standard exit code (succeeded=0, failed=1,
+  cancelled=130, client-side timeout=124). Use
+  `dikw client tasks events <id>` for raw cursor pages, and
+  `dikw client tasks wait <id>` for the same blocking UX applied to
+  an existing task_id.
+- **Per-file ingest errors are non-fatal.** A bad markdown file produces
+  one `partial` event with `kind=file_error` and lands on
+  `IngestReport.errors`, but the run continues. CLI users can pass
+  `--strict` to flip this to a hard fail.
+
+## Pointers
+
+- [`docs/architecture.md`](./docs/architecture.md) — module map, layer
+  contracts, named seams. Worth reading if you're going to do anything
+  more than basic retrieval.
+- [`docs/design.md`](./docs/design.md) — approved design doc. Source of
+  truth for *intent*; supersedes anything inferred from code shape.
+- [`docs/getting-started.md`](./docs/getting-started.md) — end-user
+  walkthrough. Useful as a script when you're guiding a user through
+  setup.
+- [`docs/providers.md`](./docs/providers.md) — per-vendor config cookbook
+  (MiniMax + Gitee, OpenAI, DeepSeek, Ollama, GLM, Gemini-compat, …) and
+  production gotchas around batch size, dim locking, retry, prompt
+  caching.
+- [`docs/server.md`](./docs/server.md) — full HTTP wire spec, security
+  posture, deployment notes.