alvis 1.0.0rc1__tar.gz

alvis-1.0.0rc1/.dockerignore

@@ -0,0 +1,13 @@
+.git
+.github
+.venv
+.mypy_cache
+.pytest_cache
+.ruff_cache
+__pycache__
+*.py[cod]
+docker
+docs
+examples
+scripts
+tests

alvis-1.0.0rc1/.github/workflows/ci.yml

@@ -0,0 +1,99 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: ruff check src tests
+
+      - name: Typecheck (mypy)
+        run: mypy src/alvis
+
+      - name: Tests (pytest)
+        run: pytest --cov=alvis --cov-report=term-missing --cov-fail-under=90
+
+  golden:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Golden tests (read-only snapshots)
+        run: pytest -m golden
+
+      - name: Golden coverage (every connector ships fixture + snapshot)
+        run: python scripts/check_golden_coverage.py
+
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install (runtime deps + extras)
+        run: pip install -e ".[documents,pgindex,observability]"
+
+      - name: Audit dependencies (pip-audit)
+        run: pip install pip-audit && pip-audit
+
+  integration:
+    name: Qdrant integration
+    runs-on: ubuntu-latest
+    services:
+      qdrant:
+        image: qdrant/qdrant:v1.13.6
+        ports:
+          - 6333:6333
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Wait for Qdrant
+        run: |
+          for attempt in {1..30}; do
+            if curl --fail --silent http://localhost:6333/readyz >/dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          docker logs "${{ job.services.qdrant.id }}"
+          exit 1
+
+      - name: Filesystem -> Qdrant -> query -> incremental -> reconcile
+        env:
+          ALVIS_QDRANT_URL: http://localhost:6333
+        run: pytest -m integration

alvis-1.0.0rc1/.github/workflows/publish.yml

@@ -0,0 +1,79 @@
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  package:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/alvis/
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Validate release tag
+        env:
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+        run: python scripts/check_release.py
+
+      - name: Install build tools
+        run: python -m pip install --upgrade build twine
+
+      - name: Build distributions
+        run: python -m build
+
+      - name: Validate distributions
+        run: python -m twine check dist/*
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  docker:
+    name: Build and publish Docker image
+    runs-on: ubuntu-latest
+    needs: package
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up QEMU (multi-arch builds)
+        uses: docker/setup-qemu-action@v3
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: bzdvdn/alvis
+          tags: |
+            type=ref,event=tag
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }}
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push multi-platform image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64,linux/arm64

alvis-1.0.0rc1/.gitignore

@@ -0,0 +1,22 @@
+# Local (never committed) — RU constitution copy
+CONSTITUTION.ru.md
+
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+htmlcov/
+# incremental ingestion state
+.alvis/
+
+# port pilot (workspace, not repo content)
+pilot/

alvis-1.0.0rc1/CHANGELOG.md

@@ -0,0 +1,40 @@
+# Changelog
+
+All notable changes to Alvis are documented here.
+
+## [1.0.0rc1] - 2026-08-20
+
+First release candidate for the stable Alvis contract.
+
+### Added
+
+- Versioned YAML pipeline schema (`schema_version: 1`) and Canonical Content Tree v1.
+- Typed Python DSL with parity to the YAML contract.
+- Filesystem, Confluence, GitHub, GitLab, S3, and static URL sources.
+- Memory, Qdrant, and optional pgvector indexes.
+- Incremental ingestion, cheap remote listing fingerprints, reconcile, and watch mode.
+- OpenAI-compatible embeddings with batching, retries, and optional caching.
+- Retrieval APIs (`alvis.query`, `alvis.answer`) and CLI commands.
+- Plugin SDK with discovery, validation, and a documented deprecation policy.
+- Structured logging, metrics, optional Prometheus/OpenTelemetry support, and status reporting.
+- Versioned Qdrant payload fields with reserved `__` system namespace and `__document_id`.
+- Multi-platform Docker image build configuration for `bzdvdn/alvis`.
+
+### Breaking Changes
+
+- Product, import package, CLI, and distribution were renamed from `winnow` to `alvis`.
+- The default incremental state directory is now `.alvis/`.
+- The default CLI command is now `alvis`.
+
+### Known Limitations
+
+- Qdrant point identity and reconcile currently use source URI; document-id-ledger
+  ownership is planned for a later release.
+- Metadata filters, hybrid search, reranking, multi-turn history, and LLM-based
+  evaluation remain v2.x scope.
+- The `default` embedder is deterministic and intended for development/testing;
+  production deployments should configure an OpenAI-compatible embedder.
+
+## [0.6.0]
+
+See the repository history for pre-Alvis development releases.

alvis-1.0.0rc1/CONSTITUTION.md

@@ -0,0 +1,110 @@
+# Alvis — Knowledge Ingestion Engine
+
+**Project Constitution**
+
+**Version:** 1.2.0  
+**Status:** Architecture baseline  
+**Language:** Python 3.10+  
+**Primary backends:** Qdrant, PostgreSQL/pgvector  
+**Primary sources:** GitLab, Confluence, S3  
+**Primary formats:** Markdown, HTML, PDF, DOCX, XLSX, CSV, source code
+
+---
+
+# 1. Purpose
+
+Alvis is an internal framework for building corporate knowledge bases from heterogeneous sources.
+
+The core pipeline:
+
+    Source
+      ↓
+    Artifact
+      ↓
+    Extraction
+      ↓
+    Canonical Content Tree
+      ↓
+    Transformation
+      ↓
+    Chunking
+      ↓
+    Embedding
+      ↓
+    Index
+
+The framework must allow users to assemble an ingestion pipeline declaratively,
+without writing Python code — a YAML descriptor must be sufficient for any
+pipeline (no-code by default). The typed Python DSL (`alvis.dsl`) is a
+first-class alternative that produces the exact same config contract, and is
+verified to run identically to its YAML twin.
+
+Example:
+
+```yaml
+pipeline:
+  source:
+    type: confluence
+
+  extract:
+    strategy: auto
+
+  chunk:
+    strategy: auto
+    config:
+      max_tokens: 500
+      overlap: 50
+
+  embed:
+    type: default
+
+  index:
+    type: qdrant
+```
+
+# 2. Scope
+
+## In scope
+- Pluggable source adapters (GitLab, Confluence, S3) and format parsers (Markdown, HTML, PDF, DOCX, XLSX, CSV, source code).
+- Configurable extraction, chunking, and transformation strategies exposed via YAML.
+- Canonical Content Tree as the intermediate representation between extraction and indexing.
+- Embedding and indexing into a pluggable vector store.
+- Idempotent, deduplicated ingestion with retry and error-tolerance semantics.
+
+## Out of scope
+- Building end-user applications (chatbots, search UIs) on top of the indexed data.
+- Managing source-of-truth systems (GitLab, Confluence, S3) themselves.
+- Embedding model training.
+
+# 3. Open Source Strategy
+
+Alvis is distributed and developed as an open-source project.
+
+## Principles
+- **Framework, not app.** The core must stay source/format-agnostic; all integration lives behind pluggable adapters.
+- **Declarative by default.** Users assemble pipelines via YAML; the typed Python DSL is a first-class alternative with the same contract. Python remains the language for plugin SDK extensions.
+- **First impression wins.** Time-to-hello-world is a hard quality gate: `pip install alvis` + one YAML must produce a running index.
+
+## License
+- **Apache 2.0.** Permissive for corporate adopters, allows proprietary integration without forcing contribution back.
+- CLA required for external contributors to keep the licensing path clean.
+- A separate managed/hosted offering (future, optional) is a distinct distribution, never a fork of the core.
+
+## Repository & governance
+- Single main repo (`alvis/alvis`) for core + CLI.
+- Adapters/bundles may live in the same repo under `plugins/` until they prove stability, then graduate to the mono-repo or their own repos as the community demands.
+- `CONTRIBUTING.md` contract: "how to write a connector" is the cover-page of the contributor guide, not an appendix.
+- Discussions/issues on GitHub; RFCs for anything that changes the YAML contract or Canonical Content Tree.
+
+## Contributor model
+- Core team: pipeline core, Canonical Content Tree, embedding/index abstraction.
+- Community: connectors, format parsers, chunking strategies.
+- Review rule: any plugin lands only with a hello-world example + test fixture + CI golden test.
+
+# 4. Roadmap
+
+- v0.1 — CLI (`alvis init/run/validate`), YAML schema validation, hello-world on Confluence → Qdrant
+- v0.2 — docs site, contributor guide, first external connector PRs
+- v1.0 — core pipeline stable contract (Canonical Content Tree v1), pgvector + GitLab/S3 sources
+- v1.1 — plugin SDK for third-party source/strategy extensions
+- v2.0 — stable plugin API, community-managed connector registry

alvis-1.0.0rc1/CONTRIBUTING.md

@@ -0,0 +1,244 @@
+# Contributing to Alvis
+
+Thanks for contributing. Alvis is a no-code knowledge ingestion engine, and
+most of its value is in breadth: the more sources, formats, and stores it
+speaks, the more useful it is. This guide is the cover page for **how to write
+a connector** — the single most common contribution.
+
+Read [CONSTITUTION.md](CONSTITUTION.md) first: the roadmap and cross-cutting
+rules live there and in [ROADMAP.md](ROADMAP.md).
+
+---
+
+## Development setup
+
+```bash
+python -m venv .venv
+.venv/bin/pip install -e ".[dev]"
+.venv/bin/ruff check src tests
+.venv/bin/mypy src/alvis
+.venv/bin/python -m pytest
+```
+
+- Python ≥ 3.10, ruff (lint), mypy `--strict`, pytest (asyncio auto mode).
+- A feature ships only with its documentation (cross-cutting rule) and its
+  tests — see "Connector fixture + golden test" below, which is enforced in CI.
+- YAML contract and Canonical Content Tree changes are RFC-first: open a
+  proposal before touching `config/models.py` or `core/models.py`.
+
+---
+
+## How to write a connector
+
+A connector turns a source of documents into Alvis `Artifact`s so the rest of
+the pipeline (extract → chunk → embed → index) can process them. Alvis never
+reads the raw bytes of a source itself — it always goes through a connector's
+`fetch()`.
+
+### 1. The contract
+
+`src/alvis/sources/base.py` defines the interface:
+
+```python
+class Source(Protocol):
+    async def fetch(self) -> list[Artifact]: ...
+```
+
+`Artifact` (`alvis.core.models`) carries exactly:
+
+| field          | meaning                                                            |
+| -------------- | ------------------------------------------------------------------ |
+| `step_id`      | stable, unique id for this document (page id, blob sha, ...)        |
+| `uri`          | canonical URL/path the document lives at (used for retrieval links) |
+| `content_type` | MIME type Alvis's extract stage understands (see step 3)           |
+| `data`         | the raw bytes to be ingested                                        |
+| `metadata`     | extra fields surfaced to users; keep it small and serialisable      |
+
+`fetch()` either returns artifacts or raises `SourceError`
+(`alvis.sources.base`) with an optional `status_code`.
+
+**Optional: cheap listing (recommended).** If your source can fingerprint a
+document without downloading its body — an S3 ETag, a GitLab/GitHub blob sha, a
+Confluence page version — also implement `ListingSource`
+(`alvis.sources.base`):
+
+```python
+class ListingSource(Source, Protocol):
+    async def list_documents(self) -> list[DocumentMeta]: ...
+    async def fetch(self, *, uris: set[str] | None = None) -> list[Artifact]: ...
+```
+
+- `list_documents()` returns a `DocumentMeta` (`uri`, `step_id`, `content_type`,
+  `fingerprint`) per document, doing **only** the cheap scan call(s).
+- `fetch(uris=...)` must download **only** the requested URIs — `uris=None`
+  means "everything". The engine's `--incremental` mode lists first, skips
+  documents whose stored fingerprint already matches, and calls
+  `fetch(uris=...)` with just the wanted URIs, so unchanged remote objects are
+  never fetched. A connector that skips this interface falls back to
+  content-hash fingerprinting (still skips processing, not downloads).
+- `gitlab.py` / `github.py` / `s3.py` / `confluence.py` are complete
+  `ListingSource` examples.
+
+### 2. Layout and naming
+
+Look at how the built-ins are organised — each adapter is one small module:
+
+- `src/alvis/sources/confluence.py` — a REST API adapter, the best template
+  for an HTTP connector.
+- `src/alvis/sources/fs.py` — the local-filesystem adapter.
+- `src/alvis/sources/github.py`, `gitlab.py`, `s3.py` — REST adapters that
+  accept a `transport=` kwarg for testing.
+
+For an HTTP connector that talks to a REST API, use `HttpClient`
+(`alvis.sources.http`): it gives you retries/backoff on 429/5xx/network
+errors, `Retry-After` honoring, token/basic auth from env vars
+(`api_token_env`, `username`), optional `verify=False` for self-signed TLS, and
+a `max_bytes` cap so oversized responses are aborted mid-read instead of
+buffered. Confluence is a 70-line example of a full adapter.
+
+Reach for `transport=` kwargs / `header_hook=` when the service needs special
+signing (see `s3.py` for a SigV4 example).
+
+### 3. Wire it in — the five touch points
+
+1. **Module** — `src/alvis/sources/<name>.py` implementing `fetch()`.
+2. **Export** — add it to `__all__` in `src/alvis/sources/__init__.py`.
+3. **Registry** — `src/alvis/registry.py`: add the type string to
+   `KNOWN_SOURCES` so `alvis validate` recognises it.
+4. **Factory** — `src/alvis/factories.py`:
+   - `build_source(...)` — map `config.type == "<name>"` to your class.
+   - `source_identity(config)` — a stable `"<name>:<key>@<host>"` string; the
+     index uses it to namespace points so `reconcile` prunes only your source's
+     stale documents. Pick discriminator keys that uniquely identify a data
+     scope (e.g. GitLab: `project@host`).
+5. **DSL** — `src/alvis/dsl.py`: add a `dsl.<name>(...)` builder mirroring the
+   YAML config keys, so the Python DSL and the YAML schema stay in parity.
+
+Config validation lives in `src/alvis/config/models.py` (`SourceConfig` is
+permissive `dict` today; type-specific keys are validated in the adapter).
+Document the YAML keys in the README source table.
+
+### 4. Content types and formats
+
+`alvis.sources.content_types` maps extensions to MIME types. Your connector
+should produce content types the `auto` extractor understands — Markdown
+(`text/markdown`), HTML (`text/html`), plain text / code (`text/plain`), or
+PDF/DOCX/XLSX for the `alvis[documents]` extra. If a remote format needs an
+extension (e.g. GitLab raw blobs), derive it from the object name and pass it
+through `content_type()`.
+
+### 5. Every connector ships two things — enforced in CI
+
+No connector lands without:
+
+1. **A fixture** — a `tests/fixtures/` corpus of realistic sample data the
+   adapter ingests (a couple of small files is plenty). Fixtures must not live
+   under a directory Alvis itself would ingest in tests.
+2. **A golden test** — a test marked `@pytest.mark.golden` that runs the
+   connector against canned service responses and diffs the `Artifact` output
+   against a committed snapshot.
+
+`src/alvis/testing.py` provides the harness (used by every built-in in
+`tests/test_golden_sources.py`):
+
+```python
+from alvis.testing import MockServer, artifacts_snapshot, assert_golden
+
+@pytest.mark.golden
+async def test_golden_my_source() -> None:
+    server = MockServer()
+    server.on("GET", "/api/v1/items", json_payload={"items": [{"id": "a", "body": "# Hi"}]})
+
+    source = MySource(url="http://example.com", transport=server.transport)
+    artifacts = await source.fetch()
+
+    assert_golden("my_source", artifacts_snapshot(artifacts))
+```
+
+- `MockServer` is an `httpx.MockTransport` under the hood — no network. Point
+  your `HttpClient` at `server.transport` (via a `transport=` kwarg, or
+  `source.client.transport = server.transport` when the constructor builds its
+  own client, as Confluence does). **Every request must have a route** —
+  unimapped routes raise, so the fixture doubles as a contract check on which
+  endpoints your connector calls. `server.request_log()` lets you assert on
+  the calls themselves.
+- `artifacts_snapshot(...)` renders the `fetch()` result as a stable,
+  serialisable list so the snapshot is diff-able in review. Pass `root=` to
+  scrub machine-specific absolute paths (see the `fs` golden test).
+- Snapshots live in `tests/golden/<name>.golden.json`. Run
+  `ALVIS_ACCEPT=1 pytest -m golden` to (re)write them, review the diff, and
+  commit. CI runs `pytest -m golden` **read-only** (no `ALVIS_ACCEPT`), so
+  drift and missing snapshots fail loudly.
+
+**Register the golden test in `tests/golden/manifest.json`** under the
+connector's type string pointing at the test file. The CI script
+`scripts/check_golden_coverage.py` fails if a connector has a manifest entry
+but no test/snapshot — keep the three in a single commit.
+
+### 6. Beyond the connector
+
+If you add a new parser, chunker, embedder, or indexer, follow the same shape:
+module → export → `KNOWN_*` in `registry.py` → `build_*` in `factories.py` →
+`dsl.*` builder → docs. Indexers that hold state (e.g. a new vector store)
+must implement idempotent upsert + a `reconcile` pass for the per-source prune
+to work.
+
+## Write a plugin, not a fork (v1.1)
+
+You do **not** need to modify this repository to ship an adapter. A plugin is
+an ordinary installed Python package that registers itself under the
+`alvis.plugins` entry-point group and declares a `alvis.plugin.Plugin`:
+
+```toml
+# pyproject.toml
+[project.entry-points."alvis.plugins"]
+kb-catalog = "kb_plugin:plugin"
+```
+
+```python
+# kb_plugin/__init__.py
+from alvis.plugin import Plugin
+
+def _catalog(*, config, max_bytes=None):   # factory contract (see alvis/plugin.py)
+    ...
+
+plugin = Plugin(
+    name="kb-catalog",
+    version="0.1.0",
+    sources={"catalog": _catalog},         # also: extractors, chunkers, embedders, indexers
+)
+```
+
+After `pip install .`, the adapter is accepted by `alvis validate`, listed by
+`alvis plugins`, and resolved by the stage factories — zero core changes.
+Plugins are validated their own adapter (mirroring built-ins), so plugin types
+may read any `config:` keys. The reference implementation is
+`examples/kb-plugin` (a `catalog` source with cheap-listing incremental
+support); copy it as your template and keep the tutorial contract: hello-world
+example + test + docs. External plugins are tracked in [docs/plugins.md]
+(docs/plugins.md).
+
+---
+
+## Contribution process
+
+1. Fork, create a branch (`add-magnolia-connector`).
+2. Implement the connector + fixture + golden test + docs + README table row
+   as one commit — the golden manifest, snapshot, and test never split.
+3. Run the full gate: `ruff check src tests`, `mypy src/alvis`,
+   `pytest` (which includes `-m golden` read-only), and
+   `python scripts/check_golden_coverage.py`.
+4. Open the pull request. CI runs lint, typecheck, the full suite on
+   Python 3.10/3.11/3.12, and a dedicated `golden` job that re-runs snapshots
+   read-only and enforces coverage.
+
+## Cross-cutting rules (from ROADMAP)
+
+- Every plugin lands only with: hello-world example + test fixture + CI golden
+  test.
+- Any change to the YAML contract or Canonical Content Tree is an RFC-first
+  change.
+- No feature ships without the chapter it documents.
+- Breaking a stable-surface name (see [docs/versioning.md](docs/versioning.md))
+  needs a semver `MAJOR` (or a `MINOR` before `1.0.0`), a `deprecated()`
+  warning cycle, and a changelog line — never a silent break.

alvis-1.0.0rc1/Dockerfile

@@ -0,0 +1,39 @@
+# syntax=docker/dockerfile:1
+
+# Alvis as a container. The product surface is YAML, so the image is a
+# fixed `alvis` CLI with every optional extra preinstalled — users configure
+# it purely by mounting pipeline files and a state volume, never by rebuilding:
+#
+#   docker build -t alvis:dev .
+#   docker run --rm \
+#     -v "$PWD/pipeline.yaml:/workspace/pipeline.yaml:ro" \
+#     -v "$PWD/.alvis:/workspace/.alvis" \
+#     alvis:dev run --incremental pipeline.yaml
+#
+# State defaults to `.alvis/state.json` under the workdir (`/workspace`),
+# which is exactly the mounted volume above.
+
+FROM python:3.12-slim AS builder
+
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+
+RUN pip install --no-cache-dir .[documents,pgindex,observability]
+
+FROM python:3.12-slim
+
+COPY --from=builder /usr/local /usr/local
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+RUN adduser --disabled-password --gecos "" --home /workspace alvis \
+    && chown -R alvis:alvis /workspace
+
+USER alvis
+WORKDIR /workspace
+
+VOLUME ["/workspace"]
+
+ENTRYPOINT ["alvis"]
+CMD ["--help"]