foliq 0.1.0a1__tar.gz

foliq-0.1.0a1/.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: CI
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+  workflow_dispatch:
+  workflow_call:  # release.yml reruns these checks on the tagged commit
+
+jobs:
+  checks:
+    name: Checks and tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.14"]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+      - run: uv sync --frozen
+      # The same hooks developers run on commit and on push (see prek.toml).
+      - name: Commit hooks (ruff, cargo fmt, file checks)
+        run: uv run --frozen prek run --all-files --show-diff-on-failure
+      - name: Push hooks (ty, clippy, cargo test, pytest)
+        run: uv run --frozen prek run --all-files --stage pre-push
+
+  binary:
+    name: Standalone binary (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v5
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo test --locked -p foliq -p foliq-core -p foliq-cli

foliq-0.1.0a1/.github/workflows/docs.yml

@@ -0,0 +1,37 @@
+name: Docs
+
+# Builds the documentation site (zensical.toml, docs/) and publishes it to GitHub Pages.
+# Needs Settings > Pages > Source: "GitHub Actions".
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: true
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+      - uses: actions/configure-pages@v5
+      # Only the docs tools: building the site doesn't need the Rust extension.
+      - run: uv run --frozen --only-group docs zensical build --clean
+      - uses: actions/upload-pages-artifact@v4
+        with:
+          path: site
+      - id: deployment
+        uses: actions/deploy-pages@v4

foliq-0.1.0a1/.github/workflows/release.yml

@@ -0,0 +1,142 @@
+name: Release
+
+# Every push to main publishes a development build to TestPyPI, like `0.1.0a1.dev42`
+# (the version in Cargo.toml plus `.dev<run number>`, since an index never takes a version twice).
+# Push a tag like `v0.1.0a1` (the Python form of the Cargo.toml version) to publish that version
+# to PyPI and create a GitHub release.
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+  workflow_dispatch:  # the same as a push to main: a development build to TestPyPI
+
+permissions:
+  contents: read
+
+# A newer push to main replaces a development build still in progress; releases always finish.
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_type != 'tag' }}
+
+jobs:
+  ci:
+    name: CI
+    uses: ./.github/workflows/ci.yml
+
+  sdist:
+    name: Source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0  # to check that the tag is on main
+      - uses: astral-sh/setup-uv@v7
+      - name: Check the tag matches the version and is on main
+        if: github.ref_type == 'tag'
+        run: |
+          # Cargo.toml says `0.1.0-alpha.1`; the tag uses the Python form, `v0.1.0a1`.
+          version=$(uv run --no-project --with packaging python -c 'import tomllib; from packaging.version import Version; print(Version(tomllib.load(open("Cargo.toml", "rb"))["workspace"]["package"]["version"]))')
+          [ "$GITHUB_REF_NAME" = "v$version" ] || { echo "::error::Tag $GITHUB_REF_NAME but Cargo.toml says $version"; exit 1; }
+          git merge-base --is-ancestor "$GITHUB_SHA" origin/main || { echo "::error::$GITHUB_REF_NAME is not on main"; exit 1; }
+      - name: Stamp a development version
+        if: github.ref_type != 'tag'
+        run: uv run --no-project --python 3.12 scripts/stamp-dev-version.py ${{ github.run_number }}
+      - uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v6
+        with:
+          name: dist-sdist
+          path: dist
+
+  wheels:
+    name: Wheel (${{ matrix.target }}, ${{ matrix.runner }} ${{ matrix.manylinux }})
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        # The extension uses the stable ABI (abi3-py311): one wheel per platform covers every CPython >= 3.11.
+        include:
+          - { runner: ubuntu-latest, target: x86_64, manylinux: auto, smoke: true }
+          - { runner: ubuntu-24.04-arm, target: aarch64, manylinux: auto, smoke: true }
+          - { runner: ubuntu-latest, target: x86_64, manylinux: musllinux_1_2 }
+          - { runner: ubuntu-24.04-arm, target: aarch64, manylinux: musllinux_1_2 }
+          - { runner: macos-latest, target: aarch64, smoke: true }
+          - { runner: macos-latest, target: x86_64 }  # cross-compiled from Apple Silicon
+          - { runner: windows-latest, target: x64, smoke: true }
+    steps:
+      - uses: actions/checkout@v5
+      - uses: astral-sh/setup-uv@v7
+      - name: Stamp a development version
+        if: github.ref_type != 'tag'
+        run: uv run --no-project --python 3.12 scripts/stamp-dev-version.py ${{ github.run_number }}
+      - uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: ${{ matrix.manylinux }}
+          args: --release --locked --strip --out dist
+      # Install the wheel on the oldest supported Python, import it and run the CLI.
+      - name: Smoke test
+        if: matrix.smoke
+        shell: bash
+        run: |
+          uv run --isolated --no-project --python 3.11 --with dist/*.whl python -c "import foliq; print(foliq.__version__)"
+          uvx --python 3.11 --from dist/*.whl foliq --version
+      - uses: actions/upload-artifact@v6
+        with:
+          name: dist-${{ strategy.job-index }}
+          path: dist
+
+  pypi:
+    name: Publish to PyPI
+    if: github.ref_type == 'tag'
+    needs: [ci, sdist, wheels]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/foliq
+    permissions:
+      id-token: write  # trusted publishing: no API token anywhere
+    steps:
+      - uses: actions/download-artifact@v7
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  testpypi:
+    name: Publish to TestPyPI
+    if: github.ref_type != 'tag'
+    needs: [ci, sdist, wheels]
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/foliq
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v7
+        with:
+          pattern: dist-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  github-release:
+    name: GitHub release
+    needs: pypi
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Create the release with generated notes
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          flags="--generate-notes --verify-tag"
+          if [[ "$GITHUB_REF_NAME" =~ (a|b|rc)[0-9]+$ ]]; then flags+=" --prerelease"; fi
+          gh release create "$GITHUB_REF_NAME" $flags

foliq-0.1.0a1/.gitignore

@@ -0,0 +1,11 @@
+/target
+__pycache__/
+*.py[cod]
+*.so
+*.pyd
+.venv/
+.pytest_cache/
+.ruff_cache/
+dist/
+.foliq/
+site/

foliq-0.1.0a1/.python-version

@@ -0,0 +1 @@
+3.11

foliq-0.1.0a1/AGENTS.md

@@ -0,0 +1,95 @@
+# Notes for coding agents
+
+foliq indexes documents (PDF, Word, PowerPoint, Excel, HTML, Markdown) and lets
+AI agents search, read and cite them. It is a Rust engine with a CLI and a
+Python API. Read this file before changing anything.
+
+## Philosophy
+
+### 1. Small core, everything pluggable (like pi)
+
+foliq is a small engine with sockets, in the spirit of the pi coding agent:
+the core does little, does it fast, and every part can be replaced.
+
+- Each socket is one small Rust trait in `crates/foliq-core`: `Source`,
+  `Parser`, `Splitter`, `Store`, `Sink` (and `Embedder`, used from 0.2).
+  Built-in plugins implement the same traits as user plugins. The engine
+  never special-cases a built-in.
+- Every socket can be filled from Rust or from Python. Python classes plug in
+  through `foliq.Index(parsers=..., splitter=..., sinks=...)`. Rust plugins
+  depend on `foliq-core` only.
+- The `foliq` command runs Rust code only. It never starts Python (like ruff
+  or uv). Python plugins work through `import foliq` (like polars).
+- Rust does the heavy work: parsing, splitting, token counting, storage,
+  search, sync. Files are parsed in parallel and only changed files are
+  re-processed.
+- Start small. Add a feature when someone needs it, as a plugin if it can be
+  one. Prefer one obvious way to do each thing. If a change makes the core
+  bigger, ask whether it could be a plugin instead.
+
+### 2. Clean, organized, consistent code that reads well the first time
+
+- Names say what things are, in plain words: `document`, `chunks`,
+  `heading_path`, `content_hash`, `is_visible_to`. No abbreviations
+  (`doc`, `cfg`, `ctx`, `idx`) and no Rust jargon in names.
+- **Rust must be readable by a Python developer who has never seen Rust.**
+  Prefer simple, straight-line code over clever generics, macros, lifetimes
+  or iterator chains. When a Rust feature would surprise a Python developer
+  (`Option`, `?`, `Arc`, traits, `#[serde(...)]`), add a short comment,
+  ideally comparing it to Python.
+- Every file starts with a `//!` comment saying what it does. Every public
+  item has a `///` comment. Comments explain *why*, not *what*.
+- One job per file. Module docs list the files of a folder in a small table
+  (see `crates/foliq/src/index/mod.rs`).
+- Be consistent: same words for the same things everywhere (a "reference" is
+  `collection/path#section`, a "citation" adds `(p4, L10-22)`, a "principal"
+  is who is asking). The Python API, the CLI and the Rust API use the same
+  names.
+- The same rules apply to Python: small functions, clear names, no magic.
+
+## Map
+
+| Path                  | What it is                                                     |
+|-----------------------|----------------------------------------------------------------|
+| `crates/foliq-core`   | Types (`Document`, `Chunk`...) and plugin traits. Small, stable |
+| `crates/foliq`        | The engine: parsers, folder source, splitter, SQLite store, sinks, `Index` |
+| `crates/foliq-cli`    | The `foliq` command (`run(args)`), and `SKILL.md` for agents    |
+| `src/*.rs`            | The Python extension `foliq._core` (PyO3)                       |
+| `src/foliq`           | The Python package: tools for agents, plugin protocols, CLI entry |
+| `tests`               | Python tests, with generated DOCX/PPTX/XLSX/PDF fixtures         |
+| `benchmarks`          | Speed and quality against other libraries, on public corpora (own uv project) |
+| `docs`                | The documentation site (Zensical, `zensical.toml`)              |
+
+## Key ideas
+
+- Every format becomes Markdown, the one canonical text of a document.
+  Headings, tables and pages (`<!-- page N -->`) are found in that text, and
+  everything (elements, sections, chunks) is a byte range of it.
+- Sections are numbered like an outline (`2.3`). References and citations use
+  those numbers, so an agent can go from a search hit to the full section.
+- Access rules live on documents (`access`: a list of principals; empty means
+  everyone). The store filters before ranking, so results never leak.
+  `index.tools(principal=...)` binds the principal; it is never a tool argument.
+- Sync is incremental: version (mtime and size), then content hash, then the
+  parser and splitter fingerprints decide what is re-processed. A source that
+  can't be listed deletes nothing.
+
+## Workflow
+
+- Tools: uv, maturin, ruff, ty, pytest, cargo fmt, clippy. Hooks run with prek
+  (`prek.toml`): `uv run prek install` once.
+- `uv run pytest` builds the extension and runs the Python tests.
+  `scripts/cargo-test.sh --workspace` runs the Rust tests.
+- Tests use real files in temporary folders, built by `tests/fixtures.py`.
+  Avoid mocks.
+- Docs: `uv run --only-group docs zensical serve`. Update them with every
+  user-facing change. Code examples live in files and are included with
+  `--8<--`, so they are compiled and tested.
+- The version lives in one place: `[workspace.package]` in `Cargo.toml`.
+- Releases (`.github/workflows/release.yml`): every push to main publishes a
+  development build to TestPyPI (`0.1.0a1.dev42`, stamped by
+  `scripts/stamp-dev-version.py`). To publish to PyPI, bump the version with
+  `cargo set-version --bump alpha` (or `beta`, `rc`, `release`, `patch`...;
+  from `cargo install cargo-edit`), which updates Cargo.toml and Cargo.lock in
+  SemVer form (`0.1.0-alpha.2`). Merge to main, then push the tag in Python
+  form (`git tag v0.1.0a2 && git push origin v0.1.0a2`).

foliq-0.1.0a1/CLAUDE.md

@@ -0,0 +1,95 @@
+# Notes for coding agents
+
+foliq indexes documents (PDF, Word, PowerPoint, Excel, HTML, Markdown) and lets
+AI agents search, read and cite them. It is a Rust engine with a CLI and a
+Python API. Read this file before changing anything.
+
+## Philosophy
+
+### 1. Small core, everything pluggable (like pi)
+
+foliq is a small engine with sockets, in the spirit of the pi coding agent:
+the core does little, does it fast, and every part can be replaced.
+
+- Each socket is one small Rust trait in `crates/foliq-core`: `Source`,
+  `Parser`, `Splitter`, `Store`, `Sink` (and `Embedder`, used from 0.2).
+  Built-in plugins implement the same traits as user plugins. The engine
+  never special-cases a built-in.
+- Every socket can be filled from Rust or from Python. Python classes plug in
+  through `foliq.Index(parsers=..., splitter=..., sinks=...)`. Rust plugins
+  depend on `foliq-core` only.
+- The `foliq` command runs Rust code only. It never starts Python (like ruff
+  or uv). Python plugins work through `import foliq` (like polars).
+- Rust does the heavy work: parsing, splitting, token counting, storage,
+  search, sync. Files are parsed in parallel and only changed files are
+  re-processed.
+- Start small. Add a feature when someone needs it, as a plugin if it can be
+  one. Prefer one obvious way to do each thing. If a change makes the core
+  bigger, ask whether it could be a plugin instead.
+
+### 2. Clean, organized, consistent code that reads well the first time
+
+- Names say what things are, in plain words: `document`, `chunks`,
+  `heading_path`, `content_hash`, `is_visible_to`. No abbreviations
+  (`doc`, `cfg`, `ctx`, `idx`) and no Rust jargon in names.
+- **Rust must be readable by a Python developer who has never seen Rust.**
+  Prefer simple, straight-line code over clever generics, macros, lifetimes
+  or iterator chains. When a Rust feature would surprise a Python developer
+  (`Option`, `?`, `Arc`, traits, `#[serde(...)]`), add a short comment,
+  ideally comparing it to Python.
+- Every file starts with a `//!` comment saying what it does. Every public
+  item has a `///` comment. Comments explain *why*, not *what*.
+- One job per file. Module docs list the files of a folder in a small table
+  (see `crates/foliq/src/index/mod.rs`).
+- Be consistent: same words for the same things everywhere (a "reference" is
+  `collection/path#section`, a "citation" adds `(p4, L10-22)`, a "principal"
+  is who is asking). The Python API, the CLI and the Rust API use the same
+  names.
+- The same rules apply to Python: small functions, clear names, no magic.
+
+## Map
+
+| Path                  | What it is                                                     |
+|-----------------------|----------------------------------------------------------------|
+| `crates/foliq-core`   | Types (`Document`, `Chunk`...) and plugin traits. Small, stable |
+| `crates/foliq`        | The engine: parsers, folder source, splitter, SQLite store, sinks, `Index` |
+| `crates/foliq-cli`    | The `foliq` command (`run(args)`), and `SKILL.md` for agents    |
+| `src/*.rs`            | The Python extension `foliq._core` (PyO3)                       |
+| `src/foliq`           | The Python package: tools for agents, plugin protocols, CLI entry |
+| `tests`               | Python tests, with generated DOCX/PPTX/XLSX/PDF fixtures         |
+| `benchmarks`          | Speed and quality against other libraries, on public corpora (own uv project) |
+| `docs`                | The documentation site (Zensical, `zensical.toml`)              |
+
+## Key ideas
+
+- Every format becomes Markdown, the one canonical text of a document.
+  Headings, tables and pages (`<!-- page N -->`) are found in that text, and
+  everything (elements, sections, chunks) is a byte range of it.
+- Sections are numbered like an outline (`2.3`). References and citations use
+  those numbers, so an agent can go from a search hit to the full section.
+- Access rules live on documents (`access`: a list of principals; empty means
+  everyone). The store filters before ranking, so results never leak.
+  `index.tools(principal=...)` binds the principal; it is never a tool argument.
+- Sync is incremental: version (mtime and size), then content hash, then the
+  parser and splitter fingerprints decide what is re-processed. A source that
+  can't be listed deletes nothing.
+
+## Workflow
+
+- Tools: uv, maturin, ruff, ty, pytest, cargo fmt, clippy. Hooks run with prek
+  (`prek.toml`): `uv run prek install` once.
+- `uv run pytest` builds the extension and runs the Python tests.
+  `scripts/cargo-test.sh --workspace` runs the Rust tests.
+- Tests use real files in temporary folders, built by `tests/fixtures.py`.
+  Avoid mocks.
+- Docs: `uv run --only-group docs zensical serve`. Update them with every
+  user-facing change. Code examples live in files and are included with
+  `--8<--`, so they are compiled and tested.
+- The version lives in one place: `[workspace.package]` in `Cargo.toml`.
+- Releases (`.github/workflows/release.yml`): every push to main publishes a
+  development build to TestPyPI (`0.1.0a1.dev42`, stamped by
+  `scripts/stamp-dev-version.py`). To publish to PyPI, bump the version with
+  `cargo set-version --bump alpha` (or `beta`, `rc`, `release`, `patch`...;
+  from `cargo install cargo-edit`), which updates Cargo.toml and Cargo.lock in
+  SemVer form (`0.1.0-alpha.2`). Merge to main, then push the tag in Python
+  form (`git tag v0.1.0a2 && git push origin v0.1.0a2`).