nibs 0.1.0__tar.gz

nibs-0.1.0/.coderabbit.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=https://www.coderabbit.ai/integrations/schema.v2.json
+# Settings not listed here keep CodeRabbit's defaults (reviews.profile: chill).
+reviews:
+  auto_review:
+    # The release pull request that release.yml opens changes only CHANGELOG.md
+    # and the release manifest, both generated.
+    ignore_title_keywords:
+      - 'chore(main): release'

nibs-0.1.0/.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  # nibs has no dependencies to install; only the actions in the workflows.
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    # Wait three days after a release: compromised releases are usually found
+    # and removed within hours or days. Three days is also Dependabot's default.
+    cooldown:
+      default-days: 3
+    # Majors come one by one: they can break the workflow.
+    groups:
+      actions:
+        patterns: ['*']
+        update-types: [minor, patch]
+    commit-message:
+      prefix: ci

nibs-0.1.0/.github/release-please-config.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json",
+  "packages": {
+    ".": {
+      "release-type": "python",
+      "include-component-in-tag": false,
+      "bump-minor-pre-major": true,
+      "initial-version": "0.1.0"
+    }
+  }
+}

nibs-0.1.0/.github/release-please-manifest.json

@@ -0,0 +1 @@
+{".":"0.1.0"}

nibs-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,148 @@
+name: Release
+
+# Merging a pull request that bumps the version (feat:, fix:, a breaking change;
+# see CONTRIBUTING.md) makes release-please open or update a release pull
+# request, which changes CHANGELOG.md and .github/release-please-manifest.json.
+# Merging that tags v<version> and creates the GitHub release; the jobs below
+# then build, test and publish it to PyPI. The version of the package comes
+# from the tag. Nothing is published from a laptop.
+#
+# Run by hand (Run workflow on main, with the tag) to publish a tag again after
+# a failure outside the tagged code, such as the PyPI settings. If the tagged
+# code is at fault, release a new version instead.
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: Release tag to publish, e.g. v1.2.3
+        required: true
+
+permissions: {}
+
+# One release at a time, and never cancel one halfway.
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  release-please:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+      pull-requests: write
+      issues: write
+    outputs:
+      release_created: ${{ steps.release.outputs.release_created }}
+      tag_name: ${{ steps.release.outputs.tag_name }}
+    steps:
+      - id: release
+        uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
+        with:
+          config-file: .github/release-please-config.json
+          manifest-file: .github/release-please-manifest.json
+
+  # The build tool runs only here, and cannot publish. publish installs and runs
+  # nothing from the repository, and publishes only the files build made: their
+  # SHA-256 sums are build's outputs, which later jobs cannot change.
+  build:
+    needs: release-please
+    if: >-
+      !cancelled() && needs.release-please.result != 'failure' &&
+      (github.event_name == 'workflow_dispatch' || needs.release-please.outputs.release_created == 'true')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    env:
+      TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || needs.release-please.outputs.tag_name }}
+    outputs:
+      sha: ${{ steps.commit.outputs.sha }}
+      sums: ${{ steps.build.outputs.sums }}
+    steps:
+      - name: Check the tag
+        run: '[[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "not a release tag: $TAG"; exit 1; }'
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: refs/tags/${{ env.TAG }}
+          # The version is read from the tag, which needs the history.
+          fetch-depth: 0
+          persist-credentials: false
+      - id: commit
+        name: Check the commit
+        run: |
+          # By hand, only from main's own workflow, and only for a commit on main.
+          if [[ "$GITHUB_EVENT_NAME" == workflow_dispatch ]]; then
+            [[ "$GITHUB_REF" == refs/heads/main ]] || { echo "run it on main, not $GITHUB_REF"; exit 1; }
+            git merge-base --is-ancestor HEAD origin/main || { echo "$TAG is not on main"; exit 1; }
+          fi
+          echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+      - name: Check the version against PyPI
+        run: |
+          status=$(curl -s -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/nibs/${TAG#v}/json")
+          [[ "$status" == 404 ]] || { echo "${TAG#v} is already on PyPI (HTTP $status)"; exit 1; }
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.15"
+      - id: build
+        run: |
+          pip install build
+          python -m build --outdir "$RUNNER_TEMP/dist"
+          cd "$RUNNER_TEMP/dist"
+          version=${TAG#v}
+          [[ -f "nibs-$version-py3-none-any.whl" && -f "nibs-$version.tar.gz" ]] || { ls; echo "the files are not version $version"; exit 1; }
+          { echo 'sums<<EOF'; sha256sum ./*; echo EOF; } >> "$GITHUB_OUTPUT"
+      # The wheel as it will be published installs, and reports the version.
+      - run: |
+          python -m venv "$RUNNER_TEMP/venv"
+          "$RUNNER_TEMP/venv/bin/pip" install "$RUNNER_TEMP"/dist/*.whl
+          [[ "$("$RUNNER_TEMP/venv/bin/nibs" --version)" == "nibs ${TAG#v}" ]]
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: dist
+          path: ${{ runner.temp }}/dist
+          if-no-files-found: error
+          retention-days: 7
+
+  # Separate from build, so that the test tools never touch what is published.
+  test:
+    needs: build
+    if: ${{ !cancelled() && needs.build.result == 'success' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ needs.build.outputs.sha }}
+          persist-credentials: false
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: "3.15"
+      - run: pip install -e . pytest
+      - run: pytest -q
+
+  publish:
+    needs: [build, test]
+    if: ${{ !cancelled() && needs.build.result == 'success' && needs.test.result == 'success' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    # PyPI's trusted publisher accepts this environment only.
+    environment: pypi
+    permissions:
+      contents: read
+      id-token: write # PyPI trusted publishing
+    steps:
+      # No checkout: only the files build made are used.
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: dist
+          path: dist
+      - run: cd dist && sha256sum --check --strict <<< "$SUMS"
+        env:
+          SUMS: ${{ needs.build.outputs.sums }}
+      - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3

nibs-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,54 @@
+name: Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  test:
+    name: test (${{ matrix.os }}, Python ${{ matrix.python }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python: ["3.11", "3.12", "3.13", "3.14", "3.15"]
+        # Every supported Python on one OS; the others only at the oldest and
+        # newest, since what differs by OS (file paths, replacing the cache
+        # file) does not depend on the Python version.
+        include:
+          - os: macos-latest
+            python: "3.11"
+          - os: macos-latest
+            python: "3.15"
+          - os: windows-latest
+            python: "3.11"
+          - os: windows-latest
+            python: "3.15"
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
+        with:
+          python-version: ${{ matrix.python }}
+      - run: pip install -e . pytest
+      - run: pytest -q
+      # The wheel as it would be published installs, and its command starts.
+      - name: Install the wheel in a new environment
+        shell: bash
+        run: |
+          pip wheel --no-deps -w "$RUNNER_TEMP/wheel" .
+          python -m venv "$RUNNER_TEMP/venv"
+          if [[ -d "$RUNNER_TEMP/venv/Scripts" ]]; then bin="$RUNNER_TEMP/venv/Scripts"; else bin="$RUNNER_TEMP/venv/bin"; fi
+          "$bin/python" -m pip install "$RUNNER_TEMP"/wheel/nibs-*.whl
+          "$bin/nibs" --help

nibs-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.pyc
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.DS_Store

nibs-0.1.0/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+## 0.1.0 (2026-11-03)
+
+
+### Features
+
+* read a web page by its headings ([6e42fc1](https://github.com/nhinata/nibs/commit/6e42fc1571dc3a17e93febf0caed0db9d289857d))
+
+
+### Bug Fixes
+
+* close a code fence only with one at least as long ([#10](https://github.com/nhinata/nibs/issues/10)) ([e4c363d](https://github.com/nhinata/nibs/commit/e4c363d66e23bc12209a5d2cf74925f532b180b4))
+* keep article headers, forms and code blocks when reading HTML ([#4](https://github.com/nhinata/nibs/issues/4)) ([7ff6281](https://github.com/nhinata/nibs/commit/7ff62811e90ba1530804212c997fb0eea10782ad))
+* read only http and https URLs ([#2](https://github.com/nhinata/nibs/issues/2)) ([11dd070](https://github.com/nhinata/nibs/commit/11dd070561fb2a1fb682ecadaeab9d1224b24aed))
+* read the charset a page declares in a meta tag ([#5](https://github.com/nhinata/nibs/issues/5)) ([478c52e](https://github.com/nhinata/nibs/commit/478c52ead8fb284a0b1d8cdfb5c4bf145122b28c))
+* show a message instead of a traceback on broken input ([#6](https://github.com/nhinata/nibs/issues/6)) ([6d603c2](https://github.com/nhinata/nibs/commit/6d603c29036f55f68e8328f5e82c5cb74ea12495))

nibs-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,29 @@
+# Contributing
+
+## Development
+
+```sh
+python -m venv .venv
+.venv/bin/pip install -e . pytest
+.venv/bin/pytest -q
+```
+
+The tests do not reach the internet: they serve pages from a fake site (`tests/conftest.py`) or from a web server on localhost (`tests/test_fetch.py`).
+
+## Pull requests
+
+- **One topic per pull request.** Name the branch `<type>/<topic>`, such as `fix/sphinx-headings`.
+- **Titles follow [Conventional Commits](https://www.conventionalcommits.org/).** Pull requests are squash-merged, so the title becomes the commit on `main`, and the release tooling reads it to pick the next version:
+
+  | Prefix | Use for | Version bump |
+  | --- | --- | --- |
+  | `feat:` | a new command, option or output | minor |
+  | `fix:` | wrong output or a crash | patch |
+  | `feat!:` / `fix!:` | a change to the output or to a command that breaks instructions written for the old one | major (minor while 0.x) |
+  | `docs:`, `test:`, `refactor:`, `ci:`, `chore:` | everything else | none |
+
+- `main` is protected: every change goes through a pull request, and the tests must pass.
+
+## Releases
+
+Merging a pull request that bumps the version makes release-please open a release pull request. Merging that publishes the version to PyPI ([release.yml](.github/workflows/release.yml)).

nibs-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 nhinata
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nibs-0.1.0/PKG-INFO

@@ -0,0 +1,93 @@
+Metadata-Version: 2.5
+Name: nibs
+Version: 0.1.0
+Summary: Read a web page by its headings: list them, then print only the section you need
+Project-URL: Repository, https://github.com/nhinata/nibs
+Author: nhinata
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent,cli,llm,markdown,tokens,web
+Classifier: Environment :: Console
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+
+# nibs
+
+Read a web page by its headings: list them, then print only the section you need.
+
+An AI agent that reads a whole documentation page to answer one question puts tens of thousands of characters into its context, most of which it never uses. nibs lets it look at the page's headings first and then take just one section, in the page's own words.
+
+```console
+$ nibs toc https://docs.python.org/3/library/argparse.html
+82,019 chars, 57 headings
+  1.     Navigation  (254)
+  2. argparse — Parser for command-line options, arguments and subcommands¶  (81,665)
+  3.   ArgumentParser objects¶  (19,217)
+  4.     prog¶  (1,483)
+  5.     usage¶  (893)
+  ...
+$ nibs find https://docs.python.org/3/library/argparse.html allow_abbrev --limit 2
+## ArgumentParser objects¶
+  class argparse.ArgumentParser(prog=None, usage=None, description=None, epilog=None, ...
+  allow_abbrev - Allows long options to be abbreviated if the abbreviation is unambiguous (default: True)
+[nibs: stopped at 2 matches; pass --limit]
+$ nibs section https://docs.python.org/3/library/argparse.html 4
+```
+
+## Install
+
+```sh
+pipx install nibs      # or: uv tool install nibs
+```
+
+Python 3.11 or later. No other dependencies.
+
+## Commands
+
+```sh
+nibs toc <url>                  # headings, numbered, with each section's size in characters
+nibs section <url> <n|heading>  # one section, by its number in toc or by heading text
+nibs find <url> <regex>         # matching lines, each under the heading it belongs to
+nibs stats                      # characters printed so far, against reading each page whole
+```
+
+- `section` prints at most 8,000 characters; `--max` changes that. A heading given as text matches exactly first, then as a case-insensitive substring.
+- `find` prints at most 30 lines; `--limit` changes that.
+
+## How a page is read
+
+- Only `http` and `https` URLs are read, also when a page redirects.
+- If the site serves Markdown, either at the URL itself or at the URL with `.md` added, nibs uses it. Many documentation sites do.
+- Otherwise nibs turns the HTML into headings, text and code blocks (from `<pre>`). It leaves out scripts, styles, navigation, sidebars, and the page's own header and footer; a header or footer inside an article is kept. Only heading tags become headings, and only `<pre>` becomes a code block: a line of text that would read as either (see below) gets a backslash in front.
+- Control characters in the page are removed before printing.
+- nibs does not run JavaScript. A page that builds its content in the browser comes back with little or no text; when no headings are found, nibs says so.
+- A fetched page is kept for 15 minutes in `~/.cache/nibs` (or `$XDG_CACHE_HOME/nibs`), so `toc`, `section` and `find` on the same page fetch it once.
+
+### Headings and code blocks
+
+nibs splits a page by its Markdown headings. It reads only what it needs for that, by this rule rather than all of CommonMark:
+
+- A heading is a line that starts with 1 to 6 `#` and a space.
+- A line that starts with three or more `` ` `` or `~` opens a code block, after any spaces. The block ends at a line of the same character, at least as long, with nothing after it. Headings inside a code block are not headings.
+- Nothing else is read as structure: not headings underlined with `===` or `---`, not code indented by four spaces.
+
+## Using it from an agent
+
+Tell the agent in its instructions, for example in `CLAUDE.md` or `AGENTS.md`:
+
+```markdown
+To read documentation on the web, run `nibs toc <url>` to see the headings,
+then `nibs section <url> <n>` for the part you need, or `nibs find <url> <regex>`.
+```
+
+## The usage record
+
+Each run adds one line to `~/.local/state/nibs/usage.jsonl` (or `$XDG_STATE_HOME/nibs/usage.jsonl`): the time, the command, the URL, the page's size and how many characters were printed. `nibs stats` reads it. Set `NIBS_LOG` to another path to keep the record there. Nothing is sent anywhere.
+
+## License
+
+MIT

nibs-0.1.0/README.md

@@ -0,0 +1,76 @@
+# nibs
+
+Read a web page by its headings: list them, then print only the section you need.
+
+An AI agent that reads a whole documentation page to answer one question puts tens of thousands of characters into its context, most of which it never uses. nibs lets it look at the page's headings first and then take just one section, in the page's own words.
+
+```console
+$ nibs toc https://docs.python.org/3/library/argparse.html
+82,019 chars, 57 headings
+  1.     Navigation  (254)
+  2. argparse — Parser for command-line options, arguments and subcommands¶  (81,665)
+  3.   ArgumentParser objects¶  (19,217)
+  4.     prog¶  (1,483)
+  5.     usage¶  (893)
+  ...
+$ nibs find https://docs.python.org/3/library/argparse.html allow_abbrev --limit 2
+## ArgumentParser objects¶
+  class argparse.ArgumentParser(prog=None, usage=None, description=None, epilog=None, ...
+  allow_abbrev - Allows long options to be abbreviated if the abbreviation is unambiguous (default: True)
+[nibs: stopped at 2 matches; pass --limit]
+$ nibs section https://docs.python.org/3/library/argparse.html 4
+```
+
+## Install
+
+```sh
+pipx install nibs      # or: uv tool install nibs
+```
+
+Python 3.11 or later. No other dependencies.
+
+## Commands
+
+```sh
+nibs toc <url>                  # headings, numbered, with each section's size in characters
+nibs section <url> <n|heading>  # one section, by its number in toc or by heading text
+nibs find <url> <regex>         # matching lines, each under the heading it belongs to
+nibs stats                      # characters printed so far, against reading each page whole
+```
+
+- `section` prints at most 8,000 characters; `--max` changes that. A heading given as text matches exactly first, then as a case-insensitive substring.
+- `find` prints at most 30 lines; `--limit` changes that.
+
+## How a page is read
+
+- Only `http` and `https` URLs are read, also when a page redirects.
+- If the site serves Markdown, either at the URL itself or at the URL with `.md` added, nibs uses it. Many documentation sites do.
+- Otherwise nibs turns the HTML into headings, text and code blocks (from `<pre>`). It leaves out scripts, styles, navigation, sidebars, and the page's own header and footer; a header or footer inside an article is kept. Only heading tags become headings, and only `<pre>` becomes a code block: a line of text that would read as either (see below) gets a backslash in front.
+- Control characters in the page are removed before printing.
+- nibs does not run JavaScript. A page that builds its content in the browser comes back with little or no text; when no headings are found, nibs says so.
+- A fetched page is kept for 15 minutes in `~/.cache/nibs` (or `$XDG_CACHE_HOME/nibs`), so `toc`, `section` and `find` on the same page fetch it once.
+
+### Headings and code blocks
+
+nibs splits a page by its Markdown headings. It reads only what it needs for that, by this rule rather than all of CommonMark:
+
+- A heading is a line that starts with 1 to 6 `#` and a space.
+- A line that starts with three or more `` ` `` or `~` opens a code block, after any spaces. The block ends at a line of the same character, at least as long, with nothing after it. Headings inside a code block are not headings.
+- Nothing else is read as structure: not headings underlined with `===` or `---`, not code indented by four spaces.
+
+## Using it from an agent
+
+Tell the agent in its instructions, for example in `CLAUDE.md` or `AGENTS.md`:
+
+```markdown
+To read documentation on the web, run `nibs toc <url>` to see the headings,
+then `nibs section <url> <n>` for the part you need, or `nibs find <url> <regex>`.
+```
+
+## The usage record
+
+Each run adds one line to `~/.local/state/nibs/usage.jsonl` (or `$XDG_STATE_HOME/nibs/usage.jsonl`): the time, the command, the URL, the page's size and how many characters were printed. `nibs stats` reads it. Set `NIBS_LOG` to another path to keep the record there. Nothing is sent anywhere.
+
+## License
+
+MIT

nibs-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "nibs"
+dynamic = ["version"]
+description = "Read a web page by its headings: list them, then print only the section you need"
+readme = "README.md"
+license = "MIT"
+authors = [{ name = "nhinata" }]
+license-files = ["LICENSE"]
+requires-python = ">=3.11"
+dependencies = []
+keywords = ["llm", "agent", "tokens", "web", "markdown", "cli"]
+classifiers = [
+  "Environment :: Console",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Topic :: Internet :: WWW/HTTP",
+  "Topic :: Utilities",
+]
+
+[project.urls]
+Repository = "https://github.com/nhinata/nibs"
+
+[project.scripts]
+nibs = "nibs.cli:main"
+
+[tool.hatch.version]
+source = "vcs"
+fallback-version = "0.0.0"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nibs"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

nibs-0.1.0/src/nibs/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("nibs")
+except PackageNotFoundError:  # running from a checkout that was never installed
+    __version__ = "unknown"

nibs-0.1.0/src/nibs/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

nibs-0.1.0/src/nibs/cache.py

@@ -0,0 +1,48 @@
+"""Pages already read, so that toc, section and find on one page fetch it once."""
+
+import hashlib
+import json
+import os
+import time
+
+from .paths import cache_dir
+
+# How long a fetched page is reused; the same default as Claude Code's WebFetch.
+TTL_SECONDS = 15 * 60
+
+
+def _path(url):
+    return cache_dir() / (hashlib.sha256(url.encode()).hexdigest() + ".json")
+
+
+def get(url):
+    try:
+        with open(_path(url), encoding="utf-8") as f:
+            entry = json.load(f)
+    except (OSError, ValueError):
+        return None
+    if entry.get("url") != url or time.time() - entry.get("time", 0) > TTL_SECONDS:
+        return None
+    return entry.get("text")
+
+
+def put(url, text):
+    try:
+        cache_dir().mkdir(parents=True, exist_ok=True)
+        _prune()
+        tmp = _path(url).with_suffix(".tmp")
+        with open(tmp, "w", encoding="utf-8") as f:
+            json.dump({"url": url, "time": time.time(), "text": text}, f)
+        os.replace(tmp, _path(url))
+    except OSError:
+        pass  # A page that cannot be cached is fetched again next time.
+
+
+def _prune():
+    now = time.time()
+    for p in cache_dir().glob("*.json"):
+        try:
+            if now - p.stat().st_mtime > TTL_SECONDS:
+                p.unlink()
+        except OSError:
+            pass