py-hamt 3.0.0__tar.gz

py_hamt-3.0.0/.github/workflows/pages-main.yaml

@@ -0,0 +1,49 @@
+name: Deploy static site generated by pdoc to GitHub Pages
+
+on:
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6
+        with:
+          version: "latest"
+
+      - name: Create project environment
+        run: uv sync
+
+      - name: Build with pdoc
+        run: uv run pdoc py_hamt -o ./_site
+
+      - name: Upload artifact
+        # Automatically uploads an artifact from the './_site' directory by default
+        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4

py_hamt-3.0.0/.github/workflows/run-checks.yaml

@@ -0,0 +1,54 @@
+name: Run checks
+run-name: Triggered on push from ${{ github.actor }} to branch/tag ${{ github.ref_name }}
+on: push
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: pre-commit/action@v3.0.1
+  test:
+    name: Create project environment, run all checks
+    needs:
+      - validate
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.12"]
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6
+        with:
+          version: "latest"
+
+      - name: Create project environment
+        run: uv sync
+
+      - name: Install IPFS
+        uses: oduwsdl/setup-ipfs@e92fedca9f61ab9184cb74940254859f4d7af4d9 # v0.6.3
+        with:
+          ipfs_version: "0.35.0"
+          run_daemon: true
+
+      - name: Run pytest with coverage
+        run: uv run pytest --ipfs --cov=py_hamt tests/ --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Check coverage
+        run: uv run coverage report --fail-under=100 --show-missing

py_hamt-3.0.0/.gitignore

@@ -0,0 +1,159 @@
+.DS_Store
+pyrightconfig.json
+
+*.prof
+memray*
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

py_hamt-3.0.0/.pre-commit-config.yaml

@@ -0,0 +1,32 @@
+repos:
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v4.0.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-ast
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.11.11
+    hooks:
+      - id: ruff-check
+      - id: ruff-format
+  - repo: https://github.com/ariebovenberg/slotscheck
+    rev: v0.17.1
+    hooks:
+      - id: slotscheck
+        name: slotscheck
+        entry: bash -c 'env PYTHONPATH=src slotscheck'
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.15.0"
+    hooks:
+      - id: mypy

py_hamt-3.0.0/.python-version

@@ -0,0 +1 @@
+3.12

py_hamt-3.0.0/AGENTS.md

@@ -0,0 +1,127 @@
+# Project Agents.md Guide for py-hamt
+
+This AGENTS.md file provides comprehensive guidance for OpenAI Codex and other AI agents working with this codebase.
+
+## Project Description
+
+This library is a python implementation of a HAMT, inspired by [rvagg's IAMap project written in JavaScript](https://github.com/rvagg/iamap).
+
+py-hamt provides efficient storage and retrieval of large sets of key-value mappings in a content-addressed storage system. The main target is IPFS, and the data model used is IPLD.
+
+dClimate primarily created this for storing large [zarrs](https://zarr.dev/) on IPFS. To see this in action, see our [data ETLs](https://github.com/dClimate/etl-scripts).
+
+## Project Architecture
+
+A user instantiates a ContentAddressedStore such as KuboCAS which is then used to instantiate a HAMT. This HAMT can then be passed to a ZarrHAMTStore to read/write zarr files. An IPFS daemon running locally can be used or a remote one can be accessed via gateway(reads) & rpc endpoints(writes) on their IPFS defaults. For tests, a local docker container is spun up if docker is available.
+
+## Project Structure for OpenAI Codex Navigation
+
+- `/py_hamt`: Source code of the py-hamt library
+    - `encryption_hamt_store.py` - Example using total encryption
+    - `hamt.py`- where the HAMT data structure is constructed and accesed
+    - `store.py`- Where various stores live that writes the data. Primarily used KuboCAS
+    - `zarr_hamt_store.py` - ZarrHAMTStore class used to directly write or read zarrs leveraging the HAMT data structure onto a Store usually a Content Addressed store like KuboCAS located in store.py.
+- `/tests`: Test files that should be maintained and extended where possible.
+
+## Coding Conventions
+
+### General Conventions for AGENTS.md Implementation
+
+- Use the latest version of Python (3.12) for all new code generated by OpenAI Codex
+- The AI Agent should follow the existing code style in each file
+- Agents.md requires meaningful variable and function names
+- The AI Agent should add comments for complex logic as guided by Agents.md
+
+### Python Guidelines
+- Use functional easy to understand functions wherever possible.
+- All functions need to be fully typed with mypy.
+- All functions require tests to be written and coverage must be full.
+- Example usage can be found in the `/tests` folder
+
+### Development Setup
+
+`py-hamt` uses uv for package management. If uv is not already present which can be checked with `uv` then it can be installed via
+
+`curl -LsSf https://astral.sh/uv/install.sh | sh`
+
+otherwise if uv is already present you can run
+
+`uv sync` to install all dependencies
+
+and then
+
+`source .venv/bin/activate` to activate the venv created by uv.
+
+Lastly while still setting up you can run
+
+```
+pre-commit install
+```
+
+to ensure pre-commit installed.
+
+Docker is also used for integration tests to test ipfs. Try to have docker installed to be able to run all tests with` pytest --ipfs`
+
+## Testing Requirements for the Agent
+
+All tests should be created within the `/tests` directory.
+
+The agent should run tests with the following commands:
+
+```bash
+# Run all tests
+pytest --ipfs
+
+# Run specific test file
+pytest test tests/<insert_file_here>
+
+# Run tests with coverage
+pytest --ipfs --cov
+
+# If IPFS is not present use
+pytest --cov
+```
+
+Tests should be kept as closed to 100% as possible.
+
+## Pull Request Guidelines for the Agent
+
+When the Agent helps create a PR, please ensure it:
+
+1. Includes a clear description of the changes as guided by AGENTS.md
+2. References any related issues that the Agent is addressing
+3. Ensures all tests pass for code generated by the Agent
+4. Keeps PRs focused on a single concern as specified in AGENTS.md
+
+## Programmatic Checks for the Agent
+
+Before submitting changes generated by the Agent, run:
+
+```bash
+bash run-checks.sh
+```
+
+**Note:** This assumes ipfs is running as `run-checks.sh` runs with the ``--ipfs` flag. If kubo ipfs is not running in a local daemon or docker was unable to instantiate simply run
+
+```
+pytest --cov
+```
+
+and
+
+```
+uv run pre-commit run --all-files --show-diff-on-failure
+```
+
+instead.
+
+
+If there is an error with formatting for ruff and it can be autofixed you can normally run
+
+```
+uv run ruff check --fix
+```
+
+to fix it.
+
+All checks must pass before the Agent generated code can be merged. AGENTS.md helps ensure the Agent follows these requirements.

py_hamt-3.0.0/Architecture.md

@@ -0,0 +1,213 @@
+# py-hamt Library Architecture Analysis
+
+## Overview
+
+The py-hamt library is a Python implementation of a Hash Array Mapped Trie (HAMT) designed for content-addressed storage systems like IPFS. It provides efficient storage and retrieval of large key-value mappings using the IPLD data model.
+
+## Core Architecture
+
+### 1. HAMT Structure (`hamt.py`)
+
+The core HAMT implementation follows a hierarchical trie structure where:
+- **Nodes** contain 256 buckets (indexed 0-255)
+- Each bucket can either be:
+  - A dictionary containing key-value mappings (when bucket size ≤ max_bucket_size)
+  - A link to a child Node (when bucket overflows)
+- Hash values are consumed 8 bits at a time to determine bucket indices
+
+#### Key Components:
+
+**Node Class** (`hamt.py:62`)
+- Fixed array of 256 elements representing buckets
+- Uses `list[IPLDKind]` where empty dicts `{}` represent empty buckets
+- Links are stored as single-element lists `[link_id]`
+- Serializes to DAG-CBOR format for content addressing
+
+**HAMT Class** (`hamt.py:287`)
+- Main interface for trie operations
+- Supports both read-only and read-write modes
+- Uses asyncio locks for thread safety in write mode
+- Implements two node storage strategies via NodeStore abstraction
+
+### 2. Storage Abstraction (`store.py`)
+
+**ContentAddressedStore** (`store.py:11`)
+- Abstract base class for content-addressed storage backends
+- Returns immutable IDs (IPLDKind) for stored content
+- Two codec types: "raw" for data, "dag-cbor" for structured content
+
+**KuboCAS** (`store.py:74`)
+- IPFS implementation using Kubo daemon
+- Uses RPC API for writes (`/api/v0/add`)
+- Uses HTTP Gateway for reads (`/ipfs/{cid}`)
+- Supports authentication and custom headers
+- Implements connection pooling and concurrency limiting
+
+**InMemoryCAS** (`store.py:37`)
+- Testing implementation using in-memory dictionary
+- Content-addressed via Blake3 hashing
+
+### 3. Node Storage Strategies
+
+**ReadCacheStore** (`hamt.py:150`)
+- Used in read-only mode
+- Implements LRU-style caching of loaded nodes
+- Cannot perform writes (throws exception)
+- Optimized for concurrent read operations
+
+**InMemoryTreeStore** (`hamt.py:180`)
+- Used in read-write mode
+- Maintains modified nodes in memory buffer
+- Uses UUID4 integers as temporary node IDs
+- Implements sophisticated flush algorithm during `vacate()`
+
+### 4. Zarr Integration (`zarr_hamt_store.py`)
+
+**ZarrHAMTStore** (`zarr_hamt_store.py:11`)
+- Implements Zarr v3 Store interface
+- Provides metadata caching for `zarr.json` files
+- Supports directory listing operations with efficient prefix matching
+- Key insight: Zarr keys map directly to HAMT keys without transformation
+
+**SimpleEncryptedZarrHAMTStore** (`encryption_hamt_store.py:12`)
+- Extends ZarrHAMTStore with ChaCha20-Poly1305 encryption
+- Encrypts all data including metadata
+- Uses 24-byte nonces and 16-byte authentication tags
+
+## Value Setting Mechanism & Data Flow
+
+### Setting Values (`_set_pointer` in `hamt.py:505`)
+
+1. **Hash and Navigate**: Key is hashed, bits extracted to determine path
+2. **Queue-based Insertion**: Uses FIFO queue to handle bucket overflows
+3. **Bucket Overflow Handling**: When bucket exceeds max_bucket_size:
+   - All existing items moved to queue for reinsertion
+   - New child node created and linked
+   - Continues insertion process at deeper level
+4. **Tree Rebalancing**: After insertion, `_reserialize_and_link` updates all affected nodes
+
+### Getting Values (`_get_pointer` in `hamt.py:618`)
+
+1. **Hash Traversal**: Follow hash-determined path through trie
+2. **Bucket Search**: Check final bucket for key
+3. **Pointer Resolution**: Return content-addressed pointer
+4. **Value Retrieval**: Load actual value from CAS using pointer
+
+### Memory Management
+
+**Read-Write Mode**:
+- Uses `InMemoryTreeStore` with UUID-based temporary IDs
+- Modified nodes stay in memory until `make_read_only()` or `cache_vacate()`
+- Flush algorithm uses DFS to preserve parent-child relationships
+
+**Read-Only Mode**:
+- Switches to `ReadCacheStore` for better read performance
+- Allows concurrent operations without locks
+- Cache size can be monitored and manually vacated
+
+## Architecture Gotchas & Edge Cases
+
+### 1. Mode Switching Complexity
+- **Problem**: HAMT can switch between read-only and read-write modes
+- **Gotcha**: In read-write mode, `root_node_id` is invalid until `make_read_only()`
+- **Why**: InMemoryTreeStore uses temporary UUIDs that aren't content-addressed
+- **Solution**: Always call `make_read_only()` before reading `root_node_id`
+
+### 2. Thread Safety Limitations
+- **Problem**: Only async-safe, not thread-safe in write mode
+- **Gotcha**: Multiple threads writing simultaneously can corrupt state
+- **Why**: Uses asyncio.Lock, not threading.Lock
+- **Solution**: Use single event loop for all write operations
+
+### 3. Hash Function Constraints
+- **Problem**: Hash must be multiple of 8 bits (bytes)
+- **Gotcha**: Custom hash functions with odd bit lengths will fail
+- **Why**: `extract_bits` assumes byte-aligned hash values
+- **Solution**: Ensure hash functions return byte-aligned results
+
+### 4. Bucket Size Tuning
+- **Problem**: max_bucket_size affects performance vs memory trade-offs
+- **Gotcha**: Very small bucket sizes (1) force deep trees, large sizes create big nodes
+- **Why**: Small buckets = more CAS operations, large buckets = bigger serialized nodes
+- **Solution**: Test with your specific workload (default 4 is reasonable)
+
+### 5. Empty Node Pruning
+- **Problem**: Deletions can leave empty nodes in tree
+- **Gotcha**: Empty nodes are automatically pruned except root
+- **Why**: Content addressing means identical empty nodes have same hash
+- **Solution**: Pruning is automatic, but be aware root can become empty
+
+### 6. Serialization Edge Cases
+- **Problem**: InMemoryTreeStore nodes contain non-serializable UUID links
+- **Gotcha**: Attempting to serialize buffer nodes directly will fail
+- **Why**: UUID4.int values exceed DAG-CBOR integer limits
+- **Solution**: Always use `vacate()` or `make_read_only()` before accessing serialized form
+
+### 7. CAS ID Immutability Requirement
+- **Problem**: CAS implementations must return immutable IDs
+- **Gotcha**: Using mutable objects (lists, dicts) as IDs breaks assumptions
+- **Why**: HAMT uses ID equality checks and as dictionary keys
+- **Solution**: Ensure CAS returns immutable types (bytes, CID, int, str, etc.)
+
+### 8. Zarr Metadata Caching
+- **Problem**: ZarrHAMTStore caches `zarr.json` files
+- **Gotcha**: Cache isn't invalidated on writes, can become stale
+- **Why**: Zarr frequently re-reads metadata, caching improves performance
+- **Solution**: Cache is updated on writes, but be aware of this optimization
+
+### 9. Concurrent Operations in Read Mode
+- **Problem**: Read-only mode allows concurrent access
+- **Gotcha**: Switching to write mode while reads are happening is unsafe
+- **Why**: Mode switch changes internal data structures
+- **Solution**: Ensure all operations complete before mode switches
+
+### 10. Key Encoding Assumptions
+- **Problem**: Keys are encoded as UTF-8 bytes for hashing
+- **Gotcha**: Non-UTF-8 string keys or binary keys need special handling
+- **Why**: `key.encode()` assumes UTF-8 encoding
+- **Solution**: Ensure keys are valid UTF-8 strings or modify key handling
+
+## Performance Characteristics
+
+### Time Complexity
+- **Get/Set/Delete**: O(log₂₅₆ n) average case, O(depth) worst case
+- **Iteration**: O(n) for all keys/values
+- **Tree depth**: Typically 1-4 levels for most datasets
+
+### Space Complexity
+- **Node overhead**: ~2KB per node (256 × 8-byte pointers)
+- **Memory efficiency**: Improves with higher bucket sizes
+- **CAS efficiency**: Content addressing deduplicates identical subtrees
+
+### Concurrency
+- **Read-only mode**: Full concurrency support
+- **Write mode**: Single-writer, async-safe
+- **Mode switching**: Blocking operation requiring exclusive access
+
+## Integration Patterns
+
+### 1. IPFS Integration
+```python
+kubo_cas = KuboCAS()
+hamt = await HAMT.build(cas=kubo_cas)
+await hamt.set("key", "value")
+await hamt.make_read_only()
+cid = hamt.root_node_id  # IPFS CID
+```
+
+### 2. Zarr Storage
+```python
+hamt = await HAMT.build(cas=kubo_cas, values_are_bytes=True)
+zarr_store = ZarrHAMTStore(hamt, read_only=False)
+dataset.to_zarr(store=zarr_store, zarr_format=3)
+```
+
+### 3. Encrypted Storage
+```python
+encryption_key = get_random_bytes(32)
+encrypted_store = SimpleEncryptedZarrHAMTStore(
+    hamt, read_only=False, encryption_key=encryption_key, header=b"app-name"
+)
+```
+
+The py-hamt library provides a robust, efficient implementation of HAMTs for content-addressed storage with careful attention to memory management, concurrency, and integration patterns. Understanding these architectural details and gotchas is crucial for successful implementation in production systems.