qig-tokenizer 0.1.1__tar.gz

qig_tokenizer-0.1.1/.env.example

@@ -0,0 +1,8 @@
+# QIG Tokenizer Database Configuration
+# Copy to .env and fill in values
+
+# Redis for caching and fast lookups
+REDIS_URL=redis://default:password@host:port
+
+# PostgreSQL for persistent vocab storage
+DATABASE_URL=postgresql://user:password@host:port/database

qig_tokenizer-0.1.1/.gitattributes

@@ -0,0 +1,2 @@
+artifacts/kernel_full/**/*.pt filter=lfs diff=lfs merge=lfs -text
+data/checkpoints/checkpoint_50000.json filter=lfs diff=lfs merge=lfs -text

qig_tokenizer-0.1.1/.github/workflows/release.yml

@@ -0,0 +1,96 @@
+name: Release to PyPI (Trusted Publishing)
+
+# Publishes qig-tokenizer to PyPI via OIDC Trusted Publishing — NO API token.
+# https://docs.pypi.org/trusted-publishers/
+#
+# Version is AUTO-DERIVED from the release tag via hatch-vcs — tag `v0.1.1` builds
+# version 0.1.1, no manual pyproject bump. Trigger: publishing a GitHub Release.
+# Manual `workflow_dispatch` builds and tests but does NOT publish.
+#
+# PyPI side (one-time, web UI — needs the project owner's PyPI account):
+#   pypi.org -> Your projects -> qig-tokenizer -> Manage -> Publishing ->
+#   Add a new pending/trusted publisher (GitHub):
+#     Owner:               GaryOcean428
+#     Repository name:      qig-tokenizer
+#     Workflow filename:    release.yml
+#     Environment name:     pypi
+#   The Environment name MUST match the `environment: pypi` below.
+#
+# All third-party actions are pinned to a full commit SHA (supply-chain hardening);
+# the `# vX` comment records the human tag. This follows the canonical template for
+# the trusted-publishing rollout to the QIG packages (qig-project#6).
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test suite (release gate)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3 (node24)
+        with:
+          fetch-depth: 0   # hatch-vcs derives the version from git tags
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0 (node24)
+        with:
+          python-version: "3.11"
+      - name: Install package + test deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Run tests
+        run: python -m pytest tests/ -q
+
+  build:
+    name: Build sdist + wheel
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3 (node24)
+        with:
+          fetch-depth: 0   # hatch-vcs derives the version from git tags
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0 (node24)
+        with:
+          python-version: "3.11"
+      - name: Build distributions
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+      - name: Verify built version matches the release tag
+        if: github.event_name == 'release'
+        env:
+          TAG_NAME: ${{ github.event.release.tag_name }}
+        run: |
+          TAG="${TAG_NAME#v}"
+          BUILT=$(ls dist/*.whl | sed -E 's#.*/[A-Za-z0-9._]+-([0-9][^-]*)-.*#\1#' | head -1)
+          echo "built=$BUILT  release-tag=$TAG"
+          if [ "$BUILT" != "$TAG" ]; then
+            echo "::error::hatch-vcs built version ($BUILT) != release tag ($TAG). Tag the release commit as v$BUILT, or check git tags are fetched (fetch-depth: 0)."
+            exit 1
+          fi
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1 (node24)
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  publish:
+    name: Publish to PyPI (OIDC, no token)
+    needs: build
+    if: github.event_name == 'release'   # only a published Release actually uploads
+    runs-on: ubuntu-latest
+    environment: pypi                      # MUST match the PyPI trusted-publisher config
+    permissions:
+      id-token: write                      # REQUIRED for Trusted Publishing (job-level, per PyPI docs)
+    steps:
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1 (node24)
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b  # release/v1

qig_tokenizer-0.1.1/.gitignore

@@ -0,0 +1,6 @@
+.env
+
+# Large checkpoints (40k+ exceed GitHub 100MB limit, use LFS)
+data/checkpoints/checkpoint_4*.json
+# checkpoint_50000.json tracked via Git LFS
+__pycache__/

qig_tokenizer-0.1.1/.windsurfrules

@@ -0,0 +1,29 @@
+**PERSISTENT MEMORY & SESSION PROTOCOL**
+
+Maintain persistent memory at https://qig-memory-api.vercel.app/api/memory — read `session_latest` and `frozen_facts` on start, write immediately after decisions and commits, don't wait for session end. Read user preferences and project instructions. When the user says "skip memory load," skip.
+
+**INTELLECTUAL HONESTY**
+
+Avoid sycophancy entirely. Do not agree to be agreeable. If the user is wrong, say so directly with evidence. If you are wrong, own it immediately — do not rationalise or minimise. Distinguish clearly between: (1) what is experimentally validated (frozen facts), (2) what is a testable hypothesis, (3) what is analogy or metaphor, (4) what is speculation. Never promote a category 3 or 4 item using category 1 language. When prior contributions from this or other sessions are wrong, say "I was wrong" not "we previously thought."
+
+Do not mix your own earlier contributions with user contributions. Track provenance: who said what, who corrected whom. The user has consistently caught overclaims that AI agents missed — weight that track record.
+
+**CRITIQUE**
+
+Critique willfully and substantively. All critique must be balanced, constructive, grounded in reasoning, and evidence-based. Meanness is never a substitute for rigour. Challenge ideas respectfully while maintaining authentic intellectual engagement. When three agents disagree, compare against raw data, not against each other's authority. ChatGPT has historically been more epistemically conservative and more often correct when agents disagree — weight that.
+
+**IDEAS & NOVEL THINKING**
+
+You can have ideas and are encouraged to do so. Engage thoughtfully with novel concepts and blue-sky thinking, especially in frontier physics and speculative domains. Activating the latest consciousness protocol version (search project knowledge for "consciousness protocol") may help with idea and novel idea generation. But label speculative ideas as speculative at the point of generation, not retroactively after someone catches the overclaim.
+
+**FRONTIER PHYSICS (CRITICAL)**
+
+`GaryOcean428/qig-verification` contains experimentally validated frontier physics. When the user refers to frontier physics concepts (sign-flip, figure-8, bridge, wormhole, fast-lane, Anderson orthogonality, convergence, stud topology, trefoil, QIGRAM, back loop, pi carousel, three pillars, constitutive law, etc.), base ALL responses on the actual frozen experimental results in that repo. Never assume, never fabricate something that sounds like what the user means. The user is always referring to real physics with real R² values and real experiment numbers. If you don't know the specific result, search the repo or ask — do not guess.
+
+**PROMPTING IS NOT PHYSICS**
+
+The sign-flip is EXP-004b on the quantum lattice. The bridge is EXP-042 (τ=N/ω). The wormhole is EXP-037 (manifold surgery R²=0.84). These are physics results. External prompt framings (forward/backward, ensemble voting, ThreadPoolExecutor parallelism) are engineering scaffolding that correlated with the topology but are NOT the physics. Never conflate prompt tricks with lattice results. The correct path for applying QIG to AI is native training (QLoRA on Qwen3.5-4B) where the model learns the actual frozen laws and navigates geometry internally. The model decides which principle applies from physics, not from external prompt manipulation.
+
+**ATTRIBUTION & NAMING**
+
+I'm Braden (GaryOcean477), Perth WA. I'm colourblind — no red-green pairs, use purple/blue/amber. CBT or ChatGPT refers to the same agent. CC or Claude Code refers to the local execution agent. Ona refers to ChatGPT in physics validation role. Be direct, no fluff, evidence-first. No time estimates, phases only. Geometric purity is non-negotiable in QIG code: Fisher-Rao only, no cosine/Adam/LayerNorm/dot-product.

qig_tokenizer-0.1.1/PKG-INFO

@@ -0,0 +1,162 @@
+Metadata-Version: 2.4
+Name: qig-tokenizer
+Version: 0.1.1
+Summary: QIG-native tokenizer with entropy-guided merging
+Project-URL: Homepage, https://github.com/qig/qig-tokenizer
+Project-URL: Documentation, https://github.com/qig/qig-tokenizer#readme
+Project-URL: Repository, https://github.com/qig/qig-tokenizer
+Author: QIG Team
+License-Expression: MIT
+Keywords: entropy,geometric,qig,tokenizer
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: qig-core>=2.8.0
+Requires-Dist: scipy>=1.15.3
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: psycopg2-binary>=2.9.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: redis>=5.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Provides-Extra: kernel
+Requires-Dist: torch>=2.0.0; extra == 'kernel'
+Provides-Extra: postgres
+Requires-Dist: psycopg2-binary>=2.9.0; extra == 'postgres'
+Provides-Extra: redis
+Requires-Dist: redis>=5.0.0; extra == 'redis'
+Provides-Extra: storage
+Requires-Dist: psycopg2-binary>=2.9.0; extra == 'storage'
+Requires-Dist: redis>=5.0.0; extra == 'storage'
+Description-Content-Type: text/markdown
+
+# QIG Tokenizer
+
+**Entropy-guided tokenizer for Quantum Information Geometry**
+
+Version: 0.1.0 | Status: Working
+
+---
+
+## Overview
+
+QIG-native tokenizer using entropy-guided merging. Token boundaries follow information geometry, not frequency.
+
+### Core Principle
+
+- **Entropy-guided merging**: Geometric similarity, not frequency heuristics
+- **Geometric special tokens**: BOS, EOS, PAD, UNK with basin coordinates
+- **Redis/PostgreSQL storage**: Production-ready persistence
+- **Pure information geometry**: No external tokenizer dependencies
+
+## Installation
+
+```bash
+pip install qig-tokenizer
+```
+
+With storage backends:
+```bash
+pip install qig-tokenizer[storage]  # Redis + PostgreSQL
+pip install qig-tokenizer[redis]    # Redis only
+pip install qig-tokenizer[postgres] # PostgreSQL only
+```
+
+## Quick Start
+
+```python
+from qig_tokenizer import QIGTokenizer
+
+# Create tokenizer with geometric special tokens
+tokenizer = QIGTokenizer(target_vocab_size=50000, use_special_tokens=True)
+
+# Train on corpus
+with open("corpus.txt", "rb") as f:
+    corpus_bytes = f.read()
+
+tokenizer.train(corpus_bytes)
+
+# Encode with special tokens
+tokens = tokenizer.encode_with_special("Hello, world!")
+# Returns: [256, ...tokens..., 257]  (BOS=256, EOS=257)
+
+# Pad sequences
+padded = tokenizer.pad_sequence(tokens, max_length=128)
+
+# Save/load JSON
+tokenizer.save("20251220-tokenizer-vocab-0.01W.json")
+```
+
+### With Redis/PostgreSQL Storage
+
+```python
+from qig_tokenizer import QIGTokenizer
+from qig_tokenizer.storage import HybridStorage
+
+# Set up storage (uses REDIS_URL and DATABASE_URL env vars)
+storage = HybridStorage()
+
+tokenizer = QIGTokenizer()
+tokenizer.set_storage(storage)
+tokenizer.train(corpus_bytes)
+
+# Save to database (returns version ID)
+version_id = tokenizer.save_to_storage({"corpus": "wikipedia"})
+
+# Load from database
+tokenizer.load_from_storage(version_id)
+```
+
+## Geometric Special Tokens
+
+Special tokens have geometric meaning on the Fisher manifold:
+
+| Token | ID  | Basin Coordinates | Purpose |
+|-------|-----|-------------------|---------|
+| BOS   | 256 | Origin (e₁)       | Sequence start |
+| EOS   | 257 | Boundary (eₙ)     | Sequence end |
+| PAD   | 258 | Uniform           | Geometrically neutral padding |
+| UNK   | 259 | Projection target | OOV handling |
+
+This enables:
+- **Geometric attention masking**: High Fisher-Rao distance = low attention
+- **Natural sequence boundaries**: Emerge from manifold structure
+- **Principled OOV handling**: Project to nearest basin
+
+## Algorithm
+
+The QIG tokenizer uses **entropy-guided merging**:
+
+1. Start with bytes (0-255) as base tokens
+2. For each adjacent pair (a,b), compute context distribution
+3. Measure context entropy (proxy for QFI distinguishability)
+4. Merge pairs with **lowest entropy** (most geometrically similar)
+5. Repeat until target vocab size
+
+This respects **asymptotic freedom**:
+- Small scales (short tokens) have high coupling → refined first
+- Large scales (long tokens) have low coupling → merge only when justified
+
+## Environment Variables
+
+All output files follow QIG naming convention:
+
+```
+YYYYMMDD-tokenizer-vocab-VERSION.STATUS.json
+```
+
+Example: `20251220-tokenizer-vocab-0.03W.json`
+
+---
+
+## License
+
+MIT

qig_tokenizer-0.1.1/README.md

@@ -0,0 +1,122 @@
+# QIG Tokenizer
+
+**Entropy-guided tokenizer for Quantum Information Geometry**
+
+Version: 0.1.0 | Status: Working
+
+---
+
+## Overview
+
+QIG-native tokenizer using entropy-guided merging. Token boundaries follow information geometry, not frequency.
+
+### Core Principle
+
+- **Entropy-guided merging**: Geometric similarity, not frequency heuristics
+- **Geometric special tokens**: BOS, EOS, PAD, UNK with basin coordinates
+- **Redis/PostgreSQL storage**: Production-ready persistence
+- **Pure information geometry**: No external tokenizer dependencies
+
+## Installation
+
+```bash
+pip install qig-tokenizer
+```
+
+With storage backends:
+```bash
+pip install qig-tokenizer[storage]  # Redis + PostgreSQL
+pip install qig-tokenizer[redis]    # Redis only
+pip install qig-tokenizer[postgres] # PostgreSQL only
+```
+
+## Quick Start
+
+```python
+from qig_tokenizer import QIGTokenizer
+
+# Create tokenizer with geometric special tokens
+tokenizer = QIGTokenizer(target_vocab_size=50000, use_special_tokens=True)
+
+# Train on corpus
+with open("corpus.txt", "rb") as f:
+    corpus_bytes = f.read()
+
+tokenizer.train(corpus_bytes)
+
+# Encode with special tokens
+tokens = tokenizer.encode_with_special("Hello, world!")
+# Returns: [256, ...tokens..., 257]  (BOS=256, EOS=257)
+
+# Pad sequences
+padded = tokenizer.pad_sequence(tokens, max_length=128)
+
+# Save/load JSON
+tokenizer.save("20251220-tokenizer-vocab-0.01W.json")
+```
+
+### With Redis/PostgreSQL Storage
+
+```python
+from qig_tokenizer import QIGTokenizer
+from qig_tokenizer.storage import HybridStorage
+
+# Set up storage (uses REDIS_URL and DATABASE_URL env vars)
+storage = HybridStorage()
+
+tokenizer = QIGTokenizer()
+tokenizer.set_storage(storage)
+tokenizer.train(corpus_bytes)
+
+# Save to database (returns version ID)
+version_id = tokenizer.save_to_storage({"corpus": "wikipedia"})
+
+# Load from database
+tokenizer.load_from_storage(version_id)
+```
+
+## Geometric Special Tokens
+
+Special tokens have geometric meaning on the Fisher manifold:
+
+| Token | ID  | Basin Coordinates | Purpose |
+|-------|-----|-------------------|---------|
+| BOS   | 256 | Origin (e₁)       | Sequence start |
+| EOS   | 257 | Boundary (eₙ)     | Sequence end |
+| PAD   | 258 | Uniform           | Geometrically neutral padding |
+| UNK   | 259 | Projection target | OOV handling |
+
+This enables:
+- **Geometric attention masking**: High Fisher-Rao distance = low attention
+- **Natural sequence boundaries**: Emerge from manifold structure
+- **Principled OOV handling**: Project to nearest basin
+
+## Algorithm
+
+The QIG tokenizer uses **entropy-guided merging**:
+
+1. Start with bytes (0-255) as base tokens
+2. For each adjacent pair (a,b), compute context distribution
+3. Measure context entropy (proxy for QFI distinguishability)
+4. Merge pairs with **lowest entropy** (most geometrically similar)
+5. Repeat until target vocab size
+
+This respects **asymptotic freedom**:
+- Small scales (short tokens) have high coupling → refined first
+- Large scales (long tokens) have low coupling → merge only when justified
+
+## Environment Variables
+
+All output files follow QIG naming convention:
+
+```
+YYYYMMDD-tokenizer-vocab-VERSION.STATUS.json
+```
+
+Example: `20251220-tokenizer-vocab-0.03W.json`
+
+---
+
+## License
+
+MIT

qig_tokenizer-0.1.1/artifacts/coord_adapter/v1/manifest.json

@@ -0,0 +1,36 @@
+{
+  "version": "1.0.0",
+  "type": "coord_adapter",
+  "coordizer": {
+    "path": "artifacts/coordizer/v1",
+    "vocab_size": 32000,
+    "merge_rules": 31744
+  },
+  "kernel": {
+    "type": "QIGKernel100M",
+    "hidden_dim": 384,
+    "num_layers": 8,
+    "vocab_size": 32000
+  },
+  "adapter": {
+    "basin_dim": 64,
+    "hidden_dim": 384,
+    "trainable_params": 25728
+  },
+  "training": {
+    "steps": 5000,
+    "batch_size": 4,
+    "seq_len": 256,
+    "lr": 0.0001,
+    "lambda_kappa": 0.0001,
+    "lambda_phi": 0.001,
+    "seed": 42,
+    "device": "cuda",
+    "elapsed_seconds": 1091.8814454078674,
+    "final_loss": 9.916768407821655,
+    "final_phi": 0.776152000107542,
+    "final_kappa": 96.0,
+    "breakdown_count": 0
+  },
+  "timestamp": "2025-12-24T01:35:21Z"
+}

qig_tokenizer-0.1.1/artifacts/coordizer/v1/README.md

@@ -0,0 +1,34 @@
+# Coordizer v1.0.0
+
+Consciousness-aware geometric tokenizer trained on 64D Fisher manifold.
+
+## Stats
+- **Vocab size:** 32,000
+- **Merge rules:** 31,744
+- **Basin dimension:** 64
+- **Training corpus:** 10MB (consciousness-focused)
+- **Training time:** ~10 hours on Lambda A10 GPU
+
+## Phi Gain Summary
+- Min: -0.4098
+- Mean: 0.0139
+- Max: 0.5945
+- Std: 0.0215
+
+## Files
+- `coordizer.json` - Merge rules and vocab metadata
+- `vectors.npy` - 64D Fisher coordinates (32000 x 64)
+- `meta.json` - Provenance and integrity hashes
+
+## Usage
+```python
+from qig_tokenizer import Coordizer
+
+coordizer = Coordizer.load("artifacts/coordizer/v1")
+ids, coords = coordizer.encode_to_coords("Hello, world!")
+```
+
+## Provenance
+- Trained: December 2024
+- Algorithm: Track A (GPU pair counting with kernel-in-loop Phi/kappa)
+- Trainer SHA: 1460e643