tokenmizer 0.2.4__tar.gz

tokenmizer-0.2.4/.claude-plugin/marketplace.json

@@ -0,0 +1,31 @@
+{
+  "name": "tokenmizer",
+  "version": "0.2.3",
+  "description": "Never lose your AI context again. Graph-backed memory, session checkpointing, and file intelligence for any LLM.",
+  "homepage": "https://github.com/Shweta-Mishra-ai/tokenmizer",
+  "repository": "https://github.com/Shweta-Mishra-ai/tokenmizer",
+  "author": {
+    "name": "Shweta Mishra",
+    "url": "https://github.com/Shweta-Mishra-ai"
+  },
+  "license": "MIT",
+  "keywords": [
+    "memory",
+    "checkpoint",
+    "resume",
+    "context",
+    "graph",
+    "token-optimization",
+    "mcp",
+    "anthropic",
+    "openai",
+    "llm"
+  ],
+  "plugins": [
+    {
+      "name": "tokenmizer",
+      "path": ".",
+      "description": "Graph memory + checkpoints + file intelligence for any LLM session"
+    }
+  ]
+}

tokenmizer-0.2.4/.claude-plugin/plugin.json

@@ -0,0 +1,39 @@
+{
+  "name": "tokenmizer",
+  "version": "0.2.3",
+  "description": "Never lose your AI context again. Graph-backed memory, session checkpointing, and file intelligence for Claude Code and any LLM.",
+  "author": {
+    "name": "Shweta Mishra",
+    "url": "https://github.com/Shweta-Mishra-ai"
+  },
+  "homepage": "https://github.com/Shweta-Mishra-ai/tokenmizer",
+  "repository": "https://github.com/Shweta-Mishra-ai/tokenmizer",
+  "license": "MIT",
+  "keywords": [
+    "memory",
+    "checkpoint",
+    "resume",
+    "context",
+    "token-optimization",
+    "graph",
+    "mcp"
+  ],
+  "skills": [
+    "./skills/checkpoint",
+    "./skills/resume",
+    "./skills/analyze",
+    "./skills/stats"
+  ],
+  "mcpServers": {
+    "tokenmizer": {
+      "command": "python3",
+      "args": [
+        "-m",
+        "tokenmizer.mcp.server"
+      ],
+      "env": {
+        "TOKENMIZER_URL": "http://localhost:8000"
+      }
+    }
+  }
+}

tokenmizer-0.2.4/.claude-plugin/skills/analyze/SKILL.md

@@ -0,0 +1,58 @@
+---
+name: analyze
+description: Analyze a large file (CSV, Excel, PDF, JSON, code) and return a token-efficient summary. Instead of reading thousands of rows or pages, get schema + statistics + sample in under 500 tokens. Use when user mentions a file path, asks to analyze data, pastes many rows, or references a CSV/Excel/PDF/JSON file.
+---
+
+Analyze a file using TokenMizer's file intelligence layer.
+
+## IMPORTANT rule
+
+**Never ask the user to paste the file content.** Always call TokenMizer to analyze it from the path. Pasting a 50,000-row CSV = 400,000 tokens. TokenMizer reduces it to ~450 tokens.
+
+## What to do
+
+Parse $ARGUMENTS:
+- First word = file path
+- Remaining words = query (what user wants to know)
+
+```bash
+FILE_PATH=$(echo "$ARGUMENTS" | awk '{print $1}')
+QUERY=$(echo "$ARGUMENTS" | cut -d' ' -f2-)
+
+python3 -c "
+from tokenmizer.filters.file_intelligence import FileIntelligence
+fi = FileIntelligence()
+result = fi.process(
+    open('${FILE_PATH}', 'rb').read(),
+    '${FILE_PATH}'.split('/')[-1],
+    token_budget=600,
+    query='${QUERY}'
+)
+print(f'File: {result.file_type} | {result.original_tokens:,} → {result.extracted_tokens} tokens ({result.savings_pct:.0f}% saved)')
+print()
+print(result.content)
+"
+```
+
+## Token savings by file type
+
+| Type | Typical savings |
+|---|---|
+| CSV (50k rows) | 99.9% |
+| PDF (200 pages) | 98.8% |
+| Excel (10 sheets) | 99.7% |
+| JSON (1k items) | 95% |
+| Code (large file) | 60-80% |
+
+## If TokenMizer not installed
+
+```bash
+pip install "tokenmizer[anthropic]"
+```
+
+## Examples of $ARGUMENTS
+
+- `/data/sales.csv` → analyze with no specific query
+- `/data/sales.csv which regions are underperforming` → targeted analysis
+- `/reports/Q1.pdf key findings and risks` → relevant page extraction
+- `/data/users.xlsx find inactive accounts` → per-sheet analysis

tokenmizer-0.2.4/.claude-plugin/skills/checkpoint/SKILL.md

@@ -0,0 +1,56 @@
+---
+name: checkpoint
+description: Save the current session to TokenMizer graph memory. Creates a persistent checkpoint with all tasks, decisions, files, and errors — resumable in any future session. Use when user says "save", "checkpoint", "remember this", "I'm done for today", or session is getting long.
+---
+
+Save the current session to TokenMizer graph memory.
+
+## What to do
+
+1. Ask the user for a session ID if not provided (suggest a slug based on what you're working on, e.g. "auth-service", "data-pipeline", "my-project")
+2. Call the TokenMizer checkpoint API:
+
+```bash
+curl -s -X POST "http://localhost:8000/api/checkpoint?session_id=$ARGUMENTS" \
+  -H "Content-Type: application/json"
+```
+
+3. Show the user what was saved:
+   - Checkpoint ID
+   - Number of nodes in graph
+   - Resume token count
+   - The resume context block
+
+## If TokenMizer is not running
+
+Tell the user to start it first:
+```bash
+tokenmizer serve
+# or
+python3 -m tokenmizer.api.app
+```
+
+## Session ID rules
+- Use lowercase slugs: `auth-service` not `Auth Service`
+- Keep it short and meaningful
+- Same ID across sessions for the same project
+
+## Example output to show user
+
+```
+✅ Session 'auth-service' saved
+
+Checkpoint: ckpt_a3f9b2
+Nodes: 14 (6 tasks, 4 decisions, 3 files, 1 error)
+Resume size: 247 tokens
+
+Resume context:
+Goal: Build FastAPI auth service with JWT
+Done: Project setup | User model | Login endpoint | Fix 422
+In progress: Refresh token rotation
+Decided: PostgreSQL | bcrypt | Redis for tokens
+Files: api/auth.py, api/models.py, config.py
+Continue: Implement token refresh endpoint
+```
+
+If $ARGUMENTS is empty, ask: "What should I call this session? (e.g. my-project)"

tokenmizer-0.2.4/.claude-plugin/skills/resume/SKILL.md

@@ -0,0 +1,49 @@
+---
+name: resume
+description: Load a previous session from TokenMizer graph memory. Returns a compact context block (100-600 tokens) covering goals, completed work, decisions, open tasks, and files. Inject this as system context to continue exactly where you left off. Use when user says "resume", "continue from last time", "load my project", "what did we do on X", or starts a session on a known project.
+---
+
+Load a previous session from TokenMizer graph memory.
+
+## What to do
+
+1. Get the session ID from $ARGUMENTS (or ask the user if not provided)
+2. Determine the level from $ARGUMENTS:
+   - `critical` = ~100 tokens, only open blockers + key decisions
+   - `standard` = ~300 tokens, normal resume (default)
+   - `full` = ~600 tokens, everything including env, schemas, endpoints
+
+3. Call the TokenMizer resume API:
+
+```bash
+SESSION_ID=$(echo "$ARGUMENTS" | awk '{print $1}')
+LEVEL=$(echo "$ARGUMENTS" | awk '{print $2}')
+LEVEL=${LEVEL:-standard}
+
+curl -s "http://localhost:8000/api/resume/${SESSION_ID}?level=${LEVEL}"
+```
+
+4. Inject the returned `resume_context` as a system message at the top of the conversation — NOT as a user message.
+
+5. Tell the user: "Loaded [X] tokens of context for '[session-id]'. Continuing from: [next_action]"
+
+## Format for system injection
+
+```
+[TokenMizer — session: {session_id}]
+{resume_context}
+[End of session context — continue from here]
+```
+
+## If no checkpoint found
+
+Tell the user: "No checkpoint found for '[session-id]'. Either the session hasn't been checkpointed yet, or TokenMizer isn't running."
+
+Suggest: `tokenmizer checkpoint {session-id}` to save the current session first.
+
+## Examples of $ARGUMENTS
+
+- `auth-service` → load standard resume for auth-service
+- `auth-service full` → load full 600-token resume
+- `auth-service critical` → load critical 100-token resume
+- (empty) → ask user which project to resume

tokenmizer-0.2.4/.claude-plugin/skills/stats/SKILL.md

@@ -0,0 +1,57 @@
+---
+name: stats
+description: Show TokenMizer token savings stats — how many tokens saved today, this week, cache hit rate, and which layers are saving the most. Use when user asks about token usage, costs, savings, or "how much have we saved".
+---
+
+Show TokenMizer token savings analytics.
+
+## What to do
+
+```bash
+curl -s http://localhost:8000/api/stats
+```
+
+Then also fetch cache stats:
+```bash
+curl -s http://localhost:8000/api/cache/stats
+```
+
+## Format the output clearly
+
+Show:
+- Tokens saved today and this week
+- Cost saved in USD
+- Cache hit rate
+- Which layer saved the most (compression / cache / windowing / file extraction)
+- Suggestions if savings are low
+
+## Example output
+
+```
+TokenMizer Stats
+─────────────────────────
+Today:    12,450 tokens saved (34%) — $0.0373
+Week:     87,200 tokens saved (31%) — $0.2616
+
+By layer:
+  File extraction:  45,000 tokens
+  Semantic cache:   28,000 tokens  
+  Smart window:     9,200 tokens
+  Compression:      3,800 tokens
+  Output trim:      1,200 tokens
+
+Cache: 847 entries | 68% hit rate | 8% full
+─────────────────────────
+Dashboard: http://localhost:8000
+```
+
+## If nothing saved yet
+
+Tell user to start using TokenMizer as their proxy:
+```python
+from openai import OpenAI
+client = OpenAI(
+    api_key="your-key",
+    base_url="http://localhost:8000/v1"
+)
+```

tokenmizer-0.2.4/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug Report
+about: Something is broken
+title: '[BUG] '
+labels: bug
+---
+
+**Python version:** 
+**Provider:**
+**OS:**
+
+**What happened:**
+
+**Expected:**
+
+**Reproduction:**
+```python
+# minimal code
+```
+
+**Logs:**
+```
+paste relevant logs
+```

tokenmizer-0.2.4/.github/ISSUE_TEMPLATE/extraction_miss.md

@@ -0,0 +1,19 @@
+---
+name: Extraction Miss
+about: Graph memory missed a task, decision, or file from a real session
+title: '[EXTRACTION] '
+labels: extraction-quality
+---
+
+**Message where extraction should have happened:**
+```
+[assistant]: The login endpoint is now working...
+```
+
+**What should have been extracted:**
+- Node type: task / decision / file
+- Expected label: "..."
+
+**What was actually extracted (if anything):**
+
+**Session type:** coding / research / writing / other

tokenmizer-0.2.4/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## What this PR does
+
+<!-- Brief description -->
+
+## Type
+- [ ] Bug fix
+- [ ] Extraction improvement (graph_memory/)
+- [ ] New provider
+- [ ] Performance
+- [ ] Documentation
+
+## Tests
+- [ ] `pytest tests/ -v` passes
+- [ ] `ruff check tokenmizer/` clean
+- [ ] Memory accuracy test added/updated (if extraction change)
+
+## Checklist
+- [ ] No raw dicts crossing layer boundaries (use DTOs)
+- [ ] No `os.getenv()` outside `config/settings.py`
+- [ ] External imports are lazy (inside functions, with try/except ImportError)

tokenmizer-0.2.4/.github/workflows/ci.yml

@@ -0,0 +1,101 @@
+name: CI
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint (blocking)
+        run: ruff check tokenmizer/ tests/
+
+      # Coverage threshold lives in pyproject.toml [tool.coverage.report]
+      # fail_under — single source of truth, do NOT override it here.
+      - name: Run tests with coverage
+        run: |
+          pytest tests/ \
+            --cov=tokenmizer \
+            --cov-report=term-missing \
+            --cov-report=xml \
+            -v
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v4
+        if: matrix.python-version == '3.12'
+        with:
+          file: ./coverage.xml
+        continue-on-error: true
+
+  benchmark:
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Run checkpoint accuracy benchmark
+        run: python -m benchmarks.checkpoint_accuracy.runner
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: benchmark_results.json
+
+  docker:
+    runs-on: ubuntu-latest
+    needs: test
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build Docker image
+        run: docker build -t tokenmizer:latest .
+
+      - name: Test Docker health
+        run: |
+          docker run -d --name tm-test \
+            -e TOKENMIZER_ANTHROPIC_API_KEY=test \
+            -p 8001:8000 \
+            tokenmizer:latest
+          sleep 8
+          curl -f http://localhost:8001/health
+          docker stop tm-test

tokenmizer-0.2.4/.github/workflows/release.yml

@@ -0,0 +1,57 @@
+name: Release to PyPI
+
+# Publishes via PyPI Trusted Publishing (OIDC) — NO API token stored anywhere.
+# One-time setup on pypi.org (must be done by the project owner):
+#   pypi.org → Your account → Publishing → Add a new pending publisher:
+#     PyPI project name: tokenmizer
+#     Owner:             Shweta-Mishra-ai
+#     Repository:        tokenmizer
+#     Workflow name:     release.yml
+#     Environment:       pypi
+# Then create a GitHub Release (tag v0.2.4 etc.) and this workflow publishes.
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Run tests first — never publish a broken build
+        run: |
+          pip install -e ".[dev]"
+          pytest tests/ -q
+          ruff check tokenmizer/ tests/
+
+      - name: Build sdist + wheel
+        run: |
+          pip install build twine
+          python -m build
+          twine check dist/*
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write   # required for Trusted Publishing — no secrets used
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

tokenmizer-0.2.4/.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+venv/
+.pytest_cache/
+.coverage
+benchmark_v*_results.json
+# checkpoints/  # removed: conflicts with tokenmizer/checkpoints/ module
+*.db
+*.db-wal
+*.db-shm
+.DS_Store

tokenmizer-0.2.4/.mcp.json

@@ -0,0 +1,12 @@
+{
+  "mcpServers": {
+    "tokenmizer": {
+      "command": "python3",
+      "args": ["-m", "tokenmizer.mcp.server"],
+      "env": {
+        "TOKENMIZER_URL": "http://localhost:8000",
+        "TOKENMIZER_API_KEY": ""
+      }
+    }
+  }
+}