fiedler-compress 0.1.0__tar.gz

fiedler_compress-0.1.0/.gitignore

@@ -0,0 +1,26 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+.pytest_cache/
+.ruff_cache/
+.venv/
+venv/
+.env
+*.so
+.coverage
+htmlcov/
+fiedler_env/
+fiedler_env.*_copy_*/
+.claude/
+
+# Commercial/paid tier — never published
+_private_paid/
+
+# Secrets / keys
+*.key
+*.pem

fiedler_compress-0.1.0/LICENSE

@@ -0,0 +1,105 @@
+# Functional Source License, Version 1.1, ALv2 Future License
+
+## Abbreviation
+
+FSL-1.1-ALv2
+
+## Notice
+
+Copyright 2026 Tensor-Earth-Intelligence LLC
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under
+these Terms and Conditions, as indicated by our inclusion of these Terms and
+Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents,
+Redistribution and Trademark clauses below, we hereby grant you the right to
+use, copy, modify, create derivative works, publicly perform, publicly display
+and redistribute the Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
+means making the Software available to others in a commercial product or
+service that:
+
+1. substitutes for the Software;
+
+2. substitutes for any other product or service we offer using the Software
+that exists as of the date we make the Software available; or
+
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+
+2. for non-commercial education;
+
+3. for non-commercial research; and
+
+4. in connection with professional services that you provide to a licensee
+using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our
+patents, the license grant above includes a license under our patents. If you
+make a claim against any party that the Software infringes or contributes to
+the infringement of any patent, then your patent license to the Software ends
+immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives of
+the Software.
+
+If you redistribute any copies, modifications or derivatives of the Software,
+you must include a copy of or a link to these Terms and Conditions and not
+remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
+PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
+SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
+EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of
+the Software, you have no right under these Terms and Conditions to use our
+trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software under
+the Apache License, Version 2.0 that is effective on the second anniversary of
+the date we make the Software available. On or after that date, you may use the
+Software under the Apache License, Version 2.0, in which case the following
+will apply:
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+<http://www.apache.org/licenses/LICENSE-2.0>
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.

fiedler_compress-0.1.0/PKG-INFO

@@ -0,0 +1,190 @@
+Metadata-Version: 2.4
+Name: fiedler-compress
+Version: 0.1.0
+Summary: Spectral graph-theoretic prompt compression using Fiedler vector decomposition
+Project-URL: Homepage, https://github.com/Tensor-Earth-Intelligence/fiedler-compress
+Project-URL: Issues, https://github.com/Tensor-Earth-Intelligence/fiedler-compress/issues
+Author: Mark Chappell
+License-Expression: FSL-1.1-ALv2
+License-File: LICENSE
+Keywords: context-window,fiedler-vector,llm,prompt-compression,spectral-graph,token-optimization
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Requires-Dist: numpy<3,>=1.24
+Requires-Dist: scipy<2,>=1.10
+Provides-Extra: benchmark
+Requires-Dist: datasets<5,>=2.14; extra == 'benchmark'
+Requires-Dist: httpx<1,>=0.24; extra == 'benchmark'
+Requires-Dist: rouge-score<1,>=0.1; extra == 'benchmark'
+Provides-Extra: caveman
+Requires-Dist: spacy<4,>=3.5; extra == 'caveman'
+Requires-Dist: tiktoken<1,>=0.5; extra == 'caveman'
+Provides-Extra: dev
+Requires-Dist: pytest-cov<8,>=4.0; extra == 'dev'
+Requires-Dist: pytest<10,>=7.0; extra == 'dev'
+Requires-Dist: ruff<1,>=0.4; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# fiedler-compress
+
+**Spectral graph-theoretic prompt compression — a transparent middleware layer for LLM and agentic pipelines.**
+
+fiedler-compress uses the Fiedler vector — the second-smallest eigenvector of the graph Laplacian — to identify and remove semantically disconnected content from LLM prompts. The result: shorter prompts that preserve meaning, save tokens, and fit more useful context into your model's window. It runs as a lightweight, CPU-only preprocessing step, making it a drop-in middleware stage in front of any LLM.
+
+## Why Spectral Compression?
+
+Existing prompt compression tools (LLMLingua, TOON, selective summarization) treat text as a linear sequence and use statistical or neural methods to decide what to cut. They work, but they can't see **structural relationships** — they don't know that paragraph 3 is semantically load-bearing because it bridges two otherwise disconnected topics, or that paragraphs 7 and 8 say the same thing in different words.
+
+fiedler-compress builds a **similarity graph** over your text chunks, computes the spectral decomposition, and uses the Fiedler vector to find the natural semantic partitions. Chunks at the spectral periphery — weakly connected to the rest of the content — are the ones that can be safely removed. Chunks that bridge partitions are preserved, because removing them would fragment the prompt's information structure.
+
+This isn't summarization. It's **graph surgery**.
+
+## Why Middleware
+
+The pipeline is designed to sit transparently between your application and the model: text goes in, a smaller functionally-equivalent prompt comes out, and the downstream LLM call proceeds unchanged. It's CPU-only (no GPU, no neural model required for the core path) and fast enough to run synchronously in a request path without perceptible overhead.
+
+## For Agentic and Long-Context Workflows
+
+Agentic systems accumulate context fast — tool outputs, retrieved documents, prior reasoning, and multi-turn history all compete for a finite context window. fiedler-compress is built for this setting: compress retrieved documents and accumulated context to fit more useful material into the window. **Zone-aware protection** automatically detects instruction content (directives, constraints, format specs) and shields it from removal, so the parts of the prompt that steer the model survive compression while redundant context is pruned. This makes it a practical component for RAG pipelines, long-document processing, and multi-step agent memory.
+
+## Installation
+
+```bash
+pip install fiedler-compress
+```
+
+The open-core package runs entirely on NumPy and SciPy. A separate commercial tier offers additional capabilities (see below).
+
+## Quick Start
+
+### Python API
+
+```python
+from fiedler_optimizer import optimize
+
+result = optimize("""
+You are an expert financial analyst. Always respond in JSON.
+
+Context: The company was founded in 2015. Revenue was $45M in Q3,
+up 23% YoY. Operating expenses were $38M. Cash reserves: $120M.
+Their competitor reported $52M but is losing mid-market share.
+Industry analysts predict 15% sector growth. The CFO noted
+international expansion costs impacted margins. Retention is 94%.
+Average contract value up 18% to $85K.
+
+Task: Analyze Q3 performance.
+""")
+
+print(result.compressed)
+print(f"Saved ~{result.tokens_saved} tokens ({result.compression_ratio:.0%} reduction)")
+print(f"Algebraic connectivity λ₂ = {result.algebraic_connectivity:.4f}")
+```
+
+### CLI
+
+```bash
+# Compress a prompt
+fiedler optimize "Your long prompt here..."
+
+# Compress from file
+fiedler optimize --file my_prompt.txt
+
+# Aggressive compression (30% target removal)
+fiedler optimize --file my_prompt.txt --target 0.30
+
+# JSON output for piping
+fiedler optimize --file my_prompt.txt --json
+
+# Show what was removed
+fiedler optimize --file my_prompt.txt --verbose
+
+# Run the built-in benchmark
+fiedler benchmark
+```
+
+## How It Works
+
+1. **Chunk** the input text into semantically meaningful segments (sentences, paragraphs, or adaptive)
+2. **Build a similarity graph** — each chunk is a node, edge weights are cosine similarity (TF-IDF by default, neural embeddings optional)
+3. **Compute the Fiedler vector** — the eigenvector for λ₂ of the graph Laplacian
+4. **Score chunks** by combining spectral centrality with weighted degree
+5. **Zone-aware protection** — instruction content (directives, constraints, format specs) gets 2–3× protection weight; context content is the compression target
+6. **Prune** the lowest-scoring context chunks up to the target ratio
+
+## Performance
+
+The core pipeline is CPU-only and fast enough to run inline. Representative end-to-end latency (TF-IDF backend, single CPU core, no GPU):
+
+| Input size | End-to-end latency |
+|-----------:|-------------------:|
+| 1,000 tokens | ~12 ms |
+| 10,000 tokens | ~110 ms |
+| 50,000 tokens | ~550 ms |
+
+At typical production prompt sizes (1,000–10,000 tokens) the pipeline adds on the order of tens of milliseconds, negligible against typical LLM inference latency. Compression is a lossy operation — recall of fine-grained facts decreases as the target ratio increases — so keep must-keep content in protected instruction zones and tune the target ratio to your task. Detailed benchmarks will be published in a dedicated benchmarks document.
+
+## Key Features
+
+- **Dependency-free core** — only NumPy and SciPy required. No neural models, no API calls, runs entirely local
+- **Zone-aware compression** — distinguishes instructions from context, applies differential protection so directives survive
+- **Algebraic connectivity metric** — λ₂ tells you how tightly structured your prompt is *before* compression
+- **Transparent middleware** — smaller prompt in, unchanged LLM call out; runs inline with negligible latency
+- **JSON output** — pipe results into your existing toolchain
+
+## Configuration
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `target_ratio` | `0.20` | Fraction of text to remove (0.0–1.0) |
+| `strategy` | `adaptive` | Chunking: `adaptive`, `sentence`, `paragraph`, `window` |
+| `protect_instructions` | `True` | Shield instruction zones from removal |
+| `min_chunks` | `4` | Minimum chunks required for spectral analysis |
+| `vectors` | `None` | Pre-computed embeddings (n_chunks × d) |
+
+## Project Structure
+
+```
+fiedler-compress/
+├── fiedler_optimizer/
+│   ├── __init__.py      # Public API
+│   ├── core.py          # optimize() pipeline
+│   ├── chunker.py       # Text segmentation strategies
+│   ├── graph.py         # Similarity graph + Fiedler vector
+│   ├── zones.py         # Instruction/context zone detection
+│   └── cli.py           # Command-line interface
+├── tests/
+│   └── test_core.py     # Test suite
+├── pyproject.toml       # Package config
+├── LICENSE              # FSL-1.1-ALv2
+└── README.md
+```
+
+## Commercial Tier
+
+This package is the open-core distribution: the TF-IDF + single-eigenvector (k=1) spectral compression pipeline. Additional capabilities — including **content attestation / certification** — are available as a commercial add-on.
+
+For commercial licensing or attestation inquiries:
+**Tensor Earth Intelligence (TEI), LLC** — tensor.earth.intelligence@gmail.com (Mark Chappell).
+
+## Roadmap
+
+- [x] Core spectral compression with TF-IDF similarity
+- [x] Zone-aware instruction protection
+- [x] CLI with JSON output
+- [ ] VS Code extension with semantic density visualization
+- [ ] Additional capabilities available in the commercial tier
+
+## Background
+
+The Fiedler vector was originally conceived by Miroslav Fiedler (1973) for graph partitioning. This implementation applies it to natural language, treating text chunks as nodes in a semantic similarity graph. The spectral decomposition reveals the natural information structure of a prompt — which parts are tightly interconnected and which are peripheral — enabling principled, structure-aware compression.
+
+## License
+
+Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (FSL-1.1-ALv2). See [LICENSE](LICENSE) for full terms.

fiedler_compress-0.1.0/README.md

@@ -0,0 +1,156 @@
+# fiedler-compress
+
+**Spectral graph-theoretic prompt compression — a transparent middleware layer for LLM and agentic pipelines.**
+
+fiedler-compress uses the Fiedler vector — the second-smallest eigenvector of the graph Laplacian — to identify and remove semantically disconnected content from LLM prompts. The result: shorter prompts that preserve meaning, save tokens, and fit more useful context into your model's window. It runs as a lightweight, CPU-only preprocessing step, making it a drop-in middleware stage in front of any LLM.
+
+## Why Spectral Compression?
+
+Existing prompt compression tools (LLMLingua, TOON, selective summarization) treat text as a linear sequence and use statistical or neural methods to decide what to cut. They work, but they can't see **structural relationships** — they don't know that paragraph 3 is semantically load-bearing because it bridges two otherwise disconnected topics, or that paragraphs 7 and 8 say the same thing in different words.
+
+fiedler-compress builds a **similarity graph** over your text chunks, computes the spectral decomposition, and uses the Fiedler vector to find the natural semantic partitions. Chunks at the spectral periphery — weakly connected to the rest of the content — are the ones that can be safely removed. Chunks that bridge partitions are preserved, because removing them would fragment the prompt's information structure.
+
+This isn't summarization. It's **graph surgery**.
+
+## Why Middleware
+
+The pipeline is designed to sit transparently between your application and the model: text goes in, a smaller functionally-equivalent prompt comes out, and the downstream LLM call proceeds unchanged. It's CPU-only (no GPU, no neural model required for the core path) and fast enough to run synchronously in a request path without perceptible overhead.
+
+## For Agentic and Long-Context Workflows
+
+Agentic systems accumulate context fast — tool outputs, retrieved documents, prior reasoning, and multi-turn history all compete for a finite context window. fiedler-compress is built for this setting: compress retrieved documents and accumulated context to fit more useful material into the window. **Zone-aware protection** automatically detects instruction content (directives, constraints, format specs) and shields it from removal, so the parts of the prompt that steer the model survive compression while redundant context is pruned. This makes it a practical component for RAG pipelines, long-document processing, and multi-step agent memory.
+
+## Installation
+
+```bash
+pip install fiedler-compress
+```
+
+The open-core package runs entirely on NumPy and SciPy. A separate commercial tier offers additional capabilities (see below).
+
+## Quick Start
+
+### Python API
+
+```python
+from fiedler_optimizer import optimize
+
+result = optimize("""
+You are an expert financial analyst. Always respond in JSON.
+
+Context: The company was founded in 2015. Revenue was $45M in Q3,
+up 23% YoY. Operating expenses were $38M. Cash reserves: $120M.
+Their competitor reported $52M but is losing mid-market share.
+Industry analysts predict 15% sector growth. The CFO noted
+international expansion costs impacted margins. Retention is 94%.
+Average contract value up 18% to $85K.
+
+Task: Analyze Q3 performance.
+""")
+
+print(result.compressed)
+print(f"Saved ~{result.tokens_saved} tokens ({result.compression_ratio:.0%} reduction)")
+print(f"Algebraic connectivity λ₂ = {result.algebraic_connectivity:.4f}")
+```
+
+### CLI
+
+```bash
+# Compress a prompt
+fiedler optimize "Your long prompt here..."
+
+# Compress from file
+fiedler optimize --file my_prompt.txt
+
+# Aggressive compression (30% target removal)
+fiedler optimize --file my_prompt.txt --target 0.30
+
+# JSON output for piping
+fiedler optimize --file my_prompt.txt --json
+
+# Show what was removed
+fiedler optimize --file my_prompt.txt --verbose
+
+# Run the built-in benchmark
+fiedler benchmark
+```
+
+## How It Works
+
+1. **Chunk** the input text into semantically meaningful segments (sentences, paragraphs, or adaptive)
+2. **Build a similarity graph** — each chunk is a node, edge weights are cosine similarity (TF-IDF by default, neural embeddings optional)
+3. **Compute the Fiedler vector** — the eigenvector for λ₂ of the graph Laplacian
+4. **Score chunks** by combining spectral centrality with weighted degree
+5. **Zone-aware protection** — instruction content (directives, constraints, format specs) gets 2–3× protection weight; context content is the compression target
+6. **Prune** the lowest-scoring context chunks up to the target ratio
+
+## Performance
+
+The core pipeline is CPU-only and fast enough to run inline. Representative end-to-end latency (TF-IDF backend, single CPU core, no GPU):
+
+| Input size | End-to-end latency |
+|-----------:|-------------------:|
+| 1,000 tokens | ~12 ms |
+| 10,000 tokens | ~110 ms |
+| 50,000 tokens | ~550 ms |
+
+At typical production prompt sizes (1,000–10,000 tokens) the pipeline adds on the order of tens of milliseconds, negligible against typical LLM inference latency. Compression is a lossy operation — recall of fine-grained facts decreases as the target ratio increases — so keep must-keep content in protected instruction zones and tune the target ratio to your task. Detailed benchmarks will be published in a dedicated benchmarks document.
+
+## Key Features
+
+- **Dependency-free core** — only NumPy and SciPy required. No neural models, no API calls, runs entirely local
+- **Zone-aware compression** — distinguishes instructions from context, applies differential protection so directives survive
+- **Algebraic connectivity metric** — λ₂ tells you how tightly structured your prompt is *before* compression
+- **Transparent middleware** — smaller prompt in, unchanged LLM call out; runs inline with negligible latency
+- **JSON output** — pipe results into your existing toolchain
+
+## Configuration
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `target_ratio` | `0.20` | Fraction of text to remove (0.0–1.0) |
+| `strategy` | `adaptive` | Chunking: `adaptive`, `sentence`, `paragraph`, `window` |
+| `protect_instructions` | `True` | Shield instruction zones from removal |
+| `min_chunks` | `4` | Minimum chunks required for spectral analysis |
+| `vectors` | `None` | Pre-computed embeddings (n_chunks × d) |
+
+## Project Structure
+
+```
+fiedler-compress/
+├── fiedler_optimizer/
+│   ├── __init__.py      # Public API
+│   ├── core.py          # optimize() pipeline
+│   ├── chunker.py       # Text segmentation strategies
+│   ├── graph.py         # Similarity graph + Fiedler vector
+│   ├── zones.py         # Instruction/context zone detection
+│   └── cli.py           # Command-line interface
+├── tests/
+│   └── test_core.py     # Test suite
+├── pyproject.toml       # Package config
+├── LICENSE              # FSL-1.1-ALv2
+└── README.md
+```
+
+## Commercial Tier
+
+This package is the open-core distribution: the TF-IDF + single-eigenvector (k=1) spectral compression pipeline. Additional capabilities — including **content attestation / certification** — are available as a commercial add-on.
+
+For commercial licensing or attestation inquiries:
+**Tensor Earth Intelligence (TEI), LLC** — tensor.earth.intelligence@gmail.com (Mark Chappell).
+
+## Roadmap
+
+- [x] Core spectral compression with TF-IDF similarity
+- [x] Zone-aware instruction protection
+- [x] CLI with JSON output
+- [ ] VS Code extension with semantic density visualization
+- [ ] Additional capabilities available in the commercial tier
+
+## Background
+
+The Fiedler vector was originally conceived by Miroslav Fiedler (1973) for graph partitioning. This implementation applies it to natural language, treating text chunks as nodes in a semantic similarity graph. The spectral decomposition reveals the natural information structure of a prompt — which parts are tightly interconnected and which are peripheral — enabling principled, structure-aware compression.
+
+## License
+
+Licensed under the Functional Source License, Version 1.1, Apache 2.0 Future License (FSL-1.1-ALv2). See [LICENSE](LICENSE) for full terms.

fiedler_compress-0.1.0/fiedler_optimizer/__init__.py

@@ -0,0 +1,36 @@
+"""
+Fiedler Optimizer — Spectral graph-theoretic prompt compression.
+
+Uses the Fiedler vector (second-smallest eigenvector of the graph Laplacian)
+to identify and remove semantically disconnected chunks from LLM prompts,
+reducing token count while preserving information fidelity.
+
+Basic usage:
+    from fiedler_optimizer import optimize
+
+    result = optimize("Your long prompt text here...")
+    print(result.compressed)
+    print(f"Saved {result.tokens_saved} tokens ({result.compression_ratio:.1%})")
+
+This is the open-core distribution: the TF-IDF + single-eigenvector (k=1)
+spectral compression pipeline. Additional capabilities are available in a
+separate commercial tier.
+"""
+
+__version__ = "0.1.0"
+
+from fiedler_optimizer.core import optimize, FiedlerResult
+from fiedler_optimizer.graph import build_similarity_graph, compute_fiedler_vector
+from fiedler_optimizer.chunker import chunk_text, ChunkingStrategy
+from fiedler_optimizer.zones import detect_zones, Zone
+
+__all__ = [
+    "optimize",
+    "FiedlerResult",
+    "build_similarity_graph",
+    "compute_fiedler_vector",
+    "chunk_text",
+    "ChunkingStrategy",
+    "detect_zones",
+    "Zone",
+]

fiedler_compress-0.1.0/fiedler_optimizer/_tier.py

@@ -0,0 +1,29 @@
+"""Commercial-tier feature gating for the open-core distribution.
+
+Some capabilities are not included in the open-core package. The modules that
+implement them are absent here, so a lazy ``import`` of one raises
+:class:`ImportError`. The helpers below translate that into a clear,
+actionable message instead of an opaque import crash.
+"""
+
+from __future__ import annotations
+
+
+class CommercialTierError(RuntimeError):
+    """Raised when an open-core caller requests a commercial-tier capability."""
+
+
+def commercial_tier_error(feature: str | None = None) -> "CommercialTierError":
+    """Return a generic :class:`CommercialTierError` for a gated capability.
+
+    The message intentionally does not name the specific capability.
+
+    Parameters
+    ----------
+    feature : str, optional
+        Accepted for backward compatibility; ignored.
+    """
+    return CommercialTierError(
+        "This feature requires a commercial license of fiedler-optimizer. "
+        "See the project README for details."
+    )

fiedler_compress-0.1.0/fiedler_optimizer/benchmarks/__init__.py

@@ -0,0 +1 @@
+"""Benchmark harnesses for Fiedler Optimizer compression quality."""