ccllm 0.1.0__tar.gz

ccllm-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wasi_Ahmad
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ccllm-0.1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: ccllm
+Version: 0.1.0
+Summary: Lossless text compression, storage, memory management, and retrieval toolkit
+Author-email: Wasi Ahmad <wasiahamd0569@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/<your-user-or-org>/<repo>
+Project-URL: Repository, https://github.com/<your-user-or-org>/<repo>
+Project-URL: Issues, https://github.com/<your-user-or-org>/<repo>/issues
+Keywords: compression,retrieval,memory,storage,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fastapi<1.0,>=0.110
+Requires-Dist: pydantic<3.0,>=2.7
+Requires-Dist: numpy<2.0,>=1.26
+Requires-Dist: scikit-learn<2.0,>=1.4
+Requires-Dist: uvicorn<1.0,>=0.29
+Provides-Extra: dev
+Requires-Dist: pytest<9,>=8; extra == "dev"
+Requires-Dist: pytest-cov<6,>=5; extra == "dev"
+Requires-Dist: ruff<0.6,>=0.4; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine<6,>=5; extra == "dev"
+Dynamic: license-file

ccllm-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=69", "wheel", "build"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ccllm"
+version = "0.1.0"
+description = "Lossless text compression, storage, memory management, and retrieval toolkit"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+  { name = "Wasi Ahmad", email = "wasiahamd0569@gmail.com" }
+]
+keywords = ["compression", "retrieval", "memory", "storage", "nlp"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12"
+]
+dependencies = [
+  "fastapi>=0.110,<1.0",
+  "pydantic>=2.7,<3.0",
+  "numpy>=1.26,<2.0",
+  "scikit-learn>=1.4,<2.0",
+  "uvicorn>=0.29,<1.0"
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=8,<9",
+  "pytest-cov>=5,<6",
+  "ruff>=0.4,<0.6",
+  "build>=1.2",
+  "twine>=5,<6"
+]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[project.urls]
+Homepage = "https://github.com/<your-user-or-org>/<repo>"
+Repository = "https://github.com/<your-user-or-org>/<repo>"
+Issues = "https://github.com/<your-user-or-org>/<repo>/issues"

ccllm-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ccllm-0.1.0/src/ccllm/api/__init__.py

@@ -0,0 +1,3 @@
+from src.ccllm.api.app import app
+
+__all__ = ["app"]

ccllm-0.1.0/src/ccllm/api/app.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from fastapi import FastAPI, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from src.ccllm.memory import MemoryManager
+from src.ccllm.retrieval import MemoryRetriever
+from src.ccllm.storage import CompressionStorage
+
+# -----------------------------
+# App setup
+# -----------------------------
+
+storage = CompressionStorage("data/compression.db")
+memory_manager = MemoryManager(storage=storage, default_method="zlib")
+
+app = FastAPI(
+    title="Compression Memory API",
+    version="1.0.0",
+    description=(
+        "API for storing, restoring, listing, deleting, and retrieving "
+        "compressed text memories."
+    ),
+)
+
+
+# -----------------------------
+# Request / Response Models
+# -----------------------------
+
+CompressionMethod = Literal["none", "zlib", "lzma", "dictionary"]
+RetrievalMode = Literal["lexical", "vector", "hybrid"]
+
+
+class StoreTextRequest(BaseModel):
+    text: str = Field(..., description="Original text to compress and store.")
+    method: CompressionMethod | None = Field(
+        default=None,
+        description="Compression method to use. Falls back to default if omitted.",
+    )
+    metadata: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional metadata stored with the record.",
+    )
+    record_id: str | None = Field(
+        default=None,
+        description="Optional custom record ID.",
+    )
+    compressor_kwargs: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional compressor configuration.",
+    )
+
+
+class StoreTextResponse(BaseModel):
+    record_id: str
+    created_at: str
+    method: str
+    original_sha256: str
+    original_length: int
+    token_count: int
+    compressed_bytes: int
+    compression_ratio: float
+    metadata_json: str
+
+
+class RetrieveRequest(BaseModel):
+    query: str = Field(..., description="Search query text.")
+    mode: RetrievalMode = Field(
+        default="hybrid",
+        description="Retrieval mode: lexical, vector, or hybrid.",
+    )
+    limit: int = Field(default=5, ge=1, description="Maximum number of results.")
+    search_limit: int = Field(
+        default=100,
+        ge=1,
+        description="Maximum number of stored memories to inspect.",
+    )
+    metadata_filter: dict[str, Any] | None = Field(
+        default=None,
+        description="Optional exact-match metadata filter.",
+    )
+    alpha: float = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="Hybrid weighting. Used only in hybrid mode.",
+    )
+
+
+class RetrievalItem(BaseModel):
+    record_id: str
+    score: float
+    method: str
+    created_at: str
+    metadata: dict[str, Any]
+    text: str
+
+
+class DeleteResponse(BaseModel):
+    deleted: bool
+    record_id: str
+
+
+class HealthResponse(BaseModel):
+    status: str
+    total_memories: int
+
+
+# -----------------------------
+# Routes
+# -----------------------------
+
+
+@app.get("/health", response_model=HealthResponse)
+def health() -> HealthResponse:
+    return HealthResponse(
+        status="ok",
+        total_memories=memory_manager.count_memories(),
+    )
+
+
+@app.post("/store", response_model=StoreTextResponse)
+def store_text(payload: StoreTextRequest) -> StoreTextResponse:
+    try:
+        record = memory_manager.save_text(
+            text=payload.text,
+            method=payload.method,
+            metadata=payload.metadata,
+            record_id=payload.record_id,
+            compressor_kwargs=payload.compressor_kwargs,
+        )
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    return StoreTextResponse(
+        record_id=record.record_id,
+        created_at=record.created_at,
+        method=record.method,
+        original_sha256=record.original_sha256,
+        original_length=record.original_length,
+        token_count=record.token_count,
+        compressed_bytes=record.compressed_bytes,
+        compression_ratio=record.compression_ratio,
+        metadata_json=record.metadata_json,
+    )
+
+
+@app.get("/memories")
+def list_memories(
+    limit: int = Query(default=100, ge=1, description="Maximum number of records."),
+) -> list[dict[str, Any]]:
+    try:
+        records = memory_manager.list_memories(limit=limit)
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    return [record.to_dict() for record in records]
+
+
+@app.get("/memories/{record_id}")
+def get_memory(record_id: str) -> dict[str, Any]:
+    bundle = memory_manager.export_record_bundle(record_id)
+    if bundle is None:
+        raise HTTPException(status_code=404, detail="Record not found")
+    return bundle
+
+
+@app.get("/memories/{record_id}/text")
+def get_memory_text(record_id: str) -> dict[str, Any]:
+    text = memory_manager.get_text(record_id)
+    if text is None:
+        raise HTTPException(status_code=404, detail="Record not found")
+    return {"record_id": record_id, "text": text}
+
+
+@app.delete("/memories/{record_id}", response_model=DeleteResponse)
+def delete_memory(record_id: str) -> DeleteResponse:
+    deleted = memory_manager.delete_memory(record_id)
+    if not deleted:
+        raise HTTPException(status_code=404, detail="Record not found")
+
+    return DeleteResponse(
+        deleted=True,
+        record_id=record_id,
+    )
+
+
+@app.post("/retrieve", response_model=list[RetrievalItem])
+def retrieve_texts(payload: RetrieveRequest) -> list[RetrievalItem]:
+    try:
+        retriever = MemoryRetriever(
+            memory_manager=memory_manager,
+            mode=payload.mode,
+            alpha=payload.alpha,
+        )
+        results = retriever.retrieve(
+            query=payload.query,
+            limit=payload.limit,
+            search_limit=payload.search_limit,
+            metadata_filter=payload.metadata_filter,
+        )
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    return [
+        RetrievalItem(
+            record_id=result.record_id,
+            score=result.score,
+            method=result.method,
+            created_at=result.created_at,
+            metadata=result.metadata,
+            text=result.text,
+        )
+        for result in results
+    ]

ccllm-0.1.0/src/ccllm/compression/__init__.py

@@ -0,0 +1,31 @@
+from src.ccllm.compression.compressor import (
+    CompressionMethod,
+    CompressionPackage,
+    CompressionStats,
+    TextCompressor,
+)
+from src.ccllm.compression.decompressor import TextDecompressor
+from src.ccllm.compression.dictionary import (
+    DictionaryEntry,
+    PhraseDictionaryBuilder,
+)
+from src.ccllm.compression.tokenizer import (
+    TextTokenizer,
+    TokenizationResult,
+)
+
+__all__ = [
+    # tokenizer
+    "TextTokenizer",
+    "TokenizationResult",
+    # dictionary
+    "PhraseDictionaryBuilder",
+    "DictionaryEntry",
+    # compressor
+    "TextCompressor",
+    "CompressionPackage",
+    "CompressionStats",
+    "CompressionMethod",
+    # decompressor
+    "TextDecompressor",
+]

ccllm-0.1.0/src/ccllm/compression/compressor.py

@@ -0,0 +1,300 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import lzma
+import zlib
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+from src.ccllm.compression.dictionary import DictionaryEntry, PhraseDictionaryBuilder
+from src.ccllm.compression.tokenizer import TextTokenizer
+
+CompressionMethod = Literal["none", "zlib", "lzma", "dictionary"]
+
+
+@dataclass(frozen=True)
+class CompressionStats:
+    original_bytes: int
+    compressed_bytes: int
+    compression_ratio: float
+    space_saving_ratio: float
+    original_token_count: int
+    compressed_token_count: int | None = None
+
+
+@dataclass(frozen=True)
+class CompressionPackage:
+    version: str
+    method: CompressionMethod
+    original_length: int
+    original_sha256: str
+    token_count: int
+    compressed_payload_b64: str
+    stats: CompressionStats
+    metadata: dict[str, Any]
+    dictionary: dict[str, list[str]] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_dict(), ensure_ascii=False, indent=2)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> CompressionPackage:
+        stats = CompressionStats(**data["stats"])
+        return cls(
+            version=data["version"],
+            method=data["method"],
+            original_length=data["original_length"],
+            original_sha256=data["original_sha256"],
+            token_count=data["token_count"],
+            compressed_payload_b64=data["compressed_payload_b64"],
+            stats=stats,
+            metadata=data.get("metadata", {}),
+            dictionary=data.get("dictionary"),
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> CompressionPackage:
+        return cls.from_dict(json.loads(json_str))
+
+
+class TextCompressor:
+    """
+    Baseline + custom dictionary text compressor.
+
+    Supported methods:
+    - none
+    - zlib
+    - lzma
+    - dictionary
+
+    The dictionary method is token-aware:
+    - tokenize text losslessly
+    - build repeated phrase dictionary
+    - replace repeated phrases with phrase IDs
+    - serialize encoded token stream as UTF-8 JSON
+    """
+
+    SUPPORTED_METHODS: tuple[CompressionMethod, ...] = (
+        "none",
+        "zlib",
+        "lzma",
+        "dictionary",
+    )
+
+    def __init__(
+        self,
+        method: CompressionMethod = "zlib",
+        zlib_level: int = 9,
+        lzma_preset: int = 6,
+        min_phrase_len: int = 2,
+        max_phrase_len: int = 8,
+        min_frequency: int = 2,
+        max_dictionary_size: int = 256,
+        min_estimated_savings: int = 1,
+        skip_all_whitespace_phrases: bool = True,
+    ) -> None:
+        if method not in self.SUPPORTED_METHODS:
+            raise ValueError(
+                f"Unsupported compression method: {method}. "
+                f"Supported: {self.SUPPORTED_METHODS}"
+            )
+
+        if not 0 <= zlib_level <= 9:
+            raise ValueError("zlib_level must be between 0 and 9")
+
+        if not 0 <= lzma_preset <= 9:
+            raise ValueError("lzma_preset must be between 0 and 9")
+
+        self.method = method
+        self.zlib_level = zlib_level
+        self.lzma_preset = lzma_preset
+        self.tokenizer = TextTokenizer()
+        self.dictionary_builder = PhraseDictionaryBuilder(
+            min_phrase_len=min_phrase_len,
+            max_phrase_len=max_phrase_len,
+            min_frequency=min_frequency,
+            max_dictionary_size=max_dictionary_size,
+            min_estimated_savings=min_estimated_savings,
+            skip_all_whitespace_phrases=skip_all_whitespace_phrases,
+        )
+
+    def compress(
+        self,
+        text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> CompressionPackage:
+        if not isinstance(text, str):
+            raise TypeError("text must be a string")
+
+        metadata = metadata or {}
+        tokenization = self.tokenizer.tokenize(text)
+        original_bytes = text.encode("utf-8")
+
+        if self.method == "dictionary":
+            return self._compress_with_dictionary(
+                text=text,
+                original_bytes=original_bytes,
+                token_count=tokenization.token_count,
+                tokens=tokenization.tokens,
+                metadata=metadata,
+            )
+
+        compressed_bytes = self._compress_bytes(original_bytes)
+        payload_b64 = base64.b64encode(compressed_bytes).decode("ascii")
+
+        stats = self._build_stats(
+            original_bytes_len=len(original_bytes),
+            compressed_bytes_len=len(compressed_bytes),
+            original_token_count=tokenization.token_count,
+            compressed_token_count=None,
+        )
+
+        return CompressionPackage(
+            version="1.1.0",
+            method=self.method,
+            original_length=len(text),
+            original_sha256=self._sha256_hex(original_bytes),
+            token_count=tokenization.token_count,
+            compressed_payload_b64=payload_b64,
+            stats=stats,
+            metadata=metadata,
+            dictionary=None,
+        )
+
+    def compress_to_json(
+        self,
+        text: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        return self.compress(text=text, metadata=metadata).to_json()
+
+    def _compress_with_dictionary(
+        self,
+        text: str,
+        original_bytes: bytes,
+        token_count: int,
+        tokens: list[str],
+        metadata: dict[str, Any],
+    ) -> CompressionPackage:
+        entries = self.dictionary_builder.build(tokens)
+        dictionary_map = self.dictionary_builder.build_reverse_lookup(entries)
+        encoded_tokens = self._encode_with_dictionary(tokens, entries)
+
+        serialized_stream = json.dumps(
+            encoded_tokens,
+            ensure_ascii=False,
+            separators=(",", ":"),
+        ).encode("utf-8")
+        payload_b64 = base64.b64encode(serialized_stream).decode("ascii")
+
+        stats = self._build_stats(
+            original_bytes_len=len(original_bytes),
+            compressed_bytes_len=len(serialized_stream),
+            original_token_count=token_count,
+            compressed_token_count=len(encoded_tokens),
+        )
+
+        return CompressionPackage(
+            version="1.1.0",
+            method="dictionary",
+            original_length=len(text),
+            original_sha256=self._sha256_hex(original_bytes),
+            token_count=token_count,
+            compressed_payload_b64=payload_b64,
+            stats=stats,
+            metadata=metadata,
+            dictionary=dictionary_map,
+        )
+
+    def _encode_with_dictionary(
+        self,
+        tokens: list[str],
+        entries: list[DictionaryEntry],
+    ) -> list[str]:
+        if not entries:
+            return list(tokens)
+
+        phrase_to_id = self.dictionary_builder.build_lookup(entries)
+        available_lengths = sorted(
+            {len(entry.phrase) for entry in entries},
+            reverse=True,
+        )
+
+        encoded_tokens: list[str] = []
+        token_index = 0
+        total_tokens = len(tokens)
+
+        while token_index < total_tokens:
+            matched_phrase_id: str | None = None
+            matched_phrase_len = 0
+
+            for phrase_len in available_lengths:
+                if token_index + phrase_len > total_tokens:
+                    continue
+
+                candidate = tuple(tokens[token_index : token_index + phrase_len])
+                phrase_id = phrase_to_id.get(candidate)
+
+                if phrase_id is not None:
+                    matched_phrase_id = phrase_id
+                    matched_phrase_len = phrase_len
+                    break
+
+            if matched_phrase_id is not None:
+                encoded_tokens.append(matched_phrase_id)
+                token_index += matched_phrase_len
+            else:
+                encoded_tokens.append(tokens[token_index])
+                token_index += 1
+
+        return encoded_tokens
+
+    def _compress_bytes(self, data: bytes) -> bytes:
+        if self.method == "none":
+            return data
+
+        if self.method == "zlib":
+            return zlib.compress(data, level=self.zlib_level)
+
+        if self.method == "lzma":
+            return lzma.compress(data, preset=self.lzma_preset)
+
+        raise ValueError(f"Unhandled compression method: {self.method}")
+
+    @staticmethod
+    def _sha256_hex(data: bytes) -> str:
+        return hashlib.sha256(data).hexdigest()
+
+    @staticmethod
+    def _build_stats(
+        original_bytes_len: int,
+        compressed_bytes_len: int,
+        original_token_count: int,
+        compressed_token_count: int | None,
+    ) -> CompressionStats:
+        if original_bytes_len < 0 or compressed_bytes_len < 0:
+            raise ValueError("byte lengths must be non-negative")
+
+        if original_bytes_len == 0:
+            compression_ratio = 1.0
+            space_saving_ratio = 0.0
+        else:
+            compression_ratio = compressed_bytes_len / original_bytes_len
+            space_saving_ratio = 1.0 - compression_ratio
+
+        return CompressionStats(
+            original_bytes=original_bytes_len,
+            compressed_bytes=compressed_bytes_len,
+            compression_ratio=compression_ratio,
+            space_saving_ratio=space_saving_ratio,
+            original_token_count=original_token_count,
+            compressed_token_count=compressed_token_count,
+        )
+
+    def available_methods(self) -> list[str]:
+        return list(self.SUPPORTED_METHODS)