EasySmallEmbeddingModel 0.1.0__tar.gz

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: EasySmallEmbeddingModel
+Version: 0.1.0
+Summary: Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation.
+Author: gomyk
+License: Apache-2.0
+Keywords: embedding,model-compression,distillation,pruning,sentence-transformers
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=4.40.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: tqdm
+Requires-Dist: sentencepiece
+Requires-Dist: protobuf
+Provides-Extra: eval
+Requires-Dist: mteb>=1.14.0; extra == "eval"
+Requires-Dist: pandas>=2.0.0; extra == "eval"
+Provides-Extra: export
+Requires-Dist: onnxruntime>=1.16.0; extra == "export"
+Requires-Dist: onnx>=1.15.0; extra == "export"
+Requires-Dist: optimum[onnxruntime]>=1.16.0; extra == "export"
+Provides-Extra: hub
+Requires-Dist: huggingface-hub>=0.20.0; extra == "hub"
+Provides-Extra: web
+Requires-Dist: flask>=3.0.0; extra == "web"
+Provides-Extra: all
+Requires-Dist: EasySmallEmbeddingModel[eval,export,hub,web]; extra == "all"
+
+# SmallModel
+
+Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation.
+
+## Features
+
+- **Layer Pruning** - Select which transformer layers to keep
+- **Vocab Pruning** - Remove unused tokens based on corpus frequency
+- **Hidden Dim Reduction** - Shrink internal dimensions (slicing or PCA)
+- **Knowledge Distillation** - MSE + Cosine loss alignment with teacher
+- **Auto Compress** - Find optimal config within size constraints
+- **2-Stage Distillation** - Progressive distillation for 10x+ compression
+- **Interactive Web UI** - Visual layer editor with real-time size estimation
+- **MTEB Evaluation** - Benchmark on Classification, Clustering, STS tasks
+
+## Installation
+
+```bash
+pip install smallmodel[all]
+```
+
+Or install specific extras:
+
+```bash
+pip install smallmodel          # core only
+pip install smallmodel[web]     # + Flask web UI
+pip install smallmodel[eval]    # + MTEB evaluation
+pip install smallmodel[export]  # + ONNX export
+pip install smallmodel[hub]     # + HuggingFace Hub upload
+```
+
+For development:
+
+```bash
+git clone https://github.com/gomyk/smallmodel.git
+cd smallmodel
+pip install -e ".[all]"
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from smallmodel import SmallModel
+
+# Auto-compress within 50MB
+sm = SmallModel.from_teacher("gte")
+sm.compress(max_fp32_mb=50.0)
+sm.distill(epochs=10)
+
+# Manual layer selection
+sm = SmallModel.from_teacher("gte", layer_indices=[0, 3, 6, 11])
+sm.create()
+
+# Register custom teacher
+from smallmodel import register_teacher
+register_teacher(
+    "my-bert",
+    model_id="my-org/my-bert-base",
+    short_name="MyBERT",
+    hidden_dim=768, num_layers=12,
+    intermediate_size=3072, vocab_size=30522,
+)
+```
+
+### Web UI
+
+```python
+from smallmodel import SmallModel
+
+sm = SmallModel.from_teacher("gte")
+sm.serve()  # http://127.0.0.1:7860
+```
+
+Or via CLI:
+
+```bash
+smallmodel serve --teacher gte --port 7860
+```
+
+The web UI lets you:
+- Select teacher model from 7+ pre-registered models
+- Toggle layers on/off with preset configurations
+- Adjust hidden dim, FFN size, and vocab size
+- See real-time size estimation and compression ratio
+- Select distillation datasets and evaluation tasks
+- Analyze vocab coverage at different vocab sizes
+- Create compressed models with one click
+
+### CLI
+
+```bash
+smallmodel list-teachers
+smallmodel compress --teacher gte --max-mb 50
+smallmodel create --teacher gte --layers 0,3,6,11
+smallmodel distill --teacher gte --student output/students/gte/gte_compressed
+smallmodel serve --teacher gte
+```
+
+## Pre-registered Teachers
+
+| Key | Model | Layers | Hidden | Vocab | FP32 MB |
+|---|---|---|---|---|---|
+| minilm | paraphrase-multilingual-MiniLM-L12-v2 | 12 | 384 | 250K | 448 |
+| modernbert | ModernBERT-base | 22 | 768 | 50K | 496 |
+| gte | gte-multilingual-base | 12 | 768 | 250K | 1058 |
+| me5 | multilingual-e5-base | 12 | 768 | 250K | 1058 |
+| me5s | multilingual-e5-small | 12 | 384 | 250K | 448 |
+| gemma_emb | embeddinggemma-300m | 24 | 768 | 262K | 1155 |
+| qwen3 | Qwen3-0.6B | 28 | 1024 | 152K | 2274 |
+
+## How It Works
+
+1. **Layer Pruning** - Copy selected layers from teacher (uniform spacing recommended)
+2. **Hidden Dim Reduction** - Shrink dimensions if needed to meet size target
+3. **Vocab Pruning** - Remove tokens not seen in training corpus
+4. **Knowledge Distillation** - Train student to reproduce teacher's embeddings
+5. **Evaluation** - MTEB benchmark (Classification, Clustering, STS)
+
+For compression ratios > 10x, a 2-stage distillation pipeline is used:
+Teacher → Intermediate (~1/5 teacher) → Final Student
+
+## License
+
+Apache-2.0

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/SOURCES.txt

@@ -0,0 +1,22 @@
+MANIFEST.in
+README.md
+pyproject.toml
+EasySmallEmbeddingModel.egg-info/PKG-INFO
+EasySmallEmbeddingModel.egg-info/SOURCES.txt
+EasySmallEmbeddingModel.egg-info/dependency_links.txt
+EasySmallEmbeddingModel.egg-info/entry_points.txt
+EasySmallEmbeddingModel.egg-info/requires.txt
+EasySmallEmbeddingModel.egg-info/top_level.txt
+smallmodel/__init__.py
+smallmodel/arch.py
+smallmodel/cli.py
+smallmodel/core.py
+smallmodel/data.py
+smallmodel/distill.py
+smallmodel/sizing.py
+smallmodel/teachers.py
+smallmodel/web/__init__.py
+smallmodel/web/app.py
+smallmodel/web/static/css/style.css
+smallmodel/web/static/js/app.js
+smallmodel/web/templates/index.html

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+smallmodel = smallmodel.cli:main

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/requires.txt

@@ -0,0 +1,25 @@
+torch>=2.0.0
+transformers>=4.40.0
+sentence-transformers>=3.0.0
+numpy>=1.24.0
+tqdm
+sentencepiece
+protobuf
+
+[all]
+EasySmallEmbeddingModel[eval,export,hub,web]
+
+[eval]
+mteb>=1.14.0
+pandas>=2.0.0
+
+[export]
+onnxruntime>=1.16.0
+onnx>=1.15.0
+optimum[onnxruntime]>=1.16.0
+
+[hub]
+huggingface-hub>=0.20.0
+
+[web]
+flask>=3.0.0

easysmallembeddingmodel-0.1.0/EasySmallEmbeddingModel.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+dist
+smallmodel

easysmallembeddingmodel-0.1.0/MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include smallmodel/web/static *
+recursive-include smallmodel/web/templates *

easysmallembeddingmodel-0.1.0/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: EasySmallEmbeddingModel
+Version: 0.1.0
+Summary: Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation.
+Author: gomyk
+License: Apache-2.0
+Keywords: embedding,model-compression,distillation,pruning,sentence-transformers
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: torch>=2.0.0
+Requires-Dist: transformers>=4.40.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: tqdm
+Requires-Dist: sentencepiece
+Requires-Dist: protobuf
+Provides-Extra: eval
+Requires-Dist: mteb>=1.14.0; extra == "eval"
+Requires-Dist: pandas>=2.0.0; extra == "eval"
+Provides-Extra: export
+Requires-Dist: onnxruntime>=1.16.0; extra == "export"
+Requires-Dist: onnx>=1.15.0; extra == "export"
+Requires-Dist: optimum[onnxruntime]>=1.16.0; extra == "export"
+Provides-Extra: hub
+Requires-Dist: huggingface-hub>=0.20.0; extra == "hub"
+Provides-Extra: web
+Requires-Dist: flask>=3.0.0; extra == "web"
+Provides-Extra: all
+Requires-Dist: EasySmallEmbeddingModel[eval,export,hub,web]; extra == "all"
+
+# SmallModel
+
+Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation.
+
+## Features
+
+- **Layer Pruning** - Select which transformer layers to keep
+- **Vocab Pruning** - Remove unused tokens based on corpus frequency
+- **Hidden Dim Reduction** - Shrink internal dimensions (slicing or PCA)
+- **Knowledge Distillation** - MSE + Cosine loss alignment with teacher
+- **Auto Compress** - Find optimal config within size constraints
+- **2-Stage Distillation** - Progressive distillation for 10x+ compression
+- **Interactive Web UI** - Visual layer editor with real-time size estimation
+- **MTEB Evaluation** - Benchmark on Classification, Clustering, STS tasks
+
+## Installation
+
+```bash
+pip install smallmodel[all]
+```
+
+Or install specific extras:
+
+```bash
+pip install smallmodel          # core only
+pip install smallmodel[web]     # + Flask web UI
+pip install smallmodel[eval]    # + MTEB evaluation
+pip install smallmodel[export]  # + ONNX export
+pip install smallmodel[hub]     # + HuggingFace Hub upload
+```
+
+For development:
+
+```bash
+git clone https://github.com/gomyk/smallmodel.git
+cd smallmodel
+pip install -e ".[all]"
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from smallmodel import SmallModel
+
+# Auto-compress within 50MB
+sm = SmallModel.from_teacher("gte")
+sm.compress(max_fp32_mb=50.0)
+sm.distill(epochs=10)
+
+# Manual layer selection
+sm = SmallModel.from_teacher("gte", layer_indices=[0, 3, 6, 11])
+sm.create()
+
+# Register custom teacher
+from smallmodel import register_teacher
+register_teacher(
+    "my-bert",
+    model_id="my-org/my-bert-base",
+    short_name="MyBERT",
+    hidden_dim=768, num_layers=12,
+    intermediate_size=3072, vocab_size=30522,
+)
+```
+
+### Web UI
+
+```python
+from smallmodel import SmallModel
+
+sm = SmallModel.from_teacher("gte")
+sm.serve()  # http://127.0.0.1:7860
+```
+
+Or via CLI:
+
+```bash
+smallmodel serve --teacher gte --port 7860
+```
+
+The web UI lets you:
+- Select teacher model from 7+ pre-registered models
+- Toggle layers on/off with preset configurations
+- Adjust hidden dim, FFN size, and vocab size
+- See real-time size estimation and compression ratio
+- Select distillation datasets and evaluation tasks
+- Analyze vocab coverage at different vocab sizes
+- Create compressed models with one click
+
+### CLI
+
+```bash
+smallmodel list-teachers
+smallmodel compress --teacher gte --max-mb 50
+smallmodel create --teacher gte --layers 0,3,6,11
+smallmodel distill --teacher gte --student output/students/gte/gte_compressed
+smallmodel serve --teacher gte
+```
+
+## Pre-registered Teachers
+
+| Key | Model | Layers | Hidden | Vocab | FP32 MB |
+|---|---|---|---|---|---|
+| minilm | paraphrase-multilingual-MiniLM-L12-v2 | 12 | 384 | 250K | 448 |
+| modernbert | ModernBERT-base | 22 | 768 | 50K | 496 |
+| gte | gte-multilingual-base | 12 | 768 | 250K | 1058 |
+| me5 | multilingual-e5-base | 12 | 768 | 250K | 1058 |
+| me5s | multilingual-e5-small | 12 | 384 | 250K | 448 |
+| gemma_emb | embeddinggemma-300m | 24 | 768 | 262K | 1155 |
+| qwen3 | Qwen3-0.6B | 28 | 1024 | 152K | 2274 |
+
+## How It Works
+
+1. **Layer Pruning** - Copy selected layers from teacher (uniform spacing recommended)
+2. **Hidden Dim Reduction** - Shrink dimensions if needed to meet size target
+3. **Vocab Pruning** - Remove tokens not seen in training corpus
+4. **Knowledge Distillation** - Train student to reproduce teacher's embeddings
+5. **Evaluation** - MTEB benchmark (Classification, Clustering, STS)
+
+For compression ratios > 10x, a 2-stage distillation pipeline is used:
+Teacher → Intermediate (~1/5 teacher) → Final Student
+
+## License
+
+Apache-2.0

easysmallembeddingmodel-0.1.0/README.md

@@ -0,0 +1,126 @@
+# SmallModel
+
+Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation.
+
+## Features
+
+- **Layer Pruning** - Select which transformer layers to keep
+- **Vocab Pruning** - Remove unused tokens based on corpus frequency
+- **Hidden Dim Reduction** - Shrink internal dimensions (slicing or PCA)
+- **Knowledge Distillation** - MSE + Cosine loss alignment with teacher
+- **Auto Compress** - Find optimal config within size constraints
+- **2-Stage Distillation** - Progressive distillation for 10x+ compression
+- **Interactive Web UI** - Visual layer editor with real-time size estimation
+- **MTEB Evaluation** - Benchmark on Classification, Clustering, STS tasks
+
+## Installation
+
+```bash
+pip install smallmodel[all]
+```
+
+Or install specific extras:
+
+```bash
+pip install smallmodel          # core only
+pip install smallmodel[web]     # + Flask web UI
+pip install smallmodel[eval]    # + MTEB evaluation
+pip install smallmodel[export]  # + ONNX export
+pip install smallmodel[hub]     # + HuggingFace Hub upload
+```
+
+For development:
+
+```bash
+git clone https://github.com/gomyk/smallmodel.git
+cd smallmodel
+pip install -e ".[all]"
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from smallmodel import SmallModel
+
+# Auto-compress within 50MB
+sm = SmallModel.from_teacher("gte")
+sm.compress(max_fp32_mb=50.0)
+sm.distill(epochs=10)
+
+# Manual layer selection
+sm = SmallModel.from_teacher("gte", layer_indices=[0, 3, 6, 11])
+sm.create()
+
+# Register custom teacher
+from smallmodel import register_teacher
+register_teacher(
+    "my-bert",
+    model_id="my-org/my-bert-base",
+    short_name="MyBERT",
+    hidden_dim=768, num_layers=12,
+    intermediate_size=3072, vocab_size=30522,
+)
+```
+
+### Web UI
+
+```python
+from smallmodel import SmallModel
+
+sm = SmallModel.from_teacher("gte")
+sm.serve()  # http://127.0.0.1:7860
+```
+
+Or via CLI:
+
+```bash
+smallmodel serve --teacher gte --port 7860
+```
+
+The web UI lets you:
+- Select teacher model from 7+ pre-registered models
+- Toggle layers on/off with preset configurations
+- Adjust hidden dim, FFN size, and vocab size
+- See real-time size estimation and compression ratio
+- Select distillation datasets and evaluation tasks
+- Analyze vocab coverage at different vocab sizes
+- Create compressed models with one click
+
+### CLI
+
+```bash
+smallmodel list-teachers
+smallmodel compress --teacher gte --max-mb 50
+smallmodel create --teacher gte --layers 0,3,6,11
+smallmodel distill --teacher gte --student output/students/gte/gte_compressed
+smallmodel serve --teacher gte
+```
+
+## Pre-registered Teachers
+
+| Key | Model | Layers | Hidden | Vocab | FP32 MB |
+|---|---|---|---|---|---|
+| minilm | paraphrase-multilingual-MiniLM-L12-v2 | 12 | 384 | 250K | 448 |
+| modernbert | ModernBERT-base | 22 | 768 | 50K | 496 |
+| gte | gte-multilingual-base | 12 | 768 | 250K | 1058 |
+| me5 | multilingual-e5-base | 12 | 768 | 250K | 1058 |
+| me5s | multilingual-e5-small | 12 | 384 | 250K | 448 |
+| gemma_emb | embeddinggemma-300m | 24 | 768 | 262K | 1155 |
+| qwen3 | Qwen3-0.6B | 28 | 1024 | 152K | 2274 |
+
+## How It Works
+
+1. **Layer Pruning** - Copy selected layers from teacher (uniform spacing recommended)
+2. **Hidden Dim Reduction** - Shrink dimensions if needed to meet size target
+3. **Vocab Pruning** - Remove tokens not seen in training corpus
+4. **Knowledge Distillation** - Train student to reproduce teacher's embeddings
+5. **Evaluation** - MTEB benchmark (Classification, Clustering, STS)
+
+For compression ratios > 10x, a 2-stage distillation pipeline is used:
+Teacher → Intermediate (~1/5 teacher) → Final Student
+
+## License
+
+Apache-2.0

easysmallembeddingmodel-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "EasySmallEmbeddingModel"
+version = "0.1.0"
+description = "Compress large embedding models into small, fast students via layer pruning, vocab pruning, hidden dim reduction, and knowledge distillation."
+readme = "README.md"
+license = {text = "Apache-2.0"}
+requires-python = ">=3.9"
+authors = [{name = "gomyk"}]
+keywords = ["embedding", "model-compression", "distillation", "pruning", "sentence-transformers"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "torch>=2.0.0",
+    "transformers>=4.40.0",
+    "sentence-transformers>=3.0.0",
+    "numpy>=1.24.0",
+    "tqdm",
+    "sentencepiece",
+    "protobuf",
+]
+
+[project.optional-dependencies]
+eval = ["mteb>=1.14.0", "pandas>=2.0.0"]
+export = ["onnxruntime>=1.16.0", "onnx>=1.15.0", "optimum[onnxruntime]>=1.16.0"]
+hub = ["huggingface-hub>=0.20.0"]
+web = ["flask>=3.0.0"]
+all = ["EasySmallEmbeddingModel[eval,export,hub,web]"]
+
+[project.scripts]
+smallmodel = "smallmodel.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["."]
+
+[tool.setuptools.package-data]
+"smallmodel.web" = ["static/**/*", "templates/**/*"]

easysmallembeddingmodel-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

easysmallembeddingmodel-0.1.0/smallmodel/__init__.py

@@ -0,0 +1,7 @@
+"""SmallModel - Compress large embedding models into small, fast students."""
+
+from smallmodel.core import SmallModel
+from smallmodel.teachers import TEACHERS, register_teacher
+
+__version__ = "0.1.0"
+__all__ = ["SmallModel", "TEACHERS", "register_teacher"]