poolin 0.1.0__tar.gz

poolin-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Wenxi Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

poolin-0.1.0/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: poolin
+Version: 0.1.0
+Summary: A local UI package for pooling existing embedding zips into grouped vectors
+Author: Wenxi Wang
+License-Expression: MIT
+Project-URL: Homepage, https://example.com/poolin
+Project-URL: Repository, https://example.com/poolin
+Keywords: pooling,embedding,rag,vectors,streamlit,ui
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: streamlit>=1.32
+Requires-Dist: numpy>=1.24
+Dynamic: license-file
+
+# poolin
+
+A local UI package for pooling existing embedding vectors from an embedding zip into grouped higher-level vectors.
+
+## Important note
+
+Standard sentence-transformer style pooling usually happens **inside the embedding model** when token embeddings are converted into one sentence embedding. This package does **post-embedding vector pooling** over already-created chunk embeddings.
+
+## What it does
+
+- launches with the `poolin` command
+- reads an embedding zip such as `RAG_chunks_recursive_chunks_embeddings.zip`
+- auto-groups related chunk embeddings by filename pattern like `RAG_chunk_001_rcs_001.md -> RAG_chunk_001`
+- pools vectors with one of these methods:
+  - `auto`
+  - `mean`
+  - `max`
+  - `weighted_char_mean`
+  - `weighted_word_mean`
+  - `mean_sqrt_len`
+- exports a zip with:
+  - `pooling_summary.json`
+  - `pooling_manifest.csv`
+  - `*_pooled_embeddings.jsonl` (optional)
+  - `*_pooled_embeddings.csv` (optional)
+  - `*_pooled_embeddings.npz` (optional)
+
+## Install
+
+```bash
+pip install poolin
+```
+
+## Run
+
+```bash
+poolin
+```
+
+## Suggested input
+
+Use a zip produced by your embedding step, containing an embeddings `.npz` or `.jsonl` payload plus the summary file.
+
+## Ownership note
+
+The package metadata and copyright notice are set to Wenxi Wang. You should still verify PyPI package-name availability, trademark questions, and any legal or patent issues yourself before publishing.

poolin-0.1.0/README.md

@@ -0,0 +1,46 @@
+# poolin
+
+A local UI package for pooling existing embedding vectors from an embedding zip into grouped higher-level vectors.
+
+## Important note
+
+Standard sentence-transformer style pooling usually happens **inside the embedding model** when token embeddings are converted into one sentence embedding. This package does **post-embedding vector pooling** over already-created chunk embeddings.
+
+## What it does
+
+- launches with the `poolin` command
+- reads an embedding zip such as `RAG_chunks_recursive_chunks_embeddings.zip`
+- auto-groups related chunk embeddings by filename pattern like `RAG_chunk_001_rcs_001.md -> RAG_chunk_001`
+- pools vectors with one of these methods:
+  - `auto`
+  - `mean`
+  - `max`
+  - `weighted_char_mean`
+  - `weighted_word_mean`
+  - `mean_sqrt_len`
+- exports a zip with:
+  - `pooling_summary.json`
+  - `pooling_manifest.csv`
+  - `*_pooled_embeddings.jsonl` (optional)
+  - `*_pooled_embeddings.csv` (optional)
+  - `*_pooled_embeddings.npz` (optional)
+
+## Install
+
+```bash
+pip install poolin
+```
+
+## Run
+
+```bash
+poolin
+```
+
+## Suggested input
+
+Use a zip produced by your embedding step, containing an embeddings `.npz` or `.jsonl` payload plus the summary file.
+
+## Ownership note
+
+The package metadata and copyright notice are set to Wenxi Wang. You should still verify PyPI package-name availability, trademark questions, and any legal or patent issues yourself before publishing.

poolin-0.1.0/poolin/__init__.py

@@ -0,0 +1,2 @@
+__all__ = ["__version__"]
+__version__ = "0.1.0"

poolin-0.1.0/poolin/app.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import streamlit as st
+
+from poolin.core import (
+    build_output_zip_bytes,
+    load_embedding_zip_bytes,
+    pool_embedding_records,
+    supported_pooling_methods,
+)
+
+st.set_page_config(
+    page_title="poolin",
+    page_icon="🧩",
+    layout="wide",
+)
+
+st.title("poolin")
+st.caption(
+    "Pool existing embedding vectors from an embedding zip into grouped higher-level vectors with a local UI."
+)
+
+with st.sidebar:
+    st.header("Settings")
+    method_options = supported_pooling_methods()
+    labels = [f"{name} — {desc}" for name, desc in method_options]
+    selected_label = st.selectbox("Pooling method", labels, index=0)
+    pooling_method = method_options[labels.index(selected_label)][0]
+
+    normalize_output = st.checkbox("Normalize pooled embeddings", value=True)
+    formats = st.multiselect(
+        "Output formats inside export zip",
+        options=["jsonl", "csv", "npz"],
+        default=["jsonl", "npz"],
+        help="jsonl = readable records, csv = spreadsheet-friendly, npz = direct NumPy arrays.",
+    )
+
+uploaded = st.file_uploader("Drag the embeddings zip here", type=["zip"])
+
+if uploaded is None:
+    st.info(
+        "Upload a zip produced by your embedding step. It should contain an embeddings .npz or .jsonl payload."
+    )
+    st.stop()
+
+if not formats:
+    st.warning("Select at least one export format.")
+    st.stop()
+
+with st.expander("What this step actually does", expanded=False):
+    st.write(
+        "This app performs post-embedding vector pooling across related chunk embeddings. "
+        "It does not replace the token-pooling layer that originally produced each sentence embedding inside the embedding model."
+    )
+
+if st.button("Generate pooled vectors", type="primary"):
+    with st.spinner("Reading embedding zip and pooling vectors..."):
+        try:
+            records, input_summary = load_embedding_zip_bytes(uploaded.read())
+            manifest, items, summary = pool_embedding_records(
+                records,
+                pooling_method=pooling_method,
+                normalize_output=normalize_output,
+            )
+            export_bytes = build_output_zip_bytes(
+                manifest,
+                items,
+                summary,
+                input_zip_name=uploaded.name,
+                formats=formats,
+            )
+        except Exception as exc:
+            st.error(str(exc))
+            st.stop()
+
+    col1, col2 = st.columns([1, 2])
+    with col1:
+        st.metric("Input embeddings", summary["input_embeddings"])
+        st.metric("Pooled outputs", summary["pooled_outputs"])
+        st.metric("Embedding dimension", summary["output_embedding_dim"])
+        st.metric("Pooling method", summary["pooling_method"])
+
+    with col2:
+        st.subheader("Input model")
+        st.write(input_summary.get("model_name", summary.get("source_model_name", "unknown")))
+        st.subheader("Preview")
+        st.dataframe(manifest[:10], use_container_width=True)
+
+    st.download_button(
+        "Download poolin zip",
+        data=export_bytes,
+        file_name=f"{uploaded.name.rsplit('.', 1)[0]}_pooled.zip",
+        mime="application/zip",
+    )

poolin-0.1.0/poolin/cli.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+import sys
+
+
+def main() -> None:
+    try:
+        from streamlit.web import cli as stcli
+    except Exception as exc:  # pragma: no cover
+        raise SystemExit(
+            "Streamlit is required to launch the UI. Install package dependencies first."
+        ) from exc
+
+    app_path = Path(__file__).with_name("app.py")
+    sys.argv = ["streamlit", "run", str(app_path)]
+    raise SystemExit(stcli.main())

poolin-0.1.0/poolin/core.py

@@ -0,0 +1,406 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from io import BytesIO, StringIO
+import csv
+import json
+from pathlib import Path
+import re
+from typing import Iterable
+import zipfile
+
+import numpy as np
+
+POOLING_METHODS: list[tuple[str, str]] = [
+    ("auto", "Auto (recommended)"),
+    ("mean", "Mean pooling across chunk embeddings"),
+    ("max", "Elementwise max pooling"),
+    ("weighted_char_mean", "Weighted mean using character counts"),
+    ("weighted_word_mean", "Weighted mean using word counts"),
+    ("mean_sqrt_len", "Mean scaled by sqrt(number of chunks)"),
+]
+
+SUMMARY_FILES = {"pooling_summary.json", "embedding_summary.json"}
+JSONL_SUFFIX = ".jsonl"
+NPZ_SUFFIX = ".npz"
+
+
+@dataclass
+class EmbeddedRecord:
+    item_id: str
+    source_file: str
+    text: str
+    char_count: int
+    word_count: int
+    embedding_dim: int
+    model_name: str
+    embedding: np.ndarray
+
+
+@dataclass
+class PooledRecord:
+    pooled_id: str
+    group_key: str
+    source_files: list[str]
+    member_count: int
+    total_characters: int
+    total_words: int
+    embedding_dim: int
+    source_model_name: str
+    pooling_method: str
+    embedding: np.ndarray
+
+
+class PoolingError(RuntimeError):
+    pass
+
+
+def supported_pooling_methods() -> list[tuple[str, str]]:
+    return POOLING_METHODS.copy()
+
+
+def infer_default_pooling(model_name: str, normalized: bool | None = None) -> str:
+    """Post-embedding pooling heuristic.
+
+    This is not token pooling inside the original embedding model. It aggregates
+    already-produced embedding vectors into higher-level grouped vectors.
+    """
+    _ = model_name.lower()
+    if normalized is True:
+        return "mean"
+    return "weighted_char_mean"
+
+
+def _safe_stem(filename: str) -> str:
+    return re.sub(r"[^A-Za-z0-9._-]+", "_", Path(filename).stem).strip("_") or "pooling_output"
+
+
+def _group_key_from_filename(source_file: str) -> str:
+    stem = Path(source_file).stem
+    patterns = [
+        r"^(?P<base>.+?)_rcs_\d+$",
+        r"^(?P<base>.+?)_chunk_\d+$",
+        r"^(?P<base>.+?)_part_\d+$",
+        r"^(?P<base>.+?)_split_\d+$",
+    ]
+    for pat in patterns:
+        m = re.match(pat, stem)
+        if m:
+            return m.group("base")
+    return stem
+
+
+def _load_summary_from_zip(zf: zipfile.ZipFile) -> dict:
+    for name in zf.namelist():
+        if Path(name).name in SUMMARY_FILES:
+            return json.loads(zf.read(name).decode("utf-8"))
+    return {}
+
+
+def _load_records_from_npz(npz_bytes: bytes, summary: dict) -> list[EmbeddedRecord]:
+    data = np.load(BytesIO(npz_bytes), allow_pickle=True)
+    if not {"ids", "source_files", "texts", "embeddings"}.issubset(set(data.files)):
+        raise PoolingError("The NPZ file does not contain the expected embedding arrays.")
+
+    ids = data["ids"]
+    source_files = data["source_files"]
+    texts = data["texts"]
+    embeddings = data["embeddings"]
+
+    model_name = str(summary.get("model_name", "unknown"))
+    records: list[EmbeddedRecord] = []
+    for i in range(len(ids)):
+        text = str(texts[i])
+        records.append(
+            EmbeddedRecord(
+                item_id=str(ids[i]),
+                source_file=str(source_files[i]),
+                text=text,
+                char_count=len(text),
+                word_count=len(text.split()),
+                embedding_dim=int(embeddings.shape[1]),
+                model_name=model_name,
+                embedding=np.asarray(embeddings[i], dtype=np.float32),
+            )
+        )
+    return records
+
+
+def _load_records_from_jsonl(jsonl_text: str) -> list[EmbeddedRecord]:
+    records: list[EmbeddedRecord] = []
+    for line in jsonl_text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        obj = json.loads(line)
+        records.append(
+            EmbeddedRecord(
+                item_id=str(obj.get("id", "")),
+                source_file=str(obj.get("source_file", "")),
+                text=str(obj.get("text", "")),
+                char_count=int(obj.get("char_count", 0)),
+                word_count=int(obj.get("word_count", 0)),
+                embedding_dim=int(obj.get("embedding_dim", 0)),
+                model_name=str(obj.get("model_name", "unknown")),
+                embedding=np.asarray(obj.get("embedding", []), dtype=np.float32),
+            )
+        )
+    return records
+
+
+def load_embedding_zip_bytes(zip_bytes: bytes) -> tuple[list[EmbeddedRecord], dict]:
+    with zipfile.ZipFile(BytesIO(zip_bytes), "r") as zf:
+        summary = _load_summary_from_zip(zf)
+
+        npz_name = next((name for name in zf.namelist() if name.lower().endswith(NPZ_SUFFIX)), None)
+        if npz_name:
+            records = _load_records_from_npz(zf.read(npz_name), summary)
+            if records:
+                return records, summary
+
+        jsonl_name = next((name for name in zf.namelist() if name.lower().endswith(JSONL_SUFFIX)), None)
+        if jsonl_name:
+            records = _load_records_from_jsonl(zf.read(jsonl_name).decode("utf-8"))
+            if records:
+                return records, summary
+
+    raise PoolingError(
+        "Could not find a supported embedding payload inside the zip. Expected an embeddings .npz or .jsonl file."
+    )
+
+
+def _l2_normalize(vec: np.ndarray) -> np.ndarray:
+    norm = float(np.linalg.norm(vec))
+    if norm <= 0:
+        return vec
+    return vec / norm
+
+
+def _pool_vectors(vectors: np.ndarray, weights: np.ndarray | None, method: str) -> np.ndarray:
+    if vectors.ndim != 2:
+        raise PoolingError("Expected a 2D embedding array for pooling.")
+    if len(vectors) == 1:
+        return vectors[0].copy()
+
+    if method == "mean":
+        return vectors.mean(axis=0)
+    if method == "max":
+        return vectors.max(axis=0)
+    if method == "weighted_char_mean" or method == "weighted_word_mean":
+        if weights is None:
+            return vectors.mean(axis=0)
+        weights = np.asarray(weights, dtype=np.float32)
+        if weights.sum() <= 0:
+            return vectors.mean(axis=0)
+        weights = weights / weights.sum()
+        return np.average(vectors, axis=0, weights=weights)
+    if method == "mean_sqrt_len":
+        return vectors.mean(axis=0) / np.sqrt(float(len(vectors)))
+    raise PoolingError(f"Unsupported pooling method: {method}")
+
+
+def pool_embedding_records(
+    records: list[EmbeddedRecord],
+    pooling_method: str = "auto",
+    normalize_output: bool = True,
+) -> tuple[list[dict], list[PooledRecord], dict]:
+    if not records:
+        raise PoolingError("No embedding records were found in the input zip.")
+
+    source_model_name = records[0].model_name
+    input_dim = records[0].embedding_dim
+    inferred_normalized = None
+    if records[0].embedding.size:
+        inferred_normalized = bool(abs(float(np.linalg.norm(records[0].embedding)) - 1.0) < 0.05)
+
+    resolved_method = pooling_method
+    if pooling_method == "auto":
+        resolved_method = infer_default_pooling(source_model_name, inferred_normalized)
+
+    grouped: dict[str, list[EmbeddedRecord]] = {}
+    for rec in records:
+        key = _group_key_from_filename(rec.source_file)
+        grouped.setdefault(key, []).append(rec)
+
+    manifest: list[dict] = []
+    pooled_records: list[PooledRecord] = []
+    for idx, key in enumerate(sorted(grouped.keys()), start=1):
+        members = grouped[key]
+        vectors = np.stack([m.embedding for m in members]).astype(np.float32)
+        weights = None
+        if resolved_method == "weighted_char_mean":
+            weights = np.asarray([max(m.char_count, 1) for m in members], dtype=np.float32)
+        elif resolved_method == "weighted_word_mean":
+            weights = np.asarray([max(m.word_count, 1) for m in members], dtype=np.float32)
+
+        pooled = _pool_vectors(vectors, weights, resolved_method)
+        if normalize_output:
+            pooled = _l2_normalize(pooled)
+
+        pooled_id = f"pool_{idx:03d}"
+        total_chars = sum(m.char_count for m in members)
+        total_words = sum(m.word_count for m in members)
+        source_files = [m.source_file for m in members]
+
+        pooled_rec = PooledRecord(
+            pooled_id=pooled_id,
+            group_key=key,
+            source_files=source_files,
+            member_count=len(members),
+            total_characters=total_chars,
+            total_words=total_words,
+            embedding_dim=int(pooled.shape[0]),
+            source_model_name=source_model_name,
+            pooling_method=resolved_method,
+            embedding=pooled.astype(np.float32),
+        )
+        pooled_records.append(pooled_rec)
+        manifest.append(
+            {
+                "pooled_id": pooled_id,
+                "group_key": key,
+                "member_count": len(members),
+                "total_characters": total_chars,
+                "total_words": total_words,
+                "embedding_dim": int(pooled.shape[0]),
+                "source_model_name": source_model_name,
+                "pooling_method": resolved_method,
+            }
+        )
+
+    summary = {
+        "input_embeddings": len(records),
+        "pooled_outputs": len(pooled_records),
+        "input_embedding_dim": input_dim,
+        "output_embedding_dim": input_dim,
+        "source_model_name": source_model_name,
+        "pooling_method": resolved_method,
+        "normalize_output": normalize_output,
+        "grouping_rule": "filename auto-grouping by removing trailing _rcs_N / _chunk_N / _part_N / _split_N",
+    }
+    return manifest, pooled_records, summary
+
+
+def _manifest_csv_bytes(manifest: list[dict]) -> bytes:
+    sio = StringIO()
+    fieldnames = [
+        "pooled_id",
+        "group_key",
+        "member_count",
+        "total_characters",
+        "total_words",
+        "embedding_dim",
+        "source_model_name",
+        "pooling_method",
+    ]
+    writer = csv.DictWriter(sio, fieldnames=fieldnames)
+    writer.writeheader()
+    for row in manifest:
+        writer.writerow(row)
+    return sio.getvalue().encode("utf-8")
+
+
+def _pooled_jsonl_bytes(items: list[PooledRecord]) -> bytes:
+    lines = []
+    for item in items:
+        lines.append(
+            json.dumps(
+                {
+                    "pooled_id": item.pooled_id,
+                    "group_key": item.group_key,
+                    "source_files": item.source_files,
+                    "member_count": item.member_count,
+                    "total_characters": item.total_characters,
+                    "total_words": item.total_words,
+                    "embedding_dim": item.embedding_dim,
+                    "source_model_name": item.source_model_name,
+                    "pooling_method": item.pooling_method,
+                    "embedding": item.embedding.tolist(),
+                },
+                ensure_ascii=False,
+            )
+        )
+    return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+def _pooled_csv_bytes(items: list[PooledRecord]) -> bytes:
+    sio = StringIO()
+    writer = csv.writer(sio)
+    writer.writerow(
+        [
+            "pooled_id",
+            "group_key",
+            "member_count",
+            "total_characters",
+            "total_words",
+            "embedding_dim",
+            "source_model_name",
+            "pooling_method",
+            "source_files",
+            "embedding",
+        ]
+    )
+    for item in items:
+        writer.writerow(
+            [
+                item.pooled_id,
+                item.group_key,
+                item.member_count,
+                item.total_characters,
+                item.total_words,
+                item.embedding_dim,
+                item.source_model_name,
+                item.pooling_method,
+                " | ".join(item.source_files),
+                json.dumps(item.embedding.tolist()),
+            ]
+        )
+    return sio.getvalue().encode("utf-8")
+
+
+def _pooled_npz_bytes(items: list[PooledRecord]) -> bytes:
+    ids = np.asarray([item.pooled_id for item in items], dtype=object)
+    group_keys = np.asarray([item.group_key for item in items], dtype=object)
+    source_files = np.asarray([" | ".join(item.source_files) for item in items], dtype=object)
+    embeddings = np.stack([item.embedding for item in items]).astype(np.float32)
+    member_counts = np.asarray([item.member_count for item in items], dtype=np.int32)
+
+    buffer = BytesIO()
+    np.savez_compressed(
+        buffer,
+        ids=ids,
+        group_keys=group_keys,
+        source_files=source_files,
+        member_counts=member_counts,
+        embeddings=embeddings,
+    )
+    return buffer.getvalue()
+
+
+def _summary_json_bytes(summary: dict) -> bytes:
+    return json.dumps(summary, ensure_ascii=False, indent=2).encode("utf-8")
+
+
+def build_output_zip_bytes(
+    manifest: list[dict],
+    items: list[PooledRecord],
+    summary: dict,
+    input_zip_name: str,
+    formats: list[str] | tuple[str, ...],
+) -> bytes:
+    selected = set(formats)
+    base_name = _safe_stem(input_zip_name)
+
+    buffer = BytesIO()
+    with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr("pooling_summary.json", _summary_json_bytes(summary))
+        zf.writestr("pooling_manifest.csv", _manifest_csv_bytes(manifest))
+
+        if "jsonl" in selected:
+            zf.writestr(f"{base_name}_pooled_embeddings.jsonl", _pooled_jsonl_bytes(items))
+        if "csv" in selected:
+            zf.writestr(f"{base_name}_pooled_embeddings.csv", _pooled_csv_bytes(items))
+        if "npz" in selected:
+            zf.writestr(f"{base_name}_pooled_embeddings.npz", _pooled_npz_bytes(items))
+
+    return buffer.getvalue()

poolin-0.1.0/poolin.egg-info/PKG-INFO

@@ -0,0 +1,72 @@
+Metadata-Version: 2.4
+Name: poolin
+Version: 0.1.0
+Summary: A local UI package for pooling existing embedding zips into grouped vectors
+Author: Wenxi Wang
+License-Expression: MIT
+Project-URL: Homepage, https://example.com/poolin
+Project-URL: Repository, https://example.com/poolin
+Keywords: pooling,embedding,rag,vectors,streamlit,ui
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: streamlit>=1.32
+Requires-Dist: numpy>=1.24
+Dynamic: license-file
+
+# poolin
+
+A local UI package for pooling existing embedding vectors from an embedding zip into grouped higher-level vectors.
+
+## Important note
+
+Standard sentence-transformer style pooling usually happens **inside the embedding model** when token embeddings are converted into one sentence embedding. This package does **post-embedding vector pooling** over already-created chunk embeddings.
+
+## What it does
+
+- launches with the `poolin` command
+- reads an embedding zip such as `RAG_chunks_recursive_chunks_embeddings.zip`
+- auto-groups related chunk embeddings by filename pattern like `RAG_chunk_001_rcs_001.md -> RAG_chunk_001`
+- pools vectors with one of these methods:
+  - `auto`
+  - `mean`
+  - `max`
+  - `weighted_char_mean`
+  - `weighted_word_mean`
+  - `mean_sqrt_len`
+- exports a zip with:
+  - `pooling_summary.json`
+  - `pooling_manifest.csv`
+  - `*_pooled_embeddings.jsonl` (optional)
+  - `*_pooled_embeddings.csv` (optional)
+  - `*_pooled_embeddings.npz` (optional)
+
+## Install
+
+```bash
+pip install poolin
+```
+
+## Run
+
+```bash
+poolin
+```
+
+## Suggested input
+
+Use a zip produced by your embedding step, containing an embeddings `.npz` or `.jsonl` payload plus the summary file.
+
+## Ownership note
+
+The package metadata and copyright notice are set to Wenxi Wang. You should still verify PyPI package-name availability, trademark questions, and any legal or patent issues yourself before publishing.

poolin-0.1.0/poolin.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+README.md
+pyproject.toml
+poolin/__init__.py
+poolin/app.py
+poolin/cli.py
+poolin/core.py
+poolin.egg-info/PKG-INFO
+poolin.egg-info/SOURCES.txt
+poolin.egg-info/dependency_links.txt
+poolin.egg-info/entry_points.txt
+poolin.egg-info/requires.txt
+poolin.egg-info/top_level.txt

poolin-0.1.0/poolin.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

poolin-0.1.0/poolin.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+poolin = poolin.cli:main

poolin-0.1.0/poolin.egg-info/requires.txt

@@ -0,0 +1,2 @@
+streamlit>=1.32
+numpy>=1.24

poolin-0.1.0/poolin.egg-info/top_level.txt

@@ -0,0 +1 @@
+poolin

poolin-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "poolin"
+version = "0.1.0"
+description = "A local UI package for pooling existing embedding zips into grouped vectors"
+authors = [{name="Wenxi Wang"}]
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+dependencies = [
+    "streamlit>=1.32",
+    "numpy>=1.24",
+]
+keywords = ["pooling", "embedding", "rag", "vectors", "streamlit", "ui"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://example.com/poolin"
+Repository = "https://example.com/poolin"
+
+[project.scripts]
+poolin = "poolin.cli:main"

poolin-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+