e5-large-v2-local 1.0.0__tar.gz

e5_large_v2_local-1.0.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: e5-large-v2-local
+Version: 1.0.0
+Summary: Local E5-large-v2 embedding model with automatic Internet Archive download
+Author: Your Name
+Project-URL: Homepage, https://archive.org/details/e5-large-v2
+Project-URL: Model, https://archive.org/details/e5-large-v2
+Project-URL: Download, https://archive.org/download/e5-large-v2/e5-large-v2.zip
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: sentence-transformers>=3.0
+Requires-Dist: requests>=2.31
+
+# E5-large-v2 Local
+
+A lightweight Python package for using the E5-large-v2 embedding model locally.
+
+The package itself does not contain the 751 MB model.
+
+On first use, the model is downloaded from Internet Archive and cached locally.
+
+## Installation
+
+```bash
+pip install e5-large-v2-local

e5_large_v2_local-1.0.0/README.md

@@ -0,0 +1,12 @@
+# E5-large-v2 Local
+
+A lightweight Python package for using the E5-large-v2 embedding model locally.
+
+The package itself does not contain the 751 MB model.
+
+On first use, the model is downloaded from Internet Archive and cached locally.
+
+## Installation
+
+```bash
+pip install e5-large-v2-local

e5_large_v2_local-1.0.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "e5-large-v2-local"
+version = "1.0.0"
+description = "Local E5-large-v2 embedding model with automatic Internet Archive download"
+readme = "README.md"
+requires-python = ">=3.10"
+
+dependencies = [
+    "sentence-transformers>=3.0",
+    "requests>=2.31"
+]
+
+authors = [
+    {name = "Your Name"}
+]
+
+[project.urls]
+Homepage = "https://archive.org/details/e5-large-v2"
+Model = "https://archive.org/details/e5-large-v2"
+Download = "https://archive.org/download/e5-large-v2/e5-large-v2.zip"
+
+[tool.setuptools.packages.find]
+where = ["src"]

e5_large_v2_local-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

e5_large_v2_local-1.0.0/src/e5_large_v2/__init__.py

@@ -0,0 +1,3 @@
+from .model import E5Model
+
+__all__ = ["E5Model"]

e5_large_v2_local-1.0.0/src/e5_large_v2/model.py

@@ -0,0 +1,501 @@
+from __future__ import annotations
+
+import hashlib
+import shutil
+import zipfile
+from pathlib import Path
+from typing import Iterable
+
+import requests
+
+
+MODEL_URL = (
+    "https://archive.org/download/"
+    "e5-large-v2/e5-large-v2.zip"
+)
+
+DEFAULT_CACHE_DIR = (
+    Path.home()
+    / ".cache"
+    / "e5-large-v2"
+)
+
+CHUNK_SIZE = 8 * 1024 * 1024
+
+
+class E5Model:
+    """
+    Local E5-large-v2 embedding model.
+
+    The model is downloaded from Internet Archive on first use
+    and cached locally.
+
+    Parameters
+    ----------
+    model_dir:
+        Optional custom directory for the model.
+
+    device:
+        SentenceTransformer device.
+        Examples: "cpu", "mps", "cuda".
+
+    local_files_only:
+        If True, never download the model.
+
+    sha256:
+        Optional SHA256 checksum of the ZIP file.
+    """
+
+    def __init__(
+        self,
+        model_dir: str | Path | None = None,
+        device: str | None = None,
+        local_files_only: bool = False,
+        sha256: str | None = None,
+    ):
+
+        self.model_dir = (
+            Path(model_dir).expanduser()
+            if model_dir
+            else DEFAULT_CACHE_DIR
+        )
+
+        self.device = device
+        self.local_files_only = local_files_only
+        self.sha256 = sha256
+
+        self.model_dir.parent.mkdir(
+            parents=True,
+            exist_ok=True
+        )
+
+        self._ensure_model()
+
+        from sentence_transformers import SentenceTransformer
+
+        kwargs = {}
+
+        if self.device:
+            kwargs["device"] = self.device
+
+        self.model = SentenceTransformer(
+            str(self.model_dir),
+            **kwargs
+        )
+
+    # ---------------------------------------------------------
+    # Model preparation
+    # ---------------------------------------------------------
+
+    def _ensure_model(self) -> None:
+
+        if self._is_model_directory(
+            self.model_dir
+        ):
+            return
+
+        if self.local_files_only:
+            raise FileNotFoundError(
+                "E5-large-v2 model was not found at:\n"
+                f"{self.model_dir}\n\n"
+                "local_files_only=True prevents downloading."
+            )
+
+        zip_path = (
+            self.model_dir.parent
+            / "e5-large-v2.zip"
+        )
+
+        temp_zip = (
+            self.model_dir.parent
+            / "e5-large-v2.zip.part"
+        )
+
+        extract_dir = (
+            self.model_dir.parent
+            / "e5-large-v2-extracted"
+        )
+
+        print()
+        print("E5-large-v2 model not found.")
+        print("Downloading model from Internet Archive.")
+        print()
+
+        try:
+
+            self._download(
+                MODEL_URL,
+                temp_zip
+            )
+
+            print("Download completed.")
+
+            if self.sha256:
+                print("Checking SHA256...")
+
+                actual = self._sha256(
+                    temp_zip
+                )
+
+                if actual.lower() != self.sha256.lower():
+                    raise RuntimeError(
+                        "SHA256 checksum mismatch.\n"
+                        f"Expected: {self.sha256}\n"
+                        f"Actual:   {actual}"
+                    )
+
+                print("SHA256 verified.")
+
+            print("Validating ZIP...")
+
+            if not zipfile.is_zipfile(
+                temp_zip
+            ):
+                raise RuntimeError(
+                    "Downloaded file is not a valid ZIP."
+                )
+
+            if extract_dir.exists():
+                shutil.rmtree(
+                    extract_dir
+                )
+
+            extract_dir.mkdir(
+                parents=True,
+                exist_ok=True
+            )
+
+            print("Extracting model...")
+
+            with zipfile.ZipFile(
+                temp_zip,
+                "r"
+            ) as archive:
+
+                self._safe_extract(
+                    archive,
+                    extract_dir
+                )
+
+            model_root = (
+                self._find_model_root(
+                    extract_dir
+                )
+            )
+
+            if model_root is None:
+                raise RuntimeError(
+                    "Could not locate a valid "
+                    "SentenceTransformer model "
+                    "inside the ZIP archive."
+                )
+
+            if self.model_dir.exists():
+                shutil.rmtree(
+                    self.model_dir
+                )
+
+            print("Installing model into cache...")
+
+            shutil.copytree(
+                model_root,
+                self.model_dir
+            )
+
+            print()
+            print(
+                "E5-large-v2 installed successfully."
+            )
+            print(
+                f"Model location: {self.model_dir}"
+            )
+            print()
+
+        finally:
+
+            temp_zip.unlink(
+                missing_ok=True
+            )
+
+            zip_path.unlink(
+                missing_ok=True
+            )
+
+            if extract_dir.exists():
+                shutil.rmtree(
+                    extract_dir,
+                    ignore_errors=True
+                )
+
+    # ---------------------------------------------------------
+    # Download
+    # ---------------------------------------------------------
+
+    @staticmethod
+    def _download(
+        url: str,
+        destination: Path
+    ) -> None:
+
+        destination.parent.mkdir(
+            parents=True,
+            exist_ok=True
+        )
+
+        with requests.get(
+            url,
+            stream=True,
+            timeout=(30, 120)
+        ) as response:
+
+            response.raise_for_status()
+
+            total = int(
+                response.headers.get(
+                    "content-length",
+                    0
+                )
+            )
+
+            downloaded = 0
+
+            with open(
+                destination,
+                "wb"
+            ) as file:
+
+                for chunk in response.iter_content(
+                    chunk_size=CHUNK_SIZE
+                ):
+
+                    if not chunk:
+                        continue
+
+                    file.write(chunk)
+
+                    downloaded += len(chunk)
+
+                    if total:
+
+                        percent = (
+                            downloaded
+                            / total
+                            * 100
+                        )
+
+                        downloaded_mb = (
+                            downloaded
+                            / 1024
+                            / 1024
+                        )
+
+                        total_mb = (
+                            total
+                            / 1024
+                            / 1024
+                        )
+
+                        print(
+                            f"\r"
+                            f"Downloading: "
+                            f"{percent:6.2f}% "
+                            f"({downloaded_mb:.1f}/"
+                            f"{total_mb:.1f} MB)",
+                            end="",
+                            flush=True
+                        )
+
+        print()
+
+    # ---------------------------------------------------------
+    # SHA256
+    # ---------------------------------------------------------
+
+    @staticmethod
+    def _sha256(
+        path: Path
+    ) -> str:
+
+        digest = hashlib.sha256()
+
+        with open(
+            path,
+            "rb"
+        ) as file:
+
+            while chunk := file.read(
+                CHUNK_SIZE
+            ):
+
+                digest.update(chunk)
+
+        return digest.hexdigest()
+
+    # ---------------------------------------------------------
+    # ZIP security
+    # ---------------------------------------------------------
+
+    @staticmethod
+    def _safe_extract(
+        archive: zipfile.ZipFile,
+        destination: Path
+    ) -> None:
+
+        destination = destination.resolve()
+
+        for member in archive.infolist():
+
+            member_path = (
+                destination
+                / member.filename
+            ).resolve()
+
+            if not str(member_path).startswith(
+                str(destination)
+            ):
+                raise RuntimeError(
+                    "Unsafe ZIP archive detected."
+                )
+
+        archive.extractall(
+            destination
+        )
+
+    # ---------------------------------------------------------
+    # Locate model
+    # ---------------------------------------------------------
+
+    @staticmethod
+    def _is_model_directory(
+        path: Path
+    ) -> bool:
+
+        if not path.is_dir():
+            return False
+
+        config = (
+            path / "config.json"
+        )
+
+        modules = (
+            path / "modules.json"
+        )
+
+        return (
+            config.exists()
+            and modules.exists()
+        )
+
+    @classmethod
+    def _find_model_root(
+        cls,
+        root: Path
+    ) -> Path | None:
+
+        if cls._is_model_directory(
+            root
+        ):
+            return root
+
+        for path in root.rglob("*"):
+
+            if path.is_dir():
+
+                if cls._is_model_directory(
+                    path
+                ):
+                    return path
+
+        return None
+
+    # ---------------------------------------------------------
+    # Encoding
+    # ---------------------------------------------------------
+
+    def encode(
+        self,
+        texts: str | Iterable[str],
+        *,
+        batch_size: int = 32,
+        normalize_embeddings: bool = True,
+        show_progress_bar: bool = False,
+        **kwargs
+    ):
+
+        if isinstance(
+            texts,
+            str
+        ):
+            texts = [texts]
+
+        return self.model.encode(
+            list(texts),
+            batch_size=batch_size,
+            normalize_embeddings=normalize_embeddings,
+            show_progress_bar=show_progress_bar,
+            **kwargs
+        )
+
+    # ---------------------------------------------------------
+    # E5 convenience methods
+    # ---------------------------------------------------------
+
+    def encode_queries(
+        self,
+        queries: str | Iterable[str],
+        **kwargs
+    ):
+
+        if isinstance(
+            queries,
+            str
+        ):
+            queries = [queries]
+
+        queries = [
+            q if q.startswith(
+                "query:"
+            )
+            else f"query: {q}"
+            for q in queries
+        ]
+
+        return self.encode(
+            queries,
+            **kwargs
+        )
+
+    def encode_passages(
+        self,
+        passages: str | Iterable[str],
+        **kwargs
+    ):
+
+        if isinstance(
+            passages,
+            str
+        ):
+            passages = [passages]
+
+        passages = [
+            p if p.startswith(
+                "passage:"
+            )
+            else f"passage: {p}"
+            for p in passages
+        ]
+
+        return self.encode(
+            passages,
+            **kwargs
+        )
+
+    @property
+    def dimension(self) -> int:
+
+        return self.model.get_sentence_embedding_dimension()
+
+    @property
+    def path(self) -> Path:
+
+        return self.model_dir

e5_large_v2_local-1.0.0/src/e5_large_v2_local.egg-info/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.4
+Name: e5-large-v2-local
+Version: 1.0.0
+Summary: Local E5-large-v2 embedding model with automatic Internet Archive download
+Author: Your Name
+Project-URL: Homepage, https://archive.org/details/e5-large-v2
+Project-URL: Model, https://archive.org/details/e5-large-v2
+Project-URL: Download, https://archive.org/download/e5-large-v2/e5-large-v2.zip
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: sentence-transformers>=3.0
+Requires-Dist: requests>=2.31
+
+# E5-large-v2 Local
+
+A lightweight Python package for using the E5-large-v2 embedding model locally.
+
+The package itself does not contain the 751 MB model.
+
+On first use, the model is downloaded from Internet Archive and cached locally.
+
+## Installation
+
+```bash
+pip install e5-large-v2-local

e5_large_v2_local-1.0.0/src/e5_large_v2_local.egg-info/SOURCES.txt

@@ -0,0 +1,9 @@
+README.md
+pyproject.toml
+src/e5_large_v2/__init__.py
+src/e5_large_v2/model.py
+src/e5_large_v2_local.egg-info/PKG-INFO
+src/e5_large_v2_local.egg-info/SOURCES.txt
+src/e5_large_v2_local.egg-info/dependency_links.txt
+src/e5_large_v2_local.egg-info/requires.txt
+src/e5_large_v2_local.egg-info/top_level.txt

e5_large_v2_local-1.0.0/src/e5_large_v2_local.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

e5_large_v2_local-1.0.0/src/e5_large_v2_local.egg-info/requires.txt

@@ -0,0 +1,2 @@
+sentence-transformers>=3.0
+requests>=2.31

e5_large_v2_local-1.0.0/src/e5_large_v2_local.egg-info/top_level.txt

@@ -0,0 +1 @@
+e5_large_v2