embedding-flow 0.1.0__tar.gz → 0.1.2__tar.gz

embedding_flow-0.1.2/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.4
+Name: embedding-flow
+Version: 0.1.2
+Summary: Pipeline to transform text chunks into embeddings and load to Qdrant
+Author: facuvega
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: qdrant-client>=1.7.0
+Requires-Dist: transformers
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Provides-Extra: cpu
+Requires-Dist: torch>=2.0.0; extra == "cpu"
+Provides-Extra: cuda
+Requires-Dist: torch>=2.0.0; extra == "cuda"
+Dynamic: license-file
+
+# embedding-flow
+
+Biblioteca para transformar chunks de texto en embeddings de 768 dimensiones y cargarlos en Qdrant.
+
+## Instalación
+
+```bash
+# Instalación básica (instala torch según tu sistema)
+pip install embedding-flow
+
+# O instalar con torch CPU (recomendado si no tenés GPU)
+pip install embedding-flow torch --index-url https://download.pytorch.org/whl/cpu
+```
+
+## Uso
+
+```python
+from embedding_flow import embedding_flow
+
+# Recibe el path del parquet con chunks y carga embeddings a Qdrant
+embedding_flow("/path/to/chunks.parquet")
+```
+
+## Variables de entorno
+
+```bash
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=embeddings_collection
+VECTOR_SIZE=768
+```
+
+## Flujo
+
+1. Lee chunks desde parquet
+2. Genera embeddings (768 dim) con `all-mpnet-base-v2`
+3. Carga embeddings a Qdrant (Docker local)
+
+## Licencia
+
+MIT
+

embedding_flow-0.1.2/README.md

@@ -0,0 +1,41 @@
+# embedding-flow
+
+Biblioteca para transformar chunks de texto en embeddings de 768 dimensiones y cargarlos en Qdrant.
+
+## Instalación
+
+```bash
+# Instalación básica (instala torch según tu sistema)
+pip install embedding-flow
+
+# O instalar con torch CPU (recomendado si no tenés GPU)
+pip install embedding-flow torch --index-url https://download.pytorch.org/whl/cpu
+```
+
+## Uso
+
+```python
+from embedding_flow import embedding_flow
+
+# Recibe el path del parquet con chunks y carga embeddings a Qdrant
+embedding_flow("/path/to/chunks.parquet")
+```
+
+## Variables de entorno
+
+```bash
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=embeddings_collection
+VECTOR_SIZE=768
+```
+
+## Flujo
+
+1. Lee chunks desde parquet
+2. Genera embeddings (768 dim) con `all-mpnet-base-v2`
+3. Carga embeddings a Qdrant (Docker local)
+
+## Licencia
+
+MIT
+

embedding_flow-0.1.2/embedding_flow/__init__.py

@@ -0,0 +1,4 @@
+from embedding_flow.main import embedding_flow
+
+__all__ = ['embedding_flow']
+

{embedding_flow-0.1.0 → embedding_flow-0.1.2/embedding_flow}/contracts/contracts.py

@@ -11,4 +11,5 @@ class load_data(ABC):
     @abstractmethod
     def load_data(self, url: str) -> bool:
         """Carga datos y retorna True si fue exitoso, False si falló"""
-        pass
+        pass
+

{embedding_flow-0.1.0 → embedding_flow-0.1.2/embedding_flow}/load/load.py

@@ -1,4 +1,4 @@
-from contracts.contracts import load_data
+from embedding_flow.contracts.contracts import load_data
 from qdrant_client import QdrantClient
 from qdrant_client.models import Distance, VectorParams, PointStruct
 import pandas as pd
@@ -98,4 +98,4 @@ class load_embedding(load_data):
             
         except Exception as e:
             logger.error(f"❌ Error al cargar embeddings a Qdrant desde {parquet_path}: {e}", exc_info=True)
-            return False
+            return False

embedding_flow-0.1.2/embedding_flow/main.py

@@ -0,0 +1,22 @@
+from embedding_flow.transform.transform import transform_embedding
+from embedding_flow.load.load import load_embedding
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,              # Nivel mínimo de logs a mostrar
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    filename='appMain.log',             # Opcional: guarda los logs en un archivo
+    filemode='a'                    # 'a' append, 'w' overwrite
+)
+
+
+def embedding_flow(parquet_path: str)-> str | None  :
+    
+    transformer = transform_embedding(parquet_path)
+    of = load_embedding(transformer)
+    if of is None:
+        logging.error("❌ Pipeline failed")
+        return None
+    else:
+        logging.info("✅ Pipeline completed successfully")
+        return of

{embedding_flow-0.1.0 → embedding_flow-0.1.2/embedding_flow}/transform/transform.py

@@ -1,4 +1,4 @@
-from contracts.contracts import transform_data
+from embedding_flow.contracts.contracts import transform_data
 from pathlib import Path
 import pandas as pd
 from sentence_transformers import SentenceTransformer
@@ -39,4 +39,4 @@ class transform_embedding(transform_data):
 
         except Exception as e:
             logger.error(f"❌ Error al transformar en embeddings {parquet_path}: {e}", exc_info=True)
-            return None
+            return None

embedding_flow-0.1.2/embedding_flow.egg-info/PKG-INFO

@@ -0,0 +1,69 @@
+Metadata-Version: 2.4
+Name: embedding-flow
+Version: 0.1.2
+Summary: Pipeline to transform text chunks into embeddings and load to Qdrant
+Author: facuvega
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=12.0.0
+Requires-Dist: sentence-transformers>=2.2.0
+Requires-Dist: qdrant-client>=1.7.0
+Requires-Dist: transformers
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+Provides-Extra: cpu
+Requires-Dist: torch>=2.0.0; extra == "cpu"
+Provides-Extra: cuda
+Requires-Dist: torch>=2.0.0; extra == "cuda"
+Dynamic: license-file
+
+# embedding-flow
+
+Biblioteca para transformar chunks de texto en embeddings de 768 dimensiones y cargarlos en Qdrant.
+
+## Instalación
+
+```bash
+# Instalación básica (instala torch según tu sistema)
+pip install embedding-flow
+
+# O instalar con torch CPU (recomendado si no tenés GPU)
+pip install embedding-flow torch --index-url https://download.pytorch.org/whl/cpu
+```
+
+## Uso
+
+```python
+from embedding_flow import embedding_flow
+
+# Recibe el path del parquet con chunks y carga embeddings a Qdrant
+embedding_flow("/path/to/chunks.parquet")
+```
+
+## Variables de entorno
+
+```bash
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=embeddings_collection
+VECTOR_SIZE=768
+```
+
+## Flujo
+
+1. Lee chunks desde parquet
+2. Genera embeddings (768 dim) con `all-mpnet-base-v2`
+3. Carga embeddings a Qdrant (Docker local)
+
+## Licencia
+
+MIT
+

embedding_flow-0.1.2/embedding_flow.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+LICENSE
+README.md
+pyproject.toml
+setup.py
+embedding_flow/__init__.py
+embedding_flow/main.py
+embedding_flow.egg-info/PKG-INFO
+embedding_flow.egg-info/SOURCES.txt
+embedding_flow.egg-info/dependency_links.txt
+embedding_flow.egg-info/requires.txt
+embedding_flow.egg-info/top_level.txt
+embedding_flow/contracts/__init__.py
+embedding_flow/contracts/contracts.py
+embedding_flow/load/__init__.py
+embedding_flow/load/load.py
+embedding_flow/transform/__init__.py
+embedding_flow/transform/transform.py
+tests/test_load.py
+tests/test_transform.py

{embedding_flow-0.1.0 → embedding_flow-0.1.2}/embedding_flow.egg-info/requires.txt

@@ -1,8 +1,14 @@
 pandas>=2.0.0
 pyarrow>=12.0.0
 sentence-transformers>=2.2.0
-torch>=2.0.0
 qdrant-client>=1.7.0
+transformers
+
+[cpu]
+torch>=2.0.0
+
+[cuda]
+torch>=2.0.0
 
 [dev]
 pytest>=7.0.0

embedding_flow-0.1.2/embedding_flow.egg-info/top_level.txt

@@ -0,0 +1 @@
+embedding_flow

{embedding_flow-0.1.0 → embedding_flow-0.1.2}/pyproject.toml

@@ -4,16 +4,17 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "embedding-flow"
-version = "0.1.0"
+version = "0.1.2"
 description = "Pipeline to transform text chunks into embeddings and load to Qdrant"
+readme = "README.md"
 authors = [{name = "facuvega"}]
 requires-python = ">=3.10"
 dependencies = [
     "pandas>=2.0.0",
     "pyarrow>=12.0.0",
     "sentence-transformers>=2.2.0",
-    "torch>=2.0.0",
     "qdrant-client>=1.7.0",
+    "transformers",
 ]
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -29,8 +30,14 @@ dev = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",
 ]
+cpu = [
+    "torch>=2.0.0",
+]
+cuda = [
+    "torch>=2.0.0",
+]
 
 [tool.setuptools.packages.find]
-include = ["contracts*", "transform*", "load*"]
+include = ["embedding_flow*"]
 exclude = ["tests*", "venv*"]
 

{embedding_flow-0.1.0 → embedding_flow-0.1.2}/tests/test_load.py

@@ -4,10 +4,10 @@ import tempfile
 import numpy as np
 from pathlib import Path
 from unittest.mock import MagicMock, patch
-from load.load import load_embedding
+from embedding_flow.load.load import load_embedding
 
 
-@patch('load.load.QdrantClient')
+@patch('embedding_flow.load.load.QdrantClient')
 def test_load_with_embeddings(mock_qdrant_client):
     """Test que load carga correctamente embeddings de 768 dims"""
     # Mock del cliente
@@ -40,7 +40,7 @@ def test_load_with_embeddings(mock_qdrant_client):
         Path(tmp_path).unlink(missing_ok=True)
 
 
-@patch('load.load.QdrantClient')
+@patch('embedding_flow.load.load.QdrantClient')
 def test_load_without_embeddings(mock_qdrant_client):
     """Test que load falla sin columna 'embedding'"""
     mock_client = MagicMock()

{embedding_flow-0.1.0 → embedding_flow-0.1.2}/tests/test_transform.py

@@ -2,7 +2,7 @@ import pytest
 import pandas as pd
 import tempfile
 from pathlib import Path
-from transform.transform import transform_embedding
+from embedding_flow.transform.transform import transform_embedding
 
 
 def test_transform_creates_embeddings():

embedding_flow-0.1.0/MANIFEST.in

@@ -1,7 +0,0 @@
-include README.md
-include LICENSE
-include requirements.txt
-recursive-include contracts *.py
-recursive-include transform *.py
-recursive-include load *.py
-

embedding_flow-0.1.0/PKG-INFO

@@ -1,22 +0,0 @@
-Metadata-Version: 2.4
-Name: embedding-flow
-Version: 0.1.0
-Summary: Pipeline to transform text chunks into embeddings and load to Qdrant
-Author: facuvega
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-License-File: LICENSE
-Requires-Dist: pandas>=2.0.0
-Requires-Dist: pyarrow>=12.0.0
-Requires-Dist: sentence-transformers>=2.2.0
-Requires-Dist: torch>=2.0.0
-Requires-Dist: qdrant-client>=1.7.0
-Provides-Extra: dev
-Requires-Dist: pytest>=7.0.0; extra == "dev"
-Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
-Dynamic: license-file

embedding_flow-0.1.0/README.md

@@ -1,40 +0,0 @@
-# embedding-flow
-
-Pipeline for transforming text chunks into 768-dimensional embeddings and loading to Qdrant.
-
-## Installation
-
-```bash
-pip install embedding-flow
-```
-
-## Usage
-
-```python
-from transform.transform import transform_embedding
-from load.load import load_embedding
-
-# Transform
-transformer = transform_embedding()
-output_path = transformer.transform_data("chunks.parquet")
-
-# Load to Qdrant
-loader = load_embedding()
-loader.load_data(output_path)
-```
-
-## Environment Variables
-
-```bash
-QDRANT_URL=http://localhost:6333
-QDRANT_COLLECTION=embeddings_collection
-VECTOR_SIZE=768
-```
-
-## Development
-
-```bash
-pip install -e ".[dev]"
-pytest tests/
-```
-

embedding_flow-0.1.0/embedding_flow.egg-info/PKG-INFO

@@ -1,22 +0,0 @@
-Metadata-Version: 2.4
-Name: embedding-flow
-Version: 0.1.0
-Summary: Pipeline to transform text chunks into embeddings and load to Qdrant
-Author: facuvega
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-License-File: LICENSE
-Requires-Dist: pandas>=2.0.0
-Requires-Dist: pyarrow>=12.0.0
-Requires-Dist: sentence-transformers>=2.2.0
-Requires-Dist: torch>=2.0.0
-Requires-Dist: qdrant-client>=1.7.0
-Provides-Extra: dev
-Requires-Dist: pytest>=7.0.0; extra == "dev"
-Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
-Dynamic: license-file

embedding_flow-0.1.0/embedding_flow.egg-info/SOURCES.txt

@@ -1,19 +0,0 @@
-LICENSE
-MANIFEST.in
-README.md
-pyproject.toml
-requirements.txt
-setup.py
-contracts/__init__.py
-contracts/contracts.py
-embedding_flow.egg-info/PKG-INFO
-embedding_flow.egg-info/SOURCES.txt
-embedding_flow.egg-info/dependency_links.txt
-embedding_flow.egg-info/requires.txt
-embedding_flow.egg-info/top_level.txt
-load/__init__.py
-load/load.py
-tests/test_load.py
-tests/test_transform.py
-transform/__init__.py
-transform/transform.py

embedding_flow-0.1.0/embedding_flow.egg-info/top_level.txt

@@ -1,3 +0,0 @@
-contracts
-load
-transform

embedding_flow-0.1.0/requirements.txt

@@ -1,17 +0,0 @@
-# Core dependencies
-pandas>=2.0.0
-pyarrow>=12.0.0
-
-# ML & Embeddings
-sentence-transformers>=2.2.0
-torch>=2.0.0
-
-# Vector Database
-qdrant-client>=1.7.0
-
-# Airflow (si se necesita localmente, sino está en el servidor)
-# apache-airflow>=2.7.0
-
-# Utilities
-python-dotenv>=1.0.0
-