vectoriz 0.0.3__tar.gz

vectoriz-0.0.3/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: vectoriz
+Version: 0.0.3
+Summary: Python library for creating vectorized data from text or files.
+Home-page: https://github.com/PedroHenriqueDevBR/vectoriz
+Author: PedroHenriqueDevBR
+Author-email: pedro.henrique.particular@gmail.com
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: faiss-cpu==1.10.0
+Requires-Dist: numpy==2.2.4
+Requires-Dist: sentence-transformers==4.0.2
+Requires-Dist: python-docx==1.1.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# RAG-vector-creator
+
+## Overview
+This project implements a RAG (Retrieval-Augmented Generation) system for creating and managing vector embeddings from documents using FAISS and NumPy libraries. It efficiently transforms text data into high-dimensional vector representations that enable semantic search capabilities, similarity matching, and context-aware document retrieval for enhanced question answering applications.
+
+## Features
+
+- Document ingestion and preprocessing
+- Vector embedding generation using state-of-the-art models
+- Efficient storage and retrieval of embeddings
+- Integration with LLM-based generation systems
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+python app.py
+```
+
+## Build lib
+
+To build the lib run the commands:
+
+```
+python setup.py sdist bdist_wheel
+```
+
+To test the install run:
+```
+pip install .
+```
+
+## License
+
+MIT

vectoriz-0.0.3/README.md

@@ -0,0 +1,35 @@
+# RAG-vector-creator
+
+## Overview
+This project implements a RAG (Retrieval-Augmented Generation) system for creating and managing vector embeddings from documents using FAISS and NumPy libraries. It efficiently transforms text data into high-dimensional vector representations that enable semantic search capabilities, similarity matching, and context-aware document retrieval for enhanced question answering applications.
+
+## Features
+
+- Document ingestion and preprocessing
+- Vector embedding generation using state-of-the-art models
+- Efficient storage and retrieval of embeddings
+- Integration with LLM-based generation systems
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+python app.py
+```
+
+## Build lib
+
+To build the lib run the commands:
+
+```
+python setup.py sdist bdist_wheel
+```
+
+To test the install run:
+```
+pip install .
+```
+
+## License
+
+MIT

vectoriz-0.0.3/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"

vectoriz-0.0.3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

vectoriz-0.0.3/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="vectoriz",
+    version="0.0.3",
+    author="PedroHenriqueDevBR",
+    author_email="pedro.henrique.particular@gmail.com",
+    description="Python library for creating vectorized data from text or files.",
+    long_description = open("README.md", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    url="https://github.com/PedroHenriqueDevBR/vectoriz",
+    packages=find_packages(),
+    classifiers=[
+        "Programming Language :: Python :: 3.12",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.12",
+    install_requires=[
+        "faiss-cpu==1.10.0",
+        "numpy==2.2.4",
+        "sentence-transformers==4.0.2",
+        "python-docx==1.1.2"
+    ],
+)

vectoriz-0.0.3/tests/test_files.py

@@ -0,0 +1,40 @@
+import pytest
+import numpy as np
+from unittest.mock import patch
+from vectoriz.files import FileArgument
+
+
+def test_add_data_appends_to_lists():
+    file_arg = FileArgument([], [], [])
+    filename = "test.txt"
+    text = "Test content"
+    
+    with patch.object(FileArgument, '_create_embedding', return_value=np.array([0.1, 0.2, 0.3])):
+        file_arg.add_data(filename, text)
+        
+        assert file_arg.chunk_names == [filename]
+        assert file_arg.text_list == [text]
+        assert len(file_arg.embeddings) == 1
+        np.testing.assert_array_equal(file_arg.embeddings[0], np.array([0.1, 0.2, 0.3]))
+
+def test_add_data_multiple_entries():
+    file_arg = FileArgument(["existing.txt"], ["existing content"], [np.array([0.5, 0.5, 0.5])])
+    filename = "new.txt"
+    text = "New content"
+    
+    with patch.object(FileArgument, '_create_embedding', return_value=np.array([0.7, 0.8, 0.9])):
+        file_arg.add_data(filename, text)
+        assert file_arg.chunk_names == ["existing.txt", "new.txt"]
+        assert file_arg.text_list == ["existing content", "New content"]
+        assert len(file_arg.embeddings) == 2
+        np.testing.assert_array_equal(file_arg.embeddings[1], np.array([0.7, 0.8, 0.9]))
+
+def test_add_data_calls_create_embedding():
+    file_arg = FileArgument([], [], [])
+    filename = "test.txt"
+    text = "Test content"
+    
+    with patch.object(FileArgument, '_create_embedding') as mock_create_embedding:
+        mock_create_embedding.return_value = np.array([0.1, 0.2, 0.3])
+        file_arg.add_data(filename, text)
+        mock_create_embedding.assert_called_once_with(text)

vectoriz-0.0.3/vectoriz/__init__.py

@@ -0,0 +1,3 @@
+# from files import FileArgument, FilesFeature
+# from token_transformer import TokenData, TokenTransformer
+# from vector_db import VectorDBClient,  VectorDB

vectoriz-0.0.3/vectoriz/files.py

@@ -0,0 +1,213 @@
+import os
+import docx
+import numpy as np
+from typing import Optional
+from vectoriz.token_transformer import TokenTransformer
+
+class FileArgument:
+    def __init__(
+        self,
+        chunk_names: list[str],
+        text_list: list[str],
+        embeddings: list[float],
+        ndarray_data: Optional[np.ndarray] = None,
+    ) -> None:
+        """
+        Initializes the FileProcessor instance with file data and embeddings.
+
+        This constructor sets up an instance with chunk_names, their text content, and associated embeddings.
+        It also initializes a TokenTransformer instance for potential token transformations.
+
+        Parameters
+        ----------
+        chunk_names : list[str]
+            List of chunk_names corresponding to processed files
+        text_list : list[str]
+            List of text content extracted from the files
+        embeddings : list[float]
+            List of embeddings (vector representations) of the text content
+        ndarray_data : Optional[np.ndarray], default=None
+            NumPy array representation of the embeddings for efficient vector operations
+
+        Returns
+        -------
+        None
+        """
+        self.chunk_names: list[str] = chunk_names
+        self.text_list: list[str] = text_list
+        self.embeddings: list[float] = embeddings
+        self.ndarray_data: np.ndarray = ndarray_data
+
+    def add_data(self, filename: str, text: str) -> None:
+        """
+        Adds text data to the vectorizer along with its filename and creates the corresponding embedding.
+        This method appends the provided filename and text to their respective lists in the object,
+        and also creates and stores the embedding vector for the text.
+        Args:
+            filename (str): The name of the file or identifier for the text data
+            text (str): The text content to be added and embedded
+        Returns:
+            None: This method doesn't return anything, it updates the internal state of the object
+        """
+
+        self.chunk_names.append(filename)
+        self.text_list.append(text)
+        self.embeddings.append(self._create_embedding(text))
+
+    def _create_embedding(self, text: str) -> np.ndarray:
+        """
+        Creates an embedding vector for the given text using the transformer model.
+        This method transforms the input text into a numerical vector representation
+        that captures semantic meaning, which can be used for similarity comparisons
+        or as input to machine learning models.
+        Args:
+            text (str): The text to be embedded.
+        Returns:
+            np.ndarray: A numpy array containing the embedding vector for the input text.
+        """
+        instance = TokenTransformer()
+        return instance.text_to_embeddings([text])[0]
+
+
+class FilesFeature:
+
+    def _extract_txt_content(self, directory: str, file: str) -> Optional[str]:
+        """
+        Extract content from a text file and add it to the response data.
+
+        This method opens a text file in read mode with UTF-8 encoding, reads its content,
+        and adds the file name and its content to the response data.
+
+        Parameters:
+        ----------
+        directory : str
+            The directory path where the file is located.
+        file : str
+            The name of the text file to read.
+
+        Returns:
+        -------
+        None
+            This method doesn't return any value but updates the internal response data.
+
+        Raises:
+        ------
+        FileNotFoundError
+            If the specified file does not exist.
+        UnicodeDecodeError
+            If the file cannot be decoded using UTF-8 encoding.
+        """
+        with open(os.path.join(directory, file), "r", encoding="utf-8") as fl:
+            text = fl.read()
+            return text
+
+    def _extract_docx_content(self, directory: str, file: str) -> Optional[str]:
+        """
+        Extracts text content from a Microsoft Word document.
+        This method opens a Word document, reads all paragraphs, and joins non-empty
+        paragraphs into a single text string. The extracted content is then stored
+        using the add_response_data method.
+        Args:
+            directory (str): The directory path where the Word file is located
+            file (str): The filename of the Word document to process
+        Returns:
+            Optional[str]: The extracted text content or None if no content is found.
+        Note:
+            Empty paragraphs (those that contain only whitespace) are skipped.
+            The python-docx library is required for this method to work.
+        """
+        file_path = os.path.join(directory, file)
+        doc = docx.Document(file_path)
+        full_text = []
+
+        for paragraph in doc.paragraphs:
+            content = paragraph.text.strip()
+            if len(content) == 0:
+                continue
+            full_text.append(paragraph.text)
+        return "\n".join(full_text)
+
+    def load_txt_files_from_directory(self, directory: str) -> FileArgument:
+        """
+        Load all text files from the specified directory and extract their content.
+        This method scans the specified directory for files with the '.txt' extension
+        and processes each of them using the extract_txt_content method.
+        Parameters:
+        ----------
+        directory : str
+            Path to the directory containing text files to be loaded.
+        Returns:
+        -------
+        None
+            This method does not return any value. It updates the internal state
+            by processing text files found in the directory.
+        """
+        argument: FileArgument = FileArgument([], [], [])
+        for file in os.listdir(directory):
+            if not file.endswith(".txt"):
+                continue
+            
+            text = self._extract_txt_content(directory, file)
+            if text is None:
+                continue
+            
+            argument.add_data(file, text)
+        return argument
+
+    def load_docx_files_from_directory(self, directory: str) -> FileArgument:
+        """
+        Load all Word (.docx) files from the specified directory and extract their content.
+
+        This method iterates through all files in the given directory, identifies those
+        with a .docx extension, and processes them using the extract_word_content method.
+
+        Args:
+            directory (str): Path to the directory containing Word files to be processed
+
+        Returns:
+            None
+
+        Examples:
+            >>> processor = DocumentProcessor()
+            >>> processor.load_word_files("/path/to/documents")
+        """
+        argument: FileArgument = FileArgument([], [], [])
+        for file in os.listdir(directory):
+            if not file.endswith(".docx"):
+                continue
+            
+            text = self._extract_docx_content(directory, file)
+            if text is None:
+                continue
+            
+            argument.add_data(file, text)
+        return argument
+
+    def load_all_files_from_directory(self, directory: str) -> FileArgument:
+        """
+        Load all supported files (.txt and .docx) from the specified directory and its subdirectories.
+
+        This method walks through the directory tree, processing all text and Word files
+        by adding them to the response data.
+
+        Args:
+            directory (str): Path to the directory containing files to be loaded
+
+        Returns:
+            None
+        """
+        argument: FileArgument = FileArgument([], [], [])
+        for root, _, files in os.walk(directory):
+            for file in files:
+                if file.endswith(".txt"):
+                    text = self._extract_txt_content(root, file)
+                    if text is not None:
+                        argument.add_data(file, text)
+                elif file.endswith(".docx"):
+                    try:
+                        text = self._extract_docx_content(root, file)
+                        if text is not None:
+                            argument.add_data(file, text)
+                    except Exception as e:
+                        print(f"Error processing {file}: {str(e)}")
+        return argument

vectoriz-0.0.3/vectoriz/token_transformer.py

@@ -0,0 +1,164 @@
+import faiss
+import numpy as np
+from typing import Self
+from sentence_transformers import SentenceTransformer
+
+
+class TokenData:
+    """
+    A class that holds text data along with their vector representations and indexing.
+    This class is designed to store and manage tokenized texts, their corresponding
+    embeddings, and a FAISS index for efficient similarity search.
+    Attributes:
+        texts (list[str]): List of text strings that have been tokenized.
+        index (faiss.IndexFlatL2): A FAISS index using L2 (Euclidean) distance metric
+                                  for similarity search.
+        embeddings (np.ndarray, optional): Matrix of vector embeddings corresponding
+                                          to the texts. Default is None.
+    """
+
+    def __init__(
+        self,
+        texts: list[str],
+        index: faiss.IndexFlatL2,
+        embeddings: np.ndarray = None,
+    ):
+        self.texts = texts
+        self.index = index
+        self.embeddings = embeddings
+
+    @staticmethod
+    def from_vector_db(vector_data) -> Self:
+        """
+        Creates a TokenData instance from a VectorDBClient.
+
+        This static method extracts the necessary components from a VectorDBClient instance
+        and uses them to instantiate a new TokenData object.
+
+        Parameters
+        ----------
+        vector_data : VectorDBClient
+            The VectorDBClient instance containing the FAISS index, embeddings, and text data.
+
+        Returns
+        -------
+        TokenData
+            A new TokenData instance initialized with texts, FAISS index, and embeddings from the
+            VectorDBClient.
+        """
+        index = vector_data.faiss_index
+        embeddings = vector_data.file_argument.embeddings
+        texts = vector_data.file_argument.text_list
+        return TokenData(texts, index, embeddings)
+
+    @staticmethod
+    def from_file_argument(file_argument, index: faiss.IndexFlatL2) -> Self:
+        """
+        Loads the FAISS index and numpy embeddings from a file argument.
+
+        Args:
+            file_argument (FileArgument): An instance of FileArgument containing
+                                            the FAISS index and numpy embeddings.
+        """
+        embeddings = file_argument.embeddings
+        texts = file_argument.text_list
+        return TokenData(texts, index, embeddings)
+
+
+class TokenTransformer:
+
+    def __init__(self):
+        self.model = SentenceTransformer("all-MiniLM-L6-v2")
+
+    def search(
+        self,
+        query: str,
+        data: TokenData,
+        context_amount: int = 1,
+    ) -> str:
+        query_embedding = self._query_to_embeddings(query)
+        _, I = data.index.search(query_embedding, k=context_amount)
+        context = ""
+
+        for i in I[0]:
+            context += data.texts[i] + "\n"
+
+        return context.strip()
+
+    def create_index(self, texts: list[str]) -> TokenData:
+        """
+        Creates a FAISS index from a list of text strings.
+
+        This method converts the input texts to embeddings and then creates a
+        FAISS IndexFlatL2 (L2 distance/Euclidean space) index from these embeddings.
+
+        Args:
+            texts (list[str]): A list of text strings to be indexed.
+
+        Returns:
+            faiss.IndexFlatL2: A FAISS index containing the embeddings of the input texts.
+        """
+        embeddings = self.text_to_embeddings(texts)
+        index = self.embeddings_to_index(embeddings)
+        return TokenData(texts, index, embeddings)
+
+    def embeddings_to_index(self, embeddings_np: np.ndarray) -> faiss.IndexFlatL2:
+        """
+        Creates a FAISS index using the provided numpy array of embeddings.
+
+        This method initializes a FAISS IndexFlatL2 (L2 distance/Euclidean) index with
+        the dimensions from the input embeddings, adds the embeddings to the index.
+
+        Args:
+            embeddings_np (np.ndarray): A numpy array of embedding vectors to be indexed.
+                The shape should be (n, dimension) where n is the number of vectors
+                and dimension is the size of each vector.
+
+        Returns:
+            faiss.IndexFlatL2: The created FAISS index containing the embeddings.
+
+        Note:
+            This method also sets the index as an instance attribute and saves it to disk
+            using the save_faiss_index method.
+        """
+        dimension = embeddings_np.shape[1]
+        index: faiss.IndexFlatL2 = faiss.IndexFlatL2(dimension)
+        index.add(embeddings_np)
+        return index
+
+    def text_to_embeddings(self, sentences: list[str]) -> np.ndarray:
+        """
+        Transforms a list of sentences into embeddings using the model.
+
+        Args:
+            sentences (list[str]): A list of sentences to be transformed into embeddings.
+
+        Returns:
+            np.ndarray: A numpy array containing the embeddings for each sentence.
+        """
+        return self.model.encode(sentences)
+
+    def get_np_vectors(self, embeddings: list[float]) -> np.ndarray:
+        """
+        Converts input embeddings to a numpy array of float32 type.
+
+        Args:
+            embeddings (list[float]): The embeddings to convert.
+
+        Returns:
+            np.ndarray: A numpy array containing the embeddings as float32 values.
+        """
+        return np.array(embeddings).astype("float32")
+
+    def _query_to_embeddings(self, query: str) -> np.ndarray:
+        """
+        Converts a text query into embeddings using the model.
+
+        Args:
+            query (str): The text query to be transformed into embeddings.
+
+        Returns:
+            np.ndarray: The embedding representation of the query reshaped to
+                        have dimensions (1, embedding_size).
+        """
+        return self.model.encode([query]).reshape(1, -1)

vectoriz-0.0.3/vectoriz/vector_db.py

@@ -0,0 +1,203 @@
+import os
+import faiss
+import numpy as np
+from typing import Optional
+
+from vectoriz.files import FileArgument
+from vectoriz.token_transformer import TokenTransformer
+
+
+class VectorDBClient:
+
+    def __init__(
+        self,
+        faiss_index: Optional[faiss.IndexFlatL2] = None,
+        file_argument: Optional[FileArgument] = None,
+    ):
+        """
+        Initialize the SavedVectorData with a FAISS index and embeddings.
+
+        Args:
+            faiss_index (faiss.IndexFlatL2): The FAISS index containing the vector data.
+            embeddings (np.ndarray): The numpy array of embeddings associated with the index.
+        """
+        self.faiss_index = faiss_index
+        self.file_argument = file_argument
+
+    def save_data(self, faiss_db_path: str, np_db_path: str) -> None:
+        """
+        Save the FAISS index and numpy embeddings to disk.
+
+        Args:
+            faiss_db_path (str): Path to save the FAISS index.
+            np_db_path (str): Path to save the numpy embeddings.
+        """
+        if self.faiss_index is None or self.file_argument is None:
+            raise ValueError("FAISS index or file argument is not initialized.")
+
+        vectorDB = VectorDB()
+        vectorDB.save_faiss_index(self.faiss_index, faiss_db_path)
+        vectorDB.save_numpy_embeddings(self.file_argument, np_db_path)
+        
+    def load_data(self, faiss_db_path: str, np_db_path: str) -> None:
+        """
+        Load the FAISS index and numpy embeddings from disk.
+
+        Args:
+            faiss_db_path (str): Path to load the FAISS index from.
+            np_db_path (str): Path to load the numpy embeddings from.
+        """
+        vectorDB = VectorDB()
+        self.faiss_index = vectorDB.load_faiss_index(faiss_db_path)
+        self.file_argument = vectorDB.load_numpy_embeddings(np_db_path)
+
+
+class VectorDB:
+
+    def __init__(self):
+        """
+        Constructor for the class.
+
+        Initializes the following attributes:
+        - transformer: A TokenTransformer instance for text transformation.
+        """
+        self.transformer = TokenTransformer()
+
+    def load_saved_data(
+        self, faiss_db_path: str, np_db_path: str
+    ) -> Optional[VectorDBClient]:
+        """
+        Load previously saved FAISS index and numpy embeddings data.
+
+        This function attempts to load a FAISS index and numpy embeddings from specified paths.
+        It combines them into a SavedVectorData object if both are successfully loaded.
+
+        Parameters:
+        ----------
+        faiss_db_path : str
+            Path to the saved FAISS index file
+        np_db_path : str
+            Path to the saved numpy embeddings file
+
+        Returns:
+        -------
+        Optional[SavedVectorData]
+            A SavedVectorData object containing the loaded index and embeddings if successful,
+            or None if either file could not be loaded.
+        """
+        index = self.load_faiss_index(faiss_db_path)
+        file_argument = self.load_numpy_embeddings(np_db_path)
+
+        if index is None or file_argument is None:
+            return None
+        return VectorDBClient(index, file_argument)
+
+    def save_faiss_index(
+        self,
+        index: faiss.IndexFlatL2,
+        faiss_db_path: str,
+    ) -> None:
+        """
+        Save a FAISS index to disk.
+
+        This method takes a FAISS index and saves it to the specified location.
+        It ensures the filename has the correct extension and the folder path ends with a slash.
+
+        Args:
+            index (faiss.IndexFlatL2): The FAISS index to save
+            faiss_db_path (str): The directory and the name of the file to save the index as where the index will be saved
+
+        Returns:
+            None: This method doesn't return anything
+
+        Note:
+            If the filename doesn't end with '.index', the extension will be added automatically.
+            If the folder_path doesn't end with '/', it will be added automatically.
+        """
+        faiss_db_path = (
+            faiss_db_path
+            if faiss_db_path.endswith(".index")
+            else faiss_db_path + ".index"
+        )
+        faiss.write_index(index, faiss_db_path)
+
+    def load_faiss_index(self, faiss_db_path: str) -> Optional[faiss.IndexFlatL2]:
+        """
+        Load a FAISS index from a specified file path.
+
+        Args:
+            faiss_db_path (str): Path to the FAISS index file.
+
+        Returns:
+            Optional[faiss.IndexFlatL2]: The loaded FAISS index if the file exists,
+            None otherwise.
+
+        Note:
+            If the file does not exist, a message will be printed to console.
+        """
+        if not os.path.exists(faiss_db_path):
+            return None
+        return faiss.read_index(faiss_db_path)
+
+    def save_numpy_embeddings(
+        self,
+        argument: FileArgument,
+        np_db_path: str,
+    ) -> None:
+        """
+        Save embeddings, chunk names, and texts to a compressed numpy file (.npz).
+        Args:
+            argument (FileArgument): An object containing embeddings, ndarray_data, chunk_names, and text_list.
+            np_db_path (str): Directory path and filename where the file will be saved.
+        Returns:
+            np.ndarray: The numpy array containing the embeddings, either from argument.ndarray_data
+                        or generated from argument.embeddings.
+        Notes:
+            The saved .npz file will contain three arrays:
+            - 'embeddings': The vector embeddings
+            - 'chunk_names': The chunk names
+            - 'texts': The text content
+        """
+        np_db_path = np_db_path if np_db_path.endswith(".npz") else np_db_path + ".npz"
+
+        embeddings_np: np.ndarray = None
+        if argument.ndarray_data is not None:
+            embeddings_np = argument.ndarray_data
+        else:
+            embeddings_np = self.transformer.get_np_vectors(argument.embeddings)
+
+        np.savez(
+            np_db_path,
+            embeddings=embeddings_np,
+            chunk_names=argument.chunk_names,
+            texts=argument.text_list,
+        )
+
+    def load_numpy_embeddings(self, np_db_path: str) -> Optional[FileArgument]:
+        """
+        Load embeddings from a NumPy archive file.
+
+        This method reads embeddings, filenames, and text data from a .npz file
+        created by a previous vectorization process.
+
+        Args:
+            np_db_path (str): Path to the NumPy archive file containing embeddings and metadata.
+
+        Returns:
+            Optional[FileArgument]: A FileArgument object containing the loaded data,
+                                   or None if the specified file does not exist.
+        """
+        if not os.path.exists(np_db_path):
+            return None
+
+        data = np.load(self.np_db_path)
+        embeddings_np = data["embeddings"]
+        chunk_names = data["chunk_names"]
+        texts = data["texts"]
+
+        return FileArgument(
+            chunk_names=chunk_names,
+            text_list=texts,
+            embeddings=[],
+            ndarray_data=embeddings_np,
+        )

vectoriz-0.0.3/vectoriz.egg-info/PKG-INFO

@@ -0,0 +1,60 @@
+Metadata-Version: 2.4
+Name: vectoriz
+Version: 0.0.3
+Summary: Python library for creating vectorized data from text or files.
+Home-page: https://github.com/PedroHenriqueDevBR/vectoriz
+Author: PedroHenriqueDevBR
+Author-email: pedro.henrique.particular@gmail.com
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: faiss-cpu==1.10.0
+Requires-Dist: numpy==2.2.4
+Requires-Dist: sentence-transformers==4.0.2
+Requires-Dist: python-docx==1.1.2
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# RAG-vector-creator
+
+## Overview
+This project implements a RAG (Retrieval-Augmented Generation) system for creating and managing vector embeddings from documents using FAISS and NumPy libraries. It efficiently transforms text data into high-dimensional vector representations that enable semantic search capabilities, similarity matching, and context-aware document retrieval for enhanced question answering applications.
+
+## Features
+
+- Document ingestion and preprocessing
+- Vector embedding generation using state-of-the-art models
+- Efficient storage and retrieval of embeddings
+- Integration with LLM-based generation systems
+
+## Installation
+
+```bash
+pip install -r requirements.txt
+python app.py
+```
+
+## Build lib
+
+To build the lib run the commands:
+
+```
+python setup.py sdist bdist_wheel
+```
+
+To test the install run:
+```
+pip install .
+```
+
+## License
+
+MIT

vectoriz-0.0.3/vectoriz.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+setup.py
+tests/__init__.py
+tests/test_files.py
+vectoriz/__init__.py
+vectoriz/files.py
+vectoriz/token_transformer.py
+vectoriz/vector_db.py
+vectoriz.egg-info/PKG-INFO
+vectoriz.egg-info/SOURCES.txt
+vectoriz.egg-info/dependency_links.txt
+vectoriz.egg-info/requires.txt
+vectoriz.egg-info/top_level.txt

vectoriz-0.0.3/vectoriz.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

vectoriz-0.0.3/vectoriz.egg-info/requires.txt

@@ -0,0 +1,4 @@
+faiss-cpu==1.10.0
+numpy==2.2.4
+sentence-transformers==4.0.2
+python-docx==1.1.2

vectoriz-0.0.3/vectoriz.egg-info/top_level.txt

@@ -0,0 +1,2 @@
+tests
+vectoriz