vector-vault 0.2.4__tar.gz → 0.2.7__tar.gz

{vector_vault-0.2.4 → vector_vault-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vector_vault
-Version: 0.2.4
+Version: 0.2.7
 Summary: VectorVault: Simplified vector database management in the cloud for machine learning and generative ai workflows
 Home-page: https://github.com/John-Rood/VectorVault
 Author: VectorVault.io

{vector_vault-0.2.4 → vector_vault-0.2.7}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="vector_vault",
-    version="0.2.4",
+    version="0.2.7",
     packages=find_packages(),
     author="VectorVault.io",
     author_email="john@johnrood.com",

{vector_vault-0.2.4 → vector_vault-0.2.7}/vector_vault.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: vector-vault
-Version: 0.2.4
+Version: 0.2.7
 Summary: VectorVault: Simplified vector database management in the cloud for machine learning and generative ai workflows
 Home-page: https://github.com/John-Rood/VectorVault
 Author: VectorVault.io

{vector_vault-0.2.4 → vector_vault-0.2.7}/vector_vault.egg-info/SOURCES.txt

@@ -11,5 +11,6 @@ vectorvault/closedai.py
 vectorvault/cloudmanager.py
 vectorvault/creds.py
 vectorvault/download.py
+vectorvault/itemize.py
 vectorvault/vault.py
 vectorvault/wrap.py

{vector_vault-0.2.4 → vector_vault-0.2.7}/vectorvault/cloudmanager.py

@@ -17,6 +17,7 @@ import tempfile
 import os
 import json
 from .creds import CustomCredentials
+from .itemize import name
 from google.cloud import storage
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
@@ -26,48 +27,46 @@ class CloudManager:
         self.user = user
         self.api = api_key
         self.vault = vault
-
-        # Create the OAuth credentials
+        # Creates the credentials
         credentials = CustomCredentials(self.user, self.api)
-
-        # Instantiates a client with the OAuth2 credentials
+        # Instantiates the client 
         self.storage_client = storage.Client(project='vectorvault-361ab', credentials=credentials)
-        self.gcloud = self.storage_client.bucket(self.user)
+        self.cloud = self.storage_client.bucket(self.user)
         print(f'Connected to Vault: {self.vault}')
 
     def vault_exists(self, vault_name):
-        return storage.Blob(bucket=self.gcloud, name=vault_name).exists(self.storage_client)
+        return storage.Blob(bucket=self.cloud, name=vault_name).exists(self.storage_client)
 
     def upload_to_cloud(self, vault_name, content):
-        blob = self.gcloud.blob(vault_name)
+        blob = self.cloud.blob(vault_name)
         blob.upload_from_string(content)
 
     def download_text_from_cloud(self, vault_name):
-        blob = self.gcloud.blob(vault_name)
+        blob = self.cloud.blob(vault_name)
         return blob.download_as_text()
 
     def upload_temp_file(self, temp_file_path, vault_name):
-        blob = self.gcloud.blob(vault_name)
+        blob = self.cloud.blob(vault_name)
         blob.upload_from_filename(temp_file_path)
         os.remove(temp_file_path)
 
     def download_to_temp_file(self, vault_name):
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            blob = self.gcloud.blob(vault_name)
+            blob = self.cloud.blob(vault_name)
             blob.download_to_filename(temp_file.name) 
             return temp_file.name
 
     def upload(self, item, text, meta):
         with ThreadPoolExecutor() as executor:
-            executor.submit(self.upload_to_cloud, f'{self.vault}/{item}/item', text)
-            executor.submit(self.upload_to_cloud, f'{self.vault}/{item}/meta', json.dumps(meta))
+            executor.submit(self.upload_to_cloud, name(self.vault, item, item=True), text)
+            executor.submit(self.upload_to_cloud, name(self.vault, item, meta=True), json.dumps(meta))
     
     def delete_blob(self, blob):
         blob.delete()
 
     def delete(self):
         # Get all objects
-        blobs = self.gcloud.list_blobs(prefix=self.vault)
+        blobs = self.cloud.list_blobs(prefix=self.vault)
         
         # Delete each object concurrently
         with ThreadPoolExecutor() as executor:
@@ -81,7 +80,7 @@ class CloudManager:
         
     def get_vaults(self, vault: str = None):
         vault = self.vault if vault is None else vault
-        blobs = self.gcloud.list_blobs(prefix=f'{vault}')
+        blobs = self.cloud.list_blobs(prefix=f'{vault}')
     
         directories = set()
         for blob in blobs:

{vector_vault-0.2.4 → vector_vault-0.2.7}/vectorvault/creds.py

@@ -34,7 +34,7 @@ class CustomCredentials(Credentials):
             return False
         return datetime.datetime.now() < self.expiry
 
-    def refresh(self, request):
+    def refresh(self):
         if not self.valid:
             data = {
                 "user_id": self.user,
@@ -46,7 +46,7 @@ class CustomCredentials(Credentials):
             self.token = response_data['access_token']
             self.expiry = datetime.datetime.fromisoformat(response_data['expiry'])
 
-    def before_request(self, request, method, url, headers):
+    def before_request(self, headers):
         self.apply(headers)
 
     def __getstate__(self):

vector_vault-0.2.7/vectorvault/itemize.py

@@ -0,0 +1,63 @@
+import datetime 
+from annoy import AnnoyIndex
+
+def itemize(vault, x, meta=None, text=None, name=None):
+    meta = {} if meta is None else meta
+
+    if 'name' not in meta and name is None:
+        name = f'{vault}-{x}'
+    elif name is not None:
+        name = __name__
+    metaize(meta, name, x)
+    return package(text, meta)
+    
+def metaize(meta, name, x):
+    meta['name'] = name
+    meta['item_id'] = x  
+    if 'created_at' not in meta:
+        meta['created_at'] = datetime.datetime.utcnow().isoformat()
+    if 'updated_at' not in meta:
+        meta['updated_at'] = datetime.datetime.utcnow().isoformat()
+
+def package(text, meta):
+    item = {
+        "text": text,
+        "meta": meta
+    }
+    return item
+
+def build_base_path(v, x):
+    return f'{v}/{x}'
+
+def append_path_suffix(base_path, is_item, is_meta):
+    if is_item:
+        suffix = 'item'
+    elif is_meta:
+        suffix = 'meta'
+    else:
+        suffix = ''
+    return f'{base_path}/{suffix}'
+
+def name(v, x, item=False, meta=False):
+    base_path = build_base_path(v, x)
+    final_path = append_path_suffix(base_path, item, meta)
+    return final_path
+
+def name_vecs(vault):
+    return f'{vault}.ann'
+
+def get_vectors(dims):
+    return AnnoyIndex(dims, 'angular')
+
+def get_item(item):
+    item_id = item["meta"]["item_id"]
+    item_text = item["text"]
+    item_meta = item["meta"]
+    return item_text, item_id, item_meta
+
+def build_return(results, item_data, meta):
+    result = {
+        "data": item_data,
+        "metadata": meta
+    }
+    results.append(result)

{vector_vault-0.2.4 → vector_vault-0.2.7}/vectorvault/vault.py

@@ -16,21 +16,20 @@
 import numpy as np
 import tempfile
 import os
-import datetime
 import time
 import json
 import re
 import openai
-from annoy import AnnoyIndex
 from concurrent.futures import ThreadPoolExecutor
 from .cloudmanager import CloudManager
 from .closedai import ClosedAI
+from .itemize import itemize, name, name_vecs, get_item, build_return, get_vectors
 
 
 class Vault:
     def __init__(self, user: str, api_key: str, vault: str = None, dims: int = 1536, verbose: bool = False):
         self.vault = vault.strip() if vault else 'home'
-        self.vectors = AnnoyIndex(dims, 'angular')
+        self.vectors = get_vectors(dims)
         self.dims = dims
         self.cloud_manager = CloudManager(user, api_key, self.vault)
         self.x = 0
@@ -46,15 +45,14 @@ class Vault:
 
     def check_index(self):
         start_time = time.time()
-        if self.cloud_manager.vault_exists(f'{self.vault}.ann'):
-            print('vault exists')
+        if self.cloud_manager.vault_exists(name_vecs(self.vault)):
             self.load_vectors()
             num_existing_items = self.vectors.get_n_items()
-            new_index = AnnoyIndex(self.dims, 'angular')
+            new_index = get_vectors(self.dims)
             for i in range(num_existing_items):
                 vector = self.vectors.get_item_vector(i)
                 new_index.add_item(i, vector)
-                self.x = i
+            self.x = i + 1
             self.vectors = new_index
         else:
             pass
@@ -67,7 +65,7 @@ class Vault:
     def load_vectors(self):
         start_time = time.time()
         # Download the .ann file from google cloud storage to a temporary file
-        temp_file_path = self.cloud_manager.download_to_temp_file(f'{self.vault}.ann')
+        temp_file_path = self.cloud_manager.download_to_temp_file(name_vecs(self.vault))
 
         # Load the AnnoyIndex object from the temporary file, then delete the temp file
         self.vectors.load(temp_file_path)
@@ -78,7 +76,7 @@ class Vault:
     
     def get_vaults(self, vault: str = None):
         vault = self.vault if vault is None else vault
-        return self.cloud_manager.get_vaults(vault=f'{vault}')
+        return self.cloud_manager.get_vaults(vault)
 
 
     def get_total_vectors(self):
@@ -91,12 +89,12 @@ class Vault:
 
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
             self.vectors.save(temp_file.name)
-            self.cloud_manager.upload_temp_file(temp_file.name, f'{self.vault}.ann')
+            self.cloud_manager.upload_temp_file(temp_file.name, name_vecs(self.vault))
 
         total_saved_items = 0
         for item in self.items:
-            item_id = item["meta"]["item_id"]
-            self.cloud_manager.upload(item_id, item["text"], item["meta"])
+            item_text, item_id, item_meta = get_item(item)
+            self.cloud_manager.upload(item_id, item_text, item_meta)
             total_saved_items += 1
 
         self.items.clear()
@@ -110,7 +108,7 @@ class Vault:
         if self.verbose == True:
                 print('Deleting started. Note: this can take a while for large datasets')
         # Clear the local vector data
-        self.vectors = AnnoyIndex(self.dims, 'angular')
+        self.vectors = get_vectors(self.dims)
         self.items.clear()
         self.x = 0
         self.cloud_manager.delete()
@@ -157,26 +155,15 @@ class Vault:
         self.load_vectors()
         start_time = time.time()
 
-        # Create an empty list to store the results
         results = []
-        indexes = self.vectors.get_nns_by_vector(vector, n)
-        print(indexes)
-        for index in indexes:
+        vecs = self.vectors.get_nns_by_vector(vector, n)
+        for vec in vecs:
             # Retrieve the item
-            item_data = self.cloud_manager.download_text_from_cloud(f'{self.vault}/{index}/item')
-
+            item_data = self.cloud_manager.download_text_from_cloud(name(self.vault, vec, item=True))
             # Retrieve the metadata
-            meta_data = self.cloud_manager.download_text_from_cloud(f'{self.vault}/{index}/meta')
+            meta_data = self.cloud_manager.download_text_from_cloud(name(self.vault, vec, meta=True))
             meta = json.loads(meta_data)
-
-            # Create a dictionary with data and metadata
-            result = {
-                "data": item_data,
-                "metadata": meta
-            }
-
-            # Append the result to the list
-            results.append(result)
+            build_return(results, item_data, meta)
 
         # Create a return dictionary
         return_dict = {
@@ -203,29 +190,8 @@ class Vault:
             self.check_index()
         else: 
             pass
-        
-        # set value if none exist
-        meta = {} if meta is None else meta
-
-        if 'name' not in meta and name is None:
-            name = f'{self.vault}-{self.x}'
-        elif name is not None:
-            name = name
-        meta['name'] = name
-        meta['item_id'] = self.x  # Store the item index in the metadata
-
-        if 'created_at' not in meta:
-            meta['created_at'] = datetime.datetime.utcnow().isoformat()
-        if 'updated_at' not in meta:
-            meta['updated_at'] = datetime.datetime.utcnow().isoformat()
-
-        # Add item and its metadata to the items list
-        item = {
-            "text": text,
-            "meta": meta
-        }
-        self.items.append(item)
 
+        self.items.append(itemize(self.vault, self.x, meta, text, name))
         self.x += 1
 
 
@@ -243,7 +209,7 @@ class Vault:
             texts = [text]
 
         for text in texts:
-            self.add_item(text)
+            self.add_item(text, meta, name)
 
     
     def add_item_with_vector(self, text: str, vector: list, meta: dict = None, name: str = None):
@@ -260,30 +226,9 @@ class Vault:
         if len(text) > 36000:
             raise 'Text length too long. Use the "split_text() function to get a list of text segments'
 
-        # set value if none exist
-        meta = {} if meta is None else meta
-
-        if 'name' not in meta and name is None:
-            name = f'{self.vault}-{self.x}'
-        elif name is not None:
-            name = name
-        meta['name'] = name
-        meta['item_id'] = self.x  # Store the item index in the metadata
-
-        if 'created_at' not in meta:
-            meta['created_at'] = datetime.datetime.utcnow().isoformat()
-        if 'updated_at' not in meta:
-            meta['updated_at'] = datetime.datetime.utcnow().isoformat()
-
         # Add vector to vectorspace
         self.vectors.add_item(self.x, vector)
-
-        # Add item and its metadata to the items list
-        item = {
-            "text": text,
-            "meta": meta
-        }
-        self.items.append(item)
+        self.items.append(itemize(self.vault, self.x, meta, text, name))
 
         self.x += 1