ddi-fw 0.0.48__py3-none-any.whl → 0.0.50__py3-none-any.whl

ddi_fw/datasets/embedding_generator_new.py

@@ -17,7 +17,7 @@ from sentence_transformers import SentenceTransformer, util
 
 from typing import Any, Dict, List, Optional
 from langchain_core.embeddings import Embeddings
-from pydantic import BaseModel, ConfigDict, Field, SecretStr
+from pydantic import BaseModel, ConfigDict, Field, SecretStr, computed_field
 from langchain.embeddings import SentenceTransformerEmbeddings
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -56,7 +56,7 @@ class SumPoolingStrategy(PoolingStrategy):
         return np.sum(embeddings, axis=0)
 
 
-class SentenceTransformerDecorator(BaseModel, Embeddings):
+class SentenceTransformerDecorator(Embeddings):
     def __init__(self, model_name="all-MiniLM-L6-v2", **kwargs: Any):
         self.embeddings = SentenceTransformerEmbeddings(model_name=model_name)
 
@@ -67,7 +67,7 @@ class SentenceTransformerDecorator(BaseModel, Embeddings):
         return self.embeddings.embed_query(text)
 
 
-class PretrainedEmbeddings(BaseModel, Embeddings):
+class PretrainedEmbeddings( Embeddings):
     def __init__(self, model_name):
         self.mmodel_name = model_name
         self.model = AutoModel.from_pretrained(model_name)
@@ -88,11 +88,23 @@ class PretrainedEmbeddings(BaseModel, Embeddings):
 
 
 class SBertEmbeddings(BaseModel, Embeddings):
-    def __init__(self, model_name):
-        self.model = SentenceTransformer(model_name)
+    # class Config:
+    #     arbitrary_types_allowed = True
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed = True,
+        protected_namespaces=()
+    )
+
+    model_name:str
+
+    @computed_field
+    @property
+    def model(self) -> SentenceTransformer:
+        return SentenceTransformer(self.model_name)
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
-        return self.model.encode(texts)
+        return self.model.encode(texts).tolist()
 
     def embed_query(self, text: str) -> List[float]:
         return self.embed_documents([text])[0]

ddi_fw/utils/__init__.py

@@ -1,4 +1,4 @@
-from .utils import create_folder_if_not_exists, utc_time_as_string,utc_time_as_string_simple_format, compress_and_save_data
+from .utils import clear_directory,create_folder_if_not_exists, utc_time_as_string,utc_time_as_string_simple_format, compress_and_save_data
 from .zip_helper import ZipHelper
 from .py7zr_helper import Py7ZipHelper
 from .enums import UMLSCodeTypes, DrugBankTextDataTypes

ddi_fw/utils/py7zr_helper.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 import math
-from ddi_fw.utils.utils import create_folder_if_not_exists
+from ddi_fw.utils.utils import clear_directory, create_folder_if_not_exists
 import py7zr
 import os
 from os.path import basename
@@ -24,9 +24,10 @@ class Py7ZipHelper:
                                                   os.path.join(folder_path, '..')))
                     # archive.write(root+"/"+file)
 
-    def create_archive_multiparts(self, zip_name, file_path, output_path, chunk_size):
+    def create_archive_multiparts(self, zip_name, file_path, output_path, chunk_size, delete_existing_files:True):
         parent_folder = os.path.dirname(file_path)
-
+        if delete_existing_files:
+            clear_directory(output_path)
         # parts_path = f"{parent_folder}/parts"
         create_folder_if_not_exists(output_path)
         # file_name, file_extension = os.path.splitext(file_path)

ddi_fw/utils/utils.py

@@ -1,11 +1,9 @@
 import gzip
 import json
 import os
-
 from datetime import datetime, timezone
-
 from matplotlib import pyplot as plt
-
+import shutil
 
 def create_folder_if_not_exists(path):
     if not os.path.exists(path):
@@ -51,6 +49,22 @@ def decompress(gzip_file):
     return data
 
 
+def clear_directory(directory_path):
+    # Check if the directory exists
+    if os.path.exists(directory_path) and os.path.isdir(directory_path):
+        # Iterate through all files and directories in the directory
+        for item in os.listdir(directory_path):
+            item_path = os.path.join(directory_path, item)
+            # Check if it's a file or a directory and remove it
+            if os.path.isfile(item_path):
+                os.remove(item_path)  # Remove file
+            elif os.path.isdir(item_path):
+                shutil.rmtree(item_path)  # Remove directory
+        print(f"Cleared contents of directory: {directory_path}")
+    else:
+        print(f"The directory does not exist: {directory_path}")
+
+
 if __name__ == "__main__":
     # json_file = f'C:\\Users\\kivanc\\Downloads\\metrics.json'
     # file_data = open(json_file, "r", 1).read()

{ddi_fw-0.0.48.dist-info → ddi_fw-0.0.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.48
+Version: 0.0.50
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.48.dist-info → ddi_fw-0.0.50.dist-info}/RECORD

@@ -2,7 +2,7 @@ ddi_fw/datasets/__init__.py,sha256=CqDrx7Ov83pXRh-n0ylembBmzhlW_yFWiheBcISrKdg,5
 ddi_fw/datasets/core.py,sha256=CFmnK0_cUxQAlTHjC2LEuKX0DNVAhGjSyQaV4jURYdI,18932
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=Jqrlv88RCu0Lg812KsA12X0cSaZuxbckJ4LNRKNy_qw,2173
-ddi_fw/datasets/embedding_generator_new.py,sha256=bdZ7dSrm0oJTRh84MyltB_hhgL_IQyvusO6aPfW8JxU,7257
+ddi_fw/datasets/embedding_generator_new.py,sha256=GOE-Io6-DBwiUJSkgmxw9ZM1exCYYVu9KyP2dH3gf1o,7506
 ddi_fw/datasets/feature_vector_generation.py,sha256=dxTHvp6uTkao9PdThs116Q3bWw_WTo9T8WigVL4G01s,3245
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
@@ -75,12 +75,12 @@ ddi_fw/test/sklearn-tfidf.py,sha256=cjtg27vLskEMXgrsqUR_EapRGVd4xgwOQ9zYsu72zjs,
 ddi_fw/test/test.py,sha256=zJh9ZBcZl-vZIFDvuftcRrRV8WAwtiFVhPPd6Et4OU4,2997
 ddi_fw/test/torch_cuda_test.py,sha256=R-4VGVErl_Ufk54DoZbgL_YXWoCYFyanIVWd6P39IEk,312
 ddi_fw/test/type_guarding_test.py,sha256=KxjyBxohDu7lwpejalCj-REjtJ-k1S1wQbOB6TGY0O8,766
-ddi_fw/utils/__init__.py,sha256=cbvKldLWGINnVWIVwRQ04zdJpqOf-Ci9X03I8OgmJcw,257
+ddi_fw/utils/__init__.py,sha256=x1ypYJRKJlbF9x4psHYGXj-YbDD8T_c28gXZkr03cdE,273
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
-ddi_fw/utils/py7zr_helper.py,sha256=rWhz3Trj6bk6B24hMuff1fKdRbC0yuvaHFlL8-eaAhg,4450
-ddi_fw/utils/utils.py,sha256=Na6Y8mY-CFbQjrgd9xC8agcrjVvTj_7KIXqFm1H_3qU,3549
+ddi_fw/utils/py7zr_helper.py,sha256=8U3fq9U6Y6vSx-InQ444SxFLHvOQNFFe8bXJgBrPC-Y,4570
+ddi_fw/utils/utils.py,sha256=szwnxMTDRrZoeNRyDuf3aCbtzriwtaRk4mHSH3asLdA,4301
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
-ddi_fw-0.0.48.dist-info/METADATA,sha256=NT2xu52EFUO2Rtk4hDZtCftemzeHYBTLZ7ktOHnEUxM,1565
-ddi_fw-0.0.48.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-ddi_fw-0.0.48.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.48.dist-info/RECORD,,
+ddi_fw-0.0.50.dist-info/METADATA,sha256=C4QuZ2hRXteDRlwN_SlebxopvrcRCo362XsgkWqU1KA,1565
+ddi_fw-0.0.50.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+ddi_fw-0.0.50.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.50.dist-info/RECORD,,