ddi-fw 0.0.23__tar.gz → 0.0.24__tar.gz

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.23
+Version: 0.0.24
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.23"
+version = "0.0.24"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/src/ddi_fw/datasets/core.py

@@ -21,8 +21,9 @@ def stack(df_column):
 
 
 class BaseDataset(ABC):
-    def __init__(self, chemical_property_columns, embedding_columns, ner_columns,
+    def __init__(self,embedding_dict, chemical_property_columns, embedding_columns, ner_columns,
                  **kwargs):
+        self.embedding_dict = embedding_dict
         self.__similarity_related_columns__ = []
         self.__similarity_related_columns__.extend(chemical_property_columns)
         self.__similarity_related_columns__.extend(ner_columns)
@@ -250,21 +251,21 @@ class BaseDataset(ABC):
 
 # her bir metin tipi için embedding oluşturursan burayı düzenle
     def prep(self):
-        if self.embedding_columns:
-            zip_helper = ZipHelper()
-            zip_helper.extract(str(HERE.joinpath('zips/embeddings')),
-                               str(HERE.joinpath('zips/embeddings')))
-
-        embedding_dict = dict()
-        for embedding_column in self.embedding_columns:
-            embedding_file = HERE.joinpath(
-                f'zips/embeddings/{embedding_column}_embeddings.pkl')
-            embedding_values = pd.read_pickle(embedding_file)
-            d = embedding_values.apply(
-                lambda x: {x.id: x[f'{embedding_column}_embedding']}, axis=1)
-            x = {k: v for l in d.values.tolist() for k, v in l.items()}
-            embedding_dict[embedding_column] = x
-
+        # if self.embedding_columns:
+        #     zip_helper = ZipHelper()
+        #     zip_helper.extract(str(HERE.joinpath('zips/embeddings')),
+        #                        str(HERE.joinpath('zips/embeddings')))
+
+       # embedding_dict = dict()
+        # for embedding_column in self.embedding_columns:
+        #     embedding_file = HERE.joinpath(
+        #         f'zips/embeddings/{embedding_column}_embeddings.pkl')
+        #     embedding_values = pd.read_pickle(embedding_file)
+        #     d = embedding_values.apply(
+        #         lambda x: {x.id: x[f'{embedding_column}_embedding']}, axis=1)
+        #     x = {k: v for l in d.values.tolist() for k, v in l.items()}
+        #     embedding_dict[embedding_column] = x
+ 
         self.ner_df = CTakesNER().load()
         drug_names = self.drugs_df['name'].to_list()
         drug_ids = self.drugs_df['id'].to_list()
@@ -362,14 +363,8 @@ class BaseDataset(ABC):
                 #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
 
         def x_fnc(row, embedding_values, embedding_column):
-            # first = embedding_values[embedding_values.id == row['id1']]
-            # second = embedding_values[embedding_values.id == row['id2']]
-            # v1 = first.iloc[0][embedding_column+'_embedding']
-            # v2 = second.iloc[0][embedding_column+'_embedding']
-            v1 = embedding_dict[embedding_column][row['id1']]
-            v2 = embedding_dict[embedding_column][row['id2']]
-            # v1 = embedding_dict[row['id1']][embedding_column+'_embedding']
-            # v2 = embedding_dict[row['id2']][embedding_column+'_embedding']
+            v1 = self.embedding_dict[embedding_column][row['id1']]
+            v2 = self.embedding_dict[embedding_column][row['id2']]
             return np.float16(np.hstack(
                 (v1, v2)))
 

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/src/ddi_fw/datasets/ddi_mdl/base.py

@@ -8,7 +8,7 @@ HERE = pathlib.Path(__file__).resolve().parent
 
 
 class DDIMDLDataset(BaseDataset):
-    def __init__(self,  chemical_property_columns=['enzyme',
+    def __init__(self, embedding_dict, chemical_property_columns=['enzyme',
                                                    'target',
                                                    'pathway',
                                                    'smile'],
@@ -16,7 +16,7 @@ class DDIMDLDataset(BaseDataset):
                  ner_columns=[],
                  **kwargs):
 
-        super().__init__(chemical_property_columns, embedding_columns,
+        super().__init__(embedding_dict, chemical_property_columns, embedding_columns,
                          ner_columns, **kwargs)
 
         # kwargs = {'index_path': str(HERE.joinpath('indexes'))}

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/src/ddi_fw/utils/zip_helper.py

@@ -7,21 +7,51 @@ import math
 from ddi_fw.utils.utils import create_folder_if_not_exists
 
 
+def get_file_name_and_folder(file_path):
+    file_path_components = file_path.split('/')
+    file_name = file_path_components[-1]
+    file_path = file_path[:len(file_name)*-1-1]
+    return file_name, file_path
+
+
 class ZipHelper:
     def __init__(self):
         pass
 
-    def zip_single_file(self, name, file_path, output_path):
+    def __zipdir__(self, file_path, zipf):
+        # ziph is zipfile handle
+        for root, dirs, files in os.walk(file_path):
+            for file in files:
+                zipf.write(os.path.join(root, file),
+                           os.path.relpath(os.path.join(root, file),
+                                           os.path.join(file_path, '..')))
+
+    def zip_dir(self, zip_name, file_path, output_path):
+        create_folder_if_not_exists(output_path)
+        with z.ZipFile(f'{output_path}/{zip_name}.zip', 'w', z.ZIP_DEFLATED) as zipf:
+            self.__zipdir__(file_path, zipf)
+
+    def zip_single_file(self, zip_name, file_path, output_path):
         if not os.path.exists(output_path):
             os.makedirs(output_path)
-        with z.ZipFile(f'{output_path}/{name}.zip', 'w', compression=z.ZIP_LZMA, compresslevel=z.ZIP_LZMA) as zipObj:
+        with z.ZipFile(f'{output_path}/{zip_name}.zip', 'w', compression=z.ZIP_LZMA, compresslevel=z.ZIP_LZMA) as zipObj:
             zipObj.write(file_path, basename(file_path))
 
-    def zip_as_multipart(self, zip_name, folder, file_name, output_path, chunk_size):
-        file_path = folder+'/'+file_name
-        parts_path=f"{folder}/parts"
-        self.zip_single_file(zip_name, file_path, output_path)
+    def zip_as_multipart(self, zip_name, file_path, output_path, chunk_size):
+        parent_folder = os.path.dirname(file_path)
+
+        parts_path = f"{parent_folder}/parts"
         create_folder_if_not_exists(parts_path)
+        # file_name, file_extension = os.path.splitext(file_path)
+        # file_name = os.path.basename(file_path)
+        file_name, folder = get_file_name_and_folder(file_path)
+
+        if os.path.isdir(file_path):
+            self.zip_dir(zip_name, file_path, output_path)
+        elif os.path.isfile(file_path):
+            self.zip_single_file(zip_name, file_path, output_path)
+        else:
+            return
         with open(file_path, 'rb') as f:
             chunk_number = 1
             while True:
@@ -67,7 +97,7 @@ class ZipHelper:
                     z1.extractall(path=output_path)
                     print(f'{file_path} has been extracted')
 
-    def extract_multiparts(self, output_path, parts_path,output_file):
+    def extract_multiparts(self, output_path, parts_path, output_file):
         input_parts = [parts_path+'/' + p for p in os.listdir(parts_path)]
         with open(f"{output_path}/{output_file}", 'wb') as outfile:
             for part in input_parts:

{ddi_fw-0.0.23 → ddi_fw-0.0.24}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.23
+Version: 0.0.24
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>