ddi-fw 0.0.23__py3-none-any.whl → 0.0.25__py3-none-any.whl

ddi_fw/datasets/core.py

@@ -21,8 +21,9 @@ def stack(df_column):
 
 
 class BaseDataset(ABC):
-    def __init__(self, chemical_property_columns, embedding_columns, ner_columns,
+    def __init__(self,embedding_dict, chemical_property_columns, embedding_columns, ner_columns,
                  **kwargs):
+        self.embedding_dict = embedding_dict
         self.__similarity_related_columns__ = []
         self.__similarity_related_columns__.extend(chemical_property_columns)
         self.__similarity_related_columns__.extend(ner_columns)
@@ -250,21 +251,21 @@ class BaseDataset(ABC):
 
 # her bir metin tipi için embedding oluşturursan burayı düzenle
     def prep(self):
-        if self.embedding_columns:
-            zip_helper = ZipHelper()
-            zip_helper.extract(str(HERE.joinpath('zips/embeddings')),
-                               str(HERE.joinpath('zips/embeddings')))
-
-        embedding_dict = dict()
-        for embedding_column in self.embedding_columns:
-            embedding_file = HERE.joinpath(
-                f'zips/embeddings/{embedding_column}_embeddings.pkl')
-            embedding_values = pd.read_pickle(embedding_file)
-            d = embedding_values.apply(
-                lambda x: {x.id: x[f'{embedding_column}_embedding']}, axis=1)
-            x = {k: v for l in d.values.tolist() for k, v in l.items()}
-            embedding_dict[embedding_column] = x
-
+        # if self.embedding_columns:
+        #     zip_helper = ZipHelper()
+        #     zip_helper.extract(str(HERE.joinpath('zips/embeddings')),
+        #                        str(HERE.joinpath('zips/embeddings')))
+
+       # embedding_dict = dict()
+        # for embedding_column in self.embedding_columns:
+        #     embedding_file = HERE.joinpath(
+        #         f'zips/embeddings/{embedding_column}_embeddings.pkl')
+        #     embedding_values = pd.read_pickle(embedding_file)
+        #     d = embedding_values.apply(
+        #         lambda x: {x.id: x[f'{embedding_column}_embedding']}, axis=1)
+        #     x = {k: v for l in d.values.tolist() for k, v in l.items()}
+        #     embedding_dict[embedding_column] = x
+ 
         self.ner_df = CTakesNER().load()
         drug_names = self.drugs_df['name'].to_list()
         drug_ids = self.drugs_df['id'].to_list()
@@ -362,14 +363,8 @@ class BaseDataset(ABC):
                 #     (value[row['id1']], value[row['id2']]), dtype=np.float16)
 
         def x_fnc(row, embedding_values, embedding_column):
-            # first = embedding_values[embedding_values.id == row['id1']]
-            # second = embedding_values[embedding_values.id == row['id2']]
-            # v1 = first.iloc[0][embedding_column+'_embedding']
-            # v2 = second.iloc[0][embedding_column+'_embedding']
-            v1 = embedding_dict[embedding_column][row['id1']]
-            v2 = embedding_dict[embedding_column][row['id2']]
-            # v1 = embedding_dict[row['id1']][embedding_column+'_embedding']
-            # v2 = embedding_dict[row['id2']][embedding_column+'_embedding']
+            v1 = self.embedding_dict[embedding_column][row['id1']]
+            v2 = self.embedding_dict[embedding_column][row['id2']]
             return np.float16(np.hstack(
                 (v1, v2)))
 

ddi_fw/datasets/ddi_mdl/base.py

@@ -8,7 +8,7 @@ HERE = pathlib.Path(__file__).resolve().parent
 
 
 class DDIMDLDataset(BaseDataset):
-    def __init__(self,  chemical_property_columns=['enzyme',
+    def __init__(self, embedding_dict, chemical_property_columns=['enzyme',
                                                    'target',
                                                    'pathway',
                                                    'smile'],
@@ -16,7 +16,7 @@ class DDIMDLDataset(BaseDataset):
                  ner_columns=[],
                  **kwargs):
 
-        super().__init__(chemical_property_columns, embedding_columns,
+        super().__init__(embedding_dict, chemical_property_columns, embedding_columns,
                          ner_columns, **kwargs)
 
         # kwargs = {'index_path': str(HERE.joinpath('indexes'))}

ddi_fw/utils/zip_helper.py

@@ -7,30 +7,62 @@ import math
 from ddi_fw.utils.utils import create_folder_if_not_exists
 
 
+def get_file_name_and_folder(file_path):
+    file_path_components = file_path.split('/')
+    file_name = file_path_components[-1]
+    file_path = file_path[:len(file_name)*-1-1]
+    return file_name, file_path
+
+
 class ZipHelper:
     def __init__(self):
         pass
 
-    def zip_single_file(self, name, file_path, output_path):
+    def __zipdir__(self, file_path, zipf):
+        # ziph is zipfile handle
+        for root, dirs, files in os.walk(file_path):
+            for file in files:
+                zipf.write(os.path.join(root, file),
+                           os.path.relpath(os.path.join(root, file),
+                                           os.path.join(file_path, '..')))
+
+    def zip_dir(self, zip_name, file_path, output_path):
+        create_folder_if_not_exists(output_path)
+        with z.ZipFile(f'{output_path}/{zip_name}.zip', 'w', z.ZIP_DEFLATED) as zipf:
+            self.__zipdir__(file_path, zipf)
+
+    def zip_single_file(self, zip_name, file_path, output_path):
         if not os.path.exists(output_path):
             os.makedirs(output_path)
-        with z.ZipFile(f'{output_path}/{name}.zip', 'w', compression=z.ZIP_LZMA, compresslevel=z.ZIP_LZMA) as zipObj:
+        with z.ZipFile(f'{output_path}/{zip_name}.zip', 'w', compression=z.ZIP_LZMA, compresslevel=z.ZIP_LZMA) as zipObj:
             zipObj.write(file_path, basename(file_path))
 
-    def zip_as_multipart(self, zip_name, folder, file_name, output_path, chunk_size):
-        file_path = folder+'/'+file_name
-        parts_path=f"{folder}/parts"
-        self.zip_single_file(zip_name, file_path, output_path)
-        create_folder_if_not_exists(parts_path)
-        with open(file_path, 'rb') as f:
+    def zip_as_multipart(self, zip_name, file_path, output_path, chunk_size):
+        parent_folder = os.path.dirname(file_path)
+
+        # parts_path = f"{parent_folder}/parts"
+        # create_folder_if_not_exists(parts_path)
+        # file_name, file_extension = os.path.splitext(file_path)
+        # file_name = os.path.basename(file_path)
+        file_name, folder = get_file_name_and_folder(file_path)
+
+        if os.path.isdir(file_path):
+            self.zip_dir(zip_name, file_path, output_path)
+        elif os.path.isfile(file_path):
+            self.zip_single_file(zip_name, file_path, output_path)
+        else:
+            return
+        with open(output_path+'/'+zip_name+'.zip', 'rb') as f:
             chunk_number = 1
             while True:
                 chunk = f.read(chunk_size)
                 if not chunk:
                     break
-                with open(f"{parts_path}/{file_name}.part{chunk_number:03}", 'wb') as chunk_file:
+                with open(f"{output_path}/{zip_name}.zip.part{chunk_number:03}", 'wb') as chunk_file:
                     chunk_file.write(chunk)
                 chunk_number += 1
+        if os.path.exists(output_path+'/'+zip_name+'.zip'):
+            os.remove(output_path+'/'+zip_name+'.zip')
 
     def zip(self, zip_prefix, input_path, output_path, chunk_size):
         files_paths = [input_path+'/' + p for p in os.listdir(input_path)]
@@ -67,7 +99,7 @@ class ZipHelper:
                     z1.extractall(path=output_path)
                     print(f'{file_path} has been extracted')
 
-    def extract_multiparts(self, output_path, parts_path,output_file):
+    def extract_multiparts(self, output_path, parts_path, output_file):
         input_parts = [parts_path+'/' + p for p in os.listdir(parts_path)]
         with open(f"{output_path}/{output_file}", 'wb') as outfile:
             for part in input_parts:

{ddi_fw-0.0.23.dist-info → ddi_fw-0.0.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.23
+Version: 0.0.25
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.23.dist-info → ddi_fw-0.0.25.dist-info}/RECORD

@@ -1,12 +1,12 @@
 ddi_fw/datasets/__init__.py,sha256=gkzHCU9-BL_bOU-RvvwdOIp_DhKRfXKU5SvgfQLVTds,505
-ddi_fw/datasets/core.py,sha256=hWvDxptCTOazcDdALdHS2siHgPB1RNUa-lfVDV0snAA,18425
+ddi_fw/datasets/core.py,sha256=Iv8l2WRM2rNyshdXd8LIz06qIGpz3EwCImEIc_Qr00w,18084
 ddi_fw/datasets/db_utils.py,sha256=OTsa3d-Iic7z3HmzSQK9UigedRbHDxYChJk0s4GfLnw,6191
 ddi_fw/datasets/embedding_generator.py,sha256=Jqrlv88RCu0Lg812KsA12X0cSaZuxbckJ4LNRKNy_qw,2173
 ddi_fw/datasets/embedding_generator_new.py,sha256=GExjmBysPWkmFxTZQPs2yEmDdFllZ-qC9lhZeRQAfbQ,4320
 ddi_fw/datasets/feature_vector_generation.py,sha256=dxTHvp6uTkao9PdThs116Q3bWw_WTo9T8WigVL4G01s,3245
 ddi_fw/datasets/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
 ddi_fw/datasets/setup_._py,sha256=khYVJuW5PlOY_i_A16F3UbSZ6s6o_ljw33Byw3C-A8E,1047
-ddi_fw/datasets/ddi_mdl/base.py,sha256=Uz1ZZS9LvxDR3EO_FqaZCKP3idQb3yytkwRzgBAYGAA,2422
+ddi_fw/datasets/ddi_mdl/base.py,sha256=_R9-CE2P2kNoxpkDWWIHfSvsRidmYqTj-Ldp4HZpoPg,2453
 ddi_fw/datasets/ddi_mdl/readme.md,sha256=WC6lpmsEKvIISnZqENY7TWtzCQr98HPpE3oRsBl8pIw,625
 ddi_fw/datasets/ddi_mdl/data/event.db,sha256=cmlSsf9MYjRzqR-mw3cUDnTnfT6FkpOG2yCl2mMwwew,30580736
 ddi_fw/datasets/ddi_mdl/indexes/test_indexes.txt,sha256=XVlDqYATckrQwNSXqMSKVBqyoN_Hg8SK6CL-XMdLADY,102176
@@ -78,8 +78,8 @@ ddi_fw/test/type_guarding_test.py,sha256=KxjyBxohDu7lwpejalCj-REjtJ-k1S1wQbOB6TG
 ddi_fw/utils/__init__.py,sha256=nhNU_sEp55xsZ5VtvhozjKg6r4GWP6SJI13v8F_jbCg,217
 ddi_fw/utils/enums.py,sha256=19eJ3fX5eRK_xPvkYcukmug144jXPH4X9zQqtsFBj5A,671
 ddi_fw/utils/utils.py,sha256=Na6Y8mY-CFbQjrgd9xC8agcrjVvTj_7KIXqFm1H_3qU,3549
-ddi_fw/utils/zip_helper.py,sha256=KnOFjf7MAw7jTjYuNJKzQL_VBGDRZiWy1a72T8_Uslg,3763
-ddi_fw-0.0.23.dist-info/METADATA,sha256=SkMnhWTRE0XsOT-PMUejmGVFdbNy8EBM-OWrkDErTgc,1541
-ddi_fw-0.0.23.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-ddi_fw-0.0.23.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.23.dist-info/RECORD,,
+ddi_fw/utils/zip_helper.py,sha256=CF6Th3ntXcx_F2bzoPhUWkVE27YqZqPNrxHgDewArfs,5123
+ddi_fw-0.0.25.dist-info/METADATA,sha256=iGtI4gPTWiD9l5XKMCduX_kEY0ErYFX3kiLf8fujuFA,1541
+ddi_fw-0.0.25.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
+ddi_fw-0.0.25.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.25.dist-info/RECORD,,