ddi-fw 0.0.97__tar.gz → 0.0.98__tar.gz

{ddi_fw-0.0.97 → ddi_fw-0.0.98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.97
+Version: 0.0.98
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
@@ -27,25 +27,25 @@ Requires-Dist: python-stopwatch==1.1.11
 Requires-Dist: lxml==5.3.0
 Requires-Dist: matplotlib==3.8.0
 Requires-Dist: mlflow==2.16.1
-Requires-Dist: nltk==3.8.1
-Requires-Dist: numpy==1.26.4
-Requires-Dist: pandas==2.2.2
+Requires-Dist: nltk>=3.8.1
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: pandas>=2.2.0
 Requires-Dist: plotly==5.24.1
 Requires-Dist: rdkit==2023.3.3
 Requires-Dist: scikit-learn==1.5.2
 Requires-Dist: scipy==1.13.1
-Requires-Dist: accelerate==0.33.0
-Requires-Dist: sentence-transformers==3.0.1
-Requires-Dist: transformers==4.42.4
+Requires-Dist: accelerate>=0.33.0
+Requires-Dist: sentence-transformers>=3.0.1
+Requires-Dist: transformers>=4.42.4
 Requires-Dist: stanza==1.9.2
-Requires-Dist: tokenizers==0.19.1
-Requires-Dist: tqdm==4.66.6
+Requires-Dist: tokenizers>=0.19.1
+Requires-Dist: tqdm>=4.66.6
 Requires-Dist: xmlschema==3.4.2
-Requires-Dist: zipp==3.20.2
+Requires-Dist: zipp>=3.20.2
 Requires-Dist: py7zr==0.22.0
-Requires-Dist: openai==1.52.2
-Requires-Dist: langchain==0.3.4
-Requires-Dist: chromadb==0.5.15
+Requires-Dist: openai>=1.52.2
+Requires-Dist: langchain>=0.3.4
+Requires-Dist: chromadb>=0.5.15
 Requires-Dist: langchain_community==0.3.3
 Requires-Dist: datasets==3.0.2
 Requires-Dist: unstructured==0.16.3

{ddi_fw-0.0.97 → ddi_fw-0.0.98}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.97"
+version = "0.0.98"
 description = "Do not use :)"
 readme = "README.md"
 authors = [
@@ -50,25 +50,25 @@ dependencies = [
 ,"lxml==5.3.0"
 ,"matplotlib==3.8.0"
 ,"mlflow==2.16.1"
-,"nltk==3.8.1"
-,"numpy==1.26.4"
-,"pandas==2.2.2"
+,"nltk>=3.8.1"
+,"numpy>=1.26.4"
+,"pandas>=2.2.0"
 ,"plotly==5.24.1"
 ,"rdkit==2023.3.3"
 ,"scikit-learn==1.5.2"
 ,"scipy==1.13.1"
-,"accelerate==0.33.0"
-,"sentence-transformers==3.0.1"
-,"transformers==4.42.4"
+,"accelerate>=0.33.0"
+,"sentence-transformers>=3.0.1"
+,"transformers>=4.42.4"
 ,"stanza==1.9.2"
-,"tokenizers==0.19.1"
-,"tqdm==4.66.6"
+,"tokenizers>=0.19.1"
+,"tqdm>=4.66.6"
 ,"xmlschema==3.4.2"
-,"zipp==3.20.2"
+,"zipp>=3.20.2"
 ,"py7zr==0.22.0"
-,"openai==1.52.2"
-,"langchain==0.3.4"
-,"chromadb==0.5.15"
+,"openai>=1.52.2"
+,"langchain>=0.3.4"
+,"chromadb>=0.5.15"
 ,"langchain_community==0.3.3"
 ,"datasets==3.0.2"
 ,"unstructured==0.16.3"

{ddi_fw-0.0.97 → ddi_fw-0.0.98}/src/ddi_fw/datasets/core.py

@@ -268,81 +268,39 @@ class BaseDataset(ABC):
 
 # her bir metin tipi için embedding oluşturursan burayı düzenle
     def prep(self):
-        # if self.embedding_columns:
-        #     zip_helper = ZipHelper()
-        #     zip_helper.extract(str(HERE.joinpath('zips/embeddings')),
-        #                        str(HERE.joinpath('zips/embeddings')))
-
-       # embedding_dict = dict()
-        # for embedding_column in self.embedding_columns:
-        #     embedding_file = HERE.joinpath(
-        #         f'zips/embeddings/{embedding_column}_embeddings.pkl')
-        #     embedding_values = pd.read_pickle(embedding_file)
-        #     d = embedding_values.apply(
-        #         lambda x: {x.id: x[f'{embedding_column}_embedding']}, axis=1)
-        #     x = {k: v for l in d.values.tolist() for k, v in l.items()}
-        #     embedding_dict[embedding_column] = x
-
-        # self.ner_df = CTakesNER().load()
         drug_names = self.drugs_df['name'].to_list()
         drug_ids = self.drugs_df['id'].to_list()
 
-        # self.ddis_df = self.ddis_df[(self.ddis_df['name1'].isin(drug_names)) & (
-        #     self.ddis_df['name2'].isin(drug_names))]
-
         filtered_df = self.drugs_df
-        filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
-            drug_ids)]
-        filtered_ner_df = self.ner_df.copy()
-
         combined_df = filtered_df.copy()
-        # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
-
-        # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
-        idf_calc = IDF(filtered_ner_df, self.ner_columns)
-        idf_calc.calculate()
-        idf_scores_df = idf_calc.to_dataframe()
-
-        # for key in filtered_ner_df.keys():
-        for key in self.ner_columns:
-            threshold = 0
-            if key.startswith('tui'):
-                threshold = self.tui_threshold
-            if key.startswith('cui'):
-                threshold = self.cui_threshold
-            if key.startswith('entities'):
-                threshold = self.entities_threshold
-            combined_df[key] = filtered_ner_df[key]
-            valid_codes = idf_scores_df[idf_scores_df[key] > threshold].index
-
-            # print(f'{key}: valid code size = {len(valid_codes)}')
-            combined_df[key] = combined_df[key].apply(lambda items:
-                                                      [item for item in items if item in valid_codes])
-
-        # Yukarıdaki koda evrildi
-        # combined_df['tui_description'] = filtered_ner_df['description_tuis']
-        # combined_df['cui_description'] = filtered_ner_df['description_cuis']
-        # combined_df['entities_description'] = filtered_ner_df['description_entities']
-
-        # tui_idf = IDF(combined_df['tui_description'], self.tui_threshold)
-        # cui_idf = IDF(combined_df['cui_description'], self.cui_threshold)
-        # entities_idf = IDF(
-        #     combined_df['entities_description'], self.entities_threshold)
-
-        # tui_idf.calculate()
-        # cui_idf.calculate()
-        # entities_idf.calculate()
-
-        # valid_tui_codes = tui_idf.find_items_over_threshold()
-        # valid_cui_codes = cui_idf.find_items_over_threshold()
-        # valid_entities_codes = entities_idf.find_items_over_threshold()
-
-        # combined_df['tui_description'] = combined_df['tui_description'].apply(lambda items:
-        #                                                                       [item for item in items if item in valid_tui_codes])
-        # combined_df['cui_description'] = combined_df['cui_description'].apply(lambda items:
-        #                                                                       [item for item in items if item in valid_cui_codes])
-        # combined_df['entities_description'] = combined_df['entities_description'].apply(lambda items:
-        #                                                                                 [item for item in items if item in valid_entities_codes])
+
+        if self.ner_df:
+            filtered_ner_df = self.ner_df[self.ner_df['drugbank_id'].isin(
+                drug_ids)]
+            filtered_ner_df = self.ner_df.copy()
+
+            # TODO: eğer kullanılan veri setinde tui, cui veya entity bilgileri yoksa o veri setine bu sütunları eklemek için aşağısı gerekli
+
+            # idf_calc = IDF(filtered_ner_df, [f for f in filtered_ner_df.keys()])
+            idf_calc = IDF(filtered_ner_df, self.ner_columns)
+            idf_calc.calculate()
+            idf_scores_df = idf_calc.to_dataframe()
+
+            # for key in filtered_ner_df.keys():
+            for key in self.ner_columns:
+                threshold = 0
+                if key.startswith('tui'):
+                    threshold = self.tui_threshold
+                if key.startswith('cui'):
+                    threshold = self.cui_threshold
+                if key.startswith('entities'):
+                    threshold = self.entities_threshold
+                combined_df[key] = filtered_ner_df[key]
+                valid_codes = idf_scores_df[idf_scores_df[key] > threshold].index
+
+                # print(f'{key}: valid code size = {len(valid_codes)}')
+                combined_df[key] = combined_df[key].apply(lambda items:
+                                                        [item for item in items if item in valid_codes])
 
         moved_columns = ['id']
         moved_columns.extend(self.__similarity_related_columns__)
@@ -409,7 +367,6 @@ class BaseDataset(ABC):
                 x_fnc, args=(embeddings_after_pooling,), axis=1)
 
         self.dataframe = self.ddis_df.copy()
-        self.dataframe['class_as_txt'] = labels
         self.dataframe['class'] = list(classes)
         print(self.dataframe.shape)
 

{ddi_fw-0.0.97 → ddi_fw-0.0.98}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ddi_fw
-Version: 0.0.97
+Version: 0.0.98
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
@@ -27,25 +27,25 @@ Requires-Dist: python-stopwatch==1.1.11
 Requires-Dist: lxml==5.3.0
 Requires-Dist: matplotlib==3.8.0
 Requires-Dist: mlflow==2.16.1
-Requires-Dist: nltk==3.8.1
-Requires-Dist: numpy==1.26.4
-Requires-Dist: pandas==2.2.2
+Requires-Dist: nltk>=3.8.1
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: pandas>=2.2.0
 Requires-Dist: plotly==5.24.1
 Requires-Dist: rdkit==2023.3.3
 Requires-Dist: scikit-learn==1.5.2
 Requires-Dist: scipy==1.13.1
-Requires-Dist: accelerate==0.33.0
-Requires-Dist: sentence-transformers==3.0.1
-Requires-Dist: transformers==4.42.4
+Requires-Dist: accelerate>=0.33.0
+Requires-Dist: sentence-transformers>=3.0.1
+Requires-Dist: transformers>=4.42.4
 Requires-Dist: stanza==1.9.2
-Requires-Dist: tokenizers==0.19.1
-Requires-Dist: tqdm==4.66.6
+Requires-Dist: tokenizers>=0.19.1
+Requires-Dist: tqdm>=4.66.6
 Requires-Dist: xmlschema==3.4.2
-Requires-Dist: zipp==3.20.2
+Requires-Dist: zipp>=3.20.2
 Requires-Dist: py7zr==0.22.0
-Requires-Dist: openai==1.52.2
-Requires-Dist: langchain==0.3.4
-Requires-Dist: chromadb==0.5.15
+Requires-Dist: openai>=1.52.2
+Requires-Dist: langchain>=0.3.4
+Requires-Dist: chromadb>=0.5.15
 Requires-Dist: langchain_community==0.3.3
 Requires-Dist: datasets==3.0.2
 Requires-Dist: unstructured==0.16.3

{ddi_fw-0.0.97 → ddi_fw-0.0.98}/src/ddi_fw.egg-info/requires.txt

@@ -3,25 +3,25 @@ python-stopwatch==1.1.11
 lxml==5.3.0
 matplotlib==3.8.0
 mlflow==2.16.1
-nltk==3.8.1
-numpy==1.26.4
-pandas==2.2.2
+nltk>=3.8.1
+numpy>=1.26.4
+pandas>=2.2.0
 plotly==5.24.1
 rdkit==2023.3.3
 scikit-learn==1.5.2
 scipy==1.13.1
-accelerate==0.33.0
-sentence-transformers==3.0.1
-transformers==4.42.4
+accelerate>=0.33.0
+sentence-transformers>=3.0.1
+transformers>=4.42.4
 stanza==1.9.2
-tokenizers==0.19.1
-tqdm==4.66.6
+tokenizers>=0.19.1
+tqdm>=4.66.6
 xmlschema==3.4.2
-zipp==3.20.2
+zipp>=3.20.2
 py7zr==0.22.0
-openai==1.52.2
-langchain==0.3.4
-chromadb==0.5.15
+openai>=1.52.2
+langchain>=0.3.4
+chromadb>=0.5.15
 langchain_community==0.3.3
 datasets==3.0.2
 unstructured==0.16.3