ddi-fw 0.0.227__py3-none-any.whl → 0.0.230__py3-none-any.whl

ddi_fw/datasets/__init__.py

@@ -1,4 +1,4 @@
-from .core import BaseDataset
+from .core import BaseDataset,TextDatasetMixin
 from .ddi_mdl.base import DDIMDLDataset 
 from .ddi_mdl_text.base import DDIMDLDatasetV2
 from .mdf_sa_ddi.base import MDFSADDIDataset

ddi_fw/pipeline/ner_pipeline.py

@@ -81,28 +81,34 @@ class NerParameterSearch(BaseModel):
             df = idf.to_dataframe()
             self.min_threshold_dict = {key: np.floor(df.describe()[key]["min"]) for key in df.describe().keys()}
             self.max_threshold_dict = {key: np.ceil(df.describe()[key]["max"]) for key in df.describe().keys()}
+    
+        print("Minimum thresholds:", self.min_threshold_dict)
+        print("Maximum thresholds:", self.max_threshold_dict)
 
         # Generate datasets and items
         for column in self.columns:
             min_threshold = self.min_threshold_dict[column]
             max_threshold = self.max_threshold_dict[column]
-            kwargs = {
+            thresholds = {
                 "threshold_method": "idf",
-                "tui_threshold": 0,
-                "cui_threshold": 0,
-                "entities_threshold": 0,
+                "tui": 0,
+                "cui": 0,
+                "entities": 0,
             }
             if self.dataset_additional_config:
-                kwargs["additional_config"]= self.dataset_additional_config
+                kwargs=  self.dataset_additional_config
 
             for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
                 if column.startswith("tui"):
-                    kwargs["tui_threshold"] = threshold
+                    thresholds["tui"] = threshold
                 if column.startswith("cui"):
-                    kwargs["cui_threshold"] = threshold
+                    thresholds["cui"] = threshold
                 if column.startswith("entities"):
-                    kwargs["entities_threshold"] = threshold
+                    thresholds["entities"] = threshold
+                kwargs['ner']['thresholds'] = thresholds
                 
+                print(f"Loading dataset for column: {column} with threshold: {threshold}")
+                # Create a new dataset instance for each threshold
                 dataset = self.dataset_type(
                     columns=[column],
                     dataset_splitter_type=self.dataset_splitter_type,

ddi_fw/vectorization/feature_vector_generation.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 from scipy.spatial.distance import pdist, squareform
+from sklearn.preprocessing import MultiLabelBinarizer
 
 # todo pd.unique kullan
 def find_distinct_elements(frame):
@@ -29,13 +30,13 @@ class SimilarityMatrixGenerator:
     def __init__(self):
         pass
 
-    def create_jaccard_similarity_matrices_ex(self, array):
+    def create_jaccard_similarity_matrices_ex_1(self, array):
         jaccard_sim = 1 - pdist(array, metric='jaccard')
         jaccard_sim_matrix = squareform(jaccard_sim)
         return jaccard_sim_matrix
 
     # https://github.com/YifanDengWHU/DDIMDL/blob/master/DDIMDL.py , def Jaccard(matrix):
-    def create_jaccard_similarity_matrices(self, matrix)->np.ndarray:
+    def create_jaccard_similarity_matrices_ex_2(self, matrix)->np.ndarray:
         matrix = np.asmatrix(matrix)
         numerator = matrix * matrix.T
         denominator = np.ones(np.shape(matrix)) * matrix.T + \
@@ -43,6 +44,37 @@ class SimilarityMatrixGenerator:
         matrix = numerator / denominator
         return np.nan_to_num(matrix, nan=0.0)
         # return matrix
+        
+    """produced from ChatGPT"""
+    def create_jaccard_similarity_matrices(self, matrix)->np.ndarray:
+        """
+        Efficiently compute the Jaccard similarity between rows of a binary matrix using vectorized operations.
+
+        Parameters:
+        matrix (np.ndarray): A 2D binary NumPy array (only 0s and 1s).
+
+        Returns:
+        np.ndarray: A 2D NumPy array containing the pairwise Jaccard similarity.
+        """
+        if not ((matrix == 0) | (matrix == 1)).all():
+            raise ValueError("Input matrix must be binary (contain only 0s and 1s).")
+
+        # Intersection: dot product (each pair of rows)
+        intersection = matrix @ matrix.T
+
+        # Row-wise sum (number of 1s per row)
+        row_sums = matrix.sum(axis=1, keepdims=True)
+
+        # Union: |A ∪ B| = |A| + |B| - |A ∩ B|
+        union = row_sums + row_sums.T - intersection
+
+        # Avoid division by zero
+        with np.errstate(divide='ignore', invalid='ignore'):
+            similarity = intersection / union
+            similarity[np.isnan(similarity)] = 1.0  # If both rows are all zeros, define similarity as 1
+
+        return similarity
+
 
 
 class VectorGenerator:
@@ -99,6 +131,7 @@ class VectorGenerator:
             bit_vectors.append(vector)
         print("array oluşturuldu")
         return np.array(bit_vectors)
+     
     
     # def generate_feature_vector(self, column):
     #     bit_vectors = []
@@ -120,6 +153,7 @@ class VectorGenerator:
             bit_vectors = self.generate_feature_vector(column)
             vectors[column] = bit_vectors
         return vectors
+    
 
 
 # generate feature vector

{ddi_fw-0.0.227.dist-info → ddi_fw-0.0.230.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.227
+Version: 0.0.230
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.227.dist-info → ddi_fw-0.0.230.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ddi_fw/datasets/__init__.py,sha256=_I3iDHARwzmg7_EL5XKtB_TgG1yAkLSOVTujLL9Wz9Q,280
+ddi_fw/datasets/__init__.py,sha256=qsiNp0JJz2djukwtBpcdryUPIXkBOxDA6whryu_jt04,297
 ddi_fw/datasets/core.py,sha256=p-e3wP5C_SCh0fMXioUHUXKvLVtyCrsQCFvKRnH4fjs,17008
 ddi_fw/datasets/dataset_splitter.py,sha256=8H8uZTAf8N9LUZeSeHOMawtJFJhnDgUUqFcnl7dquBQ,1672
 ddi_fw/datasets/db_utils.py,sha256=xRj28U_uXTRPHcz3yIICczFUHXUPiAOZtAj5BM6kH44,6465
@@ -86,7 +86,7 @@ ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,2
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=EjJnA3Vzd-WeEvUBaA2LDOy_iQ5-2eW2VhtxvvxDPfQ,9857
 ddi_fw/pipeline/multi_pipeline_org.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
-ddi_fw/pipeline/ner_pipeline.py,sha256=IVtmlBhQ73FeR0b26U33yWlNVwqiEqdvBAseTz6CVsk,6954
+ddi_fw/pipeline/ner_pipeline.py,sha256=8PKTnpD2jw9AlpLScwmlvafCm73hOmcbzAkrxNNR7tE,7240
 ddi_fw/pipeline/pipeline.py,sha256=q1kMkW9-fOlrA4BOGUku40U_PuEYfcbtH2EvlRM4uTM,6243
 ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
@@ -99,9 +99,9 @@ ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,
 ddi_fw/utils/utils.py,sha256=PY-zDawREKoXQfzX7lVkxBLVFQPkfvr9385kHCjaNXo,4391
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
-ddi_fw/vectorization/feature_vector_generation.py,sha256=EBf-XAiwQwr68az91erEYNegfeqssBR29kVgrliIyac,4765
+ddi_fw/vectorization/feature_vector_generation.py,sha256=93G3QM28uoNlvlVz_BhV6ARxldpogiNJStxHdsgqTbU,6026
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.227.dist-info/METADATA,sha256=yVVPcTBE4VRLFs4K7jWuOQWoLe-B_i8c8BV1YJCjI7U,2632
-ddi_fw-0.0.227.dist-info/WHEEL,sha256=lTU6B6eIfYoiQJTZNc-fyaR6BpL6ehTzU3xGYxn2n8k,91
-ddi_fw-0.0.227.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.227.dist-info/RECORD,,
+ddi_fw-0.0.230.dist-info/METADATA,sha256=X5O1lK9xbfCA5wvuWOWGPWJ_Jkpad8b99cd2Wos1A6E,2632
+ddi_fw-0.0.230.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+ddi_fw-0.0.230.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.230.dist-info/RECORD,,

{ddi_fw-0.0.227.dist-info → ddi_fw-0.0.230.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.1)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any