ddi-fw 0.0.228__tar.gz → 0.0.231__tar.gz

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.228
+Version: 0.0.231
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ddi_fw"
-version = "0.0.228"
+version = "0.0.231"
 description = "Do not use :)"
 readme = "README.md"
 authors = [

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/src/ddi_fw/datasets/__init__.py

@@ -1,4 +1,4 @@
-from .core import BaseDataset
+from .core import BaseDataset,TextDatasetMixin
 from .ddi_mdl.base import DDIMDLDataset 
 from .ddi_mdl_text.base import DDIMDLDatasetV2
 from .mdf_sa_ddi.base import MDFSADDIDataset

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/src/ddi_fw/pipeline/ner_pipeline.py

@@ -89,23 +89,26 @@ class NerParameterSearch(BaseModel):
         for column in self.columns:
             min_threshold = self.min_threshold_dict[column]
             max_threshold = self.max_threshold_dict[column]
-            kwargs = {
+            thresholds = {
                 "threshold_method": "idf",
-                "tui_threshold": 0,
-                "cui_threshold": 0,
-                "entities_threshold": 0,
+                "tui": 0,
+                "cui": 0,
+                "entities": 0,
             }
             if self.dataset_additional_config:
-                kwargs["additional_config"]= self.dataset_additional_config
+                additional_config=  self.dataset_additional_config
+            else:
+                additional_config={}
 
             for threshold in np.arange(min_threshold, max_threshold, self.increase_step):
                 if column.startswith("tui"):
-                    kwargs["tui_threshold"] = threshold
+                    thresholds["tui"] = threshold
                 if column.startswith("cui"):
-                    kwargs["cui_threshold"] = threshold
+                    thresholds["cui"] = threshold
                 if column.startswith("entities"):
-                    kwargs["entities_threshold"] = threshold
-                
+                    thresholds["entities"] = threshold
+                additional_config['ner']['thresholds'] = thresholds
+                kwargs = {'additional_config': additional_config}
                 print(f"Loading dataset for column: {column} with threshold: {threshold}")
                 # Create a new dataset instance for each threshold
                 dataset = self.dataset_type(

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/src/ddi_fw/vectorization/feature_vector_generation.py

@@ -1,6 +1,7 @@
 import numpy as np
 import pandas as pd
 from scipy.spatial.distance import pdist, squareform
+from sklearn.preprocessing import MultiLabelBinarizer
 
 # todo pd.unique kullan
 def find_distinct_elements(frame):
@@ -29,13 +30,13 @@ class SimilarityMatrixGenerator:
     def __init__(self):
         pass
 
-    def create_jaccard_similarity_matrices_ex(self, array):
+    def create_jaccard_similarity_matrices_ex_1(self, array):
         jaccard_sim = 1 - pdist(array, metric='jaccard')
         jaccard_sim_matrix = squareform(jaccard_sim)
         return jaccard_sim_matrix
 
     # https://github.com/YifanDengWHU/DDIMDL/blob/master/DDIMDL.py , def Jaccard(matrix):
-    def create_jaccard_similarity_matrices(self, matrix)->np.ndarray:
+    def create_jaccard_similarity_matrices_ex_2(self, matrix)->np.ndarray:
         matrix = np.asmatrix(matrix)
         numerator = matrix * matrix.T
         denominator = np.ones(np.shape(matrix)) * matrix.T + \
@@ -43,6 +44,37 @@ class SimilarityMatrixGenerator:
         matrix = numerator / denominator
         return np.nan_to_num(matrix, nan=0.0)
         # return matrix
+        
+    """produced from ChatGPT"""
+    def create_jaccard_similarity_matrices(self, matrix)->np.ndarray:
+        """
+        Efficiently compute the Jaccard similarity between rows of a binary matrix using vectorized operations.
+
+        Parameters:
+        matrix (np.ndarray): A 2D binary NumPy array (only 0s and 1s).
+
+        Returns:
+        np.ndarray: A 2D NumPy array containing the pairwise Jaccard similarity.
+        """
+        if not ((matrix == 0) | (matrix == 1)).all():
+            raise ValueError("Input matrix must be binary (contain only 0s and 1s).")
+
+        # Intersection: dot product (each pair of rows)
+        intersection = matrix @ matrix.T
+
+        # Row-wise sum (number of 1s per row)
+        row_sums = matrix.sum(axis=1, keepdims=True)
+
+        # Union: |A ∪ B| = |A| + |B| - |A ∩ B|
+        union = row_sums + row_sums.T - intersection
+
+        # Avoid division by zero
+        with np.errstate(divide='ignore', invalid='ignore'):
+            similarity = intersection / union
+            similarity[np.isnan(similarity)] = 1.0  # If both rows are all zeros, define similarity as 1
+
+        return similarity
+
 
 
 class VectorGenerator:
@@ -99,6 +131,7 @@ class VectorGenerator:
             bit_vectors.append(vector)
         print("array oluşturuldu")
         return np.array(bit_vectors)
+     
     
     # def generate_feature_vector(self, column):
     #     bit_vectors = []
@@ -120,6 +153,7 @@ class VectorGenerator:
             bit_vectors = self.generate_feature_vector(column)
             vectors[column] = bit_vectors
         return vectors
+    
 
 
 # generate feature vector

{ddi_fw-0.0.228 → ddi_fw-0.0.231}/src/ddi_fw.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.228
+Version: 0.0.231
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>