ddi-fw 0.0.232__py3-none-any.whl → 0.0.234__py3-none-any.whl

ddi_fw/pipeline/ner_pipeline.py

@@ -130,7 +130,7 @@ class NerParameterSearch(BaseModel):
 
                 for item in group_items:
                     item[0] = f"threshold_{item[0]}_{threshold}"
-                    self.datasets[item[0]] = dataset
+                    # self.datasets[item[0]] = dataset
 
                 self.items.extend(group_items)
                 

ddi_fw/vectorization/feature_vector_generation.py

@@ -1,3 +1,4 @@
+import os
 import numpy as np
 import pandas as pd
 from scipy.spatial.distance import pdist, squareform
@@ -28,7 +29,8 @@ def find_distinct_elements_count(frame):
 
 class SimilarityMatrixGenerator:
     def __init__(self):
-        pass
+        # Check if GPU usage is enabled via an environment variable
+        self.use_gpu = os.getenv("SIMILARITY_MATRIX_USE_GPU", "false").lower() == "true"
 
     def create_jaccard_similarity_matrices_ex_1(self, array):
         jaccard_sim = 1 - pdist(array, metric='jaccard')
@@ -45,8 +47,15 @@ class SimilarityMatrixGenerator:
         return np.nan_to_num(matrix, nan=0.0)
         # return matrix
         
+        
+    def create_jaccard_similarity_matrices(self, matrix: np.ndarray)->np.ndarray:
+        if self.use_gpu:
+            return self.__create_jaccard_similarity_matrices_gpu(matrix)
+        else:
+            return self.__create_jaccard_similarity_matrices(matrix)
+        
     """produced from ChatGPT"""
-    def create_jaccard_similarity_matrices(self, matrix)->np.ndarray:
+    def __create_jaccard_similarity_matrices(self, matrix: np.ndarray)->np.ndarray:
         """
         Efficiently compute the Jaccard similarity between rows of a binary matrix using vectorized operations.
 
@@ -75,6 +84,35 @@ class SimilarityMatrixGenerator:
 
         return similarity
 
+    import cupy as cp
+
+    def __create_jaccard_similarity_matrices_gpu(self,matrix: np.ndarray) -> np.ndarray:
+        """
+        Efficiently compute the Jaccard similarity between rows of a binary matrix using GPU-accelerated CuPy.
+
+        Parameters:
+        matrix (cp.ndarray): A 2D binary CuPy array (only 0s and 1s).
+
+        Returns:
+        cp.ndarray: A 2D CuPy array containing the pairwise Jaccard similarity.
+        """
+        if not ((matrix == 0) | (matrix == 1)).all():
+            raise ValueError("Input matrix must be binary (contain only 0s and 1s).")
+        matrix = cp.asarray(matrix)
+        # Intersection: dot product (each pair of rows)
+        intersection = matrix @ matrix.T
+
+        # Row-wise sum (number of 1s per row)
+        row_sums = matrix.sum(axis=1, keepdims=True)
+
+        # Union: |A ∪ B| = |A| + |B| - |A ∩ B|
+        union = row_sums + row_sums.T - intersection
+
+        # Avoid division by zero
+        similarity = cp.divide(intersection, union, out=cp.ones_like(intersection, dtype=cp.float64)) #, where=union != 0
+
+        return cp.asnumpy(similarity)
+
 
 
 class VectorGenerator:

{ddi_fw-0.0.232.dist-info → ddi_fw-0.0.234.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ddi_fw
-Version: 0.0.232
+Version: 0.0.234
 Summary: Do not use :)
 Author-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>
 Maintainer-email: Kıvanç Bayraktar <bayraktarkivanc@gmail.com>

{ddi_fw-0.0.232.dist-info → ddi_fw-0.0.234.dist-info}/RECORD

@@ -86,7 +86,7 @@ ddi_fw/pipeline/__init__.py,sha256=tKDM_rW4vPjlYTeOkNgi9PujDzb4e9O3LK1w5wqnebw,2
 ddi_fw/pipeline/multi_modal_combination_strategy.py,sha256=JSyuP71b1I1yuk0s2ecCJZTtCED85jBtkpwTUxibJvI,1706
 ddi_fw/pipeline/multi_pipeline.py,sha256=EjJnA3Vzd-WeEvUBaA2LDOy_iQ5-2eW2VhtxvvxDPfQ,9857
 ddi_fw/pipeline/multi_pipeline_org.py,sha256=AbErwu05-3YIPnCcXRsj-jxPJG8HG2H7cMZlGjzaYa8,9037
-ddi_fw/pipeline/ner_pipeline.py,sha256=BycxZvI7JRJ3s3HhYAgOxG2_lqrVnhv7ECOWSgVQhz4,8186
+ddi_fw/pipeline/ner_pipeline.py,sha256=1gBk81LeZlU1rhjJ1qBgHbFt_HqOeJ5WLnJ4AkYku4s,8188
 ddi_fw/pipeline/pipeline.py,sha256=q1kMkW9-fOlrA4BOGUku40U_PuEYfcbtH2EvlRM4uTM,6243
 ddi_fw/utils/__init__.py,sha256=WNxkQXk-694roG50D355TGLXstfdWVb_tUyr-PM-8rg,537
 ddi_fw/utils/categorical_data_encoding_checker.py,sha256=T1X70Rh4atucAuqyUZmz-iFULllY9dY0NRyV9-jTjJ0,3438
@@ -99,9 +99,9 @@ ddi_fw/utils/py7zr_helper.py,sha256=gOqaFIyJvTjUM-btO2x9AQ69jZOS8PoKN0wetYIckJw,
 ddi_fw/utils/utils.py,sha256=PY-zDawREKoXQfzX7lVkxBLVFQPkfvr9385kHCjaNXo,4391
 ddi_fw/utils/zip_helper.py,sha256=YRZA4tKZVBJwGQM0_WK6L-y5MoqkKoC-nXuuHK6CU9I,5567
 ddi_fw/vectorization/__init__.py,sha256=LcJOpLVoLvHPDw9phGFlUQGeNcST_zKV-Oi1Pm5h_nE,110
-ddi_fw/vectorization/feature_vector_generation.py,sha256=93G3QM28uoNlvlVz_BhV6ARxldpogiNJStxHdsgqTbU,6026
+ddi_fw/vectorization/feature_vector_generation.py,sha256=-M8Y82mKZB6kwhJHvPgmQZSwUCheTLbb5zYuEtHZC1Y,7623
 ddi_fw/vectorization/idf_helper.py,sha256=_Gd1dtDSLaw8o-o0JugzSKMt9FpeXewTh4wGEaUd4VQ,2571
-ddi_fw-0.0.232.dist-info/METADATA,sha256=CBSE9xsWEc0vxlCxw9NCLxTJWXmXoFTegN5wgrHVGvA,2632
-ddi_fw-0.0.232.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-ddi_fw-0.0.232.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
-ddi_fw-0.0.232.dist-info/RECORD,,
+ddi_fw-0.0.234.dist-info/METADATA,sha256=DxrU3PdEv-7FKmZ3bJ4OAI465EsN-sKaxDnTQSTO19A,2632
+ddi_fw-0.0.234.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+ddi_fw-0.0.234.dist-info/top_level.txt,sha256=PMwHICFZTZtcpzQNPV4UQnfNXYIeLR_Ste-Wfc1h810,7
+ddi_fw-0.0.234.dist-info/RECORD,,