dcs-sdk 1.6.5__py3-none-any.whl

dcs_sdk/sdk/rules/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from .rules_repository import RulesRepository

dcs_sdk/sdk/rules/rules_mappping.py

@@ -0,0 +1,31 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from dcs_sdk.sdk.rules.schema_rules import (
+    allow_equivalent_data_types,
+    ignore_column_length_difference,
+    ignore_datetime_precision_difference,
+    ignore_numeric_precision_difference,
+    ignore_numeric_scale_difference,
+)
+
+
+def get_rules_to_func_mapping():
+    return {
+        "ignore_column_length_difference": ignore_column_length_difference,
+        "allow_equivalent_data_types": allow_equivalent_data_types,
+        "ignore_numeric_precision_difference": ignore_numeric_precision_difference,
+        "ignore_numeric_scale_difference": ignore_numeric_scale_difference,
+        "ignore_datetime_precision_difference": ignore_datetime_precision_difference,
+    }

dcs_sdk/sdk/rules/rules_repository.py

@@ -0,0 +1,214 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import copy
+from collections import defaultdict
+from typing import Optional
+from uuid import UUID
+
+from loguru import logger
+
+from dcs_sdk.sdk.rules.rules_mappping import get_rules_to_func_mapping
+
+
+def map_str_type_to_generic_type(c_type):
+    c_type = c_type.lower() if c_type else str(c_type)
+    if any(
+        c_type.startswith(prefix)
+        for prefix in [
+            "int",
+            "integer",
+            "bigint",
+            "smallint",
+            "tinyint",
+            "float",
+            "double",
+            "real",
+            "numeric",
+            "decimal",
+            "money",
+            "smallmoney",
+            "number",
+        ]
+    ):
+        return "numeric"
+
+    elif any(
+        c_type.startswith(prefix)
+        for prefix in [
+            "string",
+            "varchar",
+            "char",
+            "text",
+            "str",
+            "character varying",
+            "character",
+            "nvar",
+            "nchar",
+        ]
+    ):
+        return "string"
+    elif any(c_type.startswith(prefix) for prefix in ["uuid", "uniqueidentifier", "guid"]):
+        return "uuid"
+    elif any(
+        c_type.startswith(prefix)
+        for prefix in [
+            "date",
+            "time",
+            "timestamp",
+            "datetime",
+            "timestamp with time zone",
+            "timestamp without time zone",
+            "smalldatetime",
+        ]
+    ):
+        return "datetime"
+    elif any(c_type.startswith(prefix) for prefix in ["json", "jsonb", "dict", "map"]):
+        return "json"
+    elif any(c_type.startswith(prefix) for prefix in ["array", "list"]):
+        return "array"
+    elif any(c_type.startswith(prefix) for prefix in ["binary", "blob"]):
+        return "binary"
+    elif any(c_type.startswith(prefix) for prefix in ["bytea"]):
+        return "bytea"
+    elif any(c_type.startswith(prefix) for prefix in ["boolean", "bool"]):
+        return "boolean"
+    else:
+        return c_type
+
+
+# CENTRALIZED REPO FOR ALL THE RULES
+# RULE NAME -> ALL ITS PROPERTIES
+class RulesRepository:
+    _INSTANCE = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._INSTANCE is None:
+            cls._INSTANCE = cls()
+        return cls._INSTANCE
+
+    def __init__(self):
+        self.rules = {}
+        self.rules_mapping = get_rules_to_func_mapping()
+        self.value_rules = []
+        self.schema_rules = {}
+
+    def register(self, id: UUID, rule_dict: dict):
+        """
+        Registers a rule in the centralized repository.
+        Supports both 'schema' and 'value' rules.
+
+        - For value rules: transformation is a string template
+        - For schema rules: transformation is a function name (to be resolved)
+        """
+        rule_type = rule_dict.get("type")
+
+        if not rule_type:
+            return
+
+        if rule_type == "schema_override":
+            func_name = rule_dict.get("transformation")
+            if func_name:
+                func = self.rules_mapping.get(func_name)
+                if not func:
+                    raise ValueError(f"Function '{func_name}' not found in registry")
+                rule_dict["function"] = func
+                rule_dict["function_name"] = func_name
+
+        self.rules[id] = rule_dict
+
+    def register_schema_rules(self, schema_rules: list):
+        self.schema_rules = schema_rules
+
+    def register_value_rules(self, value_rules: list):
+        self.value_rules = value_rules
+
+    def get(self, id: UUID) -> Optional[dict]:
+        return self.rules.get(id)
+
+    def apply_schema_rules(self, src_col: dict, tgt_col: dict) -> tuple[bool, str | None]:
+        """
+        Performs baseline schema checks and overrides them if corresponding rules are configured.
+        Returns (True, None) if all checks pass.
+        Returns (False, reason) if any check fails.
+        """
+
+        schema_rule_function_mapping = defaultdict(list)
+
+        for schema_rule in self.schema_rules:
+            rule_obj = self.get(schema_rule)
+            if rule_obj:
+                func_name = rule_obj.get("function_name")
+                params = rule_obj.get("params")
+                function = rule_obj.get("function")
+
+                schema_rule_function_mapping[func_name].append({"params": params, "func": function})
+
+        # def is_rule_allowed(rule_name: str, fallback_check: bool) -> bool:
+        #     if rule_name not in schema_rule_function_mapping:
+        #         return fallback_check
+        #     rule_obj = schema_rule_function_mapping[rule_name]
+        #     func = rule_obj.get("func")
+        #     params = rule_obj.get("params")
+        #     return func(src_col, tgt_col, params) if func else fallback_check
+
+        def is_rule_allowed(rule_name: str, fallback_check: bool) -> bool:
+            if rule_name not in schema_rule_function_mapping:
+                return fallback_check
+
+            rule_objs = schema_rule_function_mapping[rule_name]
+            if not rule_objs:
+                return fallback_check
+
+            for rule_obj in rule_objs:
+                func = rule_obj.get("func")
+                params = rule_obj.get("params")
+
+                if func:
+                    result = func(src_col, tgt_col, params)
+                    if result:
+                        return True
+
+            return fallback_check
+
+        source_generic_data_type = map_str_type_to_generic_type(src_col["data_type"].lower())
+        tgt_generic_data_type = map_str_type_to_generic_type(tgt_col["data_type"].lower())
+
+        if src_col["data_type"].lower() != tgt_col["data_type"].lower():
+            if not is_rule_allowed("allow_equivalent_data_types", False):
+                return False, f"Data type mismatch"
+
+            return True, None
+
+        if source_generic_data_type == "string" and tgt_generic_data_type == "string":
+            if src_col.get("character_maximum_length") != tgt_col.get("character_maximum_length"):
+                if not is_rule_allowed("ignore_column_length_difference", False):
+                    return False, f"Length mismatch"
+
+        if source_generic_data_type == "numeric" and tgt_generic_data_type == "numeric":
+            if src_col.get("numeric_precision") != tgt_col.get("numeric_precision"):
+                if not is_rule_allowed("ignore_numeric_precision_difference", False):
+                    return False, f"Numeric precision mismatch"
+
+            if src_col.get("numeric_scale") != tgt_col.get("numeric_scale"):
+                if not is_rule_allowed("ignore_numeric_scale_difference", False):
+                    return False, f"Numeric scale mismatch"
+
+        if source_generic_data_type == "datetime" and tgt_generic_data_type == "datetime":
+            if src_col.get("datetime_precision") != tgt_col.get("datetime_precision"):
+                if not is_rule_allowed("ignore_datetime_precision_difference", False):
+                    return False, f"Datetime precision mismatch"
+
+        return True, None

dcs_sdk/sdk/rules/schema_rules.py

@@ -0,0 +1,65 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from typing import Dict, Optional
+
+
+def ignore_column_length_difference(
+    src_col: Dict,
+    tgt_col: Dict,
+    params: Optional[Dict] = None,
+) -> bool:
+    if not params:
+        # IN THIS CASE ALWAYS RETURN TRUE
+        return True
+
+    # IF PARAMS THEN PROCESS
+    # FOR EG: MAX_LENGTH_DIFF = 30 SO IN THIS CASE WE CAN CACL THE DIFF AND RETURN APPROPRIATE RESPONSE
+
+    return False
+
+
+def allow_equivalent_data_types(src_col: Dict, tgt_col: Dict, params: Optional[Dict] = None) -> bool:
+
+    src_type = src_col["data_type"].lower()
+    tgt_type = tgt_col["data_type"].lower()
+
+    if params and "equivalent_groups" in params:
+        for group in params["equivalent_groups"]:
+            group_set = {t.lower() for t in group}
+            if src_type in group_set and tgt_type in group_set:
+                return True
+
+    return False
+
+
+def ignore_numeric_precision_difference(src_col: Dict, tgt_col: Dict, params: Optional[Dict] = None) -> bool:
+    if not params:
+        return True
+
+    return False
+
+
+def ignore_numeric_scale_difference(src_col: Dict, tgt_col: Dict, params: Optional[Dict] = None) -> bool:
+    if not params:
+        return True
+
+    return False
+
+
+def ignore_datetime_precision_difference(src_col: Dict, tgt_col: Dict, params: Optional[Dict] = None) -> bool:
+    if not params:
+        return True
+
+    return False

dcs_sdk/sdk/utils/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_sdk/sdk/utils/serializer.py

@@ -0,0 +1,25 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+def serialize_table_schema(info) -> dict:
+    return {
+        "column_name": info.column_name,
+        "data_type": info.data_type,
+        "datetime_precision": info.datetime_precision,
+        "numeric_precision": info.numeric_precision,
+        "numeric_scale": info.numeric_scale,
+        "collation_name": info.collation_name,
+        "character_maximum_length": info.character_maximum_length,
+    }

dcs_sdk/sdk/utils/similarity_score/__init__.py

@@ -0,0 +1,13 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

dcs_sdk/sdk/utils/similarity_score/base_provider.py

@@ -0,0 +1,153 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import string
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from typing import Any, Dict, List
+
+import nltk
+from dotenv import load_dotenv
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize
+
+from dcs_sdk.sdk.config.config_loader import SimilarityConfig
+
+
+def ensure_nltk_data():
+    load_dotenv()
+    nltk_data_dir = os.getenv("NLTK_DATA_DIR")
+
+    if not nltk_data_dir:
+        default_root = os.path.dirname(os.path.abspath(__file__))
+        nltk_data_dir = os.path.join(default_root, "nltk_data")
+        print(f"NLTK_DATA_DIR ENV variable not set. Using default path: {nltk_data_dir}")
+
+    punkt_path = os.path.join(nltk_data_dir, "tokenizers", "punkt")
+    stopwords_path = os.path.join(nltk_data_dir, "corpora", "stopwords")
+    punkt_tab_path = os.path.join(nltk_data_dir, "tokenizers", "punkt_tab")
+
+    if not os.path.exists(punkt_path):
+        nltk.download(
+            "punkt",
+            download_dir=nltk_data_dir,
+            halt_on_error=True,
+            raise_on_error=True,
+        )
+    if not os.path.exists(stopwords_path):
+        nltk.download(
+            "stopwords",
+            download_dir=nltk_data_dir,
+            halt_on_error=True,
+            raise_on_error=True,
+        )
+    if not os.path.exists(punkt_tab_path):
+        nltk.download(
+            "punkt_tab",
+            download_dir=nltk_data_dir,
+            halt_on_error=True,
+            raise_on_error=True,
+        )
+
+    nltk.data.path.append(nltk_data_dir)
+
+
+class SimilarityScoreProvider(ABC):
+    def preprocess_text(self, text: str, methods: list[str]) -> set:
+        """Applies preprocessing steps dynamically before tokenization."""
+        if "lower_case" in methods:
+            text = text.lower()
+        if "remove_punctuation" in methods:
+            text = text.translate(str.maketrans("", "", string.punctuation))
+        if "remove_stop_words" in methods:
+            stop_words = set(stopwords.words("english"))
+            text = " ".join(word for word in text.split() if word not in stop_words)
+        if "remove_extra_whitespaces" in methods:
+            text = " ".join(text.split())
+
+        tokens = set(word_tokenize(text))
+        return tokens
+
+    @abstractmethod
+    def fuzzy_match(self, str1: set, str2: set) -> float:
+        """Computes a similarity score between two sets of tokens."""
+        pass
+
+    def add_text_similarity(
+        self,
+        key: List[str],
+        data: List[Dict[str, Any]],
+        fields: List[str],
+        similarity: SimilarityConfig,
+        source_masking_cols: List[str],
+        target_masking_cols: List[str],
+        mask_char: str,
+    ) -> List[Dict[str, Any]]:
+        """Adds text similarity scores for the given fields inside the meta dictionary
+        and determines if they are a match based on the threshold.
+
+        Args:
+            key (List[str]): List of primary key column names to form a composite key.
+            data (List[Dict[str, Any]]): List of records to process.
+            fields (List[str]): List of fields to compute similarity scores for.
+            similarity (SimilarityConfig): Configuration object with pre_processing and threshold.
+            source_masking_cols (List[str]): List of cols from the source to be masked
+            target_masking_cols (List[str]): List of cols from the target to be masked
+            mask_char (str): Character to be used for masking (Default = "*")
+
+        Returns:
+            List[Dict[str, Any]]: Processed records with similarity scores added.
+        """
+        grouped_data = defaultdict(list)
+        for item in data:
+            composite_key = "_".join(str(item[k]) for k in key)
+            grouped_data[composite_key].append(item)
+
+        for _, items in grouped_data.items():
+            if len(items) == 2:
+                source, target = items
+
+                source.setdefault("meta", {}).setdefault("scores", {})
+                target.setdefault("meta", {}).setdefault("scores", {})
+
+                for field in fields:
+                    source_text = self.preprocess_text(source.get(field, ""), similarity.pre_processing)
+                    target_text = self.preprocess_text(target.get(field, ""), similarity.pre_processing)
+                    similarity_score = self.fuzzy_match(source_text, target_text)
+
+                    match_status = "match" if similarity_score >= similarity.threshold else "not matched"
+
+                    score_key = f"{field}"
+                    source["meta"]["scores"][score_key] = {"score": similarity_score, "status": match_status}
+                    target["meta"]["scores"][score_key] = {"score": similarity_score, "status": match_status}
+
+                    source_val = str(source.get(field, ""))
+                    target_val = str(target.get(field, ""))
+
+                    if field in source_masking_cols and field in target_masking_cols:
+                        if len(source_val) == len(target_val) and match_status == "not matched":
+                            source[field] = mask_char * (len(source_val) + 1)
+                            target[field] = mask_char * (len(target_val))
+                        else:
+                            source[field] = mask_char * (len(source_val))
+                            target[field] = mask_char * (len(target_val))
+
+                    if field in source_masking_cols:
+                        source[field] = mask_char * len(source_val)
+
+                    if field in target_masking_cols:
+                        target[field] = mask_char * len(target_val)
+
+        return data

dcs_sdk/sdk/utils/similarity_score/cosine_similarity_provider.py

@@ -0,0 +1,39 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import math
+from collections import Counter
+
+from dcs_sdk.sdk.utils.similarity_score.base_provider import SimilarityScoreProvider
+
+
+class CosineSimilarityProvider(SimilarityScoreProvider):
+    def fuzzy_match(self, tokens1: set, tokens2: set) -> float:
+        """Computes cosine similarity between two sets of tokens."""
+        if not tokens1 or not tokens2:
+            return 0.0
+
+        freq1 = Counter(tokens1)
+        freq2 = Counter(tokens2)
+
+        all_words = set(freq1.keys()).union(set(freq2.keys()))
+
+        dot_product = sum(freq1[word] * freq2[word] for word in all_words)
+        magnitude1 = math.sqrt(sum(freq1[word] ** 2 for word in all_words))
+        magnitude2 = math.sqrt(sum(freq2[word] ** 2 for word in all_words))
+
+        if magnitude1 == 0 or magnitude2 == 0:
+            return 0.0
+
+        return round(dot_product / (magnitude1 * magnitude2), 4)

dcs_sdk/sdk/utils/similarity_score/jaccard_provider.py

@@ -0,0 +1,24 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+from dcs_sdk.sdk.utils.similarity_score.base_provider import SimilarityScoreProvider
+
+
+class JaccardSimilarityProvider(SimilarityScoreProvider):
+    def fuzzy_match(self, set1: set, set2: set) -> float:
+        """Computes the Jaccard similarity between two sets of tokens."""
+        intersection = len(set1 & set2)
+        union = len(set1 | set2)
+        return intersection / union if union != 0 else 0.0

dcs_sdk/sdk/utils/similarity_score/levenshtein_distance_provider.py

@@ -0,0 +1,31 @@
+#  Copyright 2022-present, the Waterdip Labs Pvt. Ltd.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from nltk.metrics import edit_distance
+
+from dcs_sdk.sdk.utils.similarity_score.base_provider import SimilarityScoreProvider
+
+
+class LevenshteinDistanceProvider(SimilarityScoreProvider):
+    def fuzzy_match(self, tokens1: set, tokens2: set) -> float:
+        """Computes similarity score using Levenshtein distance."""
+        str1 = " ".join(tokens1)
+        str2 = " ".join(tokens2)
+
+        max_len = max(len(str1), len(str2))
+        if max_len == 0:
+            return 1.0
+
+        distance = edit_distance(str1, str2)
+        return round(1 - (distance / max_len), 4)