csim 3.4.0__tar.gz → 3.4.2__tar.gz

{csim-3.4.0/csim.egg-info → csim-3.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csim
-Version: 3.4.0
+Version: 3.4.2
 Summary: Code Similarity (csim) is a method designed to detect similarity between source codes
 Home-page: https://github.com/EdsonEddy/csim
 Author: Eddy Lecoña
@@ -363,6 +363,16 @@ similarity = Compare(name_a='example A', content_a=code_a, name_b='example B', c
 print(f"Similarity: {similarity}") # Output: Similarity: X.XX
 ```
 
+To see how much a program shrinks when it is normalized, pruned and hashed, count its nodes before and after:
+
+```python
+from csim import count_nodes
+
+nodes_before, nodes_after = count_nodes("example.py", code, lang="python_3_13")
+```
+
+`nodes_before` is every node of the raw ANTLR parse tree; `nodes_after` is the size of the tree handed to the tree edit distance (the same number `csim tree` prints as "Total nodes after pruning").
+
 ## Documentation
 
 - [Getting Started Guide](GETTING_STARTED.md) - Quick tutorial for new users

{csim-3.4.0 → csim-3.4.2}/README.md

@@ -324,6 +324,16 @@ similarity = Compare(name_a='example A', content_a=code_a, name_b='example B', c
 print(f"Similarity: {similarity}") # Output: Similarity: X.XX
 ```
 
+To see how much a program shrinks when it is normalized, pruned and hashed, count its nodes before and after:
+
+```python
+from csim import count_nodes
+
+nodes_before, nodes_after = count_nodes("example.py", code, lang="python_3_13")
+```
+
+`nodes_before` is every node of the raw ANTLR parse tree; `nodes_after` is the size of the tree handed to the tree edit distance (the same number `csim tree` prints as "Total nodes after pruning").
+
 ## Documentation
 
 - [Getting Started Guide](GETTING_STARTED.md) - Quick tutorial for new users

{csim-3.4.0 → csim-3.4.2}/csim/__init__.py

@@ -2,4 +2,4 @@ from .CodeSimilarity import Compare
 from .language.parser import ANTLR_parse
 from .processing.tree_processing import Normalize, PruneAndHash
 from .processing.distance_metrics import SimilarityIndex
-from .utils import group_by_exhaustive_search, report_pairwise_similarity
+from .utils import count_nodes, group_by_exhaustive_search, report_pairwise_similarity

{csim-3.4.0 → csim-3.4.2}/csim/main.py

@@ -3,6 +3,7 @@ import os
 from .language.parser import ANTLR_parse
 from .processing.tree_processing import Normalize, PruneAndHash
 from .utils import (
+    count_tree_nodes,
     group_by_exhaustive_search,
     print_antlr_tree,
     print_tree,
@@ -179,7 +180,8 @@ def main():
             print()
 
         normalized_tree = Normalize(raw_tree, args.lang)
-        pruned_tree, node_count = PruneAndHash(normalized_tree, args.lang)
+        pruned_tree, _ = PruneAndHash(normalized_tree, args.lang)
+        node_count = count_tree_nodes(pruned_tree)
 
         print("=== Normalized + Pruned Tree (input to Tree Edit Distance) ===")
         print_tree(pruned_tree, lang=args.lang)

{csim-3.4.0 → csim-3.4.2}/csim/processing/distance_metrics.py

@@ -1,4 +1,4 @@
-from zss import simple_distance
+from zss import distance as zss_distance
 
 
 def label_distance(label1, label2):
@@ -36,6 +36,35 @@ def label_distance(label1, label2):
     return 1.0
 
 
+def node_weight(node):
+    """Cost of inserting or deleting a node: 1, or the mass a hashed node kept."""
+    return node.get("weight", 1)
+
+
+def rename_cost(node1, node2):
+    """Substitution cost between two tree nodes (dicts with a 'label').
+
+    Same as label_distance, except two weighted hashed nodes of the same rule
+    but different content are charged by how much of their content differs
+    (multiset overlap of the labels they replaced) instead of a flat 0.5.
+    """
+    sig1, sig2 = node1.get("sig"), node2.get("sig")
+    if sig1 is None or sig2 is None:
+        return label_distance(node1["label"], node2["label"])
+    if node1["label"] == node2["label"]:
+        return 0.0
+    rule1, hash1 = str(node1["label"]).split("|", 1)
+    rule2, hash2 = str(node2["label"]).split("|", 1)
+    heavy = max(node1["weight"], node2["weight"])
+    if rule1 != rule2:
+        return heavy if hash1 != hash2 else 0.5 * heavy
+    common = sum((sig1 & sig2).values())
+    total = sum(sig1.values()) + sum(sig2.values())
+    # Different digest means different content even if the multisets match
+    # (same labels, other order), so it never costs less than the old flat 0.5.
+    return max(heavy * (1.0 - 2.0 * common / total), 0.5)
+
+
 def TreeEditDistance(N1, N2, ted_algorithm="apted"):
     """Calculate the tree edit distance between two trees using the specified algorithm.
     Args:
@@ -46,27 +75,14 @@ def TreeEditDistance(N1, N2, ted_algorithm="apted"):
         int: The computed tree edit distance between the two trees.
     """
     if ted_algorithm == "zss":
-        # Custom configuration for zss to work with dictionaries
-        class CustomConfigZss:
-            @staticmethod
-            def get_children(node):
-                return node["children"]
-
-            @staticmethod
-            def get_label(node):
-                return node["label"]
-
-            @staticmethod
-            def label_dist(label1, label2):
-                """Compares two labels"""
-                return label_distance(label1, label2)
-
-        d = simple_distance(
+        # zss takes per-node cost functions, so weighted hashed nodes work too
+        d = zss_distance(
             N1,
             N2,
-            get_children=CustomConfigZss.get_children,
-            get_label=CustomConfigZss.get_label,
-            label_dist=CustomConfigZss.label_dist,
+            get_children=lambda node: node["children"],
+            insert_cost=node_weight,
+            remove_cost=node_weight,
+            update_cost=rename_cost,
         )
     elif ted_algorithm == "apted":
         from apted import APTED, Config
@@ -75,7 +91,13 @@ def TreeEditDistance(N1, N2, ted_algorithm="apted"):
         class CustomConfigApted(Config):
             def rename(self, node1, node2):
                 """Compares attribute .value of trees"""
-                return label_distance(node1["label"], node2["label"])
+                return rename_cost(node1, node2)
+
+            def delete(self, node):
+                return node_weight(node)
+
+            def insert(self, node):
+                return node_weight(node)
 
             def children(self, node):
                 """Get childrens of a node"""

{csim-3.4.0 → csim-3.4.2}/csim/processing/tree_processing.py

@@ -1,4 +1,5 @@
 import hashlib
+from collections import Counter
 from antlr4 import TerminalNode
 from ..Visitors import (
     Python_3_13_ParserVisitorExtended,
@@ -16,6 +17,7 @@ from ..utils import (
     get_excluded_token_types,
     get_hash_rule_indices,
     get_excluded_rule_types,
+    get_hash_mass_alpha,
     get_relabel_fn,
     get_structural_rule_indices,
 )
@@ -121,6 +123,9 @@ def PruneAndHash(tree, lang):
     control_equivalence_rule_indices = get_control_equivalence_rule_indices(lang)
     exclude_childrens_from_rule = get_exclude_childrens_from_rule(lang)
     structural_rule_indices = get_structural_rule_indices(lang)
+    # When set, a hashed node keeps the mass of the subtree it replaced (see
+    # docs/pruning_fidelity.md, "Weighted hashes"); None keeps 1 node = 1.
+    mass_alpha = get_hash_mass_alpha(lang)
 
     def traverse_subtree(node):
         # Collect all labels in the subtree rooted at `node` into a single list
@@ -155,7 +160,8 @@ def PruneAndHash(tree, lang):
         for c in childrens:
             flat.extend(traverse_subtree(c))
         s = "|".join(map(str, flat))
-        return str(label) + "|" + hashlib.sha256(s.encode("utf-8")).hexdigest()
+        digest = str(label) + "|" + hashlib.sha256(s.encode("utf-8")).hexdigest()
+        return digest, flat
 
     # Ids of nodes whose subtree contains a structural (control-flow/body)
     # label. A hashed rule that contains one is NOT collapsed: its content is
@@ -184,8 +190,19 @@ def PruneAndHash(tree, lang):
 
         # For nodes that are in the hashed rule set, we hash their entire subtree to a single digest
         if node["label"] in hashed_rule_indices and id(node) not in has_structure:
-            digest = hash_children(label, node["children"])
-            return {"label": digest, "children": []}, 1
+            digest, flat = hash_children(label, node["children"])
+            if mass_alpha is None:
+                return {"label": digest, "children": []}, 1
+            # Weight and label multiset let the edit distance charge a hashed
+            # node in proportion to what it replaced and give partial credit
+            # to two hashed nodes of the same kind that share most content.
+            weight = (len(flat) + 1) ** mass_alpha
+            return {
+                "label": digest,
+                "children": [],
+                "weight": weight,
+                "sig": Counter(flat),
+            }, weight
 
         new_node = {"label": label, "children": []}
         count = 1

{csim-3.4.0 → csim-3.4.2}/csim/python_3/utils.py

@@ -252,6 +252,10 @@ HASHED_RULE_INDICES = {
     # without hashing anything.
 }
 
+# Exponent for the weight of a hashed node (its subtree size ** alpha) in the
+# tree edit distance; see docs/pruning_fidelity.md, "Weighted hashes".
+HASH_MASS_ALPHA = 0.6
+
 # `for` and `while` are interchangeable ways to write the same loop (the
 # jv-umsa-dataset/controlled clones rewrite one as the other), so both get the
 # same label; mirrors python_3_13 and the 2.0.0 behaviour.

{csim-3.4.0 → csim-3.4.2}/csim/utils.py

@@ -378,6 +378,19 @@ def get_relabel_fn(lang):
     return None
 
 
+def get_hash_mass_alpha(lang):
+    """Exponent applied to the size of a hashed subtree to get its weight in
+    the edit distance, or None when the language keeps every node at weight 1."""
+    import importlib
+
+    if lang not in (
+        "python_3_13", "python_3", "java_20", "java_24", "cpp_14", "kotlin", "c",
+    ):
+        return None
+    module = importlib.import_module(f".{lang}.utils", package=__package__)
+    return getattr(module, "HASH_MASS_ALPHA", None)
+
+
 def get_structural_rule_indices(lang):
     """Retrieve the rules that mark a subtree as structural (control flow,
     bodies, declarations of callables/types) for a language.
@@ -506,6 +519,53 @@ def preprocess_code(file_name, file_content, lang="python_3_13"):
     return pruned_tree, pruned_count
 
 
+def count_tree_nodes(tree):
+    """Number of nodes of a normalized/pruned tree (iterative, deep-safe).
+
+    PruneAndHash's second value is the tree's *mass* when a language weights
+    its hashed nodes, so the plain node count has to be taken from the tree.
+    """
+    count = 0
+    stack = [tree]
+    while stack:
+        node = stack.pop()
+        count += 1
+        stack.extend(node["children"])
+    return count
+
+
+def count_nodes(file_name, file_content, lang="python_3_13"):
+    """Count the nodes of a program before and after pruning.
+
+    Args:
+        file_name (str): Name of the file (used only for syntax-error messages).
+        file_content (str): Source code.
+        lang (str): Programming language identifier.
+
+    Returns:
+        tuple[int, int]: (nodes_before, nodes_after). `nodes_before` is every
+        node of the raw ANTLR parse tree (rules and tokens); `nodes_after` is
+        the size of the normalized, pruned and hashed tree that is handed to
+        the tree edit distance -- the same number `csim tree` prints as
+        "Total nodes after pruning".
+    """
+    # Local import to avoid circular dependency at module import time
+    from .CodeSimilarity import ANTLR_parse, Normalize, PruneAndHash
+
+    tree = ANTLR_parse(file_name, file_content, lang)
+
+    # Iterative: real programs can nest deeper than the recursion limit.
+    before = 0
+    stack = [tree]
+    while stack:
+        node = stack.pop()
+        before += 1
+        stack.extend(node.getChild(i) for i in range(node.getChildCount()))
+
+    pruned, _ = PruneAndHash(Normalize(tree, lang), lang)
+    return before, count_tree_nodes(pruned)
+
+
 def get_similarity_coefficient(proccesed_code1, proccesed_code2, ted_algorithm):
     N1, len_N1 = proccesed_code1
     N2, len_N2 = proccesed_code2

{csim-3.4.0 → csim-3.4.2/csim.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csim
-Version: 3.4.0
+Version: 3.4.2
 Summary: Code Similarity (csim) is a method designed to detect similarity between source codes
 Home-page: https://github.com/EdsonEddy/csim
 Author: Eddy Lecoña
@@ -363,6 +363,16 @@ similarity = Compare(name_a='example A', content_a=code_a, name_b='example B', c
 print(f"Similarity: {similarity}") # Output: Similarity: X.XX
 ```
 
+To see how much a program shrinks when it is normalized, pruned and hashed, count its nodes before and after:
+
+```python
+from csim import count_nodes
+
+nodes_before, nodes_after = count_nodes("example.py", code, lang="python_3_13")
+```
+
+`nodes_before` is every node of the raw ANTLR parse tree; `nodes_after` is the size of the tree handed to the tree edit distance (the same number `csim tree` prints as "Total nodes after pruning").
+
 ## Documentation
 
 - [Getting Started Guide](GETTING_STARTED.md) - Quick tutorial for new users

{csim-3.4.0 → csim-3.4.2}/setup.py

@@ -49,7 +49,7 @@ except ImportError:
 
 setup(
     name="csim",
-    version="3.4.0",
+    version="3.4.2",
     packages=find_packages(),
     package_data={
         # Compiled native parsers, when built (scripts/build_native_parsers.sh).