csim 3.2.0__tar.gz → 3.3.0__tar.gz

{csim-3.2.0 → csim-3.3.0}/MANIFEST.in

@@ -7,6 +7,14 @@ include scripts/build_native_parsers.sh
 include scripts/transform_grammar_for_cpp.py
 include csim/native/src/*.cpp
 
+# The C grammar's symbol-table implementation (typedef disambiguation) --
+# doesn't match the *Base.cpp/*Base.h patterns above since it isn't a
+# lexer/parser base class.
+include grammars/SymbolTable.cpp
+include grammars/SymbolTable.h
+include grammars/Symbol.h
+include grammars/TypeClassification.h
+
 include README.md
 include LICENSE
 

{csim-3.2.0/csim.egg-info → csim-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: csim
-Version: 3.2.0
+Version: 3.3.0
 Summary: Code Similarity (csim) is a method designed to detect similarity between source codes
 Home-page: https://github.com/EdsonEddy/csim
 Author: Eddy Lecoña
@@ -52,7 +52,7 @@ Code Similarity (csim) provide a module designed to detect similarities between
 - **Parse Trees:** Represents the syntactic structure of source code, enabling detailed comparisons.
 - **Tree Edit Distance:** Measures the similarity between different code structures.
 - **Hash-Based Pruning:** Optimizes the comparison process by reducing tree size while preserving essential structure.
-- **Multi-Language Support:** Supports Python 3.13, Python 3 (universal grammar), Java 20, Java 24, and C++14 source code analysis.
+- **Multi-Language Support:** Supports Python 3.13, Python 3 (universal grammar), Java 20, Java 24, C++14, Kotlin (experimental), and C (experimental) source code analysis.
 
 ## Technologies Used
 
@@ -98,7 +98,7 @@ pip install csim
 
 For detailed information about search strategies, see: [docs/STRATEGIES.md](docs/STRATEGIES.md)
 
-csim supports three main actions: **report** (for pairwise similarity analysis), **group** (for clustering similar files), and **tree**/**view** (for visualizing a file's normalized/pruned parse tree). The tool supports Python 3.13, Java 20, Java 24, and C++14 source code files.
+csim supports three main actions: **report** (for pairwise similarity analysis), **group** (for clustering similar files), and **tree**/**view** (for visualizing a file's normalized/pruned parse tree). The tool supports Python 3.13, Python 3, Java 20, Java 24, C++14, Kotlin (experimental), and C (experimental) source code files.
 
 ### General Command Structure
 ```sh
@@ -121,7 +121,7 @@ file2.py is similar to file3.py with similarity index: 0.50
 ```
 
 **Options:**
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--talg, -ta`: Tree edit distance algorithm (default: `apted`). Options: `zss`, `apted`
 
 **Example with options:**
@@ -167,7 +167,7 @@ csim group --path /path/to/directory --threshold 0.8 --strategy exhaustive
 
 - `--threshold, -t`: Similarity threshold (0.0 to 1.0). **Required.**
 - `--strategy, -s`: Grouping strategy (default: `exhaustive`). Options: `exhaustive`
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--talg, -ta`: Tree edit distance algorithm (default: `apted`). Options: `zss`, `apted`
 
 **Complete example:**
@@ -204,7 +204,7 @@ Rule and token names are resolved for readability, `LOOP` marks nodes collapsed
 
 **Options:**
 - `--path, -p`: Path to a single source code file (**required**).
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--show-raw`: Also print the raw ANTLR parse tree before normalization/pruning, for side-by-side comparison.
 
 **Example with `--show-raw`:**
@@ -261,13 +261,48 @@ until this is resolved; `java_24` is available for parse-speed experimentation o
 csim report --path /path/to/cpp/files --lang cpp_14
 ```
 
+**Kotlin (experimental):**
+```sh
+csim report --path /path/to/kotlin/files --lang kotlin
+```
+
+First-cut integration (grammars-v4/kotlin/kotlin) with a working native parser
+(no C++ base class needed at all -- this grammar declares no `superClass`).
+Unlike `java_24`/`python_3`, there is no real-world Kotlin corpus in this
+project's benchmark set to tune or validate grouping precision against yet, so
+the normalization rules (`csim/kotlin/utils.py`) follow the same *categories*
+already validated for other languages (structural punctuation, identifier
+text, import/package plumbing, body-wrapping content) but haven't been
+corpus-measured for false-positive/false-negative rates. Treat `group`/`report`
+results as a reasonable starting point, not a tuned config, until a real
+corpus drives the next pass.
+
+**C (experimental):**
+```sh
+csim report --path /path/to/c/files --lang c
+```
+
+First-cut integration (grammars-v4/c, ISO C23 + GNU/MSVC extensions), with a
+working native parser backed by a real symbol-table implementation for
+typedef disambiguation. Like Kotlin, there's no real-world C corpus in this
+project's benchmark set to tune grouping precision against yet -- same
+caveats apply, see `csim/c/utils.py`.
+
+Runs with preprocessing disabled (`--nopp`) always: a real preprocessor can't
+be assumed present in a production container, and judge submissions have no
+consistent include paths anyway. `#include`/`#define`/etc. lines are
+swallowed as hidden tokens rather than expanded, which means macro-dependent
+code (token-pasting tricks, macros used for control flow) can fail to parse
+or parse differently than a real compiler would see it -- real submissions
+essentially never rely on that, but it's a known, real limitation.
+
 ### Native Parsers
 
-For `java_20`, `java_24`, `cpp_14`, and `python_3`, csim can use a compiled C++
-ANTLR parser instead of the pure-Python one, giving a large speedup with
-identical (or, for `java_24`, not-yet-identical -- see above) output.
-`python_3_13` always uses the pure-Python parser (no C++ target is available for
-that grammar).
+For `java_20`, `java_24`, `cpp_14`, `python_3`, `kotlin`, and `c`, csim can
+use a compiled C++ ANTLR parser instead of the pure-Python one, giving a
+large speedup with identical (or, for `java_24`, not-yet-identical -- see
+above) output. `python_3_13` always uses the pure-Python parser (no C++
+target is available for that grammar).
 
 Check which backend is active for each language:
 

{csim-3.2.0 → csim-3.3.0}/README.md

@@ -13,7 +13,7 @@ Code Similarity (csim) provide a module designed to detect similarities between
 - **Parse Trees:** Represents the syntactic structure of source code, enabling detailed comparisons.
 - **Tree Edit Distance:** Measures the similarity between different code structures.
 - **Hash-Based Pruning:** Optimizes the comparison process by reducing tree size while preserving essential structure.
-- **Multi-Language Support:** Supports Python 3.13, Python 3 (universal grammar), Java 20, Java 24, and C++14 source code analysis.
+- **Multi-Language Support:** Supports Python 3.13, Python 3 (universal grammar), Java 20, Java 24, C++14, Kotlin (experimental), and C (experimental) source code analysis.
 
 ## Technologies Used
 
@@ -59,7 +59,7 @@ pip install csim
 
 For detailed information about search strategies, see: [docs/STRATEGIES.md](docs/STRATEGIES.md)
 
-csim supports three main actions: **report** (for pairwise similarity analysis), **group** (for clustering similar files), and **tree**/**view** (for visualizing a file's normalized/pruned parse tree). The tool supports Python 3.13, Java 20, Java 24, and C++14 source code files.
+csim supports three main actions: **report** (for pairwise similarity analysis), **group** (for clustering similar files), and **tree**/**view** (for visualizing a file's normalized/pruned parse tree). The tool supports Python 3.13, Python 3, Java 20, Java 24, C++14, Kotlin (experimental), and C (experimental) source code files.
 
 ### General Command Structure
 ```sh
@@ -82,7 +82,7 @@ file2.py is similar to file3.py with similarity index: 0.50
 ```
 
 **Options:**
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--talg, -ta`: Tree edit distance algorithm (default: `apted`). Options: `zss`, `apted`
 
 **Example with options:**
@@ -128,7 +128,7 @@ csim group --path /path/to/directory --threshold 0.8 --strategy exhaustive
 
 - `--threshold, -t`: Similarity threshold (0.0 to 1.0). **Required.**
 - `--strategy, -s`: Grouping strategy (default: `exhaustive`). Options: `exhaustive`
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--talg, -ta`: Tree edit distance algorithm (default: `apted`). Options: `zss`, `apted`
 
 **Complete example:**
@@ -165,7 +165,7 @@ Rule and token names are resolved for readability, `LOOP` marks nodes collapsed
 
 **Options:**
 - `--path, -p`: Path to a single source code file (**required**).
-- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`
+- `--lang, -l`: Programming language (default: `python_3_13`). Options: `python_3_13`, `python_3`, `java_20`, `java_24`, `cpp_14`, `kotlin`, `c`
 - `--show-raw`: Also print the raw ANTLR parse tree before normalization/pruning, for side-by-side comparison.
 
 **Example with `--show-raw`:**
@@ -222,13 +222,48 @@ until this is resolved; `java_24` is available for parse-speed experimentation o
 csim report --path /path/to/cpp/files --lang cpp_14
 ```
 
+**Kotlin (experimental):**
+```sh
+csim report --path /path/to/kotlin/files --lang kotlin
+```
+
+First-cut integration (grammars-v4/kotlin/kotlin) with a working native parser
+(no C++ base class needed at all -- this grammar declares no `superClass`).
+Unlike `java_24`/`python_3`, there is no real-world Kotlin corpus in this
+project's benchmark set to tune or validate grouping precision against yet, so
+the normalization rules (`csim/kotlin/utils.py`) follow the same *categories*
+already validated for other languages (structural punctuation, identifier
+text, import/package plumbing, body-wrapping content) but haven't been
+corpus-measured for false-positive/false-negative rates. Treat `group`/`report`
+results as a reasonable starting point, not a tuned config, until a real
+corpus drives the next pass.
+
+**C (experimental):**
+```sh
+csim report --path /path/to/c/files --lang c
+```
+
+First-cut integration (grammars-v4/c, ISO C23 + GNU/MSVC extensions), with a
+working native parser backed by a real symbol-table implementation for
+typedef disambiguation. Like Kotlin, there's no real-world C corpus in this
+project's benchmark set to tune grouping precision against yet -- same
+caveats apply, see `csim/c/utils.py`.
+
+Runs with preprocessing disabled (`--nopp`) always: a real preprocessor can't
+be assumed present in a production container, and judge submissions have no
+consistent include paths anyway. `#include`/`#define`/etc. lines are
+swallowed as hidden tokens rather than expanded, which means macro-dependent
+code (token-pasting tricks, macros used for control flow) can fail to parse
+or parse differently than a real compiler would see it -- real submissions
+essentially never rely on that, but it's a known, real limitation.
+
 ### Native Parsers
 
-For `java_20`, `java_24`, `cpp_14`, and `python_3`, csim can use a compiled C++
-ANTLR parser instead of the pure-Python one, giving a large speedup with
-identical (or, for `java_24`, not-yet-identical -- see above) output.
-`python_3_13` always uses the pure-Python parser (no C++ target is available for
-that grammar).
+For `java_20`, `java_24`, `cpp_14`, `python_3`, `kotlin`, and `c`, csim can
+use a compiled C++ ANTLR parser instead of the pure-Python one, giving a
+large speedup with identical (or, for `java_24`, not-yet-identical -- see
+above) output. `python_3_13` always uses the pure-Python parser (no C++
+target is available for that grammar).
 
 Check which backend is active for each language:
 

{csim-3.2.0 → csim-3.3.0}/csim/Visitors.py

@@ -3,6 +3,8 @@ from .java_20.Java20ParserVisitor import Java20ParserVisitor
 from .java_24.Java24ParserVisitor import Java24ParserVisitor
 from .cpp_14.CPP14ParserVisitor import CPP14ParserVisitor
 from .python_3.Python3ParserVisitor import Python3ParserVisitor
+from .kotlin.KotlinParserVisitor import KotlinParserVisitor
+from .c.CParserVisitor import CParserVisitor
 from antlr4 import TerminalNode
 from .java_20.utils import (
     COLLAPSED_RULE_INDICES as JAVA_20_COLLAPSED_RULES,
@@ -36,6 +38,12 @@ from .cpp_14.utils import (
     ASIGN_OP_NORMALIZED as CPP_14_ASSIGN_OP_NORMALIZED,
     RULE_ASSIGNMENT as CPP_14_RULE_ASSIGNMENT,
 )
+from .kotlin.utils import (
+    COLLAPSED_RULE_INDICES as KOTLIN_COLLAPSED_RULES,
+)
+from .c.utils import (
+    COLLAPSED_RULE_INDICES as C_COLLAPSED_RULES,
+)
 
 
 class Python_3_13_ParserVisitorExtended(PythonParserVisitor):
@@ -182,6 +190,43 @@ class CPP14ParserVisitorExtended(CPP14ParserVisitor):
             return self.visitChildren(node)
 
 
+class KotlinParserVisitorExtended(KotlinParserVisitor):
+    def visit(self, tree):
+        """Override visit to exclude certain rules from being processed.
+        This helps in reducing noise in the parse tree by skipping over
+        less relevant constructs.
+
+        No visitAssignment-style override here: Kotlin's grammar has no
+        ANTLR labeled alternatives at all (unlike java_24/python_3), so
+        there's no relabel_node() hook needed either -- see
+        csim/kotlin/utils.py's module docstring.
+        """
+        if (
+            not isinstance(tree, TerminalNode)
+            and tree.getRuleIndex() in KOTLIN_COLLAPSED_RULES
+        ):
+            return {"label": tree.getRuleIndex(), "children": []}
+        return tree.accept(self)
+
+
+class CParserVisitorExtended(CParserVisitor):
+    def visit(self, tree):
+        """Override visit to exclude certain rules from being processed.
+        This helps in reducing noise in the parse tree by skipping over
+        less relevant constructs.
+
+        No visitAssignment-style override here: no relabel_node() hook is
+        needed either -- see csim/c/utils.py's module docstring (this
+        grammar has no ANTLR labeled alternatives at all).
+        """
+        if (
+            not isinstance(tree, TerminalNode)
+            and tree.getRuleIndex() in C_COLLAPSED_RULES
+        ):
+            return {"label": tree.getRuleIndex(), "children": []}
+        return tree.accept(self)
+
+
 class Python3ParserVisitorExtended(Python3ParserVisitor):
     def visit(self, tree):
         """Override visit to exclude certain rules from being processed.