PyReprism 0.0.1__tar.gz → 0.0.2__tar.gz

{pyreprism-0.0.1 → pyreprism-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: PyReprism
-Version: 0.0.1
+Version: 0.0.2
 Summary: Framework for Source Code Preprocessing
 Project-URL: Homepage, https://github.com/unlv-evol/PyReprism
 Project-URL: Issues, https://github.com/unlv-evol/PyReprism/issues
@@ -46,7 +46,7 @@ Requires-Dist: pytest-cov; extra == 'dev'
 Description-Content-Type: text/markdown
 
 [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-
+![PyPI - Downloads](https://img.shields.io/pypi/dd/PyReprism)
 # PyReprism
 
 PyReprism is a suite of essential methods designed for common preprocessing tasks in code clone detection research.
@@ -56,10 +56,28 @@ PyReprism is a suite of essential methods designed for common preprocessing task
 pip install PyReprism
 ```
 ## Quick Usage
+Use case 1: Removing comments 
 ```python
 
 from PyReprism.languages import Python
+# from PyReprism.languages import Java
+
+source = """
+# single line comment
+x = 5 + 6
+'''
+multiline
+comment
+'''
+print(x)
+"""
+
+source = Python.remove_comments(source)
+```
 
+Use case 2: Removing whitespaces 
+```python
+from PyReprism.utils.normalizer import Normalizer
 source = """
 # single line comment
 x = 5 + 6
@@ -70,11 +88,15 @@ comment
 print(x)
 """
 
-source =''.join(Python.remove_comments(source))
+source = Normalizer.remove_whitespaces(source)
 ```
-Read the [docs]() for more usage examples. 
+
+Read the [docs]() for more usage examples. -- Coming soon!!
+
+NB: The beta versions of PyReprism is still highly unstable, we are working 24/7 to ensure the tool is usable.
 
 ## How to Contribute
+We invite you to help us build this tool and make it more extensive. Contribution is open to OSS community.
 
 ```shell
 git clone https://github.com/unlv-evol/PyReprism.git
@@ -92,3 +114,5 @@ Then, install the requirements:
 ```
 pip install -r requirements.txt
 ```
+## Issues
+If you experience any issue, feel free to report it here. We are developing the issue reporting guidelines which will be available soon.

pyreprism-0.0.2/README.md

@@ -0,0 +1,71 @@
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+![PyPI - Downloads](https://img.shields.io/pypi/dd/PyReprism)
+# PyReprism
+
+PyReprism is a suite of essential methods designed for common preprocessing tasks in code clone detection research.
+
+## Install
+```shell
+pip install PyReprism
+```
+## Quick Usage
+Use case 1: Removing comments 
+```python
+
+from PyReprism.languages import Python
+# from PyReprism.languages import Java
+
+source = """
+# single line comment
+x = 5 + 6
+'''
+multiline
+comment
+'''
+print(x)
+"""
+
+source = Python.remove_comments(source)
+```
+
+Use case 2: Removing whitespaces 
+```python
+from PyReprism.utils.normalizer import Normalizer
+source = """
+# single line comment
+x = 5 + 6
+'''
+multiline
+comment
+'''
+print(x)
+"""
+
+source = Normalizer.remove_whitespaces(source)
+```
+
+Read the [docs]() for more usage examples. -- Coming soon!!
+
+NB: The beta versions of PyReprism is still highly unstable, we are working 24/7 to ensure the tool is usable.
+
+## How to Contribute
+We invite you to help us build this tool and make it more extensive. Contribution is open to OSS community.
+
+```shell
+git clone https://github.com/unlv-evol/PyReprism.git
+cd PyReprism
+```
+**(Optional)** It is suggested to make use of virtualenv. Therefore, before installing the requirements run:
+
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+
+Then, install the requirements:
+
+```
+pip install -r requirements.txt
+```
+## Issues
+If you experience any issue, feel free to report it here. We are developing the issue reporting guidelines which will be available soon.

{pyreprism-0.0.1 → pyreprism-0.0.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "PyReprism"
-version = "0.0.1"
+version = "0.0.2"
 authors = [
   { name= "UNLV EVOL LAB", email="ogenrwot@unlv.nevada.edu" },
 ]

{pyreprism-0.0.1 → pyreprism-0.0.2}/src/PyReprism/languages/python.py

@@ -17,9 +17,8 @@ class Python:
     
     @staticmethod
     def comment_regex():
-        # return re.compile(r'(?P<comment>#.*?$)|(?P<noncomment>\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)', re.DOTALL | re.MULTILINE)
-        # pattern = re.compile(r'(?P<comment>#.*?$|"""[\s\S]*?"""|\'\'\'[\s\S]*?\'\'\')|(?P<noncomment>[^\n]*?(?=\n|$))', re.MULTILINE)
-        pattern = re.compile(r'(?P<comment>#.*?$)|(?P<noncomment>\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|[^#\'"]+)',re.DOTALL | re.MULTILINE)
+        pattern = re.compile(r'(?P<comment>#.*?$)|'r'(?P<multilinecomment1>""".*?""")|'r'(?P<multilinecomment2>\'\'\'.*?\'\'\')|'r'(?P<noncomment>\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|.[^#\'"]*)',re.DOTALL | re.MULTILINE)
+        # pattern = re.compile(r'(?P<comment>#.*?$)|(?P<noncomment>\'(\\.|[^\\\'])*\'|"(\\.|[^\\"])*"|[^#\'"]+)',re.DOTALL | re.MULTILINE)
         return pattern
     
     @staticmethod

pyreprism-0.0.2/src/PyReprism/utils/extension.py

@@ -0,0 +1,149 @@
+abap: str = ".abap"
+actionscript: str = ".as"
+ada: str = ".ada"
+apacheconf: str = ".conf"
+apl: str = ".apl"
+applescript: str = ".applescript"
+arduino: str = ".ino"
+arff: str = ".arff"
+asciidoc: str = ".adoc"
+asm6502: str = ".asm"
+aspnet: str = ".asp"
+autohotkey: str = ".ahk"
+autoit: str = ".au3"
+bash: str = ".sh"
+basic: str = ".bas"
+batch: str = ".bat"
+bison: str = ".y"
+brainfuck: str = ".bf"
+bro: str = ".bro"
+c: str = ".c"
+clike: str = ".c"  # This is generic; specific languages like C or C++ should be used.
+clojure: str = ".clj"
+coffeescript: str = ".coffee"
+cpp: str = ".cpp"
+crystal: str = ".cr"
+csharp: str = ".cs"
+csp: str = ".csp"
+css_extras: str = ".css"
+css: str = ".css"
+d: str = ".d"
+dart: str = ".dart"
+diff: str = ".diff"
+django: str = ".py"  # Django typically uses Python files
+docker: str = "Dockerfile"
+eiffel: str = ".e"
+elixir: str = ".ex"
+erb: str = ".erb"
+erlang: str = ".erl"
+flow: str = ".js"  # Flow is a static type checker for JavaScript
+fortran: str = ".f90"
+fsharp: str = ".fs"
+gedcom: str = ".ged"
+gherkin: str = ".feature"
+git: str = ".git"
+glsl: str = ".glsl"
+go: str = ".go"
+graphql: str = ".graphql"
+groovy: str = ".groovy"
+haml: str = ".haml"
+handlebars: str = ".hbs"
+haskell: str = ".hs"
+haxe: str = ".hx"
+hpkp: str = ".hpkp"
+hsts: str = ".hsts"
+html: str = ".html"
+http: str = ".http"
+ichigojam: str = ".ijs"
+icon: str = ".icon"
+inform7: str = ".ni"
+ini: str = ".ini"
+io: str = ".io"
+ipynb: str = ".ipynb"
+j: str = ".ijs"
+java: str = ".java"
+javascript: str = ".js"
+jolie: str = ".ol"
+json: str = ".json"
+jsx: str = ".jsx"
+julia: str = ".jl"
+keyman: str = ".kmn"
+kotlin: str = ".kt"
+latex: str = ".tex"
+less: str = ".less"
+liquid: str = ".liquid"
+lisp: str = ".lisp"
+livescript: str = ".ls"
+lolcode: str = ".lol"
+lua: str = ".lua"
+makefile: str = "Makefile"
+markdown: str = ".md"
+markup_templating: str = ".xml"
+markup: str = ".xml"
+matlab: str = ".m"
+mel: str = ".mel"
+mizar: str = ".miz"
+monkey: str = ".monkey"
+n4js: str = ".n4js"
+nasm: str = ".asm"
+nginx: str = ".conf"
+nim: str = ".nim"
+nix: str = ".nix"
+nsis: str = ".nsi"
+objectivec: str = ".m"
+ocaml: str = ".ml"
+opencl: str = ".cl"
+oz: str = ".oz"
+parigp: str = ".gp"
+parser: str = ".parser"
+pascal: str = ".pas"
+perl: str = ".pl"
+php_extras: str = ".php"
+php: str = ".php"
+plsql: str = ".sql"
+powershell: str = ".ps1"
+processing: str = ".pde"
+prolog: str = ".pl"
+properties: str = ".properties"
+protobuf: str = ".proto"
+pug: str = ".pug"
+puppet: str = ".pp"
+pure: str = ".pure"
+python: str = ".py"
+q: str = ".q"
+qore: str = ".q"
+r: str = ".r"
+reason: str = ".re"
+renpy: str = ".rpy"
+rest: str = ".rest"
+rip: str = ".rip"
+roboconf: str = ".graph"
+ruby: str = ".rb"
+rust: str = ".rs"
+sas: str = ".sas"
+sass: str = ".sass"
+scala: str = ".scala"
+scheme: str = ".scm"
+scss: str = ".scss"
+smalltalk: str = ".st"
+smarty: str = ".tpl"
+soy: str = ".soy"
+stylus: str = ".styl"
+swift: str = ".swift"
+tcl: str = ".tcl"
+textile: str = ".textile"
+tsx: str = ".tsx"
+twig: str = ".twig"
+typescript: str = ".ts"
+vbnet: str = ".vb"
+velocity: str = ".vm"
+verilog: str = ".v"
+vhdl: str = ".vhdl"
+vim: str = ".vim"
+visual_basic: str = ".vb"
+wasm: str = ".wasm"
+wiki: str = ".wiki"
+xeora: str = ".xeora"
+xojo: str = ".xojo"
+yaml: str = ".yaml"
+yml: str = ".yml"

pyreprism-0.0.1/README.md

@@ -1,47 +0,0 @@
-[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-
-# PyReprism
-
-PyReprism is a suite of essential methods designed for common preprocessing tasks in code clone detection research.
-
-## Install
-```shell
-pip install PyReprism
-```
-## Quick Usage
-```python
-
-from PyReprism.languages import Python
-
-source = """
-# single line comment
-x = 5 + 6
-'''
-multiline
-comment
-'''
-print(x)
-"""
-
-source =''.join(Python.remove_comments(source))
-```
-Read the [docs]() for more usage examples. 
-
-## How to Contribute
-
-```shell
-git clone https://github.com/unlv-evol/PyReprism.git
-cd PyReprism
-```
-**(Optional)** It is suggested to make use of virtualenv. Therefore, before installing the requirements run:
-
-```
-python3 -m venv venv
-source venv/bin/activate
-```
-
-Then, install the requirements:
-
-```
-pip install -r requirements.txt
-```

pyreprism-0.0.1/src/PyReprism/utils/extension.py

@@ -1,149 +0,0 @@
-abap: str = "xxxxx"
-actionscript: str = "xxxxx"
-ada: str = ".ada"
-apacheconf: str = "xxxxx"
-apl: str = "xxxxx"
-applescript: str = "xxxxx"
-arduino: str = "xxxxx"
-arff: str = "xxxxx"
-asciidoc: str = "xxxxx"
-asm6502: str = "xxxxx"
-aspnet: str = ".asp"
-autohotkey: str = "xxxxx"
-autoit: str = "xxxxx"
-bash: str = ".sh"
-basic: str = "xxxxx"
-batch: str = "xxxxx"
-bison: str = "xxxxx"
-brainfuck: str = "xxxxx"
-bro: str = "xxxxx"
-c: str = ".c"
-clike: str = "xxxxx"
-clojure: str = "xxxxx"
-coffeescript: str = "xxxxx"
-cpp: str = ".cpp"
-crystal: str = "xxxxx"
-csharp: str = ".csharp"
-csp: str = "xxxxx"
-css_extras: str = "xxxxx"
-css: str = ".css"
-d: str = "xxxxx"
-dart: str = ".dart"
-diff: str = ".diff"
-django: str = "xxxxx"
-docker: str = "xxxxx"
-eiffel: str = "xxxxx"
-elixir: str = "xxxxx"
-erb: str = "xxxxx"
-erlang: str = "xxxxx"
-flow: str = "xxxxx"
-fortran: str = "xxxxx"
-fsharp: str = "xxxxx"
-gedcom: str = "xxxxx"
-gherkin: str = "xxxxx"
-git: str = ".git"
-glsl: str = "xxxxx"
-go: str = ".go"
-graphql: str = "xxxxx"
-groovy: str = ".groovy"
-haml: str = "xxxxx"
-handlebars: str = "xxxxx"
-haskell: str = "xxxxx"
-haxe: str = "xxxxx"
-hpkp: str = "xxxxx"
-hsts: str = "xxxxx"
-html: str = "xxxxx"
-http: str = "xxxxx"
-ichigojam: str = "xxxxx"
-icon: str = "xxxxx"
-inform7: str = "xxxxx"
-ini: str = ".ini"
-io: str = ".io"
-ipynd: str = ".ipynd"
-j: str = ".j"
-java: str = ".java"
-javascript: str = ".js"
-jolie: str = "xxxxx"
-json: str = ".json"
-jsx: str = ".jsx"
-julia: str = ".julia"
-keyman: str = "xxxxx"
-kotlin: str = ".kt"
-latex: str = "xxxxx"
-less: str = ".less"
-liquid: str = "xxxxx"
-lisp: str = "xxxxx"
-livescript: str = "xxxxx"
-lolcode: str = "xxxxx"
-lua: str = "xxxxx"
-makefile: str = "xxxxx"
-markdown: str = ".md"
-markup_templating: str = "xxxxx"
-markup: str = ".xml"
-matlab: str = "xxxxx"
-mel: str = "xxxxx"
-mizar: str = "xxxxx"
-monkey: str = "xxxxx"
-n4js: str = "xxxxx"
-nasm: str = "xxxxx"
-nginx: str = "xxxxx"
-nim: str = "xxxxx"
-nix: str = "xxxxx"
-nsis: str = "xxxxx"
-objectivec: str = "xxxxx"
-ocaml: str = ".ocaml"
-opencl: str = "xxxxx"
-oz: str = "xxxxx"
-parigp: str = "xxxxx"
-parser: str = "xxxxx"
-pascal: str = "xxxxx"
-perl: str = ".pl"
-php_extras: str = "xxxxx"
-php: str = ".php"
-plsql: str = "xxxxx"
-powershell: str = "xxxxx"
-processing: str = "xxxxx"
-prolog: str = "xxxxx"
-properties: str = ".properties"
-protobuf: str = "xxxxx"
-pug: str = "xxxxx"
-puppet: str = "xxxxx"
-pure: str = "xxxxx"
-python: str = ".py"
-q: str = "xxxxx"
-qore: str = "xxxxx"
-r: str = "xxxxx"
-reason: str = "xxxxx"
-renpy: str = "xxxxx"
-rest: str = "xxxxx"
-rip: str = "xxxxx"
-roboconf: str = "xxxxx"
-ruby: str = ".rb"
-rust: str = ".rs"
-sas: str = ".sas"
-sass: str = ".sass"
-scala: str = ".scala"
-scheme: str = ".rkt"
-scss: str = ".scss"
-smalltalk: str = "xxxxx"
-smarty: str = "xxxxx"
-soy: str = "xxxxx"
-stylus: str = ".stylus"
-swift: str = ".swift"
-tcl: str = "xxxxx"
-textile: str = "xxxxx"
-tsx: str = ".tsx"
-twig: str = "xxxxx"
-typescript: str = ".ts"
-vbnet: str = "xxxxx"
-velocity: str = "xxxxx"
-verilog: str = "xxxxx"
-vhdl: str = "xxxxx"
-vim: str = ".vim"
-visual_basic: str = ".vb"
-wasm: str = "xxxxx"
-wiki: str = "xxxxx"
-xeora: str = "xxxxx"
-xojo: str = "xxxxx"
-yaml: str = ".yaml"
-yml: str = ".yml"