PyReprism 0.0.4__tar.gz → 0.1.0__tar.gz

pyreprism-0.1.0/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+show-source = True
+count = True
+statistics = True
+max-line-length = 120
+# E501: long lines are common in this project's regex/keyword tables.
+# The W1/W2/W3 and E1/E2/E3 codes below are pre-existing whitespace/indentation
+# style debt across the legacy language modules; they are ignored so flake8 can
+# focus on real correctness issues (pyflakes F-codes, undefined names, etc.).
+extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266
+exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,tests,src/PyReprism/languages/__init__.py,__test__.py

pyreprism-0.1.0/.github/scripts/changelog_notes.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Print the CHANGELOG.md section for a given version.
+
+Usage: ``python .github/scripts/changelog_notes.py 0.2.0``
+
+Extracts the body under ``## [0.2.0] ...`` up to the next ``## [`` heading.
+Falls back to a generic line if the version has no changelog section.
+"""
+import pathlib
+import re
+import sys
+
+CHANGELOG = pathlib.Path("CHANGELOG.md")
+
+
+def notes_for(version: str, text: str) -> str:
+    pattern = re.compile(
+        r"^##\s*\[" + re.escape(version) + r"\][^\n]*\n(.*?)(?=^##\s*\[|\Z)",
+        re.MULTILINE | re.DOTALL,
+    )
+    match = pattern.search(text)
+    if match:
+        body = match.group(1).strip()
+        if body:
+            return body
+    return f"Release {version}."
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print("usage: changelog_notes.py <version>", file=sys.stderr)
+        return 2
+    version = sys.argv[1]
+    text = CHANGELOG.read_text(encoding="utf-8") if CHANGELOG.exists() else ""
+    print(notes_for(version, text))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.1.0/.github/scripts/package_version.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Print the package version (``__version__`` from the package __init__)."""
+import pathlib
+import re
+import sys
+
+INIT = pathlib.Path("src/PyReprism/__init__.py")
+
+
+def main() -> int:
+    match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']',
+                      INIT.read_text(encoding="utf-8"), re.MULTILINE)
+    if not match:
+        print("could not find __version__ in", INIT, file=sys.stderr)
+        return 1
+    print(match.group(1))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# Least-privilege by default.
+permissions:
+  contents: read
+
+# Cancel superseded runs on the same ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: "Lint & test (Python ${{ matrix.python-version }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint with flake8
+        run: flake8 src/PyReprism
+
+      - name: Run tests with pytest
+        run: pytest --cov=src/PyReprism --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          fail_ci_if_error: false
+
+  build:
+    name: "Build & check package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*

pyreprism-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,91 @@
+name: Publish 📦 to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Verify the tag matches the package version
+        run: |
+          PKG_VERSION="$(python .github/scripts/package_version.py)"
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          echo "tag=$TAG_VERSION package=$PKG_VERSION"
+          if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
+            echo "::error::Tag ($TAG_VERSION) does not match package version ($PKG_VERSION). Bump __version__ in src/PyReprism/__init__.py."
+            exit 1
+          fi
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/PyReprism/
+    permissions:
+      id-token: write   # Trusted Publishing (OIDC) — no API token needed.
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write   # required to create a release
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Extract release notes from CHANGELOG
+        run: python .github/scripts/changelog_notes.py "${GITHUB_REF_NAME#v}" > release-notes.md
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create "$GITHUB_REF_NAME" \
+            --title "$GITHUB_REF_NAME" \
+            --notes-file release-notes.md \
+            dist/*

{pyreprism-0.0.4 → pyreprism-0.1.0}/.gitignore

@@ -163,4 +163,9 @@ cython_debug/
 #.idea/
 
 # others
-__test__.py
+__test__.py
+
+# macOS
+.DS_Store
+# Claude Code agent workspace
+.claude/

pyreprism-0.1.0/.pre-commit-hooks.yaml

@@ -0,0 +1,27 @@
+# PyReprism pre-commit hooks.
+#
+# Add to your .pre-commit-config.yaml, e.g.:
+#
+#   - repo: https://github.com/unlv-evol/PyReprism
+#     rev: v0.1.0
+#     hooks:
+#       - id: pyreprism-strip-comments
+#         files: ^generated/           # scope it! these hooks rewrite files
+#
+# NOTE: these hooks modify source in place. Scope them with `files:`/`exclude:`
+# to directories where stripping comments/whitespace is intended (e.g. generated
+# or vendored code), not your whole repository.
+
+- id: pyreprism-strip-comments
+  name: PyReprism – strip comments
+  description: Remove comments in place (language auto-detected from file extension).
+  entry: pyreprism remove comments --in-place
+  language: python
+  types: [text]
+
+- id: pyreprism-normalize-whitespace
+  name: PyReprism – normalize whitespace
+  description: Collapse insignificant whitespace in place.
+  entry: pyreprism preprocess --steps whitespace --in-place
+  language: python
+  types: [text]

{pyreprism-0.0.4 → pyreprism-0.1.0}/.readthedocs.yaml

@@ -23,9 +23,10 @@ sphinx:
 # formats:
 #    - pdf
 
-# Optionally declare the Python requirements required to build your docs
+# Install the package together with its documentation extra.
 python:
    install:
-   - requirements: docs/requirements.txt
    - method: pip
-     path: .
+     path: .
+     extra_requirements:
+       - docs

pyreprism-0.1.0/CHANGELOG.md

@@ -0,0 +1,78 @@
+# Changelog
+
+All notable changes to PyReprism are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **Diff processing** (`PyReprism.diffs`): parse unified/`git` diffs and analyze
+  the changed code per file in its own language. Includes churn metrics
+  (`diff_stats`: added/removed split into code, comment and blank), cosmetic
+  (comment/whitespace-only) change detection, reconstruction of the added/removed
+  code, and ML-oriented `tokenize`/`normalize`/`extract_comments` on changes.
+  Fragment mode works from the diff alone; setting a file's `new_source` /
+  `old_source` enables accurate full-file classification. New CLI command
+  `pyreprism diff` (`--json` / `--csv` / `--per-file` / `--cosmetic`).
+
+## [0.1.0] - 2026-07-07
+
+This release turns PyReprism from a comment-removal helper into a full
+source-preprocessing toolkit with a high-level API, a CLI, code metrics,
+ML-oriented normalization, an optional accurate backend, and batch processing.
+
+### Added
+- **High-level API** (`import PyReprism as pr`): `remove_*`, `extract_*`,
+  `count_*`, `match_*` for comments, keywords, numbers, operators, strings and
+  identifiers; plus `preprocess`, `remove_whitespaces`, `get_language` and
+  `detect_language`. `lang=` accepts a language name, file extension, or class.
+- **Tokenizer**: `tokenize()` on every language returns a lossless, typed
+  `Token` stream (`Token`/`TokenType` in `PyReprism.tokens`).
+- **Code metrics**: `stats()` returns a `CodeStats` (line/token counts,
+  comment-to-code ratio, comment density; `as_dict()` for JSON/dataframes).
+- **ML normalization**: `normalize()` canonicalizes code (rename identifiers to
+  `VAR1…`, mask string/number literals, drop comments) for clone/plagiarism
+  detection and code-embedding pipelines; `blank_comments()` strips comments
+  while preserving line numbers.
+- **Pluggable backends**: `engine="regex"` (default, zero-dependency) or
+  `engine="pygments"` (`pip install pyreprism[accurate]`) for higher accuracy,
+  plus `"auto"`. Available on every operation.
+- **Batch processing** (`PyReprism.batch`): `analyze()` walks a directory tree
+  and returns an aggregate report (`to_json`/`to_csv`, per-language breakdown);
+  `transform()` bulk-applies a transformation into a mirrored output directory.
+- **Command-line interface** (`pyreprism`, also `python -m PyReprism`):
+  `remove`, `extract`, `count`, `preprocess`, `tokenize`, `stats`, `normalize`,
+  `scan`, and `languages`; reads stdin/files/globs/directories; supports
+  `--lang`, `--engine`, `--in-place`, `--output`, and `--json`/`--csv`.
+- **Source-based language detection**: `detect_language(source=...)` recognizes
+  `#!` shebangs and (when Pygments is installed) makes a best-effort content
+  guess; the CLI uses this to auto-detect the language of piped input.
+- Pre-commit hooks (`.pre-commit-hooks.yaml`), a `py.typed` marker (PEP 561),
+  and a registry-wide test suite covering all supported languages.
+
+### Changed
+- Implemented real support for 25 previously-empty language stubs and migrated
+  22 legacy languages (HTML, JSON, YAML, CSS, Markdown, …) to the
+  `BaseLanguage` + registry pattern, making them importable via the public API.
+- Ambiguous file extensions now resolve to a canonical language
+  (`.py` → Python, `.js` → JavaScript, …).
+- Consolidated packaging into `pyproject.toml` (single version source, pytest
+  and coverage config), updated CI to a Python 3.8–3.12 matrix, and refreshed
+  the README.
+
+### Fixed
+- Corrected comment-stripping for ~30 languages whose regexes either failed to
+  remove line comments or consumed surrounding code, and fixed duplicate/broken
+  definitions (Perl, csp, glsl, django, apacheconf, php_extras, ruby, and the
+  Python triple-quoted-docstring bug).
+
+## [0.0.4] - 2024
+
+- Early beta releases: comment removal for an initial set of languages and the
+  `Normalizer` whitespace helper.
+
+[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0
+[0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4

{pyreprism-0.0.4 → pyreprism-0.1.0}/CONTRIBUTING.md

@@ -62,4 +62,9 @@ When you're finished with the changes, create a pull request, also known as a PR
 ### Your PR is merged!
 
 Congratulations :tada: :tada: <br />
-The PyReprism team thanks you :sparkles:.
+The PyReprism team thanks you :sparkles:.
+
+## Maintainers: cutting a release
+
+Releases are automated by pushing a `vX.Y.Z` tag. See [RELEASING.md](RELEASING.md)
+for the full process.

{pyreprism-0.0.4 → pyreprism-0.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: PyReprism
-Version: 0.0.4
+Version: 0.1.0
 Summary: Framework for Source Code Preprocessing
 Project-URL: Homepage, https://github.com/unlv-evol/PyReprism
 Project-URL: Documentation, https://pyreprism.readthedocs.io
@@ -28,19 +28,32 @@ License: MIT License
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
 License-File: LICENSE
-Classifier: Development Status :: 5 - Production/Stable
+Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Python: >=3.6
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
+Provides-Extra: accurate
+Requires-Dist: pygments; extra == 'accurate'
 Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
 Requires-Dist: flake8; extra == 'dev'
+Requires-Dist: pygments; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme; extra == 'docs'
 Description-Content-Type: text/x-rst
 
 .. _intro_toplevel:

pyreprism-0.1.0/README.md

@@ -0,0 +1,246 @@
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![Downloads](https://static.pepy.tech/badge/pyreprism)](https://pepy.tech/project/pyreprism)
+[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B46484%2Fgithub.com%2Funlv-evol%2FPyReprism.svg?type=shield&issueType=license)](https://app.fossa.com/projects/custom%2B46484%2Fgithub.com%2Funlv-evol%2FPyReprism?ref=badge_shield&issueType=license)
+[![FOSSA Status](https://app.fossa.com/api/projects/custom%2B46484%2Fgithub.com%2Funlv-evol%2FPyReprism.svg?type=shield&issueType=security)](https://app.fossa.com/projects/custom%2B46484%2Fgithub.com%2Funlv-evol%2FPyReprism?ref=badge_shield&issueType=security)
+[![Documentation Status](https://readthedocs.org/projects/pyreprism/badge/?version=latest)](https://pyreprism.readthedocs.io/en/latest/?badge=latest)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/PyReprism)
+![CI](https://github.com/unlv-evol/PyReprism/actions/workflows/ci.yml/badge.svg)
+![Publish](https://github.com/unlv-evol/PyReprism/actions/workflows/publish.yml/badge.svg)
+[![codecov](https://codecov.io/gh/unlv-evol/PyReprism/graph/badge.svg?token=J2JV31837H)](https://codecov.io/gh/unlv-evol/PyReprism)
+![PyPI - Version](https://img.shields.io/pypi/v/pyreprism)
+![GitHub last commit (branch)](https://img.shields.io/github/last-commit/unlv-evol/pyreprism/main)
+[![SWH](https://archive.softwareheritage.org/badge/origin/https://pypi.org/project/PyReprism//)](https://archive.softwareheritage.org/browse/origin/?origin_url=https://pypi.org/project/PyReprism/)
+
+# PyReprism
+
+PyReprism is a Python framework that helps researchers and developers the task of source code preprocessing. With PyReprism, you can easily match, extract, count, and remove comments, whitespaces, operators, numbers and other language specific constructs from over **150 programming languages and file extensions**.
+
+## Install
+```shell
+pip install PyReprism
+```
+## Quick Usage
+
+The simplest entry point is the top-level API. Pass `lang` as a language name
+(`"python"`), a file extension (`".py"`), or a language class.
+
+```python
+import PyReprism as pr
+
+source = """
+# single line comment
+x = 5 + 6
+print(x)  # inline comment
+"""
+
+pr.remove_comments(source, lang="python")   # -> code with comments stripped
+pr.extract_comments(source, lang="python")  # -> ['# single line comment', '# inline comment']
+pr.count_comments(source, lang="python")    # -> 2
+```
+
+Every construct supports the same **match / extract / count / remove** verbs:
+
+```python
+code = 'total = price * 42 + tax  # compute'
+
+pr.extract_numbers(code, lang="python")      # -> ['42']
+pr.extract_keywords(code, lang="python")     # -> []
+pr.extract_identifiers(code, lang="python")  # -> ['total', 'price', 'tax', 'compute']
+pr.remove_numbers(code, lang="python")       # -> 'total = price *  + tax  # compute'
+pr.extract_strings('s = "hi"', lang="python")  # -> ['"hi"']
+```
+
+`match_*` returns positioned `Token`s (`type`, `value`, `start`, `end`, `line`):
+
+```python
+for tok in pr.match_comments(source, lang="python"):
+    print(tok.line, tok.value)
+```
+
+### Tokenize
+
+`tokenize()` returns a lossless, typed token stream (comment/string/number/
+keyword/identifier/operator/whitespace/other):
+
+```python
+for tok in pr.tokenize("x = 5 + y  // c", lang="clike"):
+    print(tok.type, repr(tok.value))
+```
+
+### Pipelines
+
+Chain removal steps in one call:
+
+```python
+pr.preprocess(source, lang="java",
+              steps=["comments", "strings", "whitespace"])
+```
+
+Valid steps: `comments`, `strings`, `numbers`, `operators`, `keywords`, `whitespace`.
+
+### Code metrics
+
+```python
+s = pr.stats(source, lang="python")
+s.code_lines, s.comment_lines, s.blank_lines
+s.comment_to_code_ratio, s.comment_density
+s.as_dict()   # includes per-token-type counts, ready for JSON / dataframes
+```
+
+### Normalization for ML / clone detection
+
+Canonicalize code so that only its structure remains — rename identifiers to
+`VAR1, VAR2, …`, mask string/number literals, and drop comments:
+
+```python
+pr.normalize("total = price * 42  # cost", lang="python")
+# -> 'VAR1 = VAR2 * 0'
+```
+
+Every part is toggleable (`rename_identifiers`, `mask_numbers`, `mask_strings`,
+`drop_comments`, `collapse_whitespace`). To strip comments while keeping line
+numbers stable (useful for tools that map back to source):
+
+```python
+pr.blank_comments(source, lang="python")   # comment content removed, newlines kept
+```
+
+### Accuracy: choose a backend
+
+By default PyReprism uses its own zero-dependency regex engine (fast, no installs).
+For higher accuracy — e.g. correctly ignoring a `#` inside a string — pass
+`engine="pygments"` (install the optional extra with `pip install pyreprism[accurate]`):
+
+```python
+src = 'url = "http://x#frag"  # real comment'
+
+pr.remove_comments(src, lang="python")                     # regex (default)
+pr.remove_comments(src, lang="python", engine="pygments")  # keeps the URL, drops the comment
+```
+
+`engine` accepts `"regex"` (default), `"pygments"`, or `"auto"` (use pygments if
+installed, else regex). It works on every operation — `remove_*`, `extract_*`,
+`count_*`, `tokenize`, `normalize`, `stats`, `preprocess` — and on the CLI via
+`--engine`.
+
+### Batch / whole-directory processing
+
+Analyze or transform an entire source tree (junk folders like `.git`,
+`node_modules`, `venv` are skipped automatically):
+
+```python
+from PyReprism import batch
+
+report = batch.analyze("myproject/")     # walk the tree, compute metrics
+report.totals()                          # aggregate line/token counts
+report.by_language()                     # per-language breakdown
+report.to_json(); report.to_csv()        # export for dataframes / notebooks
+
+# Bulk-transform into a mirrored output directory:
+batch.transform("myproject/", lambda text, lang: lang.remove_comments(text),
+                output="stripped/")
+```
+
+### Process diffs / pull requests
+
+Parse a unified (`git`) diff and analyze the *changed code* per file, language-aware:
+
+```python
+from PyReprism import diffs
+
+d = diffs.parse(diff_text)          # git diff / .diff / .patch text
+
+report = diffs.diff_stats(d)        # churn split into code vs comment vs blank
+report.totals(); report.to_csv()
+
+diffs.cosmetic_files(d)             # files whose change is comment/whitespace-only
+
+f = d.files[0]
+f.language                          # detected from the file path
+f.added_text(); f.removed_text()    # reconstructed changed code
+f.normalize("added")               # canonicalize the added code for ML
+f.extract_comments("added")
+```
+
+For accuracy where a comment/string spans a hunk boundary, set
+`f.new_source` / `f.old_source` to the full file contents (e.g. from `git show`)
+and the changed lines are classified against the whole file. On the CLI:
+
+```shell
+git diff | pyreprism diff --per-file      # churn report
+git diff | pyreprism diff --json
+git diff | pyreprism diff --cosmetic      # list comment/whitespace-only changes
+```
+
+### Detect the language from a filename
+
+```python
+cls = pr.detect_language(filename="src/main.go")   # -> Go language class
+```
+
+### Language-independent whitespace normalization
+
+```python
+pr.remove_whitespaces("x = 5 + 6\n\n\nprint(x)")  # -> 'x=5+6\nprint(x)'
+```
+
+You can also import a language class directly if you prefer:
+
+```python
+from PyReprism.languages import Python
+Python.remove_comments(source)
+```
+
+## Command line
+
+Installing PyReprism also provides a `pyreprism` command (works on stdin/stdout,
+files, and globs; the language is auto-detected from a file's extension):
+
+```shell
+pyreprism remove comments file.py
+cat file.go | pyreprism remove comments --lang go
+pyreprism extract comments "src/**/*.py" --json
+pyreprism count comments file.py --lang python
+pyreprism preprocess --steps comments,strings,whitespace file.java
+pyreprism tokenize --json file.py
+pyreprism stats --json file.py                 # line/token metrics
+pyreprism normalize file.py                    # canonicalize for ML
+pyreprism remove comments file.py --in-place   # rewrite in place
+pyreprism scan myproject/ --csv                # aggregate metrics over a tree
+pyreprism remove comments src/ --output out/   # bulk-transform a directory
+pyreprism languages                            # list supported languages
+```
+
+Equivalently: `python -m PyReprism ...`.
+
+It can also be wired into [pre-commit](https://pre-commit.com) via the bundled
+`.pre-commit-hooks.yaml` (`pyreprism-strip-comments`, `pyreprism-normalize-whitespace`).
+These hooks rewrite files, so scope them with `files:`/`exclude:`.
+
+Read the [docs](https://pyreprism.readthedocs.io) for more usage examples. 
+
+NB: The beta versions of PyReprism is still unstable, but we are working 24/7 to ensure the tool is usable.
+
+## How to Contribute
+We invite you to help us build this tool and make it more extensive. Contribution is open to OSS community.
+
+```shell
+$ git clone https://github.com/unlv-evol/PyReprism.git
+$ cd PyReprism
+```
+**(Optional)** It is suggested to make use of virtualenv. Therefore, before installing the requirements run:
+
+```shell
+$ python3 -m venv venv
+$ source venv/bin/activate
+```
+
+Then, install the package in editable mode with its development tooling:
+
+```shell
+$ pip install -e ".[dev]"
+```
+For more information on how to contribute, read our [contributing guidelines](CONTRIBUTING.md).
+
+## Issues
+If you experience any issue, feel free to report it.