PyReprism 0.0.4__tar.gz → 0.2.0__tar.gz

pyreprism-0.2.0/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+show-source = True
+count = True
+statistics = True
+max-line-length = 120
+# E501: long lines are common in this project's regex/keyword tables.
+# The W1/W2/W3 and E1/E2/E3 codes below are pre-existing whitespace/indentation
+# style debt across the legacy language modules; they are ignored so flake8 can
+# focus on real correctness issues (pyflakes F-codes, undefined names, etc.).
+extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266
+exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,tests,src/PyReprism/languages/__init__.py,__test__.py

pyreprism-0.2.0/.github/scripts/changelog_notes.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Print the CHANGELOG.md section for a given version.
+
+Usage: ``python .github/scripts/changelog_notes.py 0.2.0``
+
+Extracts the body under ``## [0.2.0] ...`` up to the next ``## [`` heading.
+Falls back to a generic line if the version has no changelog section.
+"""
+import pathlib
+import re
+import sys
+
+CHANGELOG = pathlib.Path("CHANGELOG.md")
+
+
+def notes_for(version: str, text: str) -> str:
+    pattern = re.compile(
+        r"^##\s*\[" + re.escape(version) + r"\][^\n]*\n(.*?)(?=^##\s*\[|\Z)",
+        re.MULTILINE | re.DOTALL,
+    )
+    match = pattern.search(text)
+    if match:
+        body = match.group(1).strip()
+        if body:
+            return body
+    return f"Release {version}."
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print("usage: changelog_notes.py <version>", file=sys.stderr)
+        return 2
+    version = sys.argv[1]
+    text = CHANGELOG.read_text(encoding="utf-8") if CHANGELOG.exists() else ""
+    print(notes_for(version, text))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.2.0/.github/scripts/package_version.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Print the package version (``__version__`` from the package __init__)."""
+import pathlib
+import re
+import sys
+
+INIT = pathlib.Path("src/PyReprism/__init__.py")
+
+
+def main() -> int:
+    match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']',
+                      INIT.read_text(encoding="utf-8"), re.MULTILINE)
+    if not match:
+        print("could not find __version__ in", INIT, file=sys.stderr)
+        return 1
+    print(match.group(1))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# Least-privilege by default.
+permissions:
+  contents: read
+
+# Cancel superseded runs on the same ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: "Lint & test (Python ${{ matrix.python-version }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint with flake8
+        run: flake8 src/PyReprism
+
+      - name: Run tests with pytest
+        run: pytest --cov=src/PyReprism --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          fail_ci_if_error: false
+
+  build:
+    name: "Build & check package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*

pyreprism-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,91 @@
+name: Publish 📦 to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Verify the tag matches the package version
+        run: |
+          PKG_VERSION="$(python .github/scripts/package_version.py)"
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          echo "tag=$TAG_VERSION package=$PKG_VERSION"
+          if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
+            echo "::error::Tag ($TAG_VERSION) does not match package version ($PKG_VERSION). Bump __version__ in src/PyReprism/__init__.py."
+            exit 1
+          fi
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/PyReprism/
+    permissions:
+      id-token: write   # Trusted Publishing (OIDC) — no API token needed.
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write   # required to create a release
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Extract release notes from CHANGELOG
+        run: python .github/scripts/changelog_notes.py "${GITHUB_REF_NAME#v}" > release-notes.md
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create "$GITHUB_REF_NAME" \
+            --title "$GITHUB_REF_NAME" \
+            --notes-file release-notes.md \
+            dist/*

{pyreprism-0.0.4 → pyreprism-0.2.0}/.gitignore

@@ -163,4 +163,9 @@ cython_debug/
 #.idea/
 
 # others
-__test__.py
+__test__.py
+
+# macOS
+.DS_Store
+# Claude Code agent workspace
+.claude/

pyreprism-0.2.0/.pre-commit-hooks.yaml

@@ -0,0 +1,27 @@
+# PyReprism pre-commit hooks.
+#
+# Add to your .pre-commit-config.yaml, e.g.:
+#
+#   - repo: https://github.com/unlv-evol/PyReprism
+#     rev: v0.1.0
+#     hooks:
+#       - id: pyreprism-strip-comments
+#         files: ^generated/           # scope it! these hooks rewrite files
+#
+# NOTE: these hooks modify source in place. Scope them with `files:`/`exclude:`
+# to directories where stripping comments/whitespace is intended (e.g. generated
+# or vendored code), not your whole repository.
+
+- id: pyreprism-strip-comments
+  name: PyReprism – strip comments
+  description: Remove comments in place (language auto-detected from file extension).
+  entry: pyreprism remove comments --in-place
+  language: python
+  types: [text]
+
+- id: pyreprism-normalize-whitespace
+  name: PyReprism – normalize whitespace
+  description: Collapse insignificant whitespace in place.
+  entry: pyreprism preprocess --steps whitespace --in-place
+  language: python
+  types: [text]

{pyreprism-0.0.4 → pyreprism-0.2.0}/.readthedocs.yaml

@@ -23,9 +23,10 @@ sphinx:
 # formats:
 #    - pdf
 
-# Optionally declare the Python requirements required to build your docs
+# Install the package together with its documentation extra.
 python:
    install:
-   - requirements: docs/requirements.txt
    - method: pip
-     path: .
+     path: .
+     extra_requirements:
+       - docs

pyreprism-0.2.0/CHANGELOG.md

@@ -0,0 +1,118 @@
+# Changelog
+
+All notable changes to PyReprism are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.2.0] - 2026-07-08
+
+This release adds a static code-analysis toolkit on top of the tokenizer:
+diff/PR analysis, complexity metrics, n-gram "naturalness", and clone/plagiarism
+similarity — plus a comprehensive per-language test suite that hardened comment
+handling across the board.
+
+### Added
+- **Fingerprinting & similarity** (`PyReprism.fingerprints`): winnowing k-gram
+  fingerprints over the normalized token stream for clone/plagiarism detection —
+  `fingerprint()`, `similarity()`/`containment()` (rename-invariant by default),
+  and a `FingerprintIndex` for many-to-many detection over a corpus. CLI:
+  `pyreprism similarity a b` and `pyreprism clones DIR --threshold`.
+- **N-gram analysis & code naturalness** (`PyReprism.ngrams`): token/type n-gram
+  extraction and frequency counts, plus an `NgramModel` (add-k smoothing,
+  save/load) that measures cross-entropy / perplexity against a trained corpus
+  ("naturalness of software"). CLI: `pyreprism ngrams` and
+  `pyreprism perplexity --train`.
+- **Complexity metrics** computed from the token stream: `halstead()`
+  (volume/difficulty/effort/bugs), `cyclomatic_complexity()` (approximate McCabe),
+  `maintainability_index()` (0–100), `max_nesting_depth()`, and `code_metrics()`
+  which bundles them with the line/token stats. Exposed on the CLI via
+  `pyreprism stats --full`.
+- **Diff processing** (`PyReprism.diffs`): parse unified/`git` diffs and analyze
+  the changed code per file in its own language. Includes churn metrics
+  (`diff_stats`: added/removed split into code, comment and blank), cosmetic
+  (comment/whitespace-only) change detection, reconstruction of the added/removed
+  code, and ML-oriented `tokenize`/`normalize`/`extract_comments` on changes.
+  Fragment mode works from the diff alone; setting a file's `new_source` /
+  `old_source` enables accurate full-file classification. New CLI command
+  `pyreprism diff` (`--json` / `--csv` / `--per-file` / `--cosmetic`).
+
+### Changed
+- Adopted the standard `src/` layout with tests at the top level, consolidated
+  all dependencies into `pyproject.toml` extras, and rewrote the landing page.
+- Single-sourced the version from the package and automated releases: pushing a
+  `vX.Y.Z` tag now publishes to PyPI (Trusted Publishing) and creates a GitHub
+  Release. Added Python 3.13 to the CI matrix.
+- Added a comprehensive, data-driven per-language comment test suite that
+  requires every registered language to be covered.
+
+### Fixed
+- Fixed comment stripping in 8 more languages surfaced by the new test suite:
+  `smalltalk` (kept the comment and deleted the code) and
+  `eiffel`/`bro`/`coffeescript`/`io`/`nix`/`gherkin`/`gedcom` (trailing comments
+  not removed and newlines dropped).
+- Fixed a CLI argument-ordering incompatibility on Python ≤ 3.11 (options must
+  follow positionals).
+
+## [0.1.0] - 2026-07-07
+
+This release turns PyReprism from a comment-removal helper into a full
+source-preprocessing toolkit with a high-level API, a CLI, code metrics,
+ML-oriented normalization, an optional accurate backend, and batch processing.
+
+### Added
+- **High-level API** (`import PyReprism as pr`): `remove_*`, `extract_*`,
+  `count_*`, `match_*` for comments, keywords, numbers, operators, strings and
+  identifiers; plus `preprocess`, `remove_whitespaces`, `get_language` and
+  `detect_language`. `lang=` accepts a language name, file extension, or class.
+- **Tokenizer**: `tokenize()` on every language returns a lossless, typed
+  `Token` stream (`Token`/`TokenType` in `PyReprism.tokens`).
+- **Code metrics**: `stats()` returns a `CodeStats` (line/token counts,
+  comment-to-code ratio, comment density; `as_dict()` for JSON/dataframes).
+- **ML normalization**: `normalize()` canonicalizes code (rename identifiers to
+  `VAR1…`, mask string/number literals, drop comments) for clone/plagiarism
+  detection and code-embedding pipelines; `blank_comments()` strips comments
+  while preserving line numbers.
+- **Pluggable backends**: `engine="regex"` (default, zero-dependency) or
+  `engine="pygments"` (`pip install pyreprism[accurate]`) for higher accuracy,
+  plus `"auto"`. Available on every operation.
+- **Batch processing** (`PyReprism.batch`): `analyze()` walks a directory tree
+  and returns an aggregate report (`to_json`/`to_csv`, per-language breakdown);
+  `transform()` bulk-applies a transformation into a mirrored output directory.
+- **Command-line interface** (`pyreprism`, also `python -m PyReprism`):
+  `remove`, `extract`, `count`, `preprocess`, `tokenize`, `stats`, `normalize`,
+  `scan`, and `languages`; reads stdin/files/globs/directories; supports
+  `--lang`, `--engine`, `--in-place`, `--output`, and `--json`/`--csv`.
+- **Source-based language detection**: `detect_language(source=...)` recognizes
+  `#!` shebangs and (when Pygments is installed) makes a best-effort content
+  guess; the CLI uses this to auto-detect the language of piped input.
+- Pre-commit hooks (`.pre-commit-hooks.yaml`), a `py.typed` marker (PEP 561),
+  and a registry-wide test suite covering all supported languages.
+
+### Changed
+- Implemented real support for 25 previously-empty language stubs and migrated
+  22 legacy languages (HTML, JSON, YAML, CSS, Markdown, …) to the
+  `BaseLanguage` + registry pattern, making them importable via the public API.
+- Ambiguous file extensions now resolve to a canonical language
+  (`.py` → Python, `.js` → JavaScript, …).
+- Consolidated packaging into `pyproject.toml` (single version source, pytest
+  and coverage config), updated CI to a Python 3.8–3.12 matrix, and refreshed
+  the README.
+
+### Fixed
+- Corrected comment-stripping for ~30 languages whose regexes either failed to
+  remove line comments or consumed surrounding code, and fixed duplicate/broken
+  definitions (Perl, csp, glsl, django, apacheconf, php_extras, ruby, and the
+  Python triple-quoted-docstring bug).
+
+## [0.0.4] - 2024
+
+- Early beta releases: comment removal for an initial set of languages and the
+  `Normalizer` whitespace helper.
+
+[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...v0.2.0
+[0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0
+[0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4

{pyreprism-0.0.4 → pyreprism-0.2.0}/CONTRIBUTING.md

@@ -62,4 +62,9 @@ When you're finished with the changes, create a pull request, also known as a PR
 ### Your PR is merged!
 
 Congratulations :tada: :tada: <br />
-The PyReprism team thanks you :sparkles:.
+The PyReprism team thanks you :sparkles:.
+
+## Maintainers: cutting a release
+
+Releases are automated by pushing a `vX.Y.Z` tag. See [RELEASING.md](RELEASING.md)
+for the full process.

pyreprism-0.2.0/PKG-INFO

@@ -0,0 +1,158 @@
+Metadata-Version: 2.4
+Name: PyReprism
+Version: 0.2.0
+Summary: Framework for Source Code Preprocessing
+Project-URL: Homepage, https://github.com/unlv-evol/PyReprism
+Project-URL: Documentation, https://pyreprism.readthedocs.io
+Project-URL: Issues, https://github.com/unlv-evol/PyReprism/issues
+Author-email: UNLV EVOL LAB <ogenrwot@unlv.nevada.edu>
+License: MIT License
+        
+        Copyright (c) 2024 UNLV EVOL
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
+Provides-Extra: accurate
+Requires-Dist: pygments; extra == 'accurate'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: flake8; extra == 'dev'
+Requires-Dist: pygments; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme; extra == 'docs'
+Description-Content-Type: text/x-rst
+
+.. _intro_toplevel:
+
+========
+Overview
+========
+
+**PyReprism** is a Python framework for source-code preprocessing. It lets you
+**match, extract, count, and remove** comments, strings, numbers, operators,
+keywords and other language-specific constructs across **145+ programming
+languages and file formats** — through a small high-level API, a command-line
+tool, code metrics, ML-oriented normalization, and diff/pull-request analysis.
+
+Highlights
+==========
+
+* One consistent API for every language: ``remove_*`` / ``extract_*`` /
+  ``count_*`` / ``match_*`` for each construct, plus a lossless ``tokenize()``.
+* Code metrics (``stats``) and ML canonicalization (``normalize``) for
+  clone/plagiarism detection and code-embedding pipelines.
+* Optional high-accuracy `Pygments`_ backend — with **zero required
+  dependencies** by default.
+* Batch/corpus analysis and unified/``git`` diff processing.
+* A ``pyreprism`` command-line tool (also ``python -m PyReprism``).
+
+Requirements
+============
+
+* `Python`_ 3.8 or newer
+
+Install
+=======
+
+.. code-block:: shell
+
+    pip install PyReprism
+
+For the higher-accuracy tokenizer backend:
+
+.. code-block:: shell
+
+    pip install "PyReprism[accurate]"
+
+Quick start
+===========
+
+.. code-block:: python
+
+    import PyReprism as pr
+
+    source = """
+    # a comment
+    x = 5 + 6
+    print(x)  # inline
+    """
+
+    pr.remove_comments(source, lang="python")   # code without comments
+    pr.extract_comments(source, lang="python")  # ['# a comment', '# inline']
+    pr.count_comments(source, lang="python")     # 2
+
+``lang`` accepts a language name (``"python"``), a file extension (``".py"``),
+or a language class. Canonicalize code for machine learning, or measure it:
+
+.. code-block:: python
+
+    pr.normalize("total = price * 42", lang="python")   # 'VAR1 = VAR2 * 0'
+    pr.stats(source, lang="python").comment_to_code_ratio
+
+Command line
+============
+
+.. code-block:: shell
+
+    pyreprism remove comments file.py
+    pyreprism scan myproject/ --csv          # aggregate code metrics over a tree
+    git diff | pyreprism diff --cosmetic     # flag comment/whitespace-only changes
+
+Documentation
+=============
+
+Full documentation, including the API reference and the list of supported
+languages, is available at https://pyreprism.readthedocs.io.
+
+Source code
+===========
+
+PyReprism is developed on GitHub at
+https://github.com/unlv-evol/PyReprism. To work on it locally:
+
+.. code-block:: shell
+
+    git clone https://github.com/unlv-evol/PyReprism.git
+    cd PyReprism
+    python3 -m venv venv && source venv/bin/activate
+    pip install -e ".[dev]"
+    pytest
+
+See ``CONTRIBUTING.md`` for contribution guidelines.
+
+.. _Python: https://www.python.org
+.. _Pygments: https://pygments.org