PyReprism 0.0.3__tar.gz → 0.1.0__tar.gz

pyreprism-0.1.0/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+show-source = True
+count = True
+statistics = True
+max-line-length = 120
+# E501: long lines are common in this project's regex/keyword tables.
+# The W1/W2/W3 and E1/E2/E3 codes below are pre-existing whitespace/indentation
+# style debt across the legacy language modules; they are ignored so flake8 can
+# focus on real correctness issues (pyflakes F-codes, undefined names, etc.).
+extend-ignore = E501, W191, W291, W292, W293, W391, E101, E117, E128, E231, E302, E303, E265, E266
+exclude = .git,.github,venv,.venv,build,dist,docs,htmlcov,tests,src/PyReprism/languages/__init__.py,__test__.py

pyreprism-0.1.0/.github/scripts/changelog_notes.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Print the CHANGELOG.md section for a given version.
+
+Usage: ``python .github/scripts/changelog_notes.py 0.2.0``
+
+Extracts the body under ``## [0.2.0] ...`` up to the next ``## [`` heading.
+Falls back to a generic line if the version has no changelog section.
+"""
+import pathlib
+import re
+import sys
+
+CHANGELOG = pathlib.Path("CHANGELOG.md")
+
+
+def notes_for(version: str, text: str) -> str:
+    pattern = re.compile(
+        r"^##\s*\[" + re.escape(version) + r"\][^\n]*\n(.*?)(?=^##\s*\[|\Z)",
+        re.MULTILINE | re.DOTALL,
+    )
+    match = pattern.search(text)
+    if match:
+        body = match.group(1).strip()
+        if body:
+            return body
+    return f"Release {version}."
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print("usage: changelog_notes.py <version>", file=sys.stderr)
+        return 2
+    version = sys.argv[1]
+    text = CHANGELOG.read_text(encoding="utf-8") if CHANGELOG.exists() else ""
+    print(notes_for(version, text))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.1.0/.github/scripts/package_version.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Print the package version (``__version__`` from the package __init__)."""
+import pathlib
+import re
+import sys
+
+INIT = pathlib.Path("src/PyReprism/__init__.py")
+
+
+def main() -> int:
+    match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']',
+                      INIT.read_text(encoding="utf-8"), re.MULTILINE)
+    if not match:
+        print("could not find __version__ in", INIT, file=sys.stderr)
+        return 1
+    print(match.group(1))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

pyreprism-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# Least-privilege by default.
+permissions:
+  contents: read
+
+# Cancel superseded runs on the same ref.
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: "Lint & test (Python ${{ matrix.python-version }})"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint with flake8
+        run: flake8 src/PyReprism
+
+      - name: Run tests with pytest
+        run: pytest --cov=src/PyReprism --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          fail_ci_if_error: false
+
+  build:
+    name: "Build & check package"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*

pyreprism-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,91 @@
+name: Publish 📦 to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Verify the tag matches the package version
+        run: |
+          PKG_VERSION="$(python .github/scripts/package_version.py)"
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          echo "tag=$TAG_VERSION package=$PKG_VERSION"
+          if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then
+            echo "::error::Tag ($TAG_VERSION) does not match package version ($PKG_VERSION). Bump __version__ in src/PyReprism/__init__.py."
+            exit 1
+          fi
+
+      - name: Build and validate the distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/PyReprism/
+    permissions:
+      id-token: write   # Trusted Publishing (OIDC) — no API token needed.
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write   # required to create a release
+    steps:
+      - name: Check out the code
+        uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Extract release notes from CHANGELOG
+        run: python .github/scripts/changelog_notes.py "${GITHUB_REF_NAME#v}" > release-notes.md
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create "$GITHUB_REF_NAME" \
+            --title "$GITHUB_REF_NAME" \
+            --notes-file release-notes.md \
+            dist/*

{pyreprism-0.0.3 → pyreprism-0.1.0}/.gitignore

@@ -163,4 +163,9 @@ cython_debug/
 #.idea/
 
 # others
-__test__.py
+__test__.py
+
+# macOS
+.DS_Store
+# Claude Code agent workspace
+.claude/

pyreprism-0.1.0/.pre-commit-hooks.yaml

@@ -0,0 +1,27 @@
+# PyReprism pre-commit hooks.
+#
+# Add to your .pre-commit-config.yaml, e.g.:
+#
+#   - repo: https://github.com/unlv-evol/PyReprism
+#     rev: v0.1.0
+#     hooks:
+#       - id: pyreprism-strip-comments
+#         files: ^generated/           # scope it! these hooks rewrite files
+#
+# NOTE: these hooks modify source in place. Scope them with `files:`/`exclude:`
+# to directories where stripping comments/whitespace is intended (e.g. generated
+# or vendored code), not your whole repository.
+
+- id: pyreprism-strip-comments
+  name: PyReprism – strip comments
+  description: Remove comments in place (language auto-detected from file extension).
+  entry: pyreprism remove comments --in-place
+  language: python
+  types: [text]
+
+- id: pyreprism-normalize-whitespace
+  name: PyReprism – normalize whitespace
+  description: Collapse insignificant whitespace in place.
+  entry: pyreprism preprocess --steps whitespace --in-place
+  language: python
+  types: [text]

{pyreprism-0.0.3 → pyreprism-0.1.0}/.readthedocs.yaml

@@ -23,9 +23,10 @@ sphinx:
 # formats:
 #    - pdf
 
-# Optionally declare the Python requirements required to build your docs
+# Install the package together with its documentation extra.
 python:
    install:
-   - requirements: docs/requirements.txt
    - method: pip
-     path: .
+     path: .
+     extra_requirements:
+       - docs

pyreprism-0.1.0/CHANGELOG.md

@@ -0,0 +1,78 @@
+# Changelog
+
+All notable changes to PyReprism are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- **Diff processing** (`PyReprism.diffs`): parse unified/`git` diffs and analyze
+  the changed code per file in its own language. Includes churn metrics
+  (`diff_stats`: added/removed split into code, comment and blank), cosmetic
+  (comment/whitespace-only) change detection, reconstruction of the added/removed
+  code, and ML-oriented `tokenize`/`normalize`/`extract_comments` on changes.
+  Fragment mode works from the diff alone; setting a file's `new_source` /
+  `old_source` enables accurate full-file classification. New CLI command
+  `pyreprism diff` (`--json` / `--csv` / `--per-file` / `--cosmetic`).
+
+## [0.1.0] - 2026-07-07
+
+This release turns PyReprism from a comment-removal helper into a full
+source-preprocessing toolkit with a high-level API, a CLI, code metrics,
+ML-oriented normalization, an optional accurate backend, and batch processing.
+
+### Added
+- **High-level API** (`import PyReprism as pr`): `remove_*`, `extract_*`,
+  `count_*`, `match_*` for comments, keywords, numbers, operators, strings and
+  identifiers; plus `preprocess`, `remove_whitespaces`, `get_language` and
+  `detect_language`. `lang=` accepts a language name, file extension, or class.
+- **Tokenizer**: `tokenize()` on every language returns a lossless, typed
+  `Token` stream (`Token`/`TokenType` in `PyReprism.tokens`).
+- **Code metrics**: `stats()` returns a `CodeStats` (line/token counts,
+  comment-to-code ratio, comment density; `as_dict()` for JSON/dataframes).
+- **ML normalization**: `normalize()` canonicalizes code (rename identifiers to
+  `VAR1…`, mask string/number literals, drop comments) for clone/plagiarism
+  detection and code-embedding pipelines; `blank_comments()` strips comments
+  while preserving line numbers.
+- **Pluggable backends**: `engine="regex"` (default, zero-dependency) or
+  `engine="pygments"` (`pip install pyreprism[accurate]`) for higher accuracy,
+  plus `"auto"`. Available on every operation.
+- **Batch processing** (`PyReprism.batch`): `analyze()` walks a directory tree
+  and returns an aggregate report (`to_json`/`to_csv`, per-language breakdown);
+  `transform()` bulk-applies a transformation into a mirrored output directory.
+- **Command-line interface** (`pyreprism`, also `python -m PyReprism`):
+  `remove`, `extract`, `count`, `preprocess`, `tokenize`, `stats`, `normalize`,
+  `scan`, and `languages`; reads stdin/files/globs/directories; supports
+  `--lang`, `--engine`, `--in-place`, `--output`, and `--json`/`--csv`.
+- **Source-based language detection**: `detect_language(source=...)` recognizes
+  `#!` shebangs and (when Pygments is installed) makes a best-effort content
+  guess; the CLI uses this to auto-detect the language of piped input.
+- Pre-commit hooks (`.pre-commit-hooks.yaml`), a `py.typed` marker (PEP 561),
+  and a registry-wide test suite covering all supported languages.
+
+### Changed
+- Implemented real support for 25 previously-empty language stubs and migrated
+  22 legacy languages (HTML, JSON, YAML, CSS, Markdown, …) to the
+  `BaseLanguage` + registry pattern, making them importable via the public API.
+- Ambiguous file extensions now resolve to a canonical language
+  (`.py` → Python, `.js` → JavaScript, …).
+- Consolidated packaging into `pyproject.toml` (single version source, pytest
+  and coverage config), updated CI to a Python 3.8–3.12 matrix, and refreshed
+  the README.
+
+### Fixed
+- Corrected comment-stripping for ~30 languages whose regexes either failed to
+  remove line comments or consumed surrounding code, and fixed duplicate/broken
+  definitions (Perl, csp, glsl, django, apacheconf, php_extras, ruby, and the
+  Python triple-quoted-docstring bug).
+
+## [0.0.4] - 2024
+
+- Early beta releases: comment removal for an initial set of languages and the
+  `Normalizer` whitespace helper.
+
+[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0
+[0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4

{pyreprism-0.0.3 → pyreprism-0.1.0}/CONTRIBUTING.md

@@ -61,5 +61,10 @@ When you're finished with the changes, create a pull request, also known as a PR
 
 ### Your PR is merged!
 
-Congratulations :tada::tada: <br />
-The PyReprism team thanks you :sparkles:.
+Congratulations :tada: :tada: <br />
+The PyReprism team thanks you :sparkles:.
+
+## Maintainers: cutting a release
+
+Releases are automated by pushing a `vX.Y.Z` tag. See [RELEASING.md](RELEASING.md)
+for the full process.

pyreprism-0.1.0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: PyReprism
+Version: 0.1.0
+Summary: Framework for Source Code Preprocessing
+Project-URL: Homepage, https://github.com/unlv-evol/PyReprism
+Project-URL: Documentation, https://pyreprism.readthedocs.io
+Project-URL: Issues, https://github.com/unlv-evol/PyReprism/issues
+Author-email: UNLV EVOL LAB <ogenrwot@unlv.nevada.edu>
+License: MIT License
+        
+        Copyright (c) 2024 UNLV EVOL
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.8
+Provides-Extra: accurate
+Requires-Dist: pygments; extra == 'accurate'
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: flake8; extra == 'dev'
+Requires-Dist: pygments; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Requires-Dist: twine; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: sphinx; extra == 'docs'
+Requires-Dist: sphinx-rtd-theme; extra == 'docs'
+Description-Content-Type: text/x-rst
+
+.. _intro_toplevel:
+
+==================
+Overview / Install
+==================
+
+PyReprism is a Python framework that helps researchers and developers the task of source code preprocessing. With PyReprism, you can easily match, extract, count, and remove comments, whitespaces, operators, numbers and other language specific constructs from over 150 programming languages and file extensions.
+
+
+Requirements
+============
+
+* `Python`_ 3.6 or newer
+* `Git`_
+
+.. _Python: https://www.python.org
+.. _Git: https://git-scm.com/
+
+Installing PyReprism
+====================
+
+Installing PyReprism is easily done using `pip`_. Assuming it is installed, just run the following from the command-line:
+
+.. _pip: https://pip.pypa.io/en/latest/installing.html
+
+.. sourcecode:: none
+
+    $ pip install PyReprism
+
+Source Code
+===========
+
+PyReprism's git repo is available on GitHub, which can be browsed at:
+
+ * https://github.com/unlv-evol/PyReprism.git
+
+and cloned using::
+
+    $ git clone https://github.com/unlv-evol/PyReprism.git
+    $ cd PyReprism
+
+Optionally (but suggested), make use of virtual environment. Therefore, before installing the requirements run::
+    
+    $ python3 -m venv venv
+    $ source venv/bin/activate
+
+Install the requirements::
+    
+    $ pip install -r requirements.txt
+
+and run the tests using pytest::
+
+    $ pytest