PyReprism 0.2.0__tar.gz → 1.0.0__tar.gz

pyreprism-1.0.0/.dockerignore

@@ -0,0 +1,18 @@
+# Keep the build context small: only pyproject.toml, LICENSE, docs/intro.rst,
+# and src/ are needed to build the wheel (see Dockerfile).
+.git
+.github
+manuscript
+tests
+venv
+.venv
+docs/*
+!docs/intro.rst
+*.egg-info
+__pycache__
+.pytest_cache
+.mypy_cache
+dist
+build
+*.pyc
+.DS_Store

pyreprism-1.0.0/.github/pyreprism-action.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# Entry point for the "PyReprism Clone Gate" composite action (see action.yml).
+# Installs PyReprism from the action's own checked-out source (so the CLI matches
+# the pinned ref), runs `pyreprism clones` with the configured inputs, writes the
+# report, and propagates the duplication-gate exit code.
+set -euo pipefail
+
+action_path="${GITHUB_ACTION_PATH:-$(cd "$(dirname "$0")/.." && pwd)}"
+
+extras="accurate"
+if [ "${INPUT_SEMANTIC:-false}" = "true" ]; then
+  extras="accurate,ast,semantic"
+fi
+
+venv="${RUNNER_TEMP:-/tmp}/pyreprism-venv"
+echo "::group::Install PyReprism [${extras}]"
+python3 -m venv "$venv"
+"$venv/bin/python" -m pip install --quiet --upgrade pip
+"$venv/bin/python" -m pip install --quiet "${action_path}[${extras}]"
+"$venv/bin/pyreprism" --version
+echo "::endgroup::"
+
+# Build the `pyreprism clones` argument list from the inputs.
+args=(clones)
+# Intentionally word-split PATHS and EXTRA_ARGS (space-separated inputs).
+# shellcheck disable=SC2206
+args+=(${INPUT_PATHS:-.})
+args+=(--reporter "${INPUT_REPORTER:-sarif}")
+args+=(--min-tokens "${INPUT_MIN_TOKENS:-50}" --min-lines "${INPUT_MIN_LINES:-5}")
+[ -n "${INPUT_LANG:-}" ] && args+=(--lang "$INPUT_LANG")
+[ "${INPUT_NEAR:-false}" = "true" ] && args+=(--near)
+[ "${INPUT_SEMANTIC:-false}" = "true" ] && args+=(--semantic)
+[ -n "${INPUT_MAX_DUPLICATION:-}" ] && args+=(--max-duplication "$INPUT_MAX_DUPLICATION")
+# shellcheck disable=SC2206
+[ -n "${INPUT_EXTRA_ARGS:-}" ] && args+=(${INPUT_EXTRA_ARGS})
+
+out="${INPUT_OUTPUT:-}"
+echo "::group::pyreprism ${args[*]}"
+set +e
+if [ -n "$out" ]; then
+  "$venv/bin/pyreprism" "${args[@]}" > "$out"
+  code=$?
+  cat "$out"
+else
+  "$venv/bin/pyreprism" "${args[@]}"
+  code=$?
+fi
+set -e
+echo "::endgroup::"
+
+if [ -n "$out" ]; then
+  echo "report=$out" >> "${GITHUB_OUTPUT:-/dev/null}"
+fi
+
+if [ "$code" -ne 0 ]; then
+  echo "::error::PyReprism duplication gate failed (exit ${code})."
+fi
+exit "$code"

{pyreprism-0.2.0 → pyreprism-1.0.0}/.gitignore

@@ -169,3 +169,4 @@ __test__.py
 .DS_Store
 # Claude Code agent workspace
 .claude/
+manuscript/replication/

{pyreprism-0.2.0 → pyreprism-1.0.0}/CHANGELOG.md

@@ -7,6 +7,97 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.0.0] - 2026-10-12
+
+The first stable release. The public API (the top-level functions, the
+language classes, `Token`/`TokenType`, `CodeStats`, and the `batch`, `diffs`,
+`fingerprints`, `ngrams`, `clones`, `reporters`, `syntactic`, and `semantic`
+modules) and the `pyreprism` command line now follow semantic versioning:
+breaking changes only in a new major version. This release adds located clone
+detection with CI reports, optional syntactic and semantic tiers, and parallel
+processing; makes tokenization several times faster; and fixes the lexer
+defects found by validating token classes against tree-sitter parses on a
+project-stratified sample of open-source code.
+
+### Added
+- **Located clones and reports** (`PyReprism.clones`, `PyReprism.reporters`):
+  `clones.detect()` finds which line ranges are duplicated between files
+  (Rabin-Karp over the normalized token stream, with jscpd-style
+  `min_tokens`/`min_lines` floors), optionally adds Type-3 near-duplicate files
+  (`near=True`) and Type-4 semantic clones (`semantic=True`), and reports the
+  percentage of duplicated lines. Reporters render console, JSON, SARIF (one
+  rule id per clone kind), Markdown, and a self-contained HTML page. CLI:
+  `pyreprism clones --reporter ... --max-duplication PCT` (non-zero exit above
+  the threshold, for CI).
+- **Syntactic tier** (`PyReprism.syntactic`, `pip install "PyReprism[ast]"`):
+  tree-sitter ASTs for exact cyclomatic complexity and per-function metrics
+  (`cyclomatic()`, `functions()`).
+- **Semantic tier** (`PyReprism.semantic`, `pip install "PyReprism[semantic]"`):
+  code embeddings for Type-4 clone detection (`embed()`, `similarity()`, and an
+  `EmbeddingIndex` mirroring `FingerprintIndex`).
+- **GitHub Action** (`action.yml`, "PyReprism Clone Gate") and a `Dockerfile`
+  for the CLI, to gate pull requests on duplication.
+- Parallel processing: `jobs=` on `batch.analyze`, `batch.transform`, and
+  `clones.detect`, and a new `batch.map_files(paths, fn, jobs=...)` that runs any
+  per-file function on all cores. `pyreprism scan` and `pyreprism clones` take
+  `-j/--jobs` and use one process per CPU by default (`-j 1` for one process).
+  Reports are identical for any number of jobs.
+- Secondary file extensions map to their language: `.h` (C); `.cc`, `.cxx`,
+  `.hpp`, `.hh`, `.hxx` (C++); `.mjs`, `.cjs`, `.mts`, `.cts`; `.pyi`; `.bash`,
+  `.zsh`, `.ksh`; `.yml`; and others. These files were skipped before.
+
+### Changed
+- Faster tokenization (about 2.7x on one core) with identical tokens: each
+  language's string, number, identifier, and operator patterns are joined into
+  one pattern and compiled once, instead of being tried one by one at every
+  position; line numbers are tracked incrementally. `stats` (and `scan`)
+  count tokens as they are classified instead of building the token list
+  (about 4x faster than before on one core; same results). `code_metrics` computes
+  Halstead measures and complexity once instead of twice. Extension lookup no
+  longer re-imports every language module (1.4 ms to under 1 µs per file).
+- The Pygments engine is about 3x faster on large files with the same tokens:
+  it merged tokens by repeated string concatenation and counted each token's
+  line from the start of the file, both quadratic in file size.
+- `Token` is now an immutable `NamedTuple` instead of a frozen dataclass, which
+  makes tokens cheaper to create. Fields, attribute access, and immutability
+  are unchanged; a `Token` now also compares equal to a plain tuple with the
+  same values.
+
+### Fixed
+- `pyreprism --help` stated "160+ built-in languages"; it now says 187, matching
+  `pyreprism languages`. A test keeps the two in sync.
+- `pyreprism clones --max-duplication`: when stdout and stderr go to the same log
+  (as in CI), the gate message could be written into the middle of the report's
+  summary line. stdout is now flushed before the gate message.
+- Defects found by validating token classes against tree-sitter parses:
+  - `remove_comments` dropped text that a language's comment regex skips (e.g.
+    the `/` in `#include "api/z3.h"`, a `{` in Scala). The base implementation
+    now keeps it, and 20 modules that duplicated the old loop inherit it.
+  - C, C++, Java, and CSS strings containing `/` were split, so `//` or `/*`
+    inside them started a comment (Java text blocks are kept whole too).
+  - Python: only a triple-quoted string that starts a line is a docstring
+    (comment); `x = """..."""` and call arguments are strings.
+  - JavaScript, TypeScript, JSX, TSX: template literals are strings, and
+    regular-expression literals no longer start comments.
+  - Pygments engine: C preprocessor directives were typed as comments, and
+    Python docstrings as strings; both now follow the regex engine's policies.
+- Strings containing comment delimiters (`"a//b"`, `"a#b"`, `'--'`) were split
+  in 36 more languages, starting a comment inside the string: ActionScript,
+  Ada, AppleScript, Asm6502, ASP.NET, AutoHotkey, AutoIt, Bro, CoffeeScript,
+  Crystal, Elixir, Flow, GLSL, GraphQL, HPKP, HSTS, IchigoJam, Icon, Less,
+  LiveScript, Lua (including `[[...]]` long strings), Nginx, NSIS, PL/SQL, Qore,
+  Roboconf, Scheme (including `#\;` characters), SCSS (including unquoted
+  `url(...)`), and SQL. Comments now follow each language's rules in Apache
+  config, Dockerfiles, `.properties`, and Pug (own line only), Tcl (where a
+  command starts), YAML (after whitespace), and ERB (`<%# %>` tags, not `#` in
+  markup). A test checks every language with such a string.
+- Defects found on the project-stratified evaluation sample: Go raw strings
+  (`` `...` ``) containing `//` or `/*` became comments; in Bash, `#` inside a
+  word (`$#`, `${#a}`, `a#b`) started a comment; a UTF-8 byte-order mark at the
+  start of a file was counted as code, so the first line could not be a comment
+  line (both engines); and a Python triple-quoted string on the line after an
+  opening bracket, comma, or `=` was taken for a docstring.
+
 ## [0.2.0] - 2026-07-08
 
 This release adds a static code-analysis toolkit on top of the tokenizer:
@@ -112,7 +203,8 @@ ML-oriented normalization, an optional accurate backend, and batch processing.
 - Early beta releases: comment removal for an initial set of languages and the
   `Normalizer` whitespace helper.
 
-[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/unlv-evol/PyReprism/compare/v1.0.0...HEAD
+[1.0.0]: https://github.com/unlv-evol/PyReprism/compare/v0.2.0...v1.0.0
 [0.2.0]: https://github.com/unlv-evol/PyReprism/compare/v0.1.0...v0.2.0
 [0.1.0]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.1.0
 [0.0.4]: https://github.com/unlv-evol/PyReprism/releases/tag/v0.0.4

pyreprism-1.0.0/Dockerfile

@@ -0,0 +1,36 @@
+# PyReprism CLI, containerized.
+#
+#   docker build -t pyreprism .
+#   docker run --rm -v "$PWD:/work" pyreprism clones . --reporter sarif
+#   docker run --rm -v "$PWD:/work" pyreprism scan . --json
+#
+# The image ships the 187-language, zero-dependency lexical core plus the
+# optional Pygments backend ([accurate], pure Python). The heavier [ast]
+# (tree-sitter) and [semantic] (torch/transformers) tiers are intentionally
+# left out to keep the image small; derive your own image if you need them.
+
+# ---- build stage: produce a wheel from the source tree ----
+FROM python:3.12-slim AS build
+WORKDIR /src
+# Only the files hatchling needs to build the wheel and read metadata.
+COPY pyproject.toml LICENSE ./
+COPY docs/intro.rst ./docs/intro.rst
+COPY src ./src
+RUN pip install --no-cache-dir build \
+    && python -m build --wheel --outdir /dist
+
+# ---- runtime stage: install the wheel, drop the build tooling ----
+FROM python:3.12-slim
+LABEL org.opencontainers.image.title="PyReprism" \
+      org.opencontainers.image.description="Lossless token substrate for polyglot code analysis (CLI)" \
+      org.opencontainers.image.source="https://github.com/unlv-evol/PyReprism" \
+      org.opencontainers.image.licenses="MIT"
+COPY --from=build /dist/*.whl /tmp/
+RUN pip install --no-cache-dir "$(ls /tmp/*.whl)[accurate]" \
+    && rm -rf /tmp/*.whl /root/.cache/pip
+# Run as an unprivileged user; /work is where the caller mounts their code.
+RUN useradd --create-home --uid 1000 app
+USER app
+WORKDIR /work
+ENTRYPOINT ["pyreprism"]
+CMD ["--help"]

{pyreprism-0.2.0 → pyreprism-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: PyReprism
-Version: 0.2.0
+Version: 1.0.0
 Summary: Framework for Source Code Preprocessing
 Project-URL: Homepage, https://github.com/unlv-evol/PyReprism
 Project-URL: Documentation, https://pyreprism.readthedocs.io
@@ -28,7 +28,7 @@ License: MIT License
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
 License-File: LICENSE
-Classifier: Development Status :: 4 - Beta
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
@@ -44,6 +44,9 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.8
 Provides-Extra: accurate
 Requires-Dist: pygments; extra == 'accurate'
+Provides-Extra: ast
+Requires-Dist: tree-sitter; extra == 'ast'
+Requires-Dist: tree-sitter-language-pack; extra == 'ast'
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'
 Requires-Dist: flake8; extra == 'dev'
@@ -54,6 +57,9 @@ Requires-Dist: twine; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: sphinx; extra == 'docs'
 Requires-Dist: sphinx-rtd-theme; extra == 'docs'
+Provides-Extra: semantic
+Requires-Dist: torch; extra == 'semantic'
+Requires-Dist: transformers; extra == 'semantic'
 Description-Content-Type: text/x-rst
 
 .. _intro_toplevel:
@@ -64,9 +70,17 @@ Overview
 
 **PyReprism** is a Python framework for source-code preprocessing. It lets you
 **match, extract, count, and remove** comments, strings, numbers, operators,
-keywords and other language-specific constructs across **145+ programming
-languages and file formats** — through a small high-level API, a command-line
-tool, code metrics, ML-oriented normalization, and diff/pull-request analysis.
+keywords and other language-specific constructs across **187+ programming
+languages and file formats** (500+ with the optional Pygments backend) — through a
+small high-level API, a command-line tool, code metrics, ML-oriented
+normalization, and diff/pull-request analysis.
+
+Everything is derived from one lossless token stream, and the same substrate
+anchors a **fidelity ladder** of optional-cost tiers: a zero-dependency
+**lexical** tier (default), an optional **syntactic** tier (`tree-sitter ASTs
+<https://pyreprism.readthedocs.io/en/latest/syntactic.html>`__ for precise per-function metrics),
+and an optional **semantic** tier (`code embeddings
+<https://pyreprism.readthedocs.io/en/latest/semantic.html>`__ for Type-4 clone detection).
 
 Highlights
 ==========
@@ -75,11 +89,25 @@ Highlights
   ``count_*`` / ``match_*`` for each construct, plus a lossless ``tokenize()``.
 * Code metrics (``stats``) and ML canonicalization (``normalize``) for
   clone/plagiarism detection and code-embedding pipelines.
-* Optional high-accuracy `Pygments`_ backend — with **zero required
-  dependencies** by default.
-* Batch/corpus analysis and unified/``git`` diff processing.
+* Optional high-accuracy `Pygments`_ backend and an on-demand Pygments fallback
+  for 500+ languages — with **zero required dependencies** by default.
+* Optional `syntactic (AST) <https://pyreprism.readthedocs.io/en/latest/syntactic.html>`__ and
+  `semantic (embedding) <https://pyreprism.readthedocs.io/en/latest/semantic.html>`__ tiers for
+  precise per-function metrics and Type-4 clone detection.
+* Located clone detection with console, JSON, SARIF, Markdown, and HTML
+  reports and a duplication gate for CI (also as a GitHub Action).
+* Batch/corpus analysis on every core, and unified/``git`` diff processing.
 * A ``pyreprism`` command-line tool (also ``python -m PyReprism``).
 
+Stability
+=========
+
+PyReprism 1.0 is the first stable release. From 1.0 on, releases follow
+`semantic versioning`_: the public API and the ``pyreprism`` command line change
+incompatibly only in a new major version. Modules whose names start with an
+underscore are internal. Token classifications may still change in minor
+releases when they fix a lexer defect; the changelog lists every such change.
+
 Requirements
 ============
 
@@ -92,11 +120,13 @@ Install
 
     pip install PyReprism
 
-For the higher-accuracy tokenizer backend:
+The optional tiers are opt-in extras (install only what you need):
 
 .. code-block:: shell
 
-    pip install "PyReprism[accurate]"
+    pip install "PyReprism[accurate]"    # Pygments backend (higher lexical fidelity)
+    pip install "PyReprism[ast]"         # syntactic tier (tree-sitter)
+    pip install "PyReprism[semantic]"    # semantic tier (code embeddings)
 
 Quick start
 ===========
@@ -131,6 +161,7 @@ Command line
     pyreprism remove comments file.py
     pyreprism scan myproject/ --csv          # aggregate code metrics over a tree
     git diff | pyreprism diff --cosmetic     # flag comment/whitespace-only changes
+    pyreprism clones src/ --max-duplication 5   # located clones; fail above 5%
 
 Documentation
 =============
@@ -156,3 +187,4 @@ See ``CONTRIBUTING.md`` for contribution guidelines.
 
 .. _Python: https://www.python.org
 .. _Pygments: https://pygments.org
+.. _semantic versioning: https://semver.org

{pyreprism-0.2.0 → pyreprism-1.0.0}/README.md

@@ -13,12 +13,68 @@
 
 # PyReprism
 
-PyReprism is a Python framework that helps researchers and developers the task of source code preprocessing. With PyReprism, you can easily match, extract, count, and remove comments, whitespaces, operators, numbers and other language specific constructs from over **150 programming languages and file extensions**.
+PyReprism is a Python framework that helps researchers and developers the task of source code preprocessing. With PyReprism, you can easily match, extract, count, and remove comments, whitespaces, operators, numbers and other language specific constructs from over **187 programming languages and file extensions** — and **500+** with the optional Pygments backend.
+
+Everything is derived from one **lossless token stream**, and the same substrate
+anchors a **fidelity ladder** of three optional-cost tiers: a zero-dependency
+**lexical** tier (default), an optional **syntactic** tier (tree-sitter ASTs for
+precise per-function metrics), and an optional **semantic** tier (code embeddings
+for Type-4 clone detection). The core stays dependency-free; you climb the ladder
+only when a task needs it.
 
 ## Install
 ```shell
-pip install PyReprism
+pip install PyReprism                 # zero-dependency lexical core
+
+# optional tiers (install only what you need):
+pip install "PyReprism[accurate]"     # Pygments backend (higher lexical fidelity)
+pip install "PyReprism[ast]"          # syntactic tier (tree-sitter)
+pip install "PyReprism[semantic]"     # semantic tier (code embeddings)
+```
+
+### Run with Docker
+
+No Python install required. The image ships the 187-language lexical core plus
+the Pygments backend (the heavier `[ast]`/`[semantic]` tiers are omitted to keep
+it small). Mount your code at `/work`:
+
+```shell
+docker build -t pyreprism .
+
+# fail CI above 5% duplication, emit SARIF for code scanning
+docker run --rm -v "$PWD:/work" pyreprism clones . --reporter sarif --max-duplication 5
+
+# per-language metrics as JSON
+docker run --rm -v "$PWD:/work" pyreprism scan . --json
+```
+
+### GitHub Action
+
+Gate pull requests on code duplication in a few lines. The action runs
+`pyreprism clones`, writes a report, and fails the job when duplication exceeds
+your threshold:
+
+```yaml
+- uses: actions/checkout@v4
+- uses: unlv-evol/PyReprism@v1.0.0    # pin to a release tag or commit SHA
+  with:
+    paths: .
+    reporter: sarif
+    output: pyreprism-clones.sarif
+    max_duplication: 5                # fail the job above 5% duplicated lines
+    near: true                        # also flag Type-3 near-duplicate files
+
+# optional: surface the clones in the repo's Security > Code scanning tab
+- uses: github/codeql-action/upload-sarif@v3
+  if: always()
+  with:
+    sarif_file: pyreprism-clones.sarif
 ```
+
+Inputs: `paths`, `reporter` (`console`/`json`/`sarif`/`markdown`/`html`),
+`output`, `max_duplication`, `min_tokens`, `min_lines`, `near`, `semantic`,
+`lang`, and `extra_args`.
+
 ## Quick Usage
 
 The simplest entry point is the top-level API. Pass `lang` as a language name
@@ -117,6 +173,52 @@ index.similar_pairs(threshold=0.7)          # -> [(file_a, file_b, score), ...]
 On the CLI: `pyreprism similarity a.py b.py` and
 `pyreprism clones submissions/ --threshold 0.7`.
 
+#### Located clones & reports (a jscpd-style duplication detector)
+
+Beyond a similarity score, `PyReprism.clones` reports **which line ranges are
+duplicated** between files (Rabin-Karp over the normalized token stream), and
+renders a **unified report** across all three rungs of the fidelity ladder:
+
+```python
+from PyReprism import clones, reporters
+
+report = clones.detect("src/",
+                       min_tokens=50, min_lines=5,   # clone-size floor (jscpd-style)
+                       near=True,                    # + Type-3 near-duplicate files
+                       semantic=True)                # + Type-4 semantic clones
+report.duplication_pct                               # % duplicated lines
+for c in report.clones:
+    print(c.kind, c.a.path, c.a.start_line, c.b.path, c.b.start_line)
+    # kind is 'exact' (T1/2, located), 'near' (T3), or 'semantic' (T4)
+
+print(reporters.to_sarif(report))       # GitHub code scanning
+print(reporters.to_markdown(report))    # PR comment
+print(reporters.to_json(report))
+print(reporters.to_html(report))        # self-contained HTML report page
+```
+
+Every clone — **including semantic (Type-4)** ones — flows through the same
+reporters, each kind tagged with its own SARIF rule id
+(`pyreprism/exact-clone`, `.../near-clone`, `.../semantic-clone`), so CI can treat
+lexical and semantic duplication separately. The **HTML reporter** renders a
+self-contained, theme-aware page — a duplication meter with a CI-gate marker,
+per-kind counts, and a filterable card per clone with located line ranges,
+score, rule id, and a highlighted side-by-side snippet. On the CLI:
+
+```shell
+pyreprism clones src/                                  # located exact clones (console)
+pyreprism clones src/ --near --semantic                # add Type-3 and Type-4
+pyreprism clones src/ --reporter sarif  > clones.sarif # for GitHub code scanning
+pyreprism clones src/ --reporter html --max-duplication 5 > clones.html   # HTML report + CI gate
+pyreprism clones src/ --max-duplication 5              # exit non-zero above 5% (CI gate)
+pyreprism clones src/ -j 1                             # one process (default: one per CPU)
+```
+
+Fingerprints and located clones detect Type-1/2/3. For **Type-4** clones (same
+behavior, different code), `--semantic` embeds functions with the optional
+[semantic tier](#higher-fidelity-tiers-syntactic-ast--semantic-embeddings) and
+adds them to the same report.
+
 ### N-grams & code "naturalness"
 
 Token n-grams (over token text or, structurally, over token *types*) and an
@@ -173,6 +275,47 @@ installed, else regex). It works on every operation — `remove_*`, `extract_*`,
 `count_*`, `tokenize`, `normalize`, `stats`, `preprocess` — and on the CLI via
 `--engine`.
 
+When Pygments is installed, a language or extension that has no built-in module
+also resolves through an on-demand Pygments fallback, extending coverage to
+**500+** languages (`pr.get_language(".zig")`, `pr.remove_comments(src, lang="solidity")`).
+List what is reachable with `pyreprism languages` (built-in) and
+`pyreprism languages --pygments` (the fallback tier).
+
+### Higher-fidelity tiers: syntactic (AST) & semantic (embeddings)
+
+The lexical core is fast and broad but approximate. For tasks that need more, the
+same substrate feeds two optional tiers — install only what you need.
+
+**Syntactic tier** (`pip install "PyReprism[ast]"`) parses a real tree-sitter AST
+for exact, **per-function** metrics that a token stream cannot provide:
+
+```python
+from PyReprism import syntactic as ast
+
+ast.cyclomatic(src, "java")     # precise McCabe complexity (decision nodes)
+ast.functions(src, "python")    # -> [FunctionInfo(name, start_line, cyclomatic), ...]
+```
+
+**Semantic tier** (`pip install "PyReprism[semantic]"`) embeds code with a
+code-search transformer to catch **Type-4** clones (same behavior, different code)
+that token fingerprints cannot — it mirrors the `FingerprintIndex` API:
+
+```python
+from PyReprism import semantic
+
+semantic.similarity(code_a, code_b)         # cosine similarity in embedding space
+
+idx = semantic.EmbeddingIndex()
+idx.add_paths("submissions/")
+idx.similar_pairs(threshold=0.8)            # semantic near-duplicate pairs
+idx.query(code, k=5)                        # k most similar indexed snippets
+```
+
+Both tiers degrade gracefully: if the extra isn't installed, `available()` returns
+`False` and the lexical core is unaffected. Use the lexical tier for screening,
+the syntactic tier for precise/per-function metrics, and the semantic tier for
+Type-4 clones.
+
 ### Batch / whole-directory processing
 
 Analyze or transform an entire source tree (junk folders like `.git`,
@@ -191,6 +334,20 @@ batch.transform("myproject/", lambda text, lang: lang.remove_comments(text),
                 output="stripped/")
 ```
 
+Files are independent, so these calls can use every core: pass `jobs=0` (one
+process per CPU) or `jobs=N` to `batch.analyze`, `batch.transform`,
+`batch.map_files`, and `clones.detect`; results are identical to a serial run.
+With `jobs` other than 1, functions you pass must be picklable (defined at
+module level, not lambdas):
+
+```python
+def metrics_and_copy(text, lang):              # any per-file analysis
+    return lang.code_metrics(text), lang.normalize(text)
+
+for path, lang, (metrics, normalized) in batch.map_files("myproject/", metrics_and_copy, jobs=0):
+    ...
+```
+
 ### Process diffs / pull requests
 
 Parse a unified (`git`) diff and analyze the *changed code* per file, language-aware:
@@ -258,7 +415,8 @@ pyreprism normalize file.py                    # canonicalize for ML
 pyreprism remove comments file.py --in-place   # rewrite in place
 pyreprism scan myproject/ --csv                # aggregate metrics over a tree
 pyreprism remove comments src/ --output out/   # bulk-transform a directory
-pyreprism languages                            # list supported languages
+pyreprism languages                            # list built-in languages
+pyreprism languages --pygments                 # list the Pygments fallback tier
 ```
 
 Equivalently: `python -m PyReprism ...`.
@@ -267,9 +425,29 @@ It can also be wired into [pre-commit](https://pre-commit.com) via the bundled
 `.pre-commit-hooks.yaml` (`pyreprism-strip-comments`, `pyreprism-normalize-whitespace`).
 These hooks rewrite files, so scope them with `files:`/`exclude:`.
 
-Read the [docs](https://pyreprism.readthedocs.io) for more usage examples. 
-
-NB: The beta versions of PyReprism is still unstable, but we are working 24/7 to ensure the tool is usable.
+Read the [docs](https://pyreprism.readthedocs.io) for more usage examples.
+
+## Stability and versioning
+
+PyReprism 1.0 is the first stable release. From 1.0 on, releases follow
+[semantic versioning](https://semver.org): the public API (the top-level
+functions, the language classes, `Token`/`TokenType`, `CodeStats`, the
+`batch`, `diffs`, `fingerprints`, `ngrams`, `clones`, `reporters`, `syntactic`,
+and `semantic` modules) and the `pyreprism` command line change incompatibly
+only in a new major version. Modules whose names start with an underscore are
+internal. Token classifications may still change in minor releases when they
+fix a lexer defect; the [changelog](CHANGELOG.md) lists every such change.
+
+## Performance
+
+Tokenization is a single compiled pattern per language, and directory-level
+work can use every core: `pyreprism scan` and `pyreprism clones` do so by
+default (`-j N` to change it), and the Python API takes `jobs=` (`jobs=0` for
+one process per CPU).
+On an 8-core Apple M3, `pyreprism scan` processes about 2,000 files per second;
+a lexical workflow that replaces cloc, lizard, and jscpd (line counts,
+complexity, and clones over the same files) runs 3.0x faster on one core
+and 5.7x faster on 8 cores than the three tools together.
 
 ## How to Contribute
 We invite you to help us build this tool and make it more extensive. Contribution is open to OSS community.

pyreprism-1.0.0/action.yml

@@ -0,0 +1,79 @@
+name: 'PyReprism Clone Gate'
+description: >-
+  Detect duplicate/clone code across 187 languages and fail CI above a
+  duplication threshold. Emits console/JSON/SARIF/Markdown/HTML reports.
+author: 'UNLV EVOL Lab'
+branding:
+  icon: 'copy'
+  color: 'orange'
+
+inputs:
+  paths:
+    description: 'Paths to scan (space-separated).'
+    required: false
+    default: '.'
+  reporter:
+    description: 'Output format: console, json, sarif, markdown, or html.'
+    required: false
+    default: 'sarif'
+  output:
+    description: >-
+      File (relative to the workspace) to write the report to. Leave empty to
+      print to the job log only.
+    required: false
+    default: 'pyreprism-clones.sarif'
+  max_duplication:
+    description: >-
+      Fail the job if the percentage of duplicated lines exceeds this value.
+      Empty disables the gate.
+    required: false
+    default: ''
+  min_tokens:
+    description: 'Minimum clone size in tokens.'
+    required: false
+    default: '50'
+  min_lines:
+    description: 'Minimum clone size in lines.'
+    required: false
+    default: '5'
+  near:
+    description: 'Also report Type-3 near-duplicate files.'
+    required: false
+    default: 'false'
+  semantic:
+    description: >-
+      Also report Type-4 semantic clones. Installs the heavier semantic/ast
+      tiers, which take longer to set up.
+    required: false
+    default: 'false'
+  lang:
+    description: 'Force a single language instead of auto-detecting per file.'
+    required: false
+    default: ''
+  extra_args:
+    description: 'Additional raw arguments appended to `pyreprism clones`.'
+    required: false
+    default: ''
+
+outputs:
+  report:
+    description: 'Path to the written report file (empty if none).'
+    value: ${{ steps.pyreprism.outputs.report }}
+
+runs:
+  using: 'composite'
+  steps:
+    - id: pyreprism
+      shell: bash
+      run: bash "$GITHUB_ACTION_PATH/.github/pyreprism-action.sh"
+      env:
+        INPUT_PATHS: ${{ inputs.paths }}
+        INPUT_REPORTER: ${{ inputs.reporter }}
+        INPUT_OUTPUT: ${{ inputs.output }}
+        INPUT_MAX_DUPLICATION: ${{ inputs.max_duplication }}
+        INPUT_MIN_TOKENS: ${{ inputs.min_tokens }}
+        INPUT_MIN_LINES: ${{ inputs.min_lines }}
+        INPUT_NEAR: ${{ inputs.near }}
+        INPUT_SEMANTIC: ${{ inputs.semantic }}
+        INPUT_LANG: ${{ inputs.lang }}
+        INPUT_EXTRA_ARGS: ${{ inputs.extra_args }}