rustgments 0.1.0__cp39-abi3-win_amd64.whl

rustgments/__init__.py

@@ -0,0 +1,203 @@
+"""rustgments: Pygments-compatible highlighting with a Rust engine.
+
+The token tables are generated from Pygments 2.21.0 (BSD-2-Clause, see
+``rustgments.PYGMENTS_NOTICE``).  Lexers that the Rust engine cannot reproduce
+exactly fall back to Pygments itself; ``Lexer.backend`` says which one is used.
+
+Pygments submodules are imported lazily: the native highlight path never
+needs them.
+"""
+
+import pygments
+
+from . import _native
+
+__version__ = '0.1.0'
+
+__all__ = ['get_lexer_by_name', 'highlight', 'HtmlFormatter', 'Lexer', 'backend_of',
+           'EngineFailure', 'PYGMENTS_NOTICE', 'install', 'uninstall']
+
+EngineFailure = _native.EngineFailure
+# Native tables are only valid for the Pygments version they were built from,
+# and for the interpreter they were built with: Python's \w, \d, \s and its
+# case-insensitive matching follow the interpreter's Unicode database and
+# `re` module.  The IR carries one table variant per supported Python
+# version; pick ours before anything else touches the engine.
+import sys as _sys
+import unicodedata as _ud
+_PYTHON_TAG = '%d.%d' % _sys.version_info[:2]
+_VARIANT_OK = _native.select_variant(_PYTHON_TAG, _ud.unidata_version)
+_INFO = _native.info()
+PYGMENTS_NOTICE = _INFO['notice']
+PYGMENTS_VERSION = _INFO['pygments_version']
+_VERSION_OK = pygments.__version__ == PYGMENTS_VERSION and _VARIANT_OK
+
+_LEXERS = {}
+_ALIASES = {}
+for _cls, _name, _aliases, _filenames, _backend, _reason in _native.lexers():
+    _LEXERS[_cls] = (_name, tuple(_aliases), tuple(_filenames), _backend, _reason)
+    for _a in _aliases:
+        _ALIASES[_a] = _cls
+
+_NATIVE_BACKENDS = ('native', 'handwritten')
+_TOKENS = None
+
+
+def _token_types():
+    global _TOKENS
+    if _TOKENS is None:
+        from pygments.token import string_to_tokentype
+        _TOKENS = [string_to_tokentype(n) for n in _native.token_names()]
+    return _TOKENS
+
+
+def _pyg_lexers():
+    import pygments.lexers
+    return pygments.lexers
+
+
+class Lexer:
+    """A lexer that runs natively when it can, else wraps the Pygments lexer."""
+
+    def __init__(self, class_name, **options):
+        self.class_name = class_name
+        self.options = options
+        self._pyg = None
+        info = _LEXERS.get(class_name)
+        if info is None:
+            self.backend = 'fallback'
+            self.reason = 'not in the rustgments native set'
+        elif not _VERSION_OK:
+            self.backend = 'fallback'
+            self.reason = (f'no tables for Pygments {pygments.__version__} on Python {_PYTHON_TAG} '
+                           f'(Unicode {_ud.unidata_version}); tables are for Pygments {PYGMENTS_VERSION}')
+        elif options:
+            self.backend = 'fallback'
+            self.reason = f'non-default lexer options {sorted(options)}'
+        elif info[3] in _NATIVE_BACKENDS:
+            self.backend = 'native'
+            self.reason = info[4] if info[3] == 'handwritten' else None
+        else:
+            self.backend = 'fallback'
+            self.reason = info[4]
+        self.name = info[0] if info else self.pygments_lexer.name
+
+    def __repr__(self):
+        return f'<rustgments.Lexer {self.class_name} backend={self.backend}>'
+
+    @property
+    def pygments_lexer(self):
+        if self._pyg is None:
+            self._pyg = getattr(_pyg_lexers(), self.class_name)(**self.options)
+        return self._pyg
+
+    def get_tokens_unprocessed(self, text):
+        if self.backend == 'native' and isinstance(text, str):
+            try:
+                raw = _native.get_tokens_unprocessed(self.class_name, text)
+            except (EngineFailure, UnicodeEncodeError):
+                pass
+            else:
+                toks = _token_types()
+                return [(i, toks[t], v) for i, t, v in raw]
+        return list(self.pygments_lexer.get_tokens_unprocessed(text))
+
+    def get_tokens(self, text):
+        if self.backend == 'native' and isinstance(text, str):
+            try:
+                pre = _native.preprocess(text)
+                raw = _native.get_tokens_unprocessed(self.class_name, pre)
+            except (EngineFailure, UnicodeEncodeError):
+                pass
+            else:
+                toks = _token_types()
+                return [(toks[t], v) for _, t, v in raw]
+        return list(self.pygments_lexer.get_tokens(text))
+
+
+def get_lexer_by_name(alias, **options):
+    cls = _ALIASES.get(alias)
+    if cls is None:
+        cls = type(_pyg_lexers().get_lexer_by_name(alias)).__name__
+    return Lexer(cls, **options)
+
+
+def backend_of(alias):
+    return get_lexer_by_name(alias).backend
+
+
+class HtmlFormatter:
+    """HtmlFormatter; native only with default options."""
+
+    def __init__(self, **options):
+        self.options = options
+        self.native = not options
+        self._pyg = None
+
+    @property
+    def pygments_formatter(self):
+        if self._pyg is None:
+            from pygments.formatters.html import HtmlFormatter as PygHtmlFormatter
+            self._pyg = PygHtmlFormatter(**self.options)
+        return self._pyg
+
+    def get_style_defs(self, arg=None):
+        return self.pygments_formatter.get_style_defs(arg)
+
+    def format(self, tokensource, outfile):
+        return self.pygments_formatter.format(tokensource, outfile)
+
+
+def _native_lexer_class(lexer):
+    if isinstance(lexer, Lexer):
+        return lexer.class_name if lexer.backend == 'native' else None
+    # a plain Pygments lexer instance of a native class with default options
+    name = type(lexer).__name__
+    info = _LEXERS.get(name)
+    if (info and _VERSION_OK and info[3] in _NATIVE_BACKENDS
+            and type(lexer) is getattr(_pyg_lexers(), name, None)
+            and not lexer.options and not lexer.filters):
+        return name
+    return None
+
+
+def _native_formatter(formatter):
+    if isinstance(formatter, HtmlFormatter):
+        return formatter.native
+    from pygments.formatters.html import HtmlFormatter as PygHtmlFormatter
+    return type(formatter) is PygHtmlFormatter and not formatter.options
+
+
+def highlight(code, lexer, formatter, outfile=None):
+    """Like ``pygments.highlight``; native for native lexers + default HtmlFormatter."""
+    cls = _native_lexer_class(lexer)
+    if cls is not None and isinstance(code, str) and _native_formatter(formatter):
+        try:
+            html = _native.highlight_html(cls, code)
+        except (EngineFailure, UnicodeEncodeError):
+            html = None
+        if html is not None:
+            if outfile is not None:
+                outfile.write(html)
+                return None
+            return html
+    from pygments import highlight as pyg_highlight
+    pyg_lexer = lexer.pygments_lexer if isinstance(lexer, Lexer) else lexer
+    pyg_fmt = formatter.pygments_formatter if isinstance(formatter, HtmlFormatter) else formatter
+    return pyg_highlight(code, pyg_lexer, pyg_fmt, outfile)
+
+
+def install():
+    """Drop-in mode: make Pygments' own ``highlight``, ``Lexer.get_tokens``,
+    ``Lexer.get_tokens_unprocessed`` and ``HtmlFormatter`` use the native
+    engine for native lexers wherever the output is identical (see
+    ``rustgments._dropin``).  Returns False, changing nothing, when this
+    interpreter or Pygments version has no native tables."""
+    from ._dropin import install as _install
+    return _install()
+
+
+def uninstall():
+    """Undo ``install()``."""
+    from ._dropin import uninstall as _uninstall
+    _uninstall()

rustgments/_dropin.py

@@ -0,0 +1,279 @@
+"""Drop-in mode: Pygments itself uses the native engine where it is exact.
+
+``install()`` patches, for every lexer class rustgments runs natively:
+
+* ``get_tokens_unprocessed``: native tokens for instances of exactly that
+  class (subclasses, other start states and non-str input keep Pygments' own
+  method).  Code that only drives lexers, such as ``DelegatingLexer`` or
+  ``using(...)`` callbacks of other lexers, picks this up too.
+* ``get_tokens``: Pygments' own preprocessing (``_preprocess_lexer_input``
+  with the lexer's options), then a token stream that lexes natively when it
+  is first read and runs the lexer's filters on it in Python.
+
+and ``HtmlFormatter.format_unencoded``: when it receives such a stream
+untouched, ``_format_lines`` (the per-token loop) runs natively with span
+openers computed by the formatter's own code; every per-line step (line
+numbers, ``hl_lines``, anchors, ``wrap``, ``<div>``, ``full``) still runs
+the formatter's own methods.  A stream with a token of type ``Error`` and a
+``raiseonerror`` filter takes Pygments' path, so the error is raised at the
+same point with the same partial output.
+
+Everything else (other lexers, other formatters, unknown filters, overridden
+``_format_lines``, ``tagsfile``) runs Pygments untouched.  ``uninstall()``
+restores the original methods.
+"""
+
+import threading
+
+from . import _native, EngineFailure
+
+_LOCK = threading.Lock()
+_SAVED = []          # (owner, attribute, original value or _MISSING)
+_MISSING = object()
+_INSTALLED = False
+
+_HANDLES = {}        # opener configuration key -> native handle
+
+# How often each path ran (for tests and benchmarks; plain ints, no locking).
+STATS = {'lex_native': 0, 'lex_pygments': 0, 'html_native': 0, 'html_pygments': 0}
+
+
+def _save(owner, name, value):
+    _SAVED.append((owner, name, owner.__dict__.get(name, _MISSING)))
+    setattr(owner, name, value)
+
+
+class NativeTokens:
+    """The token stream of ``Lexer.get_tokens`` for a native lexer.
+
+    It behaves like the generator Pygments returns: an iterator of
+    ``(tokentype, value)`` pairs, lexed on first use, with the lexer's filters
+    applied.  The HTML formatter recognises an untouched stream and formats it
+    natively.
+    """
+
+    __slots__ = ('lexer', 'class_name', 'text', 'filters', '_it')
+
+    def __init__(self, lexer, class_name, text, filters):
+        self.lexer = lexer
+        self.class_name = class_name
+        self.text = text
+        self.filters = filters
+        self._it = None
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if self._it is None:
+            self._it = self._stream()
+        return next(self._it)
+
+    def close(self):
+        self._it = iter(())
+
+    def _stream(self):
+        from pygments.filter import apply_filters
+        toks = None
+        if type(self.text) is str:
+            try:
+                toks = _native.tokens(self.class_name, self.text)
+                STATS['lex_native'] += 1
+            except (EngineFailure, UnicodeEncodeError):
+                toks = None
+        if toks is None:
+            toks = ((t, v) for _, t, v in self.lexer.get_tokens_unprocessed(self.text))
+        stream = iter(toks)
+        if self.filters:
+            stream = apply_filters(stream, self.filters, self.lexer)
+        return stream
+
+
+def _patch_lexer(cls, class_name):
+    orig_gtu = cls.get_tokens_unprocessed
+    orig_gt = cls.get_tokens
+    from pygments.lexer import RegexLexer
+    regex = issubclass(cls, RegexLexer)
+
+    if regex:
+        def get_tokens_unprocessed(self, text, stack=('root',)):
+            if type(self) is cls and type(text) is str and stack == ('root',):
+                try:
+                    toks = _native.tokens_with_offsets(class_name, text)
+                    STATS['lex_native'] += 1
+                    return iter(toks)
+                except (EngineFailure, UnicodeEncodeError):
+                    pass
+            STATS['lex_pygments'] += 1
+            return orig_gtu(self, text, stack)
+    else:
+        def get_tokens_unprocessed(self, text):
+            if type(self) is cls and type(text) is str:
+                try:
+                    toks = _native.tokens_with_offsets(class_name, text)
+                    STATS['lex_native'] += 1
+                    return iter(toks)
+                except (EngineFailure, UnicodeEncodeError):
+                    pass
+            STATS['lex_pygments'] += 1
+            return orig_gtu(self, text)
+
+    def get_tokens(self, text, unfiltered=False):
+        if type(self) is not cls:
+            return orig_gt(self, text, unfiltered)
+        # Lexer.get_tokens: preprocess eagerly, lex lazily, then filters
+        text = self._preprocess_lexer_input(text)
+        return NativeTokens(self, class_name, text, () if unfiltered else self.filters)
+
+    get_tokens_unprocessed.__doc__ = orig_gtu.__doc__
+    get_tokens.__doc__ = orig_gt.__doc__
+    _save(cls, 'get_tokens_unprocessed', get_tokens_unprocessed)
+    _save(cls, 'get_tokens', get_tokens)
+
+
+def _opener_handle(fmt):
+    """Native handle for the span openers and line separator of `fmt`.
+
+    The openers come from the formatter's own `_format_lines` (run once over
+    every token type), so `noclasses`, styles, `classprefix` and
+    `debug_token_types` behave exactly as in Pygments.  A fresh formatter of a
+    class that does not override the CSS helpers depends only on the key
+    below, so its handle is shared; anything else is computed per instance.
+    """
+    from pygments.formatters.html import HtmlFormatter
+    from . import _token_types
+    cached = fmt.__dict__.get('_rustgments_handle')
+    if cached is not None:
+        return cached
+    t = type(fmt)
+    shareable = (not fmt.span_element_openers
+                 and t._get_css_class is HtmlFormatter._get_css_class
+                 and t._get_css_classes is HtmlFormatter._get_css_classes
+                 and t._get_css_inline_styles is HtmlFormatter._get_css_inline_styles)
+    key = None
+    if shareable:
+        key = (t, bool(fmt.noclasses), fmt.classprefix, bool(fmt.debug_token_types),
+               fmt.lineseparator, fmt.style if fmt.noclasses else None)
+        h = _HANDLES.get(key)
+        if h is not None:
+            return h
+    tts = _token_types()
+    for _ in fmt._format_lines([(tt, 'x') for tt in tts]):
+        pass
+    openers = [fmt.span_element_openers[tt] for tt in tts]
+    h = _native.register_openers(openers, fmt.lineseparator)
+    if key is not None:
+        _HANDLES[key] = h
+    else:
+        fmt.__dict__['_rustgments_handle'] = h
+    return h
+
+
+def _patch_html_formatter():
+    from pygments.filters import RaiseOnErrorTokenFilter
+    from pygments.formatters.html import HtmlFormatter
+    orig = HtmlFormatter.format_unencoded
+    orig_lines = HtmlFormatter._format_lines
+    orig_translate = HtmlFormatter._translate_parts
+
+    def native_lines(self, ts):
+        t = type(self)
+        if (t._format_lines is not orig_lines or t._translate_parts is not orig_translate
+                or self.tagsfile or type(ts.text) is not str):
+            return None
+        if any(type(f) is not RaiseOnErrorTokenFilter for f in ts.filters):
+            return None
+        try:
+            return _native.format_lines(ts.class_name, ts.text, _opener_handle(self), bool(ts.filters))
+        except (EngineFailure, UnicodeEncodeError):
+            return None
+
+    def format_unencoded(self, tokensource, outfile):
+        if type(tokensource) is NativeTokens and tokensource._it is None:
+            lines = native_lines(self, tokensource)
+            if lines is not None:
+                STATS['html_native'] += 1
+                tokensource._it = iter(())
+                source = ((1, line) for line in lines)
+                # The rest is HtmlFormatter.format_unencoded (Pygments 2.21.0)
+                # after `source = self._format_lines(tokensource)`.
+                if not self.nowrap and self.linenos == 2:
+                    source = self._wrap_inlinelinenos(source)
+                if self.hl_lines:
+                    source = self._highlight_lines(source)
+                if not self.nowrap:
+                    if self.lineanchors:
+                        source = self._wrap_lineanchors(source)
+                    if self.linespans:
+                        source = self._wrap_linespans(source)
+                    source = self.wrap(source)
+                    if self.linenos == 1:
+                        source = self._wrap_tablelinenos(source)
+                    source = self._wrap_div(source)
+                    if self.full:
+                        source = self._wrap_full(source, outfile)
+                for t, piece in source:
+                    outfile.write(piece)
+                return None
+        STATS['html_pygments'] += 1
+        return orig(self, tokensource, outfile)
+
+    format_unencoded.__doc__ = orig.__doc__
+    _save(HtmlFormatter, 'format_unencoded', format_unencoded)
+
+
+# sha256 of the source of HtmlFormatter.format_unencoded in Pygments 2.21.0,
+# whose tail is copied above; install() refuses to patch anything else.
+_FORMAT_UNENCODED_SHA256 = '70460d4f03e958a4052d16e84b873aaa1b60d8d90af84c0ef2e81e93377d860a'
+
+
+def _formatter_source_ok():
+    import hashlib
+    import inspect
+    from pygments.formatters.html import HtmlFormatter
+    try:
+        src = inspect.getsource(HtmlFormatter.format_unencoded)
+    except (OSError, TypeError):
+        return True  # no source shipped; the Pygments version pin still holds
+    return hashlib.sha256(src.encode()).hexdigest() == _FORMAT_UNENCODED_SHA256
+
+
+def install():
+    """Route Pygments' own lexers and HtmlFormatter through rustgments where
+    the output is identical.  Returns True if installed, False when this
+    interpreter / Pygments version has no native tables (Pygments then runs
+    unchanged)."""
+    global _INSTALLED
+    from . import _VERSION_OK, _LEXERS, _NATIVE_BACKENDS, _token_types
+    with _LOCK:
+        if _INSTALLED:
+            return True
+        if not _VERSION_OK:
+            return False
+        import pygments.lexers
+        if not _formatter_source_ok():
+            return False
+        _native.set_token_types(_token_types())
+        for name, info in _LEXERS.items():
+            if info[3] in _NATIVE_BACKENDS:
+                _patch_lexer(getattr(pygments.lexers, name), name)
+        _patch_html_formatter()
+        _INSTALLED = True
+        return True
+
+
+def uninstall():
+    """Restore every method install() replaced."""
+    global _INSTALLED
+    with _LOCK:
+        while _SAVED:
+            owner, name, value = _SAVED.pop()
+            if value is _MISSING:
+                delattr(owner, name)
+            else:
+                setattr(owner, name, value)
+        _INSTALLED = False
+
+
+def installed():
+    return _INSTALLED

rustgments/mkdocs_plugin.py

@@ -0,0 +1,15 @@
+"""MkDocs plugin (``plugins: [rustgments]``): calls ``rustgments.install()``.
+
+Pygments calls made by Markdown extensions (codehilite, pymdownx.highlight)
+then use the native engine where the output is identical.
+"""
+
+from mkdocs.plugins import BasePlugin
+
+import rustgments
+
+
+class RustgmentsPlugin(BasePlugin):
+    def on_config(self, config):
+        rustgments.install()
+        return config

rustgments/sphinxext.py

@@ -0,0 +1,12 @@
+"""Sphinx extension: ``extensions = [..., 'rustgments.sphinxext']``.
+
+Calls ``rustgments.install()`` so that Sphinx's Pygments calls use the native
+engine where the output is identical.  Output files do not change.
+"""
+
+import rustgments
+
+
+def setup(app):
+    rustgments.install()
+    return {'version': '0.1.0', 'parallel_read_safe': True, 'parallel_write_safe': True}

rustgments-0.1.0.dist-info/METADATA

@@ -0,0 +1,123 @@
+Metadata-Version: 2.4
+Name: rustgments
+Version: 0.1.0
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Text Processing :: Filters
+Requires-Dist: pygments>=2.21
+License-File: LICENSE
+License-File: LICENSE-pygments
+License-File: LICENSE-third-party.txt
+Summary: A Rust engine for Pygments: byte-identical output, much faster when you highlight a lot
+Keywords: pygments,syntax-highlighting,rust
+License: BSD-2-Clause
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Repository, https://github.com/b84330808/rustgments
+
+# rustgments
+
+A Rust engine for [Pygments](https://pygments.org/). It runs Pygments' own lexer tables and produces the same tokens and the same HTML, byte for byte. A lexer or option it can't reproduce exactly goes to Pygments itself, so output never changes.
+
+```python
+import rustgments
+
+lexer = rustgments.get_lexer_by_name('python')
+html = rustgments.highlight(code, lexer, rustgments.HtmlFormatter())
+```
+
+Or leave your code alone and make Pygments itself use it:
+
+```python
+import rustgments
+rustgments.install()   # pygments.highlight() and friends now use the native path
+```
+
+## When it helps
+
+It helps when one process highlights a lot of code. Python startup and imports are the same either way, so a one-shot script on a small file gains little.
+
+| Workload | Pygments | rustgments | |
+|---|---:|---:|---:|
+| `highlight()` called again in the same process, 20 KB - 1.2 MB files | 5 - 1,600 ms | 0.2 - 53 ms | 12 - 75x |
+| First `highlight()` of a process, same files | 14 - 1,588 ms | 0.3 - 52 ms | 7 - 54x |
+| 4,226 files up to 20 KB through `pygments.highlight` after `install()`, highlight loop | 32.55 s | 2.23 s | 14.6x |
+| Same, whole process | 33.41 s | 3.31 s | 10.1x |
+| One file, whole process from start to exit, 20 KB - 1.2 MB | 0.14 - 1.7 s | 0.09 - 0.18 s | 1.1 - 9.5x |
+| Django docs with Sphinx (4,199 code blocks) | 109.5 s | 100.7 s | within noise |
+
+The Sphinx row is the honest one for documentation builds: highlighting was 3.8 s of a 113 s build, and it dropped to 1.8 s. Sphinx spends its time elsewhere.
+
+The first use of each lexer costs a little more than in Pygments (median 3.8 ms against 2.7 ms over 361 lexers; a few very large lexers take 100-300 ms), so a short run that touches many languages a few times each can come out slower.
+
+Windows 11, i7-9700, CPython 3.11-3.12, base and rustgments alternating. Raw results are in [`bench/results`](bench/results).
+
+## What runs natively
+
+363 of Pygments' 602 lexers, including Python, JavaScript, TypeScript, JSON, Bash, Rust, Go, Java, Kotlin, HTML, CSS, XML, TOML, INI, Dockerfile, PowerShell, Perl and diff. **C, C++, YAML, Markdown, SQL, Ruby, PHP and Swift still run on Pygments** for now.
+
+A lexer stays on Pygments when it isn't a plain `RegexLexer` table (hand-written lexers, custom callbacks, overridden tokenizing), when Pygments has no test files to check it against, or when lexer options change its behaviour. `lexer.backend` tells you which one you got, and `lexer.reason` says why:
+
+```python
+>>> rustgments.get_lexer_by_name('c').backend
+'fallback'
+```
+
+`HtmlFormatter` runs natively with the options Sphinx and pymdownx use (`cssclass`, `nowrap`, `linenos` variants, `hl_lines`, `wrapcode`, `linespans`, `lineanchors`, `noclasses`, `style`, ...). Other formatters, and `tagsfile`, are plain Pygments.
+
+## Requirements
+
+Python 3.9 - 3.14 and **Pygments 2.21.0**. The tables are generated from that exact release, and Python's `\w`, `\d`, `\s` follow the interpreter's Unicode version, so there is one table set per Unicode version. With any other Pygments, or a Python it has no tables for, everything runs on Pygments: correct, just not faster.
+
+## Install
+
+```
+pip install rustgments
+```
+
+Wheels are built for Windows, Linux (glibc and musl, x86_64 and aarch64) and macOS (x86_64 and arm64), for any Python from 3.9. Elsewhere pip builds from source, which needs a Rust toolchain.
+
+## Using it with existing tools
+
+| | |
+|---|---|
+| Your own code | `rustgments.install()`; `rustgments.uninstall()` restores Pygments |
+| Sphinx | `extensions = ['rustgments.sphinxext']` |
+| MkDocs | `plugins: [rustgments]` |
+| Anything else | set `RUSTGMENTS_INSTALL=1` |
+
+The environment variable works through a one-line `.pth` file in the wheel. It runs at every Python startup, and all it does is check that variable. Delete `rustgments-autoinstall.pth` from `site-packages` if you'd rather not have it.
+
+`install()` patches `get_tokens`, `get_tokens_unprocessed` of the native lexers and the per-token loop of `HtmlFormatter`. Line numbers, highlighted lines, anchors and wrapping still run Pygments' own code.
+
+## How parity is checked
+
+Parity is the point of this project, so a lexer only runs natively after passing a gate on every supported Python:
+
+- `tests/gate.py`: each lexer's own Pygments test files (`tests/examplefiles`, `tests/snippets`) plus 30 generated mutants, comparing preprocessing, every `(offset, token, value)` and the final HTML. Any mismatch, engine error or timeout keeps that lexer on Pygments. The result is [`build_ir/native_gate.json`](build_ir/native_gate.json).
+- `tests/parity.py`: the same comparisons over a corpus of real code (5,171 files, 88 MB, for the five core lexers).
+- `tests/independent_check.py`: a second checker using only the public API, on other files and random mutants.
+- `tests/dropin_check.py`: Pygments' API before and after `install()`, with 23 formatter and lexer call shapes.
+- `tests/unicode_check.py`: the 9,917 characters whose classes changed between Unicode versions, inserted into real code.
+
+The corpus isn't in this repository. Point `RUSTGMENTS_CORPUS` at a directory with `python/`, `json/`, `js/`, `bash/` and `rust/` subdirectories, and `RUSTGMENTS_PYGMENTS_SRC` at a checkout of Pygments 2.21.0.
+
+CI builds every wheel and runs Pygments' own test inputs against it.
+
+## How it works
+
+A build step imports Pygments and serializes each lexer's processed token tables, so `include`, `inherit`, `words()` and friends are already resolved. Every pattern is re-emitted from Python's own regex parse tree into Rust syntax, with Python's semantics spelled out (`\w`, `\d`, `\s`, `\Z`, `$`, flags). At run time the engine matches with DFAs where it can and falls back to [fancy-regex](https://github.com/fancy-regex/fancy-regex) for look-around and back-references, following `RegexLexer.get_tokens_unprocessed` step by step.
+
+`build_ir/` regenerates the tables in `ir/`, which are checked in.
+
+## Status
+
+Early. Version 0.1.0 tracks Pygments 2.21.0 only, and the API is the small part of Pygments listed above. Please open an issue if the output ever differs from Pygments: that's the bug this project cares about most.
+
+Prior art: the `pygmentsrs` crate inside [westurner/dsport](https://github.com/westurner/dsport) takes a similar approach for a docutils port. No code from it is used here.
+
+## License
+
+BSD-2-Clause, see [LICENSE](LICENSE). The generated tables are derived from Pygments ([LICENSE-pygments](LICENSE-pygments)). Rust crate licenses are in [LICENSE-third-party.txt](LICENSE-third-party.txt).
+

rustgments-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+rustgments/__init__.py,sha256=cZbmRC8KbgZ-fCs0FWahZUS0Bixc6i-FFx-K2TGVZLo,7559
+rustgments/_dropin.py,sha256=-6dncL7O_xh_cG8H7NtqlXV2yD_S2SzLb3usM-ZQRz8,11065
+rustgments/_native.pyd,sha256=l8QcSmwalN3F4hlu8OjLO5lkmFVzMCxgHZQyoVnQfTk,33523712
+rustgments/mkdocs_plugin.py,sha256=FCNEVYoKyVkyIxl8SOg1StCdR_wQ58kj3KB-meJkkU8,412
+rustgments/sphinxext.py,sha256=P9evfxdGeq-W37FVIQATxIwwJLzJCuTNBm_Ji3y2uU8,382
+rustgments-0.1.0.dist-info/METADATA,sha256=mSnBwBVc44KyVO3rMZILqFfO51u36vSIKcoF7-nJdJ0,7358
+rustgments-0.1.0.dist-info/WHEEL,sha256=xw8F9xBWUOgm5ym2S9tihj_bLJOb723bjhA0kdANo2I,95
+rustgments-0.1.0.dist-info/entry_points.txt,sha256=wSditpSfhXlH_zcvi9_Cz0jTU1N2vO7HI3nDeodPTj0,70
+rustgments-0.1.0.dist-info/licenses/LICENSE,sha256=AXgZCU-0YJGXNge_PnwcDXRg8SLI6P5cq12PjX9KbOw,1358
+rustgments-0.1.0.dist-info/licenses/LICENSE-pygments,sha256=-BhIrkxwyHKSLs6c7ElBh-T83z7MIECFfWXgljWtsCo,1356
+rustgments-0.1.0.dist-info/licenses/LICENSE-third-party.txt,sha256=yddUo8trEbt_V0ji3TEdhD_kgZuzoTgMzTUD4BkdQ00,29194
+rustgments-0.1.0.dist-info/sboms/rustgments.cyclonedx.json,sha256=OWgOZkBy2etjCvQlpw_qC9OoTRYEKPKVhg9K8Dum22c,43111
+rustgments-autoinstall.pth,sha256=DhAh9CaC5dXH5YaLEM8ID9gJowSpU_tS-oJfpMartKw,95
+rustgments-0.1.0.dist-info/RECORD,,

rustgments-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.16.0)
+Root-Is-Purelib: false
+Tag: cp39-abi3-win_amd64

rustgments-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[mkdocs.plugins]
+rustgments=rustgments.mkdocs_plugin:RustgmentsPlugin