lhtml-markup 2.0.0__py3-none-any.whl

lhtml/patterns.py

@@ -0,0 +1,92 @@
+"""Centralized regex patterns and text transformation utilities for LHTML."""
+
+import re
+from typing import Callable
+
+
+# ---------------------------------------------------------------------------
+# Compiled regex patterns
+# ---------------------------------------------------------------------------
+
+YAML_FRONTMATTER = re.compile(r'^---\n(.*?)\n---$', re.DOTALL | re.MULTILINE)
+VERBATIM_BLOCK   = re.compile(r'verbatim::\[\](.*?)verbatim::\[-\]', re.DOTALL | re.MULTILINE)
+VERBATIM_INDEX   = re.compile(r'verbatim::\[(.*?)\]')
+CODE_BLOCK       = re.compile(r'code::(.*?)code::\[-\]', re.DOTALL | re.MULTILINE)
+CODE_INDEX       = re.compile(r'code::\[(.*?)\]')
+HEADING          = re.compile(r'^(=+)(?:\((.*?)\))? (.*?)$', re.MULTILINE)
+BOLD             = re.compile(r'\*\*(.*?)\*\*')
+ITALIC           = re.compile(r'__(.*?)__')
+INLINE_CODE      = re.compile(r'`(.*?)`')
+COMMENT          = re.compile(r'::#(.*?)$', re.MULTILINE)
+INCLUDE          = re.compile(r'include::')
+TAG_MARKER       = re.compile(r'::')
+LIST_ITEM        = re.compile(r'^(\*+) (.*)')
+
+# String constants
+VERBATIM_OPEN  = 'verbatim::[]'
+VERBATIM_CLOSE = 'verbatim::[-]'
+CODE_CLOSE     = 'code::[-]'
+SPACER_TAG     = 'nl'
+
+MAX_INCLUDE_ITERATIONS = 20
+
+
+# ---------------------------------------------------------------------------
+# Generic regex transform utility
+# ---------------------------------------------------------------------------
+
+def store_to_index(text: str, pattern: re.Pattern | str, name: str, store: list) -> str:
+    """Extract regex matches into a store, replacing with indexed placeholders.
+
+    Each match is stored in `store` and replaced with `name::[index]`.
+    Used to protect code/verbatim blocks from further processing.
+    """
+    if isinstance(pattern, str):
+        pattern = re.compile(pattern, re.DOTALL | re.MULTILINE)
+    parts = []
+    prev = 0
+    for m in pattern.finditer(text):
+        idx = len(store)
+        store.append(m.group(0))
+        parts.append(text[prev:m.start()])
+        parts.append(f'{name}::[{idx}]')
+        prev = m.end()
+    parts.append(text[prev:])
+    return ''.join(parts)
+
+
+def restore_from_index(text: str, pattern: re.Pattern | str, store: list) -> str:
+    """Restore indexed placeholders from a store.
+
+    Matches `name::[index]` patterns and replaces with the stored content.
+    """
+    if isinstance(pattern, str):
+        pattern = re.compile(pattern)
+    parts = []
+    prev = 0
+    for m in pattern.finditer(text):
+        idx = int(m.group(1))
+        parts.append(text[prev:m.start()])
+        parts.append(store[idx])
+        prev = m.end()
+    parts.append(text[prev:])
+    return ''.join(parts)
+
+
+def regex_transform(text: str, pattern: re.Pattern, transform_fn: Callable[[re.Match], str]) -> str:
+    """Apply a regex-based transformation across text.
+
+    For each match of `pattern` in `text`, calls `transform_fn(match)`
+    to produce the replacement string. Non-matching text passes through.
+
+    This eliminates the boilerplate loop duplicated across process_bold,
+    process_italic, process_code_inline, process_title, etc.
+    """
+    parts = []
+    prev = 0
+    for m in pattern.finditer(text):
+        parts.append(text[prev:m.start()])
+        parts.append(transform_fn(m))
+        prev = m.end()
+    parts.append(text[prev:])
+    return ''.join(parts)

lhtml/pipeline.py

@@ -0,0 +1,215 @@
+"""Processing pipeline and tag handler registry for LHTML.
+
+The pipeline orchestrates the multi-pass LHTML-to-HTML transformation.
+Tag handlers are registered in a plugin registry, making it easy to
+add custom element types (e.g., new :: tag names) without modifying
+the core processing code.
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from dataclasses import dataclass, field
+from typing import Callable
+
+from .errors import LHTMLIncludeLoopError
+from .patterns import MAX_INCLUDE_ITERATIONS
+
+
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+
+META_DEFAULTS = {
+    'wrap-auto': False,
+    'add_title_id': False,
+    'title': 'Webpage',
+    'css': [],
+    'js': [],
+    'wrap-custom-pre': '',
+    'wrap-custom-post': '',
+    'directory_include': [os.getcwd() + '/'],
+}
+
+
+@dataclass
+class ProcessingContext:
+    """Mutable state carried through the pipeline."""
+    text: str
+    meta: dict = field(default_factory=dict)
+    verbatim_store: list = field(default_factory=list)
+    code_store: list = field(default_factory=list)
+    current_directory: str = ''
+
+
+# ---------------------------------------------------------------------------
+# Tag handler registry (plugin system)
+# ---------------------------------------------------------------------------
+
+# A tag handler receives (element_dict, tag_to_close, current_directory)
+# and returns (html_string, is_real_tag).
+TagHandler = Callable[[dict, list, str], tuple[str, bool]]
+
+
+class TagRegistry:
+    """Registry for LHTML :: tag element handlers.
+
+    Built-in tags (div, span, link, img, video, videoplay) are
+    registered at import time. Users can register custom tags:
+
+        from lhtml.pipeline import tag_registry
+        tag_registry.register('custom', my_handler)
+    """
+
+    def __init__(self):
+        self._handlers: dict[str, TagHandler] = {}
+
+    def register(self, tag_name: str, handler: TagHandler):
+        """Register a handler for a tag name."""
+        self._handlers[tag_name] = handler
+
+    def get(self, tag_name: str) -> TagHandler | None:
+        """Look up the handler for a tag name."""
+        return self._handlers.get(tag_name)
+
+    def has(self, tag_name: str) -> bool:
+        return tag_name in self._handlers
+
+    def registered_tags(self) -> list[str]:
+        return list(self._handlers.keys())
+
+
+# Global registry instance
+tag_registry = TagRegistry()
+
+
+def _register_builtin_tags():
+    """Register the built-in LHTML tag handlers."""
+    from .export_html import (
+        export_html_generic, export_html_link,
+        export_html_img, export_html_video,
+    )
+
+    def _handle_generic(tag_name):
+        def handler(element, tag_to_close, current_directory):
+            return export_html_generic(element, tag_name, tag_to_close), True
+        return handler
+
+    tag_registry.register('div', _handle_generic('div'))
+    tag_registry.register('span', _handle_generic('span'))
+
+    def _handle_link(element, tag_to_close, current_directory):
+        return export_html_link(element), True
+    tag_registry.register('link', _handle_link)
+
+    def _handle_img(element, tag_to_close, current_directory):
+        return export_html_img(element), True
+    tag_registry.register('img', _handle_img)
+
+    def _handle_video(element, tag_to_close, current_directory):
+        return export_html_video(element, '', current_directory), True
+    tag_registry.register('video', _handle_video)
+
+    def _handle_videoplay(element, tag_to_close, current_directory):
+        return export_html_video(element, 'autoplay loop muted', current_directory), True
+    tag_registry.register('videoplay', _handle_videoplay)
+
+
+# ---------------------------------------------------------------------------
+# Code lexer registry (plugin system for syntax highlighting)
+# ---------------------------------------------------------------------------
+
+class LexerRegistry:
+    """Registry for custom Pygments lexers by language name."""
+
+    def __init__(self):
+        self._lexers: dict[str, type] = {}
+
+    def register(self, language: str, lexer_class: type):
+        self._lexers[language] = lexer_class
+
+    def get(self, language: str) -> type | None:
+        return self._lexers.get(language)
+
+
+lexer_registry = LexerRegistry()
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+class ProcessingPipeline:
+    """Orchestrates the LHTML-to-HTML transformation pipeline.
+
+    Usage:
+        pipeline = ProcessingPipeline()
+        html = pipeline.run(lhtml_text)
+        html = pipeline.run(lhtml_text, meta={'wrap-auto': True})
+    """
+
+    def __init__(self, registry: TagRegistry | None = None):
+        self.tag_registry = registry or tag_registry
+
+    def run(self, text: str, meta_arg: dict | None = None) -> str:
+        from .process import (
+            process_yaml, process_verbatim_to_index,
+            process_remove_comment, process_include,
+            process_title, process_listing,
+            process_bold, process_italic, process_code_inline,
+            process_tag, process_code,
+            process_verbatim_back_from_index,
+        )
+        from .wrap_html import wrap_auto
+        from .patterns import CODE_BLOCK, CODE_INDEX, store_to_index, restore_from_index
+
+        ctx = ProcessingContext(
+            text=text,
+            meta={**META_DEFAULTS, **(meta_arg or {})},
+        )
+
+        # Phase 1: YAML front matter
+        ctx.text, meta_yaml = process_yaml(ctx.text)
+        ctx.meta = {**ctx.meta, **meta_yaml}
+        ctx.current_directory = ctx.meta.get('current_directory', '')
+
+        # Phase 2: Verbatim / comments / includes (iterative)
+        found_include = True
+        iteration = 0
+        while found_include:
+            ctx.text = process_verbatim_to_index(ctx.text, ctx.verbatim_store)
+            ctx.text = process_remove_comment(ctx.text)
+            ctx.text, found_include = process_include(ctx.text, ctx.meta['directory_include'])
+            iteration += 1
+            if iteration > MAX_INCLUDE_ITERATIONS:
+                warnings.warn(str(LHTMLIncludeLoopError(MAX_INCLUDE_ITERATIONS)))
+                break
+
+        # Phase 3: Protect code blocks
+        ctx.text = store_to_index(ctx.text, CODE_BLOCK, 'code', ctx.code_store)
+
+        # Phase 4: Block-level elements
+        ctx.text = process_title(ctx.text)
+        ctx.text = process_listing(ctx.text)
+
+        # Phase 5: Inline elements
+        ctx.text = process_bold(ctx.text)
+        ctx.text = process_italic(ctx.text)
+        ctx.text = process_code_inline(ctx.text)
+
+        # Phase 6: Tag elements (uses tag_registry)
+        ctx.text = process_tag(ctx.text, ctx.current_directory, self.tag_registry)
+
+        # Phase 7: Restore code blocks with highlighting
+        ctx.text = restore_from_index(ctx.text, CODE_INDEX, ctx.code_store)
+        ctx.text = process_code(ctx.text)
+
+        # Phase 8: Restore verbatim blocks
+        ctx.text = process_verbatim_back_from_index(ctx.text, ctx.verbatim_store)
+
+        # Phase 9: Optional HTML wrapping
+        if ctx.meta.get('wrap-auto') is True:
+            ctx.text = wrap_auto(ctx.text, ctx.meta)
+
+        return ctx.text

lhtml/process.py

@@ -0,0 +1,236 @@
+"""Core LHTML processing functions.
+
+Each function transforms LHTML markup into HTML for one language
+feature (headings, bold, lists, tags, etc.). The pipeline execution
+order is managed by pipeline.py.
+"""
+
+import os
+import re
+import warnings
+
+import yaml
+
+from .element_extract import extract_bracket_elements
+from .export_html import (
+    export_html_element_class_and_id, export_html_generic,
+    export_html_link, export_html_img, export_html_video,
+    check_is_closing_tag,
+)
+from .listing import process_listing  # noqa: F401 — re-exported
+from .code import export_html_code
+from .errors import LHTMLFileNotFound, LHTMLTagStackError
+from .patterns import (
+    YAML_FRONTMATTER, VERBATIM_BLOCK, VERBATIM_INDEX,
+    CODE_BLOCK, HEADING, BOLD, ITALIC, INLINE_CODE,
+    COMMENT, INCLUDE, TAG_MARKER,
+    VERBATIM_OPEN, VERBATIM_CLOSE, CODE_CLOSE, SPACER_TAG,
+    regex_transform,
+)
+
+
+# ---------------------------------------------------------------------------
+# YAML front matter
+# ---------------------------------------------------------------------------
+
+def process_yaml(text):
+    """Extract YAML front matter and return (remaining_text, meta_dict)."""
+    match = YAML_FRONTMATTER.search(text)
+    if match:
+        new_text = text[:match.start()] + text[match.end():]
+        try:
+            yaml_content = yaml.load(match.group(1), Loader=yaml.FullLoader) or {}
+        except yaml.YAMLError:
+            yaml_content = {}
+        return new_text, yaml_content
+    return text, {}
+
+
+# ---------------------------------------------------------------------------
+# Verbatim blocks (protect / restore)
+# ---------------------------------------------------------------------------
+
+def process_verbatim_to_index(text, verbatim_index_store):
+    """Replace verbatim blocks with indexed placeholders."""
+    def _store(m):
+        content = m.group(0)[len(VERBATIM_OPEN):-len(VERBATIM_CLOSE)]
+        idx = len(verbatim_index_store)
+        verbatim_index_store.append(content)
+        return f'verbatim::[{idx}]'
+    return regex_transform(text, VERBATIM_BLOCK, _store)
+
+
+def process_verbatim_back_from_index(text, verbatim_index_store):
+    """Restore verbatim blocks from indexed placeholders."""
+    def _restore(m):
+        return verbatim_index_store[int(m.group(1))]
+    return regex_transform(text, VERBATIM_INDEX, _restore)
+
+
+# ---------------------------------------------------------------------------
+# Comments
+# ---------------------------------------------------------------------------
+
+def process_remove_comment(text):
+    """Remove ::#comment lines."""
+    return regex_transform(text, COMMENT, lambda m: '\n')
+
+
+# ---------------------------------------------------------------------------
+# File inclusion
+# ---------------------------------------------------------------------------
+
+def find_file(directories, filename):
+    """Locate a file in the given directory list."""
+    for d in directories:
+        pathname = os.path.join(d, filename)
+        if os.path.isfile(pathname):
+            return pathname
+    raise LHTMLFileNotFound(filename, directories)
+
+
+def process_include(text, directory):
+    """Expand include:: directives. Returns (new_text, found_any)."""
+    found_include = False
+    parts = []
+    prev = 0
+
+    for m in INCLUDE.finditer(text):
+        found_include = True
+        element = extract_bracket_elements(text, m.end())
+        filename = find_file(directory, element['text'])
+
+        with open(filename, 'r') as fid:
+            included = fid.read()
+
+        parts.append(text[prev:m.start()])
+        parts.append(included)
+        prev = element['index_end']
+
+    parts.append(text[prev:])
+    return ''.join(parts), found_include
+
+
+# ---------------------------------------------------------------------------
+# Inline formatting
+# ---------------------------------------------------------------------------
+
+def process_bold(text):
+    """Convert **text** to <strong>text</strong>."""
+    return regex_transform(text, BOLD, lambda m: f'<strong>{m.group(1)}</strong>')
+
+
+def process_italic(text):
+    """Convert __text__ to <em>text</em>."""
+    return regex_transform(text, ITALIC, lambda m: f'<em>{m.group(1)}</em>')
+
+
+def process_code_inline(text):
+    """Convert `text` to <code class="code-inline">text</code>."""
+    return regex_transform(text, INLINE_CODE,
+                           lambda m: f'<code class="code-inline">{m.group(1)}</code>')
+
+
+# ---------------------------------------------------------------------------
+# Headings
+# ---------------------------------------------------------------------------
+
+def process_title(text):
+    """Convert = Title or =(.class) Title to <h1>Title</h1>, etc."""
+    def _heading(m):
+        level = str(len(m.group(1)))
+        class_id = (m.group(2) or '').strip()
+        title = m.group(3)
+        if class_id:
+            attrs = export_html_element_class_and_id(class_id)
+            return f'<h{level}{attrs}>{title}</h{level}>\n'
+        return f'<h{level}>{title}</h{level}>\n'
+    return regex_transform(text, HEADING, _heading)
+
+
+# ---------------------------------------------------------------------------
+# Code blocks
+# ---------------------------------------------------------------------------
+
+def process_code(text):
+    """Parse and highlight code:: blocks."""
+    def _highlight(m):
+        element = extract_bracket_elements(text, m.start() + len('code::'))
+        code = text[element['index_end']:m.end() - len(CODE_CLOSE)]
+        language = element['[]']
+        return export_html_code(code, language)
+    return regex_transform(text, CODE_BLOCK, _highlight)
+
+
+# ---------------------------------------------------------------------------
+# Tag elements (the :: system)
+# ---------------------------------------------------------------------------
+
+def _dispatch_tag(element, tag_to_close, current_directory, registry=None):
+    """Dispatch a parsed tag element to the appropriate handler.
+
+    Uses the tag_registry if provided, otherwise falls back to
+    built-in dispatch for backward compatibility.
+    """
+    tag = element['tag']
+
+    # Try plugin registry first
+    if registry is not None:
+        handler = registry.get(tag)
+        if handler is not None:
+            return handler(element, tag_to_close, current_directory)
+
+    # Built-in tag dispatch (used when no registry, or tag not in registry)
+    if tag == 'div' or tag == 'span':
+        return export_html_generic(element, tag, tag_to_close), True
+    elif tag == 'link':
+        return export_html_link(element), True
+    elif tag == 'img':
+        return export_html_img(element), True
+    elif tag == 'video':
+        return export_html_video(element, '', current_directory), True
+    elif tag == 'videoplay':
+        return export_html_video(element, 'autoplay loop muted', current_directory), True
+    elif tag == '':
+        if check_is_closing_tag(element):
+            if not tag_to_close:
+                warnings.warn(
+                    str(LHTMLTagStackError(source_pos=element.get('index_start', -1))),
+                    stacklevel=3,
+                )
+                return '::??ERROR', True
+            return '</' + tag_to_close.pop() + '>', True
+        if element['text'] == SPACER_TAG:
+            return '<div style="height:1em;"></div>', True
+        if element['[]'] or element['()'] or element['{}'] or element['text']:
+            return export_html_generic(element, 'div', tag_to_close), True
+        return '', False
+
+    # Unrecognized tag — pass through
+    return '', False
+
+
+def process_tag(text, current_directory='', registry=None):
+    """Process all :: tag elements in text."""
+    parts = []
+    prev = 0
+    tag_to_close = []
+
+    for m in TAG_MARKER.finditer(text):
+        if m.start() <= prev:
+            continue  # skip overlapping matches
+
+        element = extract_bracket_elements(text, m.end())
+        html, is_real_tag = _dispatch_tag(element, tag_to_close, current_directory, registry)
+        tag = element['tag']
+        index_end = element['index_end']
+
+        if is_real_tag:
+            parts.append(text[prev:m.start() - len(tag)])
+            parts.append(html)
+        else:
+            parts.append(text[prev:index_end])
+        prev = index_end
+
+    parts.append(text[prev:])
+    return ''.join(parts)

lhtml/tag_element.lark

@@ -0,0 +1,30 @@
+// LHTML tag element grammar for the :: bracket subsystem.
+//
+// Parses the text after :: in constructs like:
+//   tag::(.class #id)[style]{attrs}text::
+//   div::[color:red;]
+//   link::url[link text]
+//   ::(.class)[style]
+//
+// Supports nested brackets: [outer[inner]still_outer]
+
+start: bracket_and_text*
+
+bracket_and_text: paren_group
+                | square_group
+                | curly_group
+                | bare_text
+
+paren_group:  "(" paren_content ")"
+square_group: "[" square_content "]"
+curly_group:  "{" curly_content "}"
+
+// Bracket content allows nested brackets of the same type
+paren_content:  (/[^()]+/ | "(" paren_content ")")*
+square_content: (/[^\[\]]+/ | "[" square_content "]")*
+curly_content:  (/[^{}]+/ | "{" curly_content "}")*
+
+// Bare text: any non-whitespace, non-bracket characters
+bare_text: /[^\s\[\](){}]+/
+
+%import common.WS_INLINE