llm-html 0.1.2__tar.gz

llm_html-0.1.2/.gitignore

@@ -0,0 +1,14 @@
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.venv/
+venv/
+*.db
+*.log
+.DS_Store

llm_html-0.1.2/PKG-INFO

@@ -0,0 +1,133 @@
+Metadata-Version: 2.4
+Name: llm-html
+Version: 0.1.2
+Summary: LLM-optimized HTML cleaning: hydration extraction, token budgets, multiple output formats
+Project-URL: Homepage, https://cmdop.com
+Project-URL: Repository, https://github.com/commandoperator/llm-html
+Author-email: CMDOP Team <team@cmdop.com>
+License-Expression: MIT
+Keywords: beautifulsoup,cleaner,cmdop,html,hydration,llm,markdown,token-budget,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Markup :: HTML
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: lxml>=4.9.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: python-toon>=0.1.3
+Provides-Extra: dev
+Requires-Dist: mypy>=1.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.1.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# llm-html
+
+LLM-optimized HTML cleaning: hydration extraction, token budgets, multiple output formats.
+
+## Install
+
+```bash
+pip install llm-html
+```
+
+## Quick Start
+
+```python
+from llm_html import HTMLCleaner, CleanerConfig, OutputFormat
+
+# Basic cleaning
+cleaner = HTMLCleaner()
+result = cleaner.clean(html)
+print(f"Reduction: {result.stats.reduction_percent}%")
+
+# Hydration-first (extracts SSR data from Next.js, Nuxt, etc.)
+if result.hydration_data:
+    data = result.hydration_data
+else:
+    cleaned = result.html
+```
+
+## Convenience Functions
+
+```python
+from llm_html import clean, clean_to_json, clean_html, clean_for_llm
+
+# Quick clean
+result = clean(html)
+
+# Get JSON if SSR data available, otherwise cleaned HTML
+data = clean_to_json(html)
+
+# Pipeline with full control
+result = clean_html(html, max_tokens=5000)
+result = clean_for_llm(html, output_format="markdown")
+```
+
+## Output Formats
+
+```python
+from llm_html import to_markdown, to_aom_yaml, to_xtree
+
+md = to_markdown(html)
+aom = to_aom_yaml(html)
+xtree = to_xtree(html)
+```
+
+## Downsampling
+
+Token-budget targeting with D2Snap algorithm:
+
+```python
+from llm_html import downsample_html, estimate_tokens
+
+tokens = estimate_tokens(html)
+if tokens > 10000:
+    html = downsample_html(html, target_tokens=8000)
+```
+
+## Semantic Chunking
+
+Split large pages into LLM-sized chunks:
+
+```python
+from llm_html import SemanticChunker, ChunkConfig
+
+config = ChunkConfig(max_tokens=8000, max_items=20)
+chunker = SemanticChunker(config)
+result = chunker.chunk(soup)
+for chunk in result.chunks:
+    process(chunk.html)
+```
+
+## Shadow DOM
+
+Flatten Web Components for LLM visibility:
+
+```python
+from llm_html import flatten_shadow_dom
+
+flat = flatten_shadow_dom(html)
+```
+
+## Helpers
+
+```python
+from llm_html import html_to_text, extract_links, extract_images, json_to_toon
+
+text = html_to_text(html)
+links = extract_links(html, base_url="https://example.com")
+images = extract_images(html)
+toon = json_to_toon({"key": "value"})
+```
+
+## License
+
+MIT

llm_html-0.1.2/README.md

@@ -0,0 +1,103 @@
+# llm-html
+
+LLM-optimized HTML cleaning: hydration extraction, token budgets, multiple output formats.
+
+## Install
+
+```bash
+pip install llm-html
+```
+
+## Quick Start
+
+```python
+from llm_html import HTMLCleaner, CleanerConfig, OutputFormat
+
+# Basic cleaning
+cleaner = HTMLCleaner()
+result = cleaner.clean(html)
+print(f"Reduction: {result.stats.reduction_percent}%")
+
+# Hydration-first (extracts SSR data from Next.js, Nuxt, etc.)
+if result.hydration_data:
+    data = result.hydration_data
+else:
+    cleaned = result.html
+```
+
+## Convenience Functions
+
+```python
+from llm_html import clean, clean_to_json, clean_html, clean_for_llm
+
+# Quick clean
+result = clean(html)
+
+# Get JSON if SSR data available, otherwise cleaned HTML
+data = clean_to_json(html)
+
+# Pipeline with full control
+result = clean_html(html, max_tokens=5000)
+result = clean_for_llm(html, output_format="markdown")
+```
+
+## Output Formats
+
+```python
+from llm_html import to_markdown, to_aom_yaml, to_xtree
+
+md = to_markdown(html)
+aom = to_aom_yaml(html)
+xtree = to_xtree(html)
+```
+
+## Downsampling
+
+Token-budget targeting with D2Snap algorithm:
+
+```python
+from llm_html import downsample_html, estimate_tokens
+
+tokens = estimate_tokens(html)
+if tokens > 10000:
+    html = downsample_html(html, target_tokens=8000)
+```
+
+## Semantic Chunking
+
+Split large pages into LLM-sized chunks:
+
+```python
+from llm_html import SemanticChunker, ChunkConfig
+
+config = ChunkConfig(max_tokens=8000, max_items=20)
+chunker = SemanticChunker(config)
+result = chunker.chunk(soup)
+for chunk in result.chunks:
+    process(chunk.html)
+```
+
+## Shadow DOM
+
+Flatten Web Components for LLM visibility:
+
+```python
+from llm_html import flatten_shadow_dom
+
+flat = flatten_shadow_dom(html)
+```
+
+## Helpers
+
+```python
+from llm_html import html_to_text, extract_links, extract_images, json_to_toon
+
+text = html_to_text(html)
+links = extract_links(html, base_url="https://example.com")
+images = extract_images(html)
+toon = json_to_toon({"key": "value"})
+```
+
+## License
+
+MIT

llm_html-0.1.2/pyproject.toml

@@ -0,0 +1,83 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "llm-html"
+version = "0.1.2"
+description = "LLM-optimized HTML cleaning: hydration extraction, token budgets, multiple output formats"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [
+    { name = "CMDOP Team", email = "team@cmdop.com" }
+]
+keywords = [
+    "html",
+    "llm",
+    "cleaner",
+    "beautifulsoup",
+    "markdown",
+    "hydration",
+    "token-budget",
+    "web-scraping",
+    "cmdop",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing :: Markup :: HTML",
+    "Typing :: Typed",
+]
+
+dependencies = [
+    "beautifulsoup4>=4.12.0",
+    "lxml>=4.9.0",
+    "pydantic>=2.0.0",
+    "python-toon>=0.1.3",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.0.0",
+    "pytest-cov>=4.0.0",
+    "ruff>=0.1.0",
+    "mypy>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://cmdop.com"
+Repository = "https://github.com/commandoperator/llm-html"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/llm_html"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/",
+    "README.md",
+    "pyproject.toml",
+    "LICENSE",
+]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP", "B", "C4", "SIM"]
+ignore = ["E501"]
+
+[tool.mypy]
+python_version = "3.10"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

llm_html-0.1.2/src/llm_html/__init__.py

@@ -0,0 +1,213 @@
+"""HTML Cleaner module for LLM-optimized HTML processing.
+
+Provides intelligent HTML cleaning optimized for Large Language Models:
+
+- **Hydration-First**: Extract SSR data (Next.js, Nuxt, etc.) before DOM parsing
+- **Token Budget**: Target specific token limits with adaptive downsampling
+- **Multiple Outputs**: HTML, Markdown, AOM YAML, XTree formats
+- **Detailed Statistics**: Track reduction, timing, element counts
+
+Example usage:
+    from llm_html import HTMLCleaner, CleanerConfig, OutputFormat
+
+    # Basic usage
+    cleaner = HTMLCleaner()
+    result = cleaner.clean(html)
+
+    # Check statistics
+    print(f"Size: {result.stats.original_size} → {result.stats.cleaned_size}")
+    print(f"Reduction: {result.stats.reduction_percent}%")
+    print(f"Scripts removed: {result.stats.scripts_removed}")
+
+    # Use hydration data if available (most efficient)
+    if result.hydration_data:
+        products = result.hydration_data.get("products", [])
+    else:
+        cleaned = result.html
+
+    # Custom configuration
+    config = CleanerConfig(
+        max_tokens=5000,
+        output_format=OutputFormat.MARKDOWN,
+        filter_classes=True,
+    )
+    cleaner = HTMLCleaner(config)
+    result = cleaner.clean(html)
+
+Convenience functions:
+    from llm_html import clean, clean_to_json
+
+    # Quick clean with default settings
+    result = clean(html)
+
+    # Get JSON if SSR data available, otherwise cleaned HTML
+    data = clean_to_json(html)
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, Union
+
+# =============================================================================
+# Cleaner subpackage (HTML cleaning pipeline)
+# =============================================================================
+
+# Models
+from .cleaner.models import (
+    OutputFormat,
+    CleanerConfig,
+    CleanerStats,
+    ChunkInfo,
+    CleanerResult,
+)
+
+# Main cleaner class
+from .cleaner.cleaner import (
+    HTMLCleaner,
+    clean,
+    clean_to_json,
+)
+
+# Extractors
+from .cleaner.extractors import (
+    # Hydration extraction
+    HydrationExtractor,
+    HydrationData,
+    Framework,
+    extract_hydration,
+    detect_framework,
+    # Context Window
+    ContextExtractor,
+    ContextWindow,
+    ContextConfig,
+    extract_context,
+    find_stable_anchor,
+    generate_selector,
+)
+
+# Transformers
+from .cleaner.transformers import (
+    # Shadow DOM
+    ShadowDOMFlattener,
+    flatten_shadow_dom,
+    # D2Snap Downsampling
+    D2SnapDownsampler,
+    D2SnapConfig,
+    downsample_html,
+    estimate_tokens,
+    # Semantic Chunking
+    SemanticChunker,
+    ChunkConfig,
+    ChunkResult,
+)
+
+# Classifiers
+from .cleaner.classifiers import (
+    ClassSemanticScorer,
+    score_class,
+    filter_classes,
+    clean_classes,
+    detect_css_framework,
+)
+
+# Output Formats
+from .cleaner.outputs import (
+    # AOM YAML (Playwright-style Aria Snapshot)
+    AOMYAMLExporter,
+    AOMConfig,
+    to_aom_yaml,
+    # Markdown
+    MarkdownExporter,
+    MarkdownConfig,
+    to_markdown,
+    # XTree
+    XTreeExporter,
+    XTreeConfig,
+    to_xtree,
+)
+
+# Pipeline
+from .cleaner.pipeline import (
+    CleaningPipeline,
+    PipelineConfig,
+    PipelineResult,
+    clean_html,
+    clean_for_llm,
+)
+
+# =============================================================================
+# Helpers (parsing utilities)
+# =============================================================================
+
+from .helpers import (
+    json_to_toon,
+    JsonCleaner,
+    html_to_text,
+    extract_links,
+    extract_images,
+)
+
+
+__all__ = [
+    # Primary API
+    "HTMLCleaner",
+    "CleanerConfig",
+    "CleanerResult",
+    "CleanerStats",
+    "ChunkInfo",
+    "OutputFormat",
+    "clean",
+    "clean_to_json",
+    # Extractors - Hydration
+    "HydrationExtractor",
+    "HydrationData",
+    "Framework",
+    "extract_hydration",
+    "detect_framework",
+    # Extractors - Context Window
+    "ContextExtractor",
+    "ContextWindow",
+    "ContextConfig",
+    "extract_context",
+    "find_stable_anchor",
+    "generate_selector",
+    # Transformers - Shadow DOM
+    "ShadowDOMFlattener",
+    "flatten_shadow_dom",
+    # Transformers - D2Snap
+    "D2SnapDownsampler",
+    "D2SnapConfig",
+    "downsample_html",
+    "estimate_tokens",
+    # Transformers - Chunking
+    "SemanticChunker",
+    "ChunkConfig",
+    "ChunkResult",
+    # Classifiers
+    "ClassSemanticScorer",
+    "score_class",
+    "filter_classes",
+    "clean_classes",
+    "detect_css_framework",
+    # Outputs
+    "AOMYAMLExporter",
+    "AOMConfig",
+    "to_aom_yaml",
+    "MarkdownExporter",
+    "MarkdownConfig",
+    "to_markdown",
+    "XTreeExporter",
+    "XTreeConfig",
+    "to_xtree",
+    # Pipeline
+    "CleaningPipeline",
+    "PipelineConfig",
+    "PipelineResult",
+    "clean_html",
+    "clean_for_llm",
+    # Helpers
+    "json_to_toon",
+    "JsonCleaner",
+    "html_to_text",
+    "extract_links",
+    "extract_images",
+]

llm_html-0.1.2/src/llm_html/cleaner/__init__.py

@@ -0,0 +1,137 @@
+"""HTML Cleaner subpackage — aggressive & focused cleaning, SSR hydration, output formats."""
+
+from __future__ import annotations
+
+# Models
+from .models import (
+    OutputFormat,
+    CleanerConfig,
+    CleanerStats,
+    ChunkInfo,
+    CleanerResult,
+)
+
+# Main cleaner class
+from .cleaner import (
+    HTMLCleaner,
+    clean,
+    clean_to_json,
+)
+
+# Extractors
+from .extractors import (
+    HydrationExtractor,
+    HydrationData,
+    Framework,
+    extract_hydration,
+    detect_framework,
+    ContextExtractor,
+    ContextWindow,
+    ContextConfig,
+    extract_context,
+    find_stable_anchor,
+    generate_selector,
+)
+
+# Transformers
+from .transformers import (
+    ShadowDOMFlattener,
+    flatten_shadow_dom,
+    D2SnapDownsampler,
+    D2SnapConfig,
+    downsample_html,
+    estimate_tokens,
+    SemanticChunker,
+    ChunkConfig,
+    ChunkResult,
+)
+
+# Classifiers
+from .classifiers import (
+    ClassSemanticScorer,
+    score_class,
+    filter_classes,
+    clean_classes,
+    detect_css_framework,
+)
+
+# Output Formats
+from .outputs import (
+    AOMYAMLExporter,
+    AOMConfig,
+    to_aom_yaml,
+    MarkdownExporter,
+    MarkdownConfig,
+    to_markdown,
+    XTreeExporter,
+    XTreeConfig,
+    to_xtree,
+)
+
+# Pipeline
+from .pipeline import (
+    CleaningPipeline,
+    PipelineConfig,
+    PipelineResult,
+    clean_html,
+    clean_for_llm,
+)
+
+__all__ = [
+    # Primary API
+    "HTMLCleaner",
+    "CleanerConfig",
+    "CleanerResult",
+    "CleanerStats",
+    "ChunkInfo",
+    "OutputFormat",
+    "clean",
+    "clean_to_json",
+    # Extractors - Hydration
+    "HydrationExtractor",
+    "HydrationData",
+    "Framework",
+    "extract_hydration",
+    "detect_framework",
+    # Extractors - Context Window
+    "ContextExtractor",
+    "ContextWindow",
+    "ContextConfig",
+    "extract_context",
+    "find_stable_anchor",
+    "generate_selector",
+    # Transformers - Shadow DOM
+    "ShadowDOMFlattener",
+    "flatten_shadow_dom",
+    # Transformers - D2Snap
+    "D2SnapDownsampler",
+    "D2SnapConfig",
+    "downsample_html",
+    "estimate_tokens",
+    # Transformers - Chunking
+    "SemanticChunker",
+    "ChunkConfig",
+    "ChunkResult",
+    # Classifiers
+    "ClassSemanticScorer",
+    "score_class",
+    "filter_classes",
+    "clean_classes",
+    "detect_css_framework",
+    # Outputs
+    "AOMYAMLExporter",
+    "AOMConfig",
+    "to_aom_yaml",
+    "MarkdownExporter",
+    "MarkdownConfig",
+    "to_markdown",
+    "XTreeExporter",
+    "XTreeConfig",
+    "to_xtree",
+    # Pipeline
+    "CleaningPipeline",
+    "PipelineConfig",
+    "PipelineResult",
+    "clean_html",
+    "clean_for_llm",
+]