segment_classifier 0.1.0__py3-none-any.whl

segment_classifier/utils/html_normalizer.py

@@ -0,0 +1,165 @@
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+from bs4 import BeautifulSoup, Tag, NavigableString
+
+
+STRUCTURAL_CLASS_PATTERN = re.compile(
+    r'\b(card|grid|list|item|hero|nav|menu|header|footer|sidebar|'
+    r'form|modal|badge|price|rating|carousel|pagination|search|'
+    r'feature|testimonial|cta|faq|pricing|article|media|table|'
+    r'product|blog|news|collection|section|widget)\b',
+    re.IGNORECASE
+)
+
+PRESENTATIONAL_CLASS_PATTERN = re.compile(
+    r'\b(mt|mb|ml|mr|mx|my|pt|pb|pl|pr|px|py|w-|h-|text-|bg-|'
+    r'border|rounded|shadow|flex|grid-cols|gap|p-|m-|font-|'
+    r'color|opacity|z-|hidden|block|inline)\b'
+)
+
+STRUCTURAL_ATTRS = {"role", "type", "aria-label", "aria-role", "data-component", "data-type"}
+
+
+@dataclass
+class NormalizedSegment:
+    skeleton: str
+    attrs_fingerprint: str
+    class_tokens: list[str]
+    child_tag_counts: dict[str, int]
+    dom_depth: int
+    root_tag: str
+    text_density_ratio: float
+    unique_tag_count: int
+
+    def fingerprint_hash(self) -> str:
+        payload = {
+            "skeleton": self.skeleton,
+            "attrs": self.attrs_fingerprint,
+            "classes": sorted(self.class_tokens),
+            "counts": self.child_tag_counts,
+            "depth": self.dom_depth,
+            "root": self.root_tag,
+        }
+        return hashlib.sha256(
+            json.dumps(payload, sort_keys=True).encode()
+        ).hexdigest()
+
+
+def normalize_segment(html: str, text_content: str) -> NormalizedSegment:
+    """
+    Parse HTML and extract structural fingerprint components.
+    """
+    soup = BeautifulSoup(html, "html.parser")
+    root = soup.find()
+    if not root or not isinstance(root, Tag):
+        return NormalizedSegment("", "", [], {}, 0, "unknown", 0.0, 0)
+
+    skeleton = _extract_skeleton(root)
+    attrs_fp = _extract_attrs_fingerprint(root)
+    class_tokens = _extract_class_tokens(root)
+    child_counts = _count_tags(root)
+    depth = _max_depth(root)
+    text_ratio = len(text_content) / max(len(html), 1)
+    unique_tags = len(child_counts)
+
+    return NormalizedSegment(
+        skeleton=skeleton,
+        attrs_fingerprint=attrs_fp,
+        class_tokens=class_tokens,
+        child_tag_counts=child_counts,
+        dom_depth=depth,
+        root_tag=root.name,
+        text_density_ratio=round(text_ratio, 4),
+        unique_tag_count=unique_tags,
+    )
+
+
+def _extract_skeleton(tag: Tag, depth: int = 0, max_depth: int = 8) -> str:
+    """Recursive tag-name-only skeleton. Siblings joined with '+', children with '>'."""
+    if depth >= max_depth:
+        return tag.name
+
+    child_skeletons = []
+    for child in tag.children:
+        if isinstance(child, Tag):
+            child_skeletons.append(_extract_skeleton(child, depth + 1, max_depth))
+
+    if not child_skeletons:
+        return tag.name
+
+    return f"{tag.name}>" + "+".join(child_skeletons)
+
+
+def _extract_attrs_fingerprint(tag: Tag) -> str:
+    """Walk all tags, keep only STRUCTURAL_ATTRS values and href/src presence booleans."""
+    parts = []
+
+    def walk(node: Tag):
+        node_parts = []
+        for attr in STRUCTURAL_ATTRS:
+            val = node.get(attr)
+            if val:
+                if isinstance(val, list):
+                    val = " ".join(val)
+                node_parts.append(f"{attr}={val}")
+
+        if node.has_attr("href"):
+            node_parts.append("has_href=true")
+        if node.has_attr("src"):
+            node_parts.append("has_src=true")
+
+        if node_parts:
+            parts.append(f"{node.name}[" + ",".join(sorted(node_parts)) + "]")
+
+        for child in node.children:
+            if isinstance(child, Tag):
+                walk(child)
+
+    walk(tag)
+    return "|".join(parts)
+
+
+def _extract_class_tokens(tag: Tag) -> list[str]:
+    """Extract class names matching STRUCTURAL_CLASS_PATTERN from all tags."""
+    tokens = set()
+
+    def walk(node: Tag):
+        classes = node.get("class", [])
+        if isinstance(classes, str):
+            classes = [classes]
+        for c in classes:
+            if STRUCTURAL_CLASS_PATTERN.search(c) and not PRESENTATIONAL_CLASS_PATTERN.search(c):
+                tokens.add(c)
+        for child in node.children:
+            if isinstance(child, Tag):
+                walk(child)
+
+    walk(tag)
+    return list(tokens)
+
+
+def _count_tags(tag: Tag) -> dict[str, int]:
+    """Count occurrences of each tag name in the full subtree."""
+    counts = {}
+
+    def walk(node: Tag):
+        counts[node.name] = counts.get(node.name, 0) + 1
+        for child in node.children:
+            if isinstance(child, Tag):
+                walk(child)
+
+    # Count tags in the full subtree including the root
+    walk(tag)
+
+    return counts
+
+
+def _max_depth(tag: Tag, current: int = 0) -> int:
+    """Return maximum nesting depth."""
+    child_depths = [current]
+    for child in tag.children:
+        if isinstance(child, Tag):
+            child_depths.append(_max_depth(child, current + 1))
+    return max(child_depths)

segment_classifier-0.1.0.dist-info/METADATA

@@ -0,0 +1,95 @@
+Metadata-Version: 2.3
+Name: segment_classifier
+Version: 0.1.0
+Summary: Async segment classifier library
+Author: Gagandeep Singh
+Author-email: gagan@innerkore.com
+Requires-Python: >=3.12,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: aiofiles (>=23.0,<24.0)
+Requires-Dist: beautifulsoup4 (>=4.12,<5.0)
+Requires-Dist: litellm (>=1.40,<2.0)
+Requires-Dist: lxml (>=5.0,<6.0)
+Requires-Dist: numpy (>=1.26,<2.0)
+Requires-Dist: pydantic (>=2.7,<3.0)
+Requires-Dist: pydantic-settings (>=2.2,<3.0)
+Requires-Dist: scikit-learn (>=1.5,<2.0)
+Description-Content-Type: text/markdown
+
+# Segment Classifier
+
+An asynchronous Python library that classifies HTML segments extracted by a page-segmenter into structured component types.
+
+## Overview
+
+The `segment_classifier` implements a 4-stage classification pipeline with progressive fallback to optimize for cost and speed:
+
+1. **Rule-based heuristics** — Zero LLM cost. Uses DOM structure, text density, siblings, and attributes.
+2. **L1 exact fingerprint cache** — Zero LLM cost. Exact matching on structural DOM fingerprint hashes.
+3. **L2 fuzzy cluster cache** — Zero LLM cost. TF-IDF and cosine similarity on fingerprint tokens.
+4. **LLM batch classification** — Batched fallback via LiteLLM with feature-based model routing based on segment complexity.
+
+## Installation
+
+You can install the package using poetry:
+```bash
+poetry install
+```
+
+Or via pip (once published):
+```bash
+pip install segment-classifier
+```
+
+## Setup
+
+The library uses `pydantic-settings` to manage configuration via a `.env` file or environment variables.
+
+Required environment variables:
+```env
+CLASSIFIER_LITELLM_API_KEY="your-api-key"
+```
+
+## Usage
+
+```python
+import asyncio
+from segment_classifier import ClassifierPipeline
+from segment_classifier.config import ClassifierSettings
+from segment_classifier.models import InputSegment, SegmentPosition
+
+async def main():
+    settings = ClassifierSettings()
+    pipeline = ClassifierPipeline(settings)
+    await pipeline.initialize()
+
+    segments = [
+        InputSegment(
+            segment_id="seg_001",
+            page_url="https://example.com/products",
+            page_slug="products",
+            raw_html="<div class='product-card'>...</div>",
+            text_content="Product Item",
+            position_hint=SegmentPosition.MIDDLE,
+            sibling_count=3,
+        )
+    ]
+
+    result = await pipeline.run(segments)
+    await pipeline.shutdown()
+
+    for seg in result.classified:
+        print(seg.component_type)
+
+asyncio.run(main())
+```
+
+## Caching
+
+Caches are stored by default in `.cache/l1_fingerprints.json` and `.cache/l2_clusters.json` / `.cache/l2_embeddings.npy`.
+
+## Stages Breakdown
+Every returned `ClassifiedSegment` will be marked with a `classification_stage` indicating which of the 4 stages resolved the query.
+

segment_classifier-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+segment_classifier/__init__.py,sha256=oMPRBM2ig-EBqXicWyrRabDqRF1sZ9cDcN5TOEgJY1M,136
+segment_classifier/cache/__init__.py,sha256=XAlXVuoePqSVjL_oteuAbkFp038MYFmuXBwPTVOSAIk,126
+segment_classifier/cache/l1_cache.py,sha256=9YWg5-I5RAmUA8pEUwcwDvKYeO2njLAnvF4iy0LO_cQ,2650
+segment_classifier/cache/l2_cache.py,sha256=cbHksUE8dOUhwpycEIz8h9JaiNR9BRAYPfFJcoQhcFc,5729
+segment_classifier/config.py,sha256=LwVhFZ5uJQqt4eXGwLzX7e-2eLy6SpiUTL1KR382q3w,1871
+segment_classifier/models.py,sha256=p3ebmFim1q3G0qYM-AleV4OwHmLnbQVkgsPgf92YgpQ,4122
+segment_classifier/pipeline.py,sha256=4R5IQcSMF1tSK5IkDQ8W8vbbpcqMMCUGGl0fpY43Sb8,7210
+segment_classifier/stages/__init__.py,sha256=5s_AD_v3uTK0VzXqYKChOyhUiQOTotTp5iPOQUB5TyE,282
+segment_classifier/stages/fingerprint.py,sha256=1JgS1WrpnD5bxAU0j1nWX-QH4Gv3ulMmoz7RhRP0IRk,483
+segment_classifier/stages/fuzzy_cluster.py,sha256=uIPo2EgE0oyQqTx_NHlUZL1pYEjRL36JhoVBdttcwbE,4409
+segment_classifier/stages/llm_classifier.py,sha256=lQl-bh0tlRrAqhBJQsKeW1hYLM1WV93kE9xPCzzZS6k,10874
+segment_classifier/stages/rule_based.py,sha256=qTwvaMr_zSi5HHWh2S6PE_2QANstdpgASDAtr84N29U,11664
+segment_classifier/utils/__init__.py,sha256=dvp1CDEiSMVjUIzDAy_DRmYQQolkPWOIbTrUgA7qLnQ,120
+segment_classifier/utils/html_normalizer.py,sha256=VTD49C0QLLOWTKc0le8ta3jlqg4Ln-7tJ1tuSG35J_E,4981
+segment_classifier-0.1.0.dist-info/METADATA,sha256=mM6Rr0nyjf8NMRRy22xQESkRvTK22ehAcZLoQCITY6U,2943
+segment_classifier-0.1.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+segment_classifier-0.1.0.dist-info/RECORD,,

segment_classifier-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.2
+Root-Is-Purelib: true
+Tag: py3-none-any