data-hydrator 0.2.0__py3-none-any.whl

data_hydrator/__init__.py

@@ -0,0 +1,4 @@
+from data_hydrator.pipeline import HydrationPipeline
+from data_hydrator.config import HydrationSettings
+
+__all__ = ["HydrationPipeline", "HydrationSettings"]

data_hydrator/collection/entry_builder.py

@@ -0,0 +1,98 @@
+from data_hydrator.models import (
+    CollectionEntry, CollectionFormat, EntryStatus,
+    SegmentExtractionResult, ImageUploadResult, ExtractionTrack,
+    ImageUploadStatus,
+)
+from data_hydrator.config import HydrationSettings
+
+
+def build_entry(
+    extraction_result: SegmentExtractionResult,
+    image_results: list[ImageUploadResult],
+    slug: str,
+    collection_name: str,
+    generated_url: str,
+    original_url: str,
+    page_title: str,
+    settings: HydrationSettings,
+) -> CollectionEntry:
+    # Build CDN URL lookup: original → cdn
+    cdn_map: dict[str, str] = {
+        r.original_url: r.cdn_url
+        for r in image_results
+        if r.status in (ImageUploadStatus.UPLOADED, ImageUploadStatus.CACHED)
+        and r.cdn_url
+    }
+
+    # Build data dict from successful extractions
+    data: dict = {}
+    body_markdown = ""
+    warnings: list[str] = []
+
+    for field in extraction_result.extracted_fields:
+        if not field.extraction_success:
+            continue
+        value = field.processed_value
+
+        # Rewrite image URLs to CDN
+        if field.content_type == "image_url" and isinstance(value, str):
+            value = cdn_map.get(value, value)
+            if value not in cdn_map and field.processed_value and field.processed_value not in cdn_map:
+                if field.processed_value:
+                    warnings.append(f"Image upload failed for {field.field_name}; keeping original URL")
+
+        data[field.field_name] = value
+
+    # For markdown entries: pop body field from data into body_markdown
+    fmt = _resolve_format(extraction_result.component_type, settings)
+    if fmt == CollectionFormat.MARKDOWN:
+        for body_field in ("body", "content", "description"):
+            if body_field in data:
+                candidate = data[body_field]
+                if isinstance(candidate, str) and len(candidate) > 50:
+                    body_markdown = candidate
+                    del data[body_field]
+                    break
+        # Also grab from Track B markdown
+        if not body_markdown:
+            for field in extraction_result.extracted_fields:
+                if field.content_type == "markdown" and field.extraction_success:
+                    body_markdown = str(field.processed_value or "")
+                    data.pop(field.field_name, None)
+                    break
+
+    # Title resolution
+    title = (
+        data.get("title")
+        or data.get("name")
+        or data.get("heading")
+        or page_title
+        or _slug_to_title(slug)
+    )
+
+    draft = len(extraction_result.missing_required_fields) > 0
+
+    return CollectionEntry(
+        slug=slug,
+        collection_name=collection_name,
+        format=fmt,
+        data=data,
+        draft=draft,
+        title=str(title),
+        body_markdown=body_markdown,
+        source_segment_id=extraction_result.segment_id,
+        source_page_url=extraction_result.page_url,
+        generated_url=generated_url,
+        original_url=original_url,
+    )
+
+
+def _resolve_format(component_type: str, settings: HydrationSettings) -> CollectionFormat:
+    content_prefixes = ("content.", "collection.blog", "collection.news")
+    if any(component_type.startswith(p) for p in content_prefixes):
+        return CollectionFormat.MARKDOWN
+    return CollectionFormat.JSON
+
+
+def _slug_to_title(slug: str) -> str:
+    return slug.replace("-", " ").title()

data_hydrator/collection/serializer.py

@@ -0,0 +1,42 @@
+import json
+import re
+import yaml
+from data_hydrator.models import CollectionEntry, CollectionFormat
+
+
+def serialize(entry: CollectionEntry) -> str:
+    if entry.format == CollectionFormat.MARKDOWN:
+        return _serialize_markdown(entry)
+    return _serialize_json(entry)
+
+
+def _serialize_json(entry: CollectionEntry) -> str:
+    payload: dict = {}
+    payload["draft"] = entry.draft
+    payload["title"] = entry.title
+    payload["generatedUrl"] = entry.generated_url
+    for k, v in entry.data.items():
+        payload[_snake_to_camel(k)] = v
+    return json.dumps(payload, indent=2, ensure_ascii=False)
+
+
+def _serialize_markdown(entry: CollectionEntry) -> str:
+    frontmatter: dict = {}
+    frontmatter["draft"] = entry.draft
+    frontmatter["title"] = entry.title
+    frontmatter["generatedUrl"] = entry.generated_url
+    for k, v in entry.data.items():
+        frontmatter[_snake_to_camel(k)] = v
+
+    yaml_block = _to_yaml_frontmatter(frontmatter)
+    body = entry.body_markdown or ""
+    return f"---\n{yaml_block}---\n\n{body}\n"
+
+
+def _to_yaml_frontmatter(data: dict) -> str:
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _snake_to_camel(s: str) -> str:
+    parts = s.split("_")
+    return parts[0] + "".join(p.capitalize() for p in parts[1:])

data_hydrator/collection/slug_generator.py

@@ -0,0 +1,33 @@
+import re
+from collections import defaultdict
+
+
+class SlugGenerator:
+
+    def __init__(self, max_length: int = 80):
+        self._max_length = max_length
+        self._used: dict[str, set[str]] = defaultdict(set)
+
+    def generate(self, title: str, collection: str) -> str:
+        base = self._slugify(title)
+        return self._make_unique(base, collection)
+
+    def _slugify(self, text: str) -> str:
+        if not text:
+            return "untitled"
+        text = text.lower()
+        text = re.sub(r"[^a-z0-9]+", "-", text)
+        text = text.strip("-")
+        return text[: self._max_length] or "untitled"
+
+    def _make_unique(self, slug: str, collection: str) -> str:
+        if slug not in self._used[collection]:
+            self._used[collection].add(slug)
+            return slug
+        counter = 2
+        while True:
+            candidate = f"{slug}-{counter}"
+            if candidate not in self._used[collection]:
+                self._used[collection].add(candidate)
+                return candidate
+            counter += 1

data_hydrator/config.py

@@ -0,0 +1,121 @@
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class PipelineDirsConfig(BaseModel):
+    scrape_dir:     str = "./scrape_output"
+    segmented_dir:  str = "./segmented"
+    classified_dir: str = "./classified"
+    mapped_dir:     str = "./mapped"
+    staging_dir:    str = "./staging"
+
+
+class ResumeConfig(BaseModel):
+    skip_segmentation_if_cached:   bool = True
+    skip_classification_if_cached: bool = True
+    skip_mapping_if_cached:        bool = True
+
+
+class SegmenterConfig(BaseModel):
+    use_llm: bool = True
+    llm_model: str = "anthropic/claude-haiku-4-5"
+    llm_confidence_threshold: float = 0.7
+    max_concurrent_pages: int = 10
+
+
+class ClassifierConfig(BaseModel):
+    rule_based_confidence_threshold: float = 0.90
+    l1_cache_path: str = ".cache/l1_fingerprints.json"
+    l2_cache_path: str = ".cache/l2_clusters.json"
+    litellm_batch_size: int = 20
+    max_concurrent_batches: int = 5
+
+
+class MapperConfig(BaseModel):
+    mcp_transport: str = "stdio"
+    signature_index_cache_path: str = ".cache/signature_index.json"
+    custom_registry_path: str = ".cache/custom_registry.json"
+    litellm_batch_size: int = 15
+
+
+class SchemaGenConfig(BaseModel):
+    model: str = "anthropic/claude-sonnet-4-5"
+    unknown_model: str = "anthropic/claude-haiku-4-5"
+    representatives_per_type: int = 3
+    unknown_batch_size: int = 20
+    max_concurrent_unknown_batches: int = 5
+    timeout_seconds: int = 60
+    schema_cache_path: str = ".cache/extraction_schemas"
+
+
+class ContentExtractionConfig(BaseModel):
+    markdown_library: str = "markdownify"
+    allowed_tags: list[str] = [
+        "p", "strong", "em", "b", "i",
+        "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5",
+        "a", "blockquote", "code", "pre",
+        "table", "thead", "tbody", "tr", "th", "td", "img",
+    ]
+    always_strip_tags: list[str] = [
+        "script", "style", "iframe", "noscript",
+        "nav", "header", "footer", "aside",
+    ]
+
+
+class URLGenConfig(BaseModel):
+    type_base_paths: dict[str, str] = {
+        "collection.product_card": "/products",
+        "collection.product_list": "/products",
+        "collection.blog_card":    "/blog",
+        "collection.blog_list":    "/blog",
+        "collection.news_item":    "/news",
+        "content.article":         "/articles",
+    }
+    category_detection_depth: int = 3
+    category_min_occurrence: int = 2
+    max_slug_length: int = 80
+
+
+class StagingConfig(BaseModel):
+    astro_content_prefix: str = "src/content"
+    astro_pages_prefix: str = "src/pages/_data"
+    create_backup_on_overwrite: bool = True
+    manifest_filename: str = "hydration_manifest.json"
+    redirects_json_filename: str = "redirects.json"
+    redirects_csv_filename: str = "redirects.csv"
+
+
+class R2Config(BaseModel):
+    bucket_name: str = ""
+    endpoint_url: str = ""
+    access_key_id: str = ""
+    secret_access_key: str = ""
+    public_base_url: str = ""
+    key_prefix: str = "images"
+    max_concurrent_uploads: int = 10
+    max_concurrent_downloads: int = 20
+    download_timeout_seconds: int = 30
+    upload_timeout_seconds: int = 60
+    url_cache_path: str = ".cache/image_url_cache.json"
+
+
+class HydrationSettings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_prefix="HYDRATOR_")
+
+    site_id: str = ""
+
+    dirs:       PipelineDirsConfig      = PipelineDirsConfig()
+    resume:     ResumeConfig            = ResumeConfig()
+    segmenter:  SegmenterConfig         = SegmenterConfig()
+    classifier: ClassifierConfig        = ClassifierConfig()
+    mapper:     MapperConfig            = MapperConfig()
+    schema_gen: SchemaGenConfig         = SchemaGenConfig()
+    content:    ContentExtractionConfig = ContentExtractionConfig()
+    url_gen:    URLGenConfig            = URLGenConfig()
+    staging:    StagingConfig           = StagingConfig()
+    r2:         R2Config                = R2Config()
+
+    content_type_prefixes: list[str] = [
+        "content.", "collection.blog", "collection.news",
+    ]
+    max_concurrent_segments: int = 50

data_hydrator/extraction/content_extractor.py

@@ -0,0 +1,95 @@
+import logging
+import re
+from urllib.parse import urljoin
+from bs4 import BeautifulSoup
+import markdownify
+from data_hydrator.models import ExtractionSchema, ExtractedField
+from data_hydrator.config import ContentExtractionConfig
+from data_hydrator.extraction.field_extractor import extract_fields
+
+logger = logging.getLogger(__name__)
+
+
+def extract_content(
+    segment_html: str,
+    schema: ExtractionSchema,
+    page_url: str,
+    settings: ContentExtractionConfig,
+) -> tuple[list[ExtractedField], str, list[str]]:
+    """Returns (metadata_fields, body_markdown, missing_required_fields). Never raises."""
+    try:
+        # Step 1: Extract metadata fields using field selectors
+        metadata_fields, missing_required = extract_fields(segment_html, schema, page_url)
+
+        # Step 2: Parse for content extraction
+        soup = BeautifulSoup(segment_html, "lxml")
+
+        # Step 3: Find content container
+        if schema.content_selector:
+            container = soup.select_one(schema.content_selector)
+            if container is None:
+                logger.warning(
+                    "content_selector '%s' found nothing; using full segment",
+                    schema.content_selector,
+                )
+                container = soup
+        else:
+            container = soup
+
+        # Step 4: Remove noise elements
+        _remove_noise(container, schema.exclude_selectors)
+
+        # Step 5: Remove always-strip tags
+        for tag_name in settings.always_strip_tags:
+            for el in container.find_all(tag_name):
+                el.decompose()
+
+        # Step 6: Convert to Markdown
+        content_html = str(container)
+        md = markdownify.markdownify(
+            content_html,
+            strip=settings.always_strip_tags,
+            convert=settings.allowed_tags,
+            heading_style="ATX",
+        )
+
+        # Post-process
+        md = re.sub(r"\n{3,}", "\n\n", md)
+        md = md.strip()
+        md = _resolve_relative_urls(md, page_url)
+
+        return metadata_fields, md, missing_required
+
+    except Exception as exc:
+        logger.error("Content extraction failed: %s", exc)
+        return [], "", []
+
+
+def _remove_noise(soup: BeautifulSoup, exclude_selectors: list[str]) -> None:
+    for selector in exclude_selectors:
+        try:
+            for el in soup.select(selector):
+                el.decompose()
+        except Exception as exc:
+            logger.warning("Exclude selector '%s' failed: %s", selector, exc)
+
+
+def _resolve_relative_urls(markdown: str, base_url: str) -> str:
+    if not base_url:
+        return markdown
+
+    def replace_image(m: re.Match) -> str:
+        alt, url = m.group(1), m.group(2)
+        resolved = urljoin(base_url, url)
+        return f"![{alt}]({resolved})"
+
+    def replace_link(m: re.Match) -> str:
+        text, url = m.group(1), m.group(2)
+        resolved = urljoin(base_url, url)
+        return f"[{text}]({resolved})"
+
+    # Images first (more specific pattern)
+    markdown = re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", replace_image, markdown)
+    # Then links
+    markdown = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", replace_link, markdown)
+    return markdown

data_hydrator/extraction/field_extractor.py

@@ -0,0 +1,168 @@
+import logging
+import re
+from urllib.parse import urljoin
+from bs4 import BeautifulSoup
+from data_hydrator.models import ExtractionSchema, ExtractedField, FieldSelector
+
+logger = logging.getLogger(__name__)
+
+
+def extract_fields(
+    segment_html: str,
+    schema: ExtractionSchema,
+    page_url: str,
+) -> tuple[list[ExtractedField], list[str]]:
+    """Returns (fields, missing_required_field_names). Never raises."""
+    try:
+        soup = BeautifulSoup(segment_html, "lxml")
+    except Exception as exc:
+        logger.error("Failed to parse segment HTML: %s", exc)
+        missing = [fs.field_name for fs in schema.field_selectors if fs.required]
+        return [], missing
+
+    fields: list[ExtractedField] = []
+    missing_required: list[str] = []
+
+    for selector in schema.field_selectors:
+        field = _extract_one(soup, selector, page_url)
+        fields.append(field)
+        if not field.extraction_success and selector.required:
+            missing_required.append(selector.field_name)
+
+    return fields, missing_required
+
+
+def _extract_one(
+    soup: BeautifulSoup,
+    selector: FieldSelector,
+    page_url: str,
+) -> ExtractedField:
+    css, attr_override = _parse_selector_attr(selector.css_selector)
+    attribute = selector.attribute or attr_override
+
+    try:
+        if selector.content_type == "list" or selector.multiple:
+            elements = soup.select(css)
+            if not elements:
+                return ExtractedField(
+                    field_name=selector.field_name,
+                    raw_value=None,
+                    processed_value=[],
+                    content_type=selector.content_type,
+                    extraction_success=False,
+                    warning=f"No elements matched selector '{css}'",
+                )
+            values = [el.get_text(strip=True) for el in elements]
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=values,
+                processed_value=values,
+                content_type=selector.content_type,
+                extraction_success=True,
+            )
+
+        element = soup.select_one(css)
+
+        if selector.content_type == "boolean":
+            val = element is not None
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=val,
+                processed_value=val,
+                content_type="boolean",
+                extraction_success=True,
+            )
+
+        if element is None:
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=None,
+                processed_value=None,
+                content_type=selector.content_type,
+                extraction_success=False,
+                warning=f"No element matched selector '{css}'",
+            )
+
+        if selector.content_type == "text":
+            text = element.get_text(strip=True)
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=text,
+                processed_value=text,
+                content_type="text",
+                extraction_success=bool(text),
+                warning="" if text else f"Empty text at selector '{css}'",
+            )
+
+        if selector.content_type == "image_url":
+            src = _get_image_src(element, attribute)
+            if src:
+                src = urljoin(page_url, src)
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=src,
+                processed_value=src,
+                content_type="image_url",
+                extraction_success=bool(src),
+                warning="" if src else f"Could not extract image URL at '{css}'",
+            )
+
+        if selector.content_type == "url":
+            attr = attribute or "href"
+            href = element.get(attr, "")
+            if href:
+                href = urljoin(page_url, href)
+            return ExtractedField(
+                field_name=selector.field_name,
+                raw_value=href,
+                processed_value=href,
+                content_type="url",
+                extraction_success=bool(href),
+                warning="" if href else f"No href at selector '{css}'",
+            )
+
+        # Fallback: treat as text
+        text = element.get_text(strip=True)
+        return ExtractedField(
+            field_name=selector.field_name,
+            raw_value=text,
+            processed_value=text,
+            content_type=selector.content_type,
+            extraction_success=bool(text),
+        )
+
+    except Exception as exc:
+        logger.warning("Field extraction failed for %s: %s", selector.field_name, exc)
+        return ExtractedField(
+            field_name=selector.field_name,
+            raw_value=None,
+            processed_value=None,
+            content_type=selector.content_type,
+            extraction_success=False,
+            warning=str(exc),
+        )
+
+
+def _get_image_src(element, attribute: str | None) -> str:
+    """Try multiple src attributes for lazy-loaded images."""
+    candidates = [attribute] if attribute else []
+    candidates += ["src", "data-src", "data-lazy-src"]
+    for attr in candidates:
+        if attr and element.get(attr):
+            return element[attr]
+
+    # Try background-image from style
+    style = element.get("style", "")
+    if "background-image" in style:
+        match = re.search(r"url\(['\"]?([^'\")\s]+)['\"]?\)", style)
+        if match:
+            return match.group(1)
+    return ""
+
+
+def _parse_selector_attr(segment_field: str) -> tuple[str, str | None]:
+    """Split 'img[src]' → ('img', 'src'), 'h2.name' → ('h2.name', None)."""
+    match = re.match(r"^(.*?)\[([^\]]+)\]$", segment_field)
+    if match:
+        return match.group(1), match.group(2)
+    return segment_field, None