spextract 0.5.16__py3-none-any.whl

spextract/__init__.py

@@ -0,0 +1,27 @@
+from .models import (
+    ParseSpec,
+    FieldSpec,
+    FieldType,
+    ProcessorSpec,
+    EngineOutput,
+    SpecValidationResult,
+    ValidationMismatch,
+    ExpectedResults,
+    FileExpectedItems,
+)
+from .engine import ParseEngine
+from .validation.spec_validate import validate_spec_output
+
+__all__ = [
+    "FieldSpec",
+    "FieldType",
+    "ParseEngine",
+    "ParseSpec",
+    "ProcessorSpec",
+    "EngineOutput",
+    "SpecValidationResult",
+    "ValidationMismatch",
+    "ExpectedResults",
+    "FileExpectedItems",
+    "validate_spec_output",
+]

spextract/cli.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+import yaml
+
+from .engine import ParseEngine
+from .models.output import EngineOutput
+from .models.parser_spec import ParseSpec
+from .models.validation import ExpectedResults, FileExpectedItems
+from .validation.true_validate import validate_files
+
+
+@click.group()
+def cli() -> None:
+    """Declarative scraper utilities."""
+
+
+@cli.command()
+@click.argument("spec_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.argument("html_path", type=click.Path(exists=True, dir_okay=True, path_type=Path))
+@click.option("--output", "output_path", type=click.Path(dir_okay=False, path_type=Path))
+def parse(spec_path: Path, html_path: Path, output_path: Path | None) -> None:
+    """Apply a spec to one or more HTML files."""
+
+    spec = ParseSpec.from_yaml_file(spec_path)
+    engine = ParseEngine(spec)
+    results: dict[str, EngineOutput] = {}
+    if html_path.is_file():
+        html_files = [html_path]
+    else:
+        html_files = sorted(html_path.glob("*.html"))
+
+    for html_file in html_files:
+        html = html_file.read_text(encoding="utf-8")
+        parsed = engine.parse(html)
+        results[html_file.name] = parsed
+
+    file_results = ExpectedResults(
+        data_path=html_path if html_path.is_dir() else html_path.parent,
+        files=[FileExpectedItems.from_engine_output(file_name, output) for file_name, output in results.items()],
+    )
+
+    if output_path is not None:
+        file_results.to_yaml_file(output_path)
+        click.echo(f"Wrote parsed results to {output_path}")
+        return
+
+    rendered_yaml = yaml.safe_dump(file_results.model_dump(), sort_keys=False, allow_unicode=True)
+    click.echo(rendered_yaml, nl=False)
+
+
+@cli.command()
+@click.argument("spec_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.argument("expected_results_path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option(
+    "--field-path", "-f", type=str, help="Optional dot path to a specific field to validate (e.g. 'items.name')."
+)
+def validate(spec_path: Path, expected_results_path: Path, field_path: str | None) -> None:
+    """Validate a spec against YAML expected extraction results."""
+
+    validation_result = validate_files(
+        expected_values_path=expected_results_path,
+        spec_file_path=spec_path,
+        field_path=field_path,
+    )
+    if validation_result.passed:
+        click.echo(
+            f"Validation passed for {validation_result.total_files} file(s) and {validation_result.total_items} extracted field(s)."
+        )
+        return
+
+    for file_result in validation_result.file_results:
+        click.echo(f"{file_result.file_name}:")
+        if file_result.passed:
+            click.echo(f"  Passed. Values checked: {file_result.item_count}")
+            continue
+
+        for error in file_result.errors:
+            click.echo(f"  - {error}")
+
+    raise click.ClickException(
+        f"Validation failed for {validation_result.failures} of {validation_result.total_files} file(s)."
+    )

spextract/engine.py

@@ -0,0 +1,152 @@
+"""Core extraction engine that applies a ParseSpec to HTML content."""
+
+from __future__ import annotations
+
+from typing import cast
+
+from bs4 import BeautifulSoup, Tag
+
+from spextract.models.validation import SpecValidationResult
+
+from .models import EngineOutput, FieldSpec, ParseSpec, ProcessorSpec, DataValue
+from .processors import apply_processor
+from .uni_selector import select
+
+
+class ParseEngine:
+    """Applies a ParseSpec to HTML content and returns extracted items."""
+
+    def __init__(self, spec: ParseSpec) -> None:
+        self.spec = spec
+
+    def parse(self, html: str, field_path: str | None = None) -> EngineOutput:
+        """Parse HTML string and return EngineOutput with typed fields.
+
+        Args:
+            html: Raw HTML content to parse.
+            field_path: Optional dot-separated path (e.g. ``"person.name"``) to
+                extract only a single field instead of the full spec.  Each
+                segment must match a key in the ``fields`` dict at that level of
+                nesting.  Raises ``KeyError`` if a segment is not found or if a
+                non-leaf segment has no child fields.
+        """
+        root = BeautifulSoup(html, "html.parser")
+        if field_path is not None:
+            data = self._extract_field_path(root, self.spec.fields, field_path.split("."))
+        else:
+            data = self._extract_fields(root, self.spec.fields)
+        return EngineOutput(spec=self.spec, data=data if data is not None else {})
+
+    def validate(self, html: str, field_path: str | None = None) -> SpecValidationResult:
+        """Validate HTML against spec, returning SpecValidationResult with details."""
+        from .validation.spec_validate import validate_spec_output  # pylint: disable=import-outside-toplevel
+
+        output = self.parse(html, field_path=field_path)
+        validation_result = validate_spec_output(self.spec, output.data, raise_=False)
+        return validation_result
+
+    def parse_and_validate(self, html: str, field_path: str | None = None) -> EngineOutput:
+        """Parse HTML and validate output against spec, returning EngineOutput with validation results.
+        Raises ValueError if validation fails."""
+
+        output = self.parse(html, field_path=field_path)
+        validation_result = self.validate(html, field_path=field_path)
+        if not validation_result.is_valid:
+            raise ValueError(f"Validation failed: {validation_result.mismatches[0]}")
+        return output
+
+    @staticmethod
+    def _extract_field_path(
+        node: Tag | BeautifulSoup,
+        fields: dict[str, FieldSpec],
+        path: list[str],
+    ) -> dict[str, DataValue]:
+        """Navigate ``fields`` and the HTML tree simultaneously following ``path``.
+
+        Returns a ``{leaf_name: value}`` dict containing only the targeted field.
+        Raises ``KeyError`` if any path segment is missing or if a non-leaf
+        segment has no child ``fields``.
+        """
+        name, *rest = path
+        if name not in fields:
+            raise KeyError(f"Field {name!r} not found. Available: {list(fields.keys())}")
+        field_spec = fields[name]
+
+        if not rest:
+            # Leaf — extract just this field
+            value = ParseEngine._extract_field(node, field_spec)
+            return {name: value} if value is not None else {}
+
+        # Intermediate — must have child fields and a navigable selector
+        if field_spec.fields is None:
+            raise KeyError(f"Field {name!r} has no child fields; cannot navigate to {'.'.join(rest)!r}")
+        sub_nodes = select(node, field_spec.selector, assert_tags=True)
+        if not sub_nodes:
+            return {}
+
+        if field_spec.multiple:
+            items = [ParseEngine._extract_field_path(sub, field_spec.fields, rest) for sub in sub_nodes]
+            return cast(dict[str, DataValue], {name: cast(DataValue, items)})
+
+        inner = ParseEngine._extract_field_path(sub_nodes[0], field_spec.fields, rest)
+        return cast(dict[str, DataValue], {name: cast(DataValue, inner)})
+
+    @staticmethod
+    def _extract_fields(node: Tag | BeautifulSoup, fields: dict[str, FieldSpec]) -> dict[str, DataValue]:
+        result: dict[str, DataValue] = {}
+        for name, field_spec in fields.items():
+            field_out = ParseEngine._extract_field(node, field_spec)
+            if field_out is not None:
+                result[name] = field_out
+        return result
+
+    @staticmethod
+    def _extract_field(node: Tag | BeautifulSoup, field_spec: FieldSpec) -> DataValue | None:
+        if field_spec.fields is not None:
+            # if this field has child fields, extraction is slightly different
+            # we ignore ::text / ::attr on the parent selector.
+            return ParseEngine._extract_nested(node, field_spec)
+
+        values = select(node, field_spec.selector)
+        if not values:
+            return [] if field_spec.multiple else None
+        if field_spec.multiple:
+            out_values = [ParseEngine.apply_processors(v, field_spec.resolved_processors()) for v in values]
+            return cast(DataValue, out_values)
+
+        all_strings = all(isinstance(v, str) for v in values)
+        if all_strings:
+            values = ["".join(cast(list[str], values))]
+        if len(values) > 1:
+            print(f"Warning: Multiple elements matched for single field: {field_spec.selector}. Using first match.")
+        out_value = ParseEngine.apply_processors(values[0], field_spec.resolved_processors())
+        return out_value
+
+    @staticmethod
+    def _extract_nested(node: Tag | BeautifulSoup, field_spec: FieldSpec) -> DataValue | None:
+        """Extract a field with child fields, applying child selectors relative to parent elements.
+        In this case we ignore ::text / ::attr on the parent selector since it doesn't make sense
+        to apply these to a parent element that we're extracting child fields from."""
+        assert field_spec.fields is not None
+
+        sub_nodes = select(node, field_spec.selector, assert_tags=True)
+        if not sub_nodes:
+            return [] if field_spec.multiple else None
+
+        if field_spec.multiple:
+            # Return a list of FieldOutput objects
+            return [ParseEngine._extract_fields(sub, field_spec.fields) for sub in sub_nodes]
+
+        if len(sub_nodes) > 1:
+            print(f"Warning: Multiple elements matched for single field: {field_spec.selector}. Using first match.")
+        return ParseEngine._extract_fields(sub_nodes[0], field_spec.fields)
+
+    @staticmethod
+    def apply_processors(value: object, processors: list[ProcessorSpec]) -> str | float | None:
+        if value is None:
+            return value
+        for proc in processors:
+            value = apply_processor(proc.name, value, proc.args if proc.args else None)
+        if isinstance(value, (str, float)) or value is None:
+            return value
+        raise ValueError(f"Unsupported value type after processing: {type(value)}")

spextract/models/__init__.py

@@ -0,0 +1,17 @@
+from .parser_spec import FieldSpec, FieldType, ParseSpec, ProcessorName, ProcessorSpec
+from .output import DataValue, EngineOutput
+from .validation import SpecValidationResult, ValidationMismatch, ExpectedResults, FileExpectedItems
+
+__all__ = [
+    "FieldSpec",
+    "FieldType",
+    "ParseSpec",
+    "ProcessorName",
+    "ProcessorSpec",
+    "DataValue",
+    "EngineOutput",
+    "SpecValidationResult",
+    "ValidationMismatch",
+    "ExpectedResults",
+    "FileExpectedItems",
+]

spextract/models/output.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import Union  # pylint: disable=unused-import
+from typing_extensions import TypeAliasType
+
+from .yaml import BaseModelWithYamlSupport
+
+from .parser_spec import ParseSpec
+
+DataValue = TypeAliasType(
+    "DataValue",
+    "Union[None, float, str, dict[str, DataValue], list[DataValue]]",
+)
+
+
+class EngineOutput(BaseModelWithYamlSupport):
+    """Output from the scraping engine for a single file,
+    including the parser spec used and the extracted data."""
+
+    spec: ParseSpec | None = None
+    data: dict[str, DataValue]

spextract/models/parser_spec.py

@@ -0,0 +1,85 @@
+"""
+Pydantic models for the declarative scraper.
+"""
+
+from __future__ import annotations
+
+import enum
+from typing import Union
+
+from pydantic import Field
+
+from .yaml import BaseModelWithYamlSupport
+from ..processors import ProcessorName
+
+
+class ProcessorSpec(BaseModelWithYamlSupport):
+    """A single field processor.
+
+    Can be a simple named processor (e.g. "strip") or a parameterised one, eg a regex.
+    """
+
+    name: ProcessorName
+    args: list[Union[str, int]] = Field(default_factory=list)
+
+
+class FieldType(enum.Enum):
+    """Type of field to extract, used to determine how to extract it from HTML."""
+
+    TEXT = "text"
+    LINK = "link"
+    NUMBER = "number"
+    DATE = "date"
+    OBJECT = "object"
+
+
+class FieldSpec(BaseModelWithYamlSupport):
+    """Specification for extracting a single field from HTML."""
+
+    selector: str = Field(
+        description="""
+            CSS selector or XPATH selector for the field.
+            If the fields attribute is not None, this selector should return a parent element or a list of parent elements.
+            The selector attribute of the child fields will be applied relative to each parent element.
+        """
+    )
+    type: FieldType = Field(
+        default=FieldType.TEXT,
+        description="Type of field to extract, used to validate extraction.",
+    )
+    required: bool = Field(default=True, description="Whether this field is required. Used for validation.")
+
+    multiple: bool = Field(
+        default=False,
+        description="Whether to extract multiple values from this field (i.e. return a list).",
+    )
+    processors: list[Union[ProcessorName, dict[ProcessorName, list[Union[str, int]]]]] = Field(
+        default_factory=list,
+        description="List of processors to apply to the extracted value(s). Each processor can be a string (processor name) or a dict mapping processor name to argument list. These are applied in order.",
+    )
+    fields: dict[str, "FieldSpec"] | None = Field(
+        default=None,
+        description="Child fields to extract from the element(s) selected by this field. \
+            Keys in this dict will be keys in the output data.",
+    )
+
+    def resolved_processors(self) -> list[ProcessorSpec]:
+        """Normalise the processor list into ProcessorSpec objects from dict[func-name=>arg list] or str."""
+        result: list[ProcessorSpec] = []
+        for p in self.processors:
+            if isinstance(p, (str, ProcessorName)):
+                result.append(ProcessorSpec(name=p))
+            elif isinstance(p, dict):
+                for name, args in p.items():
+                    result.append(ProcessorSpec(name=name, args=args))
+            else:
+                raise ValueError(f"Invalid processor spec: {p}")
+        return result
+
+
+class ParseSpec(BaseModelWithYamlSupport):
+    """Top-level declarative parser specification."""
+
+    version: int = 1
+    name: str
+    fields: dict[str, FieldSpec]

spextract/models/validation.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import BaseModel, Field, field_serializer
+
+from .output import DataValue, EngineOutput
+from .yaml import BaseModelWithYamlSupport
+
+
+class ValidationMismatch(BaseModel):
+    field: str
+    expected_type: str
+    actual_type: str
+    message: str
+
+
+class SpecValidationResult(BaseModel):
+    mismatches: list[ValidationMismatch] = Field(default_factory=list)
+
+    @property
+    def is_valid(self) -> bool:
+        return len(self.mismatches) == 0
+
+
+class FileValidationResult(BaseModel):
+    """Validation result for a single HTML file."""
+
+    file_name: str
+    item_count: int
+    errors: list[str] = Field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return not self.errors
+
+
+class TrueValidationResult(BaseModel):
+    """Aggregate validation result across all files."""
+
+    file_results: list[FileValidationResult] = Field(default_factory=list)
+
+    @property
+    def total_files(self) -> int:
+        return len(self.file_results)
+
+    @property
+    def total_items(self) -> int:
+        return sum(result.item_count for result in self.file_results)
+
+    @property
+    def failures(self) -> int:
+        return sum(1 for result in self.file_results if not result.passed)
+
+    @property
+    def passed(self) -> bool:
+        return self.failures == 0
+
+
+class FileExpectedItems(BaseModel):
+    """Expected extraction results for a single example file."""
+
+    file: str
+    items: dict[str, DataValue]
+
+    @classmethod
+    def from_engine_output(cls, file_name: str, output: EngineOutput) -> FileExpectedItems:
+        return cls(file=file_name, items=output.data)
+
+
+class ExpectedResults(BaseModelWithYamlSupport):
+    """Expected extraction output used to validate parser correctness.
+
+    Stored as YAML alongside example data files.
+    """
+
+    version: int = 1
+    data_path: Path | None = None
+    files: list[FileExpectedItems]
+
+    @field_serializer("data_path")
+    def serialize_data_path(self, value: Path | None) -> str | None:
+        """Serialize data_path as a string in YAML."""
+        return str(value) if value is not None else None

spextract/models/yaml.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+from typing import TypeVar
+
+import yaml
+from pydantic import BaseModel
+
+T = TypeVar("T", bound="BaseModelWithYamlSupport")
+
+
+class BaseModelWithYamlSupport(BaseModel):
+    """BaseModel subclass with support for loading from YAML files."""
+
+    @classmethod
+    def from_yaml_file(cls: type[T], file_path: Path) -> T:
+        """Load a model instance from a YAML file."""
+        with open(file_path, "r", encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+        return cls.model_validate(data)
+
+    def to_yaml_file(self, file_path: Path) -> None:
+        """Save a model instance to a YAML file."""
+        with open(file_path, "w", encoding="utf-8") as f:
+            yaml.safe_dump(self.model_dump(), f)

spextract/processors.py

@@ -0,0 +1,112 @@
+"""Built-in field processors for transforming extracted values."""
+
+from __future__ import annotations
+
+import re
+from enum import Enum
+from typing import Callable, Literal, Union, overload
+
+
+class ProcessorName(Enum):
+    STRIP = "strip"
+    TO_INT = "to_int"
+    TO_FLOAT = "to_float"
+    LOWERCASE = "lowercase"
+    UPPERCASE = "uppercase"
+    JOIN = "join"
+    REGEX = "regex"
+    SPLIT = "split"
+    INDEX = "index"
+    REPLACE = "replace"
+
+
+def regex_extract(value: str, pattern: str) -> str:
+    """Extract the first regex match from value. Returns empty string if no match.
+
+    If the pattern contains capturing groups, returns the first captured group.
+    Otherwise, returns the full match.
+    """
+    match = re.search(pattern, value)
+    if not match:
+        return ""
+    return match.group(1) if match.lastindex else match.group(0)
+
+
+def split_string(value: str, separator: str = " ") -> list[str]:
+    """Split a string by the given separator."""
+    return value.split(separator)
+
+
+def select_index(value: list[object], idx: str) -> object:
+    """Select an element from a list by index."""
+    int_idx = int(idx)
+    return value[int_idx] if -len(value) <= int_idx < len(value) else None
+
+
+def join_strings(value: list[str], separator: str = " ") -> str:
+    """Join a list of strings with the given separator."""
+    if not isinstance(value, list):
+        raise ValueError(
+            f"Expected a list of strings for join processor, got {type(value).__name__}, with value: {value!r}"
+        )
+    if not all(isinstance(v, str) for v in value):
+        raise ValueError(f"Expected all elements to be strings for join processor, got: {value!r}")
+    return separator.join(value)
+
+
+PROCESSOR_REGISTRY: dict[ProcessorName, Callable[..., object]] = {
+    ProcessorName.STRIP: lambda value: value.strip() if isinstance(value, str) else "",
+    ProcessorName.TO_INT: int,
+    ProcessorName.TO_FLOAT: float,
+    ProcessorName.LOWERCASE: lambda value: value.lower(),
+    ProcessorName.UPPERCASE: lambda value: value.upper(),
+    ProcessorName.JOIN: join_strings,
+    ProcessorName.REGEX: regex_extract,
+    ProcessorName.SPLIT: split_string,
+    ProcessorName.INDEX: select_index,
+    ProcessorName.REPLACE: lambda value, old, new: value.replace(old, new) if isinstance(value, str) else value,
+}
+
+
+@overload
+def apply_processor(name: Literal[ProcessorName.STRIP], value: str) -> str: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.TO_INT], value: str) -> int: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.TO_FLOAT], value: str) -> float: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.LOWERCASE], value: str) -> str: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.UPPERCASE], value: str) -> str: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.JOIN], value: list[str], args: list[str] | None = None) -> str: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.REGEX], value: str, args: list[str]) -> str: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.SPLIT], value: str, args: list[str] | None = None) -> list[str]: ...
+@overload
+def apply_processor(name: Literal[ProcessorName.REPLACE], value: str, args: list[str]) -> str: ...
+@overload
+def apply_processor(
+    name: Literal[ProcessorName.INDEX],
+    value: list[object],
+    args: list[Union[str, int]],
+) -> object: ...
+@overload
+def apply_processor(name: ProcessorName, value: object, args: list[Union[str, int]] | None = None) -> object: ...
+
+
+def apply_processor(
+    name: ProcessorName, value: object, args: list[Union[str, int]] | list[str] | None = None
+) -> object:
+    """Apply a named processor to a value.
+
+    Raises KeyError if the processor name is not registered.
+    """
+    processor_name = ProcessorName(name)
+    if processor_name not in PROCESSOR_REGISTRY:
+        raise KeyError(f"Unknown processor: {name!r}. Available: {list(PROCESSOR_REGISTRY.keys())}")
+    func = PROCESSOR_REGISTRY[processor_name]
+    if args:
+        return func(value, *args)
+    return func(value)

spextract/uni_selector.py

@@ -0,0 +1,125 @@
+import re
+from typing import Literal, cast, overload
+
+import lxml.etree
+import lxml.html
+from bs4 import BeautifulSoup, NavigableString, Tag
+from soupsieve import SelectorSyntaxError
+
+_PSEUDO_RE = re.compile(r"::(text|attr\(([^)]+)\))\s*$")
+
+
+def _xpath_results_to_tags(results: list[object] | str) -> list[Tag] | list[str]:
+    """Convert lxml XPath results to BeautifulSoup Tags."""
+    if isinstance(results, str):
+        return [results]
+    tags: list[Tag] | list[str] = []
+    all_elements = all(isinstance(r, lxml.etree._Element) for r in results)  # pylint: disable=protected-access
+    all_strings = all(isinstance(r, str) for r in results)
+    if all_strings:
+        return cast(list[str], results)
+    if all_elements:
+        for r in results:
+            html = lxml.html.tostring(r, encoding="unicode")  # type: ignore
+            soup = BeautifulSoup(html, "html.parser")
+            if soup.contents:
+                tags.append(soup.contents[0])  # type: ignore
+        return tags
+
+    raise ValueError(f"Expected all XPath results to be either strings or elements, but got: {results}")
+
+
+def _parse_selector(css: str) -> tuple[str, str | None]:
+    """Split selector into (base, mode) stripping ::text / ::attr(...)."""
+    m = _PSEUDO_RE.search(css)
+    if not m:
+        return css, None
+    base = css[: m.start()]
+    if m.group(1) == "text":
+        return base, "text"
+    return base, f"attr:{m.group(2)}"
+
+
+def _select_css(node: Tag | BeautifulSoup, css: str) -> list[str] | list[Tag]:
+    """Run a CSS selector and return matched strings."""
+
+    base, mode = _parse_selector(css)
+    tags = node.select(base) if base.strip() else []
+
+    if mode == "text":
+        results: list[str] = []
+        for tag in tags:
+            for child in tag.children:
+                if isinstance(child, NavigableString) and not isinstance(child, Tag):
+                    results.append(str(child))
+        return results
+
+    if mode is not None and mode.startswith("attr:"):
+        attr_name = mode[5:]
+        results = []
+        for tag in tags:
+            val = tag.get(attr_name)
+            if val is not None:
+                results.append(" ".join(val) if isinstance(val, list) else str(val))
+        return results
+
+    return tags
+
+
+@overload
+def select(
+    node: Tag | BeautifulSoup,
+    selector: str,
+) -> list[Tag] | list[str]: ...
+@overload
+def select(
+    node: Tag | BeautifulSoup,
+    selector: str,
+    assert_tags: Literal[True],
+    assert_strings: Literal[False] = False,
+) -> list[Tag]: ...
+@overload
+def select(
+    node: Tag | BeautifulSoup,
+    selector: str,
+    assert_tags: Literal[False] = False,
+    assert_strings: Literal[True] = True,
+) -> list[str]: ...
+@overload
+def select(
+    node: Tag | BeautifulSoup,
+    selector: str,
+    *,
+    as_strings: Literal[True] = True,
+) -> list[str]: ...
+
+
+def select(
+    node: Tag | BeautifulSoup,
+    selector: str,
+    assert_tags: bool = False,
+    assert_strings: bool = False,
+    as_strings: bool = False,
+) -> list[Tag] | list[str]:
+    """Select elements using CSS or XPath selector."""
+    results: list[Tag] | list[str] = []
+    try:
+        # CSS selector
+        tags = _select_css(node, selector)
+        results = tags
+    except (SelectorSyntaxError, NotImplementedError):
+        # XPath selector
+        root = lxml.html.fromstring(str(node))
+        xpath_results = cast(list[object] | str, root.xpath(selector))
+        str_or_tag = _xpath_results_to_tags(xpath_results)
+        results = str_or_tag
+
+    all_tags = all(isinstance(r, Tag) for r in results)
+    all_strings = all(isinstance(r, str) for r in results)
+    if assert_tags and not all_tags:
+        raise ValueError(f"Expected all results to be Tags, but got: {results}")
+    if assert_strings and not all_strings:
+        raise ValueError(f"Expected all results to be strings, but got: {results}")
+    if as_strings:
+        return [str(r) for r in results]
+    return results

spextract/validation/__init__.py

@@ -0,0 +1,15 @@
+from .true_validate import (
+    validate_spec_against_expected,
+    validate_files,
+    validate_items_against_expected,
+    TrueValidationResult,
+    FileValidationResult,
+)
+
+__all__ = [
+    "validate_spec_against_expected",
+    "validate_files",
+    "validate_items_against_expected",
+    "TrueValidationResult",
+    "FileValidationResult",
+]

spextract/validation/spec_validate.py

@@ -0,0 +1,138 @@
+"""
+Validation logic for checking if engine output matches the declared spec."""
+
+from typing import Any, List
+
+from ..models.output import DataValue
+from ..models.parser_spec import FieldSpec, ParseSpec
+from ..models.validation import SpecValidationResult, ValidationMismatch
+from .validators import validate_value, type_name
+
+
+def _expectation_for_field(field_spec: FieldSpec) -> str:
+    if field_spec.fields is not None:
+        if field_spec.multiple:
+            return "array<object>"
+        return "object"
+    base = field_spec.type.value
+    if field_spec.multiple:
+        return f"array<{base}>"
+    return base
+
+
+def _validate_field_recursive(name_path: str, field_spec: FieldSpec, value: Any) -> List[ValidationMismatch]:
+    if field_spec.required and value is None:
+        return [
+            ValidationMismatch(
+                field=name_path,
+                expected_type=_expectation_for_field(field_spec),
+                actual_type="missing",
+                message=f"Field '{name_path}' missing in output.",
+            )
+        ]
+    if value is None:
+        # not required and missing is fine, no mismatches
+        return []
+    # Delegate to nested or non-nested validators
+    if field_spec.fields is not None:
+        return _validate_nested(name_path, field_spec, value)
+    return _validate_non_nested(name_path, field_spec, value)
+
+
+def _validate_nested(name_path: str, field_spec: FieldSpec, value: object) -> List[ValidationMismatch]:
+    mismatches: List[ValidationMismatch] = []
+    expected = _expectation_for_field(field_spec)
+    actual = type_name(value)
+
+    if field_spec.multiple:
+        if not isinstance(value, list):
+            mismatches.append(
+                ValidationMismatch(
+                    field=name_path,
+                    expected_type=expected,
+                    actual_type=actual,
+                    message=f"Field '{name_path}' expected list of objects, got {actual}",
+                )
+            )
+            return mismatches
+        for idx, item in enumerate(value):
+            if not isinstance(item, dict):
+                mismatches.append(
+                    ValidationMismatch(
+                        field=f"{name_path}[{idx}]",
+                        expected_type="object",
+                        actual_type=type_name(item),
+                        message=f"Expected object at '{name_path}[{idx}]', got {type_name(item)}",
+                    )
+                )
+                continue
+            assert field_spec.fields is not None  # for mypy - we know this is not None since we're in the nested case
+            for child_name, child_spec in field_spec.fields.items():
+                child_value = item.get(child_name)
+                mismatches.extend(_validate_field_recursive(f"{name_path}.{child_name}", child_spec, child_value))
+        return mismatches
+
+    # single nested object expected
+    if not isinstance(value, dict):
+        mismatches.append(
+            ValidationMismatch(
+                field=name_path,
+                expected_type=expected,
+                actual_type=actual,
+                message=f"Field '{name_path}' expected object, got {actual}",
+            )
+        )
+        return mismatches
+
+    assert field_spec.fields is not None  # for mypy - we know this is not None since we're in the nested case
+    for child_name, child_spec in field_spec.fields.items():
+        child_value = value.get(child_name)
+        mismatches.extend(_validate_field_recursive(f"{name_path}.{child_name}", child_spec, child_value))
+    return mismatches
+
+
+def _validate_non_nested(name_path: str, field_spec: FieldSpec, value: object) -> List[ValidationMismatch]:
+    mismatches: List[ValidationMismatch] = []
+    expected = _expectation_for_field(field_spec)
+    actual = type_name(value)
+    base_type = field_spec.type
+
+    if field_spec.multiple:
+        if not isinstance(value, list):
+            mismatches.append(
+                ValidationMismatch(
+                    field=name_path,
+                    expected_type=expected,
+                    actual_type=actual,
+                    message=f"Field '{name_path}' expected list of {base_type.value}, got {actual}",
+                )
+            )
+            return mismatches
+        for idx, item in enumerate(value):
+            val_result = validate_value(f"{name_path}[{idx}]", base_type, item)
+            if val_result is not None:
+                mismatches.append(val_result)
+        return mismatches
+
+    # single value expected
+    val_result = validate_value(name_path, base_type, value)
+    if val_result is not None:
+        mismatches.append(val_result)
+    return mismatches
+
+
+def validate_spec_output(spec: ParseSpec, data: dict[str, DataValue], raise_: bool = False) -> SpecValidationResult:
+    """Validate that the engine output `data` matches the `spec`.
+
+    Returns a `SpecValidationResult` containing any mismatches found.
+    """
+    mismatches: List[ValidationMismatch] = []
+
+    for field_name, field_spec in spec.fields.items():
+        value = data[field_name] if field_name in data else None
+        mismatches.extend(_validate_field_recursive(field_name, field_spec, value))
+
+    result = SpecValidationResult(mismatches=mismatches)
+    if raise_ and not result.is_valid:
+        raise ValueError(f"Validation failed for output: {result}")
+    return result

spextract/validation/true_validate.py

@@ -0,0 +1,204 @@
+import re
+from pathlib import Path
+from typing import cast
+
+from ..engine import ParseEngine
+from ..models.output import DataValue
+from ..models.parser_spec import ParseSpec
+from ..models.validation import ExpectedResults, FileValidationResult, TrueValidationResult
+
+
+def _path_matches_target(path: str, target: str | None) -> bool:
+    """Return True if *path* matches, is a descendant of, or is an ancestor of *target*.
+
+    `items.name` matches `items[0].name`, `items[1].name`, etc.
+    `items[0].name` matches only `items[0].name`.
+    `items.meta` matches `items[0].meta.color` (target is ancestor of path).
+    `items` matches `items.meta` (path is ancestor of target — structural errors).
+    """
+    if not target:
+        return True
+    target_parts = target.split(".")
+    path_parts = path.split(".")
+    # Compare the overlapping prefix
+    for target_part, path_part in zip(target_parts, path_parts):
+        if "[" in target_part:
+            if path_part != target_part:
+                return False
+        else:
+            bracket = path_part.find("[")
+            path_base = path_part[:bracket] if bracket != -1 else path_part
+            if path_base != target_part:
+                return False
+    return True
+
+
+def _compare_values(
+    actual: object,
+    expected: DataValue,
+    path: str,
+    errors: list[str],
+    target_field_path: str | None = None,
+) -> int:
+    """Recursively compare an actual parsed value against an expected value."""
+    num_items_checked = 0
+    if expected is None:
+        if not _path_matches_target(path, target_field_path):
+            return 0
+        if not actual:
+            # Treat None, empty string, empty list, and empty dict as equivalent for convenience
+            return 1
+        if actual:
+            errors.append(f"{path}: expected None/empty, got {actual!r}")
+            return 1
+    elif isinstance(expected, str):
+        if not _path_matches_target(path, target_field_path):
+            return 0
+        if expected == "" and (actual is None or actual == ""):
+            return 1  # Treat empty string and None as equivalent for convenience
+        if actual != expected:
+            errors.append(f"{path}: expected {expected!r}, got {actual!r}")
+        return 1
+    elif isinstance(expected, dict):
+        if not isinstance(actual, dict):
+            if _path_matches_target(path, target_field_path):
+                errors.append(f"{path}: expected dict for target field, got {type(actual).__name__}: {actual!r}")
+                return 1
+            return 0
+        for key, exp_val in expected.items():
+            actual_val = actual.get(key)
+            num_items_checked += _compare_values(actual_val, exp_val, f"{path}.{key}", errors, target_field_path)
+    elif isinstance(expected, list):
+        exp_list = cast(list[DataValue], expected)
+        if not isinstance(actual, list):
+            if _path_matches_target(path, target_field_path):
+                errors.append(f"{path}: expected list, got {type(actual).__name__}: {actual!r}")
+                return 1
+            return 0
+        if len(actual) != len(expected):
+            if _path_matches_target(path, target_field_path):
+                errors.append(f"{path}: expected {len(expected)} items, got {len(actual)}")
+            return 1
+        for i, (act_item, exp_item) in enumerate(zip(actual, exp_list)):
+            num_items_checked += _compare_values(act_item, exp_item, f"{path}[{i}]", errors, target_field_path)
+    elif actual != expected and (not target_field_path or _path_matches_target(path, target_field_path)):
+        errors.append(f"{path}: expected {expected!r}, got {actual!r}")
+        return 1
+    return num_items_checked
+
+
+def validate_spec_against_expected(
+    spec: ParseSpec,
+    html: str,
+    expected: dict[str, DataValue],
+    field_path: str | None = None,
+) -> FileValidationResult:
+    # if field_path is provided, check that it exists in the spec before parsing
+    # if field_path starts with "fields.", remove that prefix for easier matching
+    if field_path and field_path.startswith("fields."):
+        field_path = field_path[len("fields.") :]
+    if field_path is not None:
+        # Support dot notation for nested fields; strip any [N] index suffix for spec lookup
+        field_parts = field_path.split(".")
+        current = spec.fields
+        for part in field_parts:
+            part_name = re.sub(r"\[\d+\]$", "", part)
+            if isinstance(current, dict) and part_name in current:
+                nested_fields = current[part_name].fields
+                current = nested_fields if nested_fields is not None else {}
+            else:
+                raise ValueError(f"Field path '{field_path}' does not exist in the spec.")
+
+    engine = ParseEngine(spec)
+    items = engine.parse(html).data
+    return validate_items_against_expected(items, expected, field_path)
+
+
+def validate_items_against_expected(
+    items: dict[str, DataValue],
+    expected: dict[str, DataValue],
+    field_path: str | None = None,
+) -> FileValidationResult:
+    """Validate a parser spec against an HTML string.
+
+    Parses the HTML using the spec and optionally compares against expected results.
+    """
+
+    errors: list[str] = []
+
+    actual = items if items else {}
+    num_items_checked = 0
+    for key, exp_val in expected.items():
+        actual_val = actual.get(key)
+        num_items_checked += _compare_values(actual_val, exp_val, key, errors, field_path)
+
+    return FileValidationResult(file_name="", item_count=num_items_checked, errors=errors)
+
+
+def validate_files(
+    expected_values_path: Path,
+    spec_file_path: Path,
+    data_dir: Path | None = None,
+    field_path: str | None = None,
+    skip_unexpected_files: bool = False,
+) -> TrueValidationResult:
+    """Validate an item directory containing parser_spec.yaml and expected.yaml.
+
+    Args:
+        expected_values_path: Path to the expected values YAML file.
+        spec_file_path: Path to the parser spec YAML file.
+        data_dir: Override for the data directory. Defaults to item_dir/../data.
+        field_path: Optional dot path to a specific field to validate.
+    """
+    expected_values = ExpectedResults.from_yaml_file(expected_values_path)
+    spec = ParseSpec.from_yaml_file(spec_file_path)
+
+    if data_dir is None:
+        data_dir = expected_values.data_path
+
+    if data_dir is None:
+        raise ValueError(
+            "Data path not specified. Either include 'data_path' in the expected values YAML or provide --data-dir."
+        )
+
+    if not data_dir.is_absolute():
+        data_dir = expected_values_path.parent / data_dir
+
+    expected_by_file: dict[str, dict[str, DataValue]] = {fe.file: fe.items for fe in expected_values.files}
+
+    html_files = sorted(data_dir.glob("*.html"))
+    result = TrueValidationResult()
+
+    if not html_files:
+        result.file_results.append(
+            FileValidationResult(
+                file_name=str(data_dir),
+                item_count=0,
+                errors=[f"No HTML files found in {data_dir}"],
+            )
+        )
+        return result
+
+    for html_file in html_files:
+        html = html_file.read_text(encoding="utf-8")
+        file_expected = expected_by_file.get(html_file.name)
+        if not file_expected:
+            if not skip_unexpected_files:
+                result.file_results.append(
+                    FileValidationResult(
+                        file_name=html_file.name,
+                        item_count=0,
+                        errors=[f"No expected results defined for {html_file.name}"],
+                    )
+                )
+            continue
+        file_result = validate_spec_against_expected(spec, html, file_expected, field_path=field_path)
+        result.file_results.append(
+            FileValidationResult(
+                file_name=html_file.name,
+                item_count=file_result.item_count,
+                errors=file_result.errors,
+            )
+        )
+
+    return result

spextract/validation/validators.py

@@ -0,0 +1,110 @@
+"""Validators for primitive (non-nested) field values.
+Factored out from validation logic so tests and code can reuse semantic checks.
+"""
+
+
+from typing import Any, Optional
+from datetime import datetime
+
+from ..models.parser_spec import FieldType
+from ..models.validation import ValidationMismatch
+
+
+def type_name(value: Any) -> str:
+    if value is None:
+        return "missing"
+    if isinstance(value, dict):
+        return "object"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, (int, float)):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    return type(value).__name__
+
+
+def validate_text(name_path: str, value: Any) -> Optional[ValidationMismatch]:
+    if not isinstance(value, str):
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="text",
+            actual_type=type_name(value),
+            message=f"Field '{name_path}' expected text, got {type_name(value)}",
+        )
+    return None
+
+
+def validate_link(name_path: str, value: Any) -> Optional[ValidationMismatch]:
+    if not isinstance(value, str):
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="link",
+            actual_type=type_name(value),
+            message=f"Field '{name_path}' expected link string, got {type_name(value)}",
+        )
+    if not (value.startswith("http://") or value.startswith("https://") or value.startswith("/")):
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="link",
+            actual_type="string",
+            message=f"Field '{name_path}' does not look like a link: '{value}'",
+        )
+    return None
+
+
+def validate_number(name_path: str, value: Any) -> Optional[ValidationMismatch]:
+    if not isinstance(value, (int, float)):
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="number",
+            actual_type=type_name(value),
+            message=f"Field '{name_path}' expected number, got {type_name(value)}",
+        )
+    return None
+
+
+def validate_date(name_path: str, value: Any) -> Optional[ValidationMismatch]:
+    if not isinstance(value, str):
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="date",
+            actual_type=type_name(value),
+            message=f"Field '{name_path}' expected date string, got {type_name(value)}",
+        )
+    try:
+        datetime.fromisoformat(value)
+    except Exception:  # pylint: disable=broad-except
+        return ValidationMismatch(
+            field=name_path,
+            expected_type="date",
+            actual_type="string",
+            message=f"Field '{name_path}' is not a valid ISO date: '{value}'",
+        )
+    return None
+
+
+def validate_value(name_path: str, base_type: FieldType, value: Any) -> Optional[ValidationMismatch]:
+    """Dispatch to the right validator for `base_type`.
+
+    Returns a `ValidationMismatch` if invalid, or None if valid.
+    """
+    if value is None:
+        return ValidationMismatch(
+            field=name_path,
+            expected_type=base_type.value,
+            actual_type="missing",
+            message=f"Field '{name_path}' is missing",
+        )
+
+    if base_type == FieldType.TEXT:
+        return validate_text(name_path, value)
+    if base_type == FieldType.LINK:
+        return validate_link(name_path, value)
+    if base_type == FieldType.NUMBER:
+        return validate_number(name_path, value)
+    if base_type == FieldType.DATE:
+        return validate_date(name_path, value)
+
+    # Fallback: treat as text
+    return validate_text(name_path, value)

spextract-0.5.16.dist-info/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright © 2026 Vincent Lonij
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

spextract-0.5.16.dist-info/METADATA

@@ -0,0 +1,83 @@
+Metadata-Version: 2.3
+Name: spextract
+Version: 0.5.16
+Summary: A declarative html scraper for python
+License: MIT
+Author: Vincent Lonij
+Author-email: 29819815+vincentropy@users.noreply.github.com
+Requires-Python: >=3.10
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: beautifulsoup4 (>=4.14.3,<5.0.0)
+Requires-Dist: click (>=8.3.2,<9.0.0)
+Requires-Dist: lxml (>=6.0.4,<7.0.0)
+Requires-Dist: pydantic (>=2.12.5,<3.0.0)
+Requires-Dist: pyyaml (>=6.0.3,<7.0.0)
+Description-Content-Type: text/markdown
+
+# A Declarative HTML Scraper for Python
+
+This package provides a simple way to declare what data should be extracted from an HTML document in a configuration file.
+
+This enables sharing of scraping logic across projects and teams without the risk of executing untrusted code. It also allows for easier maintenance and updates to scraping logic without needing to modify the underlying codebase.
+
+## CLI
+
+The package includes a Click-based CLI with two commands:
+
+```bash
+decs parse spec.yaml <path to html file or directory>
+decs validate spec.yaml expected-results.yaml
+```
+
+`parse` emits YAML in the same expected-results format used by `validate`, so you can capture known-good output and re-run validation later.
+
+## How to use
+
+### Build a configuration file
+
+You can write a configuration file with the provided ParserSpec class.
+
+```python
+import py_decs
+
+spec = py_decs.ParserSpec(
+    name="example_parser",
+    description="An example parser for demonstration purposes.",
+    fields=[
+        py_decs.FieldSpec(
+            name="title",
+            selector="h1.title::text",
+            type=py_decs.FieldType.TEXT,
+        ),
+        py_decs.FieldSpec(
+            name="links",
+            selector="a.link::attr(href)",
+            type=py_decs.FieldType.LINK,
+            multiple=True,
+        )
+        py_decs.FieldSpec(
+            name="author",
+            selector="div.author",
+            type=py_decs.FieldType.OBJECT,
+            fields=[
+                py_decs.FieldSpec(
+                    name="name",
+                    selector="span.name::text",
+                    type=py_decs.FieldType.TEXT,
+                ),
+                py_decs.FieldSpec(
+                    name="profile_url",
+                    selector="a.profile::attr(href)",
+                    type=py_decs.FieldType.LINK,
+                ),
+            ]
+        ),
+    ]
+)
+```
+

spextract-0.5.16.dist-info/RECORD

@@ -0,0 +1,20 @@
+spextract/__init__.py,sha256=4yowBnR5WkwlRe0_Ad27TkKTPmVa4mGouVudMctyON8,547
+spextract/cli.py,sha256=bOSDgqgnT3N1bQwKNLg8LtcsSSzWGwHNfqyH6_gjhXI,3088
+spextract/engine.py,sha256=yIH26wYiQPsVzhc39AZxJX83MBl4FetA6olXLV2L3f0,6979
+spextract/models/__init__.py,sha256=QeHguokPrbZ75TVEBfCueq1ASo2_0rN8JTsFKyjDHqI,479
+spextract/models/output.py,sha256=T-zygB2x5eUhXWvkR_eW5hfEDfWAA43kDPl-mbMNDdk,569
+spextract/models/parser_spec.py,sha256=pZbFnEnBtQdw34OrY0TvPreRIJoCI3Q5ZGvyX71rVog,2936
+spextract/models/validation.py,sha256=1_K5y94CKRqSMwZBysTDbV4njfI3x57j_9g23Si3S8k,2163
+spextract/models/yaml.py,sha256=xjSi3dDehF00X97nZDx1TIHJMTvGg9iVyMD8c-hi320,744
+spextract/processors.py,sha256=y41nSSWy8ch3gzlDifcGlGXLp-jHzLihlT-_rLGdyIU,4067
+spextract/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+spextract/uni_selector.py,sha256=R4q2gVxoJAlyJ7FbgqPAiMQSGOSsqbCjfPjAlUqux1w,3956
+spextract/validation/__init__.py,sha256=k13bHZzZAQ0zjXrp-gz98zOyBcfREPAH2_O5thTbIu8,346
+spextract/validation/spec_validate.py,sha256=GWyDjqx0ElDlqJkNrqQm6ys8yXaS6E1SFwTuZPEZRdk,5388
+spextract/validation/true_validate.py,sha256=H8c_lBZ9ClxCnqnaOZjM6nYoClFa6H9VB8h4fmVFDkE,7991
+spextract/validation/validators.py,sha256=FXeejkc-IQnsQxrsldz2f6nWHp6DFjFVBPbd0jF5HHE,3647
+spextract-0.5.16.dist-info/LICENSE.md,sha256=fboAcycaR0hzaSgMDYqsxxyyF1Yq3Vt7R28krR1lgXc,1064
+spextract-0.5.16.dist-info/METADATA,sha256=wuSrhbRRpMYGp0NaskcreQYqfaf8fw_uNJra6cBJpn0,2653
+spextract-0.5.16.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+spextract-0.5.16.dist-info/entry_points.txt,sha256=uzZaXL_t9zdWp3Y-P8wkZhSJmjfE7jrw8mJJmQN3-z4,40
+spextract-0.5.16.dist-info/RECORD,,

spextract-0.5.16.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any

spextract-0.5.16.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+decs=py_decs.cli:cli
+