bootstack 0.1.0__py3-none-any.whl

bootstack/data/query.py

@@ -0,0 +1,367 @@
+"""Expressive, SQL-free filtering and sorting for data sources.
+
+Build filter conditions with `col` and combine them with the usual operators::
+
+    from bootstack.data import col
+
+    ds.where(col("age") >= 25)
+    ds.where((col("status") == "active") & col("name").contains("ada"))
+    ds.where(col("department").is_in(["Sales", "Engineering"]))
+    ds.order("-salary", "name")          # '-' prefix sorts descending
+
+A condition is backend-agnostic: in-memory sources evaluate it with
+`matches()`, while SQLite renders it to a parameterized query with `to_sql()`
+(values are always bound, never interpolated, so user input can't inject SQL).
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, List, Mapping, Sequence, Tuple, Union
+
+__all__ = [
+    "col",
+    "Column",
+    "Condition",
+    "SortKey",
+    "any_of",
+    "all_of",
+]
+
+
+# ---------------------------------------------------------------------------
+# SQL rendering helpers
+# ---------------------------------------------------------------------------
+
+_SQL_OPS = {"eq": "=", "ne": "!=", "lt": "<", "le": "<=", "gt": ">", "ge": ">="}
+
+
+def _quote(identifier: str) -> str:
+    """Quote a column identifier for safe use in SQL."""
+    return '"' + str(identifier).replace('"', '""') + '"'
+
+
+def _escape_like(value: str) -> str:
+    """Escape LIKE wildcards so they match literally (paired with ESCAPE '\\')."""
+    return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+
+
+SqlFragment = Tuple[str, List[Any]]
+
+
+# ---------------------------------------------------------------------------
+# Conditions
+# ---------------------------------------------------------------------------
+
+class Condition:
+    """A filter expression. Combine with `&` (and), `|` (or), and `~` (not)."""
+
+    def __and__(self, other: "Condition") -> "Condition":
+        return And([self, other])
+
+    def __or__(self, other: "Condition") -> "Condition":
+        return Or([self, other])
+
+    def __invert__(self) -> "Condition":
+        return Not(self)
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        """Return whether `row` satisfies this condition (in-memory evaluation)."""
+        raise NotImplementedError
+
+    def to_sql(self) -> SqlFragment:
+        """Render to a parameterized SQL fragment: `(clause, params)`."""
+        raise NotImplementedError
+
+
+@dataclass
+class Compare(Condition):
+    """A value comparison: `column <op> value`."""
+
+    column: str
+    op: str  # eq, ne, lt, le, gt, ge
+    value: Any
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        actual = row.get(self.column)
+        if self.op == "eq":
+            return actual == self.value
+        if self.op == "ne":
+            return actual != self.value
+        if actual is None or self.value is None:
+            return False
+        try:
+            if self.op == "lt":
+                return actual < self.value
+            if self.op == "le":
+                return actual <= self.value
+            if self.op == "gt":
+                return actual > self.value
+            if self.op == "ge":
+                return actual >= self.value
+        except TypeError:
+            return False
+        return False
+
+    def to_sql(self) -> SqlFragment:
+        q = _quote(self.column)
+        if self.value is None and self.op in ("eq", "ne"):
+            return (f"{q} IS NULL" if self.op == "eq" else f"{q} IS NOT NULL", [])
+        return (f"{q} {_SQL_OPS[self.op]} ?", [self.value])
+
+
+@dataclass
+class Match(Condition):
+    """A case-insensitive text match: contains / startswith / endswith."""
+
+    column: str
+    kind: str  # contains, startswith, endswith
+    value: str
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        actual = row.get(self.column)
+        if actual is None:
+            return False
+        haystack = str(actual).lower()
+        needle = str(self.value).lower()
+        if self.kind == "contains":
+            return needle in haystack
+        if self.kind == "startswith":
+            return haystack.startswith(needle)
+        if self.kind == "endswith":
+            return haystack.endswith(needle)
+        return False
+
+    def to_sql(self) -> SqlFragment:
+        esc = _escape_like(str(self.value))
+        pattern = {
+            "contains": f"%{esc}%",
+            "startswith": f"{esc}%",
+            "endswith": f"%{esc}",
+        }[self.kind]
+        return (f"{_quote(self.column)} LIKE ? ESCAPE '\\'", [pattern])
+
+
+@dataclass
+class In(Condition):
+    """Membership test: `column IN (values)`."""
+
+    column: str
+    values: List[Any]
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        return row.get(self.column) in self.values
+
+    def to_sql(self) -> SqlFragment:
+        if not self.values:
+            return ("0", [])  # IN () matches nothing
+        placeholders = ", ".join("?" for _ in self.values)
+        return (f"{_quote(self.column)} IN ({placeholders})", list(self.values))
+
+
+@dataclass
+class IsNull(Condition):
+    """Null test: `column IS NULL`."""
+
+    column: str
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        return row.get(self.column) is None
+
+    def to_sql(self) -> SqlFragment:
+        return (f"{_quote(self.column)} IS NULL", [])
+
+
+@dataclass
+class And(Condition):
+    """All sub-conditions must hold."""
+
+    parts: List[Condition] = field(default_factory=list)
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        return all(p.matches(row) for p in self.parts)
+
+    def to_sql(self) -> SqlFragment:
+        return _combine(self.parts, " AND ", empty="1")
+
+
+@dataclass
+class Or(Condition):
+    """At least one sub-condition must hold."""
+
+    parts: List[Condition] = field(default_factory=list)
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        return any(p.matches(row) for p in self.parts)
+
+    def to_sql(self) -> SqlFragment:
+        return _combine(self.parts, " OR ", empty="0")
+
+
+@dataclass
+class Not(Condition):
+    """Negates a condition."""
+
+    part: Condition
+
+    def matches(self, row: Mapping[str, Any]) -> bool:
+        return not self.part.matches(row)
+
+    def to_sql(self) -> SqlFragment:
+        clause, params = self.part.to_sql()
+        return (f"NOT ({clause})", params)
+
+
+def _combine(parts: Sequence[Condition], joiner: str, *, empty: str) -> SqlFragment:
+    if not parts:
+        return (empty, [])
+    clauses: List[str] = []
+    params: List[Any] = []
+    for part in parts:
+        clause, p = part.to_sql()
+        clauses.append(clause)
+        params.extend(p)
+    return ("(" + joiner.join(clauses) + ")", params)
+
+
+# ---------------------------------------------------------------------------
+# Column reference (builds conditions / sort keys)
+# ---------------------------------------------------------------------------
+
+class Column:
+    """A reference to a column. Build conditions and sort keys from it.
+
+    Created by `col(name)`. Supports the comparison operators (`==`, `!=`,
+    `<`, `<=`, `>`, `>=`), text matching (`contains`, `startswith`, `endswith`),
+    membership (`is_in`), null tests (`is_null`, `is_not_null`), and sort
+    direction (`asc`, `desc`).
+    """
+
+    __slots__ = ("name",)
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    # comparisons
+    def __eq__(self, other: Any) -> Condition:  # type: ignore[override]
+        return Compare(self.name, "eq", other)
+
+    def __ne__(self, other: Any) -> Condition:  # type: ignore[override]
+        return Compare(self.name, "ne", other)
+
+    def __lt__(self, other: Any) -> Condition:
+        return Compare(self.name, "lt", other)
+
+    def __le__(self, other: Any) -> Condition:
+        return Compare(self.name, "le", other)
+
+    def __gt__(self, other: Any) -> Condition:
+        return Compare(self.name, "gt", other)
+
+    def __ge__(self, other: Any) -> Condition:
+        return Compare(self.name, "ge", other)
+
+    __hash__ = None  # type: ignore[assignment]
+
+    # text
+    def contains(self, value: str) -> Condition:
+        """Case-insensitive substring match."""
+        return Match(self.name, "contains", value)
+
+    def startswith(self, value: str) -> Condition:
+        """Case-insensitive prefix match."""
+        return Match(self.name, "startswith", value)
+
+    def endswith(self, value: str) -> Condition:
+        """Case-insensitive suffix match."""
+        return Match(self.name, "endswith", value)
+
+    # set / null
+    def is_in(self, values: Sequence[Any]) -> Condition:
+        """Match any value in `values`."""
+        return In(self.name, list(values))
+
+    def is_null(self) -> Condition:
+        """Match rows where this column is null/missing."""
+        return IsNull(self.name)
+
+    def is_not_null(self) -> Condition:
+        """Match rows where this column is not null."""
+        return Not(IsNull(self.name))
+
+    # sort
+    def asc(self) -> "SortKey":
+        """Sort by this column, ascending."""
+        return SortKey(self.name, False)
+
+    def desc(self) -> "SortKey":
+        """Sort by this column, descending."""
+        return SortKey(self.name, True)
+
+    def __repr__(self) -> str:
+        return f"col({self.name!r})"
+
+
+def col(name: str) -> Column:
+    """Reference a column by name for use in `where()` / `order()`."""
+    return Column(name)
+
+
+# ---------------------------------------------------------------------------
+# Sorting
+# ---------------------------------------------------------------------------
+
+@dataclass
+class SortKey:
+    """A single sort term: a column and a direction."""
+
+    column: str
+    descending: bool = False
+
+
+SortSpec = Union[str, Column, SortKey]
+
+
+def normalize_sort_keys(keys: Sequence[SortSpec]) -> List[SortKey]:
+    """Normalize mixed sort specs into a list of `SortKey`.
+
+    Accepts `"name"` (ascending), `"-name"` (descending), a `Column`
+    (ascending), or a `SortKey`.
+    """
+    result: List[SortKey] = []
+    for key in keys:
+        if isinstance(key, SortKey):
+            result.append(key)
+        elif isinstance(key, Column):
+            result.append(SortKey(key.name, False))
+        elif isinstance(key, str):
+            if key.startswith("-"):
+                result.append(SortKey(key[1:], True))
+            else:
+                result.append(SortKey(key, False))
+        else:
+            raise TypeError(f"Invalid sort key: {key!r}")
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Combinators
+# ---------------------------------------------------------------------------
+
+def any_of(*conditions: Condition) -> Condition | None:
+    """Combine conditions with OR. Returns None if none are given."""
+    parts = [c for c in conditions if c is not None]
+    if not parts:
+        return None
+    if len(parts) == 1:
+        return parts[0]
+    return Or(parts)
+
+
+def all_of(*conditions: Condition) -> Condition | None:
+    """Combine conditions with AND. Returns None if none are given."""
+    parts = [c for c in conditions if c is not None]
+    if not parts:
+        return None
+    if len(parts) == 1:
+        return parts[0]
+    return And(parts)

bootstack/data/readers.py

@@ -0,0 +1,289 @@
+"""Streaming record readers for file-backed data sources.
+
+A *reader* turns a file into a lazy iterator of record dicts, parsed a row at a
+time so large files stream with bounded memory rather than loading all at once.
+Readers are keyed by file extension in a registry: the stdlib formats
+(CSV / TSV / JSON / JSONL / XML) are always available, and the optional columnar
+and scientific formats (Parquet / Feather / HDF5) register too but raise a clear
+"install the extra" error when used without their dependency.
+
+The reader contract is intentionally small — `reader(path, config) ->
+Iterator[Record]` — so it composes directly with the streaming
+`SqliteDataSource.load(...)`, which chunks the iterator itself.
+
+Register a custom reader for your own extension::
+
+    from bootstack.data.readers import register_reader
+
+    @register_reader(".myfmt")
+    def read_myfmt(path, config):
+        for record in parse(path):
+            yield record
+"""
+
+from __future__ import annotations
+
+import csv
+import importlib
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Callable, Dict, Iterator, List, Optional
+
+from bootstack.data.types import Record
+
+if TYPE_CHECKING:
+    from bootstack.data.file_source import FileSourceConfig
+
+# A reader yields record dicts lazily: (path, config) -> Iterator[Record].
+Reader = Callable[..., Iterator[Record]]
+
+_READERS: Dict[str, Reader] = {}
+
+# Format-name aliases (FileSourceConfig.file_format) → canonical extension.
+_FORMAT_ALIASES: Dict[str, str] = {
+    "csv": ".csv",
+    "tsv": ".tsv",
+    "json": ".json",
+    "jsonl": ".jsonl",
+    "ndjson": ".jsonl",
+    "xml": ".xml",
+    "parquet": ".parquet",
+    "feather": ".feather",
+    "hdf5": ".h5",
+}
+
+
+def _norm_ext(ext: str) -> str:
+    """Normalize an extension to lowercase with a leading dot."""
+    ext = ext.lower()
+    return ext if ext.startswith(".") else "." + ext
+
+
+def register_reader(*extensions: str, reader: Optional[Reader] = None):
+    """Register a reader for one or more file extensions.
+
+    Use as a decorator (`@register_reader(".csv", ".tsv")`) or call directly with
+    `reader=`. Extensions may be given with or without the leading dot.
+
+    Args:
+        extensions: File extensions the reader handles.
+        reader: The reader callable, when not used as a decorator.
+
+    Returns:
+        The reader (so it works as a decorator).
+    """
+
+    def _apply(fn: Reader) -> Reader:
+        for ext in extensions:
+            _READERS[_norm_ext(ext)] = fn
+        return fn
+
+    return _apply(reader) if reader is not None else _apply
+
+
+def supported_read_extensions() -> List[str]:
+    """Return the sorted list of registered readable file extensions."""
+    return sorted(_READERS)
+
+
+def get_reader(path_or_ext: "str | Path") -> Reader:
+    """Resolve the reader for a path or extension.
+
+    Args:
+        path_or_ext: A file path or a bare extension (e.g. `".csv"` or `"csv"`).
+
+    Returns:
+        The registered reader callable.
+
+    Raises:
+        ValueError: If no reader is registered for the extension.
+    """
+    suffix = Path(str(path_or_ext)).suffix or str(path_or_ext)
+    ext = _norm_ext(suffix)
+    try:
+        return _READERS[ext]
+    except KeyError:
+        raise ValueError(
+            f"No reader registered for {ext!r}. Supported formats: "
+            f"{', '.join(supported_read_extensions())}."
+        ) from None
+
+
+def read_records(path: "str | Path", config: "Optional[FileSourceConfig]" = None) -> Iterator[Record]:
+    """Stream records from a file, choosing the reader by format or extension.
+
+    Honors `config.file_format` when set to something other than `'auto'`,
+    otherwise resolves the reader from the path's extension.
+
+    Args:
+        path: Path to the data file.
+        config: Parsing configuration (a default `FileSourceConfig` is used if
+            omitted).
+
+    Returns:
+        A lazy iterator of record dicts.
+    """
+    from bootstack.data.file_source import FileSourceConfig
+
+    cfg = config or FileSourceConfig()
+    fmt = getattr(cfg, "file_format", "auto")
+    if fmt and fmt != "auto":
+        reader = get_reader(_FORMAT_ALIASES.get(fmt, fmt))
+    else:
+        reader = get_reader(path)
+    return reader(path, cfg)
+
+
+def _require(module: str, extra: str):
+    """Import an optional dependency or raise a clear install hint."""
+    try:
+        return importlib.import_module(module)
+    except ImportError as exc:
+        raise ImportError(
+            f"Reading this format needs the optional {module!r} dependency. "
+            f"Install it with:  pip install bootstack[{extra}]"
+        ) from exc
+
+
+# --------------------------------------------------------------------------- stdlib
+
+
+@register_reader(".csv", ".tsv", ".txt")
+def read_csv(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from a CSV/TSV file, one row at a time."""
+    delimiter = config.delimiter
+    if delimiter is None:
+        delimiter = "\t" if _norm_ext(Path(path).suffix) == ".tsv" else ","
+    with open(path, "r", encoding=config.encoding, newline="") as f:
+        for _ in range(config.skip_rows or 0):
+            next(f, None)
+        reader = csv.DictReader(f, delimiter=delimiter, quotechar=config.quotechar)
+        for row in reader:
+            yield dict(row)
+
+
+@register_reader(".jsonl", ".ndjson")
+def read_jsonl(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from a line-delimited JSON file (one object per line)."""
+    with open(path, "r", encoding=config.encoding) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            obj = json.loads(line)
+            yield obj if isinstance(obj, dict) else {"value": obj}
+
+
+@register_reader(".json")
+def read_json(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Read records from a JSON file.
+
+    Supported shapes:
+
+    - a top-level **array of objects** → one record per element;
+    - a top-level **object** with its records under `config.json_records_key`
+      (e.g. an API response like `{"data": [...]}`) → that list;
+    - any other top-level **object** → a single record.
+
+    A JSON array is parsed in full — the stdlib parser cannot stream an array, so
+    prefer JSONL / `.ndjson` (or set `config.json_lines`) for very large data.
+    """
+    if getattr(config, "json_lines", False):
+        yield from read_jsonl(path, config)
+        return
+    with open(path, "r", encoding=config.encoding) as f:
+        data = json.load(f)
+
+    key = getattr(config, "json_records_key", None)
+    if key is not None:
+        items = data.get(key) if isinstance(data, dict) else None
+        if not isinstance(items, list):
+            raise ValueError(
+                f"json_records_key={key!r} did not resolve to a list of records in {path}."
+            )
+        for item in items:
+            yield item if isinstance(item, dict) else {"value": item}
+        return
+
+    if isinstance(data, list):
+        for item in data:
+            yield item if isinstance(item, dict) else {"value": item}
+    elif isinstance(data, dict):
+        yield data
+
+
+@register_reader(".xml")
+def read_xml(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from an XML file using incremental parsing.
+
+    Each record element (those matching `config.xml_record_tag`, or the direct
+    children of the root when it is unset) becomes one record: its attributes and
+    the text of its leaf child elements become fields. Parsed elements are cleared
+    so memory stays bounded. Nested/repeated children are skipped in this version.
+    """
+    import xml.etree.ElementTree as ET
+
+    record_tag = getattr(config, "xml_record_tag", None)
+    depth = 0
+    root = None
+    for event, elem in ET.iterparse(path, events=("start", "end")):
+        if event == "start":
+            if depth == 0:
+                root = elem
+            depth += 1
+            continue
+        # end event
+        depth -= 1
+        is_record = (elem.tag == record_tag) if record_tag else (depth == 1)
+        if not is_record:
+            continue
+        record: Record = dict(elem.attrib)
+        for child in elem:
+            if len(child) == 0:  # leaf element → a scalar field
+                record[child.tag] = child.text
+        yield record
+        elem.clear()
+        if root is not None:
+            root.clear()  # drop processed siblings to keep memory bounded
+
+
+# --------------------------------------------------------------------------- optional
+
+
+@register_reader(".parquet")
+def read_parquet(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from a Parquet file in row-group batches (needs pyarrow)."""
+    pq = _require("pyarrow.parquet", "parquet")
+    parquet_file = pq.ParquetFile(str(path))
+    for batch in parquet_file.iter_batches():
+        for row in batch.to_pylist():
+            yield row
+
+
+@register_reader(".feather", ".arrow")
+def read_feather(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from a Feather/Arrow IPC file in batches (needs pyarrow)."""
+    feather = _require("pyarrow.feather", "parquet")
+    table = feather.read_table(str(path))
+    for batch in table.to_batches():
+        for row in batch.to_pylist():
+            yield row
+
+
+@register_reader(".h5", ".hdf5", ".hdf")
+def read_hdf5(path: "str | Path", config: "FileSourceConfig") -> Iterator[Record]:
+    """Stream records from a tabular HDF5 file in chunks (needs pandas + tables)."""
+    pd = _require("pandas", "hdf5")
+    key = getattr(config, "hdf5_key", None)
+    store = pd.HDFStore(str(path), mode="r")
+    try:
+        if key is None:
+            keys = store.keys()
+            if not keys:
+                return
+            key = keys[0]
+        for chunk in store.select(key, chunksize=10_000):
+            for record in chunk.to_dict(orient="records"):
+                yield record
+    finally:
+        store.close()