vastgrid 0.2.0__py3-none-any.whl

vastgrid/__init__.py

@@ -0,0 +1,6 @@
+"""vastgrid in notebooks: tables of any size in vastgrid's grid, with the engine in the kernel (docs/notebook.md)."""
+
+from ._engine import Engine, EngineError, Source
+from ._widget import Grid, show
+
+__all__ = ["Engine", "EngineError", "Grid", "Source", "show"]

vastgrid/_block.py

@@ -0,0 +1,147 @@
+"""Blocks (docs/spec.md §8.4): rows of typed JSON (§2.2) as an Apache Arrow IPC stream with one schema and one
+record batch, written with the Arrow types of §2.1, as vastgrid-spec's `encodeBlock` writes them."""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Sequence
+from decimal import Decimal
+from typing import Any
+
+import pyarrow as pa
+
+from ._calendar import parse_date_text, parse_interval_text, parse_time_text, parse_timestamp_text
+from ._values import is_value_of, type_name
+
+UUID_EXTENSION = (b"ARROW:extension:name", b"arrow.uuid")
+
+_FLOAT_TOKENS = {"NaN": float("nan"), "Infinity": float("inf"), "-Infinity": float("-inf")}
+
+
+class BlockValueError(TypeError):
+    """A value that cannot be written without loss; the path is relative to the block's rows."""
+
+    reason = "type"
+
+    def __init__(self, path: str, column: dict[str, Any]):
+        super().__init__(f"The value at {path} is not a {type_name(column)} value within the ranges of §2.2.")
+        self.path = path
+
+
+def arrow_type(column: dict[str, Any]) -> pa.DataType:
+    """The Arrow type of a logical type (§8.4)."""
+    kind = column["type"]
+    if kind == "boolean":
+        return pa.bool_()
+    if kind == "integer":
+        return pa.int32()
+    if kind == "bigint":
+        return pa.int64()
+    if kind == "decimal":
+        return pa.decimal128(column.get("precision", 38), column.get("scale", 0))
+    if kind == "float":
+        return pa.float32() if column.get("bits") == 32 else pa.float64()
+    if kind == "string":
+        return pa.string()
+    if kind == "date":
+        return pa.date32()
+    if kind == "time":
+        return pa.time64("us")
+    if kind == "timestamp":
+        return pa.timestamp(column.get("unit", "us"))
+    if kind == "timestamptz":
+        return pa.timestamp("us", tz="UTC")
+    if kind == "interval":
+        return pa.month_day_nano_interval()
+    if kind == "uuid":
+        return pa.binary(16)
+    if kind == "binary":
+        return pa.binary()
+    raise ValueError(f"{kind} is not a logical type.")
+
+
+def _decimal(value: str) -> Decimal:
+    return Decimal(value)
+
+
+def _float(value: Any) -> float:
+    return _FLOAT_TOKENS[value] if isinstance(value, str) else float(value)
+
+
+def _count(column: dict[str, Any], text: str) -> int:
+    kind = column["type"]
+    if kind == "date":
+        return parse_date_text(text)
+    if kind == "time":
+        return parse_time_text(text)
+    if kind == "timestamptz":
+        return parse_timestamp_text(text[:-3], "us")
+    return parse_timestamp_text(text, column.get("unit", "us"))
+
+
+def _interval(text: str) -> pa.MonthDayNano:
+    parts = parse_interval_text(text)
+    return pa.MonthDayNano([parts.months, parts.days, parts.nanos])
+
+
+_CONVERT = {
+    "boolean": None,
+    "integer": None,
+    "bigint": int,
+    "decimal": _decimal,
+    "float": _float,
+    "string": None,
+    "date": None,
+    "time": None,
+    "timestamp": None,
+    "timestamptz": None,
+    "interval": _interval,
+    "uuid": lambda value: bytes.fromhex(value.replace("-", "")),
+    "binary": base64.b64decode,
+}
+
+
+def _array(column: dict[str, Any], values: list[Any]) -> pa.Array:
+    kind = column["type"]
+    if kind in ("date", "time", "timestamp", "timestamptz"):
+        counts = [None if value is None else _count(column, value) for value in values]
+        storage = pa.array(counts, pa.int32() if kind == "date" else pa.int64())
+        return storage.view(arrow_type(column))
+    convert = _CONVERT[kind]
+    if convert is not None:
+        values = [None if value is None else convert(value) for value in values]
+    return pa.array(values, arrow_type(column))
+
+
+def schema_of(columns: Sequence[dict[str, Any]]) -> pa.Schema:
+    return pa.schema(
+        [
+            pa.field(
+                column["name"],
+                arrow_type(column),
+                nullable=True,
+                metadata=dict([UUID_EXTENSION]) if column["type"] == "uuid" else None,
+            )
+            for column in columns
+        ]
+    )
+
+
+def encode_block(columns: Sequence[dict[str, Any]], rows: Sequence[Sequence[Any]]) -> bytes:
+    """Writes rows (typed JSON, §2.2) of columns as a block (§8.4); a value outside §2.2 raises `BlockValueError`."""
+    arrays = []
+    for index, column in enumerate(columns):
+        values = []
+        for at, row in enumerate(rows):
+            value = row[index]
+            if not is_value_of(value, column):
+                raise BlockValueError(f"/rows/{at}/{index}", column)
+            values.append(value)
+        arrays.append(_array(column, values))
+    schema = schema_of(columns)
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, schema) as writer:
+        # A block without columns has its schema only: the answer gives its positions (§8.4).
+        if columns:
+            writer.write_batch(pa.record_batch(arrays, schema=schema))
+    return sink.getvalue().to_pybytes()

vastgrid/_calendar.py

@@ -0,0 +1,212 @@
+"""Dates and times as integers and as text (docs/spec.md §2.2): days since 1970-01-01 in the proleptic Gregorian
+calendar, and counts of seconds, milliseconds, microseconds or nanoseconds. Years are from 1 to 9999, the years that
+the text of §2.2 writes. Python ints stand in for JavaScript's bigints."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+UNIT_PER_SECOND: dict[str, int] = {"s": 1, "ms": 1000, "us": 1_000_000, "ns": 1_000_000_000}
+"""Counts of a unit in a second."""
+
+UNIT_DIGITS: dict[str, int] = {"s": 0, "ms": 3, "us": 6, "ns": 9}
+"""Digits after the point that each unit keeps."""
+
+_DAY_SECONDS = 86_400
+_INT64_MIN = -(2**63)
+_INT64_MAX = 2**63 - 1
+_NANOS_PER_DAY = 86_400_000_000_000
+
+_DATE = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})")
+_CLOCK = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?")
+_INTERVAL = re.compile(
+    r"(?:(-?[0-9]+) years?(?: |\Z))?(?:(-?[0-9]+) months?(?: |\Z))?(?:(-?[0-9]+) days?(?: |\Z))?"
+    r"(?:(-?)([0-9]{2,}):([0-9]{2}):([0-9]{2})(?:\.([0-9]{1,9}))?)?"
+)
+
+
+def _trunc_div(a: int, b: int) -> int:
+    """`a / b` of bigints: rounded toward zero."""
+    quotient = abs(a) // abs(b)
+    return quotient if (a < 0) == (b < 0) else -quotient
+
+
+def _trunc_rem(a: int, b: int) -> int:
+    """`a % b` of bigints: the sign of `a`."""
+    return a - b * _trunc_div(a, b)
+
+
+def days_from_civil(year: int, month: int, day: int) -> int:
+    """Days from 1970-01-01 to a date (Howard Hinnant's `days_from_civil`)."""
+    y = year - 1 if month <= 2 else year
+    era = y // 400
+    yoe = y - era * 400
+    doy = (153 * (month + (-3 if month > 2 else 9)) + 2) // 5 + day - 1
+    doe = yoe * 365 + yoe // 4 - yoe // 100 + doy
+    return era * 146097 + doe - 719468
+
+
+def civil_from_days(days: int) -> tuple[int, int, int]:
+    """The year, month and day of a count of days from 1970-01-01 (Howard Hinnant's `civil_from_days`)."""
+    z = days + 719468
+    era = z // 146097
+    doe = z - era * 146097
+    yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365
+    doy = doe - (365 * yoe + yoe // 4 - yoe // 100)
+    mp = (5 * doy + 2) // 153
+    day = doy - (153 * mp + 2) // 5 + 1
+    month = mp + (3 if mp < 10 else -9)
+    return (yoe + era * 400 + (1 if month <= 2 else 0), month, day)
+
+
+def _pad(value: int, length: int) -> str:
+    return str(value).rjust(length, "0")
+
+
+def date_text(days: int) -> str:
+    """`YYYY-MM-DD` of a count of days. Raises for years outside 1 to 9999."""
+    year, month, day = civil_from_days(days)
+    if year < 1 or year > 9999:
+        raise ValueError(f"The year {year} is outside 1 to 9999.")
+    return f"{_pad(year, 4)}-{_pad(month, 2)}-{_pad(day, 2)}"
+
+
+def parse_date_text(text: str) -> int | None:
+    """The count of days of `YYYY-MM-DD`, or None when the text is not a real day of the years 1 to 9999."""
+    match = _DATE.fullmatch(text)
+    if not match:
+        return None
+    year, month, day = int(match[1]), int(match[2]), int(match[3])
+    if year < 1 or month < 1 or month > 12 or day < 1:
+        return None
+    days = days_from_civil(year, month, day)
+    return days if civil_from_days(days) == (year, month, day) else None
+
+
+def _clock(count: int, unit: str) -> str:
+    """`HH:MM:SS`, then the fraction of a second without trailing zeros, of a count of a unit within a day."""
+    per_second = UNIT_PER_SECOND[unit]
+    seconds = _trunc_div(count, per_second)
+    fraction = _trunc_rem(count, per_second)
+    hours = _trunc_div(seconds, 3600)
+    minutes = _trunc_rem(_trunc_div(seconds, 60), 60)
+    rest = _trunc_rem(seconds, 60)
+    text = f"{_pad(hours, 2)}:{_pad(minutes, 2)}:{_pad(rest, 2)}"
+    if fraction != 0:
+        text += "." + _pad(fraction, UNIT_DIGITS[unit]).rstrip("0")
+    return text
+
+
+def time_text(micros: int) -> str:
+    """The text of a time of day (§2.2) from microseconds since midnight."""
+    if micros < 0 or micros >= _DAY_SECONDS * 1_000_000:
+        raise ValueError("A time is within one day.")
+    return _clock(micros, "us")
+
+
+def _parse_clock(text: str, unit: str) -> int | None:
+    """Reads `HH:MM:SS` with up to the unit's fraction digits as a count of the unit since midnight, or None."""
+    match = _CLOCK.fullmatch(text)
+    if not match:
+        return None
+    hours, minutes, seconds = int(match[1]), int(match[2]), int(match[3])
+    fraction = match[4] or ""
+    if hours > 23 or minutes > 59 or seconds > 59 or len(fraction) > UNIT_DIGITS[unit]:
+        return None
+    whole = (hours * 3600 + minutes * 60 + seconds) * UNIT_PER_SECOND[unit]
+    return whole + (int(fraction.ljust(UNIT_DIGITS[unit], "0")) if fraction else 0)
+
+
+def parse_time_text(text: str) -> int | None:
+    """Microseconds since midnight of the text of a time (§2.2), or None."""
+    return _parse_clock(text, "us")
+
+
+def timestamp_text(count: int, unit: str = "us") -> str:
+    """The text of a timestamp (§2.2) from a count of its unit since 1970-01-01 00:00:00."""
+    per_day = _DAY_SECONDS * UNIT_PER_SECOND[unit]
+    days = count // per_day
+    return f"{date_text(days)} {_clock(count - days * per_day, unit)}"
+
+
+def parse_timestamp_text(text: str, unit: str = "us") -> int | None:
+    """The count of the unit since 1970-01-01 00:00:00 of the text of a timestamp (§2.2), or None."""
+    if text.find(" ") != 10:
+        return None
+    days = parse_date_text(text[:10])
+    time = _parse_clock(text[11:], unit)
+    if days is None or time is None:
+        return None
+    count = days * _DAY_SECONDS * UNIT_PER_SECOND[unit] + time
+    return count if _INT64_MIN <= count <= _INT64_MAX else None
+
+
+@dataclass
+class IntervalParts:
+    """An interval (§2.1): months, days and nanoseconds."""
+
+    months: int
+    days: int
+    nanos: int
+
+
+def _count(value: int, unit: str) -> str:
+    return f"{value} {unit}{'' if value in (1, -1) else 's'}"
+
+
+def interval_text(parts: IntervalParts) -> str:
+    """DuckDB's text of an interval (§2.4): years and months, days, then the time as `HH:MM:SS` with the fraction
+    without trailing zeros, each left out when it is 0 (`00:00:00` when everything is), as
+    `1 year 2 months 3 days 04:05:06.789` or `-01:30:00`. Nanoseconds beyond DuckDB's microseconds add digits."""
+    out: list[str] = []
+    years = _trunc_div(parts.months, 12)
+    rest = parts.months - years * 12
+    if years != 0:
+        out.append(_count(years, "year"))
+    if rest != 0:
+        out.append(_count(rest, "month"))
+    if parts.days != 0:
+        out.append(_count(parts.days, "day"))
+    if parts.nanos != 0:
+        magnitude = abs(parts.nanos)
+        seconds, fraction = divmod(magnitude, 1_000_000_000)
+        time = f"{_pad(seconds // 3600, 2)}:{_pad(seconds // 60 % 60, 2)}:{_pad(seconds % 60, 2)}"
+        if fraction != 0:
+            time += "." + _pad(fraction, 9).rstrip("0")
+        out.append(("-" if parts.nanos < 0 else "") + time)
+    return " ".join(out) if out else "00:00:00"
+
+
+def parse_interval_text(text: str) -> IntervalParts | None:
+    """The parts of DuckDB's text of an interval, as `interval_text` writes it, or None."""
+    match = _INTERVAL.fullmatch(text)
+    if not match or text == "" or text.endswith(" "):
+        return None
+    years, months, days = (int(part or 0) for part in (match[1], match[2], match[3]))
+    if int(match[6] or 0) > 59 or int(match[7] or 0) > 59:
+        return None
+    nanos = 0
+    if match[5] is not None:
+        seconds = int(match[5]) * 3600 + int(match[6]) * 60 + int(match[7])
+        nanos = seconds * 1_000_000_000 + int((match[8] or "").ljust(9, "0"))
+        if match[4] == "-":
+            nanos = -nanos
+    total = years * 12 + months
+    if not (-2147483648 <= total <= 2147483647 and -2147483648 <= days <= 2147483647):
+        return None
+    if not _INT64_MIN <= nanos <= _INT64_MAX:
+        return None
+    return IntervalParts(total, days, nanos)
+
+
+def normalize_interval(text: str) -> tuple[int, int, int] | None:
+    """The parts of an interval as DuckDB 2.0 compares them (§3.1): its length in nanoseconds, with months of 30 days
+    and days of 24 hours, as whole months, then the days (0 to 29) and the nanoseconds (less than a day) of the rest.
+    Intervals compare by these months, then days, then nanoseconds; `1 month` equals `30 days`, and `1 month -1 day`
+    equals `29 days`. None when the text is not an interval."""
+    parts = parse_interval_text(text)
+    if parts is None:
+        return None
+    months, rest = divmod((parts.months * 30 + parts.days) * _NANOS_PER_DAY + parts.nanos, 30 * _NANOS_PER_DAY)
+    return (months, *divmod(rest, _NANOS_PER_DAY))