python-corekit 0.1.0__py3-none-any.whl

corekit/data/dataset.py

@@ -0,0 +1,340 @@
+import random
+from typing import Any, Callable, Iterable, Iterator, Optional
+
+from corekit.data.record import BaseRecord, resolve_default
+from corekit.data.stats import DatasetStats
+
+BASE_IDENTIFICATION_KEY = "id"
+
+
+class Dataset:
+    """
+    A schema-fixed, order-preserving collection of records.
+
+    Every record in a dataset shares one schema and one generated record
+    class. Supports O(1) access by list position or by id, in-place field
+    editing, and dataset-wide schema evolution via add_field/remove_field.
+    """
+
+    # -- construction & protocol methods -----------------------------------
+
+    def __init__(
+        self,
+        id_key: str = BASE_IDENTIFICATION_KEY,
+        schema: Iterable[Any] | None = None,
+        missing_default: Any = None,
+        missing_default_factory: Callable[[], Any] | None = None,
+    ) -> None:
+        """
+        schema: explicit field order to lock in immediately. If omitted,
+                inferred from the first record passed to add().
+        missing_default / missing_default_factory: value (or zero-arg
+                factory, for mutables) used for a field a record omits.
+        """
+        self._id_key = id_key
+        self._cursor = 0
+        self._records: list[BaseRecord] = []
+        self._id_index: dict[Any, Any] = {}
+        self._missing_default_factory = resolve_default(missing_default, missing_default_factory)
+        self._schema: tuple | None = None
+        self._record_cls: type | None = None
+        if schema is not None:
+            self._set_schema(schema)
+
+    def __getitem__(self, key: int) -> Any:
+        """
+        Positional access, like a list. Use get_record() for id lookup
+        """
+        return self._records[key]
+
+    def __len__(self) -> int:
+        return len(self._records)
+
+    def __iter__(self) -> Iterator[Any]:
+        return iter(self._records)
+
+    def __contains__(self, record_id: Any) -> bool:
+        return record_id in self._id_index
+
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, Dataset):
+            return NotImplemented
+        return self._schema == other._schema and self.to_dict() == other.to_dict()
+
+    def __repr__(self) -> str:
+        return f"Dataset({len(self._records)} records, schema={self._schema})"
+
+    def __getstate__(self) -> dict:
+        """
+        _record_cls is a dynamic type() class with no module path, so it
+        can't pickle by reference -- dropped here and rebuilt in
+        __setstate__. Records are flattened to plain value tuples so every
+        record reconstructs through the same rebuilt class.
+        """
+        state = self.__dict__.copy()
+        state["_record_cls"] = None
+        if self._schema is not None:
+            state["_records"] = [tuple(rec[f] for f in self._schema) for rec in self._records]
+        return state
+
+    def __setstate__(self, state: dict) -> None:
+        self.__dict__.update(state)
+        if self._schema is not None:
+            self._record_cls = BaseRecord.create_new(self._schema)
+            rebuilt = []
+            id_index = {}
+            for values in self._records:
+                rec = self._new_record(zip(self._schema, values))
+                rebuilt.append(rec)
+                id_index[rec[self._id_key]] = rec
+            self._records = rebuilt
+            self._id_index = id_index
+
+    # -- statistics -----------------------------------------------------------
+
+    @property
+    def stats(self) -> DatasetStats:
+        """
+        A fresh, lightweight stats view over this dataset's current records.
+        """
+        return DatasetStats(self._records, self._schema)
+
+    # -- schema management --------------------------------------------------
+
+    @property
+    def schema(self) -> tuple | None:
+        return self._schema
+
+    def _set_schema(self, fields: Iterable[Any]) -> None:
+        fields = tuple(fields)
+        if self._id_key not in fields:
+            fields = (self._id_key,) + fields
+        self._schema = fields
+        self._record_cls = BaseRecord.create_new(fields)
+
+    def add_field(self, name: Any, default: Any = None, default_factory: Callable[[], Any] | None = None) -> None:
+        """
+        Add a field to the schema, seeded on every existing record. O(n)
+        """
+        if name == self._id_key:
+            raise ValueError(f"Cannot add the {self._id_key!r} field. It is set as the id key.")
+        if self._schema is not None and name in self._schema:
+            raise KeyError(f"field {name!r} already exists")
+
+        value_fn = resolve_default(default, default_factory)
+        new_fields = (self._schema or (self._id_key,)) + (name,)
+        self._migrate_schema(new_fields, fill={name: value_fn})
+
+    def remove_field(self, name: Any) -> None:
+        """
+        Drop a field from the schema across every record. O(n)
+        """
+        if name == self._id_key:
+            raise ValueError(f"Cannot remove the {self._id_key!r} field. It is set as the id key.")
+        if self._schema is None or name not in self._schema:
+            raise KeyError(name)
+
+        new_fields = tuple(f for f in self._schema if f != name)
+        self._migrate_schema(new_fields)
+
+    def _new_record(self, field_values: Iterable[tuple[Any, Any]]) -> "BaseRecord":
+        """
+        Build one instance of this dataset's record class from (field, value) pairs
+        """
+        instance = self._record_cls()
+        for field, value in field_values:
+            instance[field] = value
+        return instance
+
+    def _migrate_record(self, existing: "BaseRecord", fill: dict[str, Callable[[], Any]] | None = None) -> "BaseRecord":
+        """
+        Rebuild `existing` under the current schema.
+
+        `fill` maps a field to a zero-arg factory for values that don't
+        come from `existing` (e.g. a field add_field() just introduced).
+        Checked before reading existing[field], since a brand-new field
+        won't exist on the old record at all.
+        """
+        fill = fill or {}
+        pairs = ((field, fill[field]() if field in fill else existing[field]) for field in self.schema)
+        return self._new_record(pairs)
+
+    def _migrate_schema(self, new_fields: tuple, fill: dict[str, Callable[[], Any]] | None = None) -> None:
+        fill = fill or {}
+        old_records = self._records
+        self._set_schema(new_fields)
+        new_records = []
+        new_id_index = {}
+        for old in old_records:
+            new_record = self._migrate_record(old, fill)
+            new_records.append(new_record)
+            new_id_index[new_record[self._id_key]] = new_record
+
+        self.set_data(new_records, new_id_index)
+
+    def create_empty_clone(self) -> "Dataset":
+        """
+        A new, empty Dataset with the same schema and missing-default policy
+        """
+        return Dataset(
+            id_key=self._id_key,
+            schema=self._schema,
+            missing_default_factory=self._missing_default_factory,
+        )
+
+    def set_data(self, data: list[BaseRecord], id_to_index: dict[str, Any] | None = None) -> None:
+        """
+        Replace the dataset's records directly. Used internally by schema
+        migration, sample(), and filter().
+
+        If `id_to_index` is supplied, the caller is asserting every record
+        is already an instance of this dataset's class (true for
+        _migrate_schema's output) and no rebuild/check is done. Otherwise,
+        records may belong to a different dataset's class (sample()/filter()
+        cloning from a parent) -- each is checked and rebuilt through
+        _migrate_record if needed, with the id index accumulated in the
+        same pass.
+        """
+        if id_to_index is None:
+            records = []
+            id_to_index = {}
+            for rec in data:
+                if type(rec) is not self._record_cls:
+                    rec = self._migrate_record(rec)
+                records.append(rec)
+                id_to_index[rec[self._id_key]] = rec
+            data = records
+
+        self._records = data
+        self._id_index = id_to_index
+        self.reset_cursor()
+
+    def add(self, data: dict[str, Any]) -> Any:
+        """
+        Add a new record. Must contain the id field. If no schema is set
+        yet, this record's keys define it. Fields the schema expects but
+        this record omits get missing_default_factory(). A field not in
+        the schema raises -- evolve the schema via add_field() first.
+        """
+        if self._id_key not in data:
+            raise ValueError(f"record must contain the identity field: {self._id_key}")
+
+        if self._schema is None:
+            self._set_schema(data.keys())
+        else:
+            unknown = set(data.keys()) - set(self._schema)
+            if unknown:
+                raise KeyError(
+                    f"field(s) {unknown} not in schema {self._schema}; "
+                    f"call add_field() first if this is an intentional schema change"
+                )
+
+        if data[self._id_key] in self._id_index:
+            raise KeyError(f"Duplicate id key: {data[self._id_key]!r}")
+
+        rec = self._record_cls()
+        for field in self._schema:
+            rec[field] = data[field] if field in data else self._missing_default_factory()
+
+        self._records.append(rec)
+        self._id_index[data[self._id_key]] = rec
+        return rec
+
+    def extend(self, items: Iterable[dict[str, Any]]) -> None:
+        for item in items:
+            self.add(item)
+
+    def clear(self) -> None:
+        self._records.clear()
+        self._id_index.clear()
+        self.reset_cursor()
+
+    # -- cursor & positional/id access --------------------------------------
+
+    def reset_cursor(self) -> None:
+        self._cursor = 0
+
+    def next(self) -> Any:
+        """
+        Manual, stateful walk over records; pairs with reset_cursor().
+        Independent of __iter__, which always starts fresh -- see class docs.
+        """
+        if self._cursor >= len(self._records):
+            raise StopIteration
+        rec = self._records[self._cursor]
+        self._cursor += 1
+        return rec
+
+    def at(self, idx: int) -> Any:
+        return self._records[idx]
+
+    def get_record(self, record_id: Any) -> Any:
+        return self._id_index[record_id]
+
+    # -- removing whole records ----------------------------------------------
+
+    def remove_by_id(self, record_id: Any) -> None:
+        rec = self._id_index.pop(record_id)
+        self._records.remove(rec)  # O(n)
+
+    def remove_by_index(self, i: int) -> None:
+        rec = self._records.pop(i)  # O(n) unless last element
+        del self._id_index[rec[self._id_key]]
+
+    # -- ordering, sampling, querying -----------------------------------------
+
+    def get_random(self) -> Any:
+        return random.choice(self._records)
+
+    def shuffle(self) -> None:
+        """
+        Reorders records only. _id_index maps id -> record object
+        regardless of list position, so it needs no rebuild here.
+        """
+        random.shuffle(self._records)
+        self.reset_cursor()
+
+    def sort(self, key: Callable[[Any], Any], reverse: bool = False) -> None:
+        """
+        Same reasoning as shuffle(): _id_index stays valid untouched
+        """
+        self._records.sort(key=key, reverse=reverse)
+        self.reset_cursor()
+
+    def sample(self, n: int) -> "Dataset":
+        new = self.create_empty_clone()
+        new.set_data(random.sample(self._records, n))
+        return new
+
+    def _filter_iterator(self, func: Callable[[Any], bool]) -> Iterator[BaseRecord]:
+        for rec in self._records:
+            if func(rec):
+                yield rec
+
+    def filter(self, func: Callable[[Any], bool]) -> "Dataset":
+        """
+        `func` may be a plain callable/lambda, or a Field(...) expression
+        """
+        new = self.create_empty_clone()
+        new.set_data(list(self._filter_iterator(func)))
+        return new
+
+    def find_first(self, func: Callable[[Any], bool]) -> Optional["BaseRecord"]:
+        """
+        `func` may be a plain callable/lambda, or a Field(...) expression
+        """
+        return next(self._filter_iterator(func), None)
+
+    def group(self, field: str) -> dict[Any, "Dataset"]:
+        grouped = {}
+        for rec in self._records:
+            key = rec[field]
+            if key not in grouped:
+                grouped[key] = self.create_empty_clone()
+            grouped[key].add(rec.to_dict())
+        return grouped
+
+    # -- output ------------------------------------------------------------
+
+    def to_dict(self) -> list[dict[str, Any]]:
+        return [rec.to_dict() for rec in self._records]

corekit/data/expressions/__init__.py

@@ -0,0 +1,46 @@
+"""
+Composable predicates for filtering datasets.
+
+Expressions combine with ``&``, ``|`` and ``~``, and compare fields against
+values or against each other::
+
+    Field("age") >= 18
+    Field("first_name") == Field("nickname")
+    (Field("age") > 20) & ~Field("name").contains("test")
+
+Every node can also emit a backend query via ``to_sql``, ``to_mongo`` or
+``to_elasticsearch``.
+"""
+
+from corekit.data.expressions.comparison import (
+    Comparison,
+    Contains,
+    Equals,
+    FieldExpression,
+    GreaterThan,
+    GreaterThanOrEquals,
+    IsIn,
+    LessThan,
+    LessThanOrEquals,
+    NotEquals,
+    ValueExpression,
+)
+from corekit.data.expressions.expression import And, Expression, Not, Or
+
+__all__ = [
+    "And",
+    "Comparison",
+    "Contains",
+    "Equals",
+    "Expression",
+    "FieldExpression",
+    "GreaterThan",
+    "GreaterThanOrEquals",
+    "IsIn",
+    "LessThan",
+    "LessThanOrEquals",
+    "Not",
+    "NotEquals",
+    "Or",
+    "ValueExpression",
+]

corekit/data/expressions/comparison.py

@@ -0,0 +1,252 @@
+from typing import Any, Iterable
+
+from corekit.data.expressions.expression import Expression
+
+
+class Comparison(Expression):
+    """
+    Base for comparing two values (Equals, GreaterThan, IsIn, ...).
+
+    `left`, `right`, and `symbol` are public so external code --
+    e.g. a future translator to SQL/Mongo/Elasticsearch -- can walk the
+    tree and read out what each node represents without evaluating it.
+    `symbol` is a human-readable operator string; the class itself
+    (Equals, GreaterThan, etc.) is the canonical, type-safe identifier of
+    *which* operation a node represents, so a translator can dispatch on
+    isinstance(node, GreaterThan) rather than string-matching on symbol.
+    """
+
+    symbol: str
+
+    def __init__(self, left: Any, right: Any) -> None:
+        self.left = left
+        self.right = right
+
+    def __repr__(self) -> str:
+        return f"({self.left!r} {self.symbol} {self.right!r})"
+
+    @staticmethod
+    def _resolve_value(value: Any, *args: Any, **kwargs: Any) -> Any:
+        """
+        Resolves a value to a concrete value. This method utilizes a neat trick
+        to be able to resolve the final value of complex expressions without
+        having to evaluate the entire tree.
+        """
+        if isinstance(value, ValueExpression):
+            return value.get(*args, **kwargs)
+        return value
+
+    def get_left(self, *args: Any, **kwargs: Any) -> Any:
+        return self._resolve_value(self.left, *args, **kwargs)
+
+    def get_right(self, *args, **kwargs) -> Any:
+        """
+        Returns the right value of the comparison.
+        """
+        return self._resolve_value(self.right, *args, **kwargs)
+
+
+class Equals(Comparison):
+    symbol = "=="
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) == self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} = {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): self.get_right(*args, **kwargs)}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"term": {self.get_left(*args, **kwargs): self.get_right(*args, **kwargs)}}
+
+
+class NotEquals(Comparison):
+    symbol = "!="
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) != self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} != {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$ne": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"term": {self.get_left(*args, **kwargs): {"$ne": self.get_right(*args, **kwargs)}}}
+
+
+class LessThan(Comparison):
+    symbol = "<"
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) < self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} < {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$lt": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"range": {self.get_left(*args, **kwargs): {"lt": self.get_right(*args, **kwargs)}}}
+
+
+class LessThanOrEquals(Comparison):
+    symbol = "<="
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) <= self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} <= {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$lte": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"range": {self.get_left(*args, **kwargs): {"lte": self.get_right(*args, **kwargs)}}}
+
+
+class GreaterThan(Comparison):
+    symbol = ">"
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) > self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} > {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$gt": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"range": {self.get_left(*args, **kwargs): {"gt": self.get_right(*args, **kwargs)}}}
+
+
+class GreaterThanOrEquals(Comparison):
+    symbol = ">="
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) >= self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return f"{self.get_left(*args, **kwargs)} >= {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$gte": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"range": {self.get_left(*args, **kwargs): {"gte": self.get_right(*args, **kwargs)}}}
+
+
+class IsIn(Comparison):
+    symbol = "is in"
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_left(*args, **kwargs) in self.get_right(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        # TODO: this might not actually be valid SQL
+        return f"{self.get_left(*args, **kwargs)} IN {self.get_right(*args, **kwargs)}"
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {self.get_left(*args, **kwargs): {"$in": self.get_right(*args, **kwargs)}}
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return {"terms": {self.get_left(*args, **kwargs): self.get_right(*args, **kwargs)}}
+
+
+class Contains(Comparison):
+    symbol = "contains"
+
+    def __call__(self, *args: Any, **kwargs: Any) -> bool:
+        return self.get_right(*args, **kwargs) in self.get_left(*args, **kwargs)
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return ""
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return
+
+
+class ValueExpression(Expression):
+    """
+    Expression representing a single value to be compared.
+
+    The resulting tree (Equals/And/Or/... nodes) is also an inspectable structure,
+    not just a callable. These could be used in the future to emit structured queries
+    (SQL, MongoDB, Elasticsearch, etc.)
+    """
+
+    def __init__(self, value: Any) -> None:
+        self.value = value
+
+    def __repr__(self) -> str:
+        return f"{self.value!r}"
+
+    def __eq__(self, other: Any) -> "Comparison":
+        return Equals(self, other)
+
+    def __ne__(self, other: Any) -> "Comparison":
+        return NotEquals(self, other)
+
+    def __lt__(self, other: Any) -> "Comparison":
+        return LessThan(self, other)
+
+    def __le__(self, other: Any) -> "Comparison":
+        return LessThanOrEquals(self, other)
+
+    def __gt__(self, other: Any) -> "Comparison":
+        return GreaterThan(self, other)
+
+    def __ge__(self, other: Any) -> "Comparison":
+        return GreaterThanOrEquals(self, other)
+
+    def isin(self, values: Iterable[Any]) -> "Comparison":
+        return IsIn(self, frozenset(values))
+
+    def contains(self, substring: Any) -> "Comparison":
+        return Contains(self, substring)
+
+    def get(self, *args: Any, **kwargs: Any) -> Any:
+        return self.value
+
+    def to_sql(self, *args: Any, **kwargs: Any) -> str:
+        return str(self.value)
+
+    def to_mongo(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return self.value
+
+    def to_elasticsearch(self, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return self.value
+
+
+class FieldExpression(ValueExpression):
+    """
+    A value expression representing a named field reference for building filter predicates.
+    FieldExpression instances will evaluate against anything that supports __getitem__ by field name.
+    """
+
+    def __call__(self, record: Any, *args, **kwargs) -> bool:
+        return bool(record[self.value])
+
+    def __repr__(self) -> str:
+        return f"Field({self.value!r})"
+
+    def get(self, record: Any, *args: Any, **kwargs: Any) -> Any:
+        return record[self.value]
+
+    def to_sql(self, record: Any, *args: Any, **kwargs: Any) -> str:
+        return record[self.value]
+
+    def to_mongo(self, record: Any, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return record[self.value]
+
+    def to_elasticsearch(self, record: Any, *args: Any, **kwargs: Any) -> dict[str, Any] | str:
+        return record[self.value]