kitefs 0.2.0a1__py3-none-any.whl

kitefs/__init__.py

@@ -0,0 +1,31 @@
+"""KiteFS — a Python feature store for offline/online feature storage and serving."""
+
+from kitefs.definitions import (
+    EntityKey,
+    EventTimestamp,
+    Expect,
+    Feature,
+    FeatureGroup,
+    FeatureType,
+    JoinKey,
+    Metadata,
+    StorageTarget,
+    ValidationMode,
+)
+from kitefs.feature_store import FeatureStore
+from kitefs.registry import ApplyResult
+
+__all__ = [
+    "ApplyResult",
+    "EntityKey",
+    "EventTimestamp",
+    "Expect",
+    "Feature",
+    "FeatureGroup",
+    "FeatureStore",
+    "FeatureType",
+    "JoinKey",
+    "Metadata",
+    "StorageTarget",
+    "ValidationMode",
+]

kitefs/__main__.py

@@ -0,0 +1,5 @@
+"""Entry point for `python -m kitefs`."""
+
+from kitefs.cli import cli
+
+cli()

kitefs/cli.py

@@ -0,0 +1,253 @@
+"""CLI entry point for KiteFS — thin delegation layer over the SDK.
+
+``kitefs init`` is the only self-contained command because the project
+scaffold (including ``kitefs.yaml``) does not exist yet when it runs.
+All other commands delegate to :class:`kitefs.FeatureStore`.
+"""
+
+import json
+from pathlib import Path
+
+import click
+
+_GITIGNORE_ENTRY = "feature_store/data/"
+
+_DEFAULT_CONFIG = """\
+provider: local
+storage_root: ./feature_store/
+"""
+
+_EXAMPLE_FEATURES = '''\
+"""Example feature group definitions for KiteFS.
+
+Uncomment and modify the example below, then run ``kitefs apply``
+to register your feature groups.
+"""
+
+# from kitefs import (
+#     EntityKey,
+#     EventTimestamp,
+#     Expect,
+#     Feature,
+#     FeatureGroup,
+#     FeatureType,
+#     Metadata,
+#     StorageTarget,
+#     ValidationMode,
+# )
+#
+# example_features = FeatureGroup(
+#     name="example_features",
+#     storage_target=StorageTarget.OFFLINE,
+#     entity_key=EntityKey(name="entity_id", dtype=FeatureType.INTEGER),
+#     event_timestamp=EventTimestamp(name="event_timestamp", dtype=FeatureType.DATETIME),
+#     features=[
+#         Feature(name="feature_one", dtype=FeatureType.FLOAT, expect=Expect().not_null()),
+#         Feature(name="feature_two", dtype=FeatureType.STRING),
+#     ],
+#     ingestion_validation=ValidationMode.ERROR,
+#     metadata=Metadata(owner="your-team", description="An example feature group."),
+# )
+'''
+
+_SEED_REGISTRY = {"version": "1.0", "feature_groups": {}}
+
+
+@click.group()
+def cli() -> None:
+    """KiteFS — a Python feature store for offline/online feature storage and serving."""
+
+
+@cli.command()
+@click.argument("path", required=False, default=None, type=click.Path(file_okay=False))
+def init(path: str | None) -> None:
+    """Create a new KiteFS project at PATH (default: current directory)."""
+    project_root = Path(path).resolve() if path else Path.cwd().resolve()
+    config_path = project_root / "kitefs.yaml"
+
+    if config_path.exists():
+        click.echo("Error: KiteFS project already initialized at this location.", err=True)
+        raise SystemExit(1)
+
+    storage_root = project_root / "feature_store"
+
+    try:
+        # Create directory structure
+        (storage_root / "definitions").mkdir(parents=True, exist_ok=True)
+        (storage_root / "data" / "offline_store").mkdir(parents=True, exist_ok=True)
+        (storage_root / "data" / "online_store").mkdir(parents=True, exist_ok=True)
+
+        # Seed definitions
+        (storage_root / "definitions" / "__init__.py").write_text("", encoding="utf-8")
+        (storage_root / "definitions" / "example_features.py").write_text(_EXAMPLE_FEATURES, encoding="utf-8")
+
+        # Seed registry.json — deterministic output for meaningful Git diffs
+        registry_path = storage_root / "registry.json"
+        registry_path.write_text(json.dumps(_SEED_REGISTRY, sort_keys=True, indent=2) + "\n", encoding="utf-8")
+
+        # Create or append .gitignore — check by exact line, not substring, to avoid
+        # false positives from comments or negated rules containing the entry.
+        gitignore_path = project_root / ".gitignore"
+        if gitignore_path.exists():
+            content = gitignore_path.read_text(encoding="utf-8")
+            existing_lines = {line.strip() for line in content.splitlines()}
+            if _GITIGNORE_ENTRY not in existing_lines:
+                with gitignore_path.open("a", encoding="utf-8") as f:
+                    if content and not content.endswith("\n"):
+                        f.write("\n")
+                    f.write(_GITIGNORE_ENTRY + "\n")
+        else:
+            gitignore_path.write_text(_GITIGNORE_ENTRY + "\n", encoding="utf-8")
+
+        # Seed kitefs.yaml last — this is the sentinel file that guards against
+        # re-init. Writing it last ensures a crash mid-scaffold leaves no sentinel,
+        # so the user can retry `kitefs init` without manual cleanup.
+        config_path.write_text(_DEFAULT_CONFIG, encoding="utf-8")
+    except OSError as e:
+        click.echo(f"Error: {e}", err=True)
+        raise SystemExit(1) from None
+
+    click.echo(f"Project initialized at {project_root}")
+    click.echo("  Provider: local")
+    click.echo(f"  Config:   {config_path}")
+
+
+@cli.command()
+def apply() -> None:
+    """Register feature group definitions into the registry."""
+    from kitefs.exceptions import KiteFSError
+    from kitefs.feature_store import FeatureStore
+
+    try:
+        fs = FeatureStore()
+        result = fs.apply()
+    except KiteFSError as e:
+        click.echo(f"Error: {e}", err=True)
+        raise SystemExit(1) from None
+
+    click.echo(f"Applied {result.group_count} feature group(s) — registered successfully.")
+
+
+@cli.command(name="list")
+@click.option("--format", "fmt", default=None, type=click.Choice(["json"], case_sensitive=False), help="Output format.")
+@click.option("--target", default=None, type=click.Path(), help="File path to write output to.")
+def list_cmd(fmt: str | None, target: str | None) -> None:
+    """List all registered feature groups with summary information."""
+    from kitefs.exceptions import KiteFSError
+    from kitefs.feature_store import FeatureStore
+
+    try:
+        fs = FeatureStore()
+        result = fs.list_feature_groups(format=fmt, target=target)
+    except KiteFSError as e:
+        click.echo(f"Error: {e}", err=True)
+        raise SystemExit(1) from None
+
+    if target is not None:
+        click.echo(f"Output written to {target}")
+        return
+
+    if fmt == "json":
+        click.echo(result)
+        return
+
+    # Default: human-readable table.
+    assert isinstance(result, list)  # target/format branches already returned
+    if not result:
+        click.echo("No feature groups registered. Run `kitefs apply` first.")
+        return
+
+    _render_list_table(result)
+
+
+def _render_list_table(summaries: list[dict]) -> None:
+    """Render feature group summaries as a human-readable table."""
+    headers = ["Name", "Owner", "Entity Key", "Storage Target", "Features"]
+    keys = ["name", "owner", "entity_key", "storage_target", "feature_count"]
+
+    rows: list[list[str]] = []
+    for s in summaries:
+        rows.append([str(s.get(k) or "") for k in keys])
+
+    # Compute column widths from headers and data.
+    widths = [len(h) for h in headers]
+    for row in rows:
+        for i, cell in enumerate(row):
+            widths[i] = max(widths[i], len(cell))
+
+    def _fmt_row(cells: list[str]) -> str:
+        return "  ".join(cell.ljust(widths[i]) for i, cell in enumerate(cells))
+
+    click.echo(_fmt_row(headers))
+    click.echo("  ".join("-" * w for w in widths))
+    for row in rows:
+        click.echo(_fmt_row(row))
+
+
+@cli.command()
+@click.argument("feature_group_name")
+@click.option("--format", "fmt", default=None, type=click.Choice(["json"], case_sensitive=False), help="Output format.")
+@click.option("--target", default=None, type=click.Path(), help="File path to write output to.")
+def describe(feature_group_name: str, fmt: str | None, target: str | None) -> None:
+    """Display the full definition of a specific feature group."""
+    from kitefs.exceptions import KiteFSError
+    from kitefs.feature_store import FeatureStore
+
+    try:
+        fs = FeatureStore()
+        result = fs.describe_feature_group(feature_group_name, format=fmt, target=target)
+    except KiteFSError as e:
+        click.echo(f"Error: {e}", err=True)
+        raise SystemExit(1) from None
+
+    if target is not None:
+        click.echo(f"Output written to {target}")
+        return
+
+    if fmt == "json":
+        click.echo(result)
+        return
+
+    # Default: human-readable key-value layout.
+    assert isinstance(result, dict)  # target/format branches already returned
+    _render_describe(result)
+
+
+def _render_describe(entry: dict) -> None:
+    """Render a full feature group definition as a human-readable layout."""
+    click.echo(f"Feature Group: {entry.get('name', '?')}")
+    click.echo(f"  Storage Target:              {entry.get('storage_target', '?')}")
+
+    ek = entry.get("entity_key", {})
+    click.echo(f"  Entity Key:                  {ek.get('name', '?')} ({ek.get('dtype', '?')})")
+
+    et = entry.get("event_timestamp", {})
+    click.echo(f"  Event Timestamp:             {et.get('name', '?')} ({et.get('dtype', '?')})")
+
+    click.echo(f"  Ingestion Validation:        {entry.get('ingestion_validation', '?')}")
+    click.echo(f"  Offline Retrieval Validation: {entry.get('offline_retrieval_validation', '?')}")
+
+    meta = entry.get("metadata") or {}
+    if meta.get("owner"):
+        click.echo(f"  Owner:                       {meta['owner']}")
+    if meta.get("description"):
+        click.echo(f"  Description:                 {meta['description']}")
+    if meta.get("tags"):
+        click.echo(f"  Tags:                        {meta['tags']}")
+
+    click.echo(f"  Applied At:                  {entry.get('applied_at', '?')}")
+    click.echo(f"  Last Materialized At:        {entry.get('last_materialized_at', 'None')}")
+
+    features = entry.get("features", [])
+    click.echo(f"  Features ({len(features)}):")
+    for f in features:
+        expect_str = ""
+        if f.get("expect"):
+            expect_str = f" expect={f['expect']}"
+        click.echo(f"    - {f['name']} ({f.get('dtype', '?')}){expect_str}")
+
+    join_keys = entry.get("join_keys", [])
+    if join_keys:
+        click.echo(f"  Join Keys ({len(join_keys)}):")
+        for jk in join_keys:
+            click.echo(f"    - {jk['field_name']} -> {jk['referenced_group']}")

kitefs/config.py

@@ -0,0 +1,166 @@
+"""Configuration manager — loads, validates, and exposes kitefs.yaml settings."""
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from kitefs.exceptions import ConfigurationError
+
+_SUPPORTED_PROVIDERS = ("local", "aws")
+
+_ENV_OVERRIDES: dict[str, str] = {
+    "KITEFS_PROVIDER": "provider",
+    "KITEFS_STORAGE_ROOT": "storage_root",
+}
+
+_ENV_AWS_OVERRIDES: dict[str, str] = {
+    "KITEFS_AWS_S3_BUCKET": "s3_bucket",
+    "KITEFS_AWS_S3_PREFIX": "s3_prefix",
+    "KITEFS_AWS_DYNAMODB_TABLE_PREFIX": "dynamodb_table_prefix",
+}
+
+_REQUIRED_AWS_FIELDS = ("s3_bucket", "s3_prefix", "dynamodb_table_prefix")
+
+
+@dataclass(frozen=True)
+class AWSConfig:
+    """AWS-specific configuration for S3 and DynamoDB access."""
+
+    s3_bucket: str
+    s3_prefix: str
+    dynamodb_table_prefix: str
+
+
+@dataclass(frozen=True)
+class Config:
+    """Validated, immutable configuration loaded from kitefs.yaml."""
+
+    provider: str
+    project_root: Path
+    storage_root: Path
+    definitions_path: Path
+    aws: AWSConfig | None
+
+
+def load_config(project_root: Path) -> Config:
+    """Load and validate kitefs.yaml from the given project root.
+
+    Environment variables take precedence over file values.
+    All validation errors are collected before raising.
+    """
+    project_root = project_root.resolve()
+    config_path = project_root / "kitefs.yaml"
+
+    raw = _read_yaml(config_path)
+    env_origins = _apply_env_overrides(raw)
+    _validate(raw, config_path, env_origins)
+
+    storage_root = (project_root / raw["storage_root"]).resolve()
+
+    aws_config: AWSConfig | None = None
+    if raw["provider"] == "aws":
+        aws_section = raw.get("aws", {}) or {}
+        aws_config = AWSConfig(
+            s3_bucket=aws_section["s3_bucket"],
+            s3_prefix=aws_section["s3_prefix"],
+            dynamodb_table_prefix=aws_section["dynamodb_table_prefix"],
+        )
+
+    return Config(
+        provider=raw["provider"],
+        project_root=project_root,
+        storage_root=storage_root,
+        definitions_path=storage_root / "definitions",
+        aws=aws_config,
+    )
+
+
+def _read_yaml(config_path: Path) -> dict:
+    """Read and parse kitefs.yaml, raising ConfigurationError on failure."""
+    if not config_path.exists():
+        raise ConfigurationError(
+            f"No configuration file found at '{config_path}'. Run `kitefs init` to create a project."
+        )
+    try:
+        text = config_path.read_text(encoding="utf-8")
+        data = yaml.safe_load(text)
+    except yaml.YAMLError as exc:
+        raise ConfigurationError(f"Malformed YAML in '{config_path}': {exc}") from exc
+
+    if not isinstance(data, dict):
+        raise ConfigurationError(
+            f"Expected a YAML mapping in '{config_path}', got {type(data).__name__}. "
+            "The file must contain key-value pairs (e.g., 'provider: local')."
+        )
+    return data
+
+
+def _apply_env_overrides(raw: dict) -> dict[str, str]:
+    """Apply environment variable overrides to the raw config dict.
+
+    Returns a mapping of config field path (e.g. 'provider', 'aws.s3_bucket') to
+    the env var name that provided the value, for use in actionable error messages.
+    """
+    origins: dict[str, str] = {}
+
+    for env_var, field in _ENV_OVERRIDES.items():
+        value = os.environ.get(env_var)
+        if value is not None:
+            raw[field] = value
+            origins[field] = env_var
+
+    for env_var, field in _ENV_AWS_OVERRIDES.items():
+        value = os.environ.get(env_var)
+        if value is not None:
+            raw.setdefault("aws", {})
+            if raw["aws"] is None:
+                raw["aws"] = {}
+            raw["aws"][field] = value
+            origins[f"aws.{field}"] = env_var
+
+    return origins
+
+
+def _validate(raw: dict, config_path: Path, env_origins: dict[str, str]) -> None:
+    """Validate all config fields, collecting all errors before raising."""
+    errors: list[str] = []
+
+    # --- provider ---
+    provider = raw.get("provider")
+    if provider is None:
+        errors.append(f"Missing required field: 'provider' in {config_path}")
+    elif provider not in _SUPPORTED_PROVIDERS:
+        source = f" (set by environment variable {env_origins['provider']})" if "provider" in env_origins else ""
+        errors.append(f"Unsupported provider: '{provider}'{source}. Supported: {', '.join(_SUPPORTED_PROVIDERS)}")
+
+    # --- storage_root ---
+    storage_root = raw.get("storage_root")
+    if storage_root is None:
+        errors.append(f"Missing required field: 'storage_root' in {config_path}")
+    elif not isinstance(storage_root, str):
+        env_var = env_origins.get("storage_root")
+        source = f" (set by environment variable {env_var})" if env_var else ""
+        errors.append(
+            f"Invalid 'storage_root' value{source}: expected a string path, got {type(storage_root).__name__}"
+        )
+    elif not storage_root.strip():
+        env_var = env_origins.get("storage_root")
+        source = f" (set by environment variable {env_var})" if env_var else ""
+        errors.append(
+            f"Invalid 'storage_root' value{source}: path must not be empty. "
+            "Set a relative path such as './feature_store/'."
+        )
+
+    # --- AWS fields (only when provider is aws) ---
+    if provider == "aws":
+        aws_section = raw.get("aws", {}) or {}
+        missing_aws = [f for f in _REQUIRED_AWS_FIELDS if not aws_section.get(f)]
+        if missing_aws:
+            formatted = ", ".join(f"'aws.{f}'" for f in missing_aws)
+            errors.append(f"Missing required AWS fields when provider is 'aws': {formatted}")
+
+    if errors:
+        joined = "; ".join(errors)
+        raise ConfigurationError(f"Invalid configuration in '{config_path}': {joined}")

kitefs/definitions.py

@@ -0,0 +1,167 @@
+"""Definition types for KiteFS feature groups — the foundational data model."""
+
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class FeatureType(Enum):
+    """Supported data types for entity keys, event timestamps, and features."""
+
+    STRING = "STRING"
+    INTEGER = "INTEGER"
+    FLOAT = "FLOAT"
+    DATETIME = "DATETIME"
+
+
+class StorageTarget(Enum):
+    """Where a feature group's data is stored and served from."""
+
+    OFFLINE = "OFFLINE"
+    OFFLINE_AND_ONLINE = "OFFLINE_AND_ONLINE"
+
+
+class ValidationMode(Enum):
+    """How validation failures are handled at each gate (ingestion, retrieval)."""
+
+    ERROR = "ERROR"
+    FILTER = "FILTER"
+    NONE = "NONE"
+
+
+@dataclass(frozen=True)
+class Expect:
+    """Fluent builder for feature-level data expectations.
+
+    Each method returns a new Expect instance with the constraint appended,
+    preserving immutability. Constraints are stored as a tuple of dicts,
+    serializable via dataclasses.asdict().
+
+    Example::
+
+        Expect().not_null().gt(0)
+        Expect().gte(1900).lte(2030)
+        Expect().one_of(["apartment", "house", "land"])
+    """
+
+    _constraints: tuple[dict, ...] = ()
+
+    def not_null(self) -> "Expect":
+        """Require non-null values."""
+        return Expect(_constraints=(*self._constraints, {"type": "not_null"}))
+
+    def gt(self, value: int | float) -> "Expect":
+        """Require values strictly greater than *value*."""
+        return Expect(_constraints=(*self._constraints, {"type": "gt", "value": value}))
+
+    def gte(self, value: int | float) -> "Expect":
+        """Require values greater than or equal to *value*."""
+        return Expect(_constraints=(*self._constraints, {"type": "gte", "value": value}))
+
+    def lt(self, value: int | float) -> "Expect":
+        """Require values strictly less than *value*."""
+        return Expect(_constraints=(*self._constraints, {"type": "lt", "value": value}))
+
+    def lte(self, value: int | float) -> "Expect":
+        """Require values less than or equal to *value*."""
+        return Expect(_constraints=(*self._constraints, {"type": "lte", "value": value}))
+
+    def one_of(self, values: Sequence[str | int | float]) -> "Expect":
+        """Require values to be one of the given *values*. Stores a defensive copy as a tuple."""
+        return Expect(_constraints=(*self._constraints, {"type": "one_of", "values": tuple(values)}))
+
+
+@dataclass(frozen=True)
+class EntityKey:
+    """The single entity identifier for a feature group.
+
+    Structural column — always included in query results, implicitly non-null.
+    Does not support expectations (Expect is not available on this type).
+    """
+
+    name: str
+    dtype: FeatureType
+    description: str | None = None
+
+
+@dataclass(frozen=True)
+class EventTimestamp:
+    """The single event timestamp for a feature group.
+
+    Structural column — always included in query results, implicitly non-null.
+    dtype must be FeatureType.DATETIME (enforced at apply() time).
+    """
+
+    name: str
+    dtype: FeatureType
+    description: str | None = None
+
+
+@dataclass(frozen=True)
+class Feature:
+    """A single feature (data column) within a feature group.
+
+    Supports optional expectations via the Expect fluent builder for
+    data-level validation at ingestion and retrieval gates.
+    """
+
+    name: str
+    dtype: FeatureType
+    description: str | None = None
+    expect: Expect | None = None
+
+
+@dataclass(frozen=True)
+class JoinKey:
+    """Declares a join relationship to another feature group's entity key.
+
+    field_name must match a feature name in this group AND the entity key name
+    of referenced_group. Type matching is validated at apply() time.
+    """
+
+    field_name: str
+    referenced_group: str
+
+
+@dataclass(frozen=True)
+class Metadata:
+    """Optional metadata attached to a feature group.
+
+    All fields are optional and default to None.
+    """
+
+    description: str | None = None
+    owner: str | None = None
+    tags: dict[str, str] | None = None
+
+
+@dataclass(frozen=True)
+class FeatureGroup:
+    """Top-level definition for a feature group.
+
+    The foundational type that users create in their definitions/ directory.
+    FeatureGroup instances are discovered at apply() time via importlib.
+
+    features is normalised to a tuple sorted alphabetically by Feature.name
+    during construction, ensuring deterministic equality and serialisation
+    regardless of user-provided order.
+    """
+
+    name: str
+    storage_target: StorageTarget
+    entity_key: EntityKey
+    event_timestamp: EventTimestamp
+    features: list[Feature]
+    join_keys: list[JoinKey] = field(default_factory=list)
+    ingestion_validation: ValidationMode = ValidationMode.ERROR
+    offline_retrieval_validation: ValidationMode = ValidationMode.NONE
+    metadata: Metadata = field(default_factory=Metadata)
+
+    def __post_init__(self) -> None:
+        """Normalise features to a sorted tuple and join_keys to a tuple."""
+        object.__setattr__(
+            self,
+            "features",
+            tuple(sorted(self.features, key=lambda f: f.name)),
+        )
+        object.__setattr__(self, "join_keys", tuple(self.join_keys))