starfish-events 3.0.0a42__tar.gz

starfish_events-3.0.0a42/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: starfish-events
+Version: 3.0.0a42
+Summary: Starfish server plugin that intercepts JSON event-batch pushes and encodes them as Parquet on S3 (Python mirror of @drakkar.software/starfish-events)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Requires-Dist: starfish-server
+Requires-Dist: pyarrow>=24.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: httpx>=0.25.0; extra == "dev"
+Requires-Dist: fastapi>=0.100; extra == "dev"
+Requires-Dist: pyarrow>=24.0; extra == "dev"

starfish_events-3.0.0a42/README.md

@@ -0,0 +1,88 @@
+# starfish-events
+
+Starfish server plugin that intercepts JSON event-batch pushes and encodes them as
+[Apache Parquet](https://parquet.apache.org/) files written directly to the object
+store (typically S3).
+
+Mirrors [`@drakkar.software/starfish-events`](../ts/events) (TypeScript) with
+identical Parquet encoding — both are locked to the same test vectors.
+
+## Install
+
+```bash
+pip install starfish-events
+```
+
+## How it works
+
+1. Register a JSON-typed collection (`allowed_mime_types: ["application/json"]`) in
+   your `SyncConfig`.
+2. Attach `create_events_server_plugin` to `create_sync_router`.
+3. Each push to that collection is intercepted: the JSON event batch is encoded as
+   Parquet and written via `store.put_bytes`. The default JSON document write is
+   short-circuited — **no JSON is persisted alongside the Parquet**.
+
+One Parquet file is written per push (one file per batch). DuckDB's
+`read_parquet('s3://…/**/*.parquet')` glob treats all files under the prefix as one
+logical dataset.
+
+## Usage
+
+```python
+from starfish_server.storage.s3 import S3ObjectStore
+from starfish_server.router.sync_router import create_sync_router
+from starfish_protocol.config import SyncConfig, CollectionConfig, SyncRouterOptions
+from starfish_events import create_events_server_plugin
+
+store = S3ObjectStore(...)
+
+plugin = create_events_server_plugin(
+    store=store,
+    collection="events",
+    storage_path="events/{app}/{batchId}",
+)
+
+router = create_sync_router(
+    SyncRouterOptions(
+        store=store,
+        config=SyncConfig(
+            version=1,
+            collections=[
+                CollectionConfig(
+                    name="events",
+                    storage_path="events/{app}/{batchId}",
+                    read_roles=["admin"],
+                    write_roles=["public"],
+                    encryption="none",
+                    allowed_mime_types=["application/json"],  # JSON-typed, not Parquet
+                    max_body_bytes=8_000_000,
+                )
+            ],
+        ),
+        plugins=[plugin],
+    )
+)
+```
+
+## API
+
+### `create_events_server_plugin(*, store, collection, storage_path) -> ServerPlugin`
+
+| Parameter | Type | Description |
+|---|---|---|
+| `store` | `AbstractObjectStore` | Object store with `put_bytes`. Pass the same instance as `create_sync_router`. |
+| `collection` | `str` | Name of the collection to intercept (e.g. `"events"`). |
+| `storage_path` | `str` | Storage-path template for the Parquet key. Supports `{param}` placeholders from the push URL. The `.parquet` extension is appended automatically if absent. |
+
+The plugin adds `received_at` (ISO-8601 UTC) to every event row before encoding.
+
+## Querying with DuckDB
+
+```sql
+SELECT event_type, COUNT(*) AS n
+FROM read_parquet('s3://my-bucket/events/myapp/**/*.parquet')
+GROUP BY event_type
+ORDER BY n DESC;
+```
+
+See [Analytics — Events & Parquet](/analytics/events) for the full guide.

starfish_events-3.0.0a42/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "starfish-events"
+version = "3.0.0a42"
+description = "Starfish server plugin that intercepts JSON event-batch pushes and encodes them as Parquet on S3 (Python mirror of @drakkar.software/starfish-events)"
+requires-python = ">=3.11"
+dependencies = [
+  "starfish-protocol",
+  "starfish-server",
+  # pyarrow is the ONLY heavy dep and lives here exclusively (the Python analog
+  # of hyparquet-writer living only in packages/ts/events).  >=10.0 covers the
+  # full stable API surface used here (pa.table, pa.array, pq.write_table,
+  # io.BytesIO); tested against 21.x.
+  "pyarrow>=24.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "pytest>=7.0",
+  "pytest-asyncio>=0.21",
+  "httpx>=0.25.0",
+  "fastapi>=0.100",
+  # pyarrow is a runtime dep but we list it here too so `uv sync --extra dev`
+  # installs it for the round-trip decode assertions in tests.
+  "pyarrow>=24.0",
+]
+
+[tool.uv.sources]
+starfish-protocol = { path = "../protocol", editable = true }
+starfish-server = { path = "../server", editable = true }
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]

starfish_events-3.0.0a42/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

starfish_events-3.0.0a42/starfish_events/__init__.py

@@ -0,0 +1,33 @@
+"""``starfish-events`` — Starfish server plugin (Python) that intercepts JSON
+event-batch pushes from the SunGlasses adapter and encodes them as Parquet
+files on S3.
+
+Python mirror of ``@drakkar.software/starfish-events`` (``packages/ts/events``).
+
+Public surface
+--------------
+- :func:`create_events_server_plugin` — factory that returns a
+  :class:`~starfish_protocol.plugins.ServerPlugin` whose ``intercept_push``
+  hook encodes event batches as Parquet.
+- :func:`encode_parquet` — low-level encoder (exposed for testing and direct
+  use).
+- :data:`COLUMNS` — the fixed 10-column tuple that forms the Parquet schema.
+"""
+
+from starfish_events.encode import COLUMNS, encode_parquet
+
+
+def __getattr__(name: str):
+    """Lazy import of :func:`create_events_server_plugin`.
+
+    Keeps ``starfish_server`` off the hot path for callers that only import the
+    types or the encoder — mirrors the pattern used in ``starfish_projection``.
+    """
+    if name == "create_events_server_plugin":
+        from starfish_events.plugin import create_events_server_plugin as _f
+
+        return _f
+    raise AttributeError(f"module 'starfish_events' has no attribute {name!r}")
+
+
+__all__ = ["COLUMNS", "encode_parquet", "create_events_server_plugin"]

starfish_events-3.0.0a42/starfish_events/encode.py

@@ -0,0 +1,63 @@
+"""Parquet encoding for SunGlasses event rows.
+
+Column schema matches the ``EventRow`` produced by ``apps/ingest-server/src/schema.ts``
+so DuckDB queries are identical regardless of which backend delivered the data
+(HTTP ingest server vs. this Starfish-events plugin).
+
+All columns are STRING (VARCHAR in Parquet terms).  ``UNCOMPRESSED`` codec
+avoids a native/WASM compressor dependency and matches the TypeScript side
+(``packages/ts/events/src/encode.ts``).
+
+Privacy: the caller is responsible for never logging the contents of
+``distinct_id``, ``properties``, or ``context``.  This module stores whatever
+it receives opaquely.
+"""
+from __future__ import annotations
+
+import io
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+# Fixed column order — mirrors apps/ingest-server EventRow and
+# packages/ts/events/src/encode.ts.  The order is part of the schema contract
+# consumed by DuckDB; do not reorder.
+COLUMNS: tuple[str, ...] = (
+    "event_type",
+    "event",
+    "distinct_id",
+    "anonymous_id",
+    "ts",
+    "message_id",
+    "properties",
+    "context",
+    "dt",
+    "received_at",
+)
+
+
+def encode_parquet(rows: list[dict]) -> bytes:
+    """Encode a list of flat event-row dicts as an Apache Parquet byte string.
+
+    All values are coerced to :class:`str` so integers, floats, ``None``, and
+    other JSON types are stored as strings rather than causing a type error.
+    Missing keys default to ``""`` (empty string).
+
+    :param rows: List of row dicts; each row is one SunGlasses event as
+        flattened by the adapter's ``toStarfishRow`` (or an equivalent mapper).
+    :returns: Raw Parquet bytes starting (and ending) with the ``PAR1`` magic.
+    :raises Exception: Propagated from pyarrow on encoding failure; the plugin
+        wraps this as an HTTP 500 so the client retries.
+    """
+    table = pa.table(
+        {
+            col: pa.array(
+                [str(row[col]) if row.get(col) is not None else "" for row in rows],
+                type=pa.string(),
+            )
+            for col in COLUMNS
+        }
+    )
+    buf = io.BytesIO()
+    pq.write_table(table, buf, compression="none")
+    return buf.getvalue()

starfish_events-3.0.0a42/starfish_events/plugin.py

@@ -0,0 +1,190 @@
+"""Starfish server plugin: intercepts JSON event-batch pushes and encodes them
+as Parquet files written directly to the object store (typically S3).
+
+How it works
+------------
+1. Register a JSON-typed collection (``allowed_mime_types: ["application/json"]``)
+   with public write access.
+2. Attach this plugin to the sync router.
+3. Each push to that collection is intercepted here; the JSON event batch is
+   encoded as Parquet and stored via ``store.put_bytes``, short-circuiting the
+   default JSON document write so no JSON is persisted alongside the Parquet.
+
+Collection requirement
+-----------------------
+The intercepted collection **must** be JSON-typed — ``intercept_push`` only
+receives a populated ``raw_body`` for JSON collections (see
+``route_builder.py:847``).  A binary (Parquet-typed) collection would yield an
+empty body.
+
+One file per batch
+------------------
+Parquet's column-footer format makes in-place append impractical.  Each
+``send()`` call from the SunGlasses adapter writes a unique path (batchId in
+the storage-path template). DuckDB's
+``read_parquet('s3://…/**/*.parquet')`` glob treats all files under the prefix
+as one logical dataset.
+
+Privacy
+-------
+Never log ``distinct_id``, ``properties``, or ``context``.  Log counts only.
+These values ride as opaque strings into Parquet.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING
+
+from starfish_protocol.constants import PARQUET_MIME_TYPE
+from starfish_protocol.plugins import PushHookContext, PushHookResult, ServerPlugin
+from starfish_server.router.route_builder import resolve_document_key
+
+from starfish_events.encode import encode_parquet
+
+if TYPE_CHECKING:
+    from starfish_server.storage.base import AbstractObjectStore
+
+_log = logging.getLogger(__name__)
+
+
+def _supports_binary(store: object) -> bool:
+    """Return True when *store* has overridden ``put_bytes`` (supports binary writes).
+
+    The base :class:`~starfish_server.storage.base.AbstractObjectStore` raises
+    ``NotImplementedError`` on ``put_bytes``.  Any concrete implementation
+    (``S3ObjectStore``, ``FilesystemObjectStore``, ``MemoryObjectStore``) overrides
+    it and returns ``True`` here.
+    """
+    from starfish_server.storage.base import AbstractObjectStore as _Base
+
+    return type(store).put_bytes is not _Base.put_bytes
+
+
+def create_events_server_plugin(
+    *,
+    store: "AbstractObjectStore",
+    collection: str,
+    storage_path: str,
+) -> ServerPlugin:
+    """Build a :class:`~starfish_protocol.plugins.ServerPlugin` that encodes
+    SunGlasses event batches as Parquet and writes them to the object store.
+
+    :param store: Object store the plugin writes Parquet files to.  Must
+        implement ``put_bytes`` (e.g. ``S3ObjectStore`` or ``MemoryObjectStore``).
+        Pass the **same** store instance you pass to ``create_sync_router``.
+    :param collection: Name of the collection to intercept.  Must match the
+        ``name`` field in the ``SyncConfig.collections`` entry.
+        Example: ``"events"``.
+    :param storage_path: Storage-path template for the output Parquet key.
+        Supports ``{param}`` placeholders resolved from the push URL's path
+        params.  Example: ``"events/{app}/{batchId}"`` →
+        ``"events/myapp/<uuid>"``.  The plugin appends ``.parquet`` when the
+        resolved key does not already end with it.
+    :raises TypeError: When *store* does not override ``put_bytes``.
+
+    Example wiring::
+
+        from starfish_server.storage.s3 import S3ObjectStore
+        from starfish_events import create_events_server_plugin
+
+        store = S3ObjectStore(...)
+        plugin = create_events_server_plugin(
+            store=store,
+            collection="events",
+            storage_path="events/{app}/{batchId}",
+        )
+        router = create_sync_router(
+            SyncRouterOptions(
+                store=store,
+                config=SyncConfig(
+                    version=1,
+                    collections=[
+                        CollectionConfig(
+                            name="events",
+                            storage_path="events/{app}/{batchId}",
+                            read_roles=["admin"],
+                            write_roles=["public"],
+                            encryption="none",
+                            allowed_mime_types=["application/json"],  # JSON-typed!
+                            max_body_bytes=8_000_000,
+                        )
+                    ],
+                ),
+                plugins=[plugin],
+            )
+        )
+    """
+    if not _supports_binary(store):
+        raise TypeError(
+            "[starfish-events] the provided store does not implement put_bytes "
+            "(binary writes). Use S3ObjectStore, FilesystemObjectStore, or "
+            "MemoryObjectStore."
+        )
+
+    async def _intercept_push(ctx: PushHookContext) -> PushHookResult:
+        # Only intercept the configured collection; let everything else proceed.
+        if ctx.collection != collection:
+            return PushHookResult(action="proceed")
+
+        # Parse the push envelope: { data: { events: [...] }, baseHash }
+        try:
+            envelope = json.loads(ctx.raw_body)
+            data = (envelope.get("data") or {}) if isinstance(envelope, dict) else {}
+            raw = data.get("events") if isinstance(data, dict) else None
+            events: list[dict] = raw if isinstance(raw, list) else []
+        except (json.JSONDecodeError, AttributeError, ValueError):
+            return PushHookResult(
+                action="reject",
+                status=400,
+                error="Invalid JSON body — expected { data: { events: [...] }, baseHash }",
+            )
+
+        # Stamp ingest time server-side. Never log event contents.
+        dt_now = datetime.now(timezone.utc)
+        received_at = (
+            dt_now.strftime("%Y-%m-%dT%H:%M:%S.")
+            + f"{dt_now.microsecond // 1000:03d}Z"
+        )
+        rows = [{**e, "received_at": received_at} for e in events]
+
+        # Encode to Parquet.
+        try:
+            parquet_bytes = encode_parquet(rows)
+        except Exception as exc:
+            _log.error("[starfish-events] Parquet encoding failed: %s", exc)
+            return PushHookResult(action="reject", status=500, error="Parquet encoding failed")
+
+        # Resolve the output key from the storagePath template + URL params.
+        key = resolve_document_key(storage_path, dict(ctx.params))
+        if not key.endswith(".parquet"):
+            key += ".parquet"
+
+        # Write to the object store. On failure return a clean HTTP 500 so the
+        # SunGlasses adapter sees a non-2xx and the SDK requeues the batch
+        # (at-least-once delivery guarantee).
+        try:
+            await store.put_bytes(key, parquet_bytes, content_type=PARQUET_MIME_TYPE)
+        except Exception as exc:
+            _log.error("[starfish-events] put_bytes failed for key %r: %s", key, exc)
+            return PushHookResult(action="reject", status=500, error="Storage write failed")
+
+        # Compute SHA-256 to match the binary push response format.
+        sha = hashlib.sha256(parquet_bytes).hexdigest()
+
+        # Privacy: log only counts, never event contents.
+        _log.info(
+            "[starfish-events] wrote %d event(s) → %s (%d bytes)",
+            len(events),
+            key,
+            len(parquet_bytes),
+        )
+
+        return PushHookResult(action="respond", status=200, body={"hash": sha})
+
+    return ServerPlugin(name="starfish-events", intercept_push=_intercept_push)
+
+
+__all__ = ["create_events_server_plugin"]

starfish_events-3.0.0a42/starfish_events.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: starfish-events
+Version: 3.0.0a42
+Summary: Starfish server plugin that intercepts JSON event-batch pushes and encodes them as Parquet on S3 (Python mirror of @drakkar.software/starfish-events)
+Requires-Python: >=3.11
+Requires-Dist: starfish-protocol
+Requires-Dist: starfish-server
+Requires-Dist: pyarrow>=24.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21; extra == "dev"
+Requires-Dist: httpx>=0.25.0; extra == "dev"
+Requires-Dist: fastapi>=0.100; extra == "dev"
+Requires-Dist: pyarrow>=24.0; extra == "dev"

starfish_events-3.0.0a42/starfish_events.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+README.md
+pyproject.toml
+starfish_events/__init__.py
+starfish_events/encode.py
+starfish_events/plugin.py
+starfish_events.egg-info/PKG-INFO
+starfish_events.egg-info/SOURCES.txt
+starfish_events.egg-info/dependency_links.txt
+starfish_events.egg-info/requires.txt
+starfish_events.egg-info/top_level.txt
+tests/test_cross_language.py
+tests/test_encode.py
+tests/test_integration.py

starfish_events-3.0.0a42/starfish_events.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

starfish_events-3.0.0a42/starfish_events.egg-info/requires.txt

@@ -0,0 +1,10 @@
+starfish-protocol
+starfish-server
+pyarrow>=24.0
+
+[dev]
+pytest>=7.0
+pytest-asyncio>=0.21
+httpx>=0.25.0
+fastapi>=0.100
+pyarrow>=24.0

starfish_events-3.0.0a42/starfish_events.egg-info/top_level.txt

@@ -0,0 +1 @@
+starfish_events