rtbot-sql 0.1.14__cp314-cp314-win_amd64.whl

rtbot_sql/__init__.py

@@ -0,0 +1,52 @@
+"""rtbot-sql Python runtime."""
+
+from .catalog import ColumnDef, InMemoryCatalog, StreamSchema, TableSchema, ViewMeta
+from .compiler import compile_sql, native, validate_sql
+from .formatter import normalize_time_column
+from .pipeline_runner import InputMessage, LocalPipelineRunner, PipelineOutput
+from .runtime import InsertResult, RtBotSql, SqlError
+from .stream_store import InMemoryStreamStore, Message
+
+
+_default = RtBotSql()
+
+
+def execute(sql: str):
+  return _default.execute(sql)
+
+
+def insert_dataframe(stream_name: str, dataframe):
+  return _default.insert_dataframe(stream_name, dataframe)
+
+
+def show_graph(view_name: str):
+  return _default.show_graph(view_name)
+
+
+def explain(sql: str):
+  return _default.explain(sql)
+
+
+__all__ = [
+    "ColumnDef",
+    "StreamSchema",
+    "ViewMeta",
+    "TableSchema",
+    "InMemoryCatalog",
+    "InMemoryStreamStore",
+    "Message",
+    "InputMessage",
+    "PipelineOutput",
+    "LocalPipelineRunner",
+    "InsertResult",
+    "RtBotSql",
+    "SqlError",
+    "normalize_time_column",
+    "compile_sql",
+    "validate_sql",
+    "native",
+    "execute",
+    "insert_dataframe",
+    "show_graph",
+    "explain",
+]

rtbot_sql/catalog.py

@@ -0,0 +1,247 @@
+"""In-memory catalog for the Python runtime."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Sequence
+
+from .compiler import native
+
+
+@dataclass
+class ColumnDef:
+  name: str
+  index: int
+
+
+@dataclass
+class StreamSchema:
+  name: str
+  columns: List[ColumnDef] = field(default_factory=list)
+
+
+@dataclass
+class ViewMeta:
+  name: str
+  entity_type: Any
+  view_type: Any
+  field_map: Dict[str, int] = field(default_factory=dict)
+  source_streams: List[str] = field(default_factory=list)
+  program_json: str = ""
+  output_stream: str = ""
+  per_key_prefix: str = ""
+  known_keys: List[float] = field(default_factory=list)
+  key_index: int = -1
+
+
+@dataclass
+class TableSchema:
+  name: str
+  columns: List[ColumnDef] = field(default_factory=list)
+  changelog_stream: str = ""
+
+
+def _to_column_def(obj: Any) -> ColumnDef:
+  if isinstance(obj, ColumnDef):
+    return ColumnDef(obj.name, obj.index)
+  if isinstance(obj, native.ColumnDef):
+    return ColumnDef(obj.name, int(obj.index))
+  if isinstance(obj, dict):
+    return ColumnDef(str(obj["name"]), int(obj["index"]))
+  raise TypeError(f"Unsupported column definition: {type(obj)!r}")
+
+
+def _to_stream_schema(name: str, obj: Any) -> StreamSchema:
+  if isinstance(obj, StreamSchema):
+    return StreamSchema(name=name or obj.name, columns=[_to_column_def(c) for c in obj.columns])
+  if isinstance(obj, native.StreamSchema):
+    return StreamSchema(name=name or obj.name, columns=[_to_column_def(c) for c in obj.columns])
+  if isinstance(obj, dict):
+    cols = [_to_column_def(c) for c in obj.get("columns", [])]
+    return StreamSchema(name=name or str(obj.get("name", name)), columns=cols)
+  raise TypeError(f"Unsupported stream schema: {type(obj)!r}")
+
+
+def _to_view_meta(name: str, obj: Any) -> ViewMeta:
+  if isinstance(obj, ViewMeta):
+    return ViewMeta(
+        name=name or obj.name,
+        entity_type=obj.entity_type,
+        view_type=obj.view_type,
+        field_map=dict(obj.field_map),
+        source_streams=list(obj.source_streams),
+        program_json=obj.program_json,
+        output_stream=obj.output_stream,
+        per_key_prefix=obj.per_key_prefix,
+        known_keys=list(obj.known_keys),
+        key_index=int(obj.key_index),
+    )
+  if isinstance(obj, native.ViewMeta):
+    return ViewMeta(
+        name=name or obj.name,
+        entity_type=obj.entity_type,
+        view_type=obj.view_type,
+        field_map=dict(obj.field_map),
+        source_streams=list(obj.source_streams),
+        program_json=obj.program_json,
+        output_stream=obj.output_stream,
+        per_key_prefix=obj.per_key_prefix,
+        known_keys=list(obj.known_keys),
+        key_index=int(obj.key_index),
+    )
+  if isinstance(obj, dict):
+    return ViewMeta(
+        name=name or str(obj.get("name", name)),
+        entity_type=obj.get("entity_type", native.EntityType.VIEW),
+        view_type=obj.get("view_type", native.ViewType.SCALAR),
+        field_map=dict(obj.get("field_map", {})),
+        source_streams=list(obj.get("source_streams", [])),
+        program_json=str(obj.get("program_json", "")),
+        output_stream=str(obj.get("output_stream", "")),
+        per_key_prefix=str(obj.get("per_key_prefix", "")),
+        known_keys=[float(v) for v in obj.get("known_keys", [])],
+        key_index=int(obj.get("key_index", -1)),
+    )
+  raise TypeError(f"Unsupported view metadata: {type(obj)!r}")
+
+
+def _to_table_schema(name: str, obj: Any) -> TableSchema:
+  if isinstance(obj, TableSchema):
+    return TableSchema(
+        name=name or obj.name,
+        columns=[_to_column_def(c) for c in obj.columns],
+        changelog_stream=obj.changelog_stream,
+    )
+  if isinstance(obj, native.TableSchema):
+    return TableSchema(
+        name=name or obj.name,
+        columns=[_to_column_def(c) for c in obj.columns],
+        changelog_stream=obj.changelog_stream,
+    )
+  if isinstance(obj, dict):
+    return TableSchema(
+        name=name or str(obj.get("name", name)),
+        columns=[_to_column_def(c) for c in obj.get("columns", [])],
+        changelog_stream=str(obj.get("changelog_stream", "")),
+    )
+  raise TypeError(f"Unsupported table schema: {type(obj)!r}")
+
+
+def _column_to_native(col: ColumnDef) -> native.ColumnDef:
+  return native.ColumnDef(col.name, int(col.index))
+
+
+def _stream_to_native(schema: StreamSchema) -> native.StreamSchema:
+  out = native.StreamSchema()
+  out.name = schema.name
+  out.columns = [_column_to_native(c) for c in schema.columns]
+  return out
+
+
+def _view_to_native(meta: ViewMeta) -> native.ViewMeta:
+  out = native.ViewMeta()
+  out.name = meta.name
+  out.entity_type = meta.entity_type
+  out.view_type = meta.view_type
+  out.field_map = dict(meta.field_map)
+  out.source_streams = list(meta.source_streams)
+  out.program_json = meta.program_json
+  out.output_stream = meta.output_stream
+  out.per_key_prefix = meta.per_key_prefix
+  out.known_keys = [float(v) for v in meta.known_keys]
+  out.key_index = int(meta.key_index)
+  return out
+
+
+def _table_to_native(schema: TableSchema) -> native.TableSchema:
+  out = native.TableSchema()
+  out.name = schema.name
+  out.columns = [_column_to_native(c) for c in schema.columns]
+  out.changelog_stream = schema.changelog_stream
+  return out
+
+
+class InMemoryCatalog:
+  def __init__(self) -> None:
+    self._streams: Dict[str, StreamSchema] = {}
+    self._views: Dict[str, ViewMeta] = {}
+    self._tables: Dict[str, TableSchema] = {}
+
+  def register_stream(self, name: str, schema: Any) -> None:
+    self._streams[name] = _to_stream_schema(name, schema)
+
+  def register_view(self, name: str, meta: Any) -> None:
+    self._views[name] = _to_view_meta(name, meta)
+
+  def register_table(self, name: str, schema: Any) -> None:
+    self._tables[name] = _to_table_schema(name, schema)
+
+  def lookup_stream(self, name: str) -> Optional[StreamSchema]:
+    return self._streams.get(name)
+
+  def lookup_view(self, name: str) -> Optional[ViewMeta]:
+    return self._views.get(name)
+
+  def lookup_table(self, name: str) -> Optional[TableSchema]:
+    return self._tables.get(name)
+
+  def resolve_entity(self, name: str) -> Optional[Any]:
+    if name in self._streams:
+      return native.EntityType.STREAM
+    if name in self._views:
+      return self._views[name].entity_type
+    if name in self._tables:
+      return native.EntityType.TABLE
+    return None
+
+  def snapshot(self) -> native.CatalogSnapshot:
+    snap = native.CatalogSnapshot()
+    snap.streams = {name: _stream_to_native(schema) for name, schema in self._streams.items()}
+    snap.views = {name: _view_to_native(meta) for name, meta in self._views.items()}
+    snap.tables = {name: _table_to_native(schema) for name, schema in self._tables.items()}
+    return snap
+
+  def add_key(self, view_name: str, key: float) -> None:
+    view = self._views.get(view_name)
+    if view is None:
+      return
+    key_value = float(key)
+    if key_value not in view.known_keys:
+      view.known_keys.append(key_value)
+
+  def get_known_keys(self, view_name: str) -> set:
+    view = self._views.get(view_name)
+    if view is None:
+      return set()
+    return set(view.known_keys)
+
+  def drop_stream(self, name: str) -> None:
+    self._streams.pop(name, None)
+
+  def drop_view(self, name: str) -> None:
+    self._views.pop(name, None)
+
+  def drop_table(self, name: str) -> None:
+    self._tables.pop(name, None)
+
+  def drop(self, name: str) -> bool:
+    removed = False
+    if name in self._streams:
+      self._streams.pop(name, None)
+      removed = True
+    if name in self._views:
+      self._views.pop(name, None)
+      removed = True
+    if name in self._tables:
+      self._tables.pop(name, None)
+      removed = True
+    return removed
+
+  def streams(self) -> Dict[str, StreamSchema]:
+    return dict(self._streams)
+
+  def views(self) -> Dict[str, ViewMeta]:
+    return dict(self._views)
+
+  def tables(self) -> Dict[str, TableSchema]:
+    return dict(self._tables)

rtbot_sql/compiler.py

@@ -0,0 +1,45 @@
+"""Thin wrappers over the native rtbot-sql extension."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from importlib.machinery import ExtensionFileLoader, ModuleSpec
+from pathlib import Path
+from typing import Any
+
+try:
+  from . import _rtbot_sql_native as native  # type: ignore[attr-defined]
+except ImportError:  # pragma: no cover - fallback for Bazel runfiles layout
+  try:
+    import _rtbot_sql_native as native  # type: ignore
+  except ImportError:
+    if not sys.platform.startswith("win"):
+      raise
+    ext_path = Path(__file__).with_name("_rtbot_sql_native.so")
+    if not ext_path.exists():
+      raise
+    loader = ExtensionFileLoader("rtbot_sql._rtbot_sql_native", str(ext_path))
+    spec = ModuleSpec("rtbot_sql._rtbot_sql_native", loader, origin=str(ext_path))
+    native = importlib.util.module_from_spec(spec)
+    loader.exec_module(native)
+
+
+def compile_sql(sql: str, catalog_snapshot: Any) -> Any:
+  return native.compile_sql(sql, catalog_snapshot)
+
+
+def validate_sql(sql: str) -> dict:
+  return native.validate_sql(sql)
+
+
+def compile_select_to_program(sql: str, catalog_snapshot: Any) -> Any:
+  """Compile a SELECT by wrapping it in CREATE MATERIALIZED VIEW.
+
+  compile_sql() only emits program_json for tier-3 SELECT queries. For tier-2,
+  this wrapper forces graph compilation and lets the Python runtime execute the
+  query locally through the native pipeline runner.
+  """
+  select_sql = sql.strip().rstrip(";")
+  wrapped_sql = f"CREATE MATERIALIZED VIEW __rtbot_sql_tmp AS {select_sql}"
+  return native.compile_sql(wrapped_sql, catalog_snapshot)

rtbot_sql/formatter.py

@@ -0,0 +1,73 @@
+"""Helpers to format runtime outputs."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional, Sequence
+
+try:
+  import pandas as pd
+  _HAS_PANDAS = True
+except Exception:  # pragma: no cover - optional dependency
+  pd = None
+  _HAS_PANDAS = False
+
+
+def ordered_columns(field_map: Dict[str, int]) -> List[str]:
+  return [name for name, _ in sorted(field_map.items(), key=lambda item: item[1])]
+
+
+def project_rows(rows: Iterable[Sequence[float]], field_map: Dict[str, int]) -> List[List[float]]:
+  ordered = sorted(field_map.items(), key=lambda item: item[1])
+  indices = [index for _, index in ordered]
+
+  projected: List[List[float]] = []
+  for row in rows:
+    projected.append([
+        float(row[idx]) if 0 <= idx < len(row) else 0.0
+        for idx in indices
+    ])
+  return projected
+
+
+def format_rows(
+    rows: Iterable[Sequence[float]],
+    field_map: Dict[str, int],
+    timestamps: Optional[List[int]] = None,
+    time_values: Optional[List[Any]] = None,
+    time_column: str = "time",
+    as_dataframe: bool = True,
+):
+  columns = ordered_columns(field_map)
+  normalized = project_rows(rows, field_map)
+
+  if as_dataframe and _HAS_PANDAS:
+    df = pd.DataFrame(normalized, columns=columns)
+    if timestamps is not None:
+      values = time_values if time_values is not None else timestamps
+      df.insert(0, time_column, values)
+    # Auto-normalize: rename _time -> time and convert numeric timestamps.
+    if '_time' in df.columns and 'time' not in df.columns:
+      df = df.rename(columns={'_time': 'time'})
+      if pd.api.types.is_numeric_dtype(df['time']):
+        df['time'] = pd.to_datetime(df['time'], unit='ms')
+    return df
+
+  result = {
+      "columns": columns,
+      "rows": normalized,
+  }
+  if timestamps is not None:
+    result["timestamps"] = timestamps
+    result["time_column"] = time_column
+  return result
+
+
+def normalize_time_column(df, default_unit='ms'):
+  if 'time' in df.columns:
+    return df
+  if '_time' in df.columns:
+    out = df.rename(columns={'_time': 'time'}).copy()
+    if pd.api.types.is_numeric_dtype(out['time']):
+      out['time'] = pd.to_datetime(out['time'], unit=default_unit)
+    return out
+  raise KeyError("Expected a time column in SQL result ('time' or '_time')")

rtbot_sql/jupyter.py

@@ -0,0 +1,42 @@
+"""Jupyter display helpers."""
+
+from __future__ import annotations
+
+import json
+from typing import Dict
+
+
+def _safe_node_id(node_id: str) -> str:
+  return node_id.replace(":", "_").replace("-", "_")
+
+
+def program_json_to_mermaid(program_json: str) -> str:
+  parsed = json.loads(program_json)
+  lines = ["flowchart LR"]
+
+  operators = parsed.get("operators", [])
+  for op in operators:
+    op_id = str(op.get("id", "unknown"))
+    op_type = str(op.get("type", "Operator"))
+    lines.append(f'  {_safe_node_id(op_id)}["{op_id}\\n{op_type}"]')
+
+  for connection in parsed.get("connections", []):
+    src = _safe_node_id(str(connection.get("from", "")))
+    dst = _safe_node_id(str(connection.get("to", "")))
+    from_port = str(connection.get("fromPort", "o1"))
+    to_port = str(connection.get("toPort", "i1"))
+    lines.append(f'  {src} -->|"{from_port}->{to_port}"| {dst}')
+
+  return "\n".join(lines)
+
+
+def show_graph(program_json: str):
+  mermaid = program_json_to_mermaid(program_json)
+
+  try:
+    from IPython.display import Markdown, display
+
+    display(Markdown(f"```mermaid\n{mermaid}\n```"))
+    return None
+  except Exception:  # pragma: no cover - used outside notebooks
+    return mermaid

rtbot_sql/pipeline_runner.py

@@ -0,0 +1,98 @@
+"""Local pipeline runner backed by native rtbot::Program execution."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List
+
+from .compiler import native
+from .stream_store import Message
+
+
+@dataclass
+class InputMessage:
+  timestamp: int
+  values: List[float]
+  port: str = "i1"
+
+
+@dataclass
+class PipelineOutput:
+  timestamp: int
+  values: List[float]
+  operator_id: str = ""
+  port: str = ""
+
+
+def _normalize_input(item: Any) -> InputMessage:
+  if isinstance(item, InputMessage):
+    return InputMessage(int(item.timestamp), [float(v) for v in item.values], item.port)
+  if isinstance(item, Message):
+    return InputMessage(int(item.timestamp), [float(v) for v in item.values], "i1")
+  if isinstance(item, dict):
+    return InputMessage(
+        int(item["timestamp"]),
+        [float(v) for v in item["values"]],
+        str(item.get("port", "i1")),
+    )
+  if isinstance(item, (tuple, list)) and len(item) >= 2:
+    port = "i1" if len(item) < 3 else str(item[2])
+    return InputMessage(int(item[0]), [float(v) for v in item[1]], port)
+  raise TypeError(f"Unsupported input message: {type(item)!r}")
+
+
+class LocalPipelineRunner:
+  def __init__(self) -> None:
+    self._pipelines: Dict[str, native.NativePipeline] = {}
+    self._counter = 0
+
+  def _next_id(self) -> str:
+    self._counter += 1
+    return f"pipeline_{self._counter}"
+
+  def deploy(self, program_json: str, _input_streams: Any = None, _output_config: Any = None) -> str:
+    pipeline_id = self._next_id()
+    self._pipelines[pipeline_id] = native.NativePipeline(program_json)
+    return pipeline_id
+
+  def run_once(self, program_json: str, input_messages: Iterable[Any]) -> List[PipelineOutput]:
+    pipeline = native.NativePipeline(program_json)
+    return self._run_pipeline(pipeline, input_messages)
+
+  def feed(
+      self,
+      pipeline_id: str,
+      timestamp: int,
+      values: List[float],
+      port: str = "i1",
+  ) -> List[PipelineOutput]:
+    pipeline = self._pipelines[pipeline_id]
+    native_outputs = pipeline.feed(int(timestamp), [float(v) for v in values], str(port))
+    return [
+        PipelineOutput(
+            timestamp=int(out.timestamp),
+            values=[float(v) for v in out.values],
+            operator_id=str(out.operator_id),
+            port=str(out.port),
+        )
+        for out in native_outputs
+    ]
+
+  def destroy(self, pipeline_id: str) -> None:
+    self._pipelines.pop(pipeline_id, None)
+
+  def _run_pipeline(self, pipeline: native.NativePipeline, input_messages: Iterable[Any]) -> List[PipelineOutput]:
+    outputs: List[PipelineOutput] = []
+    for raw in input_messages:
+      msg = _normalize_input(raw)
+      native_outputs = pipeline.feed(msg.timestamp, msg.values, msg.port)
+      outputs.extend(
+          PipelineOutput(
+              timestamp=int(out.timestamp),
+              values=[float(v) for v in out.values],
+              operator_id=str(out.operator_id),
+              port=str(out.port),
+          )
+          for out in native_outputs
+      )
+    return outputs

rtbot_sql/runtime.py

@@ -0,0 +1,592 @@
+"""User-facing Python runtime API for rtbot-sql."""
+
+from __future__ import annotations
+
+import json
+import re
+import time
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+from .catalog import InMemoryCatalog, StreamSchema, ViewMeta
+from .compiler import compile_select_to_program, compile_sql, native
+from .formatter import format_rows
+from .jupyter import show_graph as display_graph
+from .pipeline_runner import LocalPipelineRunner
+from .stream_store import InMemoryStreamStore
+
+
+class SqlError(RuntimeError):
+  def __init__(self, errors: Sequence[Any]):
+    self.errors = list(errors)
+    messages = [getattr(err, "message", str(err)) for err in self.errors]
+    super().__init__("; ".join(messages) if messages else "SQL error")
+
+
+@dataclass
+class InsertResult:
+  rows: int
+  elapsed_s: float
+  rows_per_sec: float
+
+  def __repr__(self):
+    return f"Inserted {self.rows:,} rows in {self.elapsed_s:.2f}s ({self.rows_per_sec:,.0f} rows/sec)"
+
+
+def _enum_name(value: Any) -> str:
+  if hasattr(value, "name"):
+    return str(value.name)
+  text = str(value)
+  if "." in text:
+    return text.split(".")[-1]
+  return text
+
+
+def _extract_limit(sql: str) -> Optional[int]:
+  match = re.search(r"\bLIMIT\s+(\d+)\b", sql, flags=re.IGNORECASE)
+  if not match:
+    return None
+  return int(match.group(1))
+
+
+class RtBotSql:
+  def __init__(self) -> None:
+    self._catalog = InMemoryCatalog()
+    self._store = InMemoryStreamStore()
+    self._runner = LocalPipelineRunner()
+
+    self._view_pipelines: Dict[str, str] = {}
+    self._view_port_maps: Dict[str, Dict[str, str]] = {}
+    self._dependencies: Dict[str, set] = defaultdict(set)
+
+    self._last_timestamp = int(time.time() * 1000)
+    self._time_column_name = "time"
+    self._time_unit_override: Optional[str] = None
+    self._time_formatter: Optional[Callable[[List[int]], List[Any]]] = None
+
+  def _next_timestamp(self) -> int:
+    now = int(time.time() * 1000)
+    if now <= self._last_timestamp:
+      self._last_timestamp += 1
+    else:
+      self._last_timestamp = now
+    return self._last_timestamp
+
+  @staticmethod
+  def _normalize_sql(sql: str) -> str:
+    out = sql.strip()
+    out = re.sub(r"^\s*CREATE\s+STREAM\b", "CREATE TABLE", out, flags=re.IGNORECASE)
+    out = re.sub(r"^\s*DROP\s+STREAM\b", "DROP TABLE", out, flags=re.IGNORECASE)
+    return out
+
+  def execute(self, sql: str):
+    normalized = self._normalize_sql(sql)
+
+    result = compile_sql(normalized, self._catalog.snapshot())
+    if result.has_errors():
+      raise SqlError(result.errors)
+
+    statement = result.statement_type
+    if statement == native.StatementType.CREATE_STREAM:
+      self._handle_create_stream(result)
+      return None
+
+    if statement == native.StatementType.INSERT:
+      self._handle_insert(result)
+      return None
+
+    if statement in (
+        native.StatementType.CREATE_VIEW,
+        native.StatementType.CREATE_MATERIALIZED_VIEW,
+    ):
+      self._handle_create_view(result, statement)
+      return None
+
+    if statement == native.StatementType.DROP:
+      self._handle_drop(result)
+      return None
+
+    if statement == native.StatementType.SELECT:
+      return self._handle_select(normalized, result)
+
+    raise SqlError([f"Unsupported statement type: {statement}"])
+
+  def _handle_create_stream(self, result: native.CompilationResult) -> None:
+    self._catalog.register_stream(result.entity_name, result.stream_schema)
+
+  def _handle_insert(self, result: native.CompilationResult) -> None:
+    stream_name = result.entity_name
+    schema = self._catalog.lookup_stream(stream_name)
+    if schema is None:
+      raise SqlError([f"Unknown stream: {stream_name}"])
+
+    payload = [float(v) for v in result.insert_payload]
+    if schema.columns and len(payload) != len(schema.columns):
+      raise SqlError([
+          f"INSERT payload length mismatch for {stream_name}: expected {len(schema.columns)}, got {len(payload)}"
+      ])
+
+    self._append_and_propagate(stream_name, self._next_timestamp(), payload)
+
+  def _handle_create_view(
+      self,
+      result: native.CompilationResult,
+      statement: Any,
+  ) -> None:
+    name = result.entity_name
+    materialized = statement == native.StatementType.CREATE_MATERIALIZED_VIEW
+
+    view_meta = ViewMeta(
+        name=name,
+        entity_type=(
+            native.EntityType.MATERIALIZED_VIEW
+            if materialized
+            else native.EntityType.VIEW
+        ),
+        view_type=result.view_type,
+        field_map=dict(result.field_map),
+        source_streams=list(result.source_streams),
+        program_json=result.program_json,
+        output_stream=name,
+        per_key_prefix=f"{name}:key:",
+        known_keys=[],
+        key_index=int(result.key_index),
+    )
+
+    self._catalog.register_view(name, view_meta)
+    for source in view_meta.source_streams:
+      self._dependencies[source].add(name)
+
+    pipeline_id = self._runner.deploy(result.program_json, view_meta.source_streams, {"output_stream": name})
+    self._view_pipelines[name] = pipeline_id
+    source_port_map: Dict[str, str] = {}
+    for i, source in enumerate(view_meta.source_streams):
+      source_port_map[source] = f"i{i + 1}"
+    self._view_port_maps[name] = source_port_map
+
+    # Backfill the new view from already-known source data.
+    if self._uses_snapshot_sync(name):
+      source_rank = {source: i for i, source in enumerate(view_meta.source_streams)}
+      events = []
+      for source in view_meta.source_streams:
+        for msg in self._store.read(source):
+          events.append((msg.timestamp, source_rank[source], source, msg.values))
+      events.sort(key=lambda item: (item[0], item[1]))
+
+      history: Dict[str, List[Any]] = {source: [] for source in view_meta.source_streams}
+      for timestamp, _, trigger_source, trigger_values in events:
+        history[trigger_source].append((timestamp, list(trigger_values)))
+
+        snapshot: Dict[str, List[float]] = {}
+        ready = True
+        for source in view_meta.source_streams:
+          if source == trigger_source:
+            snapshot[source] = list(trigger_values)
+            continue
+
+          prior = None
+          for prior_timestamp, prior_values in reversed(history[source]):
+            if prior_timestamp <= timestamp:
+              prior = prior_values
+              break
+
+          if prior is None:
+            ready = False
+            break
+
+          snapshot[source] = list(prior)
+
+        if not ready:
+          continue
+
+        outputs = []
+        for source in view_meta.source_streams:
+          port = source_port_map.get(source, "i1")
+          outputs.extend(self._runner.feed(pipeline_id, timestamp, snapshot[source], port=port))
+
+        for out in outputs:
+          self._append_and_propagate(name, out.timestamp, out.values)
+    else:
+      for i, source in enumerate(view_meta.source_streams):
+        port = f"i{i + 1}"
+        for msg in self._store.read(source):
+          outputs = self._runner.feed(pipeline_id, msg.timestamp, msg.values, port=port)
+          for out in outputs:
+            self._append_and_propagate(name, out.timestamp, out.values)
+
+  def _handle_drop(self, result: native.CompilationResult) -> None:
+    name = result.drop_entity_name
+
+    if name in self._view_pipelines:
+      self._runner.destroy(self._view_pipelines[name])
+      self._view_pipelines.pop(name, None)
+    self._view_port_maps.pop(name, None)
+
+    self._catalog.drop(name)
+    self._store.clear(name)
+
+    for dependents in self._dependencies.values():
+      dependents.discard(name)
+
+    self._dependencies.pop(name, None)
+
+  def _uses_snapshot_sync(self, dependent: str) -> bool:
+    view = self._catalog.lookup_view(dependent)
+    if view is None or len(view.source_streams) <= 1:
+      return False
+
+    for source in view.source_streams:
+      if self._catalog.lookup_stream(source) is None:
+        return False
+
+    return True
+
+  def _feed_dependent_pipeline(
+      self,
+      dependent: str,
+      stream_name: str,
+      timestamp: int,
+      values: List[float],
+  ) -> List[Any]:
+    pipeline_id = self._view_pipelines.get(dependent)
+    if pipeline_id is None:
+      return []
+
+    port_map = self._view_port_maps.get(dependent, {})
+
+    if not self._uses_snapshot_sync(dependent):
+      port = port_map.get(stream_name, "i1")
+      return self._runner.feed(pipeline_id, timestamp, values, port=port)
+
+    view = self._catalog.lookup_view(dependent)
+    if view is None:
+      return []
+
+    snapshot: Dict[str, List[float]] = {}
+    for source in view.source_streams:
+      if source == stream_name:
+        snapshot[source] = list(values)
+        continue
+
+      prior = self._store.read_latest_before(source, timestamp)
+      if prior is None:
+        return []
+      snapshot[source] = list(prior.values)
+
+    outputs = []
+    for source in view.source_streams:
+      port = port_map.get(source, "i1")
+      outputs.extend(self._runner.feed(pipeline_id, timestamp, snapshot[source], port=port))
+    return outputs
+
+  def _append_and_propagate(self, stream_name: str, timestamp: int, values: List[float]) -> None:
+    self._store.append(stream_name, timestamp, values)
+
+    view = self._catalog.lookup_view(stream_name)
+    if (
+        view is not None
+        and view.view_type == native.ViewType.KEYED
+        and 0 <= view.key_index < len(values)
+    ):
+      self._catalog.add_key(stream_name, float(values[view.key_index]))
+
+    for dependent in list(self._dependencies.get(stream_name, set())):
+      outputs = self._feed_dependent_pipeline(dependent, stream_name, timestamp, values)
+      for out in outputs:
+        self._append_and_propagate(dependent, out.timestamp, out.values)
+
+  def _project_messages(
+      self,
+      messages: Sequence[Any],
+      field_map: Dict[str, int],
+  ) -> List[List[float]]:
+    ordered = sorted(field_map.items(), key=lambda item: item[1])
+    indices = [index for _, index in ordered]
+
+    rows: List[List[float]] = []
+    for msg in messages:
+      values = list(msg.values)
+      rows.append([
+          float(values[idx]) if 0 <= idx < len(values) else 0.0
+          for idx in indices
+      ])
+    return rows
+
+  def _resolve_field_map(self, source: str, field_map: Dict[str, int]) -> Dict[str, int]:
+    if field_map:
+      return dict(field_map)
+
+    source_view = self._catalog.lookup_view(source)
+    if source_view is not None:
+      return dict(source_view.field_map)
+
+    source_stream = self._catalog.lookup_stream(source)
+    if source_stream is not None:
+      return {column.name: column.index for column in source_stream.columns}
+
+    source_table = self._catalog.lookup_table(source)
+    if source_table is not None:
+      return {column.name: column.index for column in source_table.columns}
+
+    return {}
+
+  def _format_time_values(self, timestamps: List[int]) -> List[Any]:
+    if self._time_formatter is not None:
+      return list(self._time_formatter(list(timestamps)))
+
+    if not timestamps:
+      return []
+
+    try:
+      import pandas as pd  # type: ignore
+    except Exception:
+      return [int(ts) for ts in timestamps]
+
+    unit = self._time_unit_override
+    if unit is None:
+      max_abs = max(abs(int(ts)) for ts in timestamps)
+      if max_abs >= 10**17:
+        unit = "ns"
+      elif max_abs >= 10**14:
+        unit = "us"
+      elif max_abs >= 10**11:
+        unit = "ms"
+      else:
+        unit = "s"
+
+    return list(pd.to_datetime(timestamps, unit=unit))
+
+  def configure_time_format(
+      self,
+      *,
+      unit: Optional[str] = None,
+      formatter: Optional[Callable[[List[int]], List[Any]]] = None,
+      column_name: str = "time",
+  ) -> None:
+    if unit is not None and unit not in {"s", "ms", "us", "ns"}:
+      raise SqlError([f"Invalid time unit '{unit}'. Expected one of: s, ms, us, ns"])
+    self._time_unit_override = unit
+    self._time_formatter = formatter
+    self._time_column_name = column_name
+
+  def _execute_tier1(self, sql: str, result: native.CompilationResult):
+    source = result.source_streams[0] if result.source_streams else ""
+    if not source:
+      return format_rows([], {}, timestamps=[])
+
+    effective_field_map = self._resolve_field_map(source, dict(result.field_map))
+    limit = _extract_limit(sql)
+    source_view = self._catalog.lookup_view(source)
+
+    if (
+        source_view is not None
+        and source_view.view_type == native.ViewType.KEYED
+        and " where " not in sql.lower()
+    ):
+      known_keys = sorted(self._catalog.get_known_keys(source))
+      latest_by_key = self._store.read_latest_per_key(
+          source,
+          known_keys,
+          source_view.key_index,
+      )
+      messages = [latest_by_key[k] for k in known_keys if k in latest_by_key]
+      if limit is not None:
+        messages = messages[:limit]
+    else:
+      if limit is None:
+        messages = self._store.read(source)
+      else:
+        messages = self._store.read_latest(source, limit)
+
+    rows = self._project_messages(messages, effective_field_map)
+    timestamps = [msg.timestamp for msg in messages]
+    return format_rows(
+        rows,
+        effective_field_map,
+        timestamps=timestamps,
+        time_values=self._format_time_values(timestamps),
+        time_column=self._time_column_name,
+    )
+
+  def _execute_with_pipeline(self, sql: str, result: native.CompilationResult):
+    original_field_map = dict(result.field_map)
+    runtime_result = result
+    if not runtime_result.program_json:
+      runtime_result = compile_select_to_program(sql, self._catalog.snapshot())
+      if runtime_result.has_errors():
+        raise SqlError(runtime_result.errors)
+
+    effective_field_map = dict(runtime_result.field_map)
+    if not effective_field_map:
+      effective_field_map = original_field_map
+    if not effective_field_map and runtime_result.source_streams:
+      effective_field_map = self._resolve_field_map(runtime_result.source_streams[0], {})
+
+    if not runtime_result.source_streams:
+      return format_rows(
+          [],
+          effective_field_map,
+          timestamps=[],
+          time_values=[],
+          time_column=self._time_column_name,
+      )
+
+    inputs: List[Tuple[int, List[float], str]] = []
+    uses_snapshot_sync = (
+        len(runtime_result.source_streams) > 1
+        and all(self._catalog.lookup_stream(source) is not None for source in runtime_result.source_streams)
+    )
+
+    if uses_snapshot_sync:
+      source_rank = {source: i for i, source in enumerate(runtime_result.source_streams)}
+      events: List[Tuple[int, int, str, List[float]]] = []
+      for source in runtime_result.source_streams:
+        for msg in self._store.read(source):
+          events.append((msg.timestamp, source_rank[source], source, list(msg.values)))
+      events.sort(key=lambda item: (item[0], item[1]))
+
+      latest_by_source: Dict[str, List[float]] = {}
+      for timestamp, _, trigger_source, trigger_values in events:
+        latest_by_source[trigger_source] = list(trigger_values)
+        if len(latest_by_source) < len(runtime_result.source_streams):
+          continue
+
+        for i, source in enumerate(runtime_result.source_streams):
+          inputs.append((timestamp, list(latest_by_source[source]), f"i{i + 1}"))
+    else:
+      for i, source in enumerate(runtime_result.source_streams):
+        port = f"i{i + 1}"
+        inputs.extend((msg.timestamp, msg.values, port) for msg in self._store.read(source))
+
+    outputs = self._runner.run_once(runtime_result.program_json, inputs)
+    timestamps = [out.timestamp for out in outputs]
+    rows = [list(out.values) for out in outputs]
+
+    limit = _extract_limit(sql)
+    if limit is not None:
+      rows = rows[:limit]
+      timestamps = timestamps[:limit]
+
+    return format_rows(
+        rows,
+        effective_field_map,
+        timestamps=timestamps,
+        time_values=self._format_time_values(timestamps),
+        time_column=self._time_column_name,
+    )
+
+  def _handle_select(self, sql: str, result: native.CompilationResult):
+    if result.select_tier == native.SelectTier.TIER1_READ:
+      return self._execute_tier1(sql, result)
+    return self._execute_with_pipeline(sql, result)
+
+  def explain(self, sql: str) -> Dict[str, Any]:
+    normalized = self._normalize_sql(sql)
+    result = compile_sql(normalized, self._catalog.snapshot())
+
+    output = {
+        "statement_type": _enum_name(result.statement_type),
+        "select_tier": _enum_name(result.select_tier),
+        "entity_name": result.entity_name,
+        "source_streams": list(result.source_streams),
+        "field_map": dict(result.field_map),
+        "view_type": _enum_name(result.view_type),
+        "key_index": int(result.key_index),
+        "program_json": None,
+        "errors": [
+            {
+                "message": err.message,
+                "line": int(err.line),
+                "column": int(err.column),
+            }
+            for err in result.errors
+        ],
+    }
+
+    if result.program_json:
+      output["program_json"] = json.loads(result.program_json)
+
+    return output
+
+  def show_graph(self, view_name: str):
+    view = self._catalog.lookup_view(view_name)
+    if view is None:
+      raise SqlError([f"Unknown view: {view_name}"])
+    return display_graph(view.program_json)
+
+  def debug(self, sql: str) -> Dict[str, Any]:
+    return self.explain(sql)
+
+  def export_for_redis(self, view_name: str) -> str:
+    view = self._catalog.lookup_view(view_name)
+    if view is None:
+      raise SqlError([f"Unknown view: {view_name}"])
+
+    kind = (
+        "MATERIALIZED VIEW"
+        if view.entity_type == native.EntityType.MATERIALIZED_VIEW
+        else "VIEW"
+    )
+    sources = ", ".join(view.source_streams)
+    return (
+        "-- Original SQL text is not persisted in the Python runtime catalog.\n"
+        f"CREATE {kind} {view_name} AS /* sources: {sources} */;"
+    )
+
+  def insert_dataframe(
+      self,
+      stream_name: str,
+      dataframe: Any,
+      column_map: Optional[Dict[str, str]] = None,
+  ) -> InsertResult:
+    schema = self._catalog.lookup_stream(stream_name)
+    if schema is None:
+      raise SqlError([f"Unknown stream: {stream_name}"])
+
+    cmap = column_map or {}
+
+    time_col = None
+    if hasattr(dataframe, "columns"):
+      for candidate in ("time", "timestamp", "ts"):
+        if candidate in dataframe.columns:
+          time_col = candidate
+          break
+
+    row_count = 0
+    t0 = time.perf_counter()
+
+    if hasattr(dataframe, "iterrows"):
+      for _, row in dataframe.iterrows():
+        timestamp = int(row[time_col]) if time_col else self._next_timestamp()
+        values = [float(row[cmap.get(col.name, col.name)]) for col in schema.columns]
+        self._append_and_propagate(stream_name, timestamp, values)
+        row_count += 1
+    else:
+      for row in dataframe:
+        if isinstance(row, dict):
+          if time_col:
+            timestamp = int(row.get(time_col, self._next_timestamp()))
+          else:
+            dict_time_col = None
+            for candidate in ("time", "timestamp", "ts"):
+              if candidate in row:
+                dict_time_col = candidate
+                break
+            timestamp = int(row[dict_time_col]) if dict_time_col else self._next_timestamp()
+          values = [float(row[cmap.get(col.name, col.name)]) for col in schema.columns]
+        else:
+          timestamp = self._next_timestamp()
+          values = [float(v) for v in row]
+        self._append_and_propagate(stream_name, timestamp, values)
+        row_count += 1
+
+    elapsed = time.perf_counter() - t0
+    rps = row_count / elapsed if elapsed > 0 else float('inf')
+    return InsertResult(rows=row_count, elapsed_s=elapsed, rows_per_sec=rps)
+
+  def get_catalog(self) -> InMemoryCatalog:
+    return self._catalog
+
+  def get_store(self) -> InMemoryStreamStore:
+    return self._store

rtbot_sql/stream_store.py

@@ -0,0 +1,100 @@
+"""In-memory stream message store."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional
+
+
+@dataclass
+class Message:
+  timestamp: int
+  values: List[float]
+
+
+def _coerce_message(item: Any) -> Message:
+  if isinstance(item, Message):
+    return Message(int(item.timestamp), [float(v) for v in item.values])
+  if isinstance(item, dict):
+    return Message(int(item["timestamp"]), [float(v) for v in item["values"]])
+  if isinstance(item, (tuple, list)) and len(item) == 2:
+    return Message(int(item[0]), [float(v) for v in item[1]])
+  raise TypeError(f"Unsupported message payload: {type(item)!r}")
+
+
+class InMemoryStreamStore:
+  def __init__(self) -> None:
+    self._streams: Dict[str, List[Message]] = {}
+
+  def append(self, stream_name: str, timestamp: int, vector: List[float]) -> None:
+    messages = self._streams.setdefault(stream_name, [])
+    messages.append(Message(int(timestamp), [float(v) for v in vector]))
+
+  def append_batch(self, stream_name: str, messages: Iterable[Any]) -> None:
+    for item in messages:
+      msg = _coerce_message(item)
+      self.append(stream_name, msg.timestamp, msg.values)
+
+  def read(self, stream_name: str) -> List[Message]:
+    return list(self._streams.get(stream_name, []))
+
+  def read_latest(self, stream_name: str, count: int) -> List[Message]:
+    if count <= 0:
+      return []
+    data = self._streams.get(stream_name, [])
+    if count >= len(data):
+      return list(data)
+    return list(data[-count:])
+
+  def read_latest_before(self, stream_name: str, timestamp: int) -> Optional[Message]:
+    data = self._streams.get(stream_name, [])
+    target = int(timestamp)
+    for msg in reversed(data):
+      if msg.timestamp <= target:
+        return Message(msg.timestamp, list(msg.values))
+    return None
+
+  def read_range(
+      self,
+      stream_name: str,
+      from_time: Optional[int],
+      to_time: Optional[int],
+      limit: Optional[int] = None,
+  ) -> List[Message]:
+    result: List[Message] = []
+    for msg in self._streams.get(stream_name, []):
+      if from_time is not None and msg.timestamp < from_time:
+        continue
+      if to_time is not None and msg.timestamp > to_time:
+        continue
+      result.append(msg)
+      if limit is not None and limit > 0 and len(result) >= limit:
+        break
+    return result
+
+  def read_latest_per_key(
+      self,
+      view_name: str,
+      known_keys: Iterable[float],
+      key_index: int,
+  ) -> Dict[float, Message]:
+    wanted = {float(v) for v in known_keys}
+    out: Dict[float, Message] = {}
+
+    if not wanted:
+      return out
+
+    for msg in reversed(self._streams.get(view_name, [])):
+      if key_index < 0 or key_index >= len(msg.values):
+        continue
+      key = float(msg.values[key_index])
+      if key not in wanted or key in out:
+        continue
+      out[key] = msg
+      if len(out) == len(wanted):
+        break
+
+    return out
+
+  def clear(self, stream_name: str) -> None:
+    self._streams.pop(stream_name, None)

rtbot_sql-0.1.14.dist-info/METADATA

@@ -0,0 +1,65 @@
+Metadata-Version: 2.1
+Name: rtbot-sql
+Author: Katenaria
+Author-email: services@katenaria.com
+Home-page: https://rtbot.dev
+License: Proprietary
+Description-Content-Type: text/markdown
+Summary: Real-time SQL computation engine for streaming numerical data
+Project-URL: Documentation, https://rtbot.dev/docs
+Project-URL: Source, https://github.com/rtbot-dev/rtbot-sql
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.13
+Version: 0.1.14
+
+# RtBot SQL
+
+Real-time SQL computation engine for streaming numerical data. Write SQL to define streaming pipelines — RtBot compiles them into high-performance C++ programs that process data incrementally, one message at a time.
+
+## Quick start
+
+```python
+from rtbot_sql import RtBotSql
+
+sql = RtBotSql()
+sql.execute("CREATE STREAM sensors (temperature DOUBLE, pressure DOUBLE)")
+sql.execute("""
+  CREATE MATERIALIZED VIEW alerts AS
+    SELECT temperature, pressure,
+           MOVING_AVERAGE(temperature, 50) AS avg_temp,
+           MOVING_STD(temperature, 50) AS std_temp
+    FROM sensors
+    WHERE ABS(temperature - MOVING_AVERAGE(temperature, 50)) > 2 * MOVING_STD(temperature, 50)
+""")
+
+import pandas as pd
+df = pd.read_csv("sensor_history.csv")
+sql.insert_dataframe("sensors", df)
+results = sql.execute("SELECT * FROM alerts")
+```
+
+## Features
+
+- **One language, all stages** — the SQL you write in a notebook is the same SQL that runs in production
+- **Incremental execution** — each new data point updates pipeline state in constant time
+- **Deterministic** — same input always produces the same output, regardless of timing
+- **High performance** — native C++ engine with Python bindings via pybind11
+
+## Documentation
+
+- [Getting Started](https://rtbot.dev/docs/user-guide/getting-started)
+- [Python Quickstart](https://rtbot.dev/docs/user-guide/quickstart-python)
+- [SQL Reference](https://rtbot.dev/docs/reference/rtbot-sql/overview)
+
+## License
+
+Proprietary. See [rtbot.dev](https://rtbot.dev) for licensing options.
+

rtbot_sql-0.1.14.dist-info/RECORD

@@ -0,0 +1,12 @@
+rtbot_sql/_rtbot_sql_native.so,sha256=LwP12rSJOE6uLgjUajKHYfosUawhpo8OlnCAfn0lgo8,6317568
+rtbot_sql/__init__.py,sha256=7A40ISiExSGggL6uMctNPVmH6wtjjBpY2IGKUhOuiLI,1209
+rtbot_sql/catalog.py,sha256=GgPTJD_LIyhrOMWxualw1LOELOIs8yBleo_sfqQjH14,8170
+rtbot_sql/compiler.py,sha256=hCgOtob2uFBqbpCR7ZPnVxmJ1jBlZvfNd9Qe94HskyE,1643
+rtbot_sql/formatter.py,sha256=34T4-4hGVIRUmH1ut7xD7U-6miMT4Kbn54_Qj1H2Rcc,2340
+rtbot_sql/jupyter.py,sha256=KwiqlK62U-A-a9nHUWPYsRTQJkjuzDIjrUC-ldhBHOs,1242
+rtbot_sql/pipeline_runner.py,sha256=TKvQkk0uCE4Qr7CkjVmQUWQaI8o0t0qexyeQU6Pj3lg,3163
+rtbot_sql/runtime.py,sha256=KJCMXLRoZwJgwLHzTwFH0sspHRyN3jSK10m5pd-mipQ,20436
+rtbot_sql/stream_store.py,sha256=0XNgnNSgKKe1uJ7WgY5jL_t18dgdYAl6VsxXQq1wN-4,3136
+rtbot_sql-0.1.14.dist-info/WHEEL,sha256=N9E0YzaAkxKF8USZMMWbkNIXLjcNcbLNitkJXYC8GwI,101
+rtbot_sql-0.1.14.dist-info/METADATA,sha256=uDCfH0eIFZ0WX7mZPMAi3k2kYMK47xcbaWlrFGNE-pU,2374
+rtbot_sql-0.1.14.dist-info/RECORD,,

rtbot_sql-0.1.14.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: bazel-wheelmaker 1.0
+Root-Is-Purelib: false
+Tag: cp314-cp314-win_amd64