feedple-sdk 0.0.1__py3-none-any.whl

feedple_sdk/__about__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: 2026-present Umar Bello Kanwa <umarbellokanwa@gmail.com>
+#
+# SPDX-License-Identifier: MIT
+__version__ = "0.0.1"

feedple_sdk/__init__.py

@@ -0,0 +1,176 @@
+import asyncio
+import threading
+from sqlalchemy import Engine, text
+from typing import Optional
+from .core.schema_services import get_schema, _generate_schema_hash
+from .core.policy import Identity, PolicyEngine
+from .core.compiler import SQLCompiler
+from .core.ir_builder import build_query_from_ir
+from .core.websocket import FeedpleWebSocket
+from .core.request import FEEDPLE_WS_URL
+import logging
+import time
+
+logger = logging.getLogger("feedple_sdk")
+if not logger.handlers:
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+    logger.addHandler(handler)
+    logger.setLevel(logging.INFO)
+
+class FeedpleSDK:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        db: Engine,
+        identity: Identity,
+        auto_sync: bool = True,
+        sync_interval: int = 60,
+        # connection options
+        reconnect_enabled: bool = True,
+        max_retries: Optional[int] = None,
+        probe_before_connect: bool = False,
+    ) -> None:
+        if not api_key:
+            raise ValueError(
+                "api_key is required. Please provide a valid Feedple API key."
+            )
+        self.api_key = api_key
+        self.db = db
+        self.identity = identity
+        self.sync_interval = sync_interval
+
+        self._previous_hash: Optional[str] = None
+        self._loop = asyncio.new_event_loop()
+
+        logger.info("Feedple: initializing SDK...")
+
+        self._ws = FeedpleWebSocket(
+            ws_url=FEEDPLE_WS_URL,
+            api_key=api_key,
+        )
+        # apply connection options
+        self._ws.reconnect_enabled = reconnect_enabled
+        self._ws.max_retries = max_retries
+        self._ws.probe_before_connect = probe_before_connect
+        self._ws.on_ir_request(self._execute_ir)
+        logger.info("Feedple: WebSocket client configured")
+
+        self._verify_db_connection()
+
+        self._thread = threading.Thread(
+            target=self._run_loop,
+            daemon=True,
+            name="feedple-worker",
+        )
+        self._thread.start()
+        logger.info("Feedple: background worker thread started")
+
+        if auto_sync:
+            logger.info(f"Feedple: auto-sync enabled (interval: {self.sync_interval}s)")
+
+        logger.info("Feedple: SDK initialized successfully")
+
+    def _run_loop(self) -> None:
+        logger.info("Feedple: event loop running")
+        asyncio.set_event_loop(self._loop)
+        self._loop.run_until_complete(self._main())
+        logger.info("Feedple: event loop stopped")
+
+    async def _main(self) -> None:
+        logger.info("Feedple: starting WebSocket and sync worker...")
+        await asyncio.gather(
+            self._ws.connect(),
+            self._sync_worker(),
+        )
+
+    def _verify_db_connection(self) -> None:
+        logger.info("Feedple: verifying database connection...")
+        try:
+            with self.db.connect() as conn:
+                conn.execute(text("SELECT 1"))
+            logger.info("Feedple: database connection verified")
+        except Exception as e:
+            raise ConnectionError(f"Feedple SDK could not connect to the database: {e}")
+
+    async def sync_schema(self) -> None:
+        logger.info("Feedple: inspecting database schema...")
+
+        # run get_schema in thread executor so it doesn't block the event loop
+        # this keeps the WebSocket heartbeat running during inspection
+        loop = asyncio.get_event_loop()
+        schema = await loop.run_in_executor(
+            None,
+            lambda: get_schema(db=self.db, identity=self.identity)
+        )
+
+        logger.info(f"Feedple: schema inspected ({len(schema)} tables found)")
+
+        # check hash before sending
+        new_hash = _generate_schema_hash(schema)
+        
+        if self._previous_hash and self._previous_hash == new_hash:
+            logger.info("Feedple: schema unchanged, skipping sync")
+            return # If nothing changed, return empty (None)
+
+
+        logger.info("Feedple: sending schema via WebSocket...")
+        await self._ws.send_schema(schema)
+        self._previous_hash = new_hash  # store after successful send
+        logger.info("Feedple: schema sent successfully")
+
+    async def _sync_worker(self) -> None:
+        logger.info("Feedple: sync worker started, waiting for WebSocket authentication...")
+
+        while True:
+            try:
+                # wait for auth on every iteration, covers first connect and every reconnect
+                await self._ws.wait_until_authenticated(timeout=30)
+                logger.info("Feedple: sync worker authenticated, running sync...")
+                await self.sync_schema()
+            except asyncio.TimeoutError:
+                logger.warning("Feedple: timed out waiting for WebSocket authentication, retrying...")
+                continue
+            except Exception as e:
+                logger.warning(f"Feedple: schema sync failed: {e}")
+                self._ws._auth_event.clear()  # force re-auth wait on next iteration
+                continue
+
+            logger.info(f"Feedple: next sync in {self.sync_interval}s")
+            await asyncio.sleep(self.sync_interval)
+
+    def _build_compiler(self) -> SQLCompiler:
+        policy = PolicyEngine(self.identity)
+        return SQLCompiler(policy) # Only use for when there is direct use of SQL codes
+
+    async def _execute_ir(self, ir: dict) -> dict:
+        logger.info(f"Feedple: validating IR request...")
+        
+        policy = PolicyEngine(self.identity)
+        policy.validate_ir_access(ir=ir)
+        
+        logger.info(f"Feedple: executing IR request...")
+        stmt = build_query_from_ir(ir=ir)
+
+        logger.info(f"Feedple: executing query against database...")
+        start = time.time()
+
+        with self.db.connect() as conn:
+            result = conn.execute(stmt)  # passing the Select object directly — params stay bound, never stringified
+            rows = [dict(r._mapping) for r in result]
+
+        duration = int((time.time() - start) * 1000)
+        logger.info(f"Feedple: IR executed ({len(rows)} rows, {duration}ms)")
+
+        return {
+            "rows": rows,
+            "count": len(rows),
+            "duration_ms": duration,
+        }
+    
+    def stop(self) -> None:
+        logger.info("Feedple: stopping SDK...")
+        self._ws.stop()
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        logger.info("Feedple: SDK stopped")

feedple_sdk/core/compiler.py

@@ -0,0 +1,30 @@
+import sqlglot
+from sqlglot import exp
+from typing import List
+from .policy import PolicyEngine
+
+
+class SQLCompiler:
+    def __init__(self, policy: PolicyEngine, dialect: str = "postgres"):
+        self.policy = policy
+        self.dialect = dialect
+
+    def parse(self, sql: str) -> exp.Expression:
+        return sqlglot.parse_one(sql, read=self.dialect)
+
+    def extract_tables(self, ast: exp.Expression) -> List[str]:
+        return [t.name for t in ast.find_all(exp.Table) if t.name]
+
+    def validate_access(self, tables: List[str]) -> None:
+        if self.policy.identity.all_tables:
+            return
+        
+        for table in tables:
+            if not self.policy.can_access_table(table):
+                raise PermissionError(f"Access denied to table: {table}")
+
+    def compile(self, sql: str) -> str:
+        ast = self.parse(sql)
+        tables = self.extract_tables(ast)
+        self.validate_access(tables)
+        return ast.sql(dialect=self.dialect)

feedple_sdk/core/ir_builder.py

@@ -0,0 +1,161 @@
+# feedple_sdk/core/ir_builder.py
+from sqlalchemy import select, table, column, and_, func
+from sqlalchemy.sql import ColumnElement
+from sqlalchemy.sql.selectable import Select
+
+_ALLOWED_FUNCTIONS = {"count", "sum", "avg", "min", "max"}
+
+_OPERATOR_BUILDERS = {
+    "eq": lambda col, val: col == val,
+    "=": lambda col, val: col == val,
+    "neq": lambda col, val: col != val,
+    "!=": lambda col, val: col != val,
+    "gt": lambda col, val: col > val,
+    ">": lambda col, val: col > val,
+    "gte": lambda col, val: col >= val,
+    ">=": lambda col, val: col >= val,
+    "lt": lambda col, val: col < val,
+    "<": lambda col, val: col < val,
+    "lte": lambda col, val: col <= val,
+    "<=": lambda col, val: col <= val,
+}
+
+
+def build_query_from_ir(ir: dict) -> Select:
+    """
+    Translates the flat, clause-based SQLIntermediateRepresentation IR schema
+    into an executable SQLAlchemy Core Select construct.
+    """
+    if ir.get("operation") != "query":
+        raise NotImplementedError(f"Unsupported IR operation: {ir.get('operation')}")
+
+    # Initialize tables map starting with the primary base FROM table
+    base_table_name = ir.get("table")
+    if not base_table_name:
+        raise ValueError("IR is missing primary 'table'")
+        
+    tables_by_name = {base_table_name: table(base_table_name)}
+    for j in ir.get("joins") or []:
+        tables_by_name[j["table"]] = table(j["table"])
+
+    # 1. Build SELECT elements
+    select_cols = _build_select_columns(ir.get("fields") or [], tables_by_name)
+    stmt = select(*select_cols).select_from(tables_by_name[base_table_name])
+
+    # 2. Build JOIN clauses linearly
+    for j in ir.get("joins") or []:
+        join_tbl = tables_by_name[j["table"]]
+        
+        # Build the condition explicitly via on_left and on_right keys
+        left_col = _resolve_column(j["on_left"], tables_by_name)
+        right_col = _resolve_column(j["on_right"], tables_by_name)
+        on_clause = (left_col == right_col)
+        
+        # Determine join type
+        is_outer = j.get("join_type", "INNER").upper() in ("LEFT", "LEFT OUTER")
+        stmt = stmt.join(join_tbl, on_clause, isouter=is_outer)
+
+    # 3. Build WHERE clauses (filters)
+    filters = ir.get("filters") or []
+    if filters:
+        stmt = stmt.where(and_(*[_build_filter_condition(f, tables_by_name) for f in filters]))
+
+    # 4. Build GROUP BY clauses
+    if ir.get("group_by"):
+        group_by = ir["group_by"] if isinstance(ir["group_by"], list) else [ir["group_by"]]
+        stmt = stmt.group_by(*[_resolve_column(g, tables_by_name) for g in group_by])
+
+    # 5. Build HAVING clauses (Aggregated filters)
+    having_filters = ir.get("having") or []
+    if having_filters:
+        stmt = stmt.having(and_(*[_build_filter_condition(h, tables_by_name) for h in having_filters]))
+
+    # 6. Build ORDER BY clauses
+    if ir.get("order_by"):
+        order_by = ir["order_by"] if isinstance(ir["order_by"], list) else [ir["order_by"]]
+        stmt = stmt.order_by(*[_resolve_order_by_column(o, tables_by_name) for o in order_by])
+
+    # 7. Apply limits and offsets
+    if ir.get("limit") is not None:
+        stmt = stmt.limit(int(ir["limit"]))
+
+    if ir.get("offset") is not None:
+        stmt = stmt.offset(int(ir["offset"]))
+
+    return stmt
+
+
+def _build_select_columns(fields: list[dict], tables_by_name: dict) -> list[ColumnElement]:
+    if not fields:
+        return [tables_by_name[name] for name in tables_by_name]
+
+    result = []
+    for f in fields:
+        # Corrected key from f["field"] to f["column"]
+        col = _resolve_column(f["column"], tables_by_name)
+        expr_raw = f.get("expression")
+
+        if expr_raw:
+            # Handle case-insensitivity or distinct qualifiers (e.g., "COUNT(DISTINCT)" -> "count")
+            expr_clean = expr_raw.lower()
+            
+            if "count" in expr_clean:
+                # If the string contains distinct keywords, handle counting uniquely
+                if "distinct" in expr_clean:
+                    node = func.count(col.distinct())
+                else:
+                    node = func.count(col)
+            elif expr_clean in _ALLOWED_FUNCTIONS:
+                node = getattr(func, expr_clean)(col)
+            else:
+                raise ValueError(f"Unsupported field expression aggregation: {expr_raw}")
+        else:
+            node = col
+
+        if f.get("alias"):
+            node = node.label(f["alias"])
+
+        result.append(node)
+
+    return result
+
+
+def _build_filter_condition(f: dict, tables_by_name: dict) -> ColumnElement:
+    operator = f["operator"].lower()
+    col = _resolve_column(f["column"], tables_by_name)
+    value = f["value"]
+
+    if operator in _OPERATOR_BUILDERS:
+        return _OPERATOR_BUILDERS[operator](col, value)
+
+    if operator in ("in", "in_"):
+        return col.in_(value)
+
+    if operator in ("like", "ilike"):
+        if operator == "ilike":
+            return col.ilike(value)
+        return col.like(value)
+
+    raise ValueError(f"Unsupported filter operator: {operator}")
+
+
+def _resolve_column(field: str, tables_by_name: dict) -> ColumnElement:
+    if "." in field:
+        table_name, col_name = field.split(".", 1)
+        if table_name not in tables_by_name:
+            raise ValueError(f"Unknown table reference in field definition: {field}")
+        return column(col_name, _selectable=tables_by_name[table_name])
+    
+    # Fallback to structural context if no prefix was generated
+    return column(field)
+
+
+def _resolve_order_by_column(order_string: str, tables_by_name: dict) -> ColumnElement:
+    """Splits string parameters like 'articles.created_at DESC' into clean columns with modifiers."""
+    parts = order_string.strip().split()
+    col_ref = parts[0]
+    col_element = _resolve_column(col_ref, tables_by_name)
+    
+    if len(parts) > 1 and parts[1].upper() == "DESC":
+        return col_element.desc()
+    return col_element.asc()

feedple_sdk/core/json.py

@@ -0,0 +1,22 @@
+import json
+from uuid import UUID
+from datetime import datetime, date
+
+class SafeJSONEncoder(json.JSONEncoder):
+    def default(self, obj):
+        # 1. Handle UUIDs safely
+        if isinstance(obj, UUID):
+            return str(obj)
+            
+        # 2. Handle datetimes cleanly (strips internal timezone circular dependencies)
+        if isinstance(obj, (datetime, date)):
+            return obj.isoformat()
+            
+        # 3. Handle SQLAlchemy Row/Mapping objects if passed directly
+        if hasattr(obj, "_asdict"):
+            return obj._asdict()
+        if hasattr(obj, "__dict__"):
+            # Fallback to prevent serialization crashes on complex custom objects
+            return {k: v for k, v in obj.__dict__.items() if not k.startswith('_')}
+            
+        return super().default(obj)

feedple_sdk/core/policy.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+
+@dataclass
+class Identity:
+    name: Optional[str]
+    allowed_tables: List[str] = field(default_factory=list)
+    all_tables: bool = False
+
+class PolicyEngine:
+    def __init__(self, identity: Identity):
+        self.identity = identity
+
+    # ---------------- RBAC ----------------
+    def can_access_table(self, table: str) -> bool:
+        if self.identity.all_tables:
+            return True
+        return table in self.identity.allowed_tables
+    
+
+    def validate_ir_access(self, ir: dict) -> None:
+        if self.identity.all_tables:
+            return
+
+        tables = [ir["table"]] + [j["table"] for j in (ir.get("joins") or [])]
+        for table in tables:
+            if not self.can_access_table(table):
+                raise PermissionError(f"Access denied to table: {table}")

feedple_sdk/core/request.py

@@ -0,0 +1,50 @@
+import json
+from typing import Dict, Any, Optional
+import urllib.request
+import urllib.error
+
+DEV_ENV = True
+# Note: the server sets a FastAPI `root_path` of '/api/py'. To avoid duplicating
+# that prefix we target the internal versioned API paths here (e.g. '/api/v1/...').
+FEEDPLE_API_BASE_URL = ("localhost:8000" if DEV_ENV else "https://feedple-ai.onrender.com") + "/api/v1"
+
+SCHEMA_SYNC_URL = f"{FEEDPLE_API_BASE_URL}/schema/sync"
+FEEDPLE_WS_URL = ("ws://localhost:8000" if DEV_ENV else "wss://feedple-ai.onrender.com") + "/api/v1/tenants/ws"
+
+class SchemaSyncError(Exception):
+    pass
+
+def _post_json_schema(
+    payload: Dict[str, Any],
+    headers: Optional[Dict[str, str]] = None,
+    timeout: int = 10,
+) -> Dict[str, Any]:
+    """
+    Minimal HTTP POST using stdlib only. No external dependencies.
+    """
+    data = json.dumps(payload).encode("utf-8")
+
+    request_headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+    if headers:
+        request_headers.update(headers)
+
+    req = urllib.request.Request(
+        url=SCHEMA_SYNC_URL,
+        data=data,
+        headers=request_headers,
+        method="POST",
+    )
+
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as res:
+            return json.loads(res.read().decode("utf-8"))
+    except urllib.error.HTTPError as e:
+        body = e.read().decode("utf-8")
+        raise SchemaSyncError(f"HTTP {e.code}: {body}")
+    except Exception as e:
+        raise SchemaSyncError(f"Network error: {str(e)}")
+

feedple_sdk/core/schema_services.py

@@ -0,0 +1,151 @@
+import json
+import hashlib
+import time
+from typing import Dict, Any, Optional
+from sqlalchemy import inspect, Engine
+from .request import _post_json_schema, SchemaSyncError
+from feedple_sdk.core.policy import Identity
+from sqlalchemy.exc import NoSuchTableError
+
+def _normalize_schema(schema: Dict[str, Any]) -> str:
+    """
+    Normalize schema into a deterministic JSON string for hashing.
+    sort_keys ensures consistent ordering across environments.
+    """
+    return json.dumps(schema, sort_keys=True, separators=(",", ":"))
+
+
+def _generate_schema_hash(schema: Dict[str, Any]) -> str:
+    """
+    SHA-256 hash of the normalized schema for change detection.
+    """
+    return hashlib.sha256(_normalize_schema(schema).encode("utf-8")).hexdigest()
+
+
+def send_schema_to_api(
+    *,
+    schema: Dict[str, Any],
+    previous_hash: Optional[str] = None,
+    api_key: Optional[str] = None,
+    version: Optional[int] = None,
+    retry: int = 3,
+    backoff_seconds: float = 0.5,
+) -> Dict[str, Any]:
+    """
+    Sends database schema to your central API.
+
+    - Skips sync if schema has not changed in the central API
+    - Attaches Authorization header if api_key is provided
+    - Retries with exponential backoff on failure
+    """
+    schema_hash = _generate_schema_hash(schema)
+
+    if previous_hash and previous_hash == schema_hash:
+        return {
+            "status": "skipped",
+            "reason": "no_change",
+            "schema_hash": schema_hash,
+        }
+
+    payload = {
+        "schema": schema,
+        "schema_version": version,
+        "timestamp": int(time.time()),
+    }
+    
+    headers = {}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    last_error = None
+
+    for attempt in range(retry):
+        try:
+            response = _post_json_schema(payload, headers=headers)
+            return {
+                "status": "success",
+                "schema_hash": schema_hash,
+                "response": response,
+            }
+        except SchemaSyncError as e:
+            last_error = str(e)
+            if attempt < retry - 1:
+                time.sleep(backoff_seconds * (2 ** attempt))
+
+    raise SchemaSyncError(f"Failed after {retry} attempts: {last_error}")
+
+
+def should_sync_schema(old_schema: Dict[str, Any], new_schema: Dict[str, Any]) -> bool:
+    """
+    Lightweight pre-check before making an API call.
+    Returns True if schemas differ and a sync is needed.
+    """
+    return _generate_schema_hash(old_schema) != _generate_schema_hash(new_schema)
+
+def get_schema(db: Engine, identity: Identity) -> dict:
+    with db.connect() as conn:
+        inspector = inspect(conn)
+        tables = inspector.get_table_names() if identity.all_tables else identity.allowed_tables
+        result = {}
+
+        for table in tables:
+            try:
+                pk = inspector.get_pk_constraint(table)
+                fks = inspector.get_foreign_keys(table)
+                indexes = inspector.get_indexes(table)
+                unique = inspector.get_unique_constraints(table)
+                columns = inspector.get_columns(table)
+
+                result[table] = {
+                    "columns": [
+                        {
+                            "name": col["name"],
+                            "type": str(col["type"]),
+                            "nullable": col["nullable"],
+                            "default": str(col["default"]) if col["default"] is not None else None,
+                        }
+                        for col in columns
+                    ],
+                    "primary_key": pk.get("constrained_columns", []),
+                    "foreign_keys": [
+                        {
+                            "columns": fk["constrained_columns"],
+                            "references_table": fk["referred_table"],
+                            "references_columns": fk["referred_columns"],
+                        }
+                        for fk in fks
+                    ],
+                    "indexes": [
+                        {
+                            "name": idx["name"],
+                            "columns": idx["column_names"],
+                            "unique": idx["unique"],
+                        }
+                        for idx in indexes
+                    ],
+                    "unique_constraints": [
+                        {
+                            "name": uc["name"],
+                            "columns": uc["column_names"],
+                        }
+                        for uc in unique
+                    ],
+                }
+            except NoSuchTableError:
+                raise ValueError(f"Table '{table}' does not exist in the database")
+
+    return result
+
+SENSITIVE_COLUMNS = {"password", "token", "secret", "hash", "salt", "ssn", "credit_card"}
+
+def filter_sensitive_columns(schema: dict) -> dict:
+    for table in schema.values():
+        table["columns"] = [
+            col for col in table["columns"]
+            if col["name"].lower() not in SENSITIVE_COLUMNS
+        ]
+    return schema
+
+# def on_migration_complete():
+#     schema = introspect_db() #db
+#     sdk.sync(schema)