flatseek 0.1.0__py3-none-any.whl

flatseek/__init__.py

@@ -0,0 +1,41 @@
+"""Flatseek — dual-mode client for Flatseek trigram index.
+
+Usage (API mode):
+    from flatseek import Flatseek
+
+    client = Flatseek("http://localhost:8000")
+    result = client.search(index="solana_txs", q="program:raydium AND amount:>1000000")
+
+Usage (direct mode):
+    from flatseek import Flatseek
+
+    qe = Flatseek("./data")                    # single index
+    qe = Flatseek("./data", index="solana_txs")    # named sub-index
+    result = qe.search(q="program:raydium AND signer:*7xMg*", size=10)
+
+    # Aggregation (direct mode, with query filter)
+    result = qe.aggregate(q="status:active AND country:ID", aggs={
+        "by_campaign": {"terms": {"field": "campaign", "size": 10}},
+        "bid_stats": {"stats": {"field": "bid"}}
+    })
+    print(result.aggs["by_campaign"]["buckets"])
+"""
+
+__version__ = "0.1.0"
+
+from flatseek.client import (
+    Flatseek,
+    Response,
+    CountResponse,
+    AggsResponse,
+    Elasticsearch,
+)
+
+__all__ = [
+    "Flatseek",
+    "Response",
+    "CountResponse",
+    "AggsResponse",
+    "Elasticsearch",
+    "__version__",
+]

flatseek/api/__init__.py

@@ -0,0 +1 @@
+# Flatseek API - FastAPI wrapper for trigram inverted index

flatseek/api/deps.py

@@ -0,0 +1,128 @@
+"""Dependency injection for Flatseek API."""
+
+import os
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING, AsyncGenerator, Any
+
+from fastapi import Depends, HTTPException
+
+# Default data directory
+DEFAULT_DATA_DIR = os.environ.get("FLATSEEK_DATA_DIR", "data")
+
+# Import QueryEngine lazily to avoid circular imports
+if TYPE_CHECKING:
+    from flatseek.core.query_engine import QueryEngine
+
+
+class IndexManager:
+    """Manages QueryEngine instances per index (lazy loading)."""
+
+    def __init__(self, data_dir: str = DEFAULT_DATA_DIR):
+        self.data_dir = data_dir
+        self._engines: dict[str, Any] = {}
+        # Per-index passwords for encrypted indexes
+        self._index_passwords: dict[str, str] = {}
+
+    def set_password(self, index: str, password: str) -> None:
+        """Store password for an encrypted index."""
+        self._index_passwords[index] = password
+
+    def get_password(self, index: str) -> str | None:
+        """Get stored password for an index, or None."""
+        return self._index_passwords.get(index)
+
+    def clear_password(self, index: str) -> None:
+        """Remove stored password for an index."""
+        self._index_passwords.pop(index, None)
+
+    def is_encrypted(self, index: str) -> bool:
+        """Check if a specific index is encrypted.
+
+        Each index stores its own encryption.json inside its folder.
+        If that file exists, the index is encrypted.
+        """
+        possible_paths = [
+            os.path.join(self.data_dir, index),
+            os.path.join(self.data_dir, index, "..", index),
+        ]
+        for index_path in possible_paths:
+            enc_path = os.path.join(index_path, "encryption.json")
+            if os.path.isfile(enc_path):
+                return True
+        return False
+
+    def get_engine(self, index: str) -> "QueryEngine":
+        """Get or create QueryEngine for an index."""
+        if not index.replace("_", "").replace("-", "").replace("/", "").isalnum():
+            raise HTTPException(400, f"Invalid index name: {index}")
+
+        engine = self._engines.get(index)
+        if engine is None:
+            possible_paths = [
+                os.path.join(self.data_dir, index),
+                os.path.join(self.data_dir, index, "..", index),
+            ]
+            from flatseek.core.query_engine import QueryEngine
+            for path in possible_paths:
+                if os.path.isdir(os.path.join(path, "index")):
+                    engine = QueryEngine(path)
+                    self._engines[index] = engine
+                    break
+
+        if engine is None:
+            raise HTTPException(404, f"Index not found: {index}")
+
+        return engine
+    
+    def list_indices(self) -> list[str]:
+        """List all available indices."""
+        if not os.path.isdir(self.data_dir):
+            return []
+        
+        indices = []
+        for name in os.listdir(self.data_dir):
+            path = os.path.join(self.data_dir, name)
+            if os.path.isdir(os.path.join(path, "index")):
+                indices.append(name)
+            elif os.path.isdir(path):
+                # Check for sub-indexes
+                for sub in os.listdir(path):
+                    sub_path = os.path.join(path, sub)
+                    if os.path.isdir(os.path.join(sub_path, "index")):
+                        indices.append(f"{name}/{sub}")
+        return sorted(indices)
+
+
+# Global index manager
+_index_manager: IndexManager | None = None
+
+
+def get_index_manager() -> IndexManager:
+    """Get the global IndexManager instance, re-reading FLATSEEK_DATA_DIR each call."""
+    global _index_manager
+    if _index_manager is None:
+        _index_manager = IndexManager()
+    else:
+        # Pick up changed FLATSEEK_DATA_DIR for tests
+        _index_manager.data_dir = os.environ.get("FLATSEEK_DATA_DIR", _index_manager.data_dir)
+    return _index_manager
+
+
+async def get_query_engine(
+    index: str,
+    manager: IndexManager = Depends(get_index_manager),
+) -> "QueryEngine":
+    """Dependency to get QueryEngine for an index."""
+    return manager.get_engine(index)
+
+
+def require_index(
+    index: str,
+    manager: IndexManager = Depends(get_index_manager),
+) -> str:
+    """Dependency that validates index exists."""
+    try:
+        manager.get_engine(index)
+    except Exception as e:
+        raise HTTPException(404, str(e)) from e
+    return index

flatseek/api/main.py

@@ -0,0 +1,264 @@
+"""FastAPI application for Flatseek (Elasticsearch-like API)."""
+
+import os
+import sys
+import logging
+import re
+import tempfile
+import shutil
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel, Field
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+
+def create_app():
+    """Create and return the FastAPI app."""
+    try:
+        from fastapi import FastAPI
+        from fastapi.middleware.cors import CORSMiddleware
+
+        app = FastAPI(
+            title="Flatseek API",
+            description="Trigram inverted index API — search, aggregate, and index your data. "
+                        "Supports Solana blockchain txs, aviation ADS-B, AdTech campaigns, DevOps logs, and more.",
+            version="0.1.0",
+            terms_of_service="https://flatseek.io/terms",
+            contact={"name": "Flatseek", "url": "https://flatseek.io"},
+        )
+
+        app.add_middleware(
+            CORSMiddleware,
+            allow_origins=["*"],
+            allow_credentials=True,
+            allow_methods=["*"],
+            allow_headers=["*"],
+        )
+
+        return app
+    except ImportError:
+        return _MockFastAPI()
+
+
+# ─── OpenAPI Schemas ──────────────────────────────────────────────────────────
+
+class RootInfo(BaseModel):
+    name: str = Field(..., example="Flatseek API")
+    version: str = Field(..., example="0.10.0")
+    description: str = Field(..., example="Trigram inverted index API")
+
+
+class ClusterHealth(BaseModel):
+    status: str = Field(..., example="green")
+    number_of_indices: int = Field(..., example=3)
+    number_of_nodes: int = Field(..., example=1)
+    indices: list[str]
+
+
+class IndicesList(BaseModel):
+    indices: list[str]
+    count: int
+
+
+class _MockFastAPI:
+    """Minimal FastAPI mock for testing without FastAPI installed."""
+
+    def __init__(self):
+        self._routes = []
+        self._middlewares = []
+
+    def add_middleware(self, *args, **kwargs):
+        self._middlewares.append((args, kwargs))
+
+    def include_router(self, router):
+        self._routes.append(router)
+
+    def get(self, path):
+        def decorator(func):
+            self._routes.append((path, "GET", func))
+            return func
+        return decorator
+
+    def post(self, path):
+        def decorator(func):
+            self._routes.append((path, "POST", func))
+            return func
+        return decorator
+
+    def delete(self, path):
+        def decorator(func):
+            self._routes.append((path, "DELETE", func))
+            return func
+        return decorator
+
+
+# ─── Dashboard (flatlens) mounting ────────────────────────────────────
+
+_dashboard_attached = False
+_dashboard_temp_dir = None  # holds patched flatlens dir if we rewrite API_BASE
+
+
+def _find_flatlens():
+    """Return path to flatlens directory or None."""
+    candidates = [
+        os.environ.get("FLATSEEK_FLATLENS_DIR", ""),
+        os.environ.get("FLATLENS_DIR", ""),
+        os.path.join(os.path.expanduser("~"), ".local", "share", "flatlens"),
+        "/opt/flatlens",
+    ]
+    # Dev: sibling repo at ../../flatlens relative to flatseek repo root
+    # flatseek_pkg = flatseek/flatseek/src/flatseek
+    # flatseek_repo = flatseek/flatseek/src  → flatlens is at flatseek/flatlens (two levels up)
+    _flatseek_pkg = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    _flatseek_repo = os.path.dirname(_flatseek_pkg)
+    candidates.insert(0, os.path.normpath(os.path.join(_flatseek_repo, "..", "..", "flatlens")))
+
+    for p in candidates:
+        if p and os.path.isdir(p) and os.path.isfile(os.path.join(p, "index.html")):
+            return p
+    return None
+
+
+def _copy_and_patch(flatlens_dir, api_base):
+    """Copy flatlens to temp dir with API_BASE rewrite + logo href patch.
+
+    Always copies to a temp dir so we never modify the original flatlens source.
+    """
+    global _dashboard_temp_dir
+    if _dashboard_temp_dir:
+        shutil.rmtree(_dashboard_temp_dir, ignore_errors=True)
+
+    dest = tempfile.mkdtemp(prefix="flatlens_patched_")
+    shutil.copytree(flatlens_dir, dest, dirs_exist_ok=True)
+
+    api_js = os.path.join(dest, "js", "api.js")
+    if os.path.exists(api_js):
+        with open(api_js, "r", encoding="utf-8") as f:
+            content = f.read()
+        content = re.sub(
+            r"const API_BASE\s*=\s*['\"][^'\"]*['\"]",
+            f"const API_BASE = '{api_base}'",
+            content,
+        )
+        with open(api_js, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    # Fix logo href in index.html — "/" would conflict with API docs routes
+    index_html = os.path.join(dest, "index.html")
+    if os.path.exists(index_html):
+        with open(index_html, "r", encoding="utf-8") as f:
+            content = f.read()
+        content = content.replace(
+            'href="/" class="dashboard-title"',
+            'href="/dashboard" class="dashboard-title"',
+        )
+        with open(index_html, "w", encoding="utf-8") as f:
+            f.write(content)
+
+    _dashboard_temp_dir = dest
+    return dest
+
+
+def attach_dashboard(app, api_base):
+    """Mount flatlens dashboard at /dashboard with correct API_BASE."""
+    global _dashboard_attached
+    if _dashboard_attached:
+        return
+
+    flatlens_dir = _find_flatlens()
+    if not flatlens_dir:
+        logger.warning("flatlens dashboard not found — skipping /dashboard mount")
+        logger.warning("  Set FLATLENS_DIR or install flatlens to ~/.local/share/flatlens")
+        return
+
+    # Always copy to temp and patch — never touch the original
+    patched_dir = _copy_and_patch(flatlens_dir, api_base)
+
+    app.mount("/dashboard", StaticFiles(directory=patched_dir, html=True), name="flatlens")
+    _dashboard_attached = True
+    logger.info(f"Flatlens dashboard mounted at /dashboard (API_BASE={api_base})")
+
+
+# ─── App setup ──────────────────────────────────────────────────────────
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from flatseek.api.deps import get_index_manager, IndexManager
+from fastapi import Depends
+
+app = create_app()
+
+# Mount dashboard BEFORE routers so StaticFiles takes priority over /{index} routes.
+_api_base = os.environ.get("FLATSEEK_API_BASE", "")
+if not _api_base:
+    _api_base = f"http://localhost:{os.environ.get('FLATSEEK_PORT', '8000')}"
+
+attach_dashboard(app, _api_base)
+# Also add a redirect from /dashboard → /dashboard/ (without trailing slash)
+from starlette.responses import RedirectResponse
+@app.get("/dashboard", include_in_schema=False)
+async def redirect_dashboard():
+    return RedirectResponse(url="/dashboard/", status_code=302)
+
+from flatseek.api.routes.index import router as index_router
+from flatseek.api.routes.search import router as search_router
+from flatseek.api.routes.aggregate import router as aggregate_router
+app.include_router(index_router)
+app.include_router(search_router)
+app.include_router(aggregate_router)
+
+
+# ─── Root ───────────────────────────────────────────────────────────────
+
+@app.get("/", response_model=RootInfo)
+async def root():
+    """Root endpoint — API name and version."""
+    return {
+        "name": "Flatseek API",
+        "version": "0.1.0",
+        "description": "Trigram inverted index API",
+    }
+
+
+@app.get("/_cluster/health", response_model=ClusterHealth)
+async def cluster_health(manager: IndexManager = Depends(get_index_manager)):
+    """Cluster health (single node). Returns all indices and their count."""
+    indices = manager.list_indices()
+    return {
+        "status": "green",
+        "number_of_indices": len(indices),
+        "number_of_nodes": 1,
+        "indices": indices,
+    }
+
+
+@app.get("/_indices", response_model=IndicesList)
+async def list_indices(manager: IndexManager = Depends(get_index_manager)):
+    """List all available indices in the cluster."""
+    indices = manager.list_indices()
+    return {
+        "indices": indices,
+        "count": len(indices),
+    }
+
+
+# ─── Run ─────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    import uvicorn
+
+    port = int(os.environ.get("FLATSEEK_PORT", "8000"))
+    host = os.environ.get("FLATSEEK_HOST", "0.0.0.0")
+
+    uvicorn.run(
+        "api.main:app",
+        host=host,
+        port=port,
+        reload=os.environ.get("FLATSEEK_RELOAD", "0") == "1",
+    )

flatseek/api/routes/__init__.py

@@ -0,0 +1 @@
+# Routes package

flatseek/api/routes/aggregate.py

@@ -0,0 +1,136 @@
+"""Aggregation endpoints (Elasticsearch-inspired).
+
+All business logic is delegated to QueryEngine.aggregate() in core.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import time
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from flatseek.api.deps import get_index_manager, IndexManager
+
+router = APIRouter(tags=["aggregations"])
+logger = logging.getLogger(__name__)
+
+
+# ─── OpenAPI Schemas ──────────────────────────────────────────────────────────
+
+class AggBucket(BaseModel):
+    key: str | int = Field(..., example="raydium")
+    doc_count: int = Field(..., example=892147)
+
+
+class TermsAgg(BaseModel):
+    buckets: list[AggBucket]
+
+
+class StatsAgg(BaseModel):
+    count: int = Field(..., example=1284)
+    min: float = Field(..., example=5000.0)
+    max: float = Field(..., example=10000.0)
+    avg: float = Field(..., example=5500.0)
+    sum: float = Field(..., example=7064000000.0)
+
+
+class Aggregations(BaseModel):
+    by_program: TermsAgg | None = None
+    fee_stats: StatsAgg | None = None
+
+    class Config:
+        extra = "allow"
+
+
+class AggregateHits(BaseModel):
+    total: int = Field(..., example=1284)
+
+
+class AggregateResponse(BaseModel):
+    hits: AggregateHits
+    aggregations: dict
+
+
+@router.post("/{index}/_aggregate", response_model=AggregateResponse)
+async def aggregate(
+    index: str,
+    body: dict[str, Any],
+    request: Request = None,
+    manager: IndexManager = Depends(get_index_manager),
+):
+    """Run aggregations (facets, statistics, buckets) over matching documents.
+
+    Supports `terms` (top-N by count), `stats` (count/min/max/avg/sum), and
+    `cardinality` (HyperLogLog unique count) aggregations. Combine multiple
+    aggregations in one request for rich analytics.
+
+    Example — program breakdown with fee stats:
+    ```json
+    {
+      "query": "status:success",
+      "aggs": {
+        "by_program": {"terms": {"field": "program", "size": 10}},
+        "fee_stats": {"stats": {"field": "fee"}}
+      }
+    }
+    ```
+    """
+    if manager.is_encrypted(index):
+        stored_pass = manager.get_password(index)
+        if not stored_pass and request:
+            stored_pass = request.headers.get("x-index-password")
+        if not stored_pass:
+            raise HTTPException(
+                403,
+                f"Index '{index}' is encrypted. Submit password via POST /{index}/_authenticate"
+            )
+        try:
+            from flatseek.core.query_engine import load_encryption_key
+            index_dir = os.path.join(manager.data_dir, index)
+            key = load_encryption_key(index_dir, stored_pass)
+            manager.get_engine(index).set_key(key)
+            manager.set_password(index, stored_pass)
+        except Exception:
+            raise HTTPException(401, "Invalid passphrase for encrypted index")
+
+    try:
+        engine = manager.get_engine(index)
+    except Exception as e:
+        raise HTTPException(404, f"Index not found: {index}") from e
+
+    query = body.get("query", None)
+    size = body.get("size", 10)
+    aggs = body.get("aggs", {})
+
+    try:
+        result = await asyncio.to_thread(
+            engine.aggregate, q=query, aggs=aggs, size=size
+        )
+        return result
+    except MemoryError as e:
+        raise HTTPException(
+            503,
+            f"Memory limit exceeded: {e}. Try a narrower query or fewer aggregations."
+        ) from e
+
+
+@router.get("/{index}/_aggregate", response_model=AggregateResponse)
+async def aggregate_get(
+    index: str,
+    q: str = Query("*", description="Query string"),
+    aggs: str = Query(None, description="Aggregations as JSON"),
+    request: Request = None,
+    manager: IndexManager = Depends(get_index_manager),
+):
+    """Run aggregations via GET (query params)."""
+    body = {"query": q}
+    if aggs:
+        try:
+            body["aggs"] = json.loads(aggs)
+        except Exception:
+            raise HTTPException(400, "Invalid aggs JSON")
+    return await aggregate(index, body, request, manager)