secondmate 0.3.0__py3-none-any.whl

secondmate/__init__.py

@@ -0,0 +1 @@
+

secondmate/cli.py

@@ -0,0 +1,26 @@
+import argparse
+import uvicorn
+import os
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the SecondMate application.")
+    parser.add_argument("--port", type=int, default=4050, help="Port to run the server on (default: 4050)")
+    parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to bind to (default: 0.0.0.0)")
+    parser.add_argument("--reload", action="store_true", help="Enable auto-reload")
+
+    args = parser.parse_args()
+
+    # Set the static directory path relative to this file if not set
+    # This allows the package to find its static assets
+    package_dir = os.path.dirname(os.path.abspath(__file__))
+    static_dir = os.path.join(package_dir, "static")
+    os.environ["SECONDMATE_STATIC_DIR"] = static_dir
+
+    print(f"Starting SecondMate on http://{args.host}:{args.port}")
+    if args.reload:
+        uvicorn.run("secondmate.main:app", host=args.host, port=args.port, reload=True)
+    else:
+        uvicorn.run("secondmate.main:app", host=args.host, port=args.port)
+
+if __name__ == "__main__":
+    main()

secondmate/dependencies.py

@@ -0,0 +1,38 @@
+import os
+import importlib
+from functools import lru_cache
+from pyspark.sql import SparkSession
+from fastapi import Depends, HTTPException
+from secondmate.providers.spark_interface import SparkProvider
+from secondmate.providers.local_spark import LocalSparkProvider
+
+@lru_cache()
+def get_spark_provider() -> SparkProvider:
+    provider_class_path = os.getenv("SPARK_PROVIDER_CLASS")
+    
+    if not provider_class_path:
+        return LocalSparkProvider()
+    
+    try:
+        module_name, class_name = provider_class_path.rsplit(".", 1)
+        module = importlib.import_module(module_name)
+        provider_cls = getattr(module, class_name)
+        
+        # Instantiate the provider. 
+        # We assume it takes no arguments or we might need to support passing args via env vars too.
+        # For now, let's assume valid providers typically don't need args or handle configuration internally.
+        provider_instance = provider_cls()
+        
+        if not hasattr(provider_instance, "get_session"):
+             raise ValueError(f"Provider '{provider_class_path}' must implement 'get_session' method.")
+
+        return provider_instance
+        
+    except (ImportError, AttributeError, ValueError) as e:
+        # Raise a clear error if the configured provider cannot be loaded
+        raise ImportError(f"Could not load Spark provider '{provider_class_path}': {e}") from e
+
+def get_spark_session(
+    provider: SparkProvider = Depends(get_spark_provider)
+) -> SparkSession:
+    return provider.get_session()

secondmate/main.py

@@ -0,0 +1,264 @@
+from fastapi import FastAPI, Depends, APIRouter
+from pyspark.sql import SparkSession
+from secondmate.dependencies import get_spark_session
+import sys
+
+# Hack to make sure pyspark finds the right python executable if needed, 
+# or relying on environment. 
+# For local spark, PYSPARK_PYTHON usually defaults to sys.executable
+
+from contextlib import asynccontextmanager
+from secondmate.dependencies import get_spark_provider
+import random
+
+from secondmate.providers.local_spark import LocalSparkProvider
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup: Ensure table exists and has data
+    provider = get_spark_provider()
+    spark = provider.get_session()
+    
+    # Only create fake data if using LocalSparkProvider
+    if isinstance(provider, LocalSparkProvider):
+        # helper to create table with data
+        def create_table_if_not_exists(table_name, schema_ddl, data_records, schema_cols):
+            if not spark.catalog.tableExists(table_name):
+                print(f"Creating {table_name} table...")
+                spark.sql(f"CREATE TABLE IF NOT EXISTS {table_name} ({schema_ddl}) USING iceberg")
+            
+            # Check if empty
+            count = spark.table(table_name).count()
+            if count == 0 and data_records:
+                print(f"Populating {table_name} with {len(data_records)} rows...")
+                df = spark.createDataFrame(data_records, schema_cols)
+                df.writeTo(table_name).append()
+                print(f"Initialized {table_name} with {len(data_records)} rows.")
+            else:
+                 print(f"Table {table_name} already has {count} rows.")
+    
+        # 1. user.ipgeo
+        create_table_if_not_exists(
+            "user.ipgeo", 
+            "id LONG, ip STRING, city STRING, country STRING", 
+            [(i, f"192.168.1.{i % 255}", f"City_{i}", f"Country_{i % 10}") for i in range(1000)],
+            ["id", "ip", "city", "country"]
+        )
+    
+        # 2. user.sales.transactions
+        from datetime import date
+        spark.sql("CREATE NAMESPACE IF NOT EXISTS user.sales")
+        create_table_if_not_exists(
+            "user.sales.transactions",
+            "tx_id LONG, amount DOUBLE, currency STRING, tx_date DATE",
+            [
+                (1, 100.50, "USD", date(2023, 1, 1)),
+                (2, 200.00, "EUR", date(2023, 1, 2)),
+                (3, 50.25, "GBP", date(2023, 1, 3))
+            ],
+            ["tx_id", "amount", "currency", "tx_date"]
+        )
+    
+        # 3. user.finance.budget
+        spark.sql("CREATE NAMESPACE IF NOT EXISTS user.finance")
+        create_table_if_not_exists(
+            "user.finance.budget",
+            "dept_id LONG, dept_name STRING, budget_amount DOUBLE, fiscal_year INT",
+            [
+                (101, "Engineering", 500000.0, 2024),
+                (102, "Marketing", 200000.0, 2024),
+                (103, "HR", 150000.0, 2024)
+            ],
+            ["dept_id", "dept_name", "budget_amount", "fiscal_year"]
+        )
+
+    yield
+    # Shutdown logic if needed
+
+app = FastAPI(title="SecondMate Backend", lifespan=lifespan)
+router = APIRouter()
+
+from pydantic import BaseModel
+
+class QueryRequest(BaseModel):
+    query: str
+
+@router.post("/query/execute")
+def execute_query(request: QueryRequest, spark: SparkSession = Depends(get_spark_session)):
+    """Execute a raw SQL query and return results."""
+    try:
+        df = spark.sql(request.query)
+        # Limit to 1000 to prevent overloading
+        df = df.limit(1000)
+        
+        # Get schema
+        schema = [{"name": field.name, "type": str(field.dataType)} for field in df.schema.fields]
+        
+        # Get data
+        data = [row.asDict() for row in df.collect()]
+        
+        return {"schema": schema, "data": data}
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+        return {"error": str(e)}
+
+@router.get("/catalogs")
+def get_catalogs(spark: SparkSession = Depends(get_spark_session)):
+    """List available catalogs."""
+    # In PySpark 3.4+, spark.catalog.listCatalogs() exists but might not be standard in all setups.
+    # For now, we can try to inspect catalogs.
+    # A standard Spark setup usually has 'spark_catalog' and any defined custom catalogs.
+    # We can try to query `SHOW CATALOGS` via SQL
+    try:
+        df = spark.sql("SHOW CATALOGS")
+        catalogs = [row.catalog for row in df.collect()]
+        return {"catalogs": catalogs}
+    except Exception as e:
+        return {"catalogs": [], "error": str(e)}
+
+@router.get("/catalogs/{catalog_name}/namespaces")
+def get_namespaces(catalog_name: str, spark: SparkSession = Depends(get_spark_session)):
+    """List namespaces in a specific catalog."""
+    try:
+        # Switch to catalog to list namespaces easily, or use `SHOW NAMESPACES IN catalog`
+        df = spark.sql(f"SHOW NAMESPACES IN {catalog_name}")
+        # The column name is usually 'namespace'
+        namespaces = [row.namespace for row in df.collect()]
+        return {"namespaces": namespaces}
+    except Exception as e:
+        return {"namespaces": [], "error": str(e)}
+
+@router.get("/catalogs/{catalog_name}/namespaces/{namespace}/tables")
+def get_tables(catalog_name: str, namespace: str, spark: SparkSession = Depends(get_spark_session)):
+    """List tables in a specific namespace."""
+    try:
+        df = spark.sql(f"SHOW TABLES IN {catalog_name}.{namespace}")
+        # Columns: 'namespace', 'tableName', 'isTemporary'
+        tables = [row.tableName for row in df.collect()]
+        return {"tables": tables}
+    except Exception as e:
+        return {"tables": [], "error": str(e)}
+
+@router.get("/info")
+def get_info(spark: SparkSession = Depends(get_spark_session)):
+    return {
+        "app_name": spark.sparkContext.appName,
+        "spark_version": spark.version,
+        "master": spark.sparkContext.master,
+        "python_version": sys.version
+    }
+
+
+
+@router.get("/health")
+def health_check():
+    return {"status": "ok"}
+
+@router.get("/search")
+def search_catalog(q: str, spark: SparkSession = Depends(get_spark_session)):
+    """Search for catalogs, namespaces, and tables matching the query."""
+    query = q.lower()
+    results = []
+
+    try:
+        # 1. Search Catalogs
+        df_catalogs = spark.sql("SHOW CATALOGS")
+        catalogs = [row.catalog for row in df_catalogs.collect()]
+        
+        for cat in catalogs:
+            if query in cat.lower():
+                results.append({"type": "catalog", "catalog": cat, "name": cat})
+            
+            try:
+                # 2. Search Namespaces within Catalog
+                df_ns = spark.sql(f"SHOW NAMESPACES IN {cat}")
+                namespaces = [row.namespace for row in df_ns.collect()]
+                
+                for ns in namespaces:
+                    if query in ns.lower():
+                        results.append({"type": "namespace", "catalog": cat, "namespace": ns, "name": ns})
+                    
+                    try:
+                        # 3. Search Tables within Namespace
+                        df_tables = spark.sql(f"SHOW TABLES IN {cat}.{ns}")
+                        tables = [row.tableName for row in df_tables.collect()]
+                        
+                        for table in tables:
+                            if query in table.lower():
+                                results.append({
+                                    "type": "table", 
+                                    "catalog": cat, 
+                                    "namespace": ns, 
+                                    "table": table,
+                                    "name": table
+                                })
+                    except Exception:
+                        continue # Ignore individual failures
+            except Exception:
+                continue
+
+    except Exception as e:
+        return {"results": [], "error": str(e)}
+
+    return {"results": results}
+
+# Include the API router
+app.include_router(router, prefix="/api")
+
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse, HTMLResponse
+import os
+
+# Mount static files
+# Use environment variable or fallback to local assumption
+static_dir = os.getenv("SECONDMATE_STATIC_DIR", os.path.join(os.path.dirname(__file__), "static"))
+
+if os.path.exists(static_dir):
+    app.mount("/assets", StaticFiles(directory=os.path.join(static_dir, "assets")), name="assets")
+
+    # Catch-all for SPA
+    @app.get("/{full_path:path}")
+    async def serve_spa(full_path: str):
+        # Exclude API routes from catch-all
+        if full_path.startswith("api") or full_path.startswith("docs") or full_path.startswith("openapi.json"):
+             from fastapi import HTTPException
+             raise HTTPException(status_code=404, detail="Not Found")
+        
+        # Check if it's a static file that exists
+        file_path = os.path.join(static_dir, full_path)
+        if os.path.isfile(file_path):
+            return FileResponse(file_path)
+            
+        # Fallback to index.html for SPA routing
+        index_path = os.path.join(static_dir, "index.html")
+        if os.path.exists(index_path):
+            with open(index_path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            # Determine base path for API
+            # Priority: PROXY_PREFIX env var -> JUPYTERHUB_SERVICE_PREFIX -> default
+            # Note: JUPYTERHUB_SERVICE_PREFIX is usually /user/<name>/
+            # If running via jupyter-server-proxy, the URL is typically /user/<name>/proxy/<port>/
+            # The user should ideally set PROXY_PREFIX to the full proxy path.
+            proxy_prefix = os.getenv("PROXY_PREFIX", "")
+            if not proxy_prefix and os.getenv("JUPYTERHUB_SERVICE_PREFIX"):
+                 # Fallback/heuristic: if in jupyterhub, might still need port if using proxy
+                 pass 
+
+            # Ensure prefix has trailing slash if it exists and we're appending 'api'
+            # But wait, if prefix is "/foo", we want "/foo/api"
+            # If prefix is "/foo/", we want "/foo/api"
+            api_base = "/api"
+            if proxy_prefix:
+                clean_prefix = proxy_prefix.rstrip("/")
+                api_base = f"{clean_prefix}/api"
+
+            injection = f'<script>window.SECONDMATE_CONFIG = {{ apiBaseUrl: "{api_base}" }};</script>'
+            # Inject before </head>
+            content = content.replace("</head>", f"{injection}</head>")
+            
+            return HTMLResponse(content=content)
+        return {"error": "Frontend not built or static files missing."}
+else:
+    print(f"Warning: Static directory {static_dir} not found. Frontend will not be served.")

secondmate/providers/__init__.py

@@ -0,0 +1 @@
+

secondmate/providers/local_spark.py

@@ -0,0 +1,25 @@
+from pyspark.sql import SparkSession
+from secondmate.providers.spark_interface import SparkProvider
+
+class LocalSparkProvider(SparkProvider):
+    def __init__(self, app_name: str = "SecondMateLocal"):
+        self.app_name = app_name
+        self._session = None
+
+    def get_session(self) -> SparkSession:
+        if self._session is None:
+            self._session = (
+                SparkSession.builder
+                .appName(self.app_name)
+                .master("local[*]")
+                .config("spark.driver.bindAddress", "127.0.0.1")
+                .config("spark.driver.host", "127.0.0.1")
+                # Iceberg Configuration
+                .config("spark.jars.packages", "org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0")
+                .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
+                .config("spark.sql.catalog.user", "org.apache.iceberg.spark.SparkCatalog")
+                .config("spark.sql.catalog.user.type", "hadoop")
+                .config("spark.sql.catalog.user.warehouse", "warehouse")
+                .getOrCreate()
+            )
+        return self._session

secondmate/providers/spark_interface.py

@@ -0,0 +1,7 @@
+from typing import Protocol
+from pyspark.sql import SparkSession
+
+class SparkProvider(Protocol):
+    def get_session(self) -> SparkSession:
+        """Get or create a SparkSession."""
+        ...

secondmate/static/assets/index-Gyne2P3W.css

@@ -0,0 +1 @@
+:root{--bg-primary: #0f172a;--bg-secondary: #1e293b;--bg-tertiary: #334155;--text-primary: #f8fafc;--text-secondary: #94a3b8;--text-muted: #64748b;--accent-primary: #38bdf8;--accent-secondary: #818cf8;--accent-hover: #0ea5e9;--border-color: #334155;--border-hover: #475569;--font-sans: "Inter", system-ui, -apple-system, sans-serif;--font-mono: "JetBrains Mono", "Fira Code", monospace}body{margin:0;padding:0;background-color:var(--bg-primary);color:var(--text-primary);font-family:var(--font-sans);-webkit-font-smoothing:antialiased;height:100vh;width:100vw;overflow:hidden}*{box-sizing:border-box}::-webkit-scrollbar{width:10px;height:10px}::-webkit-scrollbar-track{background:var(--bg-primary)}::-webkit-scrollbar-thumb{background:var(--bg-tertiary);border-radius:5px;border:2px solid var(--bg-primary)}::-webkit-scrollbar-thumb:hover{background:var(--text-muted)}._sidebar_1o808_1{display:flex;flex-direction:column;height:100%;color:var(--text-secondary);font-size:13px}._header_1o808_17{padding:12px;display:flex;justify-content:space-between;align-items:center;border-bottom:1px solid var(--border-color)}._title_1o808_33{font-weight:600;color:var(--text-primary);text-transform:uppercase;font-size:11px;letter-spacing:.05em}._iconButton_1o808_49{background:none;border:none;color:var(--text-secondary);cursor:pointer;padding:4px;border-radius:4px}._iconButton_1o808_49:hover{background-color:var(--bg-tertiary);color:var(--text-primary)}._searchBox_1o808_77{padding:8px 12px;position:relative;border-bottom:1px solid var(--border-color)}._searchIcon_1o808_89{position:absolute;left:20px;top:50%;transform:translateY(-50%);color:var(--text-muted)}._searchInput_1o808_105{width:100%;background-color:var(--bg-primary);border:1px solid var(--border-color);border-radius:4px;padding:6px 30px;color:var(--text-primary);font-size:12px;outline:none}._clearIcon_1o808_127{position:absolute;right:20px;top:50%;transform:translateY(-50%);color:var(--text-muted);cursor:pointer}._clearIcon_1o808_127:hover{color:var(--text-primary)}._searchInput_1o808_105:focus{border-color:var(--accent-primary)}._tree_1o808_161{flex:1;overflow-y:auto;padding:8px 0}._treeItem_1o808_173{display:flex;flex-direction:column}._treeRow_1o808_183{display:flex;align-items:center;padding:4px 12px;cursor:pointer;gap:6px;-webkit-user-select:none;user-select:none}._treeRow_1o808_183:hover{background-color:var(--bg-tertiary);color:var(--text-primary)}._treeChildren_1o808_211{padding-left:16px}._icon_1o808_49{opacity:.9}._spinner_1o808_227{animation:_spin_1o808_227 1s linear infinite}@keyframes _spin_1o808_227{0%{transform:rotate(0)}to{transform:rotate(360deg)}}._tabs_1o808_255{display:flex;border-bottom:1px solid var(--border-color);background-color:var(--bg-secondary)}._tab_1o808_255{flex:1;padding:8px;text-align:center;cursor:pointer;font-size:11px;font-weight:500;color:var(--text-muted);border-bottom:2px solid transparent;transition:all .2s}._tab_1o808_255:hover{color:var(--text-primary);background-color:var(--bg-tertiary)}._activeTab_1o808_301{color:var(--accent-primary);border-bottom-color:var(--accent-primary);background-color:var(--bg-primary)}._container_14arr_1{height:100vh;display:flex;flex-direction:column;background-color:var(--bg-primary)}._sidebarPanel_14arr_15{background-color:var(--bg-secondary);border-right:1px solid var(--border-color);display:flex;flex-direction:column}._contentPanel_14arr_29{background-color:var(--bg-primary);display:flex;flex-direction:column}._resizeHandle_14arr_41{width:2px;background-color:var(--border-color);transition:background-color .2s;cursor:col-resize}._resizeHandle_14arr_41:hover,._resizeHandle_14arr_41:active{background-color:var(--accent-primary)}._statusBar_14arr_65{height:24px;background-color:var(--accent-primary);color:var(--bg-primary);display:flex;align-items:center;padding:0 12px;font-size:12px;font-family:var(--font-mono);justify-content:space-between;font-weight:600}._editorContainer_1tof5_1{height:100%;width:100%;overflow:hidden;background-color:var(--bg-secondary)}._gridContainer_12pjg_1{height:100%;width:100%;overflow:auto;background-color:var(--bg-primary)}._table_12pjg_15{width:100%;border-collapse:collapse;font-size:13px;color:var(--text-primary);font-family:var(--font-mono)}._table_12pjg_15 th{background-color:var(--bg-secondary);position:sticky;top:0;text-align:left;border-bottom:1px solid var(--border-color);border-right:1px solid var(--border-color);padding:8px 12px;font-weight:600;color:var(--text-secondary);-webkit-user-select:none;user-select:none}._table_12pjg_15 td{padding:6px 12px;border-bottom:1px solid var(--border-color);border-right:1px solid var(--border-color);white-space:nowrap}._table_12pjg_15 tr:hover td{background-color:var(--bg-tertiary)}._headerCell_12pjg_79{display:flex;flex-direction:column;gap:2px}._typeLabel_12pjg_91{font-size:10px;color:var(--text-muted);font-weight:400}._container_1wn1h_1{display:flex;flex-direction:column;align-items:center;justify-content:center;height:100%;width:100%;overflow:visible;position:relative;min-height:200px}._scene_1wn1h_16{position:relative;width:20%;min-width:150px;max-width:300px;aspect-ratio:4/3}._river_1wn1h_29{position:absolute;bottom:0;left:-20%;width:140%;height:20%;display:flex;align-items:flex-end;justify-content:space-around;pointer-events:none}._wave_1wn1h_44{width:20%;height:30%;background:#60a5fa;border-radius:50%;opacity:.6;animation:_waveMove_1wn1h_1 2s ease-in-out infinite alternate}._wave_1wn1h_44:nth-child(2n){animation-delay:1s;background:#93c5fd}._boatContainer_1wn1h_61{position:absolute;bottom:15%;left:50%;width:80%;transform:translate(-50%);animation:_bob_1wn1h_1 2s ease-in-out infinite;z-index:10}._smoke_1wn1h_74{position:absolute;top:-10%;left:50%;width:8%;height:8%;background:#cbd5e1;border-radius:50%;opacity:0;animation:_puff_1wn1h_1 2s infinite}._smoke_1wn1h_74:nth-child(2){animation-delay:.5s}._smoke_1wn1h_74:nth-child(3){animation-delay:1s}._text_1wn1h_95{margin-top:1rem;font-family:Courier New,Courier,monospace;font-weight:600;color:#1e40af;letter-spacing:1px;animation:_pulse_1wn1h_1 1.5s ease-in-out infinite;text-align:center}@keyframes _waveMove_1wn1h_1{0%{transform:translateY(0)}to{transform:translateY(-5px)}}@keyframes _bob_1wn1h_1{0%,to{transform:translate(-50%) translateY(0) rotate(0)}50%{transform:translate(-50%) translateY(-2%) rotate(1deg)}}@keyframes _puff_1wn1h_1{0%{transform:translateY(0) scale(1);opacity:.8}to{transform:translateY(-300%) scale(3);opacity:0}}@keyframes _pulse_1wn1h_1{0%,to{opacity:1}50%{opacity:.7}}._workspace_3frhl_1{display:flex;flex-direction:column;height:100%}._toolbar_3frhl_13{height:40px;border-bottom:1px solid var(--border-color);display:flex;align-items:center;padding:0 12px;gap:12px;background-color:var(--bg-secondary)}._runButton_3frhl_33{display:flex;align-items:center;gap:6px;background-color:#059669;color:#fff;border:none;padding:6px 12px;border-radius:4px;font-size:12px;font-weight:600;cursor:pointer;transition:background-color .2s}._runButton_3frhl_33:hover{background-color:#047857}._divider_3frhl_71{width:1px;height:20px;background-color:var(--border-color)}._tabTitle_3frhl_83{font-size:13px;color:var(--text-primary)}._content_3frhl_93{flex:1;position:relative}._resizeHandle_3frhl_103{height:2px;background-color:var(--border-color);transition:background-color .2s;cursor:row-resize}._resizeHandle_3frhl_103:hover,._resizeHandle_3frhl_103:active{background-color:var(--accent-primary)}._resultsArea_3frhl_127{display:flex;flex-direction:column;height:100%;background-color:var(--bg-primary)}._resultsHeader_3frhl_141{padding:8px 12px;border-bottom:1px solid var(--border-color);font-size:12px;font-weight:600;color:var(--text-secondary);display:flex;justify-content:space-between;background-color:var(--bg-secondary)}._meta_3frhl_163{font-weight:400;color:var(--text-muted)}._emptyState_3frhl_173{display:flex;flex-direction:column;align-items:center;justify-content:center;height:100%;padding:2rem;text-align:center;animation:_fadeInZoom_3frhl_1 .5s ease-out forwards;color:var(--text-primary)}._emptyStateHeader_3frhl_197{font-size:1.25rem;font-weight:500;color:var(--text-secondary);margin-bottom:1rem;letter-spacing:.025em}._emptyStateMain_3frhl_213{font-size:1.875rem;font-weight:700;color:var(--text-primary);max-width:42rem;line-height:1.25}@keyframes _fadeInZoom_3frhl_1{0%{opacity:0;transform:scale(.95)}to{opacity:1;transform:scale(1)}}