recce-nightly 1.10.0.20250625__py3-none-any.whl → 1.30.0.20251221__py3-none-any.whl

recce/run.py

@@ -2,7 +2,7 @@ import os
 import sys
 import time
 from datetime import datetime, timezone
-from typing import List
+from typing import Dict, List, Tuple
 
 from deepdiff import DeepDiff
 from rich import box
@@ -17,6 +17,7 @@ from recce.apis.check_func import (
 from recce.apis.run_func import submit_run
 from recce.config import RecceConfig
 from recce.core import default_context
+from recce.models import CheckDAO
 from recce.models.types import RunType
 from recce.summary import generate_markdown_summary
 
@@ -111,7 +112,7 @@ def run_should_be_approved(run):
     return False
 
 
-async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
+async def execute_preset_checks(preset_checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the preset checks
     """
@@ -155,12 +156,18 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
                     is_checked=is_check,
                 )
             else:
-                run, future = submit_run(check_type, params=check_params)
-                await future
-                is_check = run_should_be_approved(run)
-                create_check_from_run(
-                    run.run_id, check_name, check_description, check_options, is_preset=True, is_checked=is_check
-                )
+                if not is_skip_query:
+                    run, future = submit_run(check_type, params=check_params)
+                    await future
+                    is_check = run_should_be_approved(run)
+                    create_check_from_run(
+                        run.run_id, check_name, check_description, check_options, is_preset=True, is_checked=is_check
+                    )
+                else:
+                    create_check_without_run(
+                        check_name, check_description, check_type, check_params, check_options, is_preset=True
+                    )
+                    continue
 
             end = time.time()
             table.add_row(
@@ -200,7 +207,7 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
     return rc, failed_checks
 
 
-async def execute_state_checks(checks: list) -> (int, List[dict]):
+async def execute_state_checks(checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the checks from loaded state
     """
@@ -232,7 +239,7 @@ async def execute_state_checks(checks: list) -> (int, List[dict]):
                 raise ValueError(f"Invalid check type: {check_type}")
 
             start = time.time()
-            if check_type not in ["schema_diff"]:
+            if check_type not in ["schema_diff", "lineage_diff"] and not is_skip_query:
                 run, future = submit_run(check_type, params=check_params, check_id=check_id)
                 await future
 
@@ -295,7 +302,7 @@ def process_failed_checks(failed_checks: List[dict], error_log=None):
     content += markdown_table(failed_check_table).set_params(quote=False, row_sep="markdown").get_markdown()
 
     if error_log:
-        with open(error_log, "w") as f:
+        with open(error_log, "w", encoding="utf-8") as f:
             f.write(content)
         print(f"The failed checks are stored at '{error_log}'")
     else:
@@ -314,7 +321,17 @@ async def cli_run(output_state_file: str, **kwargs):
 
     ctx = load_context(**kwargs)
 
+    # Set up the checks if this is a session-based run
+    if kwargs.get("session_id") and kwargs.get("state_loader"):
+        state_loader = kwargs.get("state_loader")
+        try:
+            # Try to populate the checks from the database
+            state_loader.state.checks = CheckDAO().list()
+        except Exception as e:
+            console.print(f"[[red]Error[/red]] Failed to load checks from database: {e}")
+
     is_skip_query = kwargs.get("skip_query", False)
+    is_skip_check = kwargs.get("skip_check", False)
 
     # Prepare the artifact by collecting the lineage
     console.rule("DBT Artifacts")
@@ -327,23 +344,23 @@ async def cli_run(output_state_file: str, **kwargs):
     rc = 0
     if ctx.state_loader.state is None:
         preset_checks = RecceConfig().get("checks")
-        if is_skip_query or preset_checks is None or len(preset_checks) == 0:
+        if is_skip_check or preset_checks is None or len(preset_checks) == 0:
             # Skip the preset checks
             pass
         else:
             console.rule("Preset checks")
-            _, failed_checks = await execute_preset_checks(preset_checks)
+            _, failed_checks = await execute_preset_checks(preset_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Preset checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
     else:
         state_checks = ctx.state_loader.state.checks
-        if is_skip_query or state_checks is None or len(state_checks) == 0:
+        if is_skip_check or state_checks is None or len(state_checks) == 0:
             # Skip the checks in the state
             pass
         else:
             console.rule("Checks")
-            _, failed_checks = await execute_state_checks(state_checks)
+            _, failed_checks = await execute_state_checks(state_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
@@ -356,14 +373,17 @@ async def cli_run(output_state_file: str, **kwargs):
     console.rule("Export state")
     ctx.state_loader.state_file = output_state_file
     msg = ctx.state_loader.export(ctx.export_state())
-    console.print(msg)
+    if msg is not None:
+        console.print(msg)
+    else:
+        console.print("Export successful")
 
     summary_path = kwargs.get("summary")
     if summary_path:
         dirs = os.path.dirname(summary_path)
         if dirs:
             os.makedirs(dirs, exist_ok=True)
-        with open(summary_path, "w") as f:
+        with open(summary_path, "w", encoding="utf-8") as f:
             f.write(generate_markdown_summary(ctx))
         console.print(f"The summary is stored at '{summary_path}'")
 

recce/server.py

@@ -7,6 +7,7 @@ import uuid
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, Set
 
@@ -29,8 +30,9 @@ from starlette.middleware.gzip import GZipMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.websockets import WebSocketDisconnect
 
-from . import __latest_version__, __version__, event
+from . import __latest_version__, __version__, event, is_recce_cloud_instance
 from .apis.check_api import check_router
+from .apis.check_events_api import check_events_router
 from .apis.run_api import run_router
 from .config import RecceConfig
 from .connect_to_cloud import (
@@ -43,12 +45,31 @@ from .connect_to_cloud import (
 from .core import RecceContext, default_context, load_context
 from .event import get_recce_api_token, log_api_event, log_single_env_event
 from .exceptions import RecceException
+from .github import is_github_codespace
 from .models.types import CllData
 from .run import load_preset_checks
 from .state import RecceShareStateManager, RecceStateLoader
+from .util.startup_perf import track_timing
 
 logger = logging.getLogger("uvicorn")
 
+# Idle timeout check interval bounds (in seconds)
+MAX_CHECK_INTERVAL = 30
+MIN_CHECK_INTERVAL = 1
+
+
+class RecceServerMode(str, Enum):
+    server = "server"
+    preview = "preview"
+    read_only = "read-only"
+
+    def __str__(self):
+        return self.value
+
+    @staticmethod
+    def available_members() -> Set[str]:
+        return ["server", "preview", "read-only"]
+
 
 @dataclass
 class AppState:
@@ -59,13 +80,19 @@ class AppState:
     auth_options: Optional[dict] = None
     lifetime: Optional[int] = None
     lifetime_expired_at: Optional[datetime] = None
+    idle_timeout: Optional[int] = None
+    last_activity: Optional[dict] = None
     share_url: Optional[str] = None
+    organization_name: Optional[str] = None
+    web_url: Optional[str] = None
+    host: Optional[str] = None
+    port: Optional[int] = None
 
 
 def schedule_lifetime_termination(app_state):
     def terminating_server():
         pid = os.getpid()
-        logger.info(f"Terminating server process [{pid}] manually")
+        logger.info(f"Terminating server process [{pid}] manually due to lifetime expiration")
         os.kill(pid, signal.SIGINT)
 
     # Terminate the server process after the specified lifetime
@@ -74,6 +101,56 @@ def schedule_lifetime_termination(app_state):
     asyncio.get_running_loop().call_later(app_state.lifetime, terminating_server)
 
 
+def schedule_idle_timeout_check(app_state):
+    """
+    Schedule periodic checks for idle timeout.
+    If the server has been idle for longer than idle_timeout, terminate it.
+    """
+    # Track last activity time in app_state
+    app_state.last_activity = {"time": datetime.now(utc)}
+
+    def terminating_server_idle():
+        pid = os.getpid()
+        logger.info(f"Terminating server process [{pid}] manually due to idle timeout")
+        os.kill(pid, signal.SIGINT)
+
+    async def check_idle_timeout():
+        """Periodically check if the server has been idle for too long"""
+        # Use smaller check interval if idle_timeout is very short
+        # Check at least every MAX_CHECK_INTERVAL seconds, but also check when idle_timeout is approaching
+        check_interval = min(MAX_CHECK_INTERVAL, max(MIN_CHECK_INTERVAL, app_state.idle_timeout // 3))
+
+        logger.debug(f"[Idle Timeout] Starting idle timeout checker with {check_interval}s check interval")
+
+        while True:
+            await asyncio.sleep(check_interval)
+
+            idle_seconds = (datetime.now(utc) - app_state.last_activity["time"]).total_seconds()
+            remaining_seconds = app_state.idle_timeout - idle_seconds
+
+            # Always log the countdown for debugging
+            if remaining_seconds > 0:
+                logger.debug(
+                    f"[Idle Timeout] Server idle for {idle_seconds:.1f}s / {app_state.idle_timeout}s "
+                    f"(remaining: {remaining_seconds:.1f}s)"
+                )
+
+            if idle_seconds >= app_state.idle_timeout:
+                logger.info(
+                    f"[Idle Timeout] Threshold reached! Server has been idle for {idle_seconds:.0f} seconds "
+                    f"(threshold: {app_state.idle_timeout} seconds)"
+                )
+                terminating_server_idle()
+                break
+
+    # Start the idle timeout check task
+    logger.info(f"[Configuration] The idle timeout of the server is {app_state.idle_timeout} seconds")
+
+    # Create task using asyncio.create_task which works in async context
+    task = asyncio.create_task(check_idle_timeout())
+    logger.debug(f"[Idle Timeout] Background task created: {task}")
+
+
 def setup_server(app_state: AppState) -> RecceContext:
     from rich.console import Console
 
@@ -103,46 +180,99 @@ def setup_server(app_state: AppState) -> RecceContext:
 
     log_load_state(command="server", single_env=single_env)
 
-    if app_state.lifetime is not None and app_state.lifetime > 0:
-        schedule_lifetime_termination(app_state)
-
     return ctx
 
 
 def teardown_server(app_state: AppState, ctx: RecceContext):
-    state_loader = app_state.state_loader
+    # pull latest state, merge runs/checks and pick the newer artifacts
+    state_loader = ctx.state_loader
+    state_loader.refresh()
+    if state_loader.state:
+        ctx.import_state(state_loader.state, merge=True)
     state_loader.export(ctx.export_state())
-
     ctx.stop_monitor_artifacts()
     if app_state.flag.get("single_env_onboarding", False):
         ctx.stop_monitor_base_env()
 
 
 def setup_ready_only(app_state: AppState):
-    if app_state.lifetime is not None and app_state.lifetime > 0:
-        schedule_lifetime_termination(app_state)
+    pass
 
 
 def teardown_ready_only(app_state: AppState):
     pass
 
 
-@asynccontextmanager
-async def lifespan(fastapi: FastAPI):
-    ctx = None
-    app_state: AppState = app.state
+def setup_preview(app_state: AppState):
+    state_loader = app_state.state_loader
+    kwargs = app_state.kwargs
+    ctx = load_context(**kwargs, state_loader=state_loader)
+    return ctx
+
+
+def teardown_preview(app_state: AppState, ctx: RecceContext):
+    state_loader = app_state.state_loader
+    state_loader.export(ctx.export_state())
+    pass
 
+
+@track_timing("server_setup")
+def _do_lifespan_setup(app_state: AppState):
+    """Run server setup and return context for teardown."""
     if app_state.command == "server":
         ctx = setup_server(app_state)
-    elif app_state.command == "read_only":
+    elif app_state.command == "read-only":
         setup_ready_only(app_state)
+        ctx = None
+    elif app_state.command == "preview":
+        ctx = setup_preview(app_state)
+    else:
+        ctx = None
+
+    if app_state.lifetime is not None and app_state.lifetime > 0:
+        schedule_lifetime_termination(app_state)
+
+    if app_state.idle_timeout is not None and app_state.idle_timeout > 0:
+        logger.debug(f"[Idle Timeout] Scheduling idle timeout check with {app_state.idle_timeout} seconds")
+        schedule_idle_timeout_check(app_state)
+
+    return ctx
+
+
+@asynccontextmanager
+async def lifespan(fastapi: FastAPI):
+    from recce.core import default_context
+    from recce.event import log_performance
+    from recce.util.startup_perf import clear_startup_tracker, get_startup_tracker
+
+    app_state: AppState = app.state
+
+    # Ensure logger is at DEBUG level if debug mode is enabled
+    if app_state.kwargs and app_state.kwargs.get("debug"):
+        logger.setLevel(logging.DEBUG)
+        logger.debug("Debug mode enabled - logger set to DEBUG level")
+
+    ctx = _do_lifespan_setup(app_state)
+
+    # Log startup performance metrics
+    if tracker := get_startup_tracker():
+        tracker.command = app_state.command
+        recce_ctx = default_context()
+        if recce_ctx and recce_ctx.adapter:
+            tracker.adapter_type = type(recce_ctx.adapter).__name__
+            if hasattr(recce_ctx.adapter, "curr_manifest") and recce_ctx.adapter.curr_manifest:
+                tracker.node_count = len(recce_ctx.adapter.curr_manifest.nodes)
+        log_performance("server_startup", tracker.to_dict())
+        clear_startup_tracker()
 
     yield
 
     if app_state.command == "server":
         teardown_server(app_state, ctx)
-    elif app_state.command == "read_only":
+    elif app_state.command == "read-only":
         teardown_ready_only(app_state)
+    elif app_state.command == "preview":
+        teardown_preview(app_state, ctx)
 
 
 app = FastAPI(lifespan=lifespan)
@@ -150,7 +280,7 @@ app = FastAPI(lifespan=lifespan)
 
 def verify_json_file(file_path: str) -> bool:
     try:
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             json.load(f)
     except Exception:
         return False
@@ -207,6 +337,27 @@ app.add_middleware(
 )
 
 
+@app.middleware("http")
+async def track_activity_for_idle_timeout(request: Request, call_next):
+    """Track activity time for idle timeout check"""
+    # Exclude paths that should not reset idle timer
+    # Health checks and monitoring endpoints don't count as user activity
+    excluded_paths = ["/api/health", "/api/ws"]
+
+    # Update last activity time BEFORE processing request if idle timeout is enabled
+    # This ensures long-running requests don't get terminated mid-execution
+    app_state: AppState = app.state
+    if app_state.last_activity is not None:
+        if request.url.path not in excluded_paths:
+            app_state.last_activity["time"] = datetime.now(utc)
+            logger.debug(f"[Idle Timeout] ✓ Activity detected: {request.method} {request.url.path} - Timer reset")
+        else:
+            logger.debug(f"[Idle Timeout] Excluded path (no timer reset): {request.method} {request.url.path}")
+
+    response = await call_next(request)
+    return response
+
+
 @app.middleware("http")
 async def set_context_by_cookie(request: Request, call_next):
     response = await call_next(request)
@@ -239,12 +390,30 @@ async def health_check(request: Request):
     return {"status": "ok"}
 
 
+@app.post("/api/keep-alive")
+async def keep_alive():
+    """Endpoint to keep the session alive and reset idle timeout"""
+    app_state: AppState = app.state
+    if app_state.last_activity is not None:
+        app_state.last_activity["time"] = datetime.now(utc)
+        logger.debug("[Idle Timeout] Keep-alive request received - Timer reset")
+        return {"status": "ok", "idle_timeout_enabled": True}
+    return {"status": "ok", "idle_timeout_enabled": False}
+
+
 class RecceInstanceInfoOut(BaseModel):
+    server_mode: RecceServerMode
     read_only: bool
+    preview: bool
     single_env: bool
     authed: bool
+    cloud_instance: bool
     lifetime_expired_at: Optional[datetime] = None
+    idle_timeout: Optional[int] = None
     share_url: Optional[str] = None
+    session_id: Optional[str] = None
+    organization_name: Optional[str] = None
+    web_url: Optional[str] = None
 
 
 @app.get("/api/instance-info", response_model=RecceInstanceInfoOut, response_model_exclude_none=True)
@@ -257,11 +426,18 @@ async def recce_instance_info():
     api_token = get_recce_api_token()
 
     return {
+        "server_mode": app_state.command,
         "read_only": read_only,
+        "preview": flag.get("preview", False),
         "single_env": single_env,
         "authed": True if api_token else False,
+        "cloud_instance": is_recce_cloud_instance(),
         "lifetime_expired_at": app_state.lifetime_expired_at,  # UTC timezone
+        "idle_timeout": app_state.idle_timeout,
         "share_url": app_state.share_url,
+        "session_id": app_state.state_loader.session_id if app_state.state_loader else None,
+        "organization_name": app_state.organization_name,
+        "web_url": app_state.web_url,
         # TODO: Add more instance info which won't change during the instance lifecycle
         # review_mode
         # cloud_mode
@@ -289,6 +465,7 @@ async def get_info():
     """
     context = default_context()
     demo = os.environ.get("DEMO", False)
+    is_codespace = is_github_codespace()
 
     if demo:
         state = context.export_demo_state()
@@ -313,6 +490,7 @@ async def get_info():
             "pull_request": state.pull_request.to_dict() if state.pull_request else None,
             "lineage": lineage_diff,
             "demo": bool(demo),
+            "codespace": bool(is_codespace),
             "cloud_mode": context.state_loader.cloud_mode,
             "file_mode": context.state_loader.state_file is not None,
             "filename": filename,
@@ -337,9 +515,9 @@ class CllIn(BaseModel):
     node_id: Optional[str] = None
     column: Optional[str] = None
     change_analysis: Optional[bool] = False
-    cll: Optional[bool] = False
-    upstream: Optional[bool] = False
-    downstream: Optional[bool] = False
+    no_cll: Optional[bool] = False
+    no_upstream: Optional[bool] = False
+    no_downstream: Optional[bool] = False
 
 
 class CllOutput(BaseModel):
@@ -355,8 +533,9 @@ async def column_level_lineage_by_node(cll_input: CllIn):
         node_id=cll_input.node_id,
         column=cll_input.column,
         change_analysis=cll_input.change_analysis,
-        upstream=cll_input.upstream,
-        downstream=cll_input.downstream,
+        no_upstream=cll_input.no_upstream,
+        no_downstream=cll_input.no_downstream,
+        no_cll=cll_input.no_cll,
     )
 
     return CllOutput(current=cll)
@@ -674,9 +853,25 @@ async def generate_connect_to_cloud_url(background_tasks: BackgroundTasks):
     }
 
 
+@app.get("/api/users")
+async def get_user_info():
+    from recce.connect_to_cloud import RecceCloud
+
+    context = default_context()
+    user_token = get_recce_api_token() or context.state_loader.token
+    cloud = RecceCloud(user_token)
+    try:
+        user_info = cloud.get_user_info()
+        return user_info
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
 api_prefix = "/api"
 app.include_router(check_router, prefix=api_prefix)
+app.include_router(check_events_router, prefix=api_prefix)
 app.include_router(run_router, prefix=api_prefix)
 
 static_folder_path = Path(__file__).parent / "data"
+
 app.mount("/", StaticFiles(directory=static_folder_path, html=True), name="static")

recce/state/__init__.py

@@ -0,0 +1,31 @@
+from .cloud import (
+    CloudStateLoader,
+    RecceCloudStateManager,
+    RecceShareStateManager,
+    s3_sse_c_headers,
+)
+from .const import ErrorMessage
+from .local import FileStateLoader
+from .state import (
+    ArtifactsRoot,
+    GitRepoInfo,
+    PullRequestInfo,
+    RecceState,
+    RecceStateMetadata,
+)
+from .state_loader import RecceStateLoader
+
+__all__ = [
+    "ArtifactsRoot",
+    "ErrorMessage",
+    "RecceCloudStateManager",
+    "RecceShareStateManager",
+    "RecceState",
+    "RecceStateLoader",
+    "CloudStateLoader",
+    "FileStateLoader",
+    "RecceStateMetadata",
+    "s3_sse_c_headers",
+    "GitRepoInfo",
+    "PullRequestInfo",
+]