recce-nightly 1.9.0.20250623__py3-none-any.whl → 1.25.0.20251112a2066__py3-none-any.whl

recce/run.py

@@ -2,7 +2,7 @@ import os
 import sys
 import time
 from datetime import datetime, timezone
-from typing import List
+from typing import Dict, List, Tuple
 
 from deepdiff import DeepDiff
 from rich import box
@@ -111,7 +111,7 @@ def run_should_be_approved(run):
     return False
 
 
-async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
+async def execute_preset_checks(preset_checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the preset checks
     """
@@ -155,12 +155,18 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
                     is_checked=is_check,
                 )
             else:
-                run, future = submit_run(check_type, params=check_params)
-                await future
-                is_check = run_should_be_approved(run)
-                create_check_from_run(
-                    run.run_id, check_name, check_description, check_options, is_preset=True, is_checked=is_check
-                )
+                if not is_skip_query:
+                    run, future = submit_run(check_type, params=check_params)
+                    await future
+                    is_check = run_should_be_approved(run)
+                    create_check_from_run(
+                        run.run_id, check_name, check_description, check_options, is_preset=True, is_checked=is_check
+                    )
+                else:
+                    create_check_without_run(
+                        check_name, check_description, check_type, check_params, check_options, is_preset=True
+                    )
+                    continue
 
             end = time.time()
             table.add_row(
@@ -200,7 +206,7 @@ async def execute_preset_checks(preset_checks: list) -> (int, List[dict]):
     return rc, failed_checks
 
 
-async def execute_state_checks(checks: list) -> (int, List[dict]):
+async def execute_state_checks(checks: List, is_skip_query: bool) -> Tuple[int, List[Dict]]:
     """
     Execute the checks from loaded state
     """
@@ -232,7 +238,7 @@ async def execute_state_checks(checks: list) -> (int, List[dict]):
                 raise ValueError(f"Invalid check type: {check_type}")
 
             start = time.time()
-            if check_type not in ["schema_diff"]:
+            if check_type not in ["schema_diff", "lineage_diff"] and not is_skip_query:
                 run, future = submit_run(check_type, params=check_params, check_id=check_id)
                 await future
 
@@ -295,7 +301,7 @@ def process_failed_checks(failed_checks: List[dict], error_log=None):
     content += markdown_table(failed_check_table).set_params(quote=False, row_sep="markdown").get_markdown()
 
     if error_log:
-        with open(error_log, "w") as f:
+        with open(error_log, "w", encoding="utf-8") as f:
             f.write(content)
         print(f"The failed checks are stored at '{error_log}'")
     else:
@@ -315,6 +321,7 @@ async def cli_run(output_state_file: str, **kwargs):
     ctx = load_context(**kwargs)
 
     is_skip_query = kwargs.get("skip_query", False)
+    is_skip_check = kwargs.get("skip_check", False)
 
     # Prepare the artifact by collecting the lineage
     console.rule("DBT Artifacts")
@@ -327,23 +334,23 @@ async def cli_run(output_state_file: str, **kwargs):
     rc = 0
     if ctx.state_loader.state is None:
         preset_checks = RecceConfig().get("checks")
-        if is_skip_query or preset_checks is None or len(preset_checks) == 0:
+        if is_skip_check or preset_checks is None or len(preset_checks) == 0:
             # Skip the preset checks
             pass
         else:
             console.rule("Preset checks")
-            _, failed_checks = await execute_preset_checks(preset_checks)
+            _, failed_checks = await execute_preset_checks(preset_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Preset checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
     else:
         state_checks = ctx.state_loader.state.checks
-        if is_skip_query or state_checks is None or len(state_checks) == 0:
+        if is_skip_check or state_checks is None or len(state_checks) == 0:
             # Skip the checks in the state
             pass
         else:
             console.rule("Checks")
-            _, failed_checks = await execute_state_checks(state_checks)
+            _, failed_checks = await execute_state_checks(state_checks, is_skip_query)
             if failed_checks:
                 console.print("[[yellow]Warning[/yellow]] Checks failed. Please see the failed reason.")
                 process_failed_checks(failed_checks, error_log)
@@ -363,7 +370,7 @@ async def cli_run(output_state_file: str, **kwargs):
         dirs = os.path.dirname(summary_path)
         if dirs:
             os.makedirs(dirs, exist_ok=True)
-        with open(summary_path, "w") as f:
+        with open(summary_path, "w", encoding="utf-8") as f:
             f.write(generate_markdown_summary(ctx))
         console.print(f"The summary is stored at '{summary_path}'")
 

recce/server.py

@@ -7,6 +7,7 @@ import uuid
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from datetime import datetime, timedelta
+from enum import Enum
 from pathlib import Path
 from typing import Annotated, Any, Literal, Optional, Set
 
@@ -29,19 +30,44 @@ from starlette.middleware.gzip import GZipMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.websockets import WebSocketDisconnect
 
-from . import __latest_version__, __version__, event
+from . import __latest_version__, __version__, event, is_recce_cloud_instance
 from .apis.check_api import check_router
 from .apis.run_api import run_router
 from .config import RecceConfig
+from .connect_to_cloud import (
+    connect_to_cloud_background_task,
+    generate_key_pair,
+    get_connection_url,
+    is_callback_server_running,
+    prepare_connection_url,
+)
 from .core import RecceContext, default_context, load_context
-from .event import log_api_event, log_single_env_event
+from .event import get_recce_api_token, log_api_event, log_single_env_event
 from .exceptions import RecceException
+from .github import is_github_codespace
 from .models.types import CllData
 from .run import load_preset_checks
 from .state import RecceShareStateManager, RecceStateLoader
 
 logger = logging.getLogger("uvicorn")
 
+# Idle timeout check interval bounds (in seconds)
+MAX_CHECK_INTERVAL = 30
+MIN_CHECK_INTERVAL = 1
+
+
+class RecceServerMode(str, Enum):
+    server = "server"
+    preview = "preview"
+    read_only = "read-only"
+
+    def __str__(self):
+        return self.value
+
+    @staticmethod
+    def available_members() -> Set[str]:
+        return ["server", "preview", "read-only"]
+
 
 @dataclass
 class AppState:
@@ -52,13 +78,19 @@ class AppState:
     auth_options: Optional[dict] = None
     lifetime: Optional[int] = None
     lifetime_expired_at: Optional[datetime] = None
+    idle_timeout: Optional[int] = None
+    last_activity: Optional[dict] = None
     share_url: Optional[str] = None
+    organization_name: Optional[str] = None
+    web_url: Optional[str] = None
+    host: Optional[str] = None
+    port: Optional[int] = None
 
 
 def schedule_lifetime_termination(app_state):
     def terminating_server():
         pid = os.getpid()
-        logger.info(f"Terminating server process [{pid}] manually")
+        logger.info(f"Terminating server process [{pid}] manually due to lifetime expiration")
         os.kill(pid, signal.SIGINT)
 
     # Terminate the server process after the specified lifetime
@@ -67,6 +99,56 @@ def schedule_lifetime_termination(app_state):
     asyncio.get_running_loop().call_later(app_state.lifetime, terminating_server)
 
 
+def schedule_idle_timeout_check(app_state):
+    """
+    Schedule periodic checks for idle timeout.
+    If the server has been idle for longer than idle_timeout, terminate it.
+    """
+    # Track last activity time in app_state
+    app_state.last_activity = {"time": datetime.now(utc)}
+
+    def terminating_server_idle():
+        pid = os.getpid()
+        logger.info(f"Terminating server process [{pid}] manually due to idle timeout")
+        os.kill(pid, signal.SIGINT)
+
+    async def check_idle_timeout():
+        """Periodically check if the server has been idle for too long"""
+        # Use smaller check interval if idle_timeout is very short
+        # Check at least every MAX_CHECK_INTERVAL seconds, but also check when idle_timeout is approaching
+        check_interval = min(MAX_CHECK_INTERVAL, max(MIN_CHECK_INTERVAL, app_state.idle_timeout // 3))
+
+        logger.debug(f"[Idle Timeout] Starting idle timeout checker with {check_interval}s check interval")
+
+        while True:
+            await asyncio.sleep(check_interval)
+
+            idle_seconds = (datetime.now(utc) - app_state.last_activity["time"]).total_seconds()
+            remaining_seconds = app_state.idle_timeout - idle_seconds
+
+            # Always log the countdown for debugging
+            if remaining_seconds > 0:
+                logger.debug(
+                    f"[Idle Timeout] Server idle for {idle_seconds:.1f}s / {app_state.idle_timeout}s "
+                    f"(remaining: {remaining_seconds:.1f}s)"
+                )
+
+            if idle_seconds >= app_state.idle_timeout:
+                logger.info(
+                    f"[Idle Timeout] Threshold reached! Server has been idle for {idle_seconds:.0f} seconds "
+                    f"(threshold: {app_state.idle_timeout} seconds)"
+                )
+                terminating_server_idle()
+                break
+
+    # Start the idle timeout check task
+    logger.info(f"[Configuration] The idle timeout of the server is {app_state.idle_timeout} seconds")
+
+    # Create task using asyncio.create_task which works in async context
+    task = asyncio.create_task(check_idle_timeout())
+    logger.debug(f"[Idle Timeout] Background task created: {task}")
+
+
 def setup_server(app_state: AppState) -> RecceContext:
     from rich.console import Console
 
@@ -96,39 +178,67 @@ def setup_server(app_state: AppState) -> RecceContext:
 
     log_load_state(command="server", single_env=single_env)
 
-    if app_state.lifetime is not None and app_state.lifetime > 0:
-        schedule_lifetime_termination(app_state)
-
     return ctx
 
 
 def teardown_server(app_state: AppState, ctx: RecceContext):
-    state_loader = app_state.state_loader
+    # pull latest state, merge runs/checks and pick the newer artifacts
+    state_loader = ctx.state_loader
+    state_loader.refresh()
+    if state_loader.state:
+        ctx.import_state(state_loader.state, merge=True)
     state_loader.export(ctx.export_state())
-
     ctx.stop_monitor_artifacts()
     if app_state.flag.get("single_env_onboarding", False):
         ctx.stop_monitor_base_env()
 
 
 def setup_ready_only(app_state: AppState):
-    if app_state.lifetime is not None and app_state.lifetime > 0:
-        schedule_lifetime_termination(app_state)
+    pass
 
 
 def teardown_ready_only(app_state: AppState):
     pass
 
 
+def setup_preview(app_state: AppState):
+    state_loader = app_state.state_loader
+    kwargs = app_state.kwargs
+    ctx = load_context(**kwargs, state_loader=state_loader)
+    return ctx
+
+
+def teardown_preview(app_state: AppState, ctx: RecceContext):
+    state_loader = app_state.state_loader
+    state_loader.export(ctx.export_state())
+    pass
+
+
 @asynccontextmanager
 async def lifespan(fastapi: FastAPI):
     ctx = None
     app_state: AppState = app.state
 
+    # Ensure logger is at DEBUG level if debug mode is enabled
+    # This is needed because uvicorn might reset logger configuration
+    if hasattr(app_state, "kwargs") and app_state.kwargs.get("debug"):
+        logger.setLevel(logging.DEBUG)
+        logger.debug("Debug mode enabled - logger set to DEBUG level")
+
     if app_state.command == "server":
         ctx = setup_server(app_state)
-    elif app_state.command == "read_only":
+    elif app_state.command == "read-only":
         setup_ready_only(app_state)
+    elif app_state.command == "preview":
+        ctx = setup_preview(app_state)
+
+    if app_state.lifetime is not None and app_state.lifetime > 0:
+        schedule_lifetime_termination(app_state)
+
+    # Schedule idle timeout check if idle_timeout is set
+    if app_state.idle_timeout is not None and app_state.idle_timeout > 0:
+        logger.debug(f"[Idle Timeout] Scheduling idle timeout check with {app_state.idle_timeout} seconds")
+        schedule_idle_timeout_check(app_state)
 
     yield
 
@@ -136,6 +246,8 @@ async def lifespan(fastapi: FastAPI):
         teardown_server(app_state, ctx)
     elif app_state.command == "read_only":
         teardown_ready_only(app_state)
+    elif app_state.command == "preview":
+        teardown_preview(app_state, ctx)
 
 
 app = FastAPI(lifespan=lifespan)
@@ -143,7 +255,7 @@ app = FastAPI(lifespan=lifespan)
 
 def verify_json_file(file_path: str) -> bool:
     try:
-        with open(file_path, "r") as f:
+        with open(file_path, "r", encoding="utf-8") as f:
             json.load(f)
     except Exception:
         return False
@@ -200,6 +312,27 @@ app.add_middleware(
 )
 
 
+@app.middleware("http")
+async def track_activity_for_idle_timeout(request: Request, call_next):
+    """Track activity time for idle timeout check"""
+    # Exclude paths that should not reset idle timer
+    # Health checks and monitoring endpoints don't count as user activity
+    excluded_paths = ["/api/health", "/api/ws"]
+
+    # Update last activity time BEFORE processing request if idle timeout is enabled
+    # This ensures long-running requests don't get terminated mid-execution
+    app_state: AppState = app.state
+    if app_state.last_activity is not None:
+        if request.url.path not in excluded_paths:
+            app_state.last_activity["time"] = datetime.now(utc)
+            logger.debug(f"[Idle Timeout] ✓ Activity detected: {request.method} {request.url.path} - Timer reset")
+        else:
+            logger.debug(f"[Idle Timeout] Excluded path (no timer reset): {request.method} {request.url.path}")
+
+    response = await call_next(request)
+    return response
+
+
 @app.middleware("http")
 async def set_context_by_cookie(request: Request, call_next):
     response = await call_next(request)
@@ -233,11 +366,17 @@ async def health_check(request: Request):
 
 
 class RecceInstanceInfoOut(BaseModel):
+    server_mode: RecceServerMode
     read_only: bool
+    preview: bool
     single_env: bool
     authed: bool
+    cloud_instance: bool
     lifetime_expired_at: Optional[datetime] = None
     share_url: Optional[str] = None
+    session_id: Optional[str] = None
+    organization_name: Optional[str] = None
+    web_url: Optional[str] = None
 
 
 @app.get("/api/instance-info", response_model=RecceInstanceInfoOut, response_model_exclude_none=True)
@@ -247,15 +386,20 @@ async def recce_instance_info():
     read_only = flag.get("read_only", False)
     single_env = flag.get("single_env_onboarding", False)
 
-    auth_options = app_state.auth_options or {}
-    api_token = auth_options.get("api_token")
+    api_token = get_recce_api_token()
 
     return {
+        "server_mode": app_state.command,
         "read_only": read_only,
+        "preview": flag.get("preview", False),
         "single_env": single_env,
         "authed": True if api_token else False,
+        "cloud_instance": is_recce_cloud_instance(),
         "lifetime_expired_at": app_state.lifetime_expired_at,  # UTC timezone
         "share_url": app_state.share_url,
+        "session_id": app_state.state_loader.session_id if app_state.state_loader else None,
+        "organization_name": app_state.organization_name,
+        "web_url": app_state.web_url,
         # TODO: Add more instance info which won't change during the instance lifecycle
         # review_mode
         # cloud_mode
@@ -283,6 +427,7 @@ async def get_info():
     """
     context = default_context()
     demo = os.environ.get("DEMO", False)
+    is_codespace = is_github_codespace()
 
     if demo:
         state = context.export_demo_state()
@@ -307,6 +452,7 @@ async def get_info():
             "pull_request": state.pull_request.to_dict() if state.pull_request else None,
             "lineage": lineage_diff,
             "demo": bool(demo),
+            "codespace": bool(is_codespace),
             "cloud_mode": context.state_loader.cloud_mode,
             "file_mode": context.state_loader.state_file is not None,
             "filename": filename,
@@ -331,9 +477,9 @@ class CllIn(BaseModel):
     node_id: Optional[str] = None
     column: Optional[str] = None
     change_analysis: Optional[bool] = False
-    cll: Optional[bool] = False
-    upstream: Optional[bool] = False
-    downstream: Optional[bool] = False
+    no_cll: Optional[bool] = False
+    no_upstream: Optional[bool] = False
+    no_downstream: Optional[bool] = False
 
 
 class CllOutput(BaseModel):
@@ -349,8 +495,9 @@ async def column_level_lineage_by_node(cll_input: CllIn):
         node_id=cll_input.node_id,
         column=cll_input.column,
         change_analysis=cll_input.change_analysis,
-        upstream=cll_input.upstream,
-        downstream=cll_input.downstream,
+        no_upstream=cll_input.no_upstream,
+        no_downstream=cll_input.no_downstream,
+        no_cll=cll_input.no_cll,
     )
 
     return CllOutput(current=cll)
@@ -654,6 +801,34 @@ async def broadcast(data: str):
         await client.send_text(data)
 
 
+@app.post("/api/connect")
+async def generate_connect_to_cloud_url(background_tasks: BackgroundTasks):
+    if is_callback_server_running():
+        return {"connection_url": get_connection_url()}
+
+    private_key, public_key = generate_key_pair()
+    connection_url, callback_port = prepare_connection_url(public_key)
+
+    background_tasks.add_task(connect_to_cloud_background_task, private_key, callback_port, connection_url)
+    return {
+        "connection_url": connection_url,
+    }
+
+
+@app.get("/api/users")
+async def get_user_info():
+    from recce.connect_to_cloud import RecceCloud
+
+    context = default_context()
+    user_token = get_recce_api_token() or context.state_loader.token
+    cloud = RecceCloud(user_token)
+    try:
+        user_info = cloud.get_user_info()
+        return user_info
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
 api_prefix = "/api"
 app.include_router(check_router, prefix=api_prefix)
 app.include_router(run_router, prefix=api_prefix)

recce/state/__init__.py

@@ -0,0 +1,31 @@
+from .cloud import (
+    CloudStateLoader,
+    RecceCloudStateManager,
+    RecceShareStateManager,
+    s3_sse_c_headers,
+)
+from .const import ErrorMessage
+from .local import FileStateLoader
+from .state import (
+    ArtifactsRoot,
+    GitRepoInfo,
+    PullRequestInfo,
+    RecceState,
+    RecceStateMetadata,
+)
+from .state_loader import RecceStateLoader
+
+__all__ = [
+    "ArtifactsRoot",
+    "ErrorMessage",
+    "RecceCloudStateManager",
+    "RecceShareStateManager",
+    "RecceState",
+    "RecceStateLoader",
+    "CloudStateLoader",
+    "FileStateLoader",
+    "RecceStateMetadata",
+    "s3_sse_c_headers",
+    "GitRepoInfo",
+    "PullRequestInfo",
+]