comet-hunter 0.1.0__py3-none-any.whl

backend/api/dependencies.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+from backend.database.infrastructure.query_executor import QueryExecutor
+from backend.database.repositories.file_metadata_repository import FileMetadataRepository
+from backend.database.repositories.processed_file_repository import ProcessedFileRepository
+from backend.database.repositories.downlink_slot_repository import DownlinkSlotRepository
+from backend.services.metadata_service import MetadataService
+from backend.services.download_file_service import DownloadFileService
+from backend.services.process_file_service import ProcessFileService
+from backend.services.slot_service import SlotService
+from backend.pipeline.pipeline import Pipeline
+from backend.pipeline.scheduler import Scheduler
+from backend.jobs.job_store import job_store, JobStore
+from backend.jobs.background_job_service import BackgroundJobService
+from backend.jobs.event_bus import event_bus, EventBus
+from backend.core.storage import RAW_DIR, PROCESSED_DIR
+
+background_job_service = BackgroundJobService(job_store)
+
+executor = QueryExecutor()
+
+metadata_repo = FileMetadataRepository(executor)
+processed_repo = ProcessedFileRepository(executor)
+slot_repo = DownlinkSlotRepository(executor)
+
+metadata_service = MetadataService(metadata_repo)
+slot_service = SlotService(slot_repo)
+download_service = DownloadFileService(
+    processed_repository=processed_repo,
+    metadata_service=metadata_service,
+    download_directory=RAW_DIR,
+)
+process_service = ProcessFileService(
+    processed_repository=processed_repo,
+    metadata_service=metadata_service,
+    processed_directory=PROCESSED_DIR,
+)
+
+pipeline = Pipeline(
+    slot_service=slot_service,
+    metadata_service=metadata_service,
+    download_service=download_service,
+    process_service=process_service,
+)
+
+scheduler = Scheduler(pipeline)
+
+def get_pipeline() -> Pipeline:
+    """
+    Returns shared pipeline instance for handling API requests and background jobs.
+    """
+    return pipeline
+
+def get_scheduler() -> Scheduler:
+    """
+    Returns shared scheduler instance for triggering background jobs.
+    """
+    return scheduler
+
+def get_job_store() -> JobStore:
+    """
+    Returns a shared job_store instance.
+    """
+    return job_store
+
+def get_job_service() -> BackgroundJobService:
+    """
+    Returns shared background job service instance.
+    Used for submitting and managing background jobs.
+    """
+    return background_job_service
+
+def get_event_bus() -> EventBus:
+    """
+    Returns shared event bus instance. 
+    """
+
+    return event_bus

backend/api/dto/api_response.py

@@ -0,0 +1,17 @@
+from pydantic import BaseModel
+from pydantic.generics import GenericModel
+from typing import Generic, TypeVar
+
+T = TypeVar('T')
+
+class ApiSuccessResponse(GenericModel, Generic[T]):
+    success: bool = True
+    data: T
+
+class ApiErrorDetail(BaseModel):
+    code: str
+    message: str
+
+class ApiErrorResponse(BaseModel):
+    success: bool = False
+    error: ApiErrorDetail

backend/api/dto/response_models.py

@@ -0,0 +1,65 @@
+from backend.util.enums import Instrument, JobType, JobStatus
+from pydantic import BaseModel
+from datetime import datetime, timedelta
+from typing import Optional, Any
+
+class ProcessedFileResponse(BaseModel):
+    processed_file_name: str
+    instrument: Instrument
+    processed_file_url: str
+    datetime_of_observation: datetime 
+
+class GetFramesResponse(BaseModel):
+    files: list[ProcessedFileResponse]
+    total: int
+    limit: int
+    offset: int
+
+class SyncFramesResponse(BaseModel):
+    metadata_synced: int
+    downloaded: int
+    marked_ready: int
+    processed: int
+
+class SyncSlotsResponse(BaseModel):
+    status: str
+    slots_synced: int
+
+class SchedulerStatusResponse(BaseModel):
+    running: bool
+    next_run_at: Optional[datetime]
+    next_run_in: Optional[timedelta]
+
+class SchedulerStartResponse(BaseModel):
+    started: bool
+    running: bool
+
+class SchedulerStopResponse(BaseModel):
+    stopped: bool
+    running: bool
+
+class HealthResponse(BaseModel):
+    status: str
+    database: bool
+    scheduler_initialized: bool
+    pipeline_initialized: bool
+
+class SlotResponse(BaseModel):
+    start: Optional[datetime]
+    end: Optional[datetime]
+
+class JobQueuedResponse(BaseModel):
+    job_id: str
+    existing: bool
+    status: str
+
+class JobStatusResponse(BaseModel):
+    job_id: str
+    type: JobType
+    status: JobStatus
+    created_at: datetime
+    started_at: Optional[datetime] = None
+    stopped_at: Optional[datetime] = None
+    progress: Optional[Any] = None
+    result: Optional[Any] = None
+    error: Optional[str] = None

backend/api/dto/serializers.py

@@ -0,0 +1,31 @@
+from backend.api.dto.response_models import GetFramesResponse, ProcessedFileResponse
+from backend.database.domain.processed_file import ProcessedFile
+from backend.pipeline.models import GetProcessedFramesResult
+
+def serialize_processed_file(
+    file: ProcessedFile
+) -> ProcessedFileResponse:
+    """
+    Converts ProcessedFile domain model to ProcessedFileResponse DTO
+    """
+
+    return ProcessedFileResponse(
+        processed_file_name=file.processed_file_name,
+        instrument=file.instrument,
+        processed_file_url=f"/media/{str(file.processed_file_name)}",
+        datetime_of_observation=file.datetime_of_observation
+    )
+
+def serialize_get_frames_response(result: GetProcessedFramesResult) -> GetFramesResponse:
+    """
+    Converts GetProcessedFramesResult pipeline result domain model to GetFramesResponse DTO
+    """
+    return GetFramesResponse(
+        files=[
+            serialize_processed_file(file) 
+            for file in result.processed_files
+        ],
+        total=result.total,
+        limit=result.limit,
+        offset=result.offset
+    )

backend/api/middleware.py

@@ -0,0 +1,44 @@
+import logging
+import time
+import uuid
+from starlette.middleware.base import BaseHTTPMiddleware
+from backend.core.request_context import set_request_id
+
+logger = logging.getLogger("api")
+
+class LoggingMiddleware(BaseHTTPMiddleware):
+
+    async def dispatch(self, request, call_next):
+        request_id = str(uuid.uuid4())
+        set_request_id(request_id)
+
+        start = time.time()
+
+        try:
+            response = await call_next(request)
+            duration = time.time() - start
+
+            logger.info(
+                "Request completed",
+                extra={
+                    "request_id": request_id,
+                    "method": request.method,
+                    "path": request.url.path,
+                    "status": response.status_code,
+                    "duration_ms": round(duration * 1000, 2)
+                }
+            )
+
+            return response
+        
+        except Exception:
+            logger.exception(
+                "Request failed",
+                extra={
+                    "request_id": request_id,
+                    "method": request.method,
+                    "path": request.url.path
+                }
+            )
+
+            raise

backend/api/routes/frames.py

@@ -0,0 +1,104 @@
+from fastapi import APIRouter, Query, Depends
+from backend.api.dependencies import get_pipeline, get_job_service
+from backend.util.enums import Instrument, JobType
+from backend.api.dto.api_response import ApiSuccessResponse
+from backend.api.dto.response_models import JobQueuedResponse, GetFramesResponse
+from backend.pipeline.pipeline import Pipeline
+from backend.jobs.background_job_service import BackgroundJobService
+from backend.api.dto.serializers import serialize_get_frames_response
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+@router.get("/", response_model=ApiSuccessResponse[GetFramesResponse])
+def get_processed_frames(
+    instrument: Instrument = Query(...),
+    start: str = Query(..., description = "ISO UTC observation start time"),
+    end: str = Query(..., description = "ISO UTC observation end time"),
+    limit: int = Query(15, ge=1, le=100),
+    offset: int = Query(0, ge=0),
+    pipeline: Pipeline = Depends(get_pipeline)
+) -> ApiSuccessResponse[GetFramesResponse]:
+    """
+    Retrieve of processed frames for a given observation window and instrument
+    """
+    logger.info(
+        "Processed frames retrieval requested",
+        extra={
+            "instrument":instrument,
+            "observation_start_utc":start,
+            "observation_end_utc":end
+        }
+    )
+    try:
+
+        result = pipeline.get_processed_frames(
+            instrument,
+            start,
+            end,
+            limit,
+            offset
+        )
+
+        logger.info("Processed frames retrieval successful")
+
+        return ApiSuccessResponse[GetFramesResponse](
+            data=serialize_get_frames_response(result)
+        )
+        
+    except Exception:
+        logger.exception("Processed frames retrieval failed")
+        raise
+
+@router.post("/sync", response_model=ApiSuccessResponse[JobQueuedResponse])
+def sync_processed_frames(
+    instrument: Instrument = Query(...),
+    start: str = Query(..., description = "ISO UTC observation start time"),
+    end: str = Query(..., description = "ISO UTC observation end time"),
+    pipeline: Pipeline = Depends(get_pipeline),
+    background_job_service: BackgroundJobService = Depends(get_job_service)
+) -> ApiSuccessResponse[JobQueuedResponse]:
+    """
+    Trigger the job for syncing and updating processed frames based on 
+    instrument and observation time
+
+    - Triggers metadata sync
+    - Downloads missing files
+    - Processes eligible frames
+    - Returns details of the sync operation including counts of metadata synced, files downloaded, frames marked ready and frames processed
+    """
+    logger.info(
+        "Processed frames sync job requested",
+        extra={
+            "instrument":instrument,
+            "observation_start_utc":start,
+            "observation_end_utc":end
+        }
+    )
+    try:
+        job = background_job_service.submit(
+            JobType.SYNC_PROCESSED_FRAMES,
+            pipeline.sync_processed_frames,
+            instrument,
+            start,
+            end
+        )
+
+        if job.existing:
+            logger.warning("Processed frames sync job already running",
+                           extra={"job_id": job.job.id})
+        else:
+            logger.info("Processed frames sync job queued",
+                    extra={"job_id": job.job.id})
+
+        return ApiSuccessResponse[JobQueuedResponse](
+            data=JobQueuedResponse(
+                job_id=job.job.id,
+                existing=job.existing,
+                status=job.job.status.value
+            )
+        )
+    except Exception:
+        logger.exception("Processed frames sync failed to start")
+        raise

backend/api/routes/health.py

@@ -0,0 +1,53 @@
+import logging
+from fastapi import APIRouter, Depends
+from backend.api.dto.api_response import ApiSuccessResponse
+from backend.api.dto.response_models import HealthResponse
+from backend.api.dependencies import get_pipeline, get_scheduler
+from backend.pipeline.pipeline import Pipeline
+from backend.pipeline.scheduler import Scheduler
+from backend.database.infrastructure.base import DatabaseBase
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+@router.get("/", response_model=ApiSuccessResponse[HealthResponse])
+def health(
+    pipeline: Pipeline = Depends(get_pipeline),
+    scheduler: Scheduler = Depends(get_scheduler)
+) -> ApiSuccessResponse[HealthResponse]:
+    """
+    Checks server health and verifies initialization of
+    - database connectivity
+    - scheduler status
+    - pipeline availability
+    """
+    logger.debug("Health check requested")
+    try:
+        with DatabaseBase.get_connection() as conn:
+            conn.execute("SELECT 1")
+        db_ok = True
+    except Exception:
+        db_ok = False
+    
+    scheduler_ok = scheduler is not None
+    pipeline_ok = pipeline is not None
+    healthy = db_ok and scheduler_ok and pipeline_ok
+    status = "healthy" if healthy else "degraded"
+    
+    logger.debug(
+        "Health check completed",
+        extra={
+            "status": status,
+            "database": db_ok,
+            "scheduler_initialized": scheduler_ok,
+            "pipeline_initialized": pipeline_ok
+        }
+    )
+    return ApiSuccessResponse[HealthResponse](
+        data = HealthResponse(
+            status=status,
+            database=db_ok,
+            scheduler_initialized=scheduler_ok,
+            pipeline_initialized=pipeline_ok
+        )
+    )

backend/api/routes/jobs.py

@@ -0,0 +1,274 @@
+import logging
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import StreamingResponse
+from queue import Empty
+from typing import Generator
+from dataclasses import asdict
+import json
+import time
+from backend.jobs.background_job_service import BackgroundJobService
+from backend.jobs.event_bus import EventBus
+from backend.jobs.job_store import JobStore
+from backend.api.dependencies import get_job_service, get_job_store, get_event_bus
+from backend.api.dto.api_response import ApiSuccessResponse
+from backend.api.dto.response_models import JobStatusResponse 
+from backend.util.enums import JobStatus
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+@router.get("/{job_id}", response_model=ApiSuccessResponse[JobStatusResponse])
+def get_job(
+    job_id: str,
+    background_job_service: BackgroundJobService = Depends(get_job_service)
+) -> ApiSuccessResponse[JobStatusResponse]:
+    """
+    Retrieve background job execution details.
+
+    Returns:
+    - current lifecycle status
+    - result payload if completed
+    - failure error if failed
+
+    :param job_id:
+        Unique identifier of the background job.
+
+    :param background_job_service:
+        Background job service dependency.
+
+    :raises HTTPException:
+        404 if job does not exist.
+
+    :return:
+        Standardized API response containing job details.
+    """
+
+    logger.info(
+        "Job status requested",
+        extra={"job_id": job_id}
+    )
+
+    try:
+
+        job = background_job_service.get_job(job_id)
+
+        if job is None:
+
+            logger.warning(
+                "Job not found",
+                extra={"job_id": job_id}
+            )
+
+            raise HTTPException(
+                status_code=404,
+                detail="Job not found"
+            )
+
+        logger.info(
+            "Job status retrieved",
+            extra={
+                "job_id": job.id,
+                "job_type": job.type,
+                "job_status": job.status.value
+            }
+        )
+
+        return ApiSuccessResponse[JobStatusResponse](
+            data=JobStatusResponse(
+                job_id=job.id,
+                type=job.type.value,
+                status=job.status.value,
+                created_at=job.created_at,
+                started_at=job.started_at,
+                stopped_at=job.stopped_at,
+                progress=job.progress,
+                result=job.result,
+                error=job.error
+            )
+        )
+
+    except HTTPException:
+        raise
+
+    except Exception:
+
+        logger.exception(
+            "Failed to retrieve job status",
+            extra={"job_id": job_id}
+        )
+        raise
+
+@router.get("/{job_id}/events")
+def stream_job_events(
+    job_id: str,
+    event_bus: EventBus = Depends(get_event_bus)
+) -> StreamingResponse:
+    """
+    Stream real-time events using server-sent events.
+    
+    Responsibilities:
+    - Subscribe client to job-specific event stream
+    - Continously stream published events.
+    - Cleanup subscription on client disconnect or job completion.
+
+    Event format:
+        "event": <event_type>,
+        "data": <json_payload>
+    
+    Example streamed payload:
+        "event": "progress",
+        "data":{"download": 12}
+
+    Notes:
+    - Connection remains open until job completion or client disconnect.
+    - Intended for real-time monitoring of long-running jobs.
+    - Uses in-memory event bus subscriptions.
+    - Each connected client gets dedicated queue.
+
+    :param job_id:
+        Unique identifier of the background job.
+    
+    :param event_bus:
+        Shared event bus dependency.
+    
+    :return:
+        StreamingResponse configured for server-sent events.
+    """
+    logger.info(
+        "Job event stream requested",
+        extra={"job_id": job_id}
+    )
+
+    def event_generator() -> Generator[str, None, None]:
+        """
+        Generate SSE-formatted event stream.
+
+        Workflow:
+        - Subscribe to event bus.
+        - Wait for events.
+        - Yield events in SSE format.
+        - Cleanup / Disconnect.
+
+        Yields:
+            str: SSE-formatted event string.
+        """
+
+        queue = event_bus.subscribe(job_id)
+
+        terminal_events = {
+            JobStatus.COMPLETED,
+            JobStatus.FAILED,
+            JobStatus.CANCELLED
+        }
+
+        try:
+            while True:
+                try:
+                    event = queue.get(timeout=5)
+
+                    payload = asdict(event)
+                    payload["event"] = payload["event"].value
+                    payload["job_status"] = payload["job_status"].value
+                    payload["job_type"] = payload["job_type"].value
+                    payload["timestamp"] = payload["timestamp"].isoformat()
+
+                    yield (
+                        f"event: {payload["event"]}\n"
+                        f"data: {json.dumps(payload)}\n\n"
+                    )
+
+                    if event.job_status in terminal_events:
+                        break
+                
+                except Empty:
+                    """
+                    Heartbeat to keep connection alive.
+
+                    Prevents:
+                    - proxy timeout
+                    - connection idle termination
+                    """
+                    yield ": keepalive\n\n"
+                
+                time.sleep(0.1)
+        except GeneratorExit:
+            logger.info(
+                "job event stream disconnected",
+                extra={"job_id": job_id}
+            )
+        finally:
+
+            event_bus.unsubscribe(job_id, queue)
+            logger.info(
+                "job event subscription cleaned up",
+                extra={"job_id": job_id}
+            )
+        
+    return StreamingResponse(
+        content = event_generator(),
+        media_type = "text/event-stream",
+        headers = {
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "X-Accel-Buffering": "no"
+        }
+    )
+
+@router.post("/{job_id}/cancel", response_model=ApiSuccessResponse[dict])
+def cancel_job(
+    job_id: str,
+    job_store: JobStore = Depends(get_job_store)
+) -> ApiSuccessResponse[dict]:
+    """
+    Trigger cancellation request for a running job.
+
+    Notes:
+    - Cancellation is cooperative.
+    - Actual job termination depends on cancellation checkpoints.
+    """
+
+    logger.info(
+        "Job Cancellation triggered",
+        extra={"job_id": job_id}
+    )
+
+    try:
+
+        job = job_store.get_job(job_id)
+
+        if not job:
+            logger.warning(
+                "Job not found",
+                extra={"job_id": job_id}
+            )
+
+            raise HTTPException(
+                status_code=404,
+                detail="job not found"
+            )
+        
+        job.cancel_event.set()
+
+        job_store.update_job(
+            job_id,
+            JobStatus.CANCELLING
+        )
+
+        logger.info(
+            "Job cancellation signal sent",
+            extra={"job_id":job_id}
+        )
+
+        return ApiSuccessResponse(
+            data={
+                "job_id":job_id,
+                "message":"Cancellation Signal Sent"
+            }
+        )
+    
+    except HTTPException:
+        raise
+    
+    except Exception:
+        logger.exception("Job cancellation failed")
+        raise