pylantir 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl

pylantir/api_server.py

@@ -21,7 +21,8 @@ from typing import List, Optional, Dict, Any
 from datetime import datetime, timedelta
 
 try:
-    from fastapi import FastAPI, HTTPException, Depends, status, Query
+    from fastapi import FastAPI, HTTPException, Depends, Query
+    from fastapi import status as http_status
     from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
     from fastapi.middleware.cors import CORSMiddleware
     from pydantic import BaseModel, validator
@@ -144,6 +145,7 @@ class WorklistItemResponse(BaseModel):
     protocol_name: Optional[str]
     station_name: Optional[str]
     performed_procedure_step_status: Optional[str]
+    data_source: Optional[str] = None
 
     class Config:
         from_attributes = True
@@ -169,6 +171,7 @@ class WorklistItemCreate(BaseModel):
     protocol_name: Optional[str] = None
     station_name: Optional[str] = None
     performed_procedure_step_status: str = "SCHEDULED"
+    data_source: Optional[str] = None
 
     @validator('performed_procedure_step_status')
     def validate_status(cls, v):
@@ -195,6 +198,7 @@ class WorklistItemUpdate(BaseModel):
     protocol_name: Optional[str] = None
     station_name: Optional[str] = None
     performed_procedure_step_status: Optional[str] = None
+    data_source: Optional[str] = None
 
     @validator('performed_procedure_step_status')
     def validate_status(cls, v):
@@ -296,7 +300,7 @@ async def get_current_user(
 
         if payload is None:
             raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
+                status_code=http_status.HTTP_401_UNAUTHORIZED,
                 detail="Invalid authentication token",
                 headers={"WWW-Authenticate": "Bearer"},
             )
@@ -304,7 +308,7 @@ async def get_current_user(
         username = payload.get("sub")
         if username is None:
             raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
+                status_code=http_status.HTTP_401_UNAUTHORIZED,
                 detail="Invalid token payload",
                 headers={"WWW-Authenticate": "Bearer"},
             )
@@ -312,7 +316,7 @@ async def get_current_user(
         user = auth_db.query(User).filter(User.username == username).first()
         if user is None or not user.is_active:
             raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
+                status_code=http_status.HTTP_401_UNAUTHORIZED,
                 detail="User not found or inactive",
                 headers={"WWW-Authenticate": "Bearer"},
             )
@@ -324,7 +328,7 @@ async def get_current_user(
     except Exception as e:
         lgr.error(f"Authentication error: {e}")
         raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
+            status_code=http_status.HTTP_401_UNAUTHORIZED,
             detail="Authentication failed",
             headers={"WWW-Authenticate": "Bearer"},
         )
@@ -344,7 +348,7 @@ def require_permission(action: str, resource: str = "worklist"):
     def permission_checker(current_user: User = Depends(get_current_user)) -> User:
         if not current_user.has_permission(action, resource):
             raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
+                status_code=http_status.HTTP_403_FORBIDDEN,
                 detail=f"Insufficient permissions for {action} on {resource}"
             )
         return current_user
@@ -364,7 +368,7 @@ async def login(
 
         if not user:
             raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
+                status_code=http_status.HTTP_401_UNAUTHORIZED,
                 detail="Invalid username or password",
                 headers={"WWW-Authenticate": "Bearer"},
             )
@@ -390,7 +394,7 @@ async def login(
     except Exception as e:
         lgr.error(f"Login error: {e}")
         raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            status_code=http_status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Login failed"
         )
 
@@ -437,7 +441,7 @@ async def get_worklist_items(
     except Exception as e:
         lgr.error(f"Error retrieving worklist items: {e}")
         raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            status_code=http_status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail="Failed to retrieve worklist items"
         )
 

pylantir/cli/run.py

@@ -160,6 +160,7 @@ def parse_args():
 def load_config(config_path=None):
     """
     Load configuration file, either from a user-provided path or the default package location.
+    Auto-converts legacy configuration format to new data_sources format.
 
     Args:
         config_path (str | Path, optional): Path to the configuration JSON file.
@@ -176,6 +177,35 @@ def load_config(config_path=None):
         with config_path.open("r") as f:
             config_data = json.load(f)
         lgr.info(f"Loaded configuration from {config_path}")
+
+        # Auto-convert legacy configuration format
+        if "data_sources" not in config_data and "redcap2wl" in config_data:
+            lgr.warning(
+                "Legacy configuration format detected. "
+                "Consider migrating to 'data_sources' format for better flexibility. "
+                "See config/mwl_config_multi_source_example.json for reference."
+            )
+
+            # Convert legacy format to data_sources array
+            legacy_source = {
+                "name": "redcap_legacy",
+                "type": "redcap",
+                "enabled": True,
+                "sync_interval": config_data.get("db_update_interval", 60),
+                "operation_interval": config_data.get(
+                    "operation_interval",
+                    {"start_time": [0, 0], "end_time": [23, 59]}
+                ),
+                "config": {
+                    "site_id": config_data.get("site"),
+                    "protocol": config_data.get("protocol", {}),
+                },
+                "field_mapping": config_data.get("redcap2wl", {})
+            }
+
+            config_data["data_sources"] = [legacy_source]
+            lgr.info("Auto-converted legacy configuration to data_sources format")
+
         return config_data
 
     except FileNotFoundError:
@@ -285,52 +315,286 @@ def main() -> None:
         # Load configuration into environment variables
         update_env_with_config(config)
 
-
         from ..mwl_server import run_mwl_server
-        from ..redcap_to_db import sync_redcap_to_db_repeatedly
-
-        # Extract the database update interval (default to 60 seconds if missing)
-        db_update_interval = config.get("db_update_interval", 60)
-
-        # Extract the operation interval (default from 00:00 to 23:59 hours if missing)
-        operation_interval = config.get("operation_interval", {"start_time": [0,0], "end_time": [23,59]})
 
         # Extract allowed AE Titles (default to empty list if missing)
         allowed_aet = config.get("allowed_aet", [])
 
-        # Extract the site id
-        site = config.get("site", None)
-
-        # Extract the redcap to worklist mapping
-        redcap2wl = config.get("redcap2wl", {})
+        # Check if using new data_sources format or legacy format
+        if "data_sources" in config:
+            # NEW: Multi-source orchestration using plugin architecture
+            lgr.info("Using new data_sources configuration format")
 
-        # EXtract protocol mapping
-        protocol = config.get("protocol", {})
+            from ..data_sources import get_plugin
+            from ..data_sources.base import PluginError
+            from ..redcap_to_db import STOP_EVENT
+            import threading
 
-        # Create and update the MWL database
-        with ThreadPoolExecutor(max_workers=2) as executor:
-            future = executor.submit(
-                sync_redcap_to_db_repeatedly,
-                site_id=site,
-                protocol=protocol,
-                redcap2wl=redcap2wl,
-                interval=db_update_interval,
-                operation_interval=operation_interval,
-            )
+            data_sources = config.get("data_sources", [])
+            enabled_sources = [src for src in data_sources if src.get("enabled", True)]
 
-                # sync_redcap_to_db(
-                #     mri_visit_mapping=mri_visit_session_mapping,
-                #     site_id=site,
-                #     protocol=protocol,
-                #     redcap2wl=redcap2wl,
-                # )
-
-            run_mwl_server(
-                host=args.ip,
-                port=args.port,
-                aetitle=args.AEtitle,
-                allowed_aets=allowed_aet,
-            )
+            if not enabled_sources:
+                lgr.warning("No enabled data sources found in configuration")
+            else:
+                lgr.info(f"Found {len(enabled_sources)} enabled data source(s)")
+
+            def sync_data_source_repeatedly(source_config):
+                """
+                Generic sync loop for any data source plugin.
+
+                This function works with any plugin type (REDCap, CSV, API, etc.)
+                by using the plugin interface rather than source-specific code.
+                """
+                source_name = source_config.get("name", "unknown")
+                source_type = source_config.get("type", "unknown")
+
+                try:
+                    # Get and instantiate the plugin
+                    PluginClass = get_plugin(source_type)
+                    lgr.info(f"[{source_name}] Initializing {source_type} plugin")
+
+                    plugin = PluginClass()
+
+                    # Validate plugin configuration
+                    plugin_config = dict(source_config.get("config", {}))
+                    if "field_mapping" in source_config:
+                        plugin_config["field_mapping"] = source_config.get("field_mapping")
+                    if "window_mode" in source_config:
+                        plugin_config["window_mode"] = source_config.get("window_mode")
+                    if "daily_window" in source_config:
+                        plugin_config["daily_window"] = source_config.get("daily_window")
+
+                    is_valid, error_msg = plugin.validate_config(plugin_config)
+                    if not is_valid:
+                        lgr.error(f"[{source_name}] Configuration validation failed: {error_msg}")
+                        return
+
+                    # Extract sync settings
+                    sync_interval = source_config.get("sync_interval", 60)
+                    operation_interval = source_config.get("operation_interval", {
+                        "start_time": [0, 0],
+                        "end_time": [23, 59]
+                    })
+
+                    lgr.info(f"[{source_name}] Starting sync loop (interval: {sync_interval}s)")
+
+                    # Import database and sync utilities
+                    from ..db_setup import Session
+                    from ..models import WorklistItem
+                    from ..redcap_to_db import generate_instance_uid, cleanup_memory_and_connections
+                    from datetime import datetime, time as dt_time, timedelta
+                    import logging
+
+                    # Parse operation interval
+                    start_h, start_m = operation_interval.get("start_time", [0, 0])
+                    end_h, end_m = operation_interval.get("end_time", [23, 59])
+                    start_time = dt_time(start_h, start_m)
+                    end_time = dt_time(end_h, end_m)
+
+                    last_sync_date = datetime.now().date() - timedelta(days=1)
+                    interval_sync = sync_interval + 300  # Overlap to avoid missing data
+
+                    # Sync loop
+                    while not STOP_EVENT.is_set():
+                        is_first_run = False
+                        extended_interval = sync_interval
+
+                        now_dt = datetime.now().replace(second=0, microsecond=0)
+                        now_time = now_dt.time()
+                        today_date = now_dt.date()
+
+                        # Only sync within operation interval
+                        if start_time <= now_time <= end_time:
+                            is_first_run = (last_sync_date != today_date)
+
+                            if is_first_run and (last_sync_date is not None):
+                                yesterday = last_sync_date
+                                dt_end_yesterday = datetime.combine(yesterday, end_time)
+                                dt_start_today = datetime.combine(today_date, start_time)
+                                delta = dt_start_today - dt_end_yesterday
+                                extended_interval = delta.total_seconds()
+                                # temporary increase interval to cover gap since last sync
+                                # extended_interval += 6000000
+                                logging.info(f"[{source_name}] First sync of the day at {now_time}")
+
+                            # Fetch entries using plugin
+                            try:
+                                fetch_interval = extended_interval if is_first_run else interval_sync
+                                field_mapping = source_config.get("field_mapping", {})
+
+                                lgr.debug(f"[{source_name}] Fetching entries (interval: {fetch_interval}s)")
+                                entries = plugin.fetch_entries(
+                                    field_mapping=field_mapping,
+                                    interval=fetch_interval
+                                )
+
+                                if entries:
+                                    lgr.info(f"[{source_name}] Fetched {len(entries)} entries")
+
+                                    # Get source-specific config
+                                    site_id = source_config.get("config", {}).get("site_id")
+                                    protocol = config.get("protocol", {})
+
+                                    # Process entries (source-agnostic)
+                                    session = Session()
+                                    try:
+                                        def _format_date(value):
+                                            if value is None:
+                                                return None
+                                            if hasattr(value, "strftime"):
+                                                return value.strftime("%Y-%m-%d")
+                                            value_str = str(value).strip()
+                                            if "-" in value_str and len(value_str) >= 10:
+                                                return value_str[:10]
+                                            if len(value_str) == 8 and value_str.isdigit():
+                                                return f"{value_str[0:4]}-{value_str[4:6]}-{value_str[6:8]}"
+                                            return value_str
+
+                                        def _format_time(value):
+                                            if value is None:
+                                                return None
+                                            if hasattr(value, "strftime"):
+                                                return value.strftime("%H:%M")
+                                            value_str = str(value).strip()
+                                            if ":" in value_str:
+                                                parts = value_str.split(":")
+                                                if len(parts) >= 2:
+                                                    hh = parts[0].zfill(2)
+                                                    mm = parts[1].zfill(2)
+                                                    return f"{hh}:{mm}"
+                                            if len(value_str) == 6 and value_str.isdigit():
+                                                return f"{value_str[0:2]}:{value_str[2:4]}"
+                                            if len(value_str) == 4 and value_str.isdigit():
+                                                return f"{value_str[0:2]}:{value_str[2:4]}"
+                                            return value_str
+
+                                        for record in entries:
+                                            patient_id = record.get("patient_id")
+                                            lgr.info(f"[{source_name}] Processing record for patient_id: {patient_id}")
+                                            if not patient_id:
+                                                lgr.info(f"[{source_name}] Skipping record with missing patient_id")
+                                                continue
+
+                                            existing_entry = session.query(WorklistItem).filter_by(patient_id=patient_id).first()
+
+                                            scheduled_start_date = _format_date(record.get("scheduled_start_date"))
+                                            scheduled_start_time = _format_time(record.get("scheduled_start_time"))
+
+                                            if existing_entry:
+                                                existing_entry.data_source = record.get("data_source") or source_name
+                                                existing_entry.scheduled_start_date = scheduled_start_date
+                                                existing_entry.scheduled_start_time = scheduled_start_time
+                                            else:
+                                                new_entry = WorklistItem(
+                                                    study_instance_uid=record.get("study_instance_uid") or generate_instance_uid(),
+                                                    patient_name=record.get("patient_name"),
+                                                    patient_id=patient_id,
+                                                    patient_birth_date=record.get("patient_birth_date"),
+                                                    patient_sex=record.get("patient_sex"),
+                                                    patient_weight_lb=record.get("patient_weight_lb"),
+                                                    accession_number=record.get("accession_number"),
+                                                    referring_physician_name=record.get("referring_physician_name"),
+                                                    modality=record.get("modality", "MR"),
+                                                    study_description=record.get("study_description"),
+                                                    scheduled_station_aetitle=record.get("scheduled_station_aetitle"),
+                                                    scheduled_start_date=scheduled_start_date,
+                                                    scheduled_start_time=scheduled_start_time,
+                                                    performing_physician=record.get("performing_physician"),
+                                                    procedure_description=record.get("procedure_description"),
+                                                    protocol_name=record.get("protocol_name") or protocol.get(site_id, "DEFAULT_PROTOCOL"),
+                                                    station_name=record.get("station_name"),
+                                                    hisris_coding_designator=record.get("hisris_coding_designator"),
+                                                    performed_procedure_step_status=record.get(
+                                                        "performed_procedure_step_status"
+                                                    ) or "SCHEDULED",
+                                                    data_source=record.get("data_source") or source_name
+                                                )
+                                                session.add(new_entry)
+
+                                        session.commit()
+                                        lgr.info(f"[{source_name}] Sync completed successfully")
+                                    except Exception as e:
+                                        session.rollback()
+                                        lgr.error(f"[{source_name}] Database error: {e}")
+                                    finally:
+                                        session.expunge_all()
+                                        session.close()
+                                        cleanup_memory_and_connections()
+
+                                last_sync_date = today_date
+
+                            except Exception as e:
+                                lgr.error(f"[{source_name}] Sync error: {e}")
+                                import traceback
+                                traceback.print_exc()
+
+                        # Wait before next iteration
+                        STOP_EVENT.wait(sync_interval)
+
+                    lgr.info(f"[{source_name}] Exiting sync loop (STOP_EVENT set)")
+
+                except PluginError as e:
+                    lgr.error(f"[{source_name}] Plugin error: {e}")
+                except Exception as e:
+                    lgr.error(f"[{source_name}] Unexpected error: {e}")
+                    import traceback
+                    traceback.print_exc()
+
+            # Start a thread for each enabled data source
+            max_workers = len(enabled_sources) + 1  # +1 for MWL server
+            with ThreadPoolExecutor(max_workers=max_workers) as executor:
+                # Submit sync tasks for each data source
+                for source in enabled_sources:
+                    source_name = source.get("name", "unknown")
+                    lgr.info(f"Starting background sync for data source: {source_name}")
+                    executor.submit(sync_data_source_repeatedly, source)
+
+                # Start the MWL server in the main thread
+                run_mwl_server(
+                    host=args.ip,
+                    port=args.port,
+                    aetitle=args.AEtitle,
+                    allowed_aets=allowed_aet,
+                )
+
+        else:
+            # LEGACY: Fall back to old single-source configuration
+            lgr.warning("Using legacy configuration format. Consider migrating to data_sources format.")
+
+            from ..redcap_to_db import sync_redcap_to_db_repeatedly
+
+            # Extract the database update interval (default to 60 seconds if missing)
+            db_update_interval = config.get("db_update_interval", 60)
+
+            # Extract the operation interval (default from 00:00 to 23:59 hours if missing)
+            operation_interval = config.get("operation_interval", {"start_time": [0,0], "end_time": [23,59]})
+
+            # Extract the site id
+            site = config.get("site", None)
+
+            # Extract the redcap to worklist mapping
+            redcap2wl = config.get("redcap2wl", {})
+
+            # Extract protocol mapping
+            protocol = config.get("protocol", {})
+
+            # Create and update the MWL database
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                future = executor.submit(
+                    sync_redcap_to_db_repeatedly,
+                    site_id=site,
+                    protocol=protocol,
+                    redcap2wl=redcap2wl,
+                    interval=db_update_interval,
+                    operation_interval=operation_interval,
+                )
+
+                run_mwl_server(
+                    host=args.ip,
+                    port=args.port,
+                    aetitle=args.AEtitle,
+                    allowed_aets=allowed_aet,
+                )
 
     if (args.command == "query-db"):
         from ..mwl_server import run_mwl_server
@@ -374,14 +638,16 @@ def main() -> None:
         try:
             # Check if API dependencies are available
             import uvicorn
-            from ..api_server import app
-            from ..auth_db_setup import init_auth_database, create_initial_admin_user
 
             # Load configuration for database setup
             config = load_config(args.pylantir_config)
             update_env_with_config(config)
             users_db_path = config.get("users_db_path")  # Optional users database path
 
+            # Import API app after env vars are set (DB_PATH, DB_ECHO, etc.)
+            from ..api_server import app
+            from ..auth_db_setup import init_auth_database, create_initial_admin_user
+
             # Initialize authentication database with configured path
             init_auth_database(users_db_path)
             create_initial_admin_user(users_db_path)
@@ -482,7 +748,7 @@ def main() -> None:
             email = args.email or input("Enter email (optional): ") or None
             full_name = args.full_name or input("Enter full name (optional): ") or None
             password = args.password or getpass.getpass("Enter password for new user: ")
-            
+
             # Get user role with interactive prompt
             if args.role == "read":  # Default value, prompt for role
                 print("\nAvailable user roles:")

pylantir/config/calpendo_config_example.json

@@ -0,0 +1,65 @@
+{
+    "db_path": "~/Desktop/worklist.db",
+    "db_echo": "False",
+    "db_update_interval": 60,
+    "operation_interval": {
+        "start_time": [
+            0,
+            0
+        ],
+        "end_time": [
+            23,
+            59
+        ]
+    },
+    "allowed_aet": [
+        "MRI_SCANNER",
+        "CT_SCANNER"
+    ],
+    "data_sources": [
+        {
+            "type": "calpendo",
+            "enabled": true,
+            "config": {
+                "base_url": "https://sfc-calgary.calpendo.com",
+                "resources": [
+                    "3T Diagnostic",
+                    "EEG Lab"
+                ],
+                "status_filter": "Approved",
+                "lookback_multiplier": 2,
+                "timezone": "America/Edmonton",
+                "resource_modality_mapping": {
+                    "3T": "MR",
+                    "EEG": "EEG",
+                    "Mock": "OT"
+                },
+                "field_mapping": {
+                    "patient_id": {
+                        "source_field": "title",
+                        "_extract": {
+                            "pattern": "^([A-Z0-9]+)_.*",
+                            "group": 1
+                        }
+                    },
+                    "patient_name": {
+                        "source_field": "title",
+                        "_extract": {
+                            "pattern": "^[A-Z0-9]+_(.+)$",
+                            "group": 1
+                        }
+                    },
+                    "study_description": {
+                        "source_field": "properties.project.formattedName",
+                        "_extract": {
+                            "pattern": "^([^(]+)",
+                            "group": 1
+                        }
+                    },
+                    "accession_number": "id",
+                    "study_instance_uid": "id"
+                }
+            }
+        }
+    ]
+}

pylantir/config/mwl_config.json

@@ -53,5 +53,7 @@
         "cors_allow_headers": [
             "*"
         ]
-    }
+    },
+    "_comment_data_sources": "Optional: Configure data sources to fetch worklist entries from external systems. See calpendo_config_example.json for detailed configuration.",
+    "data_sources": []
 }

pylantir/data_sources/__init__.py

@@ -0,0 +1,84 @@
+"""
+Data Source Plugin Registry
+
+This module provides the plugin registry system for discovering and loading
+data source plugins.
+
+Version: 1.0.0
+"""
+
+from .base import DataSourcePlugin, PluginError, PluginConfigError, PluginFetchError
+from .redcap_plugin import REDCapPlugin
+from .calpendo_plugin import CalendoPlugin
+from typing import Type, Dict, List
+import logging
+
+lgr = logging.getLogger(__name__)
+
+# Plugin Registry - maps source type names to plugin classes
+PLUGIN_REGISTRY: Dict[str, Type[DataSourcePlugin]] = {
+    "redcap": REDCapPlugin,
+    "calpendo": CalendoPlugin,
+}
+
+
+def register_plugin(source_type: str, plugin_class: Type[DataSourcePlugin]) -> None:
+    """
+    Register a plugin class in the registry.
+
+    Args:
+        source_type: Type identifier for the plugin (e.g., "redcap", "csv")
+        plugin_class: Plugin class inheriting from DataSourcePlugin
+
+    Raises:
+        ValueError: If source_type already registered or plugin_class is invalid
+    """
+    if source_type in PLUGIN_REGISTRY:
+        raise ValueError(f"Plugin type '{source_type}' is already registered")
+
+    if not issubclass(plugin_class, DataSourcePlugin):
+        raise ValueError(f"Plugin class must inherit from DataSourcePlugin")
+
+    PLUGIN_REGISTRY[source_type] = plugin_class
+    lgr.info(f"Registered plugin: {source_type} -> {plugin_class.__name__}")
+
+
+def get_plugin(source_type: str) -> Type[DataSourcePlugin]:
+    """
+    Retrieve a plugin class from the registry.
+
+    Args:
+        source_type: Type identifier for the plugin
+
+    Returns:
+        Plugin class for instantiation
+
+    Raises:
+        ValueError: If source_type is not registered
+    """
+    if source_type not in PLUGIN_REGISTRY:
+        available_types = list(PLUGIN_REGISTRY.keys())
+        raise ValueError(
+            f"Unknown data source type: '{source_type}'. "
+            f"Available types: {available_types}"
+        )
+
+    return PLUGIN_REGISTRY[source_type]
+
+
+def list_available_plugins() -> List[str]:
+    """Return list of registered plugin type names."""
+    return list(PLUGIN_REGISTRY.keys())
+
+
+# Export public API
+__all__ = [
+    'DataSourcePlugin',
+    'PluginError',
+    'PluginConfigError',
+    'PluginFetchError',
+    'PLUGIN_REGISTRY',
+    'register_plugin',
+    'get_plugin',
+    'list_available_plugins',
+]