webscout 8.3.3__py3-none-any.whl → 8.3.5__py3-none-any.whl

webscout/auth/config.py

@@ -2,6 +2,7 @@
 Configuration management for the Webscout API server.
 """
 
+import os
 from typing import List, Dict, Optional, Any
 from webscout.Litlogger import Logger, LogLevel, LogFormat, ConsoleHandler
 import sys
@@ -19,6 +20,42 @@ logger = Logger(
 )
 
 
+def _get_supabase_url() -> Optional[str]:
+    """Get Supabase URL from environment variables or GitHub secrets."""
+    # Try environment variable first
+    url = os.getenv("SUPABASE_URL")
+    if url:
+        logger.info("📍 Using SUPABASE_URL from environment")
+        return url
+    
+    # Try to get from GitHub secrets (if running in GitHub Actions)
+    github_url = os.getenv("GITHUB_SUPABASE_URL")  # GitHub Actions secret
+    if github_url:
+        logger.info("📍 Using SUPABASE_URL from GitHub secrets")
+        return github_url
+    
+    # Don't log error during import - only when actually needed
+    return None
+
+
+def _get_supabase_anon_key() -> Optional[str]:
+    """Get Supabase anon key from environment variables or GitHub secrets."""
+    # Try environment variable first
+    key = os.getenv("SUPABASE_ANON_KEY")
+    if key:
+        logger.info("🔑 Using SUPABASE_ANON_KEY from environment")
+        return key
+    
+    # Try to get from GitHub secrets (if running in GitHub Actions)
+    github_key = os.getenv("GITHUB_SUPABASE_ANON_KEY")  # GitHub Actions secret
+    if github_key:
+        logger.info("🔑 Using SUPABASE_ANON_KEY from GitHub secrets")
+        return github_key
+    
+    # Don't log error during import - only when actually needed
+    return None
+
+
 class ServerConfig:
     """Centralized configuration management for the API server."""
 
@@ -33,9 +70,33 @@ class ServerConfig:
         self.cors_origins: List[str] = ["*"]
         self.max_request_size: int = 10 * 1024 * 1024  # 10MB
         self.request_timeout: int = 300  # 5 minutes
-        self.auth_required: bool = True  # New: Enable/disable authentication
-        self.rate_limit_enabled: bool = True  # New: Enable/disable rate limiting
+        self.auth_required: bool = os.getenv("WEBSCOUT_AUTH_REQUIRED", "false").lower() == "true"  # Default to no auth
+        self.rate_limit_enabled: bool = os.getenv("WEBSCOUT_RATE_LIMIT_ENABLED", "false").lower() == "true"  # Default to no rate limit
         self.default_rate_limit: int = 60  # Default rate limit for no-auth mode
+        self.request_logging_enabled: bool = os.getenv("WEBSCOUT_REQUEST_LOGGING", "true").lower() == "true"  # Enable request logging by default
+        
+        # Database configuration - lazy initialization
+        self._supabase_url: Optional[str] = None
+        self._supabase_anon_key: Optional[str] = None
+        self._supabase_url_checked: bool = False
+        self._supabase_anon_key_checked: bool = False
+        self.mongodb_url: Optional[str] = os.getenv("MONGODB_URL")
+
+    @property
+    def supabase_url(self) -> Optional[str]:
+        """Get Supabase URL with lazy initialization."""
+        if not self._supabase_url_checked:
+            self._supabase_url = _get_supabase_url()
+            self._supabase_url_checked = True
+        return self._supabase_url
+
+    @property
+    def supabase_anon_key(self) -> Optional[str]:
+        """Get Supabase anon key with lazy initialization."""
+        if not self._supabase_anon_key_checked:
+            self._supabase_anon_key = _get_supabase_anon_key()
+            self._supabase_anon_key_checked = True
+        return self._supabase_anon_key
 
     def update(self, **kwargs) -> None:
         """Update configuration with provided values."""
@@ -62,9 +123,42 @@ class AppConfig:
     default_provider = "ChatGPT"
     default_tti_provider = "PollinationsAI"  # Add default TTI provider
     base_url: Optional[str] = None
-    auth_required: bool = True  # New: Enable/disable authentication
-    rate_limit_enabled: bool = True  # New: Enable/disable rate limiting
+    auth_required: bool = os.getenv("WEBSCOUT_AUTH_REQUIRED", "false").lower() == "true"  # Default to no auth
+    rate_limit_enabled: bool = os.getenv("WEBSCOUT_RATE_LIMIT_ENABLED", "false").lower() == "true"  # Default to no rate limit
     default_rate_limit: int = 60  # Default rate limit for no-auth mode
+    request_logging_enabled: bool = os.getenv("WEBSCOUT_REQUEST_LOGGING", "true").lower() == "true"  # Enable request logging by default
+    
+    # Database configuration - lazy initialization
+    _supabase_url: Optional[str] = None
+    _supabase_anon_key: Optional[str] = None
+    _supabase_url_checked: bool = False
+    _supabase_anon_key_checked: bool = False
+    mongodb_url: Optional[str] = os.getenv("MONGODB_URL")
+
+    @classmethod
+    def get_supabase_url(cls) -> Optional[str]:
+        """Get Supabase URL with lazy initialization."""
+        if not cls._supabase_url_checked:
+            cls._supabase_url = _get_supabase_url()
+            cls._supabase_url_checked = True
+        return cls._supabase_url
+
+    @classmethod
+    def get_supabase_anon_key(cls) -> Optional[str]:
+        """Get Supabase anon key with lazy initialization."""
+        if not cls._supabase_anon_key_checked:
+            cls._supabase_anon_key = _get_supabase_anon_key()
+            cls._supabase_anon_key_checked = True
+        return cls._supabase_anon_key
+
+    # For backward compatibility, provide properties that call the methods
+    @property
+    def supabase_url(self) -> Optional[str]:
+        return self.__class__.get_supabase_url()
+
+    @property
+    def supabase_anon_key(self) -> Optional[str]:
+        return self.__class__.get_supabase_anon_key()
 
     @classmethod
     def set_config(cls, **data):
@@ -72,5 +166,10 @@ class AppConfig:
         for key, value in data.items():
             setattr(cls, key, value)
         # Sync with new config system
-        from .server import config
-        config.update(**data)
+        try:
+            from .server import get_config
+            config = get_config()
+            config.update(**data)
+        except ImportError:
+            # Handle case where server module is not available
+            pass

webscout/auth/database.py

@@ -15,7 +15,13 @@ try:
 except ImportError:
     HAS_MOTOR = False
 
-from .models import User, APIKey, RateLimitEntry
+try:
+    from supabase import create_client, Client #type: ignore
+    HAS_SUPABASE = True
+except ImportError:
+    HAS_SUPABASE = False
+
+from .models import User, APIKey, RateLimitEntry, RequestLog
 
 logger = logging.getLogger(__name__)
 
@@ -30,11 +36,12 @@ class JSONDatabase:
         self.users_file = self.data_dir / "users.json"
         self.api_keys_file = self.data_dir / "api_keys.json"
         self.rate_limits_file = self.data_dir / "rate_limits.json"
+        self.request_logs_file = self.data_dir / "request_logs.json"
         
         self._lock = threading.RLock()
         
         # Initialize files if they don't exist
-        for file_path in [self.users_file, self.api_keys_file, self.rate_limits_file]:
+        for file_path in [self.users_file, self.api_keys_file, self.rate_limits_file, self.request_logs_file]:
             if not file_path.exists():
                 self._write_json(file_path, [])
     
@@ -171,6 +178,22 @@ class JSONDatabase:
         # Only for JSONDatabase
         entries = self._read_json(self.rate_limits_file)
         return [RateLimitEntry.from_dict(e) for e in entries]
+    
+    async def create_request_log(self, request_log: RequestLog) -> RequestLog:
+        """Create a new request log entry."""
+        request_logs = self._read_json(self.request_logs_file)
+        request_logs.append(request_log.to_dict())
+        self._write_json(self.request_logs_file, request_logs)
+        return request_log
+    
+    async def get_request_logs(self, limit: int = 100, offset: int = 0) -> List[RequestLog]:
+        """Get request logs with pagination."""
+        request_logs = self._read_json(self.request_logs_file)
+        # Sort by created_at descending (newest first)
+        request_logs.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+        # Apply pagination
+        paginated_logs = request_logs[offset:offset + limit]
+        return [RequestLog.from_dict(log_data) for log_data in paginated_logs]
 
 
 class MongoDatabase:
@@ -316,41 +339,338 @@ class MongoDatabase:
         async for entry_data in cursor:
             entries.append(RateLimitEntry.from_dict(entry_data))
         return entries
+    
+    async def create_request_log(self, request_log: RequestLog) -> RequestLog:
+        """Create a new request log entry."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        await self.db.request_logs.insert_one(request_log.to_dict())
+        return request_log
+    
+    async def get_request_logs(self, limit: int = 100, offset: int = 0) -> List[RequestLog]:
+        """Get request logs with pagination."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        cursor = self.db.request_logs.find({}).sort("created_at", -1).skip(offset).limit(limit)
+        logs = []
+        async for log_data in cursor:
+            logs.append(RequestLog.from_dict(log_data))
+        return logs
+
+
+class SupabaseDatabase:
+    """Supabase database implementation."""
+    
+    def __init__(self, supabase_url: str, supabase_key: str):
+        self.supabase_url = supabase_url
+        self.supabase_key = supabase_key
+        self.client: Optional[Client] = None
+        self._connected = False
+    
+    async def connect(self) -> bool:
+        """Connect to Supabase."""
+        if not HAS_SUPABASE:
+            logger.warning("supabase package not available, cannot connect to Supabase")
+            return False
+
+        try:
+            self.client = create_client(self.supabase_url, self.supabase_key)
+            # Test connection by trying to access a table
+            self.client.table('users').select('id').limit(1).execute()
+            self._connected = True
+            logger.info("Connected to Supabase successfully")
+            return True
+        except Exception as e:
+            logger.warning(f"Failed to connect to Supabase: {e}")
+            self._connected = False
+            return False
+    
+    async def create_user(self, user: User) -> User:
+        """Create a new user."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('users').insert(user.to_dict()).execute()
+            if result.data:
+                return user
+            else:
+                raise ValueError("Failed to create user")
+        except Exception as e:
+            if "duplicate key" in str(e).lower() or "already exists" in str(e).lower():
+                raise ValueError(f"User with username '{user.username}' already exists")
+            raise e
+    
+    async def get_user_by_id(self, user_id: str) -> Optional[User]:
+        """Get user by ID."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('users').select('*').eq('id', user_id).execute()
+            if result.data:
+                return User.from_dict(result.data[0])
+            return None
+        except Exception as e:
+            logger.error(f"Error getting user by ID: {e}")
+            return None
+    
+    async def get_user_by_username(self, username: str) -> Optional[User]:
+        """Get user by username."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+
+        try:
+            result = self.client.table('users').select('*').eq('username', username).execute()
+            if result.data:
+                return User.from_dict(result.data[0])
+            return None
+        except Exception as e:
+            logger.error(f"Error getting user by username: {e}")
+            return None
+
+    async def get_user_by_telegram_id(self, telegram_id: str) -> Optional[User]:
+        """Get user by Telegram ID."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+
+        try:
+            result = self.client.table('users').select('*').eq('telegram_id', int(telegram_id)).execute()
+            if result.data:
+                return User.from_dict(result.data[0])
+            return None
+        except Exception as e:
+            logger.error(f"Error getting user by telegram_id: {e}")
+            return None
+    
+    async def create_api_key(self, api_key: APIKey) -> APIKey:
+        """Create a new API key."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('api_keys').insert(api_key.to_dict()).execute()
+            if result.data:
+                return api_key
+            else:
+                raise ValueError("Failed to create API key")
+        except Exception as e:
+            if "duplicate key" in str(e).lower() or "already exists" in str(e).lower():
+                raise ValueError("API key already exists")
+            raise e
+    
+    async def get_api_key(self, key: str) -> Optional[APIKey]:
+        """Get API key by key value."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('api_keys').select('*').eq('key', key).execute()
+            if result.data:
+                return APIKey.from_dict(result.data[0])
+            return None
+        except Exception as e:
+            logger.error(f"Error getting API key: {e}")
+            return None
+    
+    async def update_api_key(self, api_key: APIKey) -> APIKey:
+        """Update an existing API key."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('api_keys').update(api_key.to_dict()).eq('id', api_key.id).execute()
+            if result.data:
+                return api_key
+            else:
+                raise ValueError(f"API key with ID '{api_key.id}' not found")
+        except Exception as e:
+            logger.error(f"Error updating API key: {e}")
+            raise e
+    
+    async def get_api_keys_by_user(self, user_id: str) -> List[APIKey]:
+        """Get all API keys for a user."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('api_keys').select('*').eq('user_id', user_id).execute()
+            return [APIKey.from_dict(key_data) for key_data in result.data]
+        except Exception as e:
+            logger.error(f"Error getting API keys by user: {e}")
+            return []
+    
+    async def get_rate_limit_entry(self, api_key_id: str) -> Optional[RateLimitEntry]:
+        """Get rate limit entry for API key."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('rate_limits').select('*').eq('api_key_id', api_key_id).execute()
+            if result.data:
+                return RateLimitEntry.from_dict(result.data[0])
+            return None
+        except Exception as e:
+            logger.error(f"Error getting rate limit entry: {e}")
+            return None
+    
+    async def update_rate_limit_entry(self, entry: RateLimitEntry) -> RateLimitEntry:
+        """Update rate limit entry."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            # Try to update first
+            result = self.client.table('rate_limits').update(entry.to_dict()).eq('api_key_id', entry.api_key_id).execute()
+            if not result.data:
+                # If no rows were updated, insert new entry
+                result = self.client.table('rate_limits').insert(entry.to_dict()).execute()
+            return entry
+        except Exception as e:
+            logger.error(f"Error updating rate limit entry: {e}")
+            raise e
+    
+    async def get_all_rate_limit_entries(self) -> list:
+        """Return all rate limit entries (for maintenance/cleanup) from Supabase."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('rate_limits').select('*').execute()
+            return [RateLimitEntry.from_dict(entry_data) for entry_data in result.data]
+        except Exception as e:
+            logger.error(f"Error getting all rate limit entries: {e}")
+            return []
+    
+    async def create_request_log(self, request_log: RequestLog) -> RequestLog:
+        """Create a new request log entry."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('request_logs').insert(request_log.to_dict()).execute()
+            if result.data:
+                return request_log
+            else:
+                raise ValueError("Failed to create request log")
+        except Exception as e:
+            logger.error(f"Error creating request log: {e}")
+            raise e
+    
+    async def get_request_logs(self, limit: int = 100, offset: int = 0) -> List[RequestLog]:
+        """Get request logs with pagination."""
+        if not self._connected:
+            raise RuntimeError("Database not connected")
+        
+        try:
+            result = self.client.table('request_logs').select('*').order('created_at', desc=True).range(offset, offset + limit - 1).execute()
+            return [RequestLog.from_dict(log_data) for log_data in result.data]
+        except Exception as e:
+            logger.error(f"Error getting request logs: {e}")
+            return []
 
 
 class DatabaseManager:
-    """Database manager that handles MongoDB with JSON fallback."""
+    """Database manager with flexible backend support."""
     
     def __init__(self, mongo_connection_string: Optional[str] = None, data_dir: str = "data"):
-        self.mongo_connection_string = mongo_connection_string or os.getenv("MONGODB_URL")
+        self.mongo_connection_string = mongo_connection_string
         self.data_dir = data_dir
+        self.supabase_url = self._get_supabase_url()
+        self.supabase_key = self._get_supabase_key()
         
+        # Database instances
+        self.supabase_db = None
         self.mongo_db = None
-        self.json_db = JSONDatabase(data_dir)
-        self.use_mongo = False
+        self.json_db = None
+        self.active_db = None
+        
+        logger.info("🔗 Database manager initialized with flexible backend support")
+    
+    def _get_supabase_url(self) -> Optional[str]:
+        """Get Supabase URL from environment variables or GitHub secrets."""
+        # Try environment variable first
+        url = os.getenv("SUPABASE_URL")
+        if url:
+            return url
         
-        logger.info(f"Database manager initialized with data_dir: {data_dir}")
+        # Try to get from GitHub secrets (if running in GitHub Actions)
+        github_url = os.getenv("GITHUB_SUPABASE_URL")  # GitHub Actions secret
+        if github_url:
+            return github_url
+        
+        # Don't log error during initialization - only when actually needed
+        return None
+    
+    def _get_supabase_key(self) -> Optional[str]:
+        """Get Supabase key from environment variables or GitHub secrets."""
+        # Try environment variable first
+        key = os.getenv("SUPABASE_ANON_KEY")
+        if key:
+            return key
+        
+        # Try to get from GitHub secrets (if running in GitHub Actions)
+        github_key = os.getenv("GITHUB_SUPABASE_ANON_KEY")  # GitHub Actions secret
+        if github_key:
+            return github_key
+        
+        # Don't log error during initialization - only when actually needed
+        return None
     
     async def initialize(self) -> None:
-        """Initialize database connection."""
+        """Initialize database connection with fallback support."""
+        # Try Supabase first if credentials are available
+        if self.supabase_url and self.supabase_key:
+            logger.info("🔗 Attempting to connect to Supabase...")
+            try:
+                self.supabase_db = SupabaseDatabase(self.supabase_url, self.supabase_key)
+                connected = await self.supabase_db.connect()
+                
+                if connected:
+                    self.active_db = self.supabase_db
+                    logger.info("✅ Successfully connected to Supabase database")
+                    return
+                else:
+                    logger.warning("⚠️ Failed to connect to Supabase, trying fallbacks...")
+            except Exception as e:
+                logger.warning(f"⚠️ Supabase connection error: {e}, trying fallbacks...")
+        else:
+            logger.info("ℹ️ Supabase credentials not found, using fallback databases")
+        
+        # Try MongoDB if connection string is provided
         if self.mongo_connection_string:
+            logger.info("🔗 Attempting to connect to MongoDB...")
             try:
                 self.mongo_db = MongoDatabase(self.mongo_connection_string)
-                self.use_mongo = await self.mongo_db.connect()
-                if self.use_mongo:
-                    logger.info("Using MongoDB as primary database")
+                connected = await self.mongo_db.connect()
+                
+                if connected:
+                    self.active_db = self.mongo_db
+                    logger.info("✅ Successfully connected to MongoDB database")
+                    return
                 else:
-                    logger.info("MongoDB connection failed, falling back to JSON database")
+                    logger.warning("⚠️ Failed to connect to MongoDB, falling back to JSON...")
             except Exception as e:
-                logger.warning(f"MongoDB initialization failed: {e}, using JSON database")
-                self.use_mongo = False
-        else:
-            logger.info("No MongoDB connection string provided, using JSON database")
+                logger.warning(f"⚠️ MongoDB connection error: {e}, falling back to JSON...")
+        
+        # Fall back to JSON database
+        logger.info("📁 Using JSON file database as fallback")
+        try:
+            self.json_db = JSONDatabase(self.data_dir)
+            self.active_db = self.json_db
+            logger.info("✅ Successfully initialized JSON database")
+        except Exception as e:
+            logger.error(f"❌ Failed to initialize JSON database: {e}")
+            raise RuntimeError(f"Failed to initialize any database backend: {e}")
     
     @property
-    def db(self) -> Union[MongoDatabase, JSONDatabase]:
+    def db(self):
         """Get the active database instance."""
-        return self.mongo_db if self.use_mongo else self.json_db
+        if not self.active_db:
+            raise RuntimeError("Database not initialized. Call initialize() first.")
+        return self.active_db
     
     async def create_user(self, user: User) -> User:
         """Create a new user."""
@@ -392,9 +712,44 @@ class DatabaseManager:
         """Update rate limit entry."""
         return await self.db.update_rate_limit_entry(entry)
     
+    async def create_request_log(self, request_log: RequestLog) -> RequestLog:
+        """Create a new request log entry."""
+        return await self.db.create_request_log(request_log)
+    
+    async def get_request_logs(self, limit: int = 100, offset: int = 0) -> List[RequestLog]:
+        """Get request logs with pagination."""
+        return await self.db.get_request_logs(limit, offset)
+    
     def get_status(self) -> Dict[str, str]:
         """Get database status."""
-        return {
-            "type": "MongoDB" if self.use_mongo else "JSON",
-            "status": "connected" if (self.use_mongo and self.mongo_db._connected) or (not self.use_mongo) else "disconnected"
-        }
+        if not self.active_db:
+            return {
+                "type": "None",
+                "status": "not_initialized",
+                "message": "Database not initialized"
+            }
+        
+        if isinstance(self.active_db, SupabaseDatabase):
+            return {
+                "type": "Supabase",
+                "status": "connected" if self.active_db._connected else "disconnected",
+                "message": "Using Supabase database"
+            }
+        elif isinstance(self.active_db, MongoDatabase):
+            return {
+                "type": "MongoDB",
+                "status": "connected" if self.active_db._connected else "disconnected",
+                "message": "Using MongoDB database"
+            }
+        elif isinstance(self.active_db, JSONDatabase):
+            return {
+                "type": "JSON",
+                "status": "connected",
+                "message": "Using JSON file database"
+            }
+        else:
+            return {
+                "type": "Unknown",
+                "status": "unknown",
+                "message": "Unknown database type"
+            }