fustor-source-fs 0.1.4__py3-none-any.whl

fustor_source_fs/__init__.py

@@ -0,0 +1,412 @@
+"""
+Fuagent source driver for the file system.
+
+This driver implements a 'Smart Dynamic Monitoring' strategy to efficiently
+monitor large directory structures without exhausting system resources.
+"""
+import os
+import queue
+import time
+import datetime
+import logging
+import uuid
+import getpass
+import fnmatch
+import threading
+from typing import Any, Dict, Iterator, List, Tuple
+from fustor_core.drivers import SourceDriver
+from fustor_core.models.config import SourceConfig
+from fustor_event_model.models import EventBase, UpdateEvent, DeleteEvent
+
+from .components import _WatchManager, safe_path_handling
+from .event_handler import OptimizedWatchEventHandler, get_file_metadata
+
+logger = logging.getLogger("fustor_agent.driver.fs")
+            
+import threading
+
+class FSDriver(SourceDriver):
+    _instances: Dict[str, 'FSDriver'] = {}
+    _lock = threading.Lock()
+    
+    @property
+    def is_transient(self) -> bool:
+        """
+        FS driver is transient - events will be lost if not processed immediately.
+        """
+        return True
+    
+    def __new__(cls, id: str, config: SourceConfig):
+        # Generate unique signature based on URI and credentials to ensure permission isolation
+        signature = f"{config.uri}#{hash(str(config.credential))}"
+        
+        with FSDriver._lock:
+            if signature not in FSDriver._instances:
+                # Create new instance
+                instance = super().__new__(cls)
+                FSDriver._instances[signature] = instance
+            return FSDriver._instances[signature]
+    
+    def __init__(self, id: str, config: SourceConfig):
+        # Prevent re-initialization of shared instances
+        if hasattr(self, '_initialized'):
+            return
+        
+        super().__init__(id, config)
+        self.uri = self.config.uri
+        self.event_queue: queue.Queue[EventBase] = queue.Queue()
+        self.clock_offset = 0.0  # Placeholder for potential future use
+        self._stop_driver_event = threading.Event() # NEW
+        min_monitoring_window_days = self.config.driver_params.get("min_monitoring_window_days", 30.0)
+        self.watch_manager = _WatchManager(self.uri, event_handler=None, min_monitoring_window_days=min_monitoring_window_days, stop_driver_event=self._stop_driver_event)
+        self.event_handler = OptimizedWatchEventHandler(self.event_queue, self.watch_manager)
+        self.watch_manager.event_handler = self.event_handler
+        self._pre_scan_completed = False
+        self._pre_scan_lock = threading.Lock()
+        self._stop_driver_event = threading.Event() # NEW
+        
+        self._initialized = True
+
+    def _perform_pre_scan_and_schedule(self):
+        """
+        Performs a one-time scan of the directory to populate the watch manager
+        with a capacity-aware, hierarchy-complete set of the most active directories.
+        It uses a delta to normalize server mtimes to the client's time domain.
+        """
+        with self._pre_scan_lock:
+            if self._pre_scan_completed:
+                return
+
+            logger.info(f"[fs] Performing initial directory scan to build hot-directory map for: {self.uri}")
+            
+            mtime_map: Dict[str, float] = {}
+            
+            # Track statistics
+            error_count = 0
+            total_entries = 0  # Total number of entries (directories and files) processed
+            
+            def handle_walk_error(e: OSError):
+                nonlocal error_count
+                error_count += 1
+                logger.debug(f"[fs] Error during pre-scan walk, skipping path: {e.filename} - {e.strerror}")
+
+            # Step 1: Walk the entire tree to build the mtime_map with server times
+            for root, dirs, files in os.walk(self.uri, topdown=False, onerror=handle_walk_error):
+                try:
+                    latest_mtime = os.path.getmtime(root)
+                except OSError:
+                    continue
+
+                for filename in files:
+                    file_path = os.path.join(root, filename)
+                    try:
+                        stat_info = os.stat(file_path)
+                        latest_mtime = max(latest_mtime, stat_info.st_mtime)
+                    except (FileNotFoundError, PermissionError, OSError) as e:
+                        error_count += 1
+                        logger.debug(f"[fs] Error during pre-scan walk, skipping path: {e.filename} - {e.strerror}")
+
+                    # Count each file as an entry
+                    total_entries += 1
+
+                for dirname in dirs:
+                    dirpath = os.path.join(root, dirname)
+                    latest_mtime = max(latest_mtime, mtime_map.get(dirpath, 0))
+                    # Count each dir as an entry
+                    total_entries += 1
+                
+                # Count the current directory
+                mtime_map[root] = latest_mtime
+                total_entries += 1  # Increment for each directory processed
+                
+                # Log statistics every 1000 entries (using a reasonable batch size)
+                if total_entries % 10000 == 0:
+                    # Find the newest and oldest directories so far
+                    if mtime_map:
+                        newest_dir = max(mtime_map.items(), key=lambda x: x[1])
+                        oldest_dir = min(mtime_map.items(), key=lambda x: x[1])
+                        newest_age = time.time() - newest_dir[1]  # Difference in seconds
+                        oldest_age = time.time() - oldest_dir[1]  # Difference in seconds
+                        logger.info(
+                            f"[fs] Pre-scan progress: processed {total_entries} entries, "
+                            f"errors: {error_count}, newest_dir: {newest_dir[0]} (age: {newest_age/86400:.2f} days), "
+                            f"oldest_dir: {oldest_dir[0]} (age: {oldest_age/86400:.2f} days)"
+                        )
+            
+            # Step 2: Calculate baseline delta using the true recursive mtime of the root.
+            try:
+                root_recursive_mtime = mtime_map.get(self.uri, os.path.getmtime(self.uri))
+                self.clock_offset = time.time() - root_recursive_mtime
+                logger.info(f"[fs] Calculated client-server time delta: {self.clock_offset:.2f} seconds.")
+            except OSError as e:
+                logger.warning(f"[fs] Could not stat root directory to calculate time delta: {e}. Proceeding without normalization.")
+
+            # Log final statistics before sorting
+            if mtime_map:
+                newest_dir = max(mtime_map.items(), key=lambda x: x[1])
+                oldest_dir = min(mtime_map.items(), key=lambda x: x[1])
+                newest_age = time.time() - newest_dir[1]  # Difference in seconds
+                oldest_age = time.time() - oldest_dir[1]  # Difference in seconds
+                logger.info(
+                    f"[fs] Pre-scan completed: processed {total_entries} entries, "
+                    f"errors: {error_count}, newest_dir: {safe_path_handling(newest_dir[0])} (age: {newest_age/86400:.2f} days), "
+                    f"oldest_dir: {safe_path_handling(oldest_dir[0])} (age: {oldest_age/86400:.2f} days)"
+                )
+
+            logger.info(f"[fs] Found {len(mtime_map)} total directories. Building capacity-aware, hierarchy-complete watch set...")
+            sorted_dirs = sorted(mtime_map.items(), key=lambda item: item[1], reverse=True)[:self.watch_manager.watch_limit]
+            old_limit = self.watch_manager.watch_limit
+            for path, _ in sorted_dirs:
+                server_mtime = mtime_map.get(path)
+                if server_mtime:
+                    # Normalize to client time domain while preserving relative differences
+                    lru_timestamp = server_mtime + self.clock_offset
+                else:
+                    # Fallback for parents that might not have been in mtime_map (though they should be)
+                    lru_timestamp = time.time()
+                self.watch_manager.schedule(path, lru_timestamp)
+                if self.watch_manager.watch_limit < old_limit:
+                    break  # Stop if we hit the limit during scheduling
+            logger.info(f"[fs] Final watch set constructed. Total paths to watch: {len(self.watch_manager.lru_cache)}.")
+            self._pre_scan_completed = True
+
+
+    def get_snapshot_iterator(self, **kwargs) -> Iterator[EventBase]:
+        stream_id = f"snapshot-fs-{uuid.uuid4().hex[:6]}"
+        logger.info(f"[{stream_id}] Starting Snapshot Scan Phase: for path: {self.uri}")
+
+        driver_params = self.config.driver_params
+        if driver_params.get("startup_mode") == "message-only":
+            logger.info(f"[{stream_id}] Skipping snapshot due to 'message-only' mode.")
+            return
+
+        file_pattern = driver_params.get("file_pattern", "*")
+        batch_size = kwargs.get("batch_size", 100)
+        
+        logger.info(f"[{stream_id}] Scan parameters: file_pattern='{file_pattern}'")
+
+        try:
+            batch: List[Dict[str, Any]] = []
+            files_processed_count = 0
+            error_count = 0
+            snapshot_time = int(time.time() * 1000)
+
+            def handle_walk_error(e: OSError):
+                nonlocal error_count
+                error_count += 1
+                logger.debug(f"[{stream_id}] Error during snapshot walk, skipping path: {safe_path_handling(e.filename)} - {e.strerror}")
+
+            temp_mtime_map: Dict[str, float] = {}
+
+            for root, dirs, files in os.walk(self.uri, topdown=False, onerror=handle_walk_error):
+                try:
+                    dir_stat_info = os.stat(root)
+                    latest_mtime_in_subtree = dir_stat_info.st_mtime
+                except OSError:
+                    dir_stat_info = None
+                    latest_mtime_in_subtree = 0.0
+
+                for filename in files:
+                    file_path = os.path.join(root, filename)
+                    try:
+                        stat_info = os.stat(file_path)
+                        latest_mtime_in_subtree = max(latest_mtime_in_subtree, stat_info.st_mtime)
+                        if fnmatch.fnmatch(filename, file_pattern):
+                            metadata = get_file_metadata(file_path, stat_info=stat_info)
+                            if metadata:
+                                batch.append(metadata)
+                                files_processed_count += 1
+                                if len(batch) >= batch_size:
+                                    # Extract fields from the first row if batch is not empty
+                                    fields = list(batch[0].keys()) if batch else []
+                                    yield UpdateEvent(event_schema=self.uri, table="files", rows=batch, index=snapshot_time, fields=fields)
+                                    batch = []
+                    except (FileNotFoundError, PermissionError, OSError) as e:
+                        error_count += 1
+                        logger.debug(f"[fs] Error processing file during snapshot: {safe_path_handling(file_path)} - {str(e)}")
+
+                for dirname in dirs:
+                    dirpath = os.path.join(root, dirname)
+                    latest_mtime_in_subtree = max(latest_mtime_in_subtree, temp_mtime_map.get(dirpath, 0.0))
+                
+                temp_mtime_map[root] = latest_mtime_in_subtree
+                aligned_lru_timestamp = latest_mtime_in_subtree + self.clock_offset
+                self.watch_manager.touch(root, aligned_lru_timestamp, is_recursive_upward=False)
+
+                if dir_stat_info:
+                    dir_metadata = get_file_metadata(root, stat_info=dir_stat_info)
+                    if dir_metadata:
+                        batch.append(dir_metadata)
+                        files_processed_count += 1
+                
+                if len(batch) >= batch_size:
+                    # Extract fields from the first row if batch is not empty
+                    fields = list(batch[0].keys()) if batch else []
+                    yield UpdateEvent(event_schema=self.uri, table="files", rows=batch, index=snapshot_time, fields=fields)
+                    batch = []
+            
+            if batch:
+                fields = list(batch[0].keys()) if batch else []
+                yield UpdateEvent(event_schema=self.uri, table="files", rows=batch, index=snapshot_time, fields=fields)
+
+            if error_count > 0:
+                logger.warning(f"[{stream_id}] Skipped {error_count} paths in total due to permission or other errors.")
+
+            logger.info(f"[{stream_id}] Full scan complete. Processed {files_processed_count} files and directories.")
+
+        except Exception as e:
+            logger.error(f"[{stream_id}] Snapshot phase for fs failed: {e}", exc_info=True)
+
+    def get_message_iterator(self, start_position: int=-1, **kwargs) -> Iterator[EventBase]:
+        
+        # Perform pre-scan to populate watches before starting the observer.
+        # This is essential for the message-first architecture and must block
+        # until completion to prevent race conditions downstream.
+        self._perform_pre_scan_and_schedule()
+
+        def _iterator_func() -> Iterator[EventBase]:
+            # After pre-scan is complete, any new events should be considered "starting from now"
+            # If start_position is provided, use it; otherwise, start from current time
+            
+            stream_id = f"message-fs-{uuid.uuid4().hex[:6]}"
+            
+            stop_event = kwargs.get("stop_event")
+            self.watch_manager.start()
+            logger.info(f"[{stream_id}] WatchManager started.")
+
+            try:
+                # Process events normally, but use the effective start position
+                while not (stop_event and stop_event.is_set()):
+                    try:
+                        max_sync_delay_seconds = self.config.driver_params.get("max_sync_delay_seconds", 1.0)
+                        event = self.event_queue.get(timeout=max_sync_delay_seconds)
+                        
+                        if start_position!=-1 and event.index < start_position:
+                            logger.debug(f"[{stream_id}] Skipping old event: {event.event_type} index={event.index} < start_position={start_position}")
+                            continue
+                        
+                        yield event
+
+                    except queue.Empty:
+                        continue
+            finally:
+                self.watch_manager.stop()
+                logger.info(f"[{stream_id}] Stopped real-time monitoring for: {self.uri}")
+
+        return _iterator_func()
+
+    @classmethod
+    async def get_available_fields(cls, **kwargs) -> Dict[str, Any]:
+        return {"properties": {
+            "file_path": {"type": "string", "description": "The full, absolute path to the file.", "column_index": 0},
+            "size": {"type": "integer", "description": "The size of the file in bytes.", "column_index": 1},
+            "modified_time": {"type": "number", "description": "The last modification time as a Unix timestamp (float).", "column_index": 2},
+            "created_time": {"type": "number", "description": "The creation time as a Unix timestamp (float).", "column_index": 3},
+        }}
+
+    @classmethod
+    async def test_connection(cls, **kwargs) -> Tuple[bool, str]:
+        path = kwargs.get("uri")
+        if not path or not isinstance(path, str):
+            return (False, "路径未提供或格式不正确。")
+        if not os.path.exists(path):
+            return (False, f"路径不存在: {path}")
+        if not os.path.isdir(path):
+            return (False, f"路径不是一个目录: {path}")
+        if not os.access(path, os.R_OK):
+            return (False, f"没有读取权限: {path}")
+        return (True, "连接成功，路径有效且可读。")
+
+    @classmethod
+    async def check_privileges(cls, **kwargs) -> Tuple[bool, str]:
+        path = kwargs.get("uri")
+        if not path:
+            return (False, "Path not provided in arguments.")
+
+        try:
+            user = getpass.getuser()
+        except Exception:
+            user = "unknown"
+
+        logger.info(f"[fs] Checking permissions for user '{user}' on path: {safe_path_handling(path)}")
+        
+        if not os.path.exists(path):
+            return (False, f"路径不存在: {path}")
+        if not os.path.isdir(path):
+            return (False, f"路径不是一个目录: {path}")
+
+        can_read = os.access(path, os.R_OK)
+        can_execute = os.access(path, os.X_OK)
+
+        if can_read and can_execute:
+            return (True, f"权限充足：当前用户 '{user}' 可以监控该目录。")
+        
+        missing_perms = []
+        if not can_read:
+            missing_perms.append("读取")
+        if not can_execute:
+            missing_perms.append("执行(进入)")
+        
+        return (False, f"权限不足：当前用户 '{user}' 缺少 {' 和 '.join(missing_perms)} 权限。")
+
+    async def close(self):
+        """
+        Close the file system watcher and stop monitoring.
+        """
+        logger.info(f"[fs] Closing file system watcher for {self.uri}")
+        
+        # Stop the watch manager if it's running
+        if hasattr(self, 'watch_manager') and self.watch_manager:
+            self.watch_manager.stop()
+        
+        # Set the stop event to ensure any active monitoring stops
+        if hasattr(self, '_stop_driver_event') and self._stop_driver_event:
+            self._stop_driver_event.set()
+        
+        logger.info(f"[fs] Closed file system watcher for {self.uri}")
+
+    @classmethod
+    async def get_wizard_steps(cls) -> Dict[str, Any]:
+        return {
+            "steps": [
+                {
+                    "step_id": "path_setup",
+                    "title": "目录与权限",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "uri": {
+                                "type": "string",
+                                "title": "监控目录路径",
+                                "description": "请输入要监控的文件夹的绝对路径。"
+                            },
+                            "driver_params": {
+                                "type": "object",
+                                "title": "驱动参数",
+                                "properties": {
+                                    "aged_interval": {
+                                        "type": "number",
+                                        "title": "被忽略监控的陈旧文件夹的年龄 (days)",
+                                        "default": 0.5
+                                    },
+                                    "max_sync_delay_seconds": {
+                                        "type": "number",
+                                        "title": "最大同步延迟 (秒)",
+                                        "description": "实时推送的最大延迟时间。如果超过此时间没有事件，将强制推送一次。",
+                                        "default": 1.0
+                                    },
+                                    "min_monitoring_window_days": {
+                                        "type": "number",
+                                        "title": "最小监控窗口 (天)",
+                                        "description": "当需要淘汰监控目录时，确保被淘汰的目录比整个监控范围内最新的文件至少旧N天。这可以防止淘汰近期仍在活跃范围内的目录。例如，设置为30，则表示只有比最新文件早30天以上的目录才允许被淘汰。",
+                                        "default": 30.0
+                                    }
+                                }
+                            }
+                        },
+                        "required": ["uri"],
+                    },
+                    "validations": ["test_connection", "check_privileges"]
+                }
+            ]
+        }

fustor_source_fs/components.py

@@ -0,0 +1,356 @@
+import collections
+import logging
+import os
+import threading
+import time
+import dataclasses
+import heapq
+from typing import Dict, List, Optional, Set, Tuple
+from fustor_core.exceptions import DriverError # NEW IMPORT
+
+# Use the low-level inotify wrapper and high-level event types
+from watchdog.observers.inotify_c import Inotify
+from watchdog.events import (
+    FileClosedEvent,
+    FileCreatedEvent,
+    FileDeletedEvent,
+    FileModifiedEvent,
+    FileMovedEvent,
+    DirCreatedEvent,
+    DirDeletedEvent,
+    DirModifiedEvent,
+    DirMovedEvent,
+)
+
+logger = logging.getLogger("fustor_agent.driver.fs")
+
+def contains_surrogate_characters(path: str) -> bool:
+    """Checks if a string contains surrogate characters."""
+    try:
+        path.encode('utf-8')
+        return False
+    except UnicodeEncodeError:
+        return True
+
+def safe_path_encode(path: str) -> bytes:
+    """Safely encodes a path to bytes, handling surrogate characters using filesystem encoding."""
+    try:
+        return os.fsencode(path)
+    except Exception:
+        # Fallback for extreme cases or non-string inputs, though fsencode is robust
+        return path.encode('utf-8', errors='replace')
+
+def safe_path_handling(path: str) -> str:
+    """Safely handles path strings, normalizing surrogate characters if present."""
+    if contains_surrogate_characters(path):
+        # Replace surrogate characters with underscores or question marks
+        # by encoding with replacement and decoding back
+        return path.encode('utf-8', errors='replace').decode('utf-8')
+    return path
+
+@dataclasses.dataclass(frozen=True)
+class WatchEntry:
+    """Simplified entry for the LRU cache, just holds the timestamp."""
+    timestamp: float
+
+class _LRUCache:
+    """A custom cache that evicts the item with the oldest timestamp (smallest value)."""
+    def __init__(self, capacity: int):
+        self.capacity = capacity
+        self.cache: Dict[str, WatchEntry] = {}  # path -> WatchEntry
+        self.min_heap: List[Tuple[float, str]] = []  # (timestamp, path)
+        self.removed_from_heap: Set[str] = set()
+
+    def _clean_heap(self):
+        """Removes stale entries from the top of the min_heap."""
+        while self.min_heap:
+            timestamp, path = self.min_heap[0]
+            if path in self.removed_from_heap:
+                heapq.heappop(self.min_heap)
+                self.removed_from_heap.remove(path)
+            elif path not in self.cache:
+                heapq.heappop(self.min_heap)
+            else:
+                break
+
+    def get(self, key: str) -> Optional[WatchEntry]:
+        return self.cache.get(key)
+
+    def put(self, key: str, value: WatchEntry):
+        if key in self.cache:
+            self.removed_from_heap.add(key)
+        self.cache[key] = value
+        heapq.heappush(self.min_heap, (value.timestamp, key))
+
+    def evict(self) -> Optional[Tuple[str, WatchEntry]]:
+        """Removes and returns the item with the oldest timestamp."""
+        if not self.cache:
+            return None
+        self._clean_heap()
+        if not self.min_heap:
+            return None
+        _timestamp, oldest_key = heapq.heappop(self.min_heap)
+        self.removed_from_heap.discard(oldest_key)
+        oldest_entry = self.cache.pop(oldest_key)
+        return oldest_key, oldest_entry
+
+    def get_oldest(self) -> Optional[Tuple[str, WatchEntry]]:
+        """Returns the item with the oldest timestamp without removing it."""
+        if not self.cache:
+            return None
+        self._clean_heap()
+        if not self.min_heap:
+            return None
+        _timestamp, oldest_key = self.min_heap[0]
+        return oldest_key, self.cache[oldest_key]
+
+    def remove(self, key: str):
+        if key in self.cache:
+            del self.cache[key]
+            self.removed_from_heap.add(key)
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.cache
+
+    def __len__(self) -> int:
+        return len(self.cache)
+
+
+class _WatchManager:
+    """
+    Manages a single inotify instance and its watches, including LRU pruning.
+    This is a more resource-efficient implementation.
+    """
+    def __init__(self, root_path: str, event_handler, min_monitoring_window_days: float = 30.0, stop_driver_event: threading.Event = None):
+        logger.info(f"Creating a new Inotify instance for root path {root_path}.")
+        self.watch_limit = 10000000  # This now only limits watches, not instances.
+        self.lru_cache = _LRUCache(self.watch_limit)
+        self.event_handler = event_handler
+        self.root_path = root_path
+        self._lock = threading.RLock()
+        self.min_monitoring_window_days = min_monitoring_window_days
+        self.stop_driver_event = stop_driver_event # NEW
+
+        # Directly use the low-level Inotify class
+        # We watch the root path non-recursively just to initialize the instance.
+        # All other watches are added dynamically.
+        # Use safe_path_encode to handle potential surrogate characters in root_path
+        self.inotify = Inotify(safe_path_encode(root_path), recursive=False)
+
+        self._stop_event = threading.Event()
+        self.inotify_thread = threading.Thread(target=self._event_processing_loop, daemon=True)
+
+    def _event_processing_loop(self):
+        """
+        The core event loop that reads from inotify and dispatches events.
+        """
+        while not self._stop_event.is_set():
+            try:
+                raw_events = self.inotify.read_events()
+
+                # Pre-process to identify paired moves and avoid duplicate events.
+                paired_move_from_paths = set()
+                for event in raw_events:
+                    if event.is_moved_to:
+                        src_path_from = self.inotify.source_for_move(event)
+                        if src_path_from:
+                            paired_move_from_paths.add(os.fsdecode(src_path_from))
+
+                for event in raw_events:
+                        # Use fsdecode to safely decode bytes to str, it handles surrogates correctly
+                        src_path_str = os.fsdecode(event.src_path)
+
+                        # Handle paired moves (MOVED_FROM + MOVED_TO)
+                        if event.is_moved_to:
+                            src_path_from = self.inotify.source_for_move(event)
+                            if src_path_from:
+                                src_path_from_str = os.fsdecode(src_path_from)
+                                if event.is_directory:
+                                    self.event_handler.on_moved(DirMovedEvent(src_path_from_str, src_path_str))
+                                else:
+                                    self.event_handler.on_moved(FileMovedEvent(src_path_from_str, src_path_str))
+                            else:
+                                # Unmatched MOVED_TO: treat as creation
+                                if event.is_directory:
+                                    self.event_handler.on_created(DirCreatedEvent(src_path_str))
+                                else:
+                                    self.event_handler.on_created(FileCreatedEvent(src_path_str))
+                        
+                        # Handle unmatched MOVED_FROM (treat as deletion)
+                        elif event.is_moved_from:
+                            if src_path_str in paired_move_from_paths:
+                                continue # Already processed as part of a move
+                            
+                            if event.is_directory:
+                                self.event_handler.on_deleted(DirDeletedEvent(src_path_str))
+                            else:
+                                self.event_handler.on_deleted(FileDeletedEvent(src_path_str))
+
+                        # Handle creation events
+                        elif event.is_create:
+                            if event.is_directory:
+                                self.event_handler.on_created(DirCreatedEvent(src_path_str))
+                            else:
+                                self.event_handler.on_created(FileCreatedEvent(src_path_str))
+
+                        # Handle deletion events
+                        elif event.is_delete:
+                            if event.is_directory:
+                                self.event_handler.on_deleted(DirDeletedEvent(src_path_str))
+                            else:
+                                self.event_handler.on_deleted(FileDeletedEvent(src_path_str))
+
+                        # Handle modification events (attrib or modify)
+                        elif event.is_attrib or event.is_modify:
+                            if event.is_directory:
+                                self.event_handler.on_modified(DirModifiedEvent(src_path_str))
+                            else:
+                                self.event_handler.on_modified(FileModifiedEvent(src_path_str))
+
+                        # Handle file closed after write (definitive modification)
+                        elif event.is_close_write:
+                            if event.is_directory:
+                                 self.event_handler.on_closed(DirModifiedEvent(src_path_str))
+                            else:
+                                 self.event_handler.on_closed(FileClosedEvent(src_path_str))
+
+                        # Handle ignored events (watch removed)
+                        elif event.is_ignored:
+                            with self._lock:
+                                if src_path_str in self.lru_cache:
+                                    self.lru_cache.remove(src_path_str)
+                                    logger.debug(f"Removed watch for '{safe_path_handling(src_path_str)}' from LRU cache due to IGNORED event.")
+
+            except KeyError as e:
+                logger.debug(f"Ignoring event for untracked watch descriptor: {str(e)}")
+
+    def schedule(self, path: str, timestamp: Optional[float] = None):
+        with self._lock:
+            timestamp_to_use = timestamp if timestamp is not None else time.time()
+            if path in self.lru_cache:
+                existing_entry = self.lru_cache.get(path)
+                if existing_entry and existing_entry.timestamp < timestamp_to_use:
+                    self.lru_cache.put(path, WatchEntry(timestamp_to_use))
+                return
+
+            is_eviction_needed = len(self.lru_cache) >= self.watch_limit
+            
+            oldest = self.lru_cache.get_oldest()
+            if oldest and oldest[1].timestamp >= timestamp_to_use and is_eviction_needed:
+                logger.debug(f"New watch for {safe_path_handling(path)} (ts {timestamp_to_use:.2f}) is older than oldest in cache (ts {oldest[1].timestamp:.2f}). Skipping.")
+                return
+
+            if is_eviction_needed:
+                evicted_item = self.lru_cache.evict()
+                if evicted_item:
+                    evicted_path, evicted_entry = evicted_item
+                    relative_age_days = (time.time() - evicted_entry.timestamp) / 86400
+                    
+                    if relative_age_days < self.min_monitoring_window_days:
+                        error_msg = (
+                            f"Watch limit reached and an active watch for {evicted_path} "
+                            f"(relative age: {relative_age_days:.2f} days) is about to be evicted. "
+                            f"This is below the configured min_monitoring_window_days ({self.min_monitoring_window_days} days). "
+                            f"Stopping driver to prevent data loss."
+                        )
+                        logger.error(error_msg)
+                        if self.stop_driver_event:
+                            self.stop_driver_event.set()
+                        raise DriverError(error_msg)
+
+                    logger.info(f"Watch limit reached. Evicting watch for {safe_path_handling(evicted_path)} (relative age: {relative_age_days:.2f} days).")
+                    try:
+                        self.inotify.remove_watch(safe_path_encode(evicted_path))
+                    except (KeyError, OSError) as e:
+                        logger.warning(f"Error removing evicted watch for {safe_path_handling(evicted_path)}: {e}")
+                    self.unschedule_recursive(evicted_path)
+                else:
+                    logger.warning(f"Watch limit of {self.watch_limit} reached, but LRU cache is empty. Cannot schedule new watch for {safe_path_handling(path)}.")
+                    return
+
+            try:
+                self.inotify.add_watch(safe_path_encode(path))
+                self.lru_cache.put(path, WatchEntry(timestamp_to_use))
+            except OSError as e:
+                # Catch ENOENT (2) - File not found, likely deleted before we could watch it
+                # Catch ENOTDIR (20) - Not a directory (can happen if a file replaced a dir)
+                # Catch EACCES (13) - Permission denied
+                # Catch EINVAL (22) - Invalid argument, which can happen with special filesystems or very long paths
+                if e.errno == 2: # ENOENT
+                    if os.path.exists(path):
+                        # Path exists, but inotify reports ENOENT. This is problematic for inotify.
+                        logger.warning(f"[fs] Could not schedule watch for {safe_path_handling(path)} (errno={e.errno}), path exists but inotify rejected it. (Consider renaming if possible).")
+                    else:
+                        # Path truly does not exist.
+                        logger.warning(f"[fs] Could not schedule watch for {safe_path_handling(path)} (errno={e.errno}), it may strictly no longer exist or be inaccessible.")
+                    return
+                if e.errno in (20, 13): # ENOTDIR, EACCES
+                     logger.warning(f"[fs] Could not schedule watch for {safe_path_handling(path)} (errno={e.errno}), it may strictly no longer exist or be inaccessible.")
+                     return
+                if e.errno == 22: # EINVAL - Invalid argument
+                    logger.warning(f"[fs] Could not schedule watch for {safe_path_handling(path)} (errno={e.errno}), invalid argument. This can happen with special filesystems, bind mounts, or unusual path characters.")
+                    return
+
+                if e.errno == 28:
+                    new_limit = len(self.lru_cache)
+                    relative_age_days = (time.time() - timestamp_to_use) / 86400
+                    if relative_age_days < self.min_monitoring_window_days:
+                        error_msg = (
+                            f"System inotify watch limit hit. The new watch for {path} "
+                            f"(relative age: {relative_age_days:.2f} days) is about to be skipped. "
+                            f"This is below the configured min_monitoring_window_days ({self.min_monitoring_window_days} days). "
+                            f"Stopping driver to prevent data loss."
+                        )
+                        logger.error(error_msg)
+                        if self.stop_driver_event:
+                            self.stop_driver_event.set()
+                        raise DriverError(error_msg)
+
+                    logger.warning(
+                        f"System inotify watch limit hit. Dynamically adjusting watch_limit from "
+                        f"{self.watch_limit} to {new_limit}. The new watch for {path} (relative age: {relative_age_days:.2f} days) will be skipped. "
+
+                        f"Consider increasing 'fs.inotify.max_user_watchs'."
+                    )
+                    self.watch_limit = new_limit
+                    return self.schedule(path, timestamp_to_use) # Retry the schedule call after adjusting the limit
+                else:
+                    logger.error(f"OS error scheduling watch for {safe_path_handling(path)}: {e}", exc_info=True)
+                    raise
+
+    def unschedule_recursive(self, path: str):
+        with self._lock:
+            paths_to_remove_from_lru = [p for p in list(self.lru_cache.cache.keys()) if p == path or p.startswith(path + os.sep)]
+            for p in paths_to_remove_from_lru:
+                try:
+                    self.inotify.remove_watch(safe_path_encode(p))
+                except (KeyError, OSError) as e:
+                    logger.warning(f"Error removing watch during recursive unschedule for {p}: {e}")
+                self.lru_cache.remove(p)
+
+    def touch(self, path: str, timestamp: Optional[float] = None, is_recursive_upward: bool = True):
+        with self._lock:
+            current_path = path
+            while True:
+                try:
+                    if os.path.isdir(current_path):
+                        self.schedule(current_path, timestamp)
+                except (OSError, PermissionError) as e:
+                    logger.warning(f"[fs] Error accessing path during touch: {safe_path_handling(current_path)} - {str(e)}")
+                
+                if not is_recursive_upward or len(current_path) <= len(self.root_path):
+                    break
+                
+                current_path = os.path.dirname(current_path)
+
+    def start(self):
+        logger.info("WatchManager: Starting inotify event thread.")
+        self.inotify_thread.start()
+
+    def stop(self):
+        logger.info("WatchManager: Stopping inotify event thread.")
+        self._stop_event.set()
+        self.inotify.close()  # This will interrupt the blocking read_events() call
+        if self.inotify_thread.is_alive():
+            self.inotify_thread.join(timeout=5.0)
+        logger.info("WatchManager: Inotify event thread stopped.")

fustor_source_fs/event_handler.py

@@ -0,0 +1,241 @@
+import os
+import queue
+import time
+import logging
+import stat
+from typing import Any, Dict, Optional
+
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+from fustor_event_model.models import UpdateEvent, DeleteEvent
+
+from .components import _WatchManager
+
+logger = logging.getLogger("fustor_agent.driver.fs")
+
+def get_file_metadata(path: str, stat_info: Optional[os.stat_result] = None) -> Optional[Dict[str, Any]]:
+    """Get file metadata, returning mtime and ctime as float timestamps."""
+    try:
+        if stat_info is None:
+            stat_info = os.stat(path)
+        
+        is_dir = stat.S_ISDIR(stat_info.st_mode)
+        
+        return {
+            "file_path": path,
+            "size": stat_info.st_size,
+            "modified_time": stat_info.st_mtime,
+            "created_time": stat_info.st_ctime,
+            "is_dir": is_dir
+        }
+    except FileNotFoundError:
+        logger.warning(f"[fs] Could not stat file, it may have been deleted before processing: {path}")
+        return None
+
+
+class OptimizedWatchEventHandler(FileSystemEventHandler):
+    """
+    Event handler that processes watchdog events immediately using dedicated
+    on_* methods, which is the idiomatic way to use watchdog.
+    """
+    def __init__(self, event_queue: queue.Queue, watch_manager: _WatchManager):
+        super().__init__()
+        self.event_queue = event_queue
+        self.watch_manager = watch_manager
+
+    def _touch_recursive_bottom_up(self, path: str):
+        """Recursively touches a directory and its contents from bottom-up."""
+        if not os.path.exists(path): return
+
+        # First, touch all files and subdirectories
+        for dirpath, dirnames, _ in os.walk(path, topdown=False):
+            for dirname in dirnames:
+                subdir_path = os.path.join(dirpath, dirname)
+                self.watch_manager.touch(subdir_path, is_recursive_upward=False)
+        
+        # Finally, touch the root of the path itself
+        self.watch_manager.touch(path, is_recursive_upward=False)
+
+    def _generate_move_events_recursive(self, from_path: str, to_path: str):
+        """Generates DeleteEvents for inferred old paths and UpdateEvents for new paths within a moved subtree."""
+        if not os.path.exists(to_path): return
+
+        for dirpath, dirnames, filenames in os.walk(to_path, topdown=False):
+            for filename in filenames:
+                add_path = os.path.join(dirpath, filename)
+                del_path = add_path.replace(to_path, from_path, 1)
+                
+                # Generate DeleteEvent for the old path
+                row = {"file_path": del_path}
+                delete_event = DeleteEvent(
+                    schema=self.watch_manager.root_path,
+                    event_schema=self.watch_manager.root_path,
+                    table="files",
+                    rows=[row],
+                    fields=list(row.keys()),
+                    index=int(time.time() * 1000)
+                )
+                self.event_queue.put(delete_event)
+                
+                # Generate UpdateEvent for the new path
+                metadata = get_file_metadata(add_path)
+                if metadata:
+                    update_event = UpdateEvent(
+                        schema=self.watch_manager.root_path,
+                        event_schema=self.watch_manager.root_path,
+                        table="files",
+                        rows=[metadata],
+                        fields=list(metadata.keys()),
+                        index=int(time.time() * 1000)
+                    )
+                    self.event_queue.put(update_event)
+            
+            for dirname in dirnames:
+                subdir_add_path = os.path.join(dirpath, dirname)
+                subdir_del_path = subdir_add_path.replace(to_path, from_path, 1)
+
+                # Generate DeleteEvent for the old directory path
+                row = {"file_path": subdir_del_path}
+                delete_event = DeleteEvent(
+                    schema=self.watch_manager.root_path,
+                    event_schema=self.watch_manager.root_path,
+                    table="files",
+                    rows=[row],
+                    fields=list(row.keys()),
+                    index=int(time.time() * 1000)
+                )
+                self.event_queue.put(delete_event)
+                # No UpdateEvent for directories, touch handles their LRU/watch status
+                # Generate UpdateEvent for the new path
+                metadata = get_file_metadata(subdir_add_path)
+                if metadata:
+                    update_event = UpdateEvent(
+                        schema=self.watch_manager.root_path,
+                        event_schema=self.watch_manager.root_path,
+                        table="files",
+                        rows=[metadata],
+                        fields=list(metadata.keys()),
+                        index=int(time.time() * 1000)
+                    )
+                    self.event_queue.put(update_event)
+
+    def on_created(self, event: FileSystemEvent):
+        """Called when a file or directory is created."""
+        try:
+            if event.is_directory:
+                metadata = get_file_metadata(event.src_path)
+                if metadata:
+                    update_event = UpdateEvent(
+                        schema=self.watch_manager.root_path,
+                        event_schema=self.watch_manager.root_path,
+                        table="files",
+                        rows=[metadata],
+                        fields=list(metadata.keys()),
+                        index=int(time.time() * 1000)
+                    )
+                    self.event_queue.put(update_event)
+                self.watch_manager.touch(event.src_path)
+        except Exception as e:
+            logger.warning(f"[fs] Error processing file creation event for {event.src_path}: {str(e)}")
+
+    def on_deleted(self, event: FileSystemEvent):
+        """Called when a file or directory is deleted."""
+        try:
+            # For a deleted path, we should not attempt to touch/schedule a watch.
+            # Instead, we unschedule and generate a delete event.
+
+            if event.is_directory:
+                self.watch_manager.unschedule_recursive(event.src_path)
+            row = {"file_path": event.src_path}
+            delete_event = DeleteEvent(
+                schema=self.watch_manager.root_path,
+                event_schema=self.watch_manager.root_path,
+                table="files",
+                rows=[row],
+                fields=list(row.keys()),
+                index=int(time.time() * 1000)
+            )
+            self.event_queue.put(delete_event)
+            
+            # A deletion is an activity, touch the parent path to update its timestamp.
+            # We assume the parent is always a directory.
+            self.watch_manager.touch(os.path.dirname(event.src_path))
+        except Exception as e:
+            logger.warning(f"[fs] Error processing file deletion event for {event.src_path}: {str(e)}")
+
+    def on_moved(self, event: FileSystemEvent):
+        """Called when a file or a directory is moved or renamed."""
+        try:
+            # Touch the parent of the source path to update its timestamp (something disappeared).
+            self.watch_manager.touch(os.path.dirname(event.src_path))
+            # Touch the parent of the destination path to update its timestamp (something appeared).
+            self.watch_manager.touch(os.path.dirname(event.dest_path))
+            
+            # Create and queue the delete event for the old location
+            delete_row = {"file_path": event.src_path}
+            delete_event = DeleteEvent(
+                schema=self.watch_manager.root_path,
+                event_schema=self.watch_manager.root_path,
+                table="files",
+                rows=[delete_row],
+                fields=list(delete_row.keys()),
+                index=int(time.time() * 1000)
+            )
+            self.event_queue.put(delete_event)
+            
+            # Handle the creation/update event for the new location
+            if event.is_directory:
+                # For directories, process recursively
+                self._generate_move_events_recursive(event.src_path, event.dest_path)
+                # Recursively touch all contents at the new destination to ensure watches are updated/scheduled.
+                self._touch_recursive_bottom_up(event.dest_path)
+                # Unschedule the old path recursively
+                self.watch_manager.unschedule_recursive(event.src_path)
+            else:
+                # For files, create update event for new location
+                metadata = get_file_metadata(event.dest_path)
+                if metadata:
+                    update_event = UpdateEvent(
+                        schema=self.watch_manager.root_path,
+                        event_schema=self.watch_manager.root_path,
+                        table="files",
+                        rows=[metadata],
+                        fields=list(metadata.keys()),
+                        index=int(time.time() * 1000)
+                    )
+                    self.event_queue.put(update_event)
+                # Touch the file itself at its new destination
+                self.watch_manager.touch(event.dest_path)
+        except Exception as e:
+            logger.warning(f"[fs] Error processing file move event for {event.src_path} -> {event.dest_path}: {str(e)}")
+            # Note: If we get here, the delete_event may already be in the queue
+            # This is an inherent issue with partial failure in distributed systems,
+            # but we prevent the entire system from crashing
+
+    def on_modified(self, event: FileSystemEvent):
+        """
+        Called when a file or directory is modified.
+        This is intentionally ignored to wait for the 'closed' event,
+        ensuring the file is fully written.
+        """
+        pass
+
+    def on_closed(self, event: FileSystemEvent):
+        """
+        Called when a file opened for writing is closed.
+        """
+        try:
+            self.watch_manager.touch(event.src_path)
+            if not event.is_directory:
+                metadata = get_file_metadata(event.src_path)
+                if metadata:
+                    update_event = UpdateEvent(
+                        schema=self.watch_manager.root_path,
+                        event_schema=self.watch_manager.root_path,
+                        table="files",
+                        rows=[metadata],
+                        fields=list(metadata.keys()),
+                        index=int(time.time() * 1000)
+                    )
+                    self.event_queue.put(update_event)
+        except Exception as e:
+            logger.warning(f"[fs] Error processing file closed event for {event.src_path}: {str(e)}")

fustor_source_fs-0.1.4.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: fustor-source-fs
+Version: 0.1.4
+Summary: Fustor Agent FS Source Driver
+License-Expression: MIT
+Requires-Dist: fustor-core
+Requires-Dist: fustor-event-model
+Requires-Dist: watchdog

fustor_source_fs-0.1.4.dist-info/RECORD

@@ -0,0 +1,9 @@
+fustor_source_fs/__init__.py,sha256=nl3IL57-yK_j0wHvR-tXL3VqUJETgtbM5cxIXMWHD6M,20143
+fustor_source_fs/components.py,sha256=2fK7p7UWDaPh7wwKbM8YOKNrcn9Jw9PN3bMAPHgCBiI,17589
+fustor_source_fs/event_handler.py,sha256=yxLqlppaeVP6ZnJ8SeMck1LO7Ps_6flRqeFlcOfEwvI,10862
+fustor_source_fs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fustor_source_fs-0.1.4.dist-info/METADATA,sha256=KE9hfxMojQ2WJhmn2uw2WDgLQA1oYHqG3lqgkDwuai0,208
+fustor_source_fs-0.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fustor_source_fs-0.1.4.dist-info/entry_points.txt,sha256=ruXHaAbRgq5l2QL5tf2QojUQzAxmE4m5hjYvLID5SPI,62
+fustor_source_fs-0.1.4.dist-info/top_level.txt,sha256=SfFvq54lUQVNr5GjMXj1qtUegMNYULlHrNt5KbNfHdI,17
+fustor_source_fs-0.1.4.dist-info/RECORD,,

fustor_source_fs-0.1.4.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fustor_source_fs-0.1.4.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[fustor_agent.drivers.sources]
+fs = fustor_source_fs:FSDriver

fustor_source_fs-0.1.4.dist-info/top_level.txt

@@ -0,0 +1 @@
+fustor_source_fs