lsoph 0.0.1__py3-none-any.whl

lsoph/backend/lsof.py

@@ -0,0 +1,329 @@
+import logging
+import os
+import re
+import subprocess
+import time
+from typing import Dict, Iterator, List, Optional, Set, Tuple
+
+from lsoph.monitor import Monitor
+from lsoph.util.pid import get_descendants
+
+# Setup logging - using the same pattern as other modules
+logging.basicConfig(
+    level=os.environ.get("LOGLEVEL", "WARNING").upper(),
+    format="%(levelname)s:%(name)s:%(message)s",
+)
+log = logging.getLogger("lsoph.backend.lsof")  # Use package-aware logger name
+
+# Regular expressions for parsing lsof output
+# Format: COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
+LSOF_LINE_RE = re.compile(
+    r"^(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$"
+)
+FD_TYPE_RE = re.compile(r"(\d+)([rwu])?")
+
+
+def _parse_fd(fd_str: str) -> Tuple[Optional[int], str]:
+    """Parse the FD column from lsof output."""
+    if fd_str in ("cwd", "rtd", "txt", "mem"):
+        log.debug(f"Special FD type: {fd_str}")
+        return None, fd_str
+
+    match = FD_TYPE_RE.match(fd_str)
+    if match:
+        fd = int(match.group(1))
+        mode = match.group(2) or ""
+        log.debug(f"Parsed FD {fd} with mode '{mode}'")
+        return fd, mode
+
+    log.debug(f"Unparsable FD string: {fd_str}")
+    return None, fd_str
+
+
+def _run_lsof(pids: Optional[List[int]] = None) -> Iterator[Dict]:
+    """Run lsof and yield parsed output lines."""
+    cmd = ["lsof", "-n", "-F", "pcftn"]  # p=pid, c=command, f=fd, t=type, n=name
+
+    if pids:
+        cmd.extend(["-p", ",".join(map(str, pids))])
+
+    log.info(f"Running lsof command: {' '.join(cmd)}")
+
+    try:
+        # Redirect stderr to /dev/null to suppress tracefs warnings
+        with open(os.devnull, "w") as devnull:
+            proc = subprocess.Popen(
+                cmd, stdout=subprocess.PIPE, stderr=devnull, text=True
+            )
+
+        current_record = {}
+        for line in proc.stdout:
+            line = line.strip()
+            if not line:
+                continue
+
+            field_type = line[0]
+            value = line[1:]
+
+            if field_type == "p" and current_record:
+                # New process, yield previous record
+                yield current_record
+                current_record = {"pid": int(value)}
+            elif field_type == "p":
+                current_record = {"pid": int(value)}
+            elif field_type == "c":
+                current_record["command"] = value
+            elif field_type == "f":
+                current_record["fd_str"] = value
+                fd, mode = _parse_fd(value)
+                current_record["fd"] = fd
+                current_record["mode"] = mode
+            elif field_type == "t":
+                current_record["type"] = value
+            elif field_type == "n":
+                current_record["path"] = value
+                log.debug(
+                    f"Complete record: PID {current_record.get('pid')}, FD {current_record.get('fd')}, path {value}"
+                )
+                yield current_record
+                current_record = {
+                    "pid": current_record["pid"],
+                    "command": current_record["command"],
+                }
+
+        # Yield final record if any
+        if current_record and "path" in current_record:
+            log.debug(f"Final record: {current_record}")
+            yield current_record
+
+    except subprocess.SubprocessError as e:
+        log.error(f"Error running lsof: {e}")
+    except Exception as e:
+        log.exception(f"Unexpected error in _run_lsof: {e}")
+    finally:
+        if proc.stdout:
+            proc.stdout.close()
+        exit_code = proc.wait()
+        log.debug(f"lsof process exited with code: {exit_code}")
+
+
+def _process_lsof_data(record: Dict, monitor: Monitor, timestamp: float) -> None:
+    """Process a parsed lsof record and update the monitor."""
+    pid = record.get("pid")
+    path = record.get("path")
+    fd = record.get("fd")
+
+    if not (pid and path):
+        log.debug(f"Incomplete record missing pid or path: {record}")
+        return
+
+    # If fd is None, this is a special file like cwd, not an open fd
+    if fd is None:
+        fd_str = record.get("fd_str", "unknown")
+        log.debug(f"Special file {fd_str} for PID {pid}: {path}")
+        monitor.stat(pid, path, True, timestamp)
+        return
+
+    # For regular files, add as open
+    log.debug(f"Processing open file: PID {pid}, FD {fd}, path {path}")
+    monitor.open(pid, path, fd, True, timestamp)
+
+    # Check mode to determine if it's being read/written
+    mode = record.get("mode", "")
+    if "r" in mode:
+        log.debug(f"File has read mode: PID {pid}, FD {fd}")
+        monitor.read(pid, fd, path, True, timestamp)
+    if "w" in mode:
+        log.debug(f"File has write mode: PID {pid}, FD {fd}")
+        monitor.write(pid, fd, path, True, timestamp)
+
+
+def _attach_with_descendants(pids: List[int], monitor: Monitor) -> None:
+    """
+    Attach to processes, their descendants, and monitor their open files using lsof.
+    This version tracks child processes by periodically checking for descendants.
+    """
+    log.info(f"Attaching to initial PIDs: {pids}")
+
+    # Keep track of all PIDs we're monitoring (including descendants)
+    all_pids = set(pids)
+
+    # Keep track of file descriptors we've seen for each PID
+    # to detect closures on subsequent runs
+    seen_fds: Dict[int, Set[Tuple[int, str]]] = {}
+
+    try:
+        poll_interval = 1.0  # seconds
+        child_check_interval = 5.0  # check for new children every 5 polls
+        poll_count = 0
+
+        log.info(f"Starting lsof polling with interval: {poll_interval} seconds")
+
+        while True:
+            timestamp = time.time()
+            poll_count += 1
+
+            # Check for descendants periodically
+            if poll_count % child_check_interval < 1:
+                log.debug(f"Checking for new child processes of {pids}")
+                new_pids = set()
+                for parent_pid in pids:
+                    try:
+                        descendants = get_descendants(parent_pid)
+                        for child_pid in descendants:
+                            if child_pid not in all_pids:
+                                log.info(f"Found new child process: {child_pid}")
+                                new_pids.add(child_pid)
+                    except Exception as e:
+                        log.debug(
+                            f"Error checking descendants for PID {parent_pid}: {e}"
+                        )
+
+                # Add any new PIDs to our tracking set
+                if new_pids:
+                    log.info(
+                        f"Adding {len(new_pids)} new child processes to monitoring"
+                    )
+                    all_pids.update(new_pids)
+
+            # Convert set back to list for lsof
+            current_pids = list(all_pids)
+            current_fds: Dict[int, Set[Tuple[int, str]]] = {}
+
+            record_count = 0
+            for record in _run_lsof(current_pids):
+                record_count += 1
+                pid = record.get("pid")
+                fd = record.get("fd")
+                path = record.get("path")
+
+                if pid and fd is not None and path:
+                    if pid not in current_fds:
+                        current_fds[pid] = set()
+                    current_fds[pid].add((fd, path))
+
+                _process_lsof_data(record, monitor, timestamp)
+
+            log.debug(f"Processed {record_count} lsof records")
+
+            # Detect closed files (present in seen_fds but not in current_fds)
+            close_count = 0
+            for pid, fd_paths in seen_fds.items():
+                current_pid_fds = current_fds.get(pid, set())
+                for fd, path in fd_paths:
+                    if (fd, path) not in current_pid_fds:
+                        log.debug(
+                            f"Detected closed file: PID {pid}, FD {fd}, path {path}"
+                        )
+                        monitor.close(pid, fd, True, timestamp)
+                        close_count += 1
+
+            if close_count > 0:
+                log.debug(f"Detected {close_count} closed files")
+
+            seen_fds = current_fds
+            log.debug(f"Sleeping for {poll_interval} seconds")
+            time.sleep(poll_interval)
+
+    except KeyboardInterrupt:
+        log.info("lsof monitoring interrupted by user")
+    except Exception as e:
+        log.exception(f"Unexpected error in lsof attach: {e}")
+
+
+def attach(pids: List[int], monitor: Monitor) -> None:
+    """
+    Standard attach function which monitors the specified PIDs without tracking descendants.
+    For command execution with descendants tracking, use _attach_with_descendants.
+    """
+    log.info(f"Attaching to PIDs: {pids}")
+
+    # Keep track of file descriptors we've seen for each PID
+    # to detect closures on subsequent runs
+    seen_fds: Dict[int, Set[Tuple[int, str]]] = {}
+
+    try:
+        poll_interval = 1.0  # seconds
+        log.info(f"Starting lsof polling with interval: {poll_interval} seconds")
+
+        while True:
+            timestamp = time.time()
+            current_fds: Dict[int, Set[Tuple[int, str]]] = {}
+
+            record_count = 0
+            for record in _run_lsof(pids):
+                record_count += 1
+                pid = record.get("pid")
+                fd = record.get("fd")
+                path = record.get("path")
+
+                if pid and fd is not None and path:
+                    if pid not in current_fds:
+                        current_fds[pid] = set()
+                    current_fds[pid].add((fd, path))
+
+                _process_lsof_data(record, monitor, timestamp)
+
+            log.debug(f"Processed {record_count} lsof records")
+
+            # Detect closed files (present in seen_fds but not in current_fds)
+            close_count = 0
+            for pid, fd_paths in seen_fds.items():
+                current_pid_fds = current_fds.get(pid, set())
+                for fd, path in fd_paths:
+                    if (fd, path) not in current_pid_fds:
+                        log.debug(
+                            f"Detected closed file: PID {pid}, FD {fd}, path {path}"
+                        )
+                        monitor.close(pid, fd, True, timestamp)
+                        close_count += 1
+
+            if close_count > 0:
+                log.debug(f"Detected {close_count} closed files")
+
+            seen_fds = current_fds
+            log.debug(f"Sleeping for {poll_interval} seconds")
+            time.sleep(poll_interval)
+
+    except KeyboardInterrupt:
+        log.info("lsof monitoring interrupted by user")
+    except Exception as e:
+        log.exception(f"Unexpected error in lsof attach: {e}")
+
+
+def run(command: List[str], monitor: Monitor) -> None:
+    """Run a command and monitor its open files using lsof."""
+    log.info(f"Running command: {' '.join(command)}")
+
+    proc = None
+    try:
+        log.debug("Launching subprocess")
+        proc = subprocess.Popen(command)
+        pid = proc.pid
+        log.info(f"Command started with PID: {pid}")
+
+        # Track the parent process and all its descendants
+        pids = [pid]
+        # Use a custom attach function to track descendants
+        _attach_with_descendants(pids, monitor)
+
+    except subprocess.SubprocessError as e:
+        log.error(f"Error launching command: {e}")
+    except Exception as e:
+        log.exception(f"Unexpected error in lsof run: {e}")
+    finally:
+        if proc and proc.poll() is None:
+            log.info(f"Terminating command process (PID: {proc.pid})")
+            proc.terminate()
+            try:
+                proc.wait(timeout=1)
+                log.debug(
+                    f"Process terminated gracefully with exit code: {proc.returncode}"
+                )
+            except subprocess.TimeoutExpired:
+                log.debug(
+                    f"Process did not terminate gracefully, killing PID {proc.pid}"
+                )
+                proc.kill()
+                proc.wait()
+                log.debug(f"Process killed with exit code: {proc.returncode}")

lsoph/backend/psutil.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+import logging
+import os
+import threading
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set, Tuple
+
+import psutil
+
+from lsoph.monitor import Monitor
+
+# Setup logging
+log = logging.getLogger("lsoph.backend.psutil")
+
+
+class PsutilBackend:
+    """Backend implementation that uses psutil to monitor file activities."""
+
+    def __init__(self, monitor: Monitor):
+        self.monitor = monitor
+        self.cwd_map: Dict[int, str] = {}
+        self.pid_map: Dict[int, bool] = {}  # Tracks if we've seen this PID before
+        self.fd_map: Dict[int, Dict[int, Tuple[str, bool, bool]]] = (
+            {}
+        )  # pid -> {fd -> (path, read, write)}
+        self.poll_interval = 0.5  # seconds
+        self.running = False
+        self.poll_thread = None
+        self.lock = threading.RLock()
+
+    def _get_process_cwd(self, pid: int) -> Optional[str]:
+        """Get current working directory for a process."""
+        try:
+            proc = psutil.Process(pid)
+            return proc.cwd()
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return None
+        except Exception as e:
+            log.debug(f"Error getting CWD for PID {pid}: {e}")
+            return None
+
+    def _resolve_path(self, pid: int, path: str) -> str:
+        """Resolve a potentially relative path to absolute using process CWD."""
+        if path.startswith("/") or (len(path) > 1 and path[1] == ":"):  # Absolute path
+            return path
+
+        # Try to resolve using process CWD
+        if pid in self.cwd_map:
+            return os.path.normpath(os.path.join(self.cwd_map[pid], path))
+
+        # Try to get CWD if not cached
+        cwd = self._get_process_cwd(pid)
+        if cwd:
+            self.cwd_map[pid] = cwd
+            return os.path.normpath(os.path.join(cwd, path))
+
+        # Fallback to current directory
+        return os.path.abspath(path)
+
+    def _get_open_files(self, pid: int) -> List[Dict[str, Any]]:
+        """Get open files for a process with error handling."""
+        try:
+            proc = psutil.Process(pid)
+            # Cache the CWD
+            if pid not in self.cwd_map:
+                try:
+                    self.cwd_map[pid] = proc.cwd()
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    pass
+
+            open_files = []
+            # Get open files directly from psutil
+            try:
+                for f in proc.open_files():
+                    open_files.append(
+                        {
+                            "path": f.path,
+                            "fd": f.fd,
+                            "mode": getattr(f, "mode", ""),
+                        }
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            # Add connections as files too
+            try:
+                for conn in proc.connections(kind="all"):
+                    if conn.status == "ESTABLISHED":
+                        path = f"<SOCKET:{conn.laddr.ip}:{conn.laddr.port}->{conn.raddr.ip}:{conn.raddr.port}>"
+                    else:
+                        path = f"<SOCKET:{conn.laddr.ip}:{conn.laddr.port}>"
+
+                    open_files.append(
+                        {
+                            "path": path,
+                            "fd": conn.fd if hasattr(conn, "fd") else -1,
+                            "mode": "rw",  # Assuming socket connections are read-write
+                        }
+                    )
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+
+            return open_files
+        except psutil.NoSuchProcess:
+            log.debug(f"Process {pid} no longer exists.")
+            return []
+        except psutil.AccessDenied:
+            log.debug(f"Access denied for process {pid}.")
+            return []
+        except Exception as e:
+            log.debug(f"Error getting open files for PID {pid}: {e}")
+            return []
+
+    def _get_descendants(self, pid: int) -> List[int]:
+        """Get all descendant PIDs recursively."""
+        try:
+            proc = psutil.Process(pid)
+            return [p.pid for p in proc.children(recursive=True)]
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return []
+        except Exception as e:
+            log.debug(f"Error getting descendants for PID {pid}: {e}")
+            return []
+
+    def _poll_processes(self, pids: List[int]) -> None:
+        """Poll processes for open files and update monitor."""
+        with self.lock:
+            # Check for new processes and store their CWD
+            all_pids = set(pids)
+            for pid in pids:
+                descendants = self._get_descendants(pid)
+                for child_pid in descendants:
+                    if child_pid not in self.pid_map:
+                        # New process - inherit parent's CWD if available
+                        if pid in self.cwd_map and child_pid not in self.cwd_map:
+                            self.cwd_map[child_pid] = self.cwd_map[pid]
+                        self.pid_map[child_pid] = True
+                        log.debug(f"Found new child process: {child_pid}")
+                    all_pids.add(child_pid)
+
+            # Keep track of current FDs to detect closures
+            current_fds: Dict[int, Set[int]] = {}
+            timestamp = time.time()
+
+            # Process each PID
+            for pid in all_pids:
+                # Skip processes we already know don't exist
+                if pid in self.pid_map and self.pid_map[pid] is False:
+                    continue
+
+                # Get open files
+                open_files = self._get_open_files(pid)
+
+                # If process doesn't exist, mark it
+                if not open_files and not psutil.pid_exists(pid):
+                    self.pid_map[pid] = False
+                    if pid in self.fd_map:
+                        # Process exit - close all its files
+                        for fd in list(self.fd_map[pid].keys()):
+                            path, _, _ = self.fd_map[pid][fd]
+                            self.monitor.close(pid, fd, True, timestamp)
+                        del self.fd_map[pid]
+                    continue
+
+                current_fds[pid] = set()
+
+                # Process open files
+                for file_info in open_files:
+                    path = file_info["path"]
+                    fd = file_info["fd"]
+                    mode = file_info.get("mode", "")
+
+                    # Skip if invalid fd
+                    if fd < 0:
+                        continue
+
+                    current_fds[pid].add(fd)
+
+                    # Check if this is a new file or changed mode
+                    is_new = True
+                    has_changed = False
+                    if pid in self.fd_map and fd in self.fd_map[pid]:
+                        is_new = False
+                        old_path, old_read, old_write = self.fd_map[pid][fd]
+                        has_changed = old_path != path
+
+                    # Handle new file or changed file
+                    if is_new or has_changed:
+                        if not is_new and has_changed:
+                            # File path changed (might be a rename)
+                            old_path, _, _ = self.fd_map[pid][fd]
+                            # First close the old file
+                            self.monitor.close(pid, fd, True, timestamp)
+                            # Then open the new one
+                            log.debug(
+                                f"File path changed for PID {pid}, FD {fd}: {old_path} -> {path}"
+                            )
+
+                        # Register file open
+                        can_read = "r" in mode
+                        can_write = "w" in mode or "a" in mode or "+" in mode
+
+                        self.monitor.open(pid, path, fd, True, timestamp)
+                        self.fd_map.setdefault(pid, {})[fd] = (
+                            path,
+                            can_read,
+                            can_write,
+                        )
+
+                        # Register read/write access based on mode
+                        if can_read:
+                            self.monitor.read(pid, fd, path, True, timestamp, bytes=0)
+                        if can_write:
+                            self.monitor.write(pid, fd, path, True, timestamp, bytes=0)
+
+            # Detect closed files
+            for pid in list(self.fd_map.keys()):
+                if pid not in current_fds:
+                    if psutil.pid_exists(pid):
+                        # Process exists but has no open files
+                        continue
+
+                    # Process has terminated - close all its files
+                    for fd, (path, _, _) in self.fd_map[pid].items():
+                        self.monitor.close(pid, fd, True, timestamp)
+                    del self.fd_map[pid]
+                else:
+                    # Check for closed files in existing processes
+                    for fd in list(self.fd_map[pid].keys()):
+                        if fd not in current_fds[pid]:
+                            path, _, _ = self.fd_map[pid][fd]
+                            self.monitor.close(pid, fd, True, timestamp)
+                            del self.fd_map[pid][fd]
+
+    def _polling_thread(self, pids: List[int]) -> None:
+        """Background thread that periodically polls processes."""
+        try:
+            log.info(f"Starting polling thread for PIDs: {pids}")
+
+            while self.running:
+                try:
+                    self._poll_processes(pids)
+                    time.sleep(self.poll_interval)
+                except Exception as e:
+                    log.error(f"Error in polling thread: {e}")
+                    time.sleep(self.poll_interval)
+
+        except Exception as e:
+            log.exception(f"Polling thread failed: {e}")
+        finally:
+            log.info("Polling thread stopped.")
+
+    def attach(self, pids: List[int], monitor: Monitor) -> None:
+        """Attach to existing processes."""
+        self.monitor = monitor
+        self.running = True
+
+        # Initialize pid_map with starting PIDs
+        for pid in pids:
+            self.pid_map[pid] = True
+
+            # Try to get CWD for each process
+            cwd = self._get_process_cwd(pid)
+            if cwd:
+                self.cwd_map[pid] = cwd
+
+        # Start polling thread
+        self.poll_thread = threading.Thread(
+            target=self._polling_thread, args=(pids,), daemon=True
+        )
+        self.poll_thread.start()
+
+        try:
+            while self.running:
+                time.sleep(0.1)  # Keep main thread alive
+        except KeyboardInterrupt:
+            log.info("Attach interrupted by user.")
+            self.running = False
+        except Exception as e:
+            log.exception(f"Error in attach: {e}")
+            self.running = False
+
+    def run(self, command: List[str], monitor: Monitor) -> None:
+        """Run a command and monitor its file activity."""
+        import subprocess
+
+        self.monitor = monitor
+        self.running = True
+
+        proc = None
+        try:
+            # Start the process
+            log.info(f"Running command: {' '.join(command)}")
+            proc = subprocess.Popen(command)
+            pid = proc.pid
+            log.info(f"Command started with PID: {pid}")
+
+            # Get initial CWD
+            cwd = self._get_process_cwd(pid)
+            if cwd:
+                self.cwd_map[pid] = cwd
+
+            # Mark this process as active
+            self.pid_map[pid] = True
+
+            # Start polling thread for this process and its descendants
+            self.poll_thread = threading.Thread(
+                target=self._polling_thread, args=([pid],), daemon=True
+            )
+            self.poll_thread.start()
+
+            # Wait for process to complete or user interrupt
+            while self.running and proc.poll() is None:
+                time.sleep(0.1)
+
+            if proc.poll() is not None:
+                log.info(f"Command exited with code: {proc.returncode}")
+
+        except KeyboardInterrupt:
+            log.info("Run interrupted by user.")
+            self.running = False
+        except Exception as e:
+            log.exception(f"Error in run: {e}")
+            self.running = False
+        finally:
+            # Cleanup
+            if proc and proc.poll() is None:
+                log.info("Terminating command process.")
+                proc.terminate()
+                try:
+                    proc.wait(timeout=1)
+                except subprocess.TimeoutExpired:
+                    log.info("Forcibly killing command process.")
+                    proc.kill()
+                    proc.wait()
+
+            self.running = False
+
+            # Wait for polling thread to stop
+            if self.poll_thread and self.poll_thread.is_alive():
+                self.poll_thread.join(timeout=1)
+
+
+# Functions exposed by the backend
+
+
+def attach(pids: List[int], monitor: Monitor) -> None:
+    """Attach to existing processes."""
+    backend = PsutilBackend(monitor)
+    backend.attach(pids, monitor)
+
+
+def run(command: List[str], monitor: Monitor) -> None:
+    """Run a command and monitor its file activity."""
+    backend = PsutilBackend(monitor)
+    backend.run(command, monitor)