dfsync 0.4.0__tar.gz → 0.4.2__tar.gz

{dfsync-0.4.0 → dfsync-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfsync
-Version: 0.4.0
+Version: 0.4.2
 Summary: 
 Author: Mihai Balint
 Author-email: balint.mihai@gmail.com

{dfsync-0.4.0 → dfsync-0.4.2}/dfsync/backends/kube_exec.py

@@ -46,14 +46,21 @@ def main():
         print(f"[{pod_name}] Running: {cmd_str}", file=f, flush=True)
         try:
             subprocess.check_call(cmd)
+            return 0
+
+        except subprocess.CalledProcessError as e:
+            err = str(e) or type(e)
+            print(f"[{pod_name}] Error: {err}", file=f, flush=True)
+            return e.returncode
 
         except Exception as e:
             err = str(e) or type(e)
             print(f"[{pod_name}] Error: {err}", file=f, flush=True)
+            return 1
 
         finally:
             print(f"[{pod_name}] Exiting", file=f, flush=True)
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

{dfsync-0.4.0 → dfsync-0.4.2}/dfsync/backends/rsync.py

@@ -1,6 +1,7 @@
-import logging, os, os.path, subprocess
+import logging, os, os.path, select, subprocess, time
 
 from dfsync.filters import list_files_to_ignore
+from dfsync.lib import ControlledThreadedOperation
 
 EVENT_TYPE_MAP = {
     "created": "Created",
@@ -10,6 +11,107 @@ EVENT_TYPE_MAP = {
 }
 
 
+class StopStreamReader(Exception):
+    pass
+
+
+class RsyncStreamReader(ControlledThreadedOperation):
+    def __init__(self, stream):
+        super().__init__()
+        if stream is None:
+            raise ValueError("Rsync status stream must not be None")
+
+        self.stream = stream
+        self.timeout = 300.0
+
+        self.rsync_stats = None
+        self.rsync_error = None
+        self.rsync_permission_error_count = 0
+        self.stream_closed = False
+
+    def read_lines(self):
+        spoll = select.poll()
+        spoll.register(self.stream, select.POLLIN)
+        try:
+            line_timestamp = time.monotonic()
+            elapsed = 0
+
+            while elapsed < self.timeout:
+                if spoll.poll(0):
+                    line = self.stream.readline()
+                    if len(line) > 0:
+                        line_timestamp = time.monotonic()
+                        yield line.decode("utf8")
+                    else:
+                        return
+                else:
+                    time.sleep(0.1)
+                elapsed = time.monotonic() - line_timestamp
+
+            if elapsed >= self.timeout:
+                raise subprocess.TimeoutExpired(cmd="read_lines", timeout=self.timeout)
+        except ValueError:
+            pass
+
+        finally:
+            self.stream_closed = True
+
+            try:
+                spoll.unregister(self.stream)
+            except:
+                # don't care at this point
+                pass
+
+            try:
+                self.stream.close()
+            except:
+                # don't care at this point
+                pass
+
+    def run(self):
+        try:
+            for line in self.read_lines():
+                if line and line.startswith("sent "):
+                    # stdout
+                    self.rsync_stats = line.strip()
+                elif line and line.startswith("X11 forwarding"):
+                    # stdout
+                    pass
+                elif line and line.startswith("building file list"):
+                    # stdout
+                    pass
+                elif line and line.startswith("total size"):
+                    # stdout
+                    pass
+                elif line and line.startswith("rsync error"):
+                    # stderr
+                    self.rsync_error = line.strip()
+                elif line and line.startswith("rsync: ") and "failed: Permission" in line:
+                    # stderr
+                    self.rsync_permission_error_count += 1
+                elif line and line.startswith("rsync: ") and "failed: Resource busy" in line:
+                    # stderr
+                    self.rsync_permission_error_count += 1
+
+                elif line and line.strip():
+                    # echo(line.strip())
+                    pass
+
+                if not self.is_running:
+                    self.timeout = 1.0
+
+        except (subprocess.TimeoutExpired, StopStreamReader):
+            # Stop silently
+            pass
+
+    def stop(self, *args, **kwargs):
+        super().stop(*args, **kwargs)
+        self.stream.close()
+
+        while not self.stream_closed:
+            time.sleep(1.0)
+
+
 def echo(msg=""):
     print(f"{msg}")
 
@@ -76,14 +178,43 @@ class FileRsync:
         if rsync_cwd:
             popen_args.update(dict(cwd=rsync_cwd))
         try:
-            subprocess.check_call(
+            rsync_process = subprocess.Popen(
                 rsync_cmd,
                 stdin=subprocess.DEVNULL,
-                stdout=subprocess.DEVNULL,
-                stderr=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
                 env=rsh_env,
                 **popen_args,
             )
+            stdout = RsyncStreamReader(rsync_process.stdout)
+            stdout.start()
+
+            stderr = RsyncStreamReader(rsync_process.stderr)
+            stderr.start()
+
+            return_code = rsync_process.wait(timeout=None)
+
+            stdout.stop()
+            stderr.stop()
+
+            if stdout.rsync_stats:
+                echo(f"  {stdout.rsync_stats}")
+
+            if return_code != 0:
+                if stderr.rsync_error:
+                    echo(f"  {stderr.rsync_error}")
+                else:
+                    echo(f"  rsync exit code: {return_code}")
+
+                if stderr.rsync_permission_error_count > 0:
+                    files = "file" if stderr.rsync_permission_error_count == 1 else "files"
+                    echo(
+                        f"  {stderr.rsync_permission_error_count} {files} had permission or other issues on destination"
+                    )
+
+                if return_code != 23:
+                    raise subprocess.CalledProcessError(returncode=return_code, cmd=rsync_cmd)
+
         except:
             echo("Sync failed")
             env_str = "N/A"

{dfsync-0.4.0 → dfsync-0.4.2}/dfsync/filters.py

@@ -96,8 +96,12 @@ class PythonBlackFilter(LoggingFilter):
                 black.format_file_contents(contents, fast=False, mode=black.FileMode())
             return False
 
+        except (FileNotFoundError, PermissionError, black.report.NothingChanged) as e:
+            return False
+
         except Exception as e:
-            echo(f"Rejected {src_file_path}, {e}")
+            message = str(e) or type(e).__name__
+            echo(f"Rejected {src_file_path}, {message}")
             return True
 
 

{dfsync-0.4.0 → dfsync-0.4.2}/dfsync/lib.py

@@ -1,16 +1,32 @@
 import time, threading
 
 
+class ThreadedOperationsManager:
+    def __init__(self):
+        self.threads = []
+
+    def stop(self, *args, **kwargs):
+        for thread in self.threads:
+            thread.stop(*args, **kwargs)
+
+    def register_threaded_operation(self, op):
+        self.threads.append(op)
+
+
+thread_manager = ThreadedOperationsManager()
+
+
 class ControlledThreadedOperation:
     def __init__(self):
         self._running = False
         self._thread = threading.Thread(target=self.run)
+        thread_manager.register_threaded_operation(self)
 
     def start(self):
         self._running = True
         self._thread.start()
 
-    def stop(self, message: str = None):
+    def stop(self, *args, **kwargs):
         self._running = False
 
     @property

{dfsync-0.4.0 → dfsync-0.4.2}/dfsync/monitor.py

@@ -4,8 +4,10 @@ import os.path
 import sys
 import time
 import logging
+import queue
 
 from click_default_group import DefaultGroup
+from contextlib import contextmanager
 from functools import partial
 from watchdog.observers import Observer
 from watchdog.events import FileSystemEventHandler
@@ -16,6 +18,7 @@ from dfsync.filters import add_user_ignored_patterns_filter, ALL_FILTERS
 from dfsync.config import read_config
 from dfsync.char_ui import KeyController
 from dfsync.kube_credentials import contextualize_kube_credentials, update_local_kube_config
+from dfsync.lib import ControlledThreadedOperation, thread_manager
 
 logging.basicConfig(level=logging.WARN)
 
@@ -30,19 +33,50 @@ class IgnoreEvent(Exception):
     pass
 
 
-class FileChangedEventHandler(FileSystemEventHandler):
-    def __init__(self, backend: str = "log", watched_dir: str = ".", input_controller: KeyController = None, **kwargs):
+class FileChangedEventHandler(ControlledThreadedOperation, FileSystemEventHandler):
+    def __init__(
+        self,
+        backend: str = "log",
+        watched_dir: str = ".",
+        all_watched_dirs: list = None,
+        input_controller: KeyController = None,
+        **kwargs,
+    ):
+        super().__init__()
         self.filters = [*ALL_FILTERS]
 
         self.backend = backend
         self.backend_options = kwargs
         self.raised_exception = False
+        self.watched_dir = watched_dir
         self.abs_watched_dir = os.path.abspath(watched_dir)
         self.input_controller = input_controller
+        self.events = queue.Queue(maxsize=10000)
+        self.full_sync_threashold = 3
+        self.all_watched_dirs = all_watched_dirs if all_watched_dirs is not None else [watched_dir]
 
     def _log_backend(self, event):
         logging.info(event)
 
+    @contextmanager
+    def _input_lock(self):
+        if self.input_controller is None:
+            yield self
+        else:
+            with self.input_controller.getch_lock():
+                yield self
+
+    @contextmanager
+    def terminal_lock(self):
+        try:
+            with self._input_lock():
+                yield self
+        except IgnoreEvent:
+            pass
+        except:
+            self.raised_exception = True
+            raise
+
     def _sync(self, event):
         # It seems like in some contexts, event.src_path will be an absolute path
         # and in some other contexts, it will be a relative path
@@ -78,24 +112,67 @@ class FileChangedEventHandler(FileSystemEventHandler):
         except ValueError as e:
             raise IgnoreEvent() from e
 
-    def _propagate_event(self, event):
-        for file_filter in self.filters:
-            if file_filter(event=event) is False:
-                return
-        self._sync(event)
+    def _drain_queue(self, timeout=0.5):
+        event = self.events.get(block=True, timeout=timeout)
+        latest_events = {event.src_path: event}
+        try:
+            # Allow a bit of time for the queue to fill
+            if self.events.qsize() == 0:
+                time.sleep(0.25)
+            # Started filling? allow a little bit more time
+            if self.events.qsize() > 0:
+                time.sleep(0.4)
+
+            # Start draining
+            while self.events.qsize() > 0:
+                event = self.events.get_nowait()
+                # Only keep the latest event for a given file path
+                latest_events[event.src_path] = event
+
+        except queue.Empty:
+            pass
+        finally:
+            return latest_events.values()
+
+    def _filter_events(self, latest_events, stop_threashold=None):
+        sync_events = []
+        for event in latest_events:
+            filtered = False
+            for file_filter in self.filters:
+                with self.terminal_lock():
+                    if file_filter(event=event) is False:
+                        filtered = True
+                        break
+            if not filtered:
+                sync_events.append(event)
+            if stop_threashold is not None and len(sync_events) > stop_threashold:
+                return sync_events
+
+        return sync_events
+
+    def run(self):
+        while self._running:
+            try:
+                latest_events = self._drain_queue()
+                sync_events = self._filter_events(latest_events, stop_threashold=self.full_sync_threashold)
+                if len(sync_events) >= self.full_sync_threashold:
+                    with self.terminal_lock():
+                        self.backend.sync_project(self.all_watched_dirs, **self.backend_options)
+                else:
+                    for event in sync_events:
+                        with self.terminal_lock():
+                            self._sync(event)
+            except queue.Empty:
+                time.sleep(0.001)
 
     def catch_all_handler(self, event):
         try:
-            if self.input_controller is None:
-                self._propagate_event(event)
-            else:
-                with self.input_controller.getch_lock():
-                    self._propagate_event(event)
-        except IgnoreEvent:
+            # Add sync event to the sync queue
+            self.events.put(event)
+        except queue.Full:
+            # It's acceptable for events to be rejected from the queue
+            # because a busy queue will trigger a full-sync
             pass
-        except:
-            self.raised_exception = True
-            raise
 
     def on_moved(self, event):
         self.catch_all_handler(event)
@@ -256,9 +333,10 @@ def sync(source, destination, supervisor, kube_host, pod_timeout, full_sync):
     observer = Observer()
     for p in paths:
         event_handler = FileChangedEventHandler(
-            backend_engine, watched_dir=p, input_controller=controller, **backend_options
+            backend_engine, watched_dir=p, all_watched_dirs=paths, input_controller=controller, **backend_options
         )
         handlers.append(event_handler)
+        event_handler.start()
         observer.schedule(event_handler, os.path.abspath(p), recursive=True)
 
     controller.on_key(
@@ -269,7 +347,7 @@ def sync(source, destination, supervisor, kube_host, pod_timeout, full_sync):
     controller.on_key(
         "x",
         description="to exit",
-        action=partial(controller.stop, "Exiting."),
+        action=partial(thread_manager.stop, "Exiting."),
     )
 
     try:
@@ -295,16 +373,16 @@ def sync(source, destination, supervisor, kube_host, pod_timeout, full_sync):
             controller.raise_exceptions()
 
     except KeyboardInterrupt:
-        controller.stop()
+        thread_manager.stop()
         click.echo("Received [Ctrl-C], exiting.")
 
     except:
-        controller.stop()
+        thread_manager.stop()
         raise
 
     finally:
         observer.stop()
-        checker.stop()
+        thread_manager.stop()
         backend_engine.on_monitor_exit(**backend_options)
         if observer.ident is not None:
             # only join the observer if it was previously started

{dfsync-0.4.0 → dfsync-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dfsync"
-version = "0.4.0"
+version = "0.4.2"
 description = ""
 authors = ["Mihai Balint <balint.mihai@gmail.com>"]