matterbak 0.4.0__py3-none-any.whl

matterbak/__init__.py

@@ -0,0 +1,6 @@
+"""
+main entry point for the matterbak
+"""
+
+# We intentionally do not export any objects, because the package should only
+# be use as executable script

matterbak/channeldata.py

@@ -0,0 +1,148 @@
+"""
+Provide class ChannelData
+"""
+
+
+import json
+import pathlib as pl
+
+from . import dump
+
+files_subdir = pl.Path('files')
+
+
+class ChannelData:
+    # pylint: disable = too-few-public-methods, too-many-instance-attributes
+    """Class to store channel data and back it up"""
+
+    def __init__(self, init, name, channel, channels_dir):
+        """Init
+
+        init:         the Init instance
+        name:         name for the channel data file and its subdir
+        channel:      channel data
+        channels_dir: pathlib.Path with the dir to store the data in
+        """
+        self.init = init
+        self.name = name
+        self.channel = channel
+        if 'id' not in self.channel:
+            raise KeyError("key 'id' not found in channel")
+        self.channels_dir = channels_dir
+        self._threads_filename = f"{self.name}{dump.FILENAME_SEPARATOR}{dump.SUFFIX_THREADS}"
+        self.posts_dir = (self.channels_dir /
+            dump.make_filename(self.channel['id'], name=self.name))
+        self.files_dir = self.posts_dir / files_subdir
+        self.files_dir.mkdir(parents=True, exist_ok=True)
+        self._load_threads()
+
+    def _get_latest_post_id(self):
+        """Return latest ID of posts in posts_dir
+
+        This function assumes that the file names begin with a timestamps, such that
+        the latest post has the lexicographically highest name.
+
+        posts_dir: pathlib.Path of a dir with json files containing posts data
+
+        return: post ID contained in the file with the max file name or None
+        """
+        latest_post_file = self.posts_dir / ' '
+        for post_file in self.posts_dir.iterdir():
+            if post_file.suffix.lower() != dump.JSON_EXTENSION:
+                continue
+            if post_file.name > latest_post_file.name:
+                latest_post_file = post_file
+
+        if latest_post_file.exists():
+            with latest_post_file.open(encoding="utf8") as post_file:
+                post = json.load(post_file)
+                return post.get('id')
+
+        return None
+
+    def _load_threads(self):
+        """Load thread data from backup"""
+        self._threads = {}
+
+        threads_path = (self.channels_dir /
+            dump.make_filename(
+                self.channel['id'], name=self._threads_filename, extension=dump.JSON_EXTENSION))
+        if threads_path.is_file():
+            with threads_path.open(encoding="utf8") as threads_file:
+                threads_json = json.load(threads_file)
+            # Has the file the new format with root_ids as keys?
+            # (The old file contained a list of lists.)
+            # If not ignore loaded file. It will be overwritten with the new format.
+            if isinstance(threads_json, dict):
+                self._threads = {root_id: set(post_ids)
+                                 for root_id, post_ids in threads_json.items()}
+
+    def _save_post(self, post):
+        """Backup a post and its files"""
+        num_files = 0
+        for file_desc in post["metadata"].get("files", []):
+            file_id = file_desc["id"]
+            dump.dump_content(self.files_dir, file_desc)
+            file_respone = self.init.matter.get_file(file_id)
+            if file_respone.ok:
+                # extension is contained in name
+                file_dump_path = (self.files_dir /
+                    dump.make_filename(file_id, name=file_desc['name']))
+                file_dump_path.write_bytes(file_respone.content)
+                num_files += 1
+            else:
+                print(
+                    f"Cannot retrieve the file '{file_desc['name']}'"
+                    f"posted to channel '{self.name}': {file_respone.text}")
+        return num_files
+
+    def _update_threads(self, post):
+        """Update thread data with new post"""
+        root_id = post['root_id']
+        if root_id:
+            if root_id not in self._threads:
+                self._threads[root_id] = set()
+            self._threads[root_id].add(post['id'])
+
+    def backup(self):
+        """Download channel data and all its posts and files"""
+
+        dump.dump_content(self.channels_dir, self.channel, name=self.name)
+
+        members = self.init.users.get_group_members(self.channel)
+        dump.dump_content(
+            self.channels_dir, members, id_=self.channel['id'],
+            name=f"{self.name}{dump.FILENAME_SEPARATOR}{dump.SUFFIX_MEMBERS}")
+
+        if self.init.options.update_old_posts:
+            latest_id = None
+        else:
+            latest_id = self._get_latest_post_id()
+
+        num_posts = 0
+        num_files = 0
+        for post in self.init.matter.get_posts_for_channel(self.channel['id'], after=latest_id):
+            self.init.rate_limiter.wait()
+            proggress_symbol = '.'
+            old_content = dump.dump_content(
+                self.posts_dir, post, with_timestamp=True, return_old_content=True)
+            if (not old_content) or (old_content['update_at'] < post['update_at']):
+                proggress_symbol = '+'
+                num_posts += 1
+                num_files += self._save_post(post)
+
+            # We update the threads in any case although thread relations cannot be changed
+            # because this will update the thread file format.
+            self._update_threads(post)
+
+            print(proggress_symbol, end='', flush=True)
+
+        # Newline after progress dots
+        print()
+
+        threads_json = {root_id: list(post_ids)
+                        for root_id, post_ids in self._threads.items()}
+        dump.dump_content(self.channels_dir, threads_json,
+                          id_=self.channel['id'], name=self._threads_filename)
+
+        return num_posts, num_files

matterbak/dump.py

@@ -0,0 +1,129 @@
+"""
+Provide functions to dump data into JSON files
+"""
+
+
+import datetime
+import json
+import pathlib as pl
+
+from .ignoresignals import IgnoreSignals
+
+JSON_EXTENSION = '.json'
+# Separator between parts of a filename
+FILENAME_SEPARATOR = '__'
+# Format for timestamps in file names
+TIMESTAMP_FORMAT = "%Y%m%d-%H%M%S%f"
+
+# Subdirs below data_dir to store the related downloads
+teams_subdir = pl.Path('teams')
+groups_subdir = pl.Path('groups')
+direct_subdir = pl.Path('direct')
+emojis_subdir = pl.Path('emojis')
+users_subdir = pl.Path('users')
+files_subdir = pl.Path('files')
+
+# Suffixes for types of data files
+SUFFIX_MEMBERS = 'members'
+SUFFIX_THREADS = 'threads'
+SUFFIX_ICON = 'icon'
+SUFFIX_IMAGE = 'image'
+
+
+def make_filename(id_, name=None, extension='', mm_timestamp=None):
+    """Make a filename for a backup file
+
+    id_:          Mattermost ID to insert into the filename
+    name:         optional name to append
+    extension:    optional extension for the filename
+    mm_timestamp: optional Mattermost timestamp (Unix time in milliseconds)
+
+    return: filename
+    """
+    filename_parts = []
+    if mm_timestamp:
+        now = datetime.datetime.fromtimestamp(mm_timestamp / 1000)
+        filename_parts.append(now.strftime(TIMESTAMP_FORMAT))
+    filename_parts.append(id_)
+    if name:
+        filename_parts.append(name)
+
+    return FILENAME_SEPARATOR.join(filename_parts) + extension
+
+
+def dump_image(directory, id_, image_loader, label=None, skip_existing=False):
+    """Helper to download and save an image from Mattermost
+
+    Calls make_filename with id_, label as name, and extension derived from the
+    content type returned from Mattermost.
+
+    directory:     pathlib.Path of the folder to store the image in
+    id_:           Mattermost ID as prefix for the filename
+    image_loader:  function returning an image Response object from Mattermost API
+    label:         label to append to filename
+    skip_existing: if True skip download if image file already exists
+    """
+
+    found_image_files = [f for f in directory.glob(id_+'*')
+                         if f.suffix != JSON_EXTENSION and f.is_file()]
+    if skip_existing and found_image_files:
+        return
+
+    # The new image file may have a different extension so delete all existing
+    # image files.
+    for image in found_image_files:
+        image.unlink(missing_ok=True)
+
+    response = image_loader()
+    if not response.ok:
+        return
+
+    content_type_prefix = 'image/'
+    content_type = response.headers.get('content-type', '')
+    if not content_type.startswith(content_type_prefix):
+        print(f"Cannot store image of type '{content_type}' for ID {id_}")
+        return
+    extension = '.' + content_type.removeprefix(content_type_prefix)
+
+    with IgnoreSignals():
+        path = (directory /
+            make_filename(id_=id_, name=label, extension=extension))
+        path.write_bytes(response.content)
+
+
+def dump_content(directory, content, id_=None, name=None, with_timestamp=False,
+                 return_old_content=False):
+    # pylint: disable = too-many-arguments, too-many-positional-arguments
+    """Helper to save the content as JSON file
+
+    Calls make_filename with id_ (if given else content['id']), name, and
+    with_timestamp to create the filename.
+
+    directory:          pathlib.Path of the folder to store the file in
+    content:            data to store
+    id_:                Mattermost ID to be integrated into filename, if None use
+                        content['id'] instead
+    name:               name (without .json extension) of the file, can be empty
+    with_timestamp:     set to True to prefix filename with content's creation time
+    return_old_content: if True content of file to be overwritten is returned
+                        or None if there was no content file
+    """
+
+    if not id_:
+        id_ = content['id']
+    mm_timestamp = content["create_at"] if with_timestamp else None
+
+    path = (directory /
+        make_filename(id_, name=name, extension=JSON_EXTENSION,
+                      mm_timestamp=mm_timestamp))
+
+    old_content = None
+    if return_old_content and path.is_file():
+        with path.open(encoding="utf8") as old_file:
+            old_content = json.load(old_file)
+
+    with IgnoreSignals():
+        with path.open(mode="w", encoding="utf8") as dump_file:
+            json.dump(content, dump_file)
+
+    return old_content

matterbak/hashablematterdata.py

@@ -0,0 +1,16 @@
+"""
+Provide class HashableMatterData
+"""
+
+
+class HashableMatterData(dict):
+    """Extends the dict of a mattermost object by hash method to enable storing in a set
+
+    This class can be initialized with the original dict of the mattermost object.
+    """
+
+    def __eq__(self, other):
+        return self['id'] == other['id']
+
+    def __hash__(self):
+        return hash(self['id'])

matterbak/ignoresignals.py

@@ -0,0 +1,106 @@
+"""
+A simple class to temporarily ignore specific signals (e.g., SIGINT, SIGTERM)
+during critical operations like file writing.
+"""
+
+import os
+import signal
+
+
+class IgnoreSignals():
+    """Context manager to temporarily ignore specified signals"""
+    def __init__(
+            self,
+            signals=None,
+            print_message_on_signal=None,
+            delay_signals=True):
+        """
+        Temporarily ignore specified signals (e.g., Ctrl+C, kill) during
+        critical operations.
+
+        Example:
+
+        >>> with IgnoreSignals([signal.SIGINT, signal.SIGTERM]):
+        ...     # do critical work like file writing
+
+        signals (list): List of signal numbers to ignore
+                        (default: [signal.SIGINT, signal.SIGTERM]).
+                        Use `None` to accept the default.
+        print_message_on_signal (str or callable or None):
+            message to be printed on signal
+            * If None (default) the f-string
+              f'ignoring signal {signum} until write is finished'
+              is used.
+            * If a callable (e.g. lambda function):
+              called with (signum, frame). This function could print other
+              messages build with input.
+            * If bool(print_message_on_signal) is True
+              the variable print_message_on_signal is printed.
+            * Otherwise (e. g. False or '') no output.
+        delay_signals:
+            If True the default handler will be called just after reverting
+            Only the last ignored signal is re-raised.
+            Attempting to re-raise multiple signals would risk unpredictable
+            behavior and is intentionally omitted.
+
+        return: dict with actual handlers
+        """
+        if signals is None:
+            signals = [signal.SIGINT, signal.SIGTERM]
+        self.signals = signals
+        self.print_message_on_signal = print_message_on_signal
+        self.default_handlers = []
+        self.update_default_handlers()
+        self.ignored_signum = None
+        self.delay_signals = delay_signals
+
+    def __enter__(self):
+        """Enter 'with' context"""
+        self.ignore()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Leave 'with' context"""
+        self.revert()
+
+    def update_default_handlers(self):
+        """
+        store actual signal handlers as default handlers
+        """
+        self.default_handlers = []
+        for sig in self.signals:
+            self.default_handlers.append([sig, signal.getsignal(sig)])
+
+    def ignore(self):
+        """
+        set signal handlers to ignore the signals
+        """
+        for sig in self.signals:
+            signal.signal(sig, self.ignoring_handler)
+
+    def revert(self):
+        """
+        revert/restore to original/default signal handlers
+
+        More precisely the signal handlers defined during
+        class instance creation
+        or calling the method `update_default_handlers`
+        """
+        for (sig, handler) in self.default_handlers:
+            signal.signal(sig, handler)
+        if self.delay_signals and (self.ignored_signum is not None):
+            os.kill(os.getpid(), self.ignored_signum)
+            self.ignored_signum = None
+
+    def ignoring_handler(self, signum, frame):
+        """
+        handler that ignores the signal and optionally prints a message
+        or calls a function
+        """
+        self.ignored_signum = signum
+        if self.print_message_on_signal is None:
+            print(f'ignoring signal {signum} until write is finished')
+        if callable(self.print_message_on_signal):
+            self.print_message_on_signal(signum, frame)
+        elif self.print_message_on_signal:
+            print(self.print_message_on_signal)