dotsync-cli 1.0.2__py3-none-any.whl

dotsync/flists.py

@@ -0,0 +1,90 @@
+import logging
+import os
+import re
+
+import dotsync.info as info
+
+
+class Filelist:
+    def __init__(self, fname):
+        self.groups = {}
+        self.files = {}
+
+        logging.debug(f'parsing filelist in {fname}')
+
+        with open(fname, 'r') as f:
+            for line in f.readlines():
+                line = line.strip()
+
+                if not line or line.startswith('#'):
+                    continue
+
+                # group
+                if '=' in line:
+                    group, categories = line.split('=')
+                    categories = categories.split(',')
+                    if group == info.hostname:
+                        categories.append(info.hostname)
+                    self.groups[group] = categories
+                # file
+                else:
+                    split = re.split('[:|]', line)
+
+                    path, categories, plugin = split[0], ['common'], 'plain'
+                    if len(split) >= 2:
+                        if ':' in line:
+                            categories = split[1].split(',')
+                        else:
+                            plugin = split[1]
+                    if len(split) >= 3:
+                        plugin = split[2]
+
+                    if path not in self.files:
+                        self.files[path] = []
+                    self.files[path].append({
+                        'categories': categories,
+                        'plugin': plugin
+                    })
+
+    def activate(self, categories):
+        # expand groups
+        categories = [self.groups.get(c, [c]) for c in categories]
+        # flatten category list
+        categories = [c for cat in categories for c in cat]
+
+        files = {}
+        for path in self.files:
+            for group in self.files[path]:
+                cat_list = group['categories']
+                if set(categories) & set(cat_list):
+                    if path in files:
+                        logging.error('multiple category lists active for '
+                                      f'{path}: {files[path]["categories"]} '
+                                      f'and {cat_list}')
+                        raise RuntimeError
+                    else:
+                        files[path] = group
+
+        return files
+
+    # generates a list of all the filenames in each plugin for later use when
+    # cleaning the repo
+    def manifest(self):
+        manifest = {}
+
+        for path in self.files:
+            for instance in self.files[path]:
+                plugin = instance['plugin']
+                for category in instance['categories']:
+                    if category in self.groups:
+                        categories = self.groups[category]
+                    else:
+                        categories = [category]
+
+                    if plugin not in manifest:
+                        manifest[plugin] = []
+
+                    for category in categories:
+                        manifest[plugin].append(os.path.join(category, path))
+
+        return manifest

dotsync/git.py

@@ -0,0 +1,161 @@
+import os
+import subprocess
+import shlex
+import logging
+import enum
+
+
+class FileState(enum.Enum):
+    MODIFIED = 'M'
+    ADDED = 'A'
+    DELETED = 'D'
+    RENAMED = 'R'
+    COPIED = 'C'
+    UPDATED = 'U'
+    UNTRACKED = '?'
+
+
+class Git:
+    def __init__(self, repo_dir):
+        if not os.path.isdir(repo_dir):
+            raise FileNotFoundError
+
+        self.repo_dir = repo_dir
+
+    def run(self, cmd):
+        if not type(cmd) is list:
+            cmd = shlex.split(cmd)
+        logging.info(f'running git command {cmd}')
+        try:
+            proc = subprocess.run(cmd, cwd=self.repo_dir,
+                                  stdout=subprocess.PIPE, check=True)
+        except subprocess.CalledProcessError as e:
+            logging.error(e.stdout.decode())
+            logging.error(f'git command {cmd} failed with exit code '
+                          f'{e.returncode}\n')
+            raise
+        logging.debug(f'git command {cmd} succeeded')
+        return proc.stdout.decode()
+
+    def init(self):
+        self.run('git init')
+
+    def reset(self, fname=None):
+        self.run('git reset' if fname is None else f'git reset {fname}')
+
+    def add(self, fname=None):
+        self.run('git add --all' if fname is None else f'git add {fname}')
+
+    def commit(self, message=None):
+        if message is None:
+            message = self.gen_commit_message()
+        return self.run(['git', 'commit', '-m', message])
+
+    def status(self, staged=True):
+        out = self.run('git status --porcelain').strip()
+        status = []
+        
+        # Handle empty output
+        if not out:
+            return status
+        
+        for line in out.split('\n'):
+            # Skip empty lines
+            if not line.strip():
+                continue
+            
+            # git status --porcelain format: XY filename
+            # X = staged state, Y = working tree state
+            # Both can be: M=modified, A=added, D=deleted, R=renamed, C=copied, U=updated, ?=untracked
+            # Space means no change in that area
+            if len(line) < 3:
+                logging.warning(f'Unexpected git status line format: {line}')
+                continue
+            
+            state_chars = line[:2]
+            path = line[3:].strip() if len(line) > 3 else ''
+            
+            # Handle special cases that don't follow XY format
+            if state_chars in ['!!', '##']:
+                continue
+            
+            # Extract stage and work states
+            stage_char = state_chars[0]
+            work_char = state_chars[1]
+            
+            # Choose which state to use based on staged parameter
+            # If preferred state is space, try the other one
+            if staged:
+                state_char = stage_char if stage_char != ' ' else work_char
+            else:
+                state_char = work_char if work_char != ' ' else stage_char
+            
+            # Skip if both states are spaces (no change)
+            if state_char == ' ':
+                continue
+            
+            # Only process if we have a valid path
+            if not path:
+                continue
+            
+            try:
+                file_state = FileState(state_char)
+            except ValueError:
+                # Handle unknown states gracefully (might be submodule states, etc.)
+                logging.debug(f'Unknown file state: {state_char} in line: {line}, skipping')
+                continue
+            
+            status.append((file_state, path))
+        
+        return sorted(status, key=lambda s: s[1])
+
+    def has_changes(self):
+        return bool(self.run('git status -s --porcelain').strip())
+
+    def gen_commit_message(self, ignore=[]):
+        mods = []
+        for stat in self.status():
+            state, path = stat
+            # skip all untracked files since they will not be committed
+            if state == FileState.UNTRACKED:
+                continue
+            if any((path.startswith(p) for p in ignore)):
+                logging.debug(f'ignoring {path} from commit message')
+                continue
+            mods.append(f'{state.name.lower()} {path}')
+        
+        # Return empty string if no modifications after filtering
+        if not mods:
+            return ''
+        
+        return ', '.join(mods).capitalize()
+
+    def commits(self):
+        return self.run('git log -1 --pretty=%s').strip().split('\n')
+
+    def last_commit(self):
+        return self.commits()[-1]
+
+    def has_remote(self):
+        return bool(self.run('git remote').strip())
+
+    def push(self):
+        self.run('git push')
+
+    def diff(self, ignore=[]):
+        if not self.has_changes():
+            return ['no changes']
+
+        self.add()
+        status = self.status()
+        self.reset()
+
+        diff = []
+
+        for path in status:
+            # ignore the paths specified in ignore
+            if any((path[1].startswith(i) for i in ignore)):
+                continue
+            diff.append(f'{path[0].name.lower()} {path[1]}')
+
+        return diff

dotsync/info.py

@@ -0,0 +1,11 @@
+from os.path import expanduser
+import socket
+
+__version__ = '1.0.2'
+__author__ = 'Harvey'
+__author_email__ = 'harvey.wanghy@gmail.com'
+__url__ = 'https://github.com/HarveyGG/dotsync'
+__license__ = 'MIT License (Non-Commercial Use Only)'
+
+home = expanduser('~')
+hostname = socket.gethostname()

dotsync/plugin.py

@@ -0,0 +1,48 @@
+import os
+
+
+class Plugin:
+    def __init__(self, data_dir, repo_dir=None):
+        self.data_dir = data_dir
+        self.repo_dir = '/' if repo_dir is None else repo_dir
+
+        if not os.path.isdir(self.data_dir):
+            os.makedirs(self.data_dir)
+
+        self.setup_data()
+
+    # does plugin-specific setting up of data located in the data_dir
+    def setup_data(self):
+        pass
+
+    # cleans up plugin's data by removing entries that is no longer in the
+    # given manifest
+    def clean_data(self, manifest):
+        pass
+
+    # takes a source (outside the repo) and applies its operation and store the
+    # resulting file in dest (inside the repo). This operation should not
+    # remove the source file
+    def apply(self, source, dest):
+        pass
+
+    # takes a source (inside the repo) and removes its operation and stores the
+    # result in dest (outside the repo)
+    def remove(self, source, dest):
+        pass
+
+    # takes a path to a repo_file and an ext_file and compares them, should
+    # return true if they are the same file
+    def samefile(self, repo_file, ext_file):
+        pass
+
+    # takes a callable (one of the plugin's ops) and returns a string
+    # describing the op
+    def strify(self, op):
+        pass
+
+    # takes a path inside the repo and strips the repo dir as a prefix
+    def strip_repo(self, path):
+        if os.path.isabs(path):
+            return os.path.relpath(path, self.repo_dir)
+        return path

dotsync/plugins/encrypt.py

@@ -0,0 +1,230 @@
+import subprocess
+import shlex
+import logging
+import json
+import getpass
+import hashlib
+import os
+import tempfile
+
+from dotsync.plugin import Plugin
+
+
+class GPG:
+    def __init__(self, password):
+        self.password = password
+
+    def run(self, cmd):
+        if not type(cmd) is list:
+            cmd = shlex.split(cmd)
+
+        # these are needed to read the password from stdin and to not ask
+        # questions
+        pre = ['--passphrase-fd', '0', '--pinentry-mode', 'loopback',
+               '--batch', '--yes']
+        # insert pre into the gpg command string
+        cmd = cmd[:1] + pre + cmd[1:]
+
+        logging.debug(f'running gpg command {cmd}')
+
+        try:
+            proc = subprocess.run(cmd, input=self.password.encode(),
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.PIPE, check=True)
+        except subprocess.CalledProcessError as e:
+            logging.error(e.stderr.decode())
+            logging.error(f'gpg command {cmd} failed with exit code '
+                          f'{e.returncode}\n')
+            raise
+
+        logging.debug(f'gpg command {cmd} succeeded')
+        return proc.stdout.decode()
+
+    def encrypt(self, input_file, output_file):
+        self.run(f'gpg --armor --output {shlex.quote(output_file)} '
+                 f'--symmetric {shlex.quote(input_file)}')
+
+    def decrypt(self, input_file, output_file):
+        self.run(f'gpg --output {shlex.quote(output_file)} '
+                 f'--decrypt {shlex.quote(input_file)}')
+
+
+# calculates the sha256 hash of the file at fpath
+def hash_file(path):
+    h = hashlib.sha256()
+
+    with open(path, 'rb') as f:
+        while True:
+            chunk = f.read(h.block_size)
+            if not chunk:
+                break
+            h.update(chunk)
+
+    return h.hexdigest()
+
+
+# hash password using suitable key-stretching algorithm
+# salt needs to be >16 bits from a suitable cryptographically secure random
+# source, but can be stored in plaintext
+def key_stretch(password, salt):
+    if type(password) is not bytes:
+        password = password.encode()
+    if type(salt) is not bytes:
+        salt = bytes.fromhex(salt)
+    key = hashlib.pbkdf2_hmac(hash_name='sha256', password=password, salt=salt,
+                              iterations=100000)
+    return key.hex()
+
+
+class EncryptPlugin(Plugin):
+    def __init__(self, data_dir, *args, **kwargs):
+        self.gpg = None
+        self.hashes_path = os.path.join(data_dir, 'hashes')
+        self.modes_path = os.path.join(data_dir, 'modes')
+        self.pword_path = os.path.join(data_dir, 'passwd')
+        super().__init__(*args, data_dir=data_dir, **kwargs)
+
+    # reads the stored hashes
+    def setup_data(self):
+        if os.path.exists(self.hashes_path):
+            with open(self.hashes_path, 'r') as f:
+                self.hashes = json.load(f)
+        else:
+            self.hashes = {}
+
+        if os.path.exists(self.modes_path):
+            with open(self.modes_path, 'r') as f:
+                self.modes = json.load(f)
+        else:
+            self.modes = {}
+
+    # removes file entries in modes and hashes that are no longer in the
+    # manifest
+    def clean_data(self, manifest):
+        for data in [self.hashes, self.modes]:
+            diff = set(data) - set(manifest)
+            for key in diff:
+                data.pop(key)
+        self.save_data()
+
+    # saves the current hashes and modes to the data dir
+    def save_data(self):
+        with open(self.hashes_path, 'w') as f:
+            json.dump(self.hashes, f)
+        with open(self.modes_path, 'w') as f:
+            json.dump(self.modes, f)
+
+    # sets the password in the plugin's data dir. do not use directly, use
+    # change_password instead
+    def save_password(self, password):
+        # get salt from crypto-safe random source
+        salt = os.urandom(32)
+        # calculate password hash
+        key = key_stretch(password.encode(), salt)
+
+        # save salt and hash
+        with open(self.pword_path, 'w') as f:
+            d = {'pword': key, 'salt': salt.hex()}
+            json.dump(d, f)
+
+    # takes a password and checks if the correct password was entered
+    def verify_password(self, password):
+        with open(self.pword_path, 'r') as f:
+            d = json.load(f)
+        return key_stretch(password, d['salt']) == d['pword']
+
+    # asks the user for a new password and re-encrypts all the files with the
+    # new password. if repo is None no attempt is made to re-encrypt files
+    def change_password(self, repo=None):
+        while True:
+            p1 = getpass.getpass(prompt='Enter new password: ')
+            p2 = getpass.getpass(prompt='Re-enter new password: ')
+
+            if p1 != p2:
+                print('Entered passwords do not match, please try again')
+            else:
+                break
+
+        new_pword = p1
+        new_gpg = GPG(new_pword)
+
+        if repo is not None:
+            self.init_password()
+
+            for root, dirs, files in os.walk(repo):
+                for fname in files:
+                    fname = os.path.join(root, fname)
+                    logging.info(f'changing passphrase for '
+                                 f'{os.path.relpath(fname, repo)}')
+
+                    # make a secure temporary file
+                    fs, sfname = tempfile.mkstemp()
+                    # close the file-handle since we won't be using it (just
+                    # there for gpg to write to)
+                    os.close(fs)
+
+                    try:
+                        # decrypt with old passphrase and re-encrypt with new
+                        # passphrase
+                        self.gpg.decrypt(fname, sfname)
+                        new_gpg.encrypt(sfname, fname)
+                    except:  # noqa: E722
+                        raise
+                    finally:
+                        os.remove(sfname)
+
+        self.gpg = new_gpg
+        self.save_password(new_pword)
+        return new_pword
+
+    # gets the password from the user if needed
+    def init_password(self):
+        if self.gpg is not None:
+            return
+
+        if not os.path.exists(self.pword_path):
+            print('No encryption password was found for this repo. To '
+                  'continue please set an encryption password\n')
+            password = self.change_password()
+        else:
+            while True:
+                password = getpass.getpass(prompt='Encryption password: ')
+                if self.verify_password(password):
+                    break
+                print('Incorrect password entered, please try again')
+
+        self.gpg = GPG(password)
+
+    # encrypts a file from outside the repo and stores it inside the repo
+    def apply(self, source, dest):
+        self.init_password()
+        self.gpg.encrypt(source, dest)
+
+        # calculate and store file hash
+        self.hashes[self.strip_repo(dest)] = hash_file(source)
+        # store file mode data (metadata)
+        self.modes[self.strip_repo(dest)] = os.stat(source).st_mode & 0o777
+
+        self.save_data()
+
+    # decrypts source and saves it in dest
+    def remove(self, source, dest):
+        self.init_password()
+        self.gpg.decrypt(source, dest)
+        os.chmod(dest, self.modes[self.strip_repo(source)])
+
+    # compares the ext_file to repo_file and returns true if they are the same.
+    # does this by looking at the repo_file's hash and calculating the hash of
+    # the ext_file
+    def samefile(self, repo_file, ext_file):
+        ext_hash = hash_file(ext_file)
+        repo_file = self.strip_repo(repo_file)
+        return self.hashes.get(repo_file, None) == ext_hash
+
+    def strify(self, op):
+        if op == self.apply:
+            return "ENCRYPT"
+        elif op == self.remove:
+            return "DECRYPT"
+        else:
+            return ""

dotsync/plugins/plain.py

@@ -0,0 +1,48 @@
+import os
+import shutil
+import filecmp
+
+from dotsync.plugin import Plugin
+
+
+class PlainPlugin(Plugin):
+    def __init__(self, *args, **kwargs):
+        self.hard = kwargs.pop('hard', False)
+        super().__init__(*args, **kwargs)
+
+    def setup_data(self):
+        pass
+
+    # copies file from outside the repo to the repo
+    def apply(self, source, dest):
+        shutil.copy2(source, dest)
+
+    # if not in hard mode, creates a symlink in dest (outside the repo) that
+    # points to source (inside the repo)
+    # if in hard mode, copies the file from the repo to the dest.
+    def remove(self, source, dest):
+        if self.hard:
+            shutil.copy2(source, dest)
+        else:
+            os.symlink(source, dest)
+
+    # if not in hard mode, checks if symlink points to file in repo
+    # if in hard mode, a bit-by-bit comparison is made to compare the files
+    def samefile(self, repo_file, ext_file):
+        if self.hard:
+            if os.path.islink(ext_file):
+                return False
+            if not os.path.exists(repo_file):
+                return False
+            return filecmp.cmp(repo_file, ext_file, shallow=False)
+        else:
+            # not using os.samefile since it resolves repo_file as well which
+            # is not what we want
+            return os.path.realpath(ext_file) == os.path.abspath(repo_file)
+
+    def strify(self, op):
+        if op == self.apply:
+            return "COPY"
+        elif op == self.remove:
+            return "COPY" if self.hard else "LINK"
+        return ""