pyreposync 0.2.0__py3-none-any.whl

pyreposync/__init__.py

@@ -0,0 +1,511 @@
+import argparse
+import collections
+import configparser
+import datetime
+import logging
+import sys
+import threading
+import time
+
+from pyreposync.downloader import Downloader
+from pyreposync.exceptions import OSRepoSyncException, OSRepoSyncHashError
+from pyreposync.sync_rpm import SyncRPM
+from pyreposync.sync_deb import SyncDeb
+
+
+def main():
+    parser = argparse.ArgumentParser(description="OS Repo Sync Tool")
+
+    parser.add_argument(
+        "--cfg",
+        dest="cfg",
+        action="store",
+        default="/etc/pyreposync/reposync.ini",
+        help="Full path to configuration",
+    )
+
+    parser.add_argument(
+        "--repo",
+        dest="repo",
+        action="store",
+        default=None,
+        help="""
+                        execute command on this repository, if not set, 
+                        command applies to all configured repositories.
+                        
+                        This command is mutually exclusive with --tags
+                        """,
+    )
+
+    parser.add_argument(
+        "--tags",
+        dest="tags",
+        action="store",
+        default=None,
+        help="""
+                        Comma separated list of repo tags the command applies to.
+                        putting a '!' in front of a tag negates it.
+                        At least one not negated tag has to be match.
+                        
+                        This command is mutually exclusive with --repo
+                        """,
+    )
+
+    subparsers = parser.add_subparsers(
+        help="commands",
+        dest="method",
+    )
+    subparsers.required = True
+
+    migrate_parser = subparsers.add_parser(
+        "migrate",
+        help="migrate to new sync format",
+    )
+    migrate_parser.set_defaults(
+        method="migrate",
+    )
+
+    snap_cleanup_parser = subparsers.add_parser(
+        "snap_cleanup",
+        help="remove all unnamed snapshots and unreferenced rpms.",
+    )
+    snap_cleanup_parser.set_defaults(
+        method="snap_cleanup",
+    )
+
+    snap_list_parser = subparsers.add_parser(
+        "snap_list",
+        help="list snapshots",
+    )
+    snap_list_parser.set_defaults(
+        method="snap_list",
+    )
+
+    snap_name_parser = subparsers.add_parser(
+        "snap_name",
+        help="give timed snapshot a name",
+    )
+    snap_name_parser.set_defaults(
+        method="snap_name",
+    )
+    snap_name_parser.add_argument(
+        "--timestamp",
+        dest="timestamp",
+        action="store",
+        required=True,
+        default=None,
+        help="source timestampm might also be a named snapshot or latest",
+    )
+    snap_name_parser.add_argument(
+        "--name",
+        dest="snapname",
+        action="store",
+        required=True,
+        default=None,
+        help="name to be created",
+    )
+
+    snap_unname_parser = subparsers.add_parser(
+        "snap_unname",
+        help="remove name from timed snapshot",
+    )
+    snap_unname_parser.set_defaults(
+        method="snap_unname",
+    )
+    snap_unname_parser.add_argument(
+        "--name",
+        dest="snapname",
+        action="store",
+        required=True,
+        help="name to be removed",
+    )
+
+    snap_parser = subparsers.add_parser(
+        "snap",
+        help="create new snapshots",
+    )
+    snap_parser.set_defaults(
+        method="snap",
+    )
+
+    sync_parser = subparsers.add_parser(
+        "sync",
+        help="sync all repos",
+    )
+    sync_parser.set_defaults(
+        method="sync",
+    )
+
+    validate_parser = subparsers.add_parser(
+        "validate",
+        help="re validate package downloads",
+    )
+    validate_parser.set_defaults(
+        method="validate",
+    )
+
+    parsed_args = parser.parse_args()
+    try:
+        snapname = parsed_args.snapname
+    except AttributeError:
+        snapname = None
+    try:
+        timestamp = parsed_args.timestamp
+    except AttributeError:
+        timestamp = None
+
+    osreposync = PyRepoSync(
+        cfg=parsed_args.cfg,
+        method=parsed_args.method,
+        snapname=snapname,
+        repo=parsed_args.repo,
+        tags=parsed_args.tags,
+        timestamp=timestamp,
+    )
+    osreposync.work()
+
+
+class PyRepoSync:
+    def __init__(self, cfg, snapname, method, repo, tags, timestamp):
+        self._config_file = cfg
+        self._config = configparser.ConfigParser()
+        self._config_dict = None
+        self._method = method
+        self._snapname = snapname
+        self._repo = repo
+        self._tags = None
+        self._timestamp = timestamp
+        self.tags = tags
+        self.log = logging.getLogger("application")
+        self.config.read_file(open(self._config_file))
+        self._config_dict = self._cfg_to_dict(self.config)
+        self._logging()
+        if self._tags and self._repo:
+            self.log.fatal("both tags & repo have been specified, choose one")
+
+    @property
+    def method(self):
+        return self._method
+
+    @property
+    def snapname(self):
+        return self._snapname
+
+    @property
+    def repo(self):
+        return self._repo
+
+    @property
+    def tags(self):
+        return self._tags
+
+    @tags.setter
+    def tags(self, tags):
+        if tags:
+            self._tags = tags.split(",")
+
+    @property
+    def timestamp(self):
+        return self._timestamp
+
+    def _logging(self):
+        logfmt = logging.Formatter(
+            "%(asctime)sUTC - %(levelname)s - %(threadName)s - %(message)s"
+        )
+        logfmt.converter = time.gmtime
+        aap_level = self.config.get("main", "loglevel")
+        handler = logging.StreamHandler()
+
+        handler.setFormatter(logfmt)
+        self.log.addHandler(handler)
+        self.log.setLevel(aap_level)
+        self.log.debug("logger is up")
+
+    @staticmethod
+    def _cfg_to_dict(config):
+        result = {}
+        for section in config.sections():
+            result[section] = {}
+            for option in config.options(section):
+                try:
+                    result[section][option] = config.getint(section, option)
+                    continue
+                except ValueError:
+                    pass
+                try:
+                    result[section][option] = config.getfloat(section, option)
+                    continue
+                except ValueError:
+                    pass
+                try:
+                    result[section][option] = config.getboolean(section, option)
+                    continue
+                except ValueError:
+                    pass
+                try:
+                    result[section][option] = config.get(section, option)
+                    continue
+                except ValueError:
+                    pass
+        return result
+
+    @property
+    def config(self):
+        return self._config
+
+    @property
+    def config_dict(self):
+        return self._config_dict
+
+    def get_job(self, date, section):
+        self.log.info(f"section name: {section}")
+        if section.endswith(":rpm"):
+            return SyncRPM(
+                base_url=self.config.get(section, "baseurl"),
+                destination=self.config.get("main", "destination"),
+                reponame=section[:-4],
+                date=date,
+                treeinfo=self.config.get(section, "treeinfo", fallback=".treeinfo"),
+                proxy=self.config.get("main", "proxy", fallback=None),
+                client_cert=self.config.get(section, "sslclientcert", fallback=None),
+                client_key=self.config.get(section, "sslclientkey", fallback=None),
+                ca_cert=self.config.get(section, "sslcacert", fallback=None),
+            )
+        elif section.endswith(":deb"):
+            return SyncDeb(
+                base_url=self.config.get(section, "baseurl"),
+                destination=self.config.get("main", "destination"),
+                reponame=section[:-4],
+                date=date,
+                proxy=self.config.get(section, "proxy", fallback=None),
+                client_cert=self.config.get(section, "sslclientcert", fallback=None),
+                client_key=self.config.get(section, "sslclientkey", fallback=None),
+                ca_cert=self.config.get(section, "sslcacert", fallback=None),
+                suites=self.config.get(section, "suites").split(),
+                components=self.config.get(section, "components").split(),
+                binary_archs=self.config.get(section, "binary_archs").split(),
+            )
+
+    def get_sections(self):
+        sections = set()
+        for section in self.config:
+            if section.endswith(":rpm") or section.endswith(":deb"):
+                if self.repo and section != self.repo:
+                    continue
+                if self._tags:
+                    if not self.validate_tags(section):
+                        continue
+                sections.add(section)
+
+        return sections
+
+    def work(self):
+        self.log.info("starting up")
+        date = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
+        queue = collections.deque()
+        for section in self.get_sections():
+            queue.append(self.get_job(date=date, section=section))
+        workers = set()
+        if self.method == "sync":
+            num_worker = self.config.getint("main", "downloaders", fallback=1)
+        else:
+            num_worker = 1
+        for _ in range(num_worker):
+            workers.add(
+                RepoSyncThread(
+                    queue=queue,
+                    action=self.method,
+                    snapname=self.snapname,
+                    timestamp=self.timestamp,
+                )
+            )
+
+        for worker in workers:
+            worker.start()
+        return_code = 0
+        for worker in workers:
+            worker.join()
+            if worker.status != 0:
+                return_code = 1
+        sys.exit(return_code)
+
+    def validate_tags(self, section):
+        try:
+            section_tags = self.config.get(section, "tags").split(",")
+        except Exception as err:
+            return False
+        for tag in self.tags:
+            if tag.startswith("!"):
+                if tag[1:] in section_tags:
+                    return False
+            else:
+                if tag not in section_tags:
+                    return False
+        self.log.info(f"section {section} has matching tags")
+        return True
+
+
+class RepoSyncThread(threading.Thread):
+    def __init__(self, queue, action, snapname, timestamp):
+        super().__init__()
+        self._action = action
+        self._snapname = snapname
+        self._queue = queue
+        self._status = 0
+        self._timestamp = timestamp
+        self.daemon = True
+        self.log = logging.getLogger("application")
+
+    @property
+    def action(self):
+        return self._action
+
+    @property
+    def snapname(self):
+        return self._snapname
+
+    @property
+    def queue(self):
+        return self._queue
+
+    @property
+    def status(self):
+        return self._status
+
+    @status.setter
+    def status(self, value):
+        self._status = value
+
+    @property
+    def timestamp(self):
+        return self._timestamp
+
+    def do_migrate(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.migrate()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_sync(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.sync()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_cleanup(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_cleanup()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_list(self, job):
+        try:
+            self.name = job.reponame
+            referenced_timestamps = job.snap_list_get_referenced_timestamps()
+            self.log.info(f"Repository: {job.reponame}")
+            self.log.info("The following timestamp snapshots exist:")
+            for timestamp in job.snap_list_timestamp_snapshots():
+                self.log.info(
+                    f"{timestamp} -> {referenced_timestamps.get(timestamp, [])}"
+                )
+            self.log.info("The following named snapshots exist:")
+            base = f"{job.destination}/snap/{job.reponame}/"
+            for named in job.snap_list_named_snapshots():
+                timestamp = job.snap_list_named_snapshot_target(f"{base}/named/{named}")
+                self.log.info(f"named/{named} -> {timestamp}")
+            latest = f"{base}/latest"
+            self.log.info(f"latest -> {job.snap_list_named_snapshot_target(latest)}")
+
+        except OSRepoSyncException:
+            self.status = 1
+
+    def do_snap_name(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_name(self.timestamp, self.snapname)
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_unname(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_unname(self.snapname)
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_validate(self, job):
+        _downloader = Downloader()
+        packages = dict()
+        try:
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            packages.update(job.revalidate())
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+        for destination, hash_info in packages.items():
+            try:
+                self.log.info(f"validating: {destination}")
+                _downloader.check_hash(
+                    destination=destination,
+                    checksum=hash_info["hash_sum"],
+                    hash_type=hash_info["hash_algo"],
+                )
+            except OSRepoSyncHashError:
+                self.log.error(f"hash mismatch for: {destination}")
+            except FileNotFoundError:
+                self.log.error(f"file not found: {destination}")
+
+    def run(self):
+        while True:
+            try:
+                job = self.queue.pop()
+                if self.action == "migrate":
+                    self.do_migrate(job)
+                elif self.action == "sync":
+                    self.do_sync(job)
+                elif self.action == "snap_cleanup":
+                    self.do_snap_cleanup(job)
+                elif self.action == "snap_list":
+                    self.do_snap_list(job)
+                elif self.action == "snap_name":
+                    self.do_snap_name(job)
+                elif self.action == "snap_unname":
+                    self.do_snap_unname(job)
+                elif self.action == "snap":
+                    self.do_snap(job)
+                elif self.action == "validate":
+                    self.do_validate(job)
+            except IndexError:
+                break
+
+
+if __name__ == "__main__":
+    main()

pyreposync/downloader.py

@@ -0,0 +1,143 @@
+import hashlib
+import logging
+import os
+import requests
+import requests.exceptions
+import shutil
+import tempfile
+import time
+
+from pyreposync.exceptions import OSRepoSyncDownLoadError, OSRepoSyncHashError
+
+
+class Downloader(object):
+    def __init__(self, proxy=None, client_cert=None, client_key=None, ca_cert=None):
+        self.log = logging.getLogger("application")
+        if proxy:
+            self._proxy = {"http": proxy, "https": proxy}
+        else:
+            self._proxy = None
+        if client_cert and client_key:
+            self._cert = (client_cert, client_key)
+        else:
+            self._cert = None
+        if ca_cert:
+            self._ca_cert = ca_cert
+        else:
+            self._ca_cert = True
+
+    @property
+    def ca_cert(self):
+        return self._ca_cert
+
+    @property
+    def cert(self):
+        return self._cert
+
+    @property
+    def proxy(self):
+        return self._proxy
+
+    def check_hash(self, destination, checksum, hash_type):
+        self.log.debug("validating hash")
+        hasher = None
+        if hash_type == "md5":
+            hasher = hashlib.md5()
+        elif hash_type == "sha":
+            hasher = hashlib.sha1()
+        elif hash_type == "sha1":
+            hasher = hashlib.sha1()
+        elif hash_type == "sha256":
+            hasher = hashlib.sha256()
+        elif hash_type == "sha512":
+            hasher = hashlib.sha512()
+
+        with open(destination, "rb") as dest:
+            hasher.update(dest.read())
+            self.log.debug(f"expected hash: {hasher.hexdigest()}")
+            self.log.debug(f"actual hash: {checksum}")
+            if hasher.hexdigest() == checksum:
+                self.log.debug(f"download valid: {destination}")
+            else:
+                self.log.error(f"download invalid: {destination}")
+                raise OSRepoSyncHashError(f"download invalid: {destination}")
+
+    def get(
+        self,
+        url,
+        destination,
+        checksum=None,
+        hash_type=None,
+        replace=False,
+        not_found_ok=False,
+    ):
+        self.log.info(f"downloading: {url}")
+        if not replace:
+            if os.path.isfile(destination):
+                self.log.info("already there, not downloading")
+                return
+        retries = 10
+        while retries >= 0:
+            try:
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    tmp_file = os.path.join(tmp_dir, os.path.basename(destination))
+                    self._get(url, tmp_file, checksum, hash_type, not_found_ok)
+                    self.create_dir(destination)
+                    try:
+                        shutil.move(tmp_file, destination)
+                    except OSError:
+                        if not_found_ok:
+                            pass
+                        else:
+                            raise
+                self.log.info(f"done downloading: {url}")
+                return
+            except requests.exceptions.ConnectionError:
+                self.log.error("could not fetch resource, retry in 10 seconds")
+                retries -= 1
+                time.sleep(10)
+            except OSRepoSyncHashError:
+                self.log.error("download invalid, retry in 10 seconds")
+                retries -= 1
+                time.sleep(10)
+            except OSRepoSyncDownLoadError:
+                break
+        self.log.error(f"could not download: {url}")
+        raise OSRepoSyncDownLoadError(f"could not download: {url}")
+
+    def create_dir(self, destination):
+        if not os.path.isdir(os.path.dirname(destination)):
+            try:
+                os.makedirs(os.path.dirname(destination))
+            except OSError as err:
+                self.log.error(f"could not create directory: {err}")
+                raise OSRepoSyncDownLoadError(f"could not create directory: {err}")
+
+    def _get(
+        self,
+        url,
+        destination,
+        checksum=None,
+        hash_type=None,
+        not_found_ok=False,
+    ):
+        self.create_dir(destination)
+        r = requests.get(
+            url, stream=True, proxies=self.proxy, cert=self.cert, verify=self.ca_cert
+        )
+        if r.status_code == 200:
+            with open(destination, "wb", 0) as dst:
+                r.raw.decode_content = True
+                shutil.copyfileobj(r.raw, dst)
+                dst.flush()
+        else:
+            if not_found_ok:
+                if r.status_code == 404:
+                    self.log.info("not found, skipping")
+                    return
+            raise OSRepoSyncDownLoadError()
+        if checksum:
+            self.check_hash(
+                destination=destination, checksum=checksum, hash_type=hash_type
+            )
+        self.log.info(f"successfully fetched: {url}")

pyreposync/exceptions.py

@@ -0,0 +1,10 @@
+class OSRepoSyncException(Exception):
+    pass
+
+
+class OSRepoSyncDownLoadError(OSRepoSyncException):
+    pass
+
+
+class OSRepoSyncHashError(OSRepoSyncException):
+    pass

pyreposync/sync_deb.py

@@ -0,0 +1,218 @@
+from shutil import copyfile
+
+import gzip
+import os
+
+from pyreposync.sync_generic import SyncGeneric
+
+
+class SyncDeb(SyncGeneric):
+    def __init__(
+        self,
+        base_url,
+        destination,
+        reponame,
+        date,
+        suites,
+        components,
+        binary_archs,
+        proxy=None,
+        client_cert=None,
+        client_key=None,
+        ca_cert=None,
+    ):
+        super().__init__(
+            base_url,
+            destination,
+            reponame,
+            date,
+            proxy,
+            client_cert,
+            client_key,
+            ca_cert,
+        )
+        self._binary_archs = binary_archs
+        self._components = components
+        self._destination = destination
+        self._suites = suites
+
+    @property
+    def binary_archs(self) -> list[str]:
+        return self._binary_archs
+
+    @property
+    def suites(self) -> list[str]:
+        return self._suites
+
+    @property
+    def components(self) -> list[str]:
+        return self._components
+
+    def _snap(self):
+        for suite in self.suites:
+            self.snap_suites(suite=suite)
+
+    def snap_suites(self, suite):
+        self.log.info(f"creating snapshot for suite {suite}")
+        self.snap_release(suite=suite)
+        self.snap_release_files(suite=suite)
+        for arch in self.binary_archs:
+            self.snap_package_binary_files(suite=suite, arch=arch)
+        self.log.info(f"creating snapshot for suite {suite}, done")
+
+    def snap_release(self, suite):
+        self.log.info(f"creating snapshot for suite {suite} release files")
+        release_files = ["InRelease", "Release", "Release.gpg"]
+        src_path = f"{self.destination}/sync/{self.reponame}/dists/{suite}"
+        dst_path = f"{self.destination}/snap/{self.reponame}/{self.date}/dists/{suite}"
+        try:
+            os.makedirs(dst_path)
+        except OSError:
+            pass
+        for release_file in release_files:
+            src = f"{src_path}/{release_file}"
+            dst = f"{dst_path}/{release_file}"
+            copyfile(src, dst)
+        self.log.info(f"creating snapshot for suite {suite} release files, done")
+
+    def snap_release_files(self, suite):
+        self.log.info(f"creating snapshot for suite {suite} release files")
+        release_files = self.release_files_sha256(suite=suite)
+        src_path = f"{self.destination}/sync/{self.reponame}/dists/{suite}"
+        dst_path = f"{self.destination}/snap/{self.reponame}/{self.date}/dists/{suite}"
+        for filename, sha256_dict in release_files.items():
+            src = f"{src_path}/{filename}"
+            dst = f"{dst_path}/{filename}"
+            try:
+                os.makedirs(os.path.dirname(dst))
+            except OSError:
+                pass
+            try:
+                copyfile(src, dst)
+            except FileNotFoundError:
+                pass
+        self.log.info(f"creating snapshot for suite {suite} release files, done")
+
+    def snap_package_binary_files(self, suite, arch):
+        self.log.info(
+            f"creating snapshot for suite {suite} arch {arch} package binary files"
+        )
+        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        src_path = f"{self.destination}/sync/{self.reponame}"
+        dst_path = f"{self.destination}/snap/{self.reponame}/{self.date}"
+        for filename, sha256_dict in packages.items():
+            src = f"{src_path}/{filename}.sha256.{sha256_dict['sha256']}"
+            dst = f"{dst_path}/{filename}"
+            try:
+                os.makedirs(os.path.dirname(dst))
+            except OSError:
+                pass
+            try:
+                os.symlink(src, dst)
+            except FileExistsError:
+                pass
+
+    def sync(self):
+        self.log.info("starting thread")
+        for suite in self.suites:
+            self.sync_suites(suite=suite)
+        self.log.info("shutdown thread complete")
+
+    def sync_suites(self, suite):
+        self.log.info(f"syncing suite {suite}")
+        self.sync_release(suite=suite)
+        self.sync_release_files(suite=suite)
+        for arch in self.binary_archs:
+            self.sync_package_binary_files(suite=suite, arch=arch)
+        self.log.info(f"syncing suite {suite}, done")
+
+    def sync_release(self, suite):
+        self.log.info(f"syncing suite {suite} release files")
+        release_files = ["InRelease", "Release", "Release.gpg"]
+        base_path = f"{self.destination}/sync/{self.reponame}/dists/{suite}"
+        base_url = f"{self.base_url}/dists/{suite}"
+        self.log.info(base_url)
+        for release_file in release_files:
+            self.downloader.get(
+                url=f"{base_url}/{release_file}",
+                destination=f"{base_path}/{release_file}",
+                replace=True,
+            )
+        self.log.info(f"syncing suite {suite} release files, done")
+
+    def sync_package_binary_files(self, suite, arch):
+        self.log.info(f"syncing suite {suite} arch {arch} package binary files")
+        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        base_path = f"{self.destination}/sync/{self.reponame}"
+        base_url = f"{self.base_url}"
+        for filename, sha256_dict in packages.items():
+            self.downloader.get(
+                url=f"{base_url}/{filename}",
+                destination=f"{base_path}/{filename}.sha256.{sha256_dict['sha256']}",
+                checksum=sha256_dict["sha256"],
+                hash_type="sha256",
+            )
+
+        self.log.info(f"syncing suite {suite} arch {arch} package binary files, done")
+
+    def binary_files_sha256(self, suite, component, arch):
+        packages_gz_file = f"{self.destination}/sync/{self.reponame}/dists/{suite}/{component}/binary-{arch}/Packages.gz"
+        packages = dict()
+        sha256 = None
+        filename = None
+        size = None
+        with gzip.open(packages_gz_file, "rb") as source:
+            for line in source:
+                line = line.decode("utf-8")
+                if line.startswith("SHA256: "):
+                    sha256 = line.split("SHA256: ")[1].strip()
+                elif line.startswith("Filename: "):
+                    filename = line.split("Filename: ")[1].strip()
+                elif line.startswith("Size: "):
+                    size = int(line.split("Size: ")[1].strip())
+                if filename and sha256 and size:
+                    packages[filename] = {
+                        "sha256": sha256,
+                        "size": size,
+                    }
+                    sha256 = None
+                    filename = None
+                    size = None
+        return packages
+
+    def sync_release_files(self, suite):
+        self.log.info(f"syncing suite {suite} release files")
+        release_files = self.release_files_sha256(suite=suite)
+        base_path = f"{self.destination}/sync/{self.reponame}/dists/{suite}"
+        base_url = f"{self.base_url}/dists/{suite}"
+        for filename, sha256_dict in release_files.items():
+            self.downloader.get(
+                url=f"{base_url}/{filename}",
+                destination=f"{base_path}/{filename}",
+                checksum=sha256_dict["sha256"],
+                hash_type="sha256",
+                replace=True,
+                not_found_ok=True,
+            )
+        self.log.info(f"syncing suite {suite} release files, done")
+
+    def release_files_sha256(self, suite):
+        release = f"{self.destination}/sync/{self.reponame}/dists/{suite}/Release"
+        with open(release, "r") as release_file:
+            release_file_content = release_file.read()
+        sha256_dict = {}
+        in_sha256_section = False
+        for line in release_file_content.splitlines():
+            if line.startswith("SHA256:"):
+                in_sha256_section = True
+                continue
+            if in_sha256_section:
+                if line.startswith(" "):
+                    sha256, size, filename = line.split()
+                    sha256_dict[filename] = {
+                        "sha256": sha256,
+                        "size": int(size),
+                    }
+                else:
+                    break
+        return sha256_dict

pyreposync/sync_generic.py

@@ -0,0 +1,169 @@
+import logging
+import os
+import shutil
+
+from pyreposync.downloader import Downloader
+from pyreposync.exceptions import OSRepoSyncException
+
+
+class SyncGeneric:
+    def __init__(
+        self,
+        base_url,
+        destination,
+        reponame,
+        date,
+        proxy=None,
+        client_cert=None,
+        client_key=None,
+        ca_cert=None,
+    ):
+        self._base_url = base_url
+        self._date = date
+        self._destination = destination
+        self._reponame = reponame
+        self.downloader = Downloader(
+            proxy=proxy, client_cert=client_cert, client_key=client_key, ca_cert=ca_cert
+        )
+        self.log = logging.getLogger("application")
+
+    @property
+    def base_url(self):
+        return self._base_url
+
+    @property
+    def date(self):
+        return self._date
+
+    @property
+    def destination(self):
+        return self._destination
+
+    @property
+    def reponame(self):
+        return self._reponame
+
+    def migrate(self):
+        pass
+
+    def revalidate(self):
+        pass
+
+    def _snap(self):
+        raise NotImplementedError("this method must be implemented by subclasses")
+
+    def snap(self):
+        self.log.info("creating snapshot")
+        self._snap()
+        current = f"{self.destination}/snap/{self.reponame}/{self.date}"
+        latest = f"{self.destination}/snap/{self.reponame}/latest"
+        timestamp = f"{self.destination}/snap/{self.reponame}/{self.date}/timestamp"
+        self.log.info("setting latest to current release")
+        try:
+            os.unlink(latest)
+        except FileNotFoundError:
+            pass
+        os.symlink(current, latest)
+        with open(timestamp, "w") as _timestamp:
+            _timestamp.write(f"{self.destination}\n")
+        self.log.info("done creating snapshot")
+
+    def snap_cleanup(self):
+        referenced_timestamps = self.snap_list_get_referenced_timestamps()
+        for snap in self.snap_list_timestamp_snapshots():
+            if snap not in referenced_timestamps:
+                snap = f"{self.destination}/snap/{self.reponame}/{snap}"
+                shutil.rmtree(snap)
+
+    def snap_list_get_referenced_timestamps(self):
+        result = dict()
+        base = f"{self.destination}/snap/{self.reponame}/"
+        for candidate in self.snap_list_named_snapshots():
+            candidate = f"named/{candidate}"
+            timestamp = self.snap_list_named_snapshot_target(f"{base}/{candidate}")
+            if timestamp not in result:
+                result[timestamp] = [candidate]
+            else:
+                result[timestamp].append(candidate)
+        timestamp = self.snap_list_named_snapshot_target(f"{base}/latest")
+        if timestamp not in result:
+            result[timestamp] = ["latest"]
+        else:
+            result[timestamp].append("latest")
+        return result
+
+    def snap_list_named_snapshots(self):
+        try:
+            return os.listdir(f"{self.destination}/snap/{self.reponame}/named")
+        except FileNotFoundError:
+            return []
+
+    @staticmethod
+    def snap_list_named_snapshot_target(path):
+        try:
+            return os.readlink(path).split("/")[-1]
+        except FileNotFoundError:
+            return None
+
+    def snap_list_timestamp_snapshots(self):
+        try:
+            result = os.listdir(f"{self.destination}/snap/{self.reponame}/")
+            try:
+                result.remove("latest")
+            except ValueError:
+                pass
+            try:
+                result.remove("named")
+            except ValueError:
+                pass
+            return result
+        except FileNotFoundError:
+            return []
+
+    def snap_name(self, timestamp, snapname):
+        self.log.info("creating named snapshot")
+        try:
+            int(timestamp)
+            if not len(timestamp) == 14:
+                raise ValueError
+        except ValueError:
+            self.log.error(
+                f"{timestamp} is not a valid timestamp, checking if its a named snapshot"
+            )
+            source = f"{self.destination}/snap/{self.reponame}/{timestamp}"
+            _timestamp = self.snap_list_named_snapshot_target(source)
+            if _timestamp:
+                self.log.info(f"setting timestamp to {_timestamp}")
+                timestamp = _timestamp
+            else:
+                raise OSRepoSyncException(f"{snapname} is not a valid named snapshot")
+        source = f"{self.destination}/snap/{self.reponame}/{timestamp}"
+        target = f"{self.destination}/snap/{self.reponame}/named/{snapname}"
+        target_dir = f"{self.destination}/snap/{self.reponame}/named/"
+        if os.path.isdir(source):
+            self.log.debug(f"source directory exists: {source}")
+        else:
+            self.log.debug(f"source directory missing: {source}")
+            raise OSRepoSyncException(f"Source directory missing: {source}")
+        try:
+            os.makedirs(os.path.dirname(target_dir))
+        except OSError:
+            pass
+        try:
+            os.unlink(target)
+        except OSError:
+            pass
+        os.symlink(source, target)
+        self.log.info("done creating named snapshot")
+
+    def snap_unname(self, snapname):
+        self.log.info("removing named snapshot")
+        target = f"{self.destination}/snap/{self.reponame}/named/{snapname}"
+        try:
+            os.unlink(target)
+        except FileNotFoundError:
+            pass
+        self.log.info("done removing named snapshot")
+
+    def sync(self):
+        pass

pyreposync/sync_rpm.py

@@ -0,0 +1,256 @@
+from shutil import copyfile
+
+import bz2
+import gzip
+import configparser
+import os
+import shutil
+
+import xml.etree.ElementTree
+
+from pyreposync.sync_generic import SyncGeneric
+
+from pyreposync.exceptions import OSRepoSyncException
+
+
+class SyncRPM(SyncGeneric):
+    def __init__(
+        self,
+        base_url,
+        destination,
+        reponame,
+        date,
+        treeinfo,
+        proxy=None,
+        client_cert=None,
+        client_key=None,
+        ca_cert=None,
+    ):
+        super().__init__(
+            base_url,
+            destination,
+            reponame,
+            date,
+            proxy,
+            client_cert,
+            client_key,
+            ca_cert,
+        )
+        self._treeinfo = treeinfo
+
+    @property
+    def treeinfo(self):
+        return self._treeinfo
+
+    def packages(self, base_path=None):
+        if not base_path:
+            base_path = f"{self.destination}/sync/{self.reponame}"
+        primary = None
+        for location, hash_algo, hash_sum in self.repomd_files():
+            destination = f"{base_path}/{location}"
+            if "primary.xml" in destination.lower():
+                primary = destination
+        if not primary:
+            self.log.fatal("no primary.xml found in repomd.xml")
+            raise OSRepoSyncException("no primary.xml found in repomd.xml")
+
+        if primary.endswith(".gz"):
+            with gzip.open(primary, "rb") as source:
+                root = xml.etree.ElementTree.parse(source).getroot()
+        elif primary.endswith("bz2"):
+            with bz2.open(primary, "rb") as source:
+                root = xml.etree.ElementTree.parse(source).getroot()
+        else:
+            with open(primary, "rb") as source:
+                root = xml.etree.ElementTree.parse(source).getroot()
+        packages = root.findall("{http://linux.duke.edu/metadata/common}package")
+        for package in packages:
+            checksum = package.find("{http://linux.duke.edu/metadata/common}checksum")
+            hash_algo = checksum.get("type")
+            hash_sum = checksum.text
+            location = package.find("{http://linux.duke.edu/metadata/common}location")
+            yield location.get("href"), hash_algo, hash_sum
+
+    def migrate(self):
+        migrated_file = f"{self.destination}/sync/{self.reponame}/migrated"
+        if os.path.isfile(migrated_file):
+            self.log.info("migration already done")
+            return
+
+        for location, hash_algo, hash_sum in self.packages():
+            destination_old = f"{self.destination}/sync/{self.reponame}/{location}"
+            destination_new = f"{self.destination}/sync/{self.reponame}/{location}.{hash_algo}.{hash_sum}"
+            try:
+                os.rename(destination_old, destination_new)
+            except FileNotFoundError:
+                self.log.error(
+                    f"could not migrate {location}: {destination_old} not found"
+                )
+                continue
+            except OSError as err:
+                self.log.error(f"could not migrate {location}: {err}")
+                continue
+
+        for snap in self.snap_list_timestamp_snapshots():
+            self.log.info(f"migrating {snap}")
+            base_path = f"{self.destination}/snap/{self.reponame}/{snap}"
+            for location, hash_algo, hash_sum in self.packages(base_path=base_path):
+                dst = f"{base_path}/{location}"
+                src = f"{self.destination}/sync/{self.reponame}/{location}.{hash_algo}.{hash_sum}"
+                try:
+                    os.unlink(dst)
+                    os.symlink(src, dst)
+                except OSError:
+                    pass
+                try:
+                    os.symlink(src, dst)
+                except OSError:
+                    pass
+
+        with open(migrated_file, "w") as _migrated:
+            _migrated.write("migrated\n")
+
+    def sync_packages(self):
+        for location, hash_algo, hash_sum in self.packages():
+            url = f"{self.base_url}{location}"
+            destination = f"{self.destination}/sync/{self.reponame}/{location}.{hash_algo}.{hash_sum}"
+            self.downloader.get(url, destination, hash_sum, hash_algo, replace=False)
+
+    def treeinfo_files(self):
+        treeinfo_file = f"{self.destination}/sync/{self.reponame}/{self.treeinfo}"
+        treeinfo = configparser.ConfigParser()
+        treeinfo.optionxform = str
+        try:
+            treeinfo.read_file(open(treeinfo_file))
+        except FileNotFoundError:
+            return
+        try:
+            for file in treeinfo.options("checksums"):
+                if file == "repodata/repomd.xml":
+                    continue
+                hash_algo, hash_sum = treeinfo.get("checksums", file).split(":", 1)
+                yield file, hash_algo, hash_sum
+        except configparser.NoSectionError:
+            files = set()
+            for section in treeinfo.sections():
+                if section.startswith("images-") or section.startswith("stage2"):
+                    for option in treeinfo.options(section):
+                        files.add(treeinfo.get(section, option))
+            for file in files:
+                yield file, None, None
+
+    def sync_treeinfo(self):
+        url = f"{self.base_url}{self.treeinfo}"
+        destination = f"{self.destination}/sync/{self.reponame}/{self.treeinfo}"
+        try:
+            self.downloader.get(url, destination, replace=True)
+        except OSRepoSyncException:
+            return
+        for file, hash_algo, hash_sum in self.treeinfo_files():
+            if file == "repodata/repomd.xml":
+                continue
+            url = f"{self.base_url}{file}"
+            destination = f"{self.destination}/sync/{self.reponame}/{file}"
+            self.downloader.get(url, destination, hash_sum, hash_algo, replace=True)
+
+    def repomd_files(self):
+        base_path = f"{self.destination}/sync/{self.reponame}/repodata/repomd.xml"
+        repomd = xml.etree.ElementTree.parse(base_path).getroot()
+        datas = repomd.findall("{http://linux.duke.edu/metadata/repo}data")
+        for data in datas:
+            checksum = data.find("{http://linux.duke.edu/metadata/repo}checksum")
+            hash_algo = checksum.get("type")
+            hash_sum = checksum.text
+            location = data.find("{http://linux.duke.edu/metadata/repo}location")
+            yield location.get("href"), hash_algo, hash_sum
+
+    def revalidate(self):
+        packages = dict()
+        try:
+            for location, hash_algo, hash_sum in self.packages():
+                destination = f"{self.destination}/sync/{self.reponame}/{location}.{hash_algo}.{hash_sum}"
+                packages[destination] = {"hash_algo": hash_algo, "hash_sum": hash_sum}
+        except FileNotFoundError:
+            self.log.error("no repodata found")
+        return packages
+
+    def sync_repomd(self):
+        url = f"{self.base_url}repodata/repomd.xml"
+        destination = f"{self.destination}/sync/{self.reponame}/repodata/repomd.xml"
+        try:
+            shutil.rmtree(f"{self.destination}/sync/{self.reponame}/repodata/")
+        except FileNotFoundError:
+            pass
+        self.downloader.get(url, destination, replace=True)
+        for location, hash_algo, hash_sum in self.repomd_files():
+            url = f"{self.base_url}{location}"
+            destination = f"{self.destination}/sync/{self.reponame}/{location}"
+            self.downloader.get(url, destination, hash_sum, hash_algo, replace=True)
+        self.sync_packages()
+        self.sync_treeinfo()
+
+    def _snap(self):
+        self.snap_repodata()
+        self.snap_treeinfo()
+        self.snap_packages()
+
+    def snap_repodata(self):
+        self.log.info("copy repodata")
+        repomd_dst = (
+            f"{self.destination}/snap/{self.reponame}/{self.date}/repodata/repomd.xml"
+        )
+        repomd_src = f"{self.destination}/sync/{self.reponame}/repodata/repomd.xml"
+        try:
+            os.makedirs(os.path.dirname(repomd_dst))
+        except OSError:
+            pass
+        copyfile(repomd_src, repomd_dst)
+        for location, hash_algo, hash_sum in self.repomd_files():
+            dst = f"{self.destination}/snap/{self.reponame}/{self.date}/{location}"
+            src = f"{self.destination}/sync/{self.reponame}/{location}"
+            try:
+                os.makedirs(os.path.dirname(dst))
+            except OSError:
+                pass
+            copyfile(src, dst)
+        self.log.info("done copy repodata")
+
+    def snap_treeinfo(self):
+        self.log.info("copy treeinfo")
+        try:
+            dst = f"{self.destination}/snap/{self.reponame}/{self.destination}/{self.treeinfo}"
+            src = f"{self.destination}/sync/{self.reponame}/{self.treeinfo}"
+            copyfile(src, dst)
+        except (OSError, FileNotFoundError) as err:
+            self.log.error(f"could not copy {self.treeinfo}: {err}")
+        for location, hash_algo, hash_sum in self.treeinfo_files():
+            dst = (
+                f"{self.destination}/snap/{self.reponame}/{self.destination}/{location}"
+            )
+            src = f"{self.destination}/sync/{self.reponame}/{location}"
+            try:
+                os.makedirs(os.path.dirname(dst))
+            except OSError:
+                pass
+            copyfile(src, dst)
+        self.log.info("done copy treeinfo")
+
+    def snap_packages(self):
+        self.log.info("copy packages")
+        for location, hash_algo, hash_sum in self.packages():
+            dst = f"{self.destination}/snap/{self.reponame}/{self.date}/{location}"
+            src = f"{self.destination}/sync/{self.reponame}/{location}.{hash_algo}.{hash_sum}"
+            try:
+                os.makedirs(os.path.dirname(dst))
+            except OSError:
+                pass
+            try:
+                os.symlink(src, dst)
+            except FileExistsError as err:
+                self.log.error(f"could not copy {location}: {err}")
+        self.log.info("done copy packages")
+
+    def sync(self):
+        self.log.info("starting thread")
+        self.sync_repomd()
+        self.log.info("shutdown thread complete")

pyreposync-0.2.0.dist-info/METADATA

@@ -0,0 +1,31 @@
+Metadata-Version: 2.4
+Name: pyreposync
+Version: 0.2.0
+Summary: Orbit Api
+Project-URL: Source, https://github.com/schlitzered/pyreposync
+Author-email: "Stephan.Schultchen" <sschultchen@gmail.com>
+License: The MIT License (MIT)
+        
+        Copyright (c) 2015 Stephan Schultchen
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in
+        all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+        THE SOFTWARE.
+License-File: LICENSE.txt
+Classifier: Programming Language :: Python
+Requires-Python: >=3.9
+Requires-Dist: requests

pyreposync-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+pyreposync/__init__.py,sha256=GdNdSibEs53cwekuUQI9SqM0gXSgO7kDQgWU0o-vWNU,16258
+pyreposync/downloader.py,sha256=828bIMCctd1X6I1w91-XHytQ1M5fmxkEw0YDKd5nvAY,4764
+pyreposync/exceptions.py,sha256=IlnvhNaffQQ6geOgrjCciNFVbFpNcycH4ijSuMTbrGA,169
+pyreposync/sync_deb.py,sha256=taH4Tf0i6alSR0Z_iJ21LfiAWZelcU74DnYCQ6EXwhs,8201
+pyreposync/sync_generic.py,sha256=vNNuh-hJp3Qj_AqrPc7S6OibHhsFKSFRaTv_2Aj9H9Y,5502
+pyreposync/sync_rpm.py,sha256=3aBKwO_wdyrOXR_3MZlup26WoMTG8p1qdiN165YigrA,10105
+pyreposync-0.2.0.dist-info/METADATA,sha256=0sRFsPPiLmjuLvljPnmCxVJXdwmVbUZwKuKHlSOkBXw,1565
+pyreposync-0.2.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pyreposync-0.2.0.dist-info/entry_points.txt,sha256=9LsBDWOF3O6_3ONP3Lc-4v1MTt5ay0Xv-TMcFbOIt2s,47
+pyreposync-0.2.0.dist-info/licenses/LICENSE.txt,sha256=lwnJoIo7uwc0h6y6gC_RYqJkvjplViV3Ad6u7pQM4Bw,1084
+pyreposync-0.2.0.dist-info/RECORD,,

pyreposync-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pyreposync-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pyreposync = pyreposync:main

pyreposync-0.2.0.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Stephan Schultchen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.