pyreposync 0.1.0__tar.gz → 0.2.0__tar.gz

{pyreposync-0.1.0 → pyreposync-0.2.0}/.gitignore

@@ -1,6 +1,9 @@
 .idea
 *__pycache__*
 pyreposync.egg-info
+.eggs
+_data
+deb_mirror
 dist
 venv
 *xml

{pyreposync-0.1.0 → pyreposync-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyreposync
-Version: 0.1.0
+Version: 0.2.0
 Summary: Orbit Api
 Project-URL: Source, https://github.com/schlitzered/pyreposync
 Author-email: "Stephan.Schultchen" <sschultchen@gmail.com>

pyreposync-0.2.0/contrib/reposync_2.ini

@@ -0,0 +1,15 @@
+[main]
+destination = /home/schlitzer/PycharmProjects/pyreposync/_data
+downloaders = 10
+log = /home/schlitzer/PycharmProjects/pyreposync/_data/pyreposync.log
+logretention = 7
+loglevel = INFO
+#proxy = http://proxy.example.com:3128
+
+[AlmaLinux/9/AppStream/x86_64:rpm]
+baseurl = https://mirror.23m.com/almalinux/9/AppStream/x86_64/os/
+tags = EXTERNAL,OS:AlmaLinux9,BASE,X86_64
+
+[AlmaLinux/9/BaseOS/x86_64:rpm]
+baseurl = https://mirror.23m.com/almalinux/9/BaseOS/x86_64/os/
+tags = EXTERNAL,OS:AlmaLinux9,BASE,X86_64

{pyreposync-0.1.0 → pyreposync-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pyreposync"
-version = "0.1.0"
+version = "0.2.0"
 requires-python = ">=3.9"
 authors = [
   {name = "Stephan.Schultchen", email = "sschultchen@gmail.com"},

{pyreposync-0.1.0 → pyreposync-0.2.0}/pyreposync/__init__.py

@@ -9,7 +9,8 @@ import time
 
 from pyreposync.downloader import Downloader
 from pyreposync.exceptions import OSRepoSyncException, OSRepoSyncHashError
-from pyreposync.rpm_sync import RPMSync
+from pyreposync.sync_rpm import SyncRPM
+from pyreposync.sync_deb import SyncDeb
 
 
 def main():
@@ -56,6 +57,14 @@ def main():
     )
     subparsers.required = True
 
+    migrate_parser = subparsers.add_parser(
+        "migrate",
+        help="migrate to new sync format",
+    )
+    migrate_parser.set_defaults(
+        method="migrate",
+    )
+
     snap_cleanup_parser = subparsers.add_parser(
         "snap_cleanup",
         help="remove all unnamed snapshots and unreferenced rpms.",
@@ -248,34 +257,13 @@ class PyRepoSync:
     def config_dict(self):
         return self._config_dict
 
-    def get_jobs(self, date, section):
-        versions = self.config.get(section, "versions", fallback=None)
-        jobs = set()
-        if versions:
-            for version in versions.split(","):
-                job = RPMSync(
-                    base_url=self.config.get(section, "baseurl").replace(
-                        ":VERSION:", version
-                    ),
-                    destination=self.config.get("main", "destination"),
-                    reponame=section[:-4].replace(":VERSION:", version),
-                    syncdir=self.config.get(section, "syncdir", fallback=None),
-                    date=date,
-                    treeinfo=self.config.get(section, "treeinfo", fallback=".treeinfo"),
-                    proxy=self.config.get("main", "proxy", fallback=None),
-                    client_cert=self.config.get(
-                        section, "sslclientcert", fallback=None
-                    ),
-                    client_key=self.config.get(section, "sslclientkey", fallback=None),
-                    ca_cert=self.config.get(section, "sslcacert", fallback=None),
-                )
-                jobs.add(job)
-        else:
-            job = RPMSync(
+    def get_job(self, date, section):
+        self.log.info(f"section name: {section}")
+        if section.endswith(":rpm"):
+            return SyncRPM(
                 base_url=self.config.get(section, "baseurl"),
                 destination=self.config.get("main", "destination"),
                 reponame=section[:-4],
-                syncdir=self.config.get(section, "syncdir", fallback=None),
                 date=date,
                 treeinfo=self.config.get(section, "treeinfo", fallback=".treeinfo"),
                 proxy=self.config.get("main", "proxy", fallback=None),
@@ -283,19 +271,32 @@ class PyRepoSync:
                 client_key=self.config.get(section, "sslclientkey", fallback=None),
                 ca_cert=self.config.get(section, "sslcacert", fallback=None),
             )
-            jobs.add(job)
-        return jobs
+        elif section.endswith(":deb"):
+            return SyncDeb(
+                base_url=self.config.get(section, "baseurl"),
+                destination=self.config.get("main", "destination"),
+                reponame=section[:-4],
+                date=date,
+                proxy=self.config.get(section, "proxy", fallback=None),
+                client_cert=self.config.get(section, "sslclientcert", fallback=None),
+                client_key=self.config.get(section, "sslclientkey", fallback=None),
+                ca_cert=self.config.get(section, "sslcacert", fallback=None),
+                suites=self.config.get(section, "suites").split(),
+                components=self.config.get(section, "components").split(),
+                binary_archs=self.config.get(section, "binary_archs").split(),
+            )
 
     def get_sections(self):
         sections = set()
         for section in self.config:
-            if section.endswith(":rpm"):
+            if section.endswith(":rpm") or section.endswith(":deb"):
                 if self.repo and section != self.repo:
                     continue
                 if self._tags:
                     if not self.validate_tags(section):
                         continue
                 sections.add(section)
+
         return sections
 
     def work(self):
@@ -303,7 +304,7 @@ class PyRepoSync:
         date = datetime.datetime.utcnow().strftime("%Y%m%d%H%M%S")
         queue = collections.deque()
         for section in self.get_sections():
-            queue.append(self.get_jobs(date=date, section=section))
+            queue.append(self.get_job(date=date, section=section))
         workers = set()
         if self.method == "sync":
             num_worker = self.config.getint("main", "downloaders", fallback=1)
@@ -324,7 +325,7 @@ class PyRepoSync:
         return_code = 0
         for worker in workers:
             worker.join()
-            if worker.status is not 0:
+            if worker.status != 0:
                 return_code = 1
         sys.exit(return_code)
 
@@ -340,7 +341,7 @@ class PyRepoSync:
             else:
                 if tag not in section_tags:
                     return False
-        self.log.info("section {0} has matching tags".format(section))
+        self.log.info(f"section {section} has matching tags")
         return True
 
 
@@ -379,146 +380,132 @@ class RepoSyncThread(threading.Thread):
     def timestamp(self):
         return self._timestamp
 
-    def do_sync(self, jobs):
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                self.name = job.reponame
-                job.sync()
-                self.log.info("{0} done repo {1}".format(self.action, job.reponame))
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
-                )
-                self.status = 1
-
-    def do_snap(self, jobs):
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                self.name = job.reponame
-                job.snap()
-                self.log.info("{0} done repo {1}".format(self.action, job.reponame))
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
-                )
-                self.status = 1
-
-    def do_snap_cleanup(self, jobs):
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                self.name = job.reponame
-                job.snap_cleanup()
-                self.log.info("{0} done repo {1}".format(self.action, job.reponame))
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
-                )
-                self.status = 1
-
-    def do_snap_list(self, jobs):
-        for job in jobs:
-            try:
-                self.name = job.reponame
-                referenced_timestamps = job.snap_list_get_referenced_timestamps()
-                self.log.info("Repository: {0}".format(job.reponame))
-                self.log.info("The following timestamp snapshots exist:")
-                for timestamp in job.snap_list_timestamp_snapshots():
-                    self.log.info(
-                        "{0} -> {1}".format(
-                            timestamp, referenced_timestamps.get(timestamp, [])
-                        )
-                    )
-                self.log.info("The following named snapshots exist:")
-                base = "{0}/snap/{1}/".format(job.destination, job.reponame)
-                for named in job.snap_list_named_snapshots():
-                    named = "named/{0}".format(named)
-                    self.log.info(
-                        "{0} -> {1}".format(
-                            named,
-                            job.snap_list_named_snapshot_target(
-                                "{0}/{1}".format(base, named)
-                            ),
-                        )
-                    )
-                latest = "{0}/latest".format(base)
+    def do_migrate(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.migrate()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_sync(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.sync()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_cleanup(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_cleanup()
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_list(self, job):
+        try:
+            self.name = job.reponame
+            referenced_timestamps = job.snap_list_get_referenced_timestamps()
+            self.log.info(f"Repository: {job.reponame}")
+            self.log.info("The following timestamp snapshots exist:")
+            for timestamp in job.snap_list_timestamp_snapshots():
                 self.log.info(
-                    "latest -> {0}".format(job.snap_list_named_snapshot_target(latest))
-                )
-
-            except OSRepoSyncException:
-                self.status = 1
-
-    def do_snap_name(self, jobs):
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                self.name = job.reponame
-                job.snap_name(self.timestamp, self.snapname)
-                self.log.info("{0} done repo {1}".format(self.action, job.reponame))
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
+                    f"{timestamp} -> {referenced_timestamps.get(timestamp, [])}"
                 )
-                self.status = 1
-
-    def do_snap_unname(self, jobs):
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                self.name = job.reponame
-                job.snap_unname(self.snapname)
-                self.log.info("{0} done repo {1}".format(self.action, job.reponame))
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
-                )
-                self.status = 1
-
-    def do_validate(self, jobs):
+            self.log.info("The following named snapshots exist:")
+            base = f"{job.destination}/snap/{job.reponame}/"
+            for named in job.snap_list_named_snapshots():
+                timestamp = job.snap_list_named_snapshot_target(f"{base}/named/{named}")
+                self.log.info(f"named/{named} -> {timestamp}")
+            latest = f"{base}/latest"
+            self.log.info(f"latest -> {job.snap_list_named_snapshot_target(latest)}")
+
+        except OSRepoSyncException:
+            self.status = 1
+
+    def do_snap_name(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_name(self.timestamp, self.snapname)
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_snap_unname(self, job):
+        try:
+            self.name = job.reponame
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            job.snap_unname(self.snapname)
+            self.log.info(f"{self.action} done repo {job.reponame}")
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
+
+    def do_validate(self, job):
         _downloader = Downloader()
         packages = dict()
-        for job in jobs:
-            try:
-                self.log.info("{0} start repo {1}".format(self.action, job.reponame))
-                packages.update(job.revalidate2())
-            except OSRepoSyncException:
-                self.log.fatal(
-                    "could not {0} repo {1}".format(self.action, job.reponame)
-                )
-                self.status = 1
+        try:
+            self.log.info(f"{self.action} start repo {job.reponame}")
+            packages.update(job.revalidate())
+        except OSRepoSyncException:
+            self.log.fatal(f"could not {self.action} repo {job.reponame}")
+            self.status = 1
         for destination, hash_info in packages.items():
             try:
-                self.log.info("validating: {0}".format(destination))
+                self.log.info(f"validating: {destination}")
                 _downloader.check_hash(
                     destination=destination,
                     checksum=hash_info["hash_sum"],
                     hash_type=hash_info["hash_algo"],
                 )
             except OSRepoSyncHashError:
-                self.log.error("hash mismatch for: {0}".format(destination))
+                self.log.error(f"hash mismatch for: {destination}")
             except FileNotFoundError:
-                self.log.error("file not found: {0}".format(destination))
+                self.log.error(f"file not found: {destination}")
 
     def run(self):
         while True:
             try:
-                jobs = self.queue.pop()
-                if self.action is "sync":
-                    self.do_sync(jobs)
-                elif self.action is "snap_cleanup":
-                    self.do_snap_cleanup(jobs)
-                elif self.action is "snap_list":
-                    self.do_snap_list(jobs)
-                elif self.action is "snap_name":
-                    self.do_snap_name(jobs)
-                elif self.action is "snap_unname":
-                    self.do_snap_unname(jobs)
-                elif self.action is "snap":
-                    self.do_snap(jobs)
-                elif self.action is "validate":
-                    self.do_validate(jobs)
+                job = self.queue.pop()
+                if self.action == "migrate":
+                    self.do_migrate(job)
+                elif self.action == "sync":
+                    self.do_sync(job)
+                elif self.action == "snap_cleanup":
+                    self.do_snap_cleanup(job)
+                elif self.action == "snap_list":
+                    self.do_snap_list(job)
+                elif self.action == "snap_name":
+                    self.do_snap_name(job)
+                elif self.action == "snap_unname":
+                    self.do_snap_unname(job)
+                elif self.action == "snap":
+                    self.do_snap(job)
+                elif self.action == "validate":
+                    self.do_validate(job)
             except IndexError:
                 break
+
+
+if __name__ == "__main__":
+    main()

{pyreposync-0.1.0 → pyreposync-0.2.0}/pyreposync/downloader.py

@@ -4,6 +4,7 @@ import os
 import requests
 import requests.exceptions
 import shutil
+import tempfile
 import time
 
 from pyreposync.exceptions import OSRepoSyncDownLoadError, OSRepoSyncHashError
@@ -53,16 +54,24 @@ class Downloader(object):
 
         with open(destination, "rb") as dest:
             hasher.update(dest.read())
-            self.log.debug("expected hash: {0}".format(hasher.hexdigest()))
-            self.log.debug("actual hash: {0}".format(checksum))
+            self.log.debug(f"expected hash: {hasher.hexdigest()}")
+            self.log.debug(f"actual hash: {checksum}")
             if hasher.hexdigest() == checksum:
-                self.log.debug("download valid: {0}".format(destination))
+                self.log.debug(f"download valid: {destination}")
             else:
-                self.log.error("download invalid: {0}".format(destination))
-                raise OSRepoSyncHashError("download invalid: {0}".format(destination))
+                self.log.error(f"download invalid: {destination}")
+                raise OSRepoSyncHashError(f"download invalid: {destination}")
 
-    def get(self, url, destination, checksum=None, hash_type=None, replace=False):
-        self.log.info("downloading: {0}".format(url))
+    def get(
+        self,
+        url,
+        destination,
+        checksum=None,
+        hash_type=None,
+        replace=False,
+        not_found_ok=False,
+    ):
+        self.log.info(f"downloading: {url}")
         if not replace:
             if os.path.isfile(destination):
                 self.log.info("already there, not downloading")
@@ -70,8 +79,18 @@ class Downloader(object):
         retries = 10
         while retries >= 0:
             try:
-                self._get(url, destination, checksum, hash_type)
-                self.log.info("done downloading: {0}".format(url))
+                with tempfile.TemporaryDirectory() as tmp_dir:
+                    tmp_file = os.path.join(tmp_dir, os.path.basename(destination))
+                    self._get(url, tmp_file, checksum, hash_type, not_found_ok)
+                    self.create_dir(destination)
+                    try:
+                        shutil.move(tmp_file, destination)
+                    except OSError:
+                        if not_found_ok:
+                            pass
+                        else:
+                            raise
+                self.log.info(f"done downloading: {url}")
                 return
             except requests.exceptions.ConnectionError:
                 self.log.error("could not fetch resource, retry in 10 seconds")
@@ -83,15 +102,26 @@ class Downloader(object):
                 time.sleep(10)
             except OSRepoSyncDownLoadError:
                 break
-        self.log.error("could not download: {0}".format(url))
-        raise OSRepoSyncDownLoadError("could not download: {0}".format(url))
+        self.log.error(f"could not download: {url}")
+        raise OSRepoSyncDownLoadError(f"could not download: {url}")
 
-    def _get(self, url, destination, checksum=None, hash_type=None):
+    def create_dir(self, destination):
         if not os.path.isdir(os.path.dirname(destination)):
             try:
                 os.makedirs(os.path.dirname(destination))
-            except OSError:
-                pass
+            except OSError as err:
+                self.log.error(f"could not create directory: {err}")
+                raise OSRepoSyncDownLoadError(f"could not create directory: {err}")
+
+    def _get(
+        self,
+        url,
+        destination,
+        checksum=None,
+        hash_type=None,
+        not_found_ok=False,
+    ):
+        self.create_dir(destination)
         r = requests.get(
             url, stream=True, proxies=self.proxy, cert=self.cert, verify=self.ca_cert
         )
@@ -101,9 +131,13 @@ class Downloader(object):
                 shutil.copyfileobj(r.raw, dst)
                 dst.flush()
         else:
+            if not_found_ok:
+                if r.status_code == 404:
+                    self.log.info("not found, skipping")
+                    return
             raise OSRepoSyncDownLoadError()
         if checksum:
             self.check_hash(
                 destination=destination, checksum=checksum, hash_type=hash_type
             )
-        self.log.info("successfully fetched: {0}".format(url))
+        self.log.info(f"successfully fetched: {url}")