pyreposync 0.2.7__py3-none-any.whl → 0.2.8__py3-none-any.whl

pyreposync/downloader.py

@@ -71,13 +71,11 @@ class Downloader(object):
 
         with open(destination, "rb") as dest:
             hasher.update(dest.read())
-            self.log.debug(f"expected hash: {hasher.hexdigest()}")
-            self.log.debug(f"actual hash: {checksum}")
             if hasher.hexdigest() == checksum:
-                self.log.debug(f"download valid: {destination}")
+                self.log.debug(f"download valid: {destination}, expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
             else:
-                self.log.error(f"download invalid: {destination}")
-                raise OSRepoSyncHashError(f"download invalid: {destination}")
+                self.log.error(f"download invalid: {destination} expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
+                raise OSRepoSyncHashError(f"download invalid: {destination} expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
 
     def get(
         self,
@@ -88,11 +86,11 @@ class Downloader(object):
         replace=False,
         not_found_ok=False,
     ):
-        self.log.info(f"downloading: {url}")
         if not replace:
             if os.path.isfile(destination):
-                self.log.info("already there, not downloading")
+                self.log.debug(f"{url} already there, not downloading")
                 return
+        self.log.info(f"{url} downloading")
         retries = 10
         while retries >= 0:
             try:
@@ -107,7 +105,7 @@ class Downloader(object):
                             pass
                         else:
                             raise
-                self.log.info(f"done downloading: {url}")
+                self.log.info(f"{url} download done")
                 return
             except requests.exceptions.ConnectionError:
                 self.log.error("could not fetch resource, retry in 10 seconds")
@@ -119,8 +117,8 @@ class Downloader(object):
                 time.sleep(10)
             except OSRepoSyncDownLoadError:
                 break
-        self.log.error(f"could not download: {url}")
-        raise OSRepoSyncDownLoadError(f"could not download: {url}")
+        self.log.error(f"{url} could not download")
+        raise OSRepoSyncDownLoadError(f"{url} could not download")
 
     def create_dir(self, destination):
         if not os.path.isdir(os.path.dirname(destination)):

pyreposync/sync_deb822.py

@@ -1,8 +1,11 @@
 from shutil import copyfile
+import threading
 
 import gzip
+import lzma
 import os
 
+from pyreposync import OSRepoSyncException
 from pyreposync.sync_generic import SyncGeneric
 
 
@@ -55,15 +58,34 @@ class SyncDeb822(SyncGeneric):
         return self._components
 
     def _snap(self):
+        jobs = []
         for suite in self.suites:
-            self.snap_suites(suite=suite)
+            job = threading.Thread(
+                name=f"{threading.current_thread().name}_{suite}",
+                target=self.snap_suites,
+                kwargs={"suite": suite},
+            )
+            job.start()
+            jobs.append(job)
+        for job in jobs:
+            job.join()
 
     def snap_suites(self, suite):
         self.log.info(f"creating snapshot for suite {suite}")
         self.snap_release(suite=suite)
         self.snap_release_files(suite=suite)
-        for arch in self.binary_archs:
-            self.snap_package_binary_files(suite=suite, arch=arch)
+        jobs = []
+        for component in self.components:
+            for arch in self.binary_archs:
+                job = threading.Thread(
+                    name=f"{threading.current_thread().name}_{component}_{arch}",
+                    target=self.snap_package_binary_files,
+                    kwargs={"suite": suite, "component": component, "arch": arch},
+                )
+                job.start()
+                jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info(f"creating snapshot for suite {suite}, done")
 
     def snap_release(self, suite):
@@ -99,11 +121,11 @@ class SyncDeb822(SyncGeneric):
                 pass
         self.log.info(f"creating snapshot for suite {suite} release files, done")
 
-    def snap_package_binary_files(self, suite, arch):
+    def snap_package_binary_files(self, suite, component, arch):
         self.log.info(
-            f"creating snapshot for suite {suite} arch {arch} package binary files"
+            f"creating snapshot for suite {suite} component {component} arch {arch} package binary files"
         )
-        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        packages = self.binary_files_sha256(suite=suite, component=component, arch=arch)
         src_path = f"{self.destination}/sync/{self.reponame}"
         dst_path = f"{self.destination}/snap/{self.reponame}/{self.date}"
         for filename, sha256_dict in packages.items():
@@ -120,16 +142,35 @@ class SyncDeb822(SyncGeneric):
 
     def sync(self):
         self.log.info("starting thread")
+        jobs = []
         for suite in self.suites:
-            self.sync_suites(suite=suite)
+            job = threading.Thread(
+                name=f"{threading.current_thread().name}_{suite}",
+                target=self.sync_suites,
+                kwargs={"suite": suite},
+            )
+            job.start()
+            jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info("shutdown thread complete")
 
     def sync_suites(self, suite):
         self.log.info(f"syncing suite {suite}")
         self.sync_release(suite=suite)
         self.sync_release_files(suite=suite)
-        for arch in self.binary_archs:
-            self.sync_package_binary_files(suite=suite, arch=arch)
+        jobs = []
+        for component in self.components:
+            for arch in self.binary_archs:
+                job = threading.Thread(
+                    name=f"{threading.current_thread().name}_{component}_{arch}",
+                    target=self.sync_package_binary_files,
+                    kwargs={"suite": suite, "component": component, "arch": arch},
+                )
+                job.start()
+                jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info(f"syncing suite {suite}, done")
 
     def sync_release(self, suite):
@@ -146,9 +187,9 @@ class SyncDeb822(SyncGeneric):
             )
         self.log.info(f"syncing suite {suite} release files, done")
 
-    def sync_package_binary_files(self, suite, arch):
+    def sync_package_binary_files(self, suite, component, arch):
         self.log.info(f"syncing suite {suite} arch {arch} package binary files")
-        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        packages = self.binary_files_sha256(suite=suite, component=component, arch=arch)
         base_path = f"{self.destination}/sync/{self.reponame}"
         base_url = f"{self.base_url}"
         for filename, sha256_dict in packages.items():
@@ -162,12 +203,26 @@ class SyncDeb822(SyncGeneric):
         self.log.info(f"syncing suite {suite} arch {arch} package binary files, done")
 
     def binary_files_sha256(self, suite, component, arch):
-        packages_gz_file = f"{self.destination}/sync/{self.reponame}/dists/{suite}/{component}/binary-{arch}/Packages.gz"
+        packages_file = f"{self.destination}/sync/{self.reponame}/dists/{suite}/{component}/binary-{arch}/Packages."
+        if os.path.isfile(f"{packages_file}gz"):
+            comp = gzip
+            packages_file += "gz"
+        elif os.path.isfile(f"{packages_file}xz"):
+            comp = lzma
+            packages_file += "xz"
+        else:
+            self.log.error(
+                f"no Packages.gz or Packages.xz file found for suite {suite}, component {component}, arch {arch}"
+            )
+            raise OSRepoSyncException(
+                f"no Packages.gz or Packages.xz file found for suite {suite}, component {component}, arch {arch}"
+            )
+
         packages = dict()
         sha256 = None
         filename = None
         size = None
-        with gzip.open(packages_gz_file, "rb") as source:
+        with comp.open(packages_file, "rb") as source:
             for line in source:
                 line = line.decode("utf-8")
                 if line.startswith("SHA256: "):

pyreposync/sync_rpm.py

@@ -252,14 +252,15 @@ class SyncRPM(SyncGeneric):
     def snap_treeinfo(self):
         self.log.info("copy treeinfo")
         try:
-            dst = f"{self.destination}/snap/{self.reponame}/{self.destination}/{self.treeinfo}"
+            dst = f"{self.destination}/snap/{self.reponame}/{self.date}/{self.treeinfo}"
             src = f"{self.destination}/sync/{self.reponame}/{self.treeinfo}"
             copyfile(src, dst)
         except (OSError, FileNotFoundError) as err:
             self.log.error(f"could not copy {self.treeinfo}: {err}")
+            self.log.error(dst)
         for location, hash_algo, hash_sum in self.treeinfo_files():
             dst = (
-                f"{self.destination}/snap/{self.reponame}/{self.destination}/{location}"
+                f"{self.destination}/snap/{self.reponame}/{self.date}/{location}"
             )
             src = f"{self.destination}/sync/{self.reponame}/{location}"
             try:

{pyreposync-0.2.7.dist-info → pyreposync-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyreposync
-Version: 0.2.7
+Version: 0.2.8
 Summary: rpm and deb822 repository sync tool
 Project-URL: Source, https://github.com/schlitzered/pyreposync
 Author-email: "Stephan.Schultchen" <sschultchen@gmail.com>

pyreposync-0.2.8.dist-info/RECORD

@@ -0,0 +1,11 @@
+pyreposync/__init__.py,sha256=fHirsBBkEk806RJwVKQZHgujQZRk13YnokX-8rp_QJE,17117
+pyreposync/downloader.py,sha256=rEv9lOg2X1VwCtADO63-yceZAZCeSJOBIGbyRsEnKQU,5275
+pyreposync/exceptions.py,sha256=IlnvhNaffQQ6geOgrjCciNFVbFpNcycH4ijSuMTbrGA,169
+pyreposync/sync_deb822.py,sha256=7Cuzg-1pC8WDiFTxw-qH26VsfzOEY9IxfZztGfLkdKg,10513
+pyreposync/sync_generic.py,sha256=qe-ERp8O2kCnhOuToSZWhdQzGY7aYb7G7nUxuKD3lxY,5880
+pyreposync/sync_rpm.py,sha256=bl1mVX8LZt5AF33o-oVsmaNwzC9teLN-quQMkQrwyUk,11781
+pyreposync-0.2.8.dist-info/METADATA,sha256=4ZJYmZtU02e0YaE1-awb39qUPIwAqKwl2fdhiyGfimY,1591
+pyreposync-0.2.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pyreposync-0.2.8.dist-info/entry_points.txt,sha256=9LsBDWOF3O6_3ONP3Lc-4v1MTt5ay0Xv-TMcFbOIt2s,47
+pyreposync-0.2.8.dist-info/licenses/LICENSE.txt,sha256=lwnJoIo7uwc0h6y6gC_RYqJkvjplViV3Ad6u7pQM4Bw,1084
+pyreposync-0.2.8.dist-info/RECORD,,

pyreposync-0.2.7.dist-info/RECORD

@@ -1,11 +0,0 @@
-pyreposync/__init__.py,sha256=fHirsBBkEk806RJwVKQZHgujQZRk13YnokX-8rp_QJE,17117
-pyreposync/downloader.py,sha256=cRUtbQ4PYbGDczZl5PsBtTQFhqagnX9IQvbwSCd0OBc,5212
-pyreposync/exceptions.py,sha256=IlnvhNaffQQ6geOgrjCciNFVbFpNcycH4ijSuMTbrGA,169
-pyreposync/sync_deb822.py,sha256=Ynwok1fiZktO3P_0lYWfBOHgaJOFpzfVNkhgKL5Y56s,8517
-pyreposync/sync_generic.py,sha256=qe-ERp8O2kCnhOuToSZWhdQzGY7aYb7G7nUxuKD3lxY,5880
-pyreposync/sync_rpm.py,sha256=FXWEJd2YelMsZeTOnjcNjkN2D_ECtX9HEp73GpiLeK4,11763
-pyreposync-0.2.7.dist-info/METADATA,sha256=t_2vG9j5arm-bG1SyZ0DDqADLwQKEiqS66EZ-q1VnJE,1591
-pyreposync-0.2.7.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pyreposync-0.2.7.dist-info/entry_points.txt,sha256=9LsBDWOF3O6_3ONP3Lc-4v1MTt5ay0Xv-TMcFbOIt2s,47
-pyreposync-0.2.7.dist-info/licenses/LICENSE.txt,sha256=lwnJoIo7uwc0h6y6gC_RYqJkvjplViV3Ad6u7pQM4Bw,1084
-pyreposync-0.2.7.dist-info/RECORD,,