pyreposync 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

pyreposync/__init__.py

@@ -269,6 +269,12 @@ class PyRepoSync:
                     section, "allow_missing_packages", fallback=False
                 ),
                 treeinfo=self.config.get(section, "treeinfo", fallback=".treeinfo"),
+                basic_auth_user=self.config.get(
+                    section, "basic_auth_user", fallback=None
+                ),
+                basic_auth_pass=self.config.get(
+                    section, "basic_auth_pass", fallback=None
+                ),
                 proxy=self.config.get("main", "proxy", fallback=None),
                 client_cert=self.config.get(section, "sslclientcert", fallback=None),
                 client_key=self.config.get(section, "sslclientkey", fallback=None),
@@ -283,6 +289,12 @@ class PyRepoSync:
                 allow_missing_packages=self.config.getboolean(
                     section, "allow_missing_packages", fallback=False
                 ),
+                basic_auth_user=self.config.get(
+                    section, "basic_auth_user", fallback=None
+                ),
+                basic_auth_pass=self.config.get(
+                    section, "basic_auth_pass", fallback=None
+                ),
                 proxy=self.config.get(section, "proxy", fallback=None),
                 client_cert=self.config.get(section, "sslclientcert", fallback=None),
                 client_key=self.config.get(section, "sslclientkey", fallback=None),
@@ -291,6 +303,7 @@ class PyRepoSync:
                 components=self.config.get(section, "components").split(),
                 binary_archs=self.config.get(section, "binary_archs").split(),
             )
+        return None
 
     def get_sections(self):
         sections = set()

pyreposync/downloader.py

@@ -2,6 +2,7 @@ import hashlib
 import logging
 import os
 import requests
+import requests.auth
 import requests.exceptions
 import shutil
 import tempfile
@@ -11,8 +12,20 @@ from pyreposync.exceptions import OSRepoSyncDownLoadError, OSRepoSyncHashError
 
 
 class Downloader(object):
-    def __init__(self, proxy=None, client_cert=None, client_key=None, ca_cert=None):
+    def __init__(
+        self,
+        basic_auth_user=None,
+        basic_auth_pass=None,
+        proxy=None,
+        client_cert=None,
+        client_key=None,
+        ca_cert=None,
+    ):
         self.log = logging.getLogger("application")
+        if basic_auth_user and basic_auth_pass:
+            self._basic_auth = (basic_auth_user, basic_auth_pass)
+        else:
+            self._basic_auth = None
         if proxy:
             self._proxy = {"http": proxy, "https": proxy}
         else:
@@ -26,6 +39,10 @@ class Downloader(object):
         else:
             self._ca_cert = True
 
+    @property
+    def basic_auth(self):
+        return self._basic_auth
+
     @property
     def ca_cert(self):
         return self._ca_cert
@@ -54,13 +71,11 @@ class Downloader(object):
 
         with open(destination, "rb") as dest:
             hasher.update(dest.read())
-            self.log.debug(f"expected hash: {hasher.hexdigest()}")
-            self.log.debug(f"actual hash: {checksum}")
             if hasher.hexdigest() == checksum:
-                self.log.debug(f"download valid: {destination}")
+                self.log.debug(f"download valid: {destination}, expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
             else:
-                self.log.error(f"download invalid: {destination}")
-                raise OSRepoSyncHashError(f"download invalid: {destination}")
+                self.log.error(f"download invalid: {destination} expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
+                raise OSRepoSyncHashError(f"download invalid: {destination} expected hash: {hasher.hexdigest()}, actual hash: {checksum}")
 
     def get(
         self,
@@ -71,11 +86,11 @@ class Downloader(object):
         replace=False,
         not_found_ok=False,
     ):
-        self.log.info(f"downloading: {url}")
         if not replace:
             if os.path.isfile(destination):
-                self.log.info("already there, not downloading")
+                self.log.debug(f"{url} already there, not downloading")
                 return
+        self.log.info(f"{url} downloading")
         retries = 10
         while retries >= 0:
             try:
@@ -90,7 +105,7 @@ class Downloader(object):
                             pass
                         else:
                             raise
-                self.log.info(f"done downloading: {url}")
+                self.log.info(f"{url} download done")
                 return
             except requests.exceptions.ConnectionError:
                 self.log.error("could not fetch resource, retry in 10 seconds")
@@ -102,8 +117,8 @@ class Downloader(object):
                 time.sleep(10)
             except OSRepoSyncDownLoadError:
                 break
-        self.log.error(f"could not download: {url}")
-        raise OSRepoSyncDownLoadError(f"could not download: {url}")
+        self.log.error(f"{url} could not download")
+        raise OSRepoSyncDownLoadError(f"{url} could not download")
 
     def create_dir(self, destination):
         if not os.path.isdir(os.path.dirname(destination)):
@@ -123,7 +138,12 @@ class Downloader(object):
     ):
         self.create_dir(destination)
         r = requests.get(
-            url, stream=True, proxies=self.proxy, cert=self.cert, verify=self.ca_cert
+            url,
+            auth=self.basic_auth,
+            stream=True,
+            proxies=self.proxy,
+            cert=self.cert,
+            verify=self.ca_cert,
         )
         if r.status_code == 200:
             with open(destination, "wb", 0) as dst:

pyreposync/sync_deb822.py

@@ -1,8 +1,11 @@
 from shutil import copyfile
+import threading
 
 import gzip
+import lzma
 import os
 
+from pyreposync import OSRepoSyncException
 from pyreposync.sync_generic import SyncGeneric
 
 
@@ -17,6 +20,8 @@ class SyncDeb822(SyncGeneric):
         components,
         binary_archs,
         allow_missing_packages,
+        basic_auth_user=None,
+        basic_auth_pass=None,
         proxy=None,
         client_cert=None,
         client_key=None,
@@ -28,6 +33,8 @@ class SyncDeb822(SyncGeneric):
             reponame=reponame,
             date=date,
             allow_missing_packages=allow_missing_packages,
+            basic_auth_user=basic_auth_user,
+            basic_auth_pass=basic_auth_pass,
             proxy=proxy,
             client_cert=client_cert,
             client_key=client_key,
@@ -51,15 +58,34 @@ class SyncDeb822(SyncGeneric):
         return self._components
 
     def _snap(self):
+        jobs = []
         for suite in self.suites:
-            self.snap_suites(suite=suite)
+            job = threading.Thread(
+                name=f"{threading.current_thread().name}_{suite}",
+                target=self.snap_suites,
+                kwargs={"suite": suite},
+            )
+            job.start()
+            jobs.append(job)
+        for job in jobs:
+            job.join()
 
     def snap_suites(self, suite):
         self.log.info(f"creating snapshot for suite {suite}")
         self.snap_release(suite=suite)
         self.snap_release_files(suite=suite)
-        for arch in self.binary_archs:
-            self.snap_package_binary_files(suite=suite, arch=arch)
+        jobs = []
+        for component in self.components:
+            for arch in self.binary_archs:
+                job = threading.Thread(
+                    name=f"{threading.current_thread().name}_{component}_{arch}",
+                    target=self.snap_package_binary_files,
+                    kwargs={"suite": suite, "component": component, "arch": arch},
+                )
+                job.start()
+                jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info(f"creating snapshot for suite {suite}, done")
 
     def snap_release(self, suite):
@@ -95,11 +121,11 @@ class SyncDeb822(SyncGeneric):
                 pass
         self.log.info(f"creating snapshot for suite {suite} release files, done")
 
-    def snap_package_binary_files(self, suite, arch):
+    def snap_package_binary_files(self, suite, component, arch):
         self.log.info(
-            f"creating snapshot for suite {suite} arch {arch} package binary files"
+            f"creating snapshot for suite {suite} component {component} arch {arch} package binary files"
         )
-        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        packages = self.binary_files_sha256(suite=suite, component=component, arch=arch)
         src_path = f"{self.destination}/sync/{self.reponame}"
         dst_path = f"{self.destination}/snap/{self.reponame}/{self.date}"
         for filename, sha256_dict in packages.items():
@@ -116,16 +142,35 @@ class SyncDeb822(SyncGeneric):
 
     def sync(self):
         self.log.info("starting thread")
+        jobs = []
         for suite in self.suites:
-            self.sync_suites(suite=suite)
+            job = threading.Thread(
+                name=f"{threading.current_thread().name}_{suite}",
+                target=self.sync_suites,
+                kwargs={"suite": suite},
+            )
+            job.start()
+            jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info("shutdown thread complete")
 
     def sync_suites(self, suite):
         self.log.info(f"syncing suite {suite}")
         self.sync_release(suite=suite)
         self.sync_release_files(suite=suite)
-        for arch in self.binary_archs:
-            self.sync_package_binary_files(suite=suite, arch=arch)
+        jobs = []
+        for component in self.components:
+            for arch in self.binary_archs:
+                job = threading.Thread(
+                    name=f"{threading.current_thread().name}_{component}_{arch}",
+                    target=self.sync_package_binary_files,
+                    kwargs={"suite": suite, "component": component, "arch": arch},
+                )
+                job.start()
+                jobs.append(job)
+        for job in jobs:
+            job.join()
         self.log.info(f"syncing suite {suite}, done")
 
     def sync_release(self, suite):
@@ -142,9 +187,9 @@ class SyncDeb822(SyncGeneric):
             )
         self.log.info(f"syncing suite {suite} release files, done")
 
-    def sync_package_binary_files(self, suite, arch):
+    def sync_package_binary_files(self, suite, component, arch):
         self.log.info(f"syncing suite {suite} arch {arch} package binary files")
-        packages = self.binary_files_sha256(suite=suite, component="main", arch=arch)
+        packages = self.binary_files_sha256(suite=suite, component=component, arch=arch)
         base_path = f"{self.destination}/sync/{self.reponame}"
         base_url = f"{self.base_url}"
         for filename, sha256_dict in packages.items():
@@ -158,12 +203,26 @@ class SyncDeb822(SyncGeneric):
         self.log.info(f"syncing suite {suite} arch {arch} package binary files, done")
 
     def binary_files_sha256(self, suite, component, arch):
-        packages_gz_file = f"{self.destination}/sync/{self.reponame}/dists/{suite}/{component}/binary-{arch}/Packages.gz"
+        packages_file = f"{self.destination}/sync/{self.reponame}/dists/{suite}/{component}/binary-{arch}/Packages."
+        if os.path.isfile(f"{packages_file}gz"):
+            comp = gzip
+            packages_file += "gz"
+        elif os.path.isfile(f"{packages_file}xz"):
+            comp = lzma
+            packages_file += "xz"
+        else:
+            self.log.error(
+                f"no Packages.gz or Packages.xz file found for suite {suite}, component {component}, arch {arch}"
+            )
+            raise OSRepoSyncException(
+                f"no Packages.gz or Packages.xz file found for suite {suite}, component {component}, arch {arch}"
+            )
+
         packages = dict()
         sha256 = None
         filename = None
         size = None
-        with gzip.open(packages_gz_file, "rb") as source:
+        with comp.open(packages_file, "rb") as source:
             for line in source:
                 line = line.decode("utf-8")
                 if line.startswith("SHA256: "):

pyreposync/sync_generic.py

@@ -14,6 +14,8 @@ class SyncGeneric:
         reponame,
         date,
         allow_missing_packages,
+        basic_auth_user=None,
+        basic_auth_pass=None,
         proxy=None,
         client_cert=None,
         client_key=None,
@@ -25,7 +27,12 @@ class SyncGeneric:
         self._destination = destination
         self._reponame = reponame
         self.downloader = Downloader(
-            proxy=proxy, client_cert=client_cert, client_key=client_key, ca_cert=ca_cert
+            basic_auth_user=basic_auth_user,
+            basic_auth_pass=basic_auth_pass,
+            proxy=proxy,
+            client_cert=client_cert,
+            client_key=client_key,
+            ca_cert=ca_cert,
         )
         self.log = logging.getLogger("application")
 

pyreposync/sync_rpm.py

@@ -24,6 +24,8 @@ class SyncRPM(SyncGeneric):
         date,
         treeinfo,
         allow_missing_packages,
+        basic_auth_user=None,
+        basic_auth_pass=None,
         proxy=None,
         client_cert=None,
         client_key=None,
@@ -35,6 +37,8 @@ class SyncRPM(SyncGeneric):
             reponame=reponame,
             date=date,
             allow_missing_packages=allow_missing_packages,
+            basic_auth_user=basic_auth_user,
+            basic_auth_pass=basic_auth_pass,
             proxy=proxy,
             client_cert=client_cert,
             client_key=client_key,
@@ -248,14 +252,15 @@ class SyncRPM(SyncGeneric):
     def snap_treeinfo(self):
         self.log.info("copy treeinfo")
         try:
-            dst = f"{self.destination}/snap/{self.reponame}/{self.destination}/{self.treeinfo}"
+            dst = f"{self.destination}/snap/{self.reponame}/{self.date}/{self.treeinfo}"
             src = f"{self.destination}/sync/{self.reponame}/{self.treeinfo}"
             copyfile(src, dst)
         except (OSError, FileNotFoundError) as err:
             self.log.error(f"could not copy {self.treeinfo}: {err}")
+            self.log.error(dst)
         for location, hash_algo, hash_sum in self.treeinfo_files():
             dst = (
-                f"{self.destination}/snap/{self.reponame}/{self.destination}/{location}"
+                f"{self.destination}/snap/{self.reponame}/{self.date}/{location}"
             )
             src = f"{self.destination}/sync/{self.reponame}/{location}"
             try:

{pyreposync-0.2.6.dist-info → pyreposync-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyreposync
-Version: 0.2.6
+Version: 0.2.8
 Summary: rpm and deb822 repository sync tool
 Project-URL: Source, https://github.com/schlitzered/pyreposync
 Author-email: "Stephan.Schultchen" <sschultchen@gmail.com>

pyreposync-0.2.8.dist-info/RECORD

@@ -0,0 +1,11 @@
+pyreposync/__init__.py,sha256=fHirsBBkEk806RJwVKQZHgujQZRk13YnokX-8rp_QJE,17117
+pyreposync/downloader.py,sha256=rEv9lOg2X1VwCtADO63-yceZAZCeSJOBIGbyRsEnKQU,5275
+pyreposync/exceptions.py,sha256=IlnvhNaffQQ6geOgrjCciNFVbFpNcycH4ijSuMTbrGA,169
+pyreposync/sync_deb822.py,sha256=7Cuzg-1pC8WDiFTxw-qH26VsfzOEY9IxfZztGfLkdKg,10513
+pyreposync/sync_generic.py,sha256=qe-ERp8O2kCnhOuToSZWhdQzGY7aYb7G7nUxuKD3lxY,5880
+pyreposync/sync_rpm.py,sha256=bl1mVX8LZt5AF33o-oVsmaNwzC9teLN-quQMkQrwyUk,11781
+pyreposync-0.2.8.dist-info/METADATA,sha256=4ZJYmZtU02e0YaE1-awb39qUPIwAqKwl2fdhiyGfimY,1591
+pyreposync-0.2.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pyreposync-0.2.8.dist-info/entry_points.txt,sha256=9LsBDWOF3O6_3ONP3Lc-4v1MTt5ay0Xv-TMcFbOIt2s,47
+pyreposync-0.2.8.dist-info/licenses/LICENSE.txt,sha256=lwnJoIo7uwc0h6y6gC_RYqJkvjplViV3Ad6u7pQM4Bw,1084
+pyreposync-0.2.8.dist-info/RECORD,,

pyreposync-0.2.6.dist-info/RECORD

@@ -1,11 +0,0 @@
-pyreposync/__init__.py,sha256=hhsb3ZqssxKAcSZ_rEiG_2XjgbRK1SveMtL_sI9pMnI,16577
-pyreposync/downloader.py,sha256=828bIMCctd1X6I1w91-XHytQ1M5fmxkEw0YDKd5nvAY,4764
-pyreposync/exceptions.py,sha256=IlnvhNaffQQ6geOgrjCciNFVbFpNcycH4ijSuMTbrGA,169
-pyreposync/sync_deb822.py,sha256=XkIZfYfgshYFwdk1oL8TsysUge2_6iPiz5zOmKNaZRk,8367
-pyreposync/sync_generic.py,sha256=wN5hPcIacvQiMQoajcf7WSbiBGqxI86CYwfVnq1fR38,5693
-pyreposync/sync_rpm.py,sha256=1yf2-lQShKyY3CI_FW1i0tyWQ0sg69iBUn-4iWfoevk,11613
-pyreposync-0.2.6.dist-info/METADATA,sha256=_iKcUStZxLy2goNOQgaIMgRE3CzmG34iDz-nXKN51Jw,1591
-pyreposync-0.2.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pyreposync-0.2.6.dist-info/entry_points.txt,sha256=9LsBDWOF3O6_3ONP3Lc-4v1MTt5ay0Xv-TMcFbOIt2s,47
-pyreposync-0.2.6.dist-info/licenses/LICENSE.txt,sha256=lwnJoIo7uwc0h6y6gC_RYqJkvjplViV3Ad6u7pQM4Bw,1084
-pyreposync-0.2.6.dist-info/RECORD,,