taskcluster-taskgraph 9.1.0__tar.gz → 9.2.1__tar.gz

{taskcluster-taskgraph-9.1.0/src/taskcluster_taskgraph.egg-info → taskcluster-taskgraph-9.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 9.1.0
+Version: 9.2.1
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-9.1.0 → taskcluster-taskgraph-9.2.1/src/taskcluster_taskgraph.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 9.1.0
+Version: 9.2.1
 Summary: Build taskcluster taskgraphs
 Home-page: https://github.com/taskcluster/taskgraph
 Classifier: Development Status :: 5 - Production/Stable

{taskcluster-taskgraph-9.1.0 → taskcluster-taskgraph-9.2.1}/src/taskgraph/__init__.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-__version__ = "9.1.0"
+__version__ = "9.2.1"
 
 # Maximum number of dependencies a single task can have
 # https://docs.taskcluster.net/reference/platform/taskcluster-queue/references/api#createTask

{taskcluster-taskgraph-9.1.0 → taskcluster-taskgraph-9.2.1}/src/taskgraph/run-task/fetch-content

@@ -194,6 +194,25 @@ def stream_download(url, sha256=None, size=None, headers=None):
     ) if certifi else urllib.request.urlopen(req, timeout=60) as fh:
         if not url.endswith(".gz") and fh.info().get("Content-Encoding") == "gzip":
             fh = gzip.GzipFile(fileobj=fh)
+        else:
+            # when using gzip we can't compare size or length (inflated) against content-length (compressed)
+            content_length = fh.getheader("content-length")
+            if content_length:
+                try:
+                    content_length = int(content_length)
+                except ValueError:
+                    raise IntegrityError(
+                        "content-length header for %s is not an integer; got %s"
+                        % (url, content_length)
+                    )
+                if size:
+                    if size != content_length:
+                        raise IntegrityError(
+                            "size mismatch on %s: wanted %d; content-length is %d"
+                            % (url, size, content_length)
+                        )
+                else:
+                    size = content_length
 
         while True:
             chunk = fh.read(65536)
@@ -252,8 +271,6 @@ def download_to_path(url, path, sha256=None, size=None, headers=None):
                     fh.write(chunk)
 
             return
-        except IntegrityError:
-            raise
         except Exception as e:
             log("Download failed: {}".format(e))
             continue
@@ -264,17 +281,15 @@ def download_to_path(url, path, sha256=None, size=None, headers=None):
 def download_to_memory(url, sha256=None, size=None):
     """Download a URL to memory, possibly with verification."""
 
-    data = b""
     for _ in retrier(attempts=5, sleeptime=60):
-        try:
-            log("Downloading %s" % (url))
+        data = b""
+        log("Downloading %s" % (url))
 
+        try:
             for chunk in stream_download(url, sha256=sha256, size=size):
                 data += chunk
 
             return data
-        except IntegrityError:
-            raise
         except Exception as e:
             log("Download failed: {}".format(e))
             continue

{taskcluster-taskgraph-9.1.0 → taskcluster-taskgraph-9.2.1}/test/test_scripts_fetch_content.py

@@ -73,7 +73,13 @@ def test_stream_download(
         # create a mock context manager
         cm = MagicMock()
         cm.getcode.return_value = 200
+
+        def getheader(field):
+            if field.lower() == "content-length":
+                return size
+
         # simulates chunking
+        cm.getheader = getheader
         cm.read.side_effect = [b"foo", b"bar", None]
         cm.__enter__.return_value = cm
         return cm