dxpy 0.392.0__tar.gz → 0.394.0__tar.gz

{dxpy-0.392.0 → dxpy-0.394.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.392.0
+Version: 0.394.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.392.0 → dxpy-0.394.0}/dxpy/__init__.py

@@ -351,7 +351,7 @@ def _calculate_retry_delay(response, num_attempts):
         resource, including the most recent failed one
     :type num_attempts: int
     '''
-    if response is not None and response.status == 503 and 'retry-after' in response.headers:
+    if response is not None and response.status in (503, 429) and 'retry-after' in response.headers:
         try:
             return int(response.headers['retry-after'])
         except ValueError:
@@ -553,10 +553,10 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
 
     # Maintain two separate counters for the number of tries...
 
-    try_index = 0  # excluding 503 errors. The number of tries as given here
+    try_index = 0  # excluding 503/429 errors. The number of tries as given here
                    # cannot exceed (max_retries + 1).
-    try_index_including_503 = 0  # including 503 errors. This number is used to
-                                 # do exponential backoff.
+    try_index_including_throttled = 0  # including 503/429 errors. This number is used to
+                                       # do exponential backoff.
 
     retried_responses = []
     _url = None
@@ -730,7 +730,7 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                        or isinstance(e, _RETRYABLE_WITH_RESPONSE)):
                         ok_to_retry = is_retryable
                     else:
-                        ok_to_retry = 500 <= response.status < 600
+                        ok_to_retry = response.status == 429 or 500 <= response.status < 600
 
                     # The server has closed the connection prematurely
                     if (response is not None
@@ -753,10 +753,10 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
                     if rewind_input_buffer_offset is not None:
                         data.seek(rewind_input_buffer_offset)
 
-                    delay = _calculate_retry_delay(response, try_index_including_503 + 1)
+                    delay = _calculate_retry_delay(response, try_index_including_throttled + 1)
 
                     range_str = (' (range=%s)' % (headers['Range'],)) if 'Range' in headers else ''
-                    if response is not None and response.status == 503:
+                    if response is not None and response.status in (503, 429):
                         waiting_msg = 'Waiting %d seconds before retry...' % (delay,)
                     else:
                         waiting_msg = 'Waiting %d seconds before retry %d of %d...' % (
@@ -768,8 +768,8 @@ def DXHTTPRequest(resource, data, method='POST', headers=None, auth=True,
 
                     logger.warning(log_msg)
                     time.sleep(delay)
-                    try_index_including_503 += 1
-                    if response is None or response.status != 503:
+                    try_index_including_throttled += 1
+                    if response is None or response.status not in (503, 429):
                         try_index += 1
                     continue
 

{dxpy-0.392.0 → dxpy-0.394.0}/dxpy/bindings/dxfile.py

@@ -253,6 +253,8 @@ class DXFile(DXDataObject):
         self._file_length = None
         self._cur_part = 1
         self._num_uploaded_parts = 0
+        # Cache for object existence in project
+        self._exists_in_proj = None  
 
     def _new(self, dx_hash, media_type=None, **kwargs):
         """
@@ -314,39 +316,19 @@ class DXFile(DXDataObject):
             raise
 
     def __iter__(self):
-        _buffer = self.read(self._read_bufsize)
-        done = False
-        if USING_PYTHON2:
-            while not done:
-                if b"\n" in _buffer:
-                    lines = _buffer.splitlines()
-                    for i in range(len(lines) - 1):
-                        yield lines[i]
-                    _buffer = lines[len(lines) - 1]
-                else:
-                    more = self.read(self._read_bufsize)
-                    if more == b"":
-                        done = True
-                    else:
-                        _buffer = _buffer + more
-        else:
-            if self._binary_mode:
-                raise DXFileError("Cannot read lines when file opened in binary mode")
-            # python3 is much stricter about distinguishing
-            # 'bytes' from 'str'.
-            while not done:
-                if "\n" in _buffer:
-                    lines = _buffer.splitlines()
-                    for i in range(len(lines) - 1):
-                        yield lines[i]
-                    _buffer = lines[len(lines) - 1]
-                else:
-                    more = self.read(self._read_bufsize)
-                    if more == "":
-                        done = True
-                    else:
-                        _buffer = _buffer + more
-
+        _buffer = ""
+        if self._binary_mode:
+            raise DXFileError("Cannot read lines when file opened in binary mode")
+        while True:
+            more = self.read(self._read_bufsize)
+            if not more:
+                break
+            _buffer += more
+            parts = _buffer.split("\n")
+            for i in range(len(parts) - 1):
+                yield parts[i]
+            # The final piece may be incomplete, so keep it in the buffer
+            _buffer = parts[-1]
         if _buffer:
             yield _buffer
 
@@ -954,18 +936,14 @@ class DXFile(DXDataObject):
             buf.seek(orig_buf_pos)
             return buf.read()
 
-        # Debug fallback
-        # import urllib2
-        # req = urllib2.Request(url, headers=headers)
-        # response = urllib2.urlopen(req)
-        # return response.read()
-
     def read(self, length=None, use_compression=None, project=None, **kwargs):
-        if project is None and object_exists_in_project(self.get_id(), self.get_proj_id()):
+        # Check if the file is present in dxfile project attribute if the project arg not specified 
+        if project is None and self._exists_in_proj is None and self.get_proj_id() is not None:
+            self._exists_in_proj = object_exists_in_project(self.get_id(), self.get_proj_id())
+        # Use the DXFile attribute if the project arg is not provided
+        if project is None and self._exists_in_proj:
             project = self.get_proj_id()
         data = self._read2(length=length, use_compression=use_compression, project=project, **kwargs)
-        if USING_PYTHON2:
-            return data
         # In python3, the underlying system methods use the 'bytes' type, not 'string'
         if self._binary_mode is True:
             return data

{dxpy-0.392.0 → dxpy-0.394.0}/dxpy/nextflow/nextflow_builder.py

@@ -182,7 +182,8 @@ def prepare_custom_inputs(schema_file="./nextflow_schema.json"):
 
     with open(schema_file, "r") as fh:
         schema = json.load(fh)
-    for d_key, d_schema in schema.get("definitions", {}).items():
+    defs_key = "definitions" if "definitions" in schema else "$defs" if "$defs" in schema else {}
+    for d_key, d_schema in schema.get(defs_key).items():
         required_inputs = d_schema.get("required", [])
         for property_key, property in d_schema.get("properties", {}).items():
             dx_input = {}

dxpy-0.394.0/dxpy/toolkit_version.py

@@ -0,0 +1 @@
+version = '0.394.0'

{dxpy-0.392.0 → dxpy-0.394.0}/dxpy.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dxpy
-Version: 0.392.0
+Version: 0.394.0
 Summary: DNAnexus Platform API bindings for Python
 Home-page: https://github.com/dnanexus/dx-toolkit
 Author: Aleksandra Zalcman, Andrey Kislyuk, Anurag Biyani, Geet Duggal, Katherine Lai, Kurt Jensen, Marek Hrvol, Ohad Rodeh, Phil Sung

{dxpy-0.392.0 → dxpy-0.394.0}/test/test_dxclient.py

@@ -1964,6 +1964,25 @@ dxpy.run()
         run("cd {wd}; rm test; touch test".format(wd=wd))
         run("cd {wd}; dx download -f test".format(wd=wd))
         assert_md5_checksum(os.path.join(wd, "test"), hashlib.md5(part1 + part2))
+        
+    def test_dx_download_api_call_count(self):
+        """
+        Ensure that dx download does not make more than 4 API calls as observed in _DX_DEBUG output.
+        """
+        with chdir(tempfile.mkdtemp()):
+            # Create a test file and upload it
+            test_file = dxpy.upload_string("test content", name="test_file", project=self.project, wait_on_close=True)
+            test_file_id = test_file.get_id()
+
+            # Run dx download with _DX_DEBUG enabled
+            (stdout, stderr) = run(f"_DX_DEBUG=1 dx download {test_file_id}", also_return_stderr=True)
+
+            # Count the number of API calls in the debug output
+            api_call_count = len([line for line in stderr.splitlines() if line.startswith(">") and dxpy.APISERVER in line])
+
+            # Assert that no more than 4 API calls were made
+            self.assertLessEqual(api_call_count, 4, f"dx download made {api_call_count} API calls, exceeding the limit of 4.\n_DX_DEBUG:\n{stderr}")
+
 
 
 class TestDXClientDownloadDataEgressBilling(DXTestCase):

{dxpy-0.392.0 → dxpy-0.394.0}/test/test_dxpy.py

@@ -19,7 +19,7 @@
 
 from __future__ import print_function, unicode_literals, division, absolute_import
 
-import os, unittest, tempfile, filecmp, time, json, sys
+import os, unittest, tempfile, filecmp, time, json, sys, random
 import shutil
 import string
 import subprocess
@@ -27,6 +27,7 @@ import platform
 import pytest
 import re
 import certifi
+from mock import patch
 
 from urllib3.exceptions import SSLError, NewConnectionError
 
@@ -859,6 +860,47 @@ class TestDXFile(testutil.DXTestCaseCompat):
             dxpy.WORKSPACE_ID = workspace_id
 
         del os.environ['DX_JOB_ID']
+    
+    def test_dxfile_read_api_call_count(self):
+        """
+        Ensure that dxfile.read() does not make more than 4 API calls by counting DXHTTPRequest invocations
+        """
+        print(sys.version)
+        with testutil.temporary_project() as project:
+            # Create a random 1KiB upload string
+            upload_string = ''.join(random.choices(string.ascii_lowercase + string.digits, k=1024))
+            dxfile = dxpy.upload_string(upload_string, project=project.get_id(), wait_on_close=True)
+
+            # Patch DXHTTPRequest to count its invocations
+            with patch("dxpy.DXHTTPRequest", wraps=dxpy.DXHTTPRequest) as mock_request:
+                # DXFile instantiated with project-id
+                dxfile.read()
+                # Count the number of DXHTTPRequest calls
+                api_call_count = mock_request.call_count
+                print(f"dxfile.read() made {api_call_count} API calls")
+                self.assertLessEqual(api_call_count, 4, f"dxfile.read() made {api_call_count} API calls, exceeding the limit.")
+                mock_request.reset_mock()
+
+                # DXFile instantiated without project-id
+                dxfile = dxpy.DXFile(dxfile.get_id(), project=None)
+                dxfile.read()
+                api_call_count = mock_request.call_count
+                print(f"dxfile.read() made {api_call_count} API calls")
+                self.assertLessEqual(api_call_count, 4, f"dxfile.read() made {api_call_count} API calls, exceeding the limit.")
+                mock_request.reset_mock()
+
+                # Call DXFile.read() 1024 times
+                dxfile = dxpy.DXFile(dxfile.get_id(), project=project.get_id())
+                while True:
+                    data = dxfile.read(1)
+                    if not data:
+                        break
+                api_call_count = mock_request.call_count
+                print(f"dxfile.read() made {api_call_count} API calls")
+                self.assertLessEqual(api_call_count, 4, f"dxfile.read() made {api_call_count} API calls, exceeding the limit.")
+                mock_request.reset_mock()
+
+    
 
 
 class TestFolder(unittest.TestCase):
@@ -2551,26 +2593,6 @@ class TestHTTPResponses(testutil.DXTestCaseCompat):
             with self.assertRaisesRegex((SSLError, IOError, OpenSSL.SSL.Error), "file"):
                 dxpy.DXHTTPRequest("/system/whoami", {}, cert_file="nonexistent")
 
-    def test_fake_errors(self):
-        dxpy.DXHTTPRequest('/system/fakeError', {'errorType': 'Valid JSON'}, always_retry=True)
-
-        # Minimal latency with retries, in seconds. This makes sure we actually did a retry.
-        min_sec_with_retries = 1
-        max_num_retries = 2
-        start_time = time.time()
-        with self.assertRaises(ValueError):
-            dxpy.DXHTTPRequest('/system/fakeError', {'errorType': 'Invalid JSON'},
-                               max_retries=max_num_retries, always_retry=True)
-        end_time = time.time()
-        self.assertGreater(end_time - start_time, min_sec_with_retries)
-
-        start_time = time.time()
-        with self.assertRaises(ValueError):
-            dxpy.DXHTTPRequest('/system/fakeError', {'errorType': 'Error not decodeable'},
-                               max_retries=max_num_retries, always_retry=True)
-        end_time = time.time()
-        self.assertGreater(end_time - start_time, min_sec_with_retries)
-
     def test_system_headers_user_agent(self):
         headers = dxpy.api.system_headers()
         self.assertTrue('user-agent' in headers)

dxpy-0.392.0/dxpy/toolkit_version.py

@@ -1 +0,0 @@
-version = '0.392.0'