megadetector 5.0.7__py3-none-any.whl → 5.0.9__py3-none-any.whl

md_utils/url_utils.py

@@ -1,10 +1,10 @@
-########
-#
-# url_utils.py
-#
-# Frequently-used functions for downloading or manipulating URLs
-#
-########
+"""
+
+url_utils.py
+
+Frequently-used functions for downloading or manipulating URLs
+
+"""
 
 #%% Imports and constants
 
@@ -14,8 +14,11 @@ import urllib
 import tempfile
 import requests
 
+from functools import partial
 from tqdm import tqdm
 from urllib.parse import urlparse
+from multiprocessing.pool import ThreadPool
+from multiprocessing.pool import Pool
 
 url_utils_temp_dir = None
 max_path_len = 255
@@ -25,6 +28,8 @@ max_path_len = 255
 
 class DownloadProgressBar():
     """
+    Progress updater based on the progressbar2 package.
+    
     https://stackoverflow.com/questions/37748105/how-to-use-progressbar-module-with-urlretrieve
     """
     
@@ -48,7 +53,15 @@ class DownloadProgressBar():
             
 
 def get_temp_folder(preferred_name='url_utils'):
-
+    """
+    Gets a temporary folder for use within this module.
+    
+    Args:
+        preferred_name (str, optional): subfolder to use within the system temp folder
+        
+    Returns:
+        str: the full path to the temporary subfolder
+    """
     global url_utils_temp_dir
     
     if url_utils_temp_dir is None:
@@ -58,15 +71,31 @@ def get_temp_folder(preferred_name='url_utils'):
     return url_utils_temp_dir
     
            
-def download_url(url, destination_filename=None, progress_updater=None, 
-                 force_download=False, verbose=True):
+def download_url(url, 
+                 destination_filename=None, 
+                 progress_updater=None, 
+                 force_download=False, 
+                 verbose=True):
     """
-    Download a URL to a file.  If no file is specified, creates a temporary file, 
-    with a semi-best-effort to avoid filename collisions.
+    Downloads a URL to a file.  If no file is specified, creates a temporary file, 
+    making a best effort to avoid filename collisions.
     
     Prints some diagnostic information and makes sure to omit SAS tokens from printouts.
     
-    progress_updater can be "None", "True", or a specific callback.
+    Args:
+        url (str): the URL to download
+        destination_filename (str, optional): the target filename; if None, will create
+            a file in system temp space        
+        progress_updater (object or bool, optional): can be "None", "False", "True", or a 
+            specific callable object.  If None or False, no progress updated will be 
+            displayed.  If True, a default progress bar will be created.
+        force_download (bool, optional): download this file even if [destination_filename]
+            exists.
+        verbose (bool, optional): enable additional debug console output
+        
+    Returns:
+        str: the filename to which [url] was downloaded, the same as [destination_filename]
+        if [destination_filename] was not None
     """
     
     if progress_updater is not None and isinstance(progress_updater,bool):
@@ -109,34 +138,186 @@ def download_url(url, destination_filename=None, progress_updater=None,
 
 def download_relative_filename(url, output_base, verbose=False):
     """
-    Download a URL to output_base, preserving relative path
+    Download a URL to output_base, preserving relative path.  Path is relative to 
+    the site, so:
+        
+        https://abc.com/xyz/123.txt
+    
+    ...will get downloaded to:
+        
+        output_base/xyz/123.txt        
+        
+    Args:
+        url (str): the URL to download
+        output_base (str): the base folder to which we should download this file
+        verbose (bool, optional): enable additional debug console output
+        
+    Returns:
+        str: the local destination filename
     """
     
     p = urlparse(url)
     # remove the leading '/'
     assert p.path.startswith('/'); relative_filename = p.path[1:]
     destination_filename = os.path.join(output_base,relative_filename)
-    download_url(url, destination_filename, verbose=verbose)
+    return download_url(url, destination_filename, verbose=verbose)
 
 
-def test_urls(urls, error_on_failure=True):
+def _do_parallelized_download(download_info,overwrite=False,verbose=False):
     """
-    Verify that a list of URLs is available (returns status 200).  By default,
-    errors if any URL is unavailable.  If error_on_failure is False, returns
-    status codes for each URL.
+    Internal function for download parallelization.
+    """
+    
+    url = download_info['url']
+    target_file = download_info['target_file']
+    result = {'status':'unknown','url':url,'target_file':target_file}
+    
+    if ((os.path.isfile(target_file)) and (not overwrite)):
+        if verbose:
+            print('Skipping existing file {}'.format(target_file))
+        result['status'] = 'skipped'
+        return result
+    try:
+        download_url(url=url, 
+                     destination_filename=target_file,
+                     verbose=verbose, 
+                     force_download=overwrite)
+    except Exception as e:
+        print('Warning: error downloading URL {}: {}'.format(
+            url,str(e)))     
+        result['status'] = 'error: {}'.format(str(e))
+        return result
+    
+    result['status'] = 'success'
+    return result
+
+
+def parallel_download_urls(url_to_target_file,verbose=False,overwrite=False,
+                           n_workers=20,pool_type='thread'):
+    """
+    Downloads a list of URLs to local files.
+    
+    Catches exceptions and reports them in the returned "results" array.    
     
-    TODO: trivially parallelizable.
+    Args:
+        url_to_target_file: a dict mapping URLs to local filenames.
+        verbose (bool, optional): enable additional debug console output
+        overwrite (bool, optional): whether to overwrite existing local files
+        n_workers (int, optional): number of concurrent workers, set to <=1 to disable
+            parallelization
+        pool_type (str, optional): worker type to use; should be 'thread' or 'process'
+        
+    Returns:
+        list: list of dicts with keys:
+            - 'url': the url this item refers to
+            - 'status': 'skipped', 'success', or a string starting with 'error'
+            - 'target_file': the local filename to which we downloaded (or tried to 
+              download) this URL            
     """
     
-    status_codes = []
+    all_download_info = []
     
-    for url in tqdm(urls):
+    print('Preparing download list')
+    for url in tqdm(url_to_target_file):
+        download_info = {}
+        download_info['url'] = url
+        download_info['target_file'] = url_to_target_file[url]
+        all_download_info.append(download_info)
         
-        r = requests.get(url)
+    print('Downloading {} images on {} workers'.format(
+        len(all_download_info),n_workers))
+
+    if n_workers <= 1:
+
+        results = []
         
-        if error_on_failure and r.status_code != 200:        
-            raise ValueError('Could not access {}: error {}'.format(url,r.status_code))
-        status_codes.append(r.status_code)
+        for download_info in tqdm(all_download_info):        
+            result = _do_parallelized_download(download_info,overwrite=overwrite,verbose=verbose)
+            results.append(result)
         
-    return status_codes
+    else:
 
+        if pool_type == 'thread':
+            pool = ThreadPool(n_workers)
+        else:
+            assert pool_type == 'process', 'Unsupported pool type {}'.format(pool_type)
+            pool = Pool(n_workers)
+        
+        print('Starting a {} pool with {} workers'.format(pool_type,n_workers))
+        
+        results = list(tqdm(pool.imap(
+            partial(_do_parallelized_download,overwrite=overwrite,verbose=verbose),
+            all_download_info), total=len(all_download_info)))
+                
+    return results
+
+    
+def test_url(url, error_on_failure=True, timeout=None):
+    """
+    Tests the availability of [url], returning an http status code.
+    
+    Args:
+        url (str): URL to test
+        error_on_failure (bool, optional): whether to error (vs. just returning an
+            error code) if accessing this URL fails
+        timeout (int, optional): timeout in seconds to wait before considering this 
+            access attempt to be a failure; see requests.head() for precise documentation
+    
+    Returns:
+        int: http status code (200 for success)
+    """
+    
+    # r = requests.get(url, stream=True, verify=True, timeout=timeout)
+    r = requests.head(url, stream=True, verify=True, timeout=timeout)
+    
+    if error_on_failure and r.status_code != 200:        
+        raise ValueError('Could not access {}: error {}'.format(url,r.status_code))
+    return r.status_code
+    
+
+def test_urls(urls, error_on_failure=True, n_workers=1, pool_type='thread', timeout=None):
+    """
+    Verify that URLs are available (i.e., returns status 200).  By default,
+    errors if any URL is unavailable.  
+    
+    Args:
+        urls (list): list of URLs to test
+        error_on_failure (bool, optional): whether to error (vs. just returning an
+            error code) if accessing this URL fails
+        n_workers (int, optional): number of concurrent workers, set to <=1 to disable
+            parallelization
+        pool_type (str, optional): worker type to use; should be 'thread' or 'process'
+        timeout (int, optional): timeout in seconds to wait before considering this 
+            access attempt to be a failure; see requests.head() for precise documentation
+    
+    Returns:
+        list: a list of http status codes, the same length and order as [urls]
+    """
+        
+    if n_workers <= 1:
+
+        status_codes = []
+        
+        for url in tqdm(urls):
+            
+            r = requests.get(url, timeout=timeout)
+            
+            if error_on_failure and r.status_code != 200:        
+                raise ValueError('Could not access {}: error {}'.format(url,r.status_code))
+            status_codes.append(r.status_code)
+                
+    else:
+
+        if pool_type == 'thread':
+            pool = ThreadPool(n_workers)
+        else:
+            assert pool_type == 'process', 'Unsupported pool type {}'.format(pool_type)
+            pool = Pool(n_workers)
+        
+        print('Starting a {} pool with {} workers'.format(pool_type,n_workers))
+        
+        status_codes = list(tqdm(pool.imap(
+            partial(test_url,error_on_failure=error_on_failure,timeout=timeout),
+            urls), total=len(urls)))
+                
+    return status_codes

md_utils/write_html_image_list.py

@@ -1,14 +1,14 @@
-########
-#
-# write_html_image_list.py
-#
-# Given a list of image file names, writes an HTML file that
-# shows all those images, with optional one-line headers above each.
-#
-# Each "filename" can also be a dict with elements 'filename','title',
-# 'imageStyle','textStyle', 'linkTarget'
-#
-########
+"""
+
+write_html_image_list.py
+
+Given a list of image file names, writes an HTML file that
+shows all those images, with optional one-line headers above each.
+
+Each "filename" can also be a dict with elements 'filename','title',
+'imageStyle','textStyle', 'linkTarget'
+
+"""
 
 #%% Constants and imports
 
@@ -23,26 +23,34 @@ from md_utils import path_utils
 
 def write_html_image_list(filename=None,images=None,options=None):
     """
-    filename: the output file
-    
-    image: a list of image filenames or dictionaries with one or more of the following fields:
-        
-        filename
-        imageStyle
-        textStyle
-        title
-        linkTarget
-        
-    options: a dict with one or more of the following fields:
-        
-        fHtml
-        headerHtml
-        trailerHtml
-        defaultTextStyle
-        defaultImageStyle
-        maxFiguresPerHtmlFile
-        urlEncodeFilenames (default True, e.g. '#' will be replaced by '%23')
-        
+    Given a list of image file names, writes an HTML file that shows all those images, 
+    with optional one-line headers above each.
+
+    Args:
+        filename (str, optional): the .html output file; if None, just returns a valid 
+            options dict
+        images (list, optional): the images to write to the .html file; if None, just returns 
+            a valid options dict.  This can be a flat list of image filenames, or this can
+            be a list of dictionaries with one or more of the following fields:
+                
+            - filename (image filename) (required, all other fields are optional)
+            - imageStyle (css style for this image)
+            - textStyle (css style for the title associated with this image)
+            - title (text label for this image)
+            - linkTarget (URL to which this image should link on click)
+            
+        options (dict, optional): a dict with one or more of the following fields:        
+            
+            - fHtml (file pointer to write to, used for splitting write operations over multiple calls)
+            - headerHtml (html text to include before the image list)
+            - trailerHtml (html text to include after the image list)
+            - defaultImageStyle (default css style for images)
+            - defaultTextStyle (default css style for image titles)
+            - maxFiguresPerHtmlFile (max figures for a single HTML file; overflow will be handled by creating
+              multiple files and a TOC with links)
+            - urlEncodeFilenames (default True, e.g. '#' will be replaced by '%23')
+            - urlEncodeLinkTargets (default True, e.g. '#' will be replaced by '%23')
+            
     """
     
     # returns an options struct
@@ -68,13 +76,16 @@ def write_html_image_list(filename=None,images=None,options=None):
     
     if 'urlEncodeFilenames' not in options or options['urlEncodeFilenames'] is None:
         options['urlEncodeFilenames'] = True
-        
+    
+    if 'urlEncodeLinkTargets' not in options or options['urlEncodeLinkTargets'] is None:
+        options['urlEncodeLinkTargets'] = True
+    
     # Possibly split the html output for figures into multiple files; Chrome gets sad with
     # thousands of images in a single tab.        
     if 'maxFiguresPerHtmlFile' not in options or options['maxFiguresPerHtmlFile'] is None:
         options['maxFiguresPerHtmlFile'] = math.inf    
     
-    if filename is None:
+    if filename is None or images is None:
         return options
     
     # images may be a list of images or a list of image/style/title dictionaries, 
@@ -176,8 +187,8 @@ def write_html_image_list(filename=None,images=None,options=None):
         title = title.encode('ascii','ignore').decode('ascii')
         filename = filename.encode('ascii','ignore').decode('ascii')
         
-        if options['urlEncodeFilenames']:
-            filename = filename.replace('\\','/')
+        filename = filename.replace('\\','/')
+        if options['urlEncodeFilenames']:            
             filename = urllib.parse.quote(filename)
         
         if len(title) > 0:       
@@ -185,6 +196,11 @@ def write_html_image_list(filename=None,images=None,options=None):
                     '<p style="{}">{}</p>\n'\
                     .format(textStyle,title))            
 
+        linkTarget = linkTarget.replace('\\','/')
+        if options['urlEncodeLinkTargets']:
+            # These are typically absolute paths, so we only want to mess with certain characters
+            linkTarget = urllib.parse.quote(linkTarget,safe=':/')
+            
         if len(linkTarget) > 0:
             fHtml.write('<a href="{}">'.format(linkTarget))
             # imageStyle.append(';border:0px;')