megadetector 5.0.6__py3-none-any.whl → 5.0.8__py3-none-any.whl

data_management/lila/download_lila_subset.py

@@ -9,7 +9,7 @@
 # what you want to query for, etc., is very application-specific; this is just meant as a 
 # demo.
 #
-# Can download from either Azure or GCP.
+# Can download from GCP (all datasets), AWS (all datasets), or Azure (most datasets).
 #
 ########
 
@@ -20,15 +20,16 @@ import random
 
 from tqdm import tqdm
 from multiprocessing.pool import ThreadPool
-from urllib.parse import urlparse
 from collections import defaultdict
 
-from data_management.lila.lila_common import \
-    read_lila_all_images_file, read_lila_metadata, is_empty, azure_url_to_gcp_http_url
+from data_management.lila.lila_common import read_lila_all_images_file, is_empty, lila_base_urls
 from md_utils.url_utils import download_url
 
+for s in lila_base_urls.values():
+    assert s.endswith('/')
+
 # If any of these strings appear in the common name of a species, we'll download that image
-species_of_interest = ['grey fox','red fox','leopard cat']
+species_of_interest = ['grey fox','gray fox','cape fox','red fox','kit fox']
 
 # We'll write images, metadata downloads, and temporary files here
 lila_local_base = os.path.expanduser('~/lila')
@@ -40,30 +41,74 @@ output_dir = os.path.join(lila_local_base,'lila_downloads_by_dataset')
 os.makedirs(output_dir,exist_ok=True)
 
 # Number of concurrent download threads
-n_download_threads = 50
+n_download_threads = 20
+
+verbose = False
 
 max_images_per_dataset = 10 # None
 
 # This impacts the data download, but not the metadata download
-image_download_source = 'azure' # 'azure' or 'gcp'
+#
+# Setting this to "Azure" really means "Azure if available"; some datasets are
+# not available on Azure.
+preferred_provider = 'gcp' # 'azure', 'gcp', 'aws'
 
 random.seed(0)
 
 
-#%% Download and open the giant table of image metadata
+#%% Support functions
 
-# Opening this huge .csv file make take ~30 seconds
-df = read_lila_all_images_file(metadata_dir)
+def download_relative_url(relative_url, output_base, provider='gcp', 
+                          verbose=False, overwrite=False):
+    """
+    Download a URL to output_base, preserving the path relative to the common LILA root.
+    """
+    
+    assert not relative_url.startswith('/')
+    
+    # Not all datasets are available on Azure, fall back in these cases.  The decision
+    # to fall back to GCP rather than AWS is arbitrary.
+    if provider == 'azure':
+        nominal_provider = relative_url_to_nominal_provider[relative_url]
+        if nominal_provider != 'azure':
+            if verbose:
+                print('URL {} not available on Azure, falling back to GCP'.format(
+                    relative_url))
+            provider = 'gcp'
+            
+    url = lila_base_urls[provider] + relative_url
+    
+    result = {'status':'unknown','url':url,'destination_filename':None}
+    
+    destination_filename = os.path.join(output_base,relative_url)
+    result['destination_filename'] = destination_filename
+    
+    if ((os.path.isfile(destination_filename)) and (not overwrite)):
+        result['status'] = 'skipped'
+        return result
+    try:
+        download_url(url, destination_filename, verbose=verbose, force_download=overwrite)
+    except Exception as e:
+        print('Warning: error downloading URL {}: {}'.format(
+            url,str(e)))     
+        result['status'] = 'error: {}'.format(str(e))
+        return result
+    
+    result['status'] = 'success'
+    return result
 
 
-#%% Download and parse the metadata file
+#%% Download and open the giant table of image URLs and labels
 
-metadata_table = read_lila_metadata(metadata_dir)
+# ~60 seconds to download, unzip, and open
+df = read_lila_all_images_file(metadata_dir)
 
 
 #%% Find all the images we want to download
 
-# Searching over the giant table can take a couple of minutes
+# ~2 minutes
+
+common_name_to_count = defaultdict(int)
 
 ds_name_to_urls = defaultdict(list)
 
@@ -77,6 +122,7 @@ def find_items(row):
     for species_name in species_of_interest:
         if species_name in row['common_name']:
             match = True
+            common_name_to_count[species_name] += 1
             break
     
     if match:
@@ -85,15 +131,19 @@ def find_items(row):
 tqdm.pandas()
 _ = df.progress_apply(find_items,axis=1)
 
+# We have a list of URLs for each dataset, flatten them all into a list of URLs
 all_urls = list(ds_name_to_urls.values())
 all_urls = [item for sublist in all_urls for item in sublist]
 print('Found {} matching URLs across {} datasets'.format(len(all_urls),len(ds_name_to_urls)))
 
+for common_name in common_name_to_count:
+    print('{}: {}'.format(common_name,common_name_to_count[common_name]))
+    
 from copy import deepcopy
 ds_name_to_urls_raw = deepcopy(ds_name_to_urls)
 
 
-#%% Trim to a fixed number of URLs per dataset
+#%% Optionally trim to a fixed number of URLs per dataset
 
 if max_images_per_dataset is None:
     pass
@@ -104,47 +154,90 @@ else:
             ds_name_to_urls[ds_name] = random.sample(ds_name_to_urls[ds_name],max_images_per_dataset)
 
 
-#%% Download those image files
+#%% Convert URLs to be relative to the common LILA base 
 
-def download_relative_filename(url, output_base, verbose=False, url_base=None):
-    """
-    Download a URL to output_base, preserving relative path
-    """
-    
-    if url_base is None:
-        url_base = '/'
-    assert url_base.startswith('/') and url_base.endswith('/')
-    
-    p = urlparse(url)
-    relative_filename = str(p.path)
-    # remove the leading '/'
-    assert relative_filename.startswith(url_base)
-    relative_filename = relative_filename.replace(url_base,'',1)        
-    
-    destination_filename = os.path.join(output_base,relative_filename)
-    download_url(url, destination_filename, verbose=verbose)
-    
 all_urls = list(ds_name_to_urls.values())
 all_urls = [item for sublist in all_urls for item in sublist]
 
-url_base = '/'
+all_urls_relative = []
 
-# Convert Azure URLs to GCP URLs if necessary
-if image_download_source != 'azure':
-    assert image_download_source == 'gcp'
-    url_base = '/public-datasets-lila/'
-    all_urls = [azure_url_to_gcp_http_url(url) for url in all_urls]    
+# Each file has a nominal URL in the .csv file.  For now, the only thing this tells is
+# is that if the nominal URL isn't an Azure URL, the file isn't on Azure.  All files are on
+# GCP and AWS.
+#
+# Keep track of the nominal provider for each URL.
+relative_url_to_nominal_provider = {}
+
+for url in all_urls:
+    found_base = False
+    for provider in lila_base_urls.keys():
+        base = lila_base_urls[provider]
+        if url.startswith(base):
+            relative_url = url.replace(base,'')
+            all_urls_relative.append(relative_url)
+            relative_url_to_nominal_provider[relative_url] = provider
+            found_base = True
+            break
+    assert found_base
+    
+assert len(all_urls) == len(all_urls_relative)
+
+
+#%% Download image files
 
-print('Downloading {} images with Python requests'.format(len(all_urls)))
+print('Downloading {} images on {} workers, preferred provider is {}'.format(
+    len(all_urls),n_download_threads,preferred_provider))
 
 if n_download_threads <= 1:
 
-    # url = all_urls[0]
-    for url in tqdm(all_urls):        
-        download_relative_filename(url,output_dir,verbose=True,url_base=url_base)
+    results = []
+    
+    # url_relative = all_urls_relative[0]
+    for url_relative in tqdm(all_urls_relative):        
+        result = download_relative_url(url_relative,
+                                       output_base=output_dir,
+                                       provider=preferred_provider,
+                                       verbose=verbose)
+        results.append(result)
     
 else:
 
     pool = ThreadPool(n_download_threads)        
-    tqdm(pool.imap(lambda s: download_relative_filename(s,output_dir,verbose=False,url_base=url_base), 
-                   all_urls), total=len(all_urls))
+    results = list(tqdm(pool.imap(lambda s: download_relative_url(
+        s,output_base=output_dir,provider=preferred_provider,verbose=verbose),
+        all_urls_relative), total=len(all_urls_relative)))
+
+
+#%% Scrap
+
+if False:
+    
+    pass
+
+    #%% Find all the reptiles on LILA
+
+    reptile_rows = df.loc[df['class'] == 'reptilia']
+    
+    # i_row = 0; row = reptile_rows.iloc[i_row]
+    
+    common_name_to_count = defaultdict(int)
+    dataset_to_count = defaultdict(int)
+    for i_row,row in reptile_rows.iterrows():
+        common_name_to_count[row['common_name']] += 1
+        dataset_to_count[row['dataset_name']] += 1
+        
+    from md_utils.ct_utils import sort_dictionary_by_value
+    
+    print('Found {} reptiles\n'.format(len(reptile_rows)))
+    
+    common_name_to_count = sort_dictionary_by_value(common_name_to_count,reverse=True)
+    dataset_to_count = sort_dictionary_by_value(dataset_to_count,reverse=True)
+    
+    print('Common names by count:\n')
+    for k in common_name_to_count:
+        print('{} ({})'.format(k,common_name_to_count[k]))
+    
+    print('\nDatasets by count:\n')    
+    for k in dataset_to_count:
+        print('{} ({})'.format(k,dataset_to_count[k]))
+    

data_management/lila/generate_lila_per_image_labels.py

@@ -22,6 +22,9 @@ import json
 import pandas as pd
 import numpy as np
 import dateparser
+import csv
+import urllib
+import urllib.request
 
 from collections import defaultdict
 from tqdm import tqdm
@@ -30,6 +33,9 @@ from data_management.lila.lila_common import read_lila_metadata, \
     read_metadata_file_for_dataset, \
     read_lila_taxonomy_mapping
 
+from md_utils import write_html_image_list
+from md_utils.path_utils import zip_file
+from md_utils.path_utils import open_file
 from md_utils.url_utils import download_url
 
 # We'll write images, metadata downloads, and temporary files here
@@ -56,7 +62,7 @@ ds_name_to_annotation_level['NACTI'] = 'unknown'
 
 known_unmapped_labels = set(['WCS Camera Traps:#ref!'])
 
-debug_max_images_per_dataset = 0
+debug_max_images_per_dataset = -1
 if debug_max_images_per_dataset > 0:
     print('Running in debug mode')
     output_file = output_file.replace('.csv','_debug.csv')
@@ -72,7 +78,7 @@ if False:
     metadata_table = {k:metadata_table[k]}
 
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['metadata_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -101,8 +107,6 @@ for i_row,row in taxonomy_df.iterrows():
 
 # Takes several hours
 
-import csv
-
 header = ['dataset_name','url','image_id','sequence_id','location_id','frame_num','original_label',\
           'scientific_name','common_name','datetime','annotation_level']
 
@@ -122,7 +126,7 @@ def clearnan(v):
     assert isinstance(v,str)
     return v
 
-with open(output_file,'w') as f:
+with open(output_file,'w',encoding='utf-8',newline='') as f:
     
     csv_writer = csv.writer(f)
     csv_writer.writerow(header)
@@ -334,6 +338,8 @@ with open(output_file,'w') as f:
 
 # ...with open()
 
+print('Processed {} datasets'.format(len(metadata_table)))
+
 
 #%% Read the .csv back
 
@@ -352,6 +358,8 @@ def isint(v):
 
 valid_annotation_levels = set(['sequence','image','unknown'])
 
+# Collect a list of locations within each dataset; we'll use this
+# in the next cell to look for datasets that only have a single location
 dataset_name_to_locations = defaultdict(set)
 
 def check_row(row):
@@ -386,6 +394,8 @@ else:
 
 #%% Check for datasets that have only one location string
 
+# Expected: ENA24, Missouri Camera Traps
+
 for ds_name in dataset_name_to_locations.keys():
     if len(dataset_name_to_locations[ds_name]) == 1:
         print('No location information for {}'.format(ds_name))
@@ -440,8 +450,8 @@ print('Selected {} total images'.format(len(images_to_download)))
 
 # Expect a few errors for images with human or vehicle labels (or things like "ignore" that *could* be humans)
 
-import urllib.request
-
+# TODO: trivially parallelizable
+#
 # i_image = 10; image = images_to_download[i_image]
 for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_download)):
     
@@ -450,17 +460,17 @@ for i_image,image in tqdm(enumerate(images_to_download),total=len(images_to_down
     image_file = os.path.join(preview_folder,'image_{}'.format(str(i_image).zfill(4)) + ext)
     relative_file = os.path.relpath(image_file,preview_folder)
     try:
-        download_url(url,output_file,verbose=False)
+        download_url(url,image_file,verbose=False)
         image['relative_file'] = relative_file
     except urllib.error.HTTPError:
         print('Image {} does not exist ({}:{})'.format(
             i_image,image['dataset_name'],image['original_label']))
         image['relative_file'] = None
 
+# ...for each image we need to download
 
-#%% Write preview HTML
 
-from md_utils import write_html_image_list
+#%% Write preview HTML
 
 html_filename = os.path.join(preview_folder,'index.html')
 
@@ -475,19 +485,18 @@ for im in images_to_download:
     output_im = {}
     output_im['filename'] = im['relative_file']
     output_im['linkTarget'] = im['url']
-    output_im['title'] = str(im)
+    output_im['title'] = '<b>{}: {}</b><br/><br/>'.format(im['dataset_name'],im['original_label']) + str(im)
     output_im['imageStyle'] = 'width:600px;'
     output_im['textStyle'] = 'font-weight:normal;font-size:100%;'
     html_images.append(output_im)
     
 write_html_image_list.write_html_image_list(html_filename,html_images)
 
-from md_utils.path_utils import open_file
 open_file(html_filename)
 
 
 #%% Zip output file
 
-from md_utils.path_utils import zip_file
+zipped_output_file = zip_file(output_file,verbose=True)
 
-zip_file(output_file,verbose=True)
+print('Zipped {} to {}'.format(output_file,zipped_output_file))

data_management/lila/get_lila_annotation_counts.py

@@ -34,18 +34,9 @@ os.makedirs(metadata_dir,exist_ok=True)
 
 output_file = os.path.join(output_dir,'lila_dataset_to_categories.json')
 
-# Created by get_lila_category_list.py... contains counts for each category
-category_list_dir = os.path.join(lila_local_base,'lila_categories_list')
-lila_dataset_to_categories_file = os.path.join(category_list_dir,'lila_dataset_to_categories.json')
-
-assert os.path.isfile(lila_dataset_to_categories_file)
-
 
 #%% Load category and taxonomy files
 
-with open(lila_dataset_to_categories_file,'r') as f:
-    lila_dataset_to_categories = json.load(f)
-
 taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
 
 
@@ -55,9 +46,13 @@ ds_query_to_scientific_name = {}
 
 unmapped_queries = set()
 
+datasets_with_taxonomy_mapping = set()
+
 # i_row = 1; row = taxonomy_df.iloc[i_row]; row
 for i_row,row in taxonomy_df.iterrows():
     
+    datasets_with_taxonomy_mapping.add(row['dataset_name'])
+    
     ds_query = row['dataset_name'] + ':' + row['query']
     ds_query = ds_query.lower()
     
@@ -68,13 +63,17 @@ for i_row,row in taxonomy_df.iterrows():
         
     ds_query_to_scientific_name[ds_query] = row['scientific_name']
     
+print('Loaded taxonomy mappings for {} datasets'.format(len(datasets_with_taxonomy_mapping)))
     
+
 #%% Download and parse the metadata file
 
 metadata_table = read_lila_metadata(metadata_dir)
 
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
 
-#%% Download and extract metadata for the datasets we're interested in
+#%% Download and extract metadata for each dataset
 
 for ds_name in metadata_table.keys():    
     metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
@@ -91,6 +90,11 @@ dataset_to_categories = {}
 # ds_name = 'NACTI'
 for ds_name in metadata_table.keys():
     
+    taxonomy_mapping_available = (ds_name in datasets_with_taxonomy_mapping)
+    
+    if not taxonomy_mapping_available:
+        print('Warning: taxonomy mapping not available for {}'.format(ds_name))
+        
     print('Finding categories in {}'.format(ds_name))
     
     json_filename = metadata_table[ds_name]['json_filename']
@@ -122,6 +126,8 @@ for ds_name in metadata_table.keys():
        # always redundant with the class-level data sets.
        if 'bbox' in ds_name:
            c['scientific_name_from_taxonomy_mapping'] = None
+       elif not taxonomy_mapping_available:
+           c['scientific_name_from_taxonomy_mapping'] = None
        else:
            taxonomy_query_string = ds_name.lower().strip() + ':' + c['name'].lower()
            if taxonomy_query_string not in ds_query_to_scientific_name:

data_management/lila/lila_common.py

@@ -21,7 +21,7 @@ from md_utils.path_utils import unzip_file
 
 # LILA camera trap primary metadata file
 lila_metadata_url = 'http://lila.science/wp-content/uploads/2023/06/lila_camera_trap_datasets.csv'
-lila_taxonomy_mapping_url = 'https://lila.science/wp-content/uploads/2022/07/lila-taxonomy-mapping_release.csv'
+lila_taxonomy_mapping_url = 'https://lila.science/public/lila-taxonomy-mapping_release.csv'
 lila_all_images_url = 'https://lila.science/public/lila_image_urls_and_labels.csv.zip'
 
 wildlife_insights_page_size = 30000
@@ -31,9 +31,13 @@ wildlife_insights_taxonomy_local_json_filename = 'wi_taxonomy.json'
 wildlife_insights_taxonomy_local_csv_filename = \
     wildlife_insights_taxonomy_local_json_filename.replace('.json','.csv')
 
-lila_azure_storage_account = 'https://lilablobssc.blob.core.windows.net'
-gcp_bucket_api_url = 'https://storage.googleapis.com/public-datasets-lila'
-gcp_bucket_gs_url = 'gs://public-datasets-lila'
+# Filenames are consistent across clouds relative to these URLs
+lila_base_urls = {
+    'azure':'https://lilablobssc.blob.core.windows.net/',
+    'gcp':'https://storage.googleapis.com/public-datasets-lila/',
+    'aws':'http://us-west-2.opendata.source.coop.s3.amazonaws.com/agentmorris/lila-wildlife/'
+}
+
 
 
 #%% Common functions
@@ -165,16 +169,18 @@ def read_lila_all_images_file(metadata_dir):
     return df
 
 
-def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
+def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None,json_url=None):
     """
     Downloads if necessary - then unzips if necessary - the .json file for a specific dataset.
     Returns the .json filename on the local disk.
     """
     
-    if metadata_table is None:
-        metadata_table = read_lila_metadata(metadata_dir)
+    if json_url is None:
         
-    json_url = metadata_table[ds_name]['metadata_url']
+        if metadata_table is None:
+            metadata_table = read_lila_metadata(metadata_dir)
+            
+        json_url = metadata_table[ds_name]['metadata_url']
     
     p = urlparse(json_url)
     json_filename = os.path.join(metadata_dir,os.path.basename(p.path))
@@ -196,27 +202,6 @@ def read_metadata_file_for_dataset(ds_name,metadata_dir,metadata_table=None):
     return json_filename
 
 
-def azure_url_to_gcp_http_url(url):
-    """
-    Most URLs point to Azure by default, but most files are available on both Azure and GCP.
-    This function converts an Azure URL to the corresponding GCP http:// url.
-    """
-    
-    assert url.startswith(lila_azure_storage_account)
-    gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
-    return gcp_url
-
-
-def azure_url_to_gcp_gs_url(url):
-    """
-    Most URLs point to Azure by default, but most files are available on both Azure and GCP.
-    This function converts an Azure URL to the corresponding GCP gs:// url.
-    """
-    
-    return azure_url_to_gcp_http_url(url).replace(gcp_bucket_api_url,
-                                                  gcp_bucket_gs_url,1)
-
-
 #%% Interactive test driver
 
 if False:
@@ -249,16 +234,4 @@ if False:
             urls_to_test.append(ds_info['bbox_url'])
             
     status_codes = url_utils.test_urls(urls_to_test)    
-    
-    
-    #%% Verify that the GCP versions of all metadata files exist
-    
-    gcp_urls = []
-    
-    # url = urls_to_test[0]
-    for url in urls_to_test:
-        assert url.startswith(lila_azure_storage_account)
-        gcp_url = url.replace(lila_azure_storage_account,gcp_bucket_api_url,1)
-        gcp_urls.append(gcp_url)
-        
-    status_codes = url_utils.test_urls(gcp_urls)
+    

data_management/lila/test_lila_metadata_urls.py

@@ -0,0 +1,116 @@
+########
+#
+# test_lila_metadata_urls.py
+#
+# Test that all the metadata URLs for LILA camera trap datasets are valid, and 
+# test that at least one image within each URL is valid, including MegaDetector results
+# files.
+#
+########
+
+#%% Constants and imports
+
+import json
+import os
+
+from data_management.lila.lila_common import read_lila_metadata,\
+    read_metadata_file_for_dataset, read_lila_taxonomy_mapping
+
+# We'll write images, metadata downloads, and temporary files here
+lila_local_base = os.path.expanduser('~/lila')
+
+output_dir = os.path.join(lila_local_base,'lila_metadata_tests')
+os.makedirs(output_dir,exist_ok=True)
+
+metadata_dir = os.path.join(lila_local_base,'metadata')
+os.makedirs(metadata_dir,exist_ok=True)
+
+md_results_dir = os.path.join(lila_local_base,'md_results')
+os.makedirs(md_results_dir,exist_ok=True)
+
+md_results_keys = ['mdv4_results_raw','mdv5a_results_raw','mdv5b_results_raw','md_results_with_rde']
+
+
+#%% Load category and taxonomy files
+
+taxonomy_df = read_lila_taxonomy_mapping(metadata_dir)
+
+
+#%% Download and parse the metadata file
+
+metadata_table = read_lila_metadata(metadata_dir)
+
+print('Loaded metadata URLs for {} datasets'.format(len(metadata_table)))
+
+
+#%% Download and extract metadata and MD results for each dataset
+
+for ds_name in metadata_table.keys():    
+
+    metadata_table[ds_name]['json_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                         metadata_dir=metadata_dir,
+                                                                         metadata_table=metadata_table)
+    for k in md_results_keys:
+        md_results_url = metadata_table[ds_name][k]
+        if md_results_url is None:
+            metadata_table[ds_name][k + '_filename'] = None
+        else:
+            metadata_table[ds_name][k + '_filename'] = read_metadata_file_for_dataset(ds_name=ds_name,
+                                                                        metadata_dir=md_results_dir,
+                                                                        json_url=md_results_url)
+
+
+#%% Build up a list of URLs to test
+
+url_to_source = {}
+
+# The first image in a dataset is disproportionately likely to be human (and thus 404)
+image_index = 1000
+
+# ds_name = list(metadata_table.keys())[0]
+for ds_name in metadata_table.keys():
+    
+    if 'bbox' in ds_name:
+        print('Skipping bbox dataset {}'.format(ds_name))
+        continue
+
+    print('Processing dataset {}'.format(ds_name))
+    
+    json_filename = metadata_table[ds_name]['json_filename']
+    with open(json_filename, 'r') as f:
+        data = json.load(f)
+
+    image_base_url = metadata_table[ds_name]['image_base_url']
+    assert not image_base_url.endswith('/')
+    # Download a test image
+    test_image_relative_path = data['images'][image_index]['file_name']
+    test_image_url = image_base_url + '/' + test_image_relative_path
+    
+    url_to_source[test_image_url] = ds_name + ' metadata'
+    
+    # k = md_results_keys[2]
+    for k in md_results_keys:
+        k_fn = k + '_filename'
+        if metadata_table[ds_name][k_fn] is not None:
+            with open(metadata_table[ds_name][k_fn],'r') as f:
+                md_results = json.load(f)
+                im = md_results['images'][image_index]
+                md_image_url = image_base_url + '/' + im['file']
+                url_to_source[md_image_url] = ds_name + ' ' + k
+    
+# ...for each dataset
+
+
+#%% Test URLs
+
+from md_utils.url_utils import test_urls
+
+urls_to_test = sorted(url_to_source.keys())
+urls_to_test = [fn.replace('\\','/') for fn in urls_to_test]
+
+status_codes = test_urls(urls_to_test,error_on_failure=False)
+
+for i_url,url in enumerate(urls_to_test):
+    if status_codes[i_url] != 200:
+        print('Status {} for {} ({})'.format(
+            status_codes[i_url],url,url_to_source[url]))