fusion-platform-python-sdk 1.15.12__tar.gz → 1.16.6__tar.gz

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/LICENSE.txt

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Digital Content Analysis Technology Ltd
+Copyright (c) 2025 Digital Content Analysis Technology Ltd
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{fusion_platform_python_sdk-1.15.12/fusion_platform_python_sdk.egg-info → fusion_platform_python_sdk-1.16.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: fusion-platform-python-sdk
-Version: 1.15.12
+Version: 1.16.6
 Summary: Python SDK used to interact with the Fusion Platform(r)
 Home-page: https://github.com/d-cat-support/fusion-platform-python-sdk
 Author: Digital Content Analysis Technology Ltd
@@ -24,6 +24,15 @@ Requires-Dist: python-i18n[YAML]
 Requires-Dist: requests
 Requires-Dist: tenacity
 Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # Fusion Platform<sup>&reg;</sup> Python SDK
 

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/__init__.py

@@ -11,8 +11,8 @@ API, and this is defined within the corresponding OpenAPI 3.0 specification, whi
 """
 
 # Do not modify the following two lines as they are maintained by the version.sh script.
-__version__ = '1.15.12'
-__version_date__ = '2024-12-20T11:33:56Z'
+__version__ = '1.16.6'
+__version_date__ = '2025-02-11T12:17:15Z'
 
 # Exclude certain sub-modules from documentation.
 # @formatter:off

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/command.py

@@ -126,6 +126,11 @@ class Command(Base):
     _YAML_OPTIONS = 'options'
     _YAML_DISPATCHERS = 'dispatchers'
 
+    # Progress bar constants.
+    __PROGRESS_BAR_FORMAT = '{l_bar}{bar:50}{r_bar}'
+    __PROGRESS_BAR_COLOUR_ACTIVE = '#3379B6'
+    __PROGRESS_BAR_COLOUR_COMPLETE = '#5BB85C'
+
     def __init__(self):
         """
         Initialises the object.
@@ -271,26 +276,33 @@ class Command(Base):
             extracted_inputs = [(k + 1, input) for k, input in enumerate(process.inputs)]
 
             self.__print(logging.INFO, i18n.t('command.define_inputs', files=len(extracted_inputs)))
-            progress_bar = tqdm(total=len(extracted_inputs), ncols=100)
-
-            for k, input in extracted_inputs:
-                # Get the data item for this input.
-                data, _ = organisation.find_data(id=input.id)
-                data.check_analysis_complete(wait=True)
-
-                # Get the list of files which we filter to only use those files that are not sources. This will include any files which do not have an alternative.
-                data_files = [data_file for data_file in data.files if data_file.file_type.lower().strip() == input.file_type.lower().strip()]
-                file_ids_to_ignore = [str(data_file.source) for data_file in data_files if hasattr(data_file, 'source') and (data_file.source is not None)]
-                data_files = [data_file for data_file in data_files if str(data_file.file_id) not in file_ids_to_ignore]
+            progress_bar = tqdm(total=len(extracted_inputs), bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE,
+                                unit='input')  # Unit matches keyword argument.
+            progress_bar.set_postfix(input=1)
 
-                # Build the list of downloads. Here we take just the first file found on the assumption that there should be only one.
-                if len(data_files) > 0:
-                    downloads.append((data_files[0], data_files[0].file_name))
-                    inputs.append({Model._FIELD_NAME: data.name, Model._FIELD_FILE: data_files[0].file_name})
-
-                progress_bar.update(1)
+            try:
+                for k, input in extracted_inputs:
+                    # Get the data item for this input.
+                    data, _ = organisation.find_data(id=input.id)
+                    data.check_analysis_complete(wait=True)
+
+                    # Get the list of files which we filter to only use those files that are not sources. This will include any files which do not have an alternative.
+                    data_files = [data_file for data_file in data.files if data_file.file_type.lower().strip() == input.file_type.lower().strip()]
+                    file_ids_to_ignore = [str(data_file.source) for data_file in data_files if hasattr(data_file, 'source') and (data_file.source is not None)]
+                    data_files = [data_file for data_file in data_files if str(data_file.file_id) not in file_ids_to_ignore]
+
+                    # Build the list of downloads. Here we take just the first file found on the assumption that there should be only one.
+                    if len(data_files) > 0:
+                        downloads.append((data_files[0], data_files[0].file_name, data_files[0].size))
+                        inputs.append({Model._FIELD_NAME: data.name, Model._FIELD_FILE: data_files[0].file_name})
+
+                    progress_bar.set_postfix(input=k)
+                    progress_bar.update(1)
 
-            progress_bar.close()
+            finally:
+                progress_bar.set_postfix()
+                progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+                progress_bar.close()
 
         else:
             # Just get the names of the inputs.
@@ -306,29 +318,36 @@ class Command(Base):
             non_aggregator_chains = process.chains if hasattr(process, Model._FIELD_CHAINS) else []
 
             self.__print(logging.INFO, i18n.t('command.define_storage', slices=non_aggregator_count))
-            progress_bar = tqdm(total=non_aggregator_count, ncols=100)
+            progress_bar = tqdm(total=non_aggregator_count, bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE,
+                                unit='slice')  # Unit matches keyword argument.
+            progress_bar.set_postfix(slice=1)
 
-            for group_index in range(1, non_aggregator_count + 1):
-                ssd_ids = [(i, non_aggregator_chains[i].get(Model._FIELD_SSD_ID)) for i in range(len(non_aggregator_chains))]
-                chain_digits = len(str(len(ssd_ids)))
+            try:
+                for group_index in range(1, non_aggregator_count + 1):
+                    ssd_ids = [(i, non_aggregator_chains[i].get(Model._FIELD_SSD_ID)) for i in range(len(non_aggregator_chains))]
+                    chain_digits = len(str(len(ssd_ids)))
 
-                for chain_index, ssd_id in ssd_ids:
-                    storage_data_item = self.__get_storage_data_item(process, group_index, non_aggregator_count, ssd_id, chain_index)
+                    for chain_index, ssd_id in ssd_ids:
+                        storage_data_item = self.__get_storage_data_item(process, group_index, non_aggregator_count, ssd_id, chain_index)
 
-                    if storage_data_item is not None:
-                        # Add the storage file to the downloads. We assume there are zero or one file for the storage data item.
-                        storage_data_item.check_analysis_complete(wait=True)
-                        storage_data_file = next(storage_data_item.files, None)
+                        if storage_data_item is not None:
+                            # Add the storage file to the downloads. We assume there are zero or one file for the storage data item.
+                            storage_data_item.check_analysis_complete(wait=True)
+                            storage_data_file = next(storage_data_item.files, None)
 
-                        if storage_data_file is not None:
-                            path = f"{str(group_index).zfill(non_aggregator_digits)}_{str(chain_index).zfill(chain_digits)}_{storage_data_file.file_name.split('_')[-1]}"
-                            key = f"{group_index}_{non_aggregator_count}_{ssd_id}_{chain_index}"
-                            downloads.append((storage_data_file, path))
-                            storage[key] = path
+                            if storage_data_file is not None:
+                                path = f"{str(group_index).zfill(non_aggregator_digits)}_{str(chain_index).zfill(chain_digits)}_{storage_data_file.file_name.split('_')[-1]}"
+                                key = f"{group_index}_{non_aggregator_count}_{ssd_id}_{chain_index}"
+                                downloads.append((storage_data_file, path, storage_data_file.size))
+                                storage[key] = path
 
-                progress_bar.update(1)
+                    progress_bar.set_postfix(slice=group_index)
+                    progress_bar.update(1)
 
-            progress_bar.close()
+            finally:
+                progress_bar.set_postfix()
+                progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+                progress_bar.close()
 
         # Download the files.
         self.__download_files(downloads)
@@ -539,11 +558,16 @@ class Command(Base):
                     elif data_type == Command._DOWNLOAD_OUTPUT:
                         output_sizes.append(file.size)
 
-                    downloads.append((file, path))
-                    stac_definitions.append(file.get_stac_item())
+                    downloads.append((file, path, file.size))
+                    stac_item, stac_item_file = file.get_stac_item()
 
-                stac_collection = data_item.get_stac_collection(stac_definitions, owner=component.name, created_at=component.created_at, detail=component_options)
-                stac_definitions.append(stac_collection)
+                    if (stac_item is not None) and (stac_item_file is not None):
+                        stac_definitions.append((stac_item, stac_item_file))
+
+                if len(stac_definitions) > 0:
+                    stac_collection = data_item.get_stac_collection(stac_definitions, owner=component.name, created_at=component.created_at,
+                                                                    detail=component_options)
+                    stac_definitions.append(stac_collection)
 
                 # Write the STAC definitions.
                 for stac_definition, stac_file_name in stac_definitions:
@@ -561,29 +585,68 @@ class Command(Base):
         Downloads the files from the downloads list.
 
         Args:
-            downloads: A list of tuples with the file model and the download path.
+            downloads: A list of tuples with the file model, the download path and file size.
         """
         downloads = [] if downloads is None else downloads
 
         if len(downloads) > 0:
-            # Report progress.
             self.__print(logging.INFO, i18n.t('command.download_files', files=len(downloads)))
-            progress_bar = tqdm(total=len(downloads), ncols=100)
 
-            # Do all the downloads in batches. We use small batches so that we do not swamp memory.
-            batch_size = 2 * os.cpu_count()  # Downloads don't swamp CPU.
+            # Initialise the progress bar to show bytes downloaded.
+            total_size = sum([size for _, _, size in downloads if size is not None])
+            progress_bar = tqdm(total=total_size, bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE, unit='B', unit_scale=True,
+                                unit_divisor=1024)  # Show progress in bytes.
 
-            for batch in [downloads[i:i + batch_size] for i in range(0, len(downloads), batch_size)]:
-                # Start downloading the batch.
-                for file, path in batch:
-                    file.download(path=path, wait=False)
+            try:
+                # Do all the downloads in batches. We use small batches so that we do not swamp memory.
+                batch_size = 2 * os.cpu_count()  # Downloads don't swamp CPU.
 
-                # Wait for the download to complete.
-                for file, path in batch:
-                    progress_bar.update(1)
-                    file.download_complete(wait=True)
+                for batch in [downloads[i:i + batch_size] for i in range(0, len(downloads), batch_size)]:
+                    download_files = []
 
-            progress_bar.close()
+                    # Start downloading the batch.
+                    for file, path, size in batch:
+                        file.download(path=path, wait=False)
+                        download_files.append((file, False, 0, size))
+
+                    # Check the status of the batch of downloads so that we know when we have finished, updating the progress bar as we go.
+                    finished = False
+                    last_total_current_size = 0
+
+                    while not finished:
+                        # Wait a bit before checking progress again.
+                        sleep(0.25)
+
+                        # Now work out haw far we've got.
+                        total_current_size = 0
+                        count_complete = 0
+
+                        for i in range(len(download_files)):
+                            file, complete, current_size, download_size = download_files[i]
+
+                            if not complete:
+                                # Get the current download size.
+                                _, _, current_size = file.download_progress()
+
+                                # Now check for completion.
+                                complete = file.download_complete()
+                                download_files[i] = (file, complete, current_size, download_size)
+
+                            count_complete += 1 if complete else 0
+                            total_current_size += current_size
+
+                        # Increment the progress bar.
+                        update_current_size = total_current_size - last_total_current_size
+                        last_total_current_size = total_current_size
+                        progress_bar.update(update_current_size)
+
+                        # Check if we have finished.
+                        finished = count_complete == len(download_files)
+
+            finally:
+                # Make sure the progress bar is closed.
+                progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+                progress_bar.close()
 
     def download_process_execution(self, organisation, process_name, download_inputs, download_outputs, download_storage, download_intermediate,
                                    download_components, save_metrics):
@@ -645,19 +708,26 @@ class Command(Base):
         # Do all the downloads in batches. We use small batches so that we do not swamp memory.
         batch_size = 2 * os.cpu_count()  # Downloads don't swamp CPU.
         self.__print(logging.INFO, i18n.t('command.download_executions', executions=len(executions)))
-        progress_bar = tqdm(total=len(executions), ncols=100)
+        progress_bar = tqdm(total=len(executions), bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE,
+                            unit='execution')  # Unit matches keyword argument.
+        progress_bar.set_postfix(execution=1)
 
-        # Now download each execution in batches.
-        partial_download_execution = partial(self.__download_execution, process, download_inputs, download_outputs, download_storage, download_intermediate,
-                                             download_components, output_dir)
-        results = []
-
-        with Pool(nodes=batch_size) as pool:
-            for result in pool.uimap(partial_download_execution, executions):
-                progress_bar.update(1)
-                results.append(result)
+        try:
+            # Now download each execution in batches.
+            partial_download_execution = partial(self.__download_execution, process, download_inputs, download_outputs, download_storage, download_intermediate,
+                                                 download_components, output_dir)
+            results = []
+
+            with Pool(nodes=batch_size) as pool:
+                for i, result in enumerate(pool.uimap(partial_download_execution, executions)):
+                    progress_bar.set_postfix(execution=i + 1)
+                    progress_bar.update(1)
+                    results.append(result)
 
-        progress_bar.close()
+        finally:
+            progress_bar.set_postfix()
+            progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+            progress_bar.close()
 
         # Combine the results.
         downloads = []
@@ -1173,9 +1243,14 @@ class Command(Base):
         # Form the status of the execution.
         status = ''
         group = ''
+        abort_reason = i18n.t('command.log_execution_abort_reason', abort_reason=execution.abort_reason) if hasattr(execution, Model._FIELD_ABORT_REASON) else ''
+        exit_type = i18n.t('command.log_execution_exit_type', exit_type=execution.exit_type) if hasattr(execution, Model._FIELD_EXIT_TYPE) else ''
 
         if execution.success:
-            status = i18n.t('command.log_execution_success')
+            if len(abort_reason) <= 0:
+                status = i18n.t('command.log_execution_success')
+            else:
+                status = i18n.t('command.log_execution_warning', exit_type=exit_type, abort_reason=abort_reason)
 
         elif execution.stopped:
             status = i18n.t('command.log_execution_stopped')
@@ -1184,9 +1259,6 @@ class Command(Base):
             status = i18n.t('command.log_execution_progress', progress=execution.progress)
 
         else:
-            abort_reason = i18n.t('command.log_execution_abort_reason', abort_reason=execution.abort_reason) if hasattr(execution,
-                                                                                                                        Model._FIELD_ABORT_REASON) else ''
-            exit_type = i18n.t('command.log_execution_exit_type', exit_type=execution.exit_type) if hasattr(execution, Model._FIELD_EXIT_TYPE) else ''
             status = i18n.t('command.log_execution_failed', exit_type=exit_type, abort_reason=abort_reason)
 
         started_at = execution.started_at.strftime('%Y-%m-%d %H:%M:%S')
@@ -1475,78 +1547,97 @@ class Command(Base):
         Returns:
             The found data items in the same order as the input files.
         """
-        data_items = []
-        created_items = []
-
-        self.__print(logging.INFO, i18n.t('command.upload_inputs', files=len(input_files)))
-        progress_bar = tqdm(total=len(input_files), ncols=100)
+        data_items = [None] * len(input_files)
+        find_items = []
+        create_items = []
+
+        for i, input_file in enumerate(input_files if input_files is not None else []):
+            # The input file can either be a string representing a name or a file, or a dictionary with explicit values for both.
+            input_name = None
+            filename = None
+            extension = None
+
+            if isinstance(input_file, dict):
+                input_name = input_file.get(Model._FIELD_NAME)
+                filename = input_file.get(Model._FIELD_FILE)
+                extension = os.path.splitext(filename.lower())[1] if filename is not None else None
+            else:
+                # An input name is not assumed to have an extension.
+                extension = os.path.splitext(input_file.lower())[1]
 
-        try:
-            for i, input_file in enumerate(input_files if input_files is not None else []):
-                # The input file can either be a string representing a name or a file, or a dictionary with explicit values for both.
-                input_name = None
-                filename = None
-                extension = None
-
-                if isinstance(input_file, dict):
-                    input_name = input_file.get(Model._FIELD_NAME)
-                    filename = input_file.get(Model._FIELD_FILE)
-                    extension = os.path.splitext(filename.lower())[1]
+                if len(extension) > 0:
+                    filename = input_file
                 else:
-                    # An input name is not assumed to have an extension.
-                    extension = os.path.splitext(input_file.lower())[1]
-
-                    if len(extension) > 0:
-                        filename = input_file
-                    else:
-                        input_name = input_file
-                        extension = None
-
-                if extension is not None:
-                    # Create a data item for each input.
-                    if (extension == '.tif') or (extension == '.tiff'):
-                        file_type = fusion_platform.FILE_TYPE_GEOTIFF
-                    elif extension == '.jp2':
-                        file_type = fusion_platform.FILE_TYPE_JPEG2000
-                    elif extension == '.dem':
-                        file_type = fusion_platform.FILE_TYPE_DEM
-                    elif (extension == '.json') or (extension == '.geojson'):
-                        file_type = fusion_platform.FILE_TYPE_GEOJSON
-                    elif extension == '.kml':
-                        file_type = fusion_platform.FILE_TYPE_KML
-                    elif extension == '.kmz':
-                        file_type = fusion_platform.FILE_TYPE_KMZ
-                    elif extension == '.csv':
-                        file_type = fusion_platform.FILE_TYPE_CSV
-                    elif extension == '.zip':
-                        file_type = fusion_platform.FILE_TYPE_ESRI_SHAPEFILE
-                    elif (extension == '.jpeg') or (extension == '.jpg'):
-                        file_type = fusion_platform.FILE_TYPE_JPEG
-                    elif extension == '.png':
-                        file_type = fusion_platform.FILE_TYPE_PNG
-                    else:
-                        file_type = fusion_platform.FILE_TYPE_OTHER
-
-                    input_name = f"{process_name} {i + 1}" if input_name is None else input_name
-                    created = organisation.create_data(name=input_name, file_type=file_type, files=[filename], wait=False)
-                    data_items.append(created)
-                    created_items.append(created)
+                    input_name = input_file
+                    extension = None
+
+            if extension is not None:
+                # Create a data item for each input.
+                if (extension == '.tif') or (extension == '.tiff'):
+                    file_type = fusion_platform.FILE_TYPE_GEOTIFF
+                elif extension == '.jp2':
+                    file_type = fusion_platform.FILE_TYPE_JPEG2000
+                elif extension == '.dem':
+                    file_type = fusion_platform.FILE_TYPE_DEM
+                elif (extension == '.json') or (extension == '.geojson'):
+                    file_type = fusion_platform.FILE_TYPE_GEOJSON
+                elif extension == '.kml':
+                    file_type = fusion_platform.FILE_TYPE_KML
+                elif extension == '.kmz':
+                    file_type = fusion_platform.FILE_TYPE_KMZ
+                elif extension == '.csv':
+                    file_type = fusion_platform.FILE_TYPE_CSV
+                elif extension == '.zip':
+                    file_type = fusion_platform.FILE_TYPE_ESRI_SHAPEFILE
+                elif (extension == '.jpeg') or (extension == '.jpg'):
+                    file_type = fusion_platform.FILE_TYPE_JPEG
+                elif extension == '.png':
+                    file_type = fusion_platform.FILE_TYPE_PNG
                 else:
+                    file_type = fusion_platform.FILE_TYPE_OTHER
+
+                input_name = f"{process_name} {i + 1}" if input_name is None else input_name
+                create_items.append((i, input_name, file_type, filename))
+            else:
+                find_items.append((i, input_name))
+
+        # Find all required items with a progress bar.
+        if len(find_items) > 0:
+            self.__print(logging.INFO, i18n.t('command.find_inputs', inputs=len(find_items)))
+            progress_bar = tqdm(total=len(find_items), bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE,
+                                unit='input')  # Unit matches keyword argument.
+            progress_bar.set_postfix(input=1)
+            k = 1
+
+            try:
+                for i, input_name in find_items:
                     data_item, _ = organisation.find_data(name=input_name)
 
                     if data_item is None:
                         raise CommandError(i18n.t('command.no_such_input', input_name=input_name))
 
-                    data_items.append(data_item)
+                    data_items[i] = data_item
+
+                    k += 1
+                    progress_bar.set_postfix(input=k)
                     progress_bar.update(1)
+            finally:
+                progress_bar.set_postfix()
+                progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+                progress_bar.close()
 
-            # Wait for the analysis of any created items to complete.
-            for created in created_items:
-                created.create_complete(wait=True)
-                progress_bar.update(1)
+        # Create all required items. The upload has a progress bar.
+        if len(create_items) > 0:
+            self.__print(logging.INFO, i18n.t('command.upload_inputs', inputs=len(create_items)))
+            created_items = []
 
-        finally:
-            progress_bar.close()
+            for i, input_name, file_type, filename in create_items:
+                created = organisation.create_data(name=input_name, file_type=file_type, files=[filename], wait=False)
+                created_items.append((created, False, 0, os.path.getsize(filename)))
+                data_items[i] = created
+
+            # Wait for the analysis of any created items to complete.
+            self.__wait_for_uploads(created_items)
 
         return data_items
 
@@ -1559,42 +1650,35 @@ class Command(Base):
             storage_files: The list of storage files to be uploaded.
             process: The newly created process.
         """
-        created_items = []
-
         self.__print(logging.INFO, i18n.t('command.upload_storage', files=len(storage_files)))
-        progress_bar = tqdm(total=len(storage_files), ncols=100)
+        storage_files = {} if storage_files is None else storage_files
+        created_items = []
 
-        try:
-            for storage_key, storage_file in (storage_files if storage_files is not None else {}).items():
-                # The key represents the values used to generate the new storage id, without the process id, which will be different.
-                elements = storage_key.split('_')
-                group_index = int(elements[0])
-                group_count = int(elements[1])
-                ssd_id = elements[2]
-                chain_index = int(elements[3])
-                storage_name = i18n.t('command.storage_name', process=process.name, group_index=group_index, chain_index=chain_index)
+        for storage_key, storage_file in storage_files.items():
+            # The key represents the values used to generate the new storage id, without the process id, which will be different.
+            elements = storage_key.split('_')
+            group_index = int(elements[0])
+            group_count = int(elements[1])
+            ssd_id = elements[2]
+            chain_index = int(elements[3])
+            storage_name = i18n.t('command.storage_name', process=process.name, group_index=group_index, chain_index=chain_index)
 
-                group_index = None if group_count == 1 else group_index  # If not in a group, then this needs to be None.
-                group_count = None if group_count == 1 else group_count  # If not in a group, then this needs to be None.
-                storage_id = self.__get_storage_id(process.id, group_index, group_count, ssd_id, chain_index)
+            group_index = None if group_count == 1 else group_index  # If not in a group, then this needs to be None.
+            group_count = None if group_count == 1 else group_count  # If not in a group, then this needs to be None.
+            storage_id = self.__get_storage_id(process.id, group_index, group_count, ssd_id, chain_index)
 
-                # The destination filename must be correct for everything to work, so we must make a copy with the right name.
-                with tempfile.TemporaryDirectory() as tmp_dir:
-                    upload_file = os.path.join(tmp_dir, Command._STORAGE_NAME)
-                    shutil.copy(storage_file, upload_file)
+            # The destination filename must be correct for everything to work, so we must make a copy with the right name.
+            with tempfile.TemporaryDirectory() as tmp_dir:
+                upload_file = os.path.join(tmp_dir, Command._STORAGE_NAME)
+                shutil.copy(storage_file, upload_file)
 
-                    created = organisation.create_data(name=storage_name, file_type=fusion_platform.FILE_TYPE_GZIP, files=[upload_file], wait=False, id=storage_id)
-                    created_items.append(created)
+                created = organisation.create_data(name=storage_name, file_type=fusion_platform.FILE_TYPE_GZIP, files=[upload_file], wait=False, id=storage_id)
+                created_items.append((created, False, 0, os.path.getsize(upload_file)))
 
-                    os.remove(upload_file)
+                os.remove(upload_file)
 
-            # Wait for the analysis of any created items to complete.
-            for created in created_items:
-                created.create_complete(wait=True)
-                progress_bar.update(1)
-
-        finally:
-            progress_bar.close()
+        # Wait for the analysis of any created items to complete.
+        self.__wait_for_uploads(created_items)
 
     def __validate_constrained(self, values, text):
         """
@@ -1609,6 +1693,60 @@ class Command(Base):
         """
         return text.lower() in [value.lower() for value in values]
 
+    def __wait_for_uploads(self, upload_data):
+        """
+        Waits for all the specified uploads to complete, showing a progress bar of bytes uploaded.
+
+        Args:
+            upload_data: A list of tuples of the data items being uploaded with the data model, if the upload has completed, the upload size and total size.
+        """
+
+        # Initialise the progress bar to show bytes uploaded.
+        total_size = sum([size for _, _, _, size in upload_data if size is not None])
+        progress_bar = tqdm(total=total_size, bar_format=Command.__PROGRESS_BAR_FORMAT, colour=Command.__PROGRESS_BAR_COLOUR_ACTIVE, unit='B', unit_scale=True,
+                            unit_divisor=1024)  # Show progress in bytes.
+
+        try:
+            # Check the status of all uploads so that we know when we have finished, updating the progress bar as we go.
+            finished = False
+            last_total_current_size = 0
+
+            while not finished:
+                # Wait a bit before checking progress again.
+                sleep(0.25)
+
+                # Now work out haw far we've got.
+                total_current_size = 0
+                count_complete = 0
+
+                for i in range(len(upload_data)):
+                    data, complete, current_size, upload_size = upload_data[i]
+
+                    if not complete:
+                        # Check every file being uploaded for the data model.
+                        upload_progress = data.upload_progress()
+                        current_size = sum([size for _, size in upload_progress])
+
+                        # Now check for completion.
+                        complete = data.create_complete()
+                        upload_data[i] = (data, complete, current_size, upload_size)
+
+                    count_complete += 1 if complete else 0
+                    total_current_size += current_size
+
+                # Increment the progress bar.
+                update_current_size = total_current_size - last_total_current_size
+                last_total_current_size = total_current_size
+                progress_bar.update(update_current_size)
+
+                # Check if we have finished.
+                finished = count_complete == len(upload_data)
+
+        finally:
+            # Make sure the progress bar is closed.
+            progress_bar.colour = Command.__PROGRESS_BAR_COLOUR_COMPLETE
+            progress_bar.close()
+
 
 def main():
     """

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/models/data.py

@@ -100,6 +100,7 @@ class Data(Model):
         super(Data, self).__init__(session)
 
         # Initialise the fields.
+        self.__upload_progress = {}
         self.__upload_threads = {}
 
     def __add_file(self, file_type, file):
@@ -138,10 +139,11 @@ class Data(Model):
             raise ModelError(i18n.t('models.data.failed_add_file_not_unique'))
 
         # Create and start a thread for the upload.
+        self.__upload_progress[file] = (file, 0)  # Indexed by file path, not file id.
         thread = None
 
         try:
-            thread = RaiseThread(target=self._session.upload_file, args=(url, file))
+            thread = RaiseThread(target=self._session.upload_file, args=(url, file, self.__upload_callback))
             thread.start()
 
         finally:
@@ -393,6 +395,12 @@ class Data(Model):
         interval_end = None
 
         for item, item_file_name in items:
+            # So that we can extract things correctly, turn any mapping proxy into a dict (and relevant sub-mapping proxies).
+            item = dict(item)
+
+            if item.get('properties') is not None:
+                item['properties'] = dict(item.get('properties'))
+
             # Calculate the spatial extent.
             item_bbox = item.get('bbox')
 
@@ -422,3 +430,30 @@ class Data(Model):
             'extent': {'spatial': {'bbox': [bbox]}, 'temporal': {'interval': [[interval_start, interval_end]]}},
             'links': links
         }, collection_file_name
+
+    def __upload_callback(self, url, source, size):
+        """
+        Callback method used to receive progress from upload. Updates the upload progress.
+
+        Args:
+            url: The URL to upload as a file.
+            source: The source file path.
+            size: The total size in bytes so far uploaded.
+        """
+        self.__upload_progress[source] = (source, size)
+
+    def upload_progress(self):
+        """
+        Returns current upload progress.
+
+        Returns:
+            A list of tuples of the source and total number of bytes uploaded so far.
+
+        Raises:
+            ModelError: if no upload is in progress.
+        """
+        # Make sure at least one upload is in progress.
+        if (self.__upload_threads is None) or (len(self.__upload_threads) == 0):
+            raise ModelError(i18n.t('models.data.no_upload'))
+
+        return list(self.__upload_progress.values())

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/models/data_file.md

@@ -39,3 +39,5 @@
 * **geojson**: The content of the associated file if it is a GeoJSON file (of limited size).
 * **title**: The title for the selector.
 * **description**: The description of the selector.
+* **stac_item**: The STAC item associated with this file.
+* **stac_item_file**: The name of the STAC item file.

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/models/data_file.py

@@ -93,9 +93,14 @@ class DataFileSchema(Schema):
     downloads = fields.Integer(allow_none=True, metadata={'read_only': True})  # Changed to prevent this being updated.
     geojson = fields.String(allow_none=True, metadata={'read_only': True})  # Changed to prevent this being updated.
 
+    # Removed detail.
+
     title = fields.String(allow_none=True, metadata={'read_only': True})  # Changed to prevent this being updated.
     description = fields.String(allow_none=True, metadata={'read_only': True})  # Changed to prevent this being updated.
 
+    stac_item = fields.Dict(allow_none=True)  # Added pseudo-parameter.
+    stac_item_file = fields.String(allow_none=True)  # Added pseudo-parameter.
+
     class Meta:
         """
         When loading an object, make sure we exclude any unknown fields, rather than raising an exception, and put fields in their definition order.
@@ -293,135 +298,8 @@ class DataFile(Model):
             A tuple of the STAC item definition as a dictionary and the item file name used in the definition.
         """
 
-        # Obtain all the relevant information.
-        bounds = self.bounds if hasattr(self, Model._FIELD_BOUNDS) else None
-        crs = self.crs if hasattr(self, Model._FIELD_CRS) else None
-        file_name = self.file_name if hasattr(self, Model._FIELD_FILE_NAME) else None
-        size = self.size if hasattr(self, Model._FIELD_SIZE) else None
-
-        # Attempt to find the file type to get its extension and geospatial type. Default to "Other" if not found.
-        found_file_type = DataFile._FILE_TYPES.get(self.file_type)
-
-        if found_file_type is None:
-            found_file_type = DataFile._FILE_TYPES.get(fusion_platform.FILE_TYPE_OTHER)
-
-        geospatial = found_file_type[1]
-
-        # Get the file extension.
-        extension = os.path.splitext(file_name)[1] if file_name is not None else None
-        extension = found_file_type[0] if extension is None else extension
-
-        if (extension is None) or (extension == '*'):
-            extension = ''
-        else:
-            extension = f".{extension}" if not extension.startswith('.') else extension
+        # Extract the STAC item from the model.
+        stac_item = self.stac_item if hasattr(self, Model._FIELD_STAC_ITEM) else None
+        stac_item_file = self.stac_item_file if hasattr(self, Model._FIELD_STAC_ITEM_FILE) else None
 
-        # Work out a suitable name for the file if one is not available.
-        if file_name is None:
-            file_name = f"{self.id}{extension}"
-
-        item_file_name = item_file_name_format.format(file_name=file_name)
-
-        # Start to build the STAC extensions list.
-        stac_extensions = ['https://stac-extensions.github.io/processing/v1.2.0/schema.json']  # For properties#processing.[software,version].
-
-        # Get the bounding box and geometry, if there is one.
-        bbox = []
-        geometry = {}
-
-        if (bounds is not None) and (len(bounds) >= 4):
-            bbox = bounds
-            geometry = {'type': 'Polygon', 'coordinates': [[[bbox[0], bbox[1]], [bbox[2], bbox[1]], [bbox[2], bbox[3]], [bbox[0], bbox[3]], [bbox[0], bbox[1]]]]}
-
-        # Get the datetime from the filename, if possible. We try various iterations of sections the file name until we find something.
-        for file_part in file_name.split('_'):
-            extracted_datetime = datetime_parse(file_part) or datetime_parse(f"{file_part}T00:00:00Z")  # Assume UTC.
-
-            if extracted_datetime is not None:
-                break
-
-        if extracted_datetime is None:
-            extracted_datetime = self.created_at
-
-        # Start to build the properties.
-        properties = {'processing.software': i18n.t('fusion_platform.sdk'), 'processing.version': fusion_platform.__version__, 'datetime': extracted_datetime}
-
-        # Add in the CRS, if there is one.
-        if crs is not None:
-            properties['proj:epsg'] = crs
-            stac_extensions.append('https://stac-extensions.github.io/projection/v1.1.0/schema.json')  # For properties#proj:epsg.
-
-        # Build the links.
-        links = [{'rel': 'self', 'href': item_file_name, 'type': 'application/json'}]
-
-        # Add in the optional owning collection.
-        if collection_file_name is not None:
-            links.append({'rel': 'root', 'href': collection_file_name, 'type': 'application/json'})
-            links.append({'rel': 'parent', 'href': collection_file_name, 'type': 'application/json'})
-
-        # Start to build the asset from the associated file.
-        asset = {'href': file_name, 'roles': []}
-
-        # Extract out the relevant values depending upon the file type.
-        raster_bands = None
-        eo_bands = None
-
-        if (geospatial == DataFile._GEOSPATIAL_RASTER) and hasattr(self, Model._FIELD_SELECTORS) and (self.selectors is not None):
-            for selector in self.selectors:
-                raster_band = {}
-                eo_band = {}
-
-                if hasattr(self, Model._FIELD_RESOLUTION):
-                    raster_band['spatial_resolution'] = self.resolution
-
-                if selector.get(Model._FIELD_UNIT) is not None:
-                    raster_band['unit'] = selector.get(Model._FIELD_UNIT)
-
-                if (selector.get(Model._FIELD_MINIMUM) is not None) and (selector.get(Model._FIELD_MAXIMUM) is not None) and (
-                        selector.get(Model._FIELD_MEAN) is not None) and (selector.get(Model._FIELD_SD) is not None):
-                    raster_band['statistics'] = {
-                        'minimum': selector.get(Model._FIELD_MINIMUM),
-                        'maximum': selector.get(Model._FIELD_MAXIMUM),
-                        'mean': selector.get(Model._FIELD_MEAN),
-                        'stddev': selector.get(Model._FIELD_SD)
-                    }
-
-                if (selector.get(Model._FIELD_HISTOGRAM_MINIMUM) is not None) and (selector.get(Model._FIELD_HISTOGRAM_MAXIMUM) is not None) and (
-                        selector.get(Model._FIELD_HISTOGRAM) is not None):
-                    raster_band['histogram'] = {
-                        'count': len(selector.get(Model._FIELD_HISTOGRAM)),
-                        'min': selector.get(Model._FIELD_HISTOGRAM_MINIMUM),
-                        'max': selector.get(Model._FIELD_HISTOGRAM_MAXIMUM),
-                        'buckets': selector.get(Model._FIELD_HISTOGRAM),
-                    }
-
-                if selector.get(Model._FIELD_CATEGORY) is not None:
-                    eo_band['name'] = selector.get(Model._FIELD_CATEGORY)
-
-                raster_bands = [raster_band] if raster_bands is None else [raster_band] + raster_bands
-                eo_bands = [eo_band] if eo_bands is None else [eo_band] + eo_bands
-
-            stac_extensions.append('https://stac-extensions.github.io/eo/v1.1.0/schema.json')  # For assets#eo:bands.
-            stac_extensions.append('https://stac-extensions.github.io/raster/v1.1.0/schema.json')  # For assets#raster:bands.
-
-        if raster_bands is not None:
-            asset['raster:bands'] = raster_bands
-            asset['eo:bands'] = eo_bands
-
-        # Add in the file size, if there is one.
-        if size is not None:
-            asset['file:size'] = size
-            stac_extensions.append('https://stac-extensions.github.io/file/v2.1.0/schema.json')  # For assets#file:size.
-
-        # Build the STAC item.
-        return {
-            'type': 'Feature',
-            'stac_version': '1.0.0',
-            'stac_extensions': stac_extensions,
-            'id': self.id,
-            'bbox': bbox,
-            'geometry': geometry,
-            'properties': properties,
-            'links': links,
-            'assets': {'asset': asset}
-        }, item_file_name
+        return stac_item, stac_item_file

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/models/model.py

@@ -96,6 +96,8 @@ class Model(Base):
     _FIELD_SELECTOR = 'selector'
     _FIELD_SELECTORS = 'selectors'
     _FIELD_SSD_ID = 'ssd_id'
+    _FIELD_STAC_ITEM = 'stac_item'
+    _FIELD_STAC_ITEM_FILE = 'stac_item_file'
     _FIELD_STARTED_AT = 'started_at'
     _FIELD_PUBLISHABLE = 'publishable'
     _FIELD_SIZE = 'size'

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/models/process_execution.py

@@ -186,13 +186,18 @@ class ProcessExecution(Model):
             complete = self.progress >= 100
 
             # Raise an exception if the execution failed.
+            abort_reason = self.abort_reason if hasattr(self, Model._FIELD_ABORT_REASON) else None
+
             if complete and not self.success:
-                self._logger.error(i18n.t('models.process_execution.execution_failed'))
-                raise ModelError(i18n.t('models.process_execution.execution_failed'))
+                self._logger.error(i18n.t('models.process_execution.execution_failed', abort_reason=abort_reason))
+                raise ModelError(i18n.t('models.process_execution.execution_failed', abort_reason=abort_reason))
 
             if complete:
                 self._logger.debug('execution %s is complete', self.id)
 
+                if abort_reason is not None:
+                    self._logger.warning(i18n.t('models.process_execution.execution_warning', abort_reason=abort_reason))
+
             if (not wait) or complete:
                 break
 

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/session.py

@@ -13,6 +13,7 @@ import logging
 import os
 import requests
 from tenacity import before_sleep_log, retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential
+from tqdm.utils import CallbackIOWrapper
 
 import fusion_platform
 from fusion_platform.base import Base
@@ -47,6 +48,36 @@ class ValueError(SessionError):
     pass
 
 
+class UploadCallback:
+    """
+    Provides a callback mechanism for uploads using the CallbackIOWrapper.
+    """
+
+    def __init__(self, url, source, callback):
+        """
+        Initialises the object.
+
+        Args:
+            url: The URL being uploaded to.
+            source: The source file path.
+            callback: The callback method used to receive upload progress.
+        """
+        self.__url = url
+        self.__source = source
+        self.__callback = callback
+        self.__upload_size = 0
+
+    def callback(self, size):
+        """
+        Receives the upload callback every time data is read from the associated file.
+
+        Args:
+            size: The number of bytes which have been read from the associated file.
+        """
+        self.__upload_size += size
+        self.__callback(self.__url, self.__source, self.__upload_size)
+
+
 class Session(Base):
     """
     Provides a session for use in interfacing with the Fusion Platform<sup>&reg;</sup> API.
@@ -257,19 +288,22 @@ class Session(Base):
         # Return the payload.
         return payload
 
-    def upload_file(self, url, source):
+    def upload_file(self, url, source, callback=None):
         """
         Uploads a file from the source path.
 
         Args:
             url: The URL to download as a file.
             source: The source file path.
+            callback: The optional callback method used to receive upload progress.
         """
         try:
             # Upload the file as a data stream.
             self._logger.info('uploading %s -> %s', url, source)
 
             with open(source, 'rb') as file:
+                # Wrap the file reader with a callback to give progress.
+                file = CallbackIOWrapper(UploadCallback(url, source, callback).callback, file, 'read') if callback is not None else file
                 response = requests.put(url, data=file)
 
                 # Raise any errors.

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/fusion_platform/translations.py

@@ -24,6 +24,7 @@ i18n.add_translation('command.log_execution_group', ' (group \'%{group_id}\' %{g
 i18n.add_translation('command.log_execution_duration', ' - %{minutes} minute(s)', 'en')
 i18n.add_translation('command.log_execution_period', ' (%{started_at}%{ended_at})', 'en')
 i18n.add_translation('command.log_execution_ended_at', ' -> %{ended_at}', 'en')
+i18n.add_translation('command.log_execution_warning', 'warning%{exit_type}%{abort_reason}', 'en')
 i18n.add_translation('command.log_execution_failed', 'failed%{exit_type}%{abort_reason}', 'en')
 i18n.add_translation('command.log_execution_exit_type', ' (%{exit_type})', 'en')
 i18n.add_translation('command.log_execution_abort_reason', ': %{abort_reason}', 'en')
@@ -50,7 +51,7 @@ i18n.add_translation('command.using_storage', 'Uploading storage \'%{filename}\'
 i18n.add_translation('command.using_service', 'Using service \'%{service}\'', 'en')
 i18n.add_translation('command.using_organisation', 'Using organisation \'%{organisation}\'', 'en')
 i18n.add_translation('command.upload_storage', 'Uploading %{files} storage file(s)...', 'en')
-i18n.add_translation('command.upload_inputs', 'Uploading/finding %{files} input file(s)...', 'en')
+i18n.add_translation('command.upload_inputs', 'Uploading %{inputs} input file(s)...', 'en')
 i18n.add_translation('command.storage_name', '%{process} Storage %{group_index} %{chain_index}', 'en')
 i18n.add_translation('command.start_process', 'Starting \'%{process}\' from service \'%{service}\'', 'en')
 i18n.add_translation('command.remove_process', 'Removing input \'%{process}\'', 'en')
@@ -59,6 +60,7 @@ i18n.add_translation('command.prompt_yes', 'Yes', 'en')
 i18n.add_translation('command.prompt_no', 'No', 'en')
 i18n.add_translation('command.password', 'Password', 'en')
 i18n.add_translation('command.organisation', 'Organisation', 'en')
+i18n.add_translation('command.find_inputs', 'Finding %{inputs} input file(s)...', 'en')
 i18n.add_translation('command.executing', 'Executing...', 'en')
 i18n.add_translation('command.email', 'Email', 'en')
 i18n.add_translation('command.download_process_execution', 'Downloading \'%{process}\' (inputs %{inputs}, outputs %{outputs}, storage %{storage}, intermediate %{intermediate}, metrics %{metrics}, components %{components})', 'en')
@@ -237,13 +239,15 @@ i18n.add_translation('models.data_file.no_download', 'No download is in progress
 i18n.add_translation('models.data_file.download_already_in_progress', 'Cannot download file as the download is already in progress', 'en')
 i18n.add_translation('models.data_file.organisation_id.description', 'The owning organisation.', 'en')
 i18n.add_translation('models.data_file.organisation_id.title', 'Organisation', 'en')
+i18n.add_translation('models.data.no_upload', 'No upload is in progress', 'en')
 i18n.add_translation('models.data.no_create', 'No create is in progress', 'en')
 i18n.add_translation('models.data.failed_add_missing_file', 'Failed to add file as the file does not exist: %{file}', 'en')
 i18n.add_translation('models.data.failed_add_file_not_unique', 'Failed to add file as the id is not unique', 'en')
 i18n.add_translation('models.data.failed_add_file_url', 'Failed to get URL from add file response', 'en')
 i18n.add_translation('models.data.failed_add_file_id', 'Failed to get id from add file response', 'en')
 i18n.add_translation('models.data.failed_copy_id', 'Failed to get data id from copy response', 'en')
-i18n.add_translation('models.process_execution.execution_failed', 'Execution has failed', 'en')
+i18n.add_translation('models.process_execution.execution_warning', 'Execution has completed successfully but with a warning: %{abort_reason}', 'en')
+i18n.add_translation('models.process_execution.execution_failed', 'Execution has failed: %{abort_reason}', 'en')
 i18n.add_translation('models.fields.uuid.invalid_uuid', 'Not a valid utf-8 string', 'en')
 i18n.add_translation('models.fields.url.invalid_url', 'Not a valid URL', 'en')
 i18n.add_translation('models.fields.tuple.invalid', 'Not a valid tuple', 'en')

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6/fusion_platform_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: fusion-platform-python-sdk
-Version: 1.15.12
+Version: 1.16.6
 Summary: Python SDK used to interact with the Fusion Platform(r)
 Home-page: https://github.com/d-cat-support/fusion-platform-python-sdk
 Author: Digital Content Analysis Technology Ltd
@@ -24,6 +24,15 @@ Requires-Dist: python-i18n[YAML]
 Requires-Dist: requests
 Requires-Dist: tenacity
 Requires-Dist: tqdm
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 # Fusion Platform<sup>&reg;</sup> Python SDK
 

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/setup.py

@@ -19,7 +19,7 @@ PACKAGE_DIR = pathlib.Path(__file__).parent
 # @formatter:off
 setup(
     name='fusion-platform-python-sdk',
-    version='1.15.12',
+    version='1.16.6',
     description='Python SDK used to interact with the Fusion Platform(r)',
     long_description=(PACKAGE_DIR / 'README.md').read_text(),
     long_description_content_type='text/markdown',

{fusion_platform_python_sdk-1.15.12 → fusion_platform_python_sdk-1.16.6}/tests/test_session.py

@@ -302,3 +302,9 @@ class TestSession(CustomTestCase):
             adapter = mock.put(url, status_code=200)
             session.upload_file(url, source)
             self.assertIsNotNone(adapter.last_request.text)
+
+            def callback(url, source, upload_size):
+                pass
+
+            session.upload_file(url, source, callback=callback)  # Because the request is mocked, the callback will not be called because there are no reads.
+            self.assertIsNotNone(adapter.last_request.text)