ciocore 5.1.1__py2.py3-none-any.whl → 10.0.0b3__py2.py3-none-any.whl

ciocore/downloader/job_downloader.py

@@ -0,0 +1,119 @@
+"""
+Job Downloader
+
+Download output files from a Conductor job.
+
+ENDPOINT
+The outputs of a Conductor job are described in the response from the /jobs/{job_id}/downloads endpoint. The response is a list of tasks. Each task has a list of files. Each file (dict) has a signed URL plus other fields such as the md5 and fields describing the original path, size, and more.
+
+PAGING
+A job may contain thousands of tasks, each with several files. To reduce the time it takes to get started, this downloader makes requests for download information in batches, or pages. The number of tasks in each page is controlled by the page_size parameter. As soon as the first page of tasks is retrieved, we start downloading the files in threads. While the files are downloading, we fetch the next page of tasks. When the current page of tasks is exhausted, we start downloading the files in the next page of tasks. We continue until all tasks have been downloaded.
+
+The get_some_tasks method is responsible for fetching the next page of tasks. It is called by the base class. It returns a list of tasks, and a locator. For this implementation, the locator is a dictionary containing the index of the current job, and the cursor for the next page of tasks for the job. A new locator is returned to the calling method so that it can be passed back to this method the next time it is called. When the calling method receives a falsey value for the locator, it knows that there are no more tasks to download.
+
+See the documentation for the base downloader for more information about the locator and other behavior.
+"""
+
+import json
+import logging
+from cioseq.sequence import Sequence
+from ciocore.downloader.base_downloader import BaseDownloader
+from ciocore.downloader.log import LOGGER_NAME
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class JobDownloader(BaseDownloader):
+    CLIENT_NAME = "JobDownloader"
+
+    def __init__(self, jobs, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        """Initialize the downloader."""
+        logger.debug("Initializing paged job downloader")
+        self.jobs = flatten(jobs)
+        self.location = None # location is not used in this downloader
+
+    def get_some_tasks(self, locator):
+        """Fetch the next page of tasks from the server.
+
+        locator: a dictionary containing the index of the current job, and the cursor for the next page of tasks for the job.
+ 
+
+        # What is a locator? It's the information needed to request the next page of tasks. It consists of the index of the current job, and the cursor for the next page of tasks. It is provided to this method as a parameter, and when we're done, a new locator is returned to the run loop. The run loop passes it back to us the next time it is called.
+
+        # On the first call, the provided locator is None. In that case, we start with the first job, and no cursor.
+
+        # We return the locator to the run loop in the base class, along with any tasks to be downloaded. If we return a falsy locator, the run loop is exited, since it means we downloaded everything OR there was an error fetching tasks.
+
+        # If we got to the end of the current job, we increment the job index and reset the cursor to None. The next time this method is called, we'll start with the next job.
+
+        # If we got a next_cursor from the request, we return it in the locator along with the current job index. This is what we'll be given on the next call.
+        """
+        
+        if not locator:
+            locator = {}
+
+        job_index = locator.get("job_index", 0)
+        cursor = locator.get("cursor", None)
+        if job_index >= len(self.jobs):
+            # return no tasks and no locator. Ends the download.
+            return [], None
+
+        # we have a job to download
+        job_info = self.jobs[job_index]
+        job_id = job_info["job_id"]
+        task_ids = job_info["task_ids"]
+        url = f"/jobs/{job_id}/downloads"
+        data = json.dumps({"tids": task_ids})
+        params = {"limit": self.page_size, "start_cursor": cursor}
+        try:
+            response, code = self.client.make_request(
+                url, verb="POST", params=params, data=data, use_api_key=True
+            )
+            if code != 201:
+                # we have an error. Return null locator to end the download
+                raise Exception(f"Code: {code}")
+        except Exception as exc:
+            logger.error("Error fetching download info for job ID: %s : %s : %s", job_id, url, exc)
+            return [], None
+        page = json.loads(response)
+        tasks = page.get("downloads", [])
+
+        tasks = self.filter(tasks)
+        
+        next_cursor = page.get("next_cursor")
+
+        if not next_cursor:
+            # we're done with this job
+            job_index += 1
+
+        return tasks, {"job_index": job_index, "cursor": next_cursor}
+
+
+def flatten(job_specs):
+    """Create a list of job objects with keys: job_id and tasks.
+
+    See tests/test_downloader.py for examples.
+
+    Example input:  ["1234", "1235:12-15"]
+
+    Example result:
+    [
+        {"job_id": "01234", "task_ids":None},
+        {"job_id": "01235", "task_ids":["012","013","014","015"]}
+    ]
+    """
+    result = []
+    for job_spec in job_specs:
+        if ":" in job_spec:
+            job_id, range_spec = job_spec.split(":")
+            try:
+                seq = Sequence.create(range_spec)
+                task_ids = seq.expand("###")
+            except (ValueError, TypeError):
+                task_ids = None
+        else:
+            job_id, task_ids = job_spec, None
+            task_ids = None
+        result.append({"job_id": job_id.zfill(5), "task_ids": task_ids})
+    return result

ciocore/{downloader.py → downloader/legacy_downloader.py}

@@ -393,7 +393,7 @@ class Downloader(object):
     def start_reporter_thread(self, download_data):
         reporter_thread_name = "ReporterThread"
         current_thread_name = threading.current_thread().name
-        thread_number_match = re.match("Thread-(\d+)", current_thread_name)
+        thread_number_match = re.match(r"Thread-(\d+)", current_thread_name)
         if thread_number_match:
             reporter_thread_name += "-%s" % thread_number_match.groups()[0]
 
@@ -493,7 +493,6 @@ class Downloader(object):
                 downloads = _get_job_download(endpoint, self.api_client, job_id, tid)
                 if downloads:
                     for task_download in downloads.get("downloads", []):
-                        print("putting in queue: %s" % task_download)
                         self.pending_queue.put(task_download, block=True)
 
     @common.dec_catch_exception(raise_=True)
@@ -1230,9 +1229,14 @@ def run_downloader(args):
 
     # Set up logging
     log_level_name = args.get("log_level")
-    log_level = loggeria.LEVEL_MAP.get(log_level_name)
-    log_dirpath = args.get("log_dir")
-    set_logging(log_level, log_dirpath)
+
+    loggeria.setup_conductor_logging(
+        logger_level=loggeria.LEVEL_MAP.get(log_level_name),
+        log_dirpath=args.get("log_dir"),
+        log_filename="conductor_downloader.log",
+        console_formatter=LOG_FORMATTER,
+        file_formatter=LOG_FORMATTER,
+    )    
     
     api_client.ApiClient.register_client(client_name = Downloader.CLIENT_NAME, client_version=ciocore.__version__)
 
@@ -1256,14 +1260,13 @@ def run_downloader(args):
 
 
 def set_logging(level=None, log_dirpath=None):
-    log_filepath = None
-    if log_dirpath:
-        log_filepath = os.path.join(log_dirpath, "conductor_dl_log")
+
     loggeria.setup_conductor_logging(
         logger_level=level,
+        log_dirpath=log_dirpath,
+        log_filename="conductor_downloader.log",
         console_formatter=LOG_FORMATTER,
         file_formatter=LOG_FORMATTER,
-        log_filepath=log_filepath,
     )
 
 

ciocore/downloader/log.py

@@ -0,0 +1,73 @@
+import logging
+import colorlog
+import sys
+LOGGER_NAME = "cw.download"
+
+LOG_COLORS	={
+		'DEBUG':    'purple',
+		'INFO':     'blue',
+		'WARNING':  'yellow',
+		'ERROR':    'red',
+		'CRITICAL': 'red,bg_white',
+}
+
+DEBUG_FORMATTER = colorlog.ColoredFormatter(
+	"%(log_color)s%(asctime)s %(name)s %(levelname)8s %(filename)s:%(lineno)d %(threadName)s> %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    log_colors=LOG_COLORS,
+ )
+
+INFO_FORMATTER = colorlog.ColoredFormatter(
+	'%(log_color)s%(levelname)s:%(name)s> %(message)s',
+    log_colors=LOG_COLORS,
+)
+
+LEVEL_MAP = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+    "NOTSET": logging.NOTSET,
+}
+
+class GracefulLogger(logging.Logger):
+    def setLevel(self, level):
+        super().setLevel(level)
+
+        # Define formatters based on level
+        formatter = DEBUG_FORMATTER if level == logging.DEBUG else INFO_FORMATTER
+        for handler in self.handlers:
+            handler.setFormatter(formatter)
+            
+
+class GracefulStreamHandler(colorlog.StreamHandler):
+    """
+    A custom StreamHandler that suppresses BrokenPipeError.
+    
+    This handler extends the standard logging.StreamHandler to gracefully handle 
+    BrokenPipeErrors that can occur when output streams are closed prematurely. 
+    It overrides the emit method to catch and ignore BrokenPipeError, allowing 
+    the program to continue without interruption.
+    """
+ 
+    def emit(self, record):
+        """
+        Overrides the StreamHandler.emit method to gracefully handle BrokenPipeError.
+
+        Args:
+            record (logging.LogRecord): The log record to be emitted.
+        """
+        try:
+            super().emit(record)
+        except BrokenPipeError:
+            pass
+
+logging.setLoggerClass(GracefulLogger)
+logger = colorlog.getLogger(LOGGER_NAME)
+logger.propagate = False 
+ 
+if not any(isinstance(handler, GracefulStreamHandler) for handler in logger.handlers):
+    stream_handler = GracefulStreamHandler(sys.stdout)
+    logger.addHandler(stream_handler)
+ 

ciocore/downloader/logging_download_runner.py

@@ -0,0 +1,87 @@
+"""
+Logging Download runner
+
+This module contains the LoggingDownloadRunner class. 
+
+The LoggingDownloadRunner is a derived class of DownloadRunnerBase.
+
+It registers callbacks that are called when certain events occur during the download. 
+It uses these callbacks to display progress via the logging module.
+
+"""
+
+import logging
+from ciocore.downloader.download_runner_base import DownloadRunnerBase
+from ciocore.downloader.log import LOGGER_NAME
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class LoggingDownloadRunner(DownloadRunnerBase):
+    CLIENT_NAME = "LoggingDownloadRunner"
+
+    def __init__(self, jobids=None, location=None, **kwargs):
+
+        super().__init__(jobids, location, **kwargs)
+
+        logger.debug("Assigning callbacks")
+        self.downloader.on("start", self.on_start)
+        self.downloader.on("start_task", self.on_start_task)
+        self.downloader.on("progress", self.on_progress)
+        self.downloader.on("file_done", self.on_file_done)
+        self.downloader.on("task_done", self.on_task_done)
+        self.downloader.on("done", self.on_done)
+
+    def on_start(self, evt):
+        logger.info("Starting download")
+
+    def on_start_task(self, evt):
+        logger.info("Starting task %s:%s", evt["job_id"], evt["task_id"])
+
+    def on_progress(self, evt):
+        percent = 0
+        if evt["size"] and evt["progress_bytes"]:
+            percent = round(evt["progress_bytes"] / evt["size"] * 100, 2)
+        logger.info("Progress: %s %.2f%%", evt["filepath"], percent)
+
+    def on_file_done(self, evt):
+        if evt["error"]:
+            logger.warning(
+                "File done with error: %s:%s:%s %s",
+                evt["job_id"],
+                evt["task_id"],
+                evt["filepath"],
+                evt["error"],
+            )
+        else:
+            logger.info(
+                "File done %s:%s:%s", evt["job_id"], evt["task_id"], evt["filepath"]
+            )
+
+    def on_task_done(self, evt):
+        if evt["preexisting"]:
+            logger.info(
+                "Task already existed locally %s:%s", evt["job_id"], evt["task_id"]
+            )
+        else:
+            logger.info("Task done %s:%s", evt["job_id"], evt["task_id"])
+
+    def on_done(self, evt):
+        """
+        When the job is done, check to see if any tasks were not completed.
+        """
+        logger.info("Download finished")
+        empty = True
+        for job_id, task_id, task in evt["registry"].each():
+            if task["completed_files"] < task["filecount"]:
+                logger.warning(
+                    "Task not fully downloaded %s:%s: %s/%s files.",
+                    job_id,
+                    task_id,
+                    task["completed_files"],
+                    task["filecount"],
+                )
+                empty = False
+
+        if empty:
+            logger.info("No failed tasks.")

ciocore/downloader/perpetual_downloader.py

@@ -0,0 +1,63 @@
+"""
+Perpetual Downloader
+
+Not yet tested
+"""
+import json
+import logging
+import time
+import sys
+from ciocore.downloader.base_downloader import BaseDownloader
+from ciocore.downloader.log import LOGGER_NAME
+
+logger = logging.getLogger(LOGGER_NAME)
+
+def spinning_cursor():
+    while True:
+        for cursor in '|/-\\':
+            yield cursor
+
+class PerpetualDownloader(BaseDownloader):
+    CLIENT_NAME = "PerpetualDownloader"
+    POLL_INTERVAL = 15
+    URL = "/downloads/next"
+    spinner = spinning_cursor()
+
+    def __init__(self, location, *args, **kwargs):
+        """Initialize the downloader."""
+        super().__init__(*args, **kwargs)
+        self.location = location
+        logger.debug("Initializing perpetual downloader")
+
+    def get_some_tasks(self, _):
+        """Fetch the next batch of tasks from the server.
+
+        Always set the return locator to True to signal that we should keep running this function.
+
+        This function never throws an error. If something goes wrong, it just sets the task array to be empty.
+
+        If tasks array is empty for any reason (error, filter, no tasks ready, etc.), it waits for POLL_INTERVAL seconds before trying again.
+        """
+        logger.debug("Fetching the next page of tasks")
+        params = {"count": self.page_size, "location": self.location}
+        tasks = []
+        try:
+            response, code = self.client.make_request(
+                self.URL, params=params, use_api_key=True
+            )
+            if code <= 201:
+                tasks = json.loads(response).get("data", [])
+                tasks = self.filter(tasks)
+        except Exception as exc:
+            logger.error("Error fetching download info from: %s : %s", self.URL, exc)
+
+        if not tasks:
+            for _ in range(self.POLL_INTERVAL):
+                spin_char = next(self.spinner)
+                line = f"Listening for files to download... ({spin_char})"
+                sys.stdout.write(line)
+                sys.stdout.flush()
+                sys.stdout.write('\b' * len(line))
+                time.sleep(1)
+
+        return tasks, True

ciocore/downloader/registry.py

@@ -0,0 +1,97 @@
+import copy
+
+import threading
+import logging
+from ciocore.downloader.log import LOGGER_NAME
+
+logger = logging.getLogger(LOGGER_NAME)
+
+
+class Registry(object):
+
+    def __init__(self):
+        self.data = {}
+        self.lock = threading.Lock()
+
+    def get_copy(self):
+        """
+        Get a copy of the registry.
+
+        Use a lock to ensure the registry is not modified while we're copying it.
+        """
+        with self.lock:
+            return copy.deepcopy(self.data)
+
+    def each(self):
+        """
+        Iterate over all tasks in the registry.
+
+        Use a lock to ensure the registry is not modified while we're iterating over it.
+        """
+        with self.lock:
+            for job_id, job in self.data.items():
+                for task_id, task in job.items():
+                    yield job_id, task_id, task
+
+    def register_task(self, task_info):
+        """
+        Register a task as active
+
+        The registry is accessed in a thread-safe manner using a lock.
+        """
+        job_id = task_info["job_id"]
+        task_id = task_info["task_id"]
+        with self.lock:
+            if job_id not in self.data:
+                self.data[job_id] = {}
+
+            if task_id in self.data[job_id]:
+                logger.debug(
+                    "Task %s for job %s is already in registry. Skipping.",
+                    task_id,
+                    job_id,
+                )
+                return False
+
+            self.data[job_id][task_id] = {
+                "download_id": task_info["download_id"],
+                "filecount": len(task_info["files"]),
+                "completed_files": 0,
+                "preexisting_files": 0,
+                "size": task_info["size"],
+            }
+        return True
+
+    def update_task(self, file_done_event):
+        """
+        Update the registry each time a file is done.
+
+        Access the registry in a thread-safe manner using a lock.
+
+        Steps:
+        1. Get the task from the registry
+        2. Increment the completed_files count
+        3. If the file was preexisting, increment the preexisting_files count too
+        4. If the task is now complete:
+            c. Remove the task from the registry
+        5. Return the task copy so that the event_dispatcher can let handlers know the task is done.
+
+        """
+
+        job_id = file_done_event["job_id"]
+        task_id = file_done_event["task_id"]
+        with self.lock:
+            task = self.data.get(job_id, {}).get(task_id)
+            if  not task:
+                return None
+            task["completed_files"] += 1
+            if file_done_event["preexisting"]:
+                task["preexisting_files"] += 1
+
+            task_copy = task.copy()
+
+            # Only really need ==, but I'm paranoid
+            if task["completed_files"] >= task["filecount"]:
+                del self.data[job_id][task_id]
+
+        return task_copy

ciocore/downloader/reporter.py

@@ -0,0 +1,135 @@
+"""
+This module contains the Reporter class. 
+
+It registers callbacks with the with the provided downloader instance that allow it to report "downloaded" or "pending" status back to the server.
+
+It is set up in the download_runner_base module. Classes that derive from DownloadRunnerBase, such as LoggingDownloadRunner, do not need to be concerned with the details of the Reporter class.
+"""
+
+import json
+import logging
+from concurrent.futures import ThreadPoolExecutor
+
+from ciocore import api_client
+from ciocore.downloader.log import LOGGER_NAME
+
+STATUS_ENDPOINT = "/downloads/status"
+STATUS_DOWNLOADED = "downloaded"
+STATUS_PENDING = "pending"
+
+logger = logging.getLogger(LOGGER_NAME)
+ 
+class Reporter(object):
+
+    def __init__(self, downloader, client=api_client.ApiClient(), num_threads=1):
+
+        self.downloader = downloader
+
+        self.num_threads = num_threads
+        self.client = client
+        self.executor = None
+
+        logger.debug("Assigning reporter callbacks")
+        self.downloader.on("task_done", self.on_task_done)
+        self.downloader.on("done", self.on_done)
+
+    def __enter__(self):
+        self.executor = ThreadPoolExecutor(max_workers=self.num_threads)
+        return self  # Optionally return this reporter
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.executor.shutdown()
+        # Handle exceptions, from inside the with block
+        if exc_type:
+            logger.exception("Error running downloader: %s", exc_value)
+            # return False to propagate the exception
+            return False
+            
+
+
+    def report_task_status(
+        self, download_id, status=STATUS_DOWNLOADED, bytes_in_task=0
+    ):
+        """
+        Make a request to the server to report the status of a task.
+
+        If the user interrupted the download, then we set the task status to pending to be safe.
+        """
+        if self.downloader.interrupt_flag.is_set():
+            status = STATUS_PENDING
+
+        bytes_to_download = 0 if status == STATUS_DOWNLOADED else bytes_in_task
+
+        data = {
+            "download_id": download_id,
+            "status": status,
+            "bytes_downloaded": 0,
+            "bytes_to_download": bytes_to_download,
+        }
+        json_data = json.dumps(data)
+        try:
+            self.client.make_request(STATUS_ENDPOINT, data=json_data, use_api_key=True)
+        except Exception as exc:
+            data["error"] = str(exc)
+        return data
+
+    def on_task_done(self, evt):
+        """
+        Callback to run on a task-done event. Report status back to the server.
+        
+        Note, the task may consist entirely of preexisting files. Nevertheless, we report the task as downloaded.
+        """
+
+        future = self.executor.submit(
+            self.report_task_status,
+            evt["download_id"],
+            status=STATUS_DOWNLOADED,
+            bytes_in_task=evt["size"],
+        )
+        future.add_done_callback(
+            lambda f, job_id=evt["job_id"], task_id=evt["task_id"]: log_report_result(
+                f.result(), job_id, task_id
+            )
+        )
+
+    def on_done(self, evt):
+        """
+        When the job is done, check to see if any tasks were not completed.
+
+        If we find any, then report them back to the server as pending.
+        """
+        logger.debug("Download done. Reporting remaining task statuses to server")
+        for job_id, task_id, task in evt["registry"].each():
+            if task["completed_files"] < task["filecount"]:
+
+                future = self.executor.submit(
+                    self.report_task_status,
+                    task["download_id"],
+                    status=STATUS_PENDING,
+                    bytes_in_task=task["size"],
+                )
+                future.add_done_callback(
+                    lambda f, job_id=job_id, task_id=task_id: log_report_result(
+                        f.result(), job_id, task_id
+                    )
+                )
+
+
+def log_report_result(report_result, job_id, task_id):
+    """Log the report result."""
+    if report_result.get("error"):
+        logger.error(
+            "Error reporting task to server:  %s:%s (%s) %s",
+            job_id,
+            task_id,
+            report_result["download_id"],
+            report_result["error"],
+        )
+        return
+    logger.debug(
+        "Reported task to server: %s:%s (%s) %s",
+        job_id,
+        task_id,
+        report_result["download_id"],
+        report_result["status"],
+    )

ciocore/exceptions.py

@@ -49,13 +49,19 @@ class InvalidPathException(Exception):
     pass
 
 
-class UploaderMissingFile(Exception):
+class UploadError(Exception):
+    """
+    An upload failed
+    """
+
+
+class UploaderMissingFile(UploadError):
     """
     A file is missing
     """
 
 
-class UploaderFileModified(Exception):
+class UploaderFileModified(UploadError):
     """
     Something wrong with a local file
     """