idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/analysis/csv_analyzer.py

@@ -0,0 +1,103 @@
+"""idmtools CSVAnalyzer.
+
+Example of a csv analyzer to concatenate csv results into one csv from your experiment's simulations.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+from typing import Dict
+import pandas as pd
+from idmtools.entities import IAnalyzer
+from idmtools.entities.ianalyzer import ANALYSIS_ITEM_MAP_DATA_TYPE, ANALYZABLE_ITEM
+
+
+class CSVAnalyzer(IAnalyzer):
+    """
+    Provides an analyzer for CSV output.
+
+    Examples:
+        .. _simple-csv-example:
+
+        Simple Example
+          This example covers the basic usage of the CSVAnalyzer
+
+          .. literalinclude:: ../../examples/analyzers/example_analysis_CSVAnalyzer.py
+
+        .. _multiple-csvs:
+
+        Multiple CSVs
+            This example covers analyzing multiple CSVs
+
+            .. literalinclude:: ../../examples/analyzers/example_analysis_MultiCSVAnalyzer.py
+    """
+    # Arg option for analyzer init are uid, working_dir, parse (True to leverage the :class:`OutputParser`;
+    # False to get the raw data in the :meth:`select_simulation_data`), and filenames
+    # In this case, we want parse=True, and the filename(s) to analyze
+    def __init__(self, filenames, output_path="output_csv"):
+        """
+        Initialize our analyzer.
+
+        Args:
+            filenames: Filenames we want to pull
+            output_path: Output path to write the csv
+        """
+        super().__init__(parse=True, filenames=filenames)
+        # Raise exception early if files are not csv files
+        if not all(['csv' in os.path.splitext(f)[1].lower() for f in self.filenames]):
+            raise Exception('Please ensure all filenames provided to CSVAnalyzer have a csv extension.')
+
+        self.output_path = output_path
+
+    def initialize(self):
+        """
+        Initialize on run. Create an output directory.
+
+        Returns:
+            None
+        """
+        self.output_path = os.path.join(self.working_dir, self.output_path)
+
+        # Create the output path
+        if not os.path.exists(self.output_path):
+            os.makedirs(self.output_path)
+
+    # Map is called to get for each simulation a data object (all the metadata of the simulations) and simulation object
+    def map(self, data: ANALYSIS_ITEM_MAP_DATA_TYPE, simulation: ANALYZABLE_ITEM) -> pd.DataFrame:
+        """
+        Map each simulation/workitem data here.
+
+        The data is a mapping of files -> content(in this case, dataframes since it is csvs parsed).
+
+        Args:
+            data: Data mapping of files -> content
+            simulation: Simulation/Workitem we are mapping
+
+        Returns:
+            Items joined together into a dataframe.
+        """
+        # If there are 1 to many csv files, concatenate csv data columns into one dataframe
+        concatenated_df = pd.concat(list(data.values()), axis=0, ignore_index=True, sort=True)
+        return concatenated_df
+
+    # In reduce, we are printing the simulation and result data filtered in map
+    def reduce(self, all_data: Dict[ANALYZABLE_ITEM, pd.DataFrame]):
+        """
+        Reduce(combine) all the data from our mapping.
+
+        Args:
+            all_data: Mapping of our data in form Item(Simulation/Workitem) -> Mapped dataframe
+
+        Returns:
+            None
+        """
+        results = pd.concat(list(all_data.values()), axis=0,  # Combine a list of all the sims csv data column values
+                            keys=[str(k.uid) for k in all_data.keys()],  # Add a hierarchical index with the keys option
+                            names=['SimId'])  # Label the index keys you create with the names option
+        results.index = results.index.droplevel(1)  # Remove default index
+
+        # Make a directory labeled the exp id to write the csv results to
+        first_sim = list(all_data.keys())[0]  # get first Simulation
+        exp_id = first_sim.experiment.id  # Set the exp id from the first sim data
+        output_folder = os.path.join(self.output_path, exp_id)
+        os.makedirs(output_folder, exist_ok=True)
+        results.to_csv(os.path.join(output_folder, self.__class__.__name__ + '.csv'))

idmtools/analysis/download_analyzer.py

@@ -0,0 +1,96 @@
+"""idmtools Download analyzer.
+
+Download Analyzer.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+from logging import getLogger
+from idmtools.entities.ianalyzer import IAnalyzer, ANALYSIS_REDUCE_DATA_TYPE, ANALYZABLE_ITEM, ANALYSIS_ITEM_MAP_DATA_TYPE
+
+logger = getLogger(__name__)
+
+
+class DownloadAnalyzer(IAnalyzer):
+    """
+    A simple base class that will download the files specified in filenames without further treatment.
+
+    Can be used by creating a child class:
+
+    .. code-block:: python
+
+        class InsetDownloader(DownloadAnalyzer):
+            filenames = ['output/InsetChart.json']
+
+    Or by directly calling it:
+
+    .. code-block:: python
+
+        analyzer = DownloadAnalyzer(filenames=['output/InsetChart.json'])
+
+
+    Examples:
+        .. literalinclude:: ../../examples/analyzers/example_analysis_DownloadAnalyzer.py
+    """
+
+    def reduce(self, all_data: ANALYSIS_REDUCE_DATA_TYPE):
+        """
+        Combine the :meth:`map` data for a set of items into an aggregate result. In this case, for downloading, we just ignore it because there is no reduction.
+
+        Args:
+            all_data: Dictionary in form item->map result where item is Simulations or WorkItems
+
+        Returns:
+            None
+        """
+        pass
+
+    def __init__(self, filenames=None, output_path="output", **kwargs):
+        """Constructor of the analyzer."""
+        super().__init__(filenames=filenames, parse=False, **kwargs)
+        self.output_path = output_path
+
+    def initialize(self):
+        """
+        Initialize our sim. In this case, we create our output directory.
+
+        Returns:
+            None
+        """
+        self.output_path = os.path.join(self.working_dir, self.output_path)
+        os.makedirs(self.output_path, exist_ok=True)
+
+    def get_item_folder(self, item: ANALYZABLE_ITEM):
+        """
+        Concatenate the specified top-level output folder with the item ID.
+
+        Args:
+            item: A simulation output parsing thread.
+
+        Returns:
+            The name of the folder to download this simulation's output to.
+        """
+        return os.path.join(self.output_path, str(item.uid))
+
+    def map(self, data: ANALYSIS_ITEM_MAP_DATA_TYPE, item: ANALYZABLE_ITEM):
+        """
+        Provide a map of filenames->data for each item. We then download each of these files to our output folder.
+
+        Args:
+            data: Map filenames->data
+            item: Item we are mapping.
+
+        Returns:
+            None
+        """
+        # Create a folder for the current simulation/item
+        sim_folder = self.get_item_folder(item)
+        os.makedirs(sim_folder, exist_ok=True)
+
+        # Create the requested files
+        for filename in self.filenames:
+            file_path = os.path.join(sim_folder, os.path.basename(filename))
+
+            logger.debug(f'Writing to path: {file_path}')
+            with open(file_path, 'wb') as outfile:
+                outfile.write(data[filename])

idmtools/analysis/map_worker_entry.py

@@ -0,0 +1,100 @@
+"""
+We define our map entry items here for analysis framework.
+
+Most of these function are used either to initialize a thread or to handle exceptions while executing.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import itertools
+from logging import getLogger, DEBUG
+from idmtools.core.interfaces.ientity import IEntity
+from idmtools.utils.file_parser import FileParser
+from typing import TYPE_CHECKING, Dict
+from idmtools.core.interfaces.iitem import IItem
+from idmtools.entities.ianalyzer import TAnalyzerList
+from idmtools.utils.general import FilterSafeItem
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools.entities.iplatform import IPlatform
+
+logger = getLogger(__name__)
+
+
+def map_item(item: IItem) -> Dict[str, Dict]:
+    """
+    Initialize some worker-global values; a worker process entry point for analyzer item-mapping.
+
+    Args:
+        item: The item (often simulation) to process.
+
+    Returns:
+        Dict[str, Dict]
+    """
+    # Retrieve the global variables coming from the pool initialization
+
+    if logger.isEnabledFor(DEBUG):
+        logger.debug(f"Init item {item.uid} in worker")
+    analyzers = map_item.analyzers
+    platform = map_item.platform
+    return _get_mapped_data_for_item(item, analyzers, platform)
+
+
+def _get_mapped_data_for_item(item: IEntity, analyzers: TAnalyzerList, platform: 'IPlatform') -> Dict[str, Dict]:
+    """
+    Get mapped data from an item.
+
+    Args:
+        item: The :class:`~idmtools.entities.iitem.IItem` object to call analyzer
+            :meth:`~idmtools.analysis.AddAnalyzer.map` methods on.
+        analyzers: The :class:`~idmtools.analysis.IAnalyzer` items with
+            :meth:`~idmtools.analysis.AddAnalyzer.map` methods to call on the provided items.
+        platform: A platform object to query for information.
+
+    Returns:
+        Dict[str, Dict] - Array mapping file data to from str to contents
+
+    """
+    try:
+        # determine which analyzers (and by extension, which filenames) are applicable to this item
+        # ensure item has a platform
+        item.platform = platform
+        analyzers_to_use = [a for a in analyzers if a.filter(FilterSafeItem(item))]
+        analyzer_uids = [a.uid for a in analyzers]
+
+        filenames = set(itertools.chain(*(a.filenames for a in analyzers_to_use)))
+        filenames = [f.replace("\\", '/') for f in filenames]
+
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f"Analyzers to use on item: {str(analyzer_uids)}")
+            logger.debug(f"Filenames to analyze: {filenames}")
+
+        # The byte_arrays will associate filename with content
+        if len(filenames) > 0:
+            file_data = platform.get_files(item, filenames)
+        else:
+            file_data = dict()
+
+        # Selected data will be a dict with analyzer.uid: data  entries
+        selected_data = {}
+        for analyzer in analyzers_to_use:
+            # If the analyzer needs the parsed data, parse
+            if analyzer.parse:
+                logger.debug(f'Parsing content for {analyzer.uid}')
+                data = {filename: FileParser.parse(filename, content) for filename, content in file_data.items() if filename in analyzer.filenames}
+            else:
+                # If the analyzer doesnt wish to parse, give the raw data
+                data = {filename: content for filename, content in file_data.items() if filename in analyzer.filenames}
+
+            # run the mapping routine for this analyzer and item
+            logger.debug("Running map on selected data")
+            selected_data[analyzer.uid] = analyzer.map(data, item)
+
+        # Store all analyzer results for this item in the result cache
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f"Setting result to cache on {item.id}")
+        logger.debug(f"Wrote Setting result to cache on {item.id}")
+    except Exception as e:
+        e.item = item
+        logger.error(e)
+        raise e
+    return selected_data

idmtools/analysis/platform_analysis_bootstrap.py

@@ -0,0 +1,94 @@
+"""
+This script is executed as entrypoint in the docker SSMT worker.
+
+Its role is to collect the experiment ids and analyzers and run the analysis.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import argparse
+import os
+import pickle
+import sys
+from logging import getLogger
+from pydoc import locate
+
+
+sys.path.append(os.path.dirname(__file__))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser("PlatformAnalysis bootstrap")
+    parser.add_argument("--experiment-ids", default=None, help="A comma separated list of experiments to analyze")
+    parser.add_argument("--simulation-ids", default=None, help="A comma separated list of simulations to analyze")
+    parser.add_argument("--work-item-ids", default=None, help="A comma separated list of work items to analyze")
+    parser.add_argument("--analyzers", help="Commas separated list of analyzers")
+    parser.add_argument("--block", help="Configuration block to use")
+    parser.add_argument("--verbose", default=False, action="store_true", help="Verbose logging")
+    parser.add_argument("--pre-run-func", default=None, help="List of function to run before starting analysis. Useful to load packages up in docker container before run")
+    parser.add_argument("--analyzer-manager-args-file", default=None, help="Path to extra arguments for analyzer manager")
+    parser.add_argument("--platform-args", default=None, help="Arguments used to create Platform")
+
+    args = parser.parse_args()
+    if args.verbose:
+        # enable verbose logging before we load idmtools
+        os.environ['IDMTOOLS_LOGGING_LEVEL'] = 'DEBUG'
+        os.environ['IDMTOOLS_LOGGING_CONSOLE'] = '1'
+
+    # delay loading idmtools so we can change log level through environment
+    from idmtools.core import ItemType
+    from idmtools.core.platform_factory import Platform
+    from idmtools.analysis.analyze_manager import AnalyzeManager
+
+    logger = getLogger('SSMT Analysis')
+
+    if args.pre_run_func:
+        import pre_run
+        getattr(pre_run, args.pre_run_func)()
+
+    item_ids = []
+    # Get the experiments, analyzers and platform
+    if args.experiment_ids:
+        experiments = args.experiment_ids.split(",")
+        for experiment in experiments:
+            experiment_tuple = (experiment, ItemType.EXPERIMENT)
+            item_ids.append(experiment_tuple)
+
+    # Get the simulations, analyzers and platform
+    if args.simulation_ids:
+        simulations = args.simulation_ids.split(",")
+        for simulation in simulations:
+            simulation_tuple = (simulation, ItemType.SIMULATION)
+            item_ids.append(simulation_tuple)
+
+    # Get the experiments, analyzers and platform
+    if args.work_item_ids:
+        work_items = args.work_item_ids.split(",")
+        for wi in work_items:
+            wi_tuple = (wi, ItemType.WORKFLOW_ITEM)
+            item_ids.append(wi_tuple)
+
+    # load analyzer args pickle file
+    analyzer_config = pickle.load(open(r"analyzer_args.pkl", 'rb'))
+
+    # Create analyzers
+    analyzers = []
+    for analyzer in args.analyzers.split(","):
+        A = locate(analyzer)
+        a = A(**analyzer_config[analyzer])
+        analyzers.append(a)
+
+    if not all(analyzers):
+        raise Exception("Not all analyzers could be found...\n{}".format(",".join(analyzers)))
+
+    if args.analyzer_manager_args_file is not None:
+        logger.info(f"Loading extra AnalyzerManager args from {args.analyzer_manager_args_file}")
+        with open(args.analyzer_manager_args_file, 'rb') as pin:
+            extra_args = pickle.load(pin)
+
+    # load platform args pickle file
+    platform_args = pickle.load(open(r"platform_args.pkl", 'rb'))
+
+    # get platform
+    platform = Platform(args.block, **platform_args)
+    logger.info(f"Analyzer Manager called with the following extra arguments: {extra_args}")
+    am = AnalyzeManager(platform=platform, ids=item_ids, analyzers=analyzers, **extra_args)
+    am.analyze()

idmtools/analysis/platform_anaylsis.py

@@ -0,0 +1,291 @@
+"""
+Platform Analysis is a wrapper to allow execution of analysis through SSMT vs Locally.
+
+Running remotely has great advantages over local execution with the biggest being more compute resources and less data transfer.
+Platform Analysis tries to make the process of running remotely similar to local execution.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import re
+from typing import List, Callable, Union, Type, Dict, Any
+import inspect
+import os
+import pickle
+from logging import getLogger, DEBUG
+from idmtools.assets import Asset, AssetCollection
+from idmtools.assets.file_list import FileList
+from idmtools.config import IdmConfigParser
+from idmtools.entities import IAnalyzer
+from idmtools.entities.iplatform import IPlatform
+from idmtools.entities.iplatform_default import AnalyzerManagerPlatformDefault
+from idmtools.utils.info import get_help_version_url
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+class PlatformAnalysis:
+    """
+    PlatformAnalysis allows remote Analysis on the server.
+
+    See Also:
+        :py:class:`idmtools.analysis.analyze_manager.AnalyzeManager`
+    """
+
+    def __init__(self, platform: IPlatform, analyzers: List[Type[IAnalyzer]],
+                 experiment_ids: List['str'] = [], simulation_ids: List['str'] = [], work_item_ids: List['str'] = [],
+                 analyzers_args=None, analysis_name: str = 'WorkItem Test', tags=None,
+                 additional_files: Union[FileList, AssetCollection, List[str]] = None, asset_collection_id=None,
+                 asset_files: Union[FileList, AssetCollection, List[str]] = None, wait_till_done: bool = True,
+                 idmtools_config: str = None, pre_run_func: Callable = None, wrapper_shell_script: str = None,
+                 verbose: bool = False, extra_args: Dict[str, Any] = None):
+        """
+        Initialize our platform analysis.
+
+        Args:
+            platform: Platform
+            experiment_ids: Experiment ids
+            simulation_ids: Simulation ids
+            work_item_ids: WorkItem ids
+            analyzers: Analyzers to run
+            analyzers_args: Arguments for our analyzers
+            analysis_name: Analysis name
+            tags: Tags for the workitem
+            additional_files: Additional files for server analysis
+            asset_collection_id: Asset Collection to use
+            asset_files: Asset files to attach
+            wait_till_done: Wait until analysis is done
+            idmtools_config: Optional path to idmtools.ini to use on server. Mostly useful for development
+            pre_run_func: A function (with no arguments) to be executed before analysis starts on the remote server
+            wrapper_shell_script: Optional path to a wrapper shell script. This script should redirect all arguments to command passed to it. Mostly useful for development purposes
+            verbose: Enables verbose logging remotely
+            extra_args: Optional extra arguments to pass to AnalyzerManager on the server side. See :meth:`~idmtools.analysis.analyze_manager.AnalyzeManager.__init__`
+
+        See Also:
+            :meth:`idmtools.analysis.analyze_manager.AnalyzeManager.__init__`
+        """
+        self.platform = platform
+        self.experiment_ids = experiment_ids or []
+        self.simulation_ids = simulation_ids or []
+        self.work_item_ids = work_item_ids or []
+        self.analyzers = analyzers
+        self.analyzers_args = analyzers_args
+        self.analysis_name = analysis_name
+        self.tags = tags
+        if isinstance(additional_files, list):
+            additional_files = AssetCollection(additional_files)
+        elif isinstance(additional_files, FileList):
+            additional_files = additional_files.to_asset_collection()
+        self.additional_files: AssetCollection = additional_files or AssetCollection()
+        self.asset_collection_id = asset_collection_id
+        if isinstance(asset_files, list):
+            asset_files = AssetCollection(asset_files)
+        elif isinstance(asset_files, FileList):
+            asset_files = asset_files.to_asset_collection()
+        self.asset_files: AssetCollection = asset_files or AssetCollection()
+        self.wi = None
+        self.wait_till_done = wait_till_done
+        self.idmtools_config = idmtools_config
+        self.pre_run_func = pre_run_func
+        self.wrapper_shell_script = wrapper_shell_script
+        self.shell_script_binary = "/bin/bash"
+        self.verbose = verbose
+        self.extra_args = extra_args if extra_args else dict()
+
+        self.validate_args()
+
+    def analyze(self, check_status=True):
+        """
+        Analyze remotely.
+
+        Args:
+            check_status: Should we check status
+
+        Returns:
+            None
+
+        Notes:
+            TODO: check_status is not being used
+        """
+        command = self._prep_analyze()
+
+        logger.debug(f"Command: {command}")
+        from idmtools_platform_comps.ssmt_work_items.comps_workitems import SSMTWorkItem
+
+        ac = AssetCollection.from_id(self.asset_collection_id,
+                                     platform=self.platform) if self.asset_collection_id else AssetCollection()
+        ac.add_assets(self.asset_files)
+        self.wi = SSMTWorkItem(name=self.analysis_name, command=command, tags=self.tags,
+                               transient_assets=self.additional_files, assets=ac,
+                               related_experiments=self.experiment_ids,
+                               related_simulations=self.simulation_ids,
+                               related_work_items=self.work_item_ids
+                               )
+
+        # Run the workitem
+        self.platform.run_items(self.wi)
+        if self.wait_till_done:
+            self.platform.wait_till_done(self.wi)
+        logger.debug(f"Status: {self.wi.status}")
+
+    def _prep_analyze(self):
+        """
+        Prepare for analysis.
+
+        Returns:
+            None
+        """
+        # Add the platform_analysis_bootstrap.py file to the collection
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        self.additional_files.add_or_replace_asset(os.path.join(dir_path, "platform_analysis_bootstrap.py"))
+        # check if user gave us an override to idmtools config
+        if self.idmtools_config:
+            self.additional_files.add_or_replace_asset(self.idmtools_config)
+        else:
+            # look for one from idmtools.
+            config_path = IdmConfigParser.get_config_path()
+            if config_path and os.path.exists(config_path):
+                if logger.isEnabledFor(DEBUG):
+                    logger.debug(f"Adding config file: {config_path}")
+                self.additional_files.add_or_replace_asset(config_path)
+
+        # build analyzer args dict
+        args_dict = {}
+        a_args = zip(self.analyzers, self.analyzers_args)
+        for a, g in a_args:
+            args_dict[f"{inspect.getmodulename(inspect.getfile(a))}.{a.__name__}"] = g
+        if self.pre_run_func:
+            self.__pickle_pre_run()
+        # save pickle file as a temp file
+        self.__pickle_analyzers(args_dict)
+
+        # Add all the analyzers files
+        for a in self.analyzers:
+            self.additional_files.add_or_replace_asset(inspect.getfile(a))
+
+        # add our extra arguments for analyzer manager
+        if 'max_workers' not in self.extra_args:
+            am_defaults: List[AnalyzerManagerPlatformDefault] = self.platform.get_defaults_by_type(
+                AnalyzerManagerPlatformDefault)
+            if len(am_defaults):
+                if logger.isEnabledFor(DEBUG):
+                    logger.debug(f"Setting max workers to comps default of: {am_defaults[0].max_workers}")
+                self.extra_args['max_workers'] = am_defaults[0].max_workers
+
+        # Create the command
+        command = ''
+        if self.wrapper_shell_script:
+            self.additional_files.add_or_replace_asset(self.wrapper_shell_script)
+            command += f'{self.shell_script_binary} {os.path.basename(self.wrapper_shell_script)} '
+        command += "python3 platform_analysis_bootstrap.py"
+        # Add the experiments
+        if self.experiment_ids:
+            command += f' --experiment-ids {",".join(self.experiment_ids)}'
+        # Add the simulations
+        if self.simulation_ids:
+            command += f' --simulation-ids {",".join(self.simulation_ids)}'
+        # Add the work items
+        if self.work_item_ids:
+            command += f' --work-item-ids {",".join(self.work_item_ids)}'
+        # Add the analyzers
+        command += " --analyzers {}".format(
+            ",".join(f"{inspect.getmodulename(inspect.getfile(a))}.{a.__name__}" for a in self.analyzers))
+
+        if self.pre_run_func:
+            command += f" --pre-run-func {self.pre_run_func.__name__}"
+
+        # Pickle the extra args
+        if len(self.extra_args):
+            from idmtools.analysis.analyze_manager import AnalyzeManager
+            argspec = inspect.signature(AnalyzeManager.__init__)
+            for argname, value in self.extra_args.items():
+                if argname not in argspec.parameters:
+                    raise ValueError(
+                        f"AnalyzerManager does not support the argument {argname}. Valid args are {' '.join([str(s) for s in argspec.parameters.keys()])}. See {get_help_version_url('idmtools.analysis.analyze_manager.html#idmtools.analysis.analyze_manager.AnalyzeManager')} for a valid list of arguments.")
+                # TODO do type validations later
+            self.additional_files.add_or_replace_asset(
+                Asset(filename="extra_args.pkl", content=pickle.dumps(self.extra_args)))
+            command += " --analyzer-manager-args-file extra_args.pkl"
+
+        self.__pickle_platform_args()
+        command += " --platform-args platform_args.pkl"
+
+        # Add platform
+        ssmt_config_block = f"{self.platform._config_block}_SSMT"
+        command += " --block {}".format(ssmt_config_block)
+        if self.verbose:
+            command += " --verbose"
+
+        return command
+
+    def __pickle_analyzers(self, args_dict):
+        """
+        Pickle our analyzers and add as assets.
+
+        Args:
+            args_dict: Analyzer and args
+
+        Returns:
+            None
+        """
+        self.additional_files.add_or_replace_asset(Asset(filename='analyzer_args.pkl', content=pickle.dumps(args_dict)))
+
+    def __pickle_pre_run(self):
+        """
+        Pickle objects before we run and add items as assets.
+
+        Returns:
+            None
+        """
+        source = inspect.getsource(self.pre_run_func).splitlines()
+        space_base = 0
+        while source[0][space_base] == " ":
+            space_base += 1
+        replace_expr = re.compile("^[ ]{" + str(space_base) + "}")
+        new_source = []
+        for line in source:
+            new_source.append(replace_expr.sub("", line))
+
+        self.additional_files.add_or_replace_asset(Asset(filename="pre_run.py", content="\n".join(new_source)))
+
+    def __pickle_platform_args(self):
+        """
+        Pickle platform args and add as assets.
+
+        Returns:
+            None
+        """
+        # Pickle the platform args
+        platform_kwargs = self.platform._kwargs
+        self.additional_files.add_or_replace_asset(
+            Asset(filename="platform_args.pkl", content=pickle.dumps(platform_kwargs)))
+
+    def validate_args(self):
+        """
+        Validate arguments for the platform analysis and analyzers.
+
+        Returns:
+            None
+        """
+        if self.analyzers_args is None:
+            self.analyzers_args = [{}] * len(self.analyzers)
+            return
+
+        self.analyzers_args = [g if g is not None else {} for g in self.analyzers_args]
+
+        if len(self.analyzers_args) < len(self.analyzers):
+            self.analyzers_args = self.analyzers_args + [{}] * (len(self.analyzers) - len(self.analyzers_args))
+            return
+
+        if len(self.analyzers) < len(self.analyzers_args):
+            user_logger.error("two list 'analyzers' and 'analyzers_args' must have the same length.")
+            exit()
+
+    def get_work_item(self):
+        """
+        Get work item being using to run analysis job on server.
+
+        Returns:
+            Workflow item
+        """
+        return self.wi