idmtools-test 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools_test/inputs/python/ye_seir_model/Assets/SEIR_model.py

@@ -0,0 +1,252 @@
+import sys
+import os
+import random
+current_directory = os.path.dirname(__file__)
+LIBRARY_PATH = os.path.join(current_directory, 'site-packages')  # Need to site-packages level!!!
+
+sys.path.insert(0, LIBRARY_PATH)
+sys.path.append(os.path.join(current_directory))
+import pandas as pd
+from pathlib import Path
+# import utils
+
+assets_dir = os.path.abspath(os.path.dirname(__file__))
+#current_directory = os.getcwd()
+p_version = sys.version_info
+if p_version.major == 3:
+    if p_version.minor == 7:
+        sys.path.append(str(Path(os.path.join(assets_dir, './MyExternalLibrary/Python37')).resolve().absolute()))
+    elif p_version.minor == 6:
+        sys.path.append(str(Path(os.path.join(assets_dir, './MyExternalLibrary/Python36')).resolve().absolute()))
+else:
+    print("Sorry, this model only supports Python 3.6, 3.7")
+import dtk_nodedemog as nd  # noqa
+import dtk_generic_intrahost as gi  # noqa
+from config_sim import configure_simulation  # noqa
+
+
+class Constant():
+    SIMULATION_TIMESTEP = "SIMULATION_TIMESTEP"
+    STATISTICAL_POPULATION = "STATISTICAL_POPULATION"
+    CONTAGION = "CONTAGION"
+    NUM_INFECTED = "NUM_INFECTED"
+    NUM_NEW_INFECTIONS = "NUM_NEW_INFECTIONS"
+    hum_id = "individual_id"
+    is_infected = "is_infected"
+    infections = "infections"
+    infectiousness = "infectiousness"
+    immunity = "immunity"
+
+
+class Persion():
+    def __init__(self, mcw, age, gender, id):
+        self.mcw = mcw
+        self.age = age
+        self.sex = gender
+        self.id = id
+
+
+# Write a SEIR class using dtk_nodedemog and dtk_generic_intrahost
+class SEIR():
+    def __init__(self, config_template="config.json", simulation_duration=10, initial_population=1000,
+                 outbreak_timestep=0, outbreak_demographic_coverage=0.01, outbreak_ignore_immunity=True,
+                 other_config_params: dict = None):
+        """
+        Define a SEIR model with the following parameeters
+        :param config_template: template file for configuration
+        :param simulation_duration: number of time step for one simulation
+        :param initial_population: number of initial population
+        :param outbreak_timestep: the day to start distributing the outbreak
+        :param outbreak_demographic_coverage: the fraction of individuals that will receive the outbreak
+        :param outbreak_ignore_immunity: iIndividuals will be force-infected regardless of actual immunity level when set to true
+        :param other_config_params: other parameter/value pairs in config.json
+        """
+        self.human_pop = {}  # dictionary of individual objects at run time
+        self.well_mixed_contagion_pool = []
+        self.statistical_population = []
+        self.num_infected = []
+        self.num_new_infections = []
+        self.individual_df = None
+        self.node_df = None
+        self.timestep = 0
+        self.config_template = config_template
+        self.simulation_duration = simulation_duration
+        self.initial_population = initial_population
+        self.outbreak_timestep = outbreak_timestep
+        self.outbreak_demographic_coverage = outbreak_demographic_coverage
+        self.outbreak_ignore_immunity = outbreak_ignore_immunity
+        self.other_config_params = other_config_params
+
+    def create_person_callback(self, mcw, age, gender):
+        new_id = gi.create((gender, age, mcw))
+        person = Persion(mcw, age, gender, new_id)
+        if new_id in self.human_pop:
+            raise Exception(" individual {0} is already created.".format(new_id))
+        else:
+            self.human_pop[new_id] = person
+
+    def expose_callback(self, action, prob, individual_id):
+        random_draw = random.random()
+        if self.timestep == self.outbreak_timestep:
+            print(f"expose {individual_id} with outbreak.")
+            if random_draw < self.outbreak_demographic_coverage:
+                if self.outbreak_ignore_immunity:
+                    print(f"Let's infect {individual_id} with outbreak ignore immunity.")
+                    return 1
+                else:
+                    return self.infect_base_on_immunity(individual_id)
+            else:
+                print(f"Let's NOT infect {individual_id} with outbreak.")
+                return 0
+        else:
+            print(f"expose {individual_id} with transmission.")
+            if random_draw < self.well_mixed_contagion_pool[-1]:
+                return self.infect_base_on_immunity(individual_id)
+            else:
+                print(f"Let's NOT infect {individual_id} based on random draw.")
+                return 0
+
+    def deposit_callback(self, contagion, individual):
+        self.well_mixed_contagion_pool[-1] += contagion
+        print(f"Depositing {contagion} contagion creates total of {self.well_mixed_contagion_pool[-1]}.")
+        return
+
+    def infect_base_on_immunity(self, individual_id):
+        random_draw = random.random()
+        if random_draw < gi.get_immunity(individual_id):
+            print(f"Let's infect {individual_id} base on immunity.")
+            return 1
+        else:
+            print(f"Let's NOT infect {individual_id} base on immunity.")
+            return 0
+
+    def run(self):
+        """
+        This is the method to run the SEIR model and generate output files.
+        """
+        print("\tWe cleared out human_pop. Should get populated via populate_from_files and callback...")
+        gi.reset()
+        nd.reset()
+        self.human_pop = {}
+
+        configure_simulation(initial_population=self.initial_population,
+                             nd_template_filename=os.path.join(current_directory, self.config_template),
+                             demo_template_filename=os.path.join(assets_dir, "templates/demographics_template.json"),
+                             other_config_params=self.other_config_params)
+
+        # set callbacks
+        nd.set_callback(self.create_person_callback)
+        nd.populate_from_files()
+        gi.my_set_callback(self.expose_callback)
+        gi.set_deposit_callback(self.deposit_callback)
+
+        data = {Constant.SIMULATION_TIMESTEP: [],
+                Constant.hum_id: [],
+                Constant.is_infected: [],
+                Constant.infectiousness: [],
+                Constant.immunity: []
+                }
+        infected = dict()
+        for t in range(self.simulation_duration):
+            self.timestep = t
+            # logging.info("Updating individuals at timestep {0}.".format(t))
+            self.statistical_population.append(len(self.human_pop))
+            self.well_mixed_contagion_pool.append(0)  # 100% decay at the end of every time step
+
+            # this is for shedding only
+            print("Updating individuals (shedding) at timestep {0}.".format(t))
+            for hum_id in self.human_pop:
+                nd.update_node_stats(
+                    (1.0, 0.0, gi.is_possible_mother(hum_id), 0))  # mcw, infectiousness, is_poss_mom, is_infected
+                gi.update1(hum_id)  # this should do shedding
+
+            # Normalize contagion
+            if self.well_mixed_contagion_pool[-1] > 0:
+                self.well_mixed_contagion_pool[-1] /= len(self.human_pop)
+            print("well_mixed_contagion_pool = {0}.".format(self.well_mixed_contagion_pool[-1]))
+
+            print("Updating individuals (exposing) at timestep {0}.".format(t))
+            self.num_infected.append(0)
+            self.num_new_infections.append(0)
+            for hum_id in list(self.human_pop.keys()):  # avoid "RuntimeError: dictionary changed size during iteration"
+                gi.update2(hum_id)  # this should do exposure & vital-dynamics(turn off in this example)
+                # Collect individual level data for every time step
+                data[Constant.SIMULATION_TIMESTEP].append(t)
+                data[Constant.hum_id].append(hum_id)
+                data[Constant.is_infected].append(gi.is_infected(hum_id))
+                data[Constant.infectiousness].append(round(gi.get_infectiousness(hum_id), 6))
+                data[Constant.immunity].append(gi.get_immunity(hum_id))
+                if gi.is_infected(hum_id):
+                    self.num_infected[-1] += 1
+                    if hum_id not in infected or not infected[hum_id]:
+                        self.num_new_infections[-1] += 1
+                    infected[hum_id] = 1
+                else:
+                    infected[hum_id] = 0
+
+            print(f"num_infected = {self.num_infected[-1]}.")
+            # End of one timestep
+
+        # save individual level and node level data
+        self.individual_df = pd.DataFrame.from_dict(data)
+        self.individual_df.index.name = "index"
+
+        self.node_df = pd.DataFrame.from_dict({Constant.STATISTICAL_POPULATION: self.statistical_population,
+                                               Constant.CONTAGION: self.well_mixed_contagion_pool,
+                                               Constant.NUM_INFECTED: self.num_infected,
+                                               Constant.NUM_NEW_INFECTIONS: self.num_new_infections})
+        self.node_df.index.name = "TimeStep"
+
+        if self.individual_df.empty or self.node_df.empty:
+            print("BAD: Simulation data is empty.")
+
+        print("writing result:")
+        self.write()
+        print("Simulation exits.")
+        pass
+
+    def write(self, output_path="output", node_filename="node.csv", individual_filename="individual.csv"):
+        output_path = os.path.join(current_directory, output_path)
+        if not os.path.exists(output_path):
+            print(f"making output folder {output_path}")
+            os.makedirs(output_path)
+
+        with open(os.path.join(output_path, individual_filename), 'w') as individual_file:
+            print(f"writing to {os.path.join(output_path, individual_filename)} now")
+            self.individual_df.to_csv(individual_file, line_terminator="\n")
+
+        with open(os.path.join(output_path, node_filename), 'w') as node_file:
+            print(f"writing to {os.path.join(output_path, node_filename)} now")
+            self.node_df.to_csv(node_file, line_terminator="\n")
+
+
+if __name__ == "__main__":
+    # pathname = os.path.dirname(".")
+    # file = os.path.join(pathname, 'config.json')
+    # if os.path.isfile(file):
+    #     config = utils.readJson(file)
+    #     print('configfile:', config)
+    #     parameters = config["parameters"]
+
+    # execute only if run as a script
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', default="config.json", help="config file name(default to config.json)")
+    parser.add_argument('-d', '--duration', default=10, help="simulation duration(number of time steps)(default to 10)")
+    parser.add_argument('-p', '--population', default=1000, help="number of initial population(default to 100)")
+    parser.add_argument('-o', '--outbreak', default=0, help="outbreak time step(default to 0)")
+    parser.add_argument('-b', '--outbreak_coverage', default=0.01,
+                        help="demographic coverage for outbreak(default to 0.1)")
+    parser.add_argument('-i', '--outbreak_ignore_immunity',
+                        help="if outbreak ignore the immunity status(default to True)", action='store_true')
+    args = parser.parse_args()
+
+    model = SEIR(config_template=args.config,
+                 simulation_duration=int(args.duration),
+                 initial_population=int(args.population),
+                 outbreak_timestep=int(args.outbreak),
+                 outbreak_demographic_coverage=float(args.outbreak_coverage),
+                 outbreak_ignore_immunity=args.outbreak_ignore_immunity)
+    model.run()
+    # The local platform needs to know the resulting status of a work item. We provide it through a return code
+    sys.exit(0)

idmtools_test/inputs/python/ye_seir_model/Assets/SEIR_model_slurm.py

@@ -0,0 +1,242 @@
+import os
+import random
+import sys
+
+assets_dir = os.path.abspath(os.path.dirname(__file__))
+current_directory = os.getcwd()
+p_version = sys.version_info
+CURRENT_DIRECTORY = os.path.dirname(__file__)
+LIBRARY_PATH = os.path.join(CURRENT_DIRECTORY, 'site-packages')  # Need to site-packages level!!!
+
+sys.path.insert(0, LIBRARY_PATH)  # Very Important!
+import pandas as pd  # noqa
+import dtk_nodedemog as nd  # noqa
+import dtk_generic_intrahost as gi  # noqa
+from config_sim import configure_simulation  # noqa
+
+
+class Constant():
+    SIMULATION_TIMESTEP = "SIMULATION_TIMESTEP"
+    STATISTICAL_POPULATION = "STATISTICAL_POPULATION"
+    CONTAGION = "CONTAGION"
+    NUM_INFECTED = "NUM_INFECTED"
+    NUM_NEW_INFECTIONS = "NUM_NEW_INFECTIONS"
+    hum_id = "individual_id"
+    is_infected = "is_infected"
+    infections = "infections"
+    infectiousness = "infectiousness"
+    immunity = "immunity"
+
+
+class Persion():
+    def __init__(self, mcw, age, gender, id):
+        self.mcw = mcw
+        self.age = age
+        self.sex = gender
+        self.id = id
+
+
+# Write a SEIR class using dtk_nodedemog and dtk_generic_intrahost
+class SEIR():
+    def __init__(self, config_template="config.json", simulation_duration=10, initial_population=1000,
+                 outbreak_timestep=0, outbreak_demographic_coverage=0.01, outbreak_ignore_immunity=True,
+                 other_config_params: dict = None):
+        """
+        Define a SEIR model with the following parameeters
+        :param config_template: template file for configuration
+        :param simulation_duration: number of time step for one simulation
+        :param initial_population: number of initial population
+        :param outbreak_timestep: the day to start distributing the outbreak
+        :param outbreak_demographic_coverage: the fraction of individuals that will receive the outbreak
+        :param outbreak_ignore_immunity: iIndividuals will be force-infected regardless of actual immunity level when set to true
+        :param other_config_params: other parameter/value pairs in config.json
+        """
+        self.human_pop = {}  # dictionary of individual objects at run time
+        self.well_mixed_contagion_pool = []
+        self.statistical_population = []
+        self.num_infected = []
+        self.num_new_infections = []
+        self.individual_df = None
+        self.node_df = None
+        self.timestep = 0
+        self.config_template = config_template
+        self.simulation_duration = simulation_duration
+        self.initial_population = initial_population
+        self.outbreak_timestep = outbreak_timestep
+        self.outbreak_demographic_coverage = outbreak_demographic_coverage
+        self.outbreak_ignore_immunity = outbreak_ignore_immunity
+        self.other_config_params = other_config_params
+
+    def create_person_callback(self, mcw, age, gender):
+        new_id = gi.create((gender, age, mcw))
+        person = Persion(mcw, age, gender, new_id)
+        if new_id in self.human_pop:
+            raise Exception(" individual {0} is already created.".format(new_id))
+        else:
+            self.human_pop[new_id] = person
+
+    def expose_callback(self, action, prob, individual_id):
+        random_draw = random.random()
+        if self.timestep == self.outbreak_timestep:
+            print(f"expose {individual_id} with outbreak.")
+            if random_draw < self.outbreak_demographic_coverage:
+                if self.outbreak_ignore_immunity:
+                    print(f"Let's infect {individual_id} with outbreak ignore immunity.")
+                    return 1
+                else:
+                    return self.infect_base_on_immunity(individual_id)
+            else:
+                print(f"Let's NOT infect {individual_id} with outbreak.")
+                return 0
+        else:
+            print(f"expose {individual_id} with transmission.")
+            if random_draw < self.well_mixed_contagion_pool[-1]:
+                return self.infect_base_on_immunity(individual_id)
+            else:
+                print(f"Let's NOT infect {individual_id} based on random draw.")
+                return 0
+
+    def deposit_callback(self, contagion, individual):
+        self.well_mixed_contagion_pool[-1] += contagion
+        print(f"Depositing {contagion} contagion creates total of {self.well_mixed_contagion_pool[-1]}.")
+        return
+
+    def infect_base_on_immunity(self, individual_id):
+        random_draw = random.random()
+        if random_draw < gi.get_immunity(individual_id):
+            print(f"Let's infect {individual_id} base on immunity.")
+            return 1
+        else:
+            print(f"Let's NOT infect {individual_id} base on immunity.")
+            return 0
+
+    def run(self):
+        """
+        This is the method to run the SEIR model and generate output files.
+        """
+        print("\tWe cleared out human_pop. Should get populated via populate_from_files and callback...")
+        gi.reset()
+        nd.reset()
+        self.human_pop = {}
+
+        configure_simulation(initial_population=self.initial_population,
+                             nd_template_filename=os.path.join(current_directory, self.config_template),
+                             demo_template_filename=os.path.join(assets_dir, "templates/demographics_template.json"),
+                             other_config_params=self.other_config_params)
+
+        # set callbacks
+        nd.set_callback(self.create_person_callback)
+        nd.populate_from_files()
+        gi.my_set_callback(self.expose_callback)
+        gi.set_deposit_callback(self.deposit_callback)
+
+        data = {Constant.SIMULATION_TIMESTEP: [],
+                Constant.hum_id: [],
+                Constant.is_infected: [],
+                Constant.infectiousness: [],
+                Constant.immunity: []
+                }
+        infected = dict()
+        for t in range(self.simulation_duration):
+            self.timestep = t
+            # logging.info("Updating individuals at timestep {0}.".format(t))
+            self.statistical_population.append(len(self.human_pop))
+            self.well_mixed_contagion_pool.append(0)  # 100% decay at the end of every time step
+
+            # this is for shedding only
+            print("Updating individuals (shedding) at timestep {0}.".format(t))
+            for hum_id in self.human_pop:
+                nd.update_node_stats(
+                    (1.0, 0.0, gi.is_possible_mother(hum_id), 0))  # mcw, infectiousness, is_poss_mom, is_infected
+                gi.update1(hum_id)  # this should do shedding
+
+            # Normalize contagion
+            if self.well_mixed_contagion_pool[-1] > 0:
+                self.well_mixed_contagion_pool[-1] /= len(self.human_pop)
+            print("well_mixed_contagion_pool = {0}.".format(self.well_mixed_contagion_pool[-1]))
+
+            print("Updating individuals (exposing) at timestep {0}.".format(t))
+            self.num_infected.append(0)
+            self.num_new_infections.append(0)
+            for hum_id in list(self.human_pop.keys()):  # avoid "RuntimeError: dictionary changed size during iteration"
+                gi.update2(hum_id)  # this should do exposure & vital-dynamics(turn off in this example)
+                # Collect individual level data for every time step
+                data[Constant.SIMULATION_TIMESTEP].append(t)
+                data[Constant.hum_id].append(hum_id)
+                data[Constant.is_infected].append(gi.is_infected(hum_id))
+                data[Constant.infectiousness].append(round(gi.get_infectiousness(hum_id), 6))
+                data[Constant.immunity].append(gi.get_immunity(hum_id))
+                if gi.is_infected(hum_id):
+                    self.num_infected[-1] += 1
+                    if hum_id not in infected or not infected[hum_id]:
+                        self.num_new_infections[-1] += 1
+                    infected[hum_id] = 1
+                else:
+                    infected[hum_id] = 0
+
+            print(f"num_infected = {self.num_infected[-1]}.")
+            # End of one timestep
+
+        # save individual level and node level data
+        self.individual_df = pd.DataFrame.from_dict(data)
+        self.individual_df.index.name = "index"
+
+        self.node_df = pd.DataFrame.from_dict({Constant.STATISTICAL_POPULATION: self.statistical_population,
+                                               Constant.CONTAGION: self.well_mixed_contagion_pool,
+                                               Constant.NUM_INFECTED: self.num_infected,
+                                               Constant.NUM_NEW_INFECTIONS: self.num_new_infections})
+        self.node_df.index.name = "TimeStep"
+
+        if self.individual_df.empty or self.node_df.empty:
+            print("BAD: Simulation data is empty.")
+
+        print("writing result:")
+        self.write()
+        print("Simulation exits.")
+        pass
+
+    def write(self, output_path="output", node_filename="node.csv", individual_filename="individual.csv"):
+        output_path = os.path.join(current_directory, output_path)
+        if not os.path.exists(output_path):
+            print(f"making output folder {output_path}")
+            os.makedirs(output_path)
+
+        with open(os.path.join(output_path, individual_filename), 'w') as individual_file:
+            print(f"writing to {os.path.join(output_path, individual_filename)} now")
+            self.individual_df.to_csv(individual_file, line_terminator="\n")
+
+        with open(os.path.join(output_path, node_filename), 'w') as node_file:
+            print(f"writing to {os.path.join(output_path, node_filename)} now")
+            self.node_df.to_csv(node_file, line_terminator="\n")
+
+
+if __name__ == "__main__":
+    # pathname = os.path.dirname(".")
+    # file = os.path.join(pathname, 'config.json')
+    # if os.path.isfile(file):
+    #     config = utils.readJson(file)
+    #     print('configfile:', config)
+    #     parameters = config["parameters"]
+
+    # execute only if run as a script
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--config', default="config.json", help="config file name(default to config.json)")
+    parser.add_argument('-d', '--duration', default=10, help="simulation duration(number of time steps)(default to 10)")
+    parser.add_argument('-p', '--population', default=1000, help="number of initial population(default to 100)")
+    parser.add_argument('-o', '--outbreak', default=0, help="outbreak time step(default to 0)")
+    parser.add_argument('-b', '--outbreak_coverage', default=0.01,
+                        help="demographic coverage for outbreak(default to 0.1)")
+    parser.add_argument('-i', '--outbreak_ignore_immunity',
+                        help="if outbreak ignore the immunity status(default to True)", action='store_true')
+    args = parser.parse_args()
+
+    model = SEIR(config_template=args.config,
+                 simulation_duration=int(args.duration),
+                 initial_population=int(args.population),
+                 outbreak_timestep=int(args.outbreak),
+                 outbreak_demographic_coverage=float(args.outbreak_coverage),
+                 outbreak_ignore_immunity=args.outbreak_ignore_immunity)
+    model.run()
+    # The local platform needs to know the resulting status of a work item. We provide it through a return code
+    sys.exit(0)

idmtools_test/inputs/python/ye_seir_model/Assets/config_sim.py

@@ -0,0 +1,48 @@
+import json
+
+
+class DemographicsParameters():
+    Nodes = "Nodes"
+    NodeAttributes = "NodeAttributes"
+    InitialPopulation = "InitialPopulation"
+
+
+def get_json_template(json_filename="demographics_template.json"):
+    with open(json_filename) as infile:
+        j_file_obj = json.load(infile)
+    return j_file_obj
+
+
+def set_demographics_file(demographics, demo_filename="demographics.json"):
+    with open(demo_filename, 'w') as outfile:
+        json.dump(demographics, outfile, indent=4, sort_keys=True)
+
+
+def set_config_file(config, config_filename="nd.json"):
+    with open(config_filename, 'w') as outfile:
+        json.dump(config, outfile, indent=4, sort_keys=True)
+
+
+def set_gi_file(config, gi_filename="gi.json"):
+    with open(gi_filename, 'w') as outfile:
+        json.dump(config, outfile, indent=4, sort_keys=True)
+
+
+def configure_simulation(initial_population,
+                         nd_template_filename,
+                         demo_template_filename,
+                         other_config_params: dict = None):
+    print("configure demographics.json.\n")
+    demographics = get_json_template(json_filename=demo_template_filename)
+    demographics[DemographicsParameters.Nodes][0][DemographicsParameters.NodeAttributes][
+        DemographicsParameters.InitialPopulation] = initial_population
+    set_demographics_file(demographics)
+
+    print("configure nd.json and gi.json.\n")
+    config = get_json_template(json_filename=nd_template_filename)
+
+    if other_config_params:
+        for param in other_config_params:
+            config[param] = other_config_params[param]
+    set_config_file(config)
+    set_gi_file(config)

idmtools_test/inputs/python/ye_seir_model/Assets/custom_csv_analyzer.py

@@ -0,0 +1,133 @@
+# Example of two csv analyzers to concatenate csv results into one csv and plot the data from your experiment
+
+# First, import some necessary system and idmtools packages.
+import os
+from sys import platform
+import matplotlib
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+from idmtools.entities import IAnalyzer
+if platform == "linux" or platform == "linux2":
+    print('Linux OS. Using non-interactive Agg backend')
+    matplotlib.use('Agg')
+
+
+# Create a class for a base CSV analyzer
+class CSVAnalyzer(IAnalyzer):
+    def __init__(self, filenames, uid=None, working_dir=None, parse=True):
+        super().__init__(uid, working_dir, parse, filenames=filenames)
+        # Raise exception early if files are not csv files
+        if not all(['csv' in os.path.splitext(f)[1].lower() for f in self.filenames]):
+            raise Exception('Please ensure all filenames provided to CSVAnalyzer have a csv extension.')
+
+
+# Create a class for node level analyzer
+class NodeCSVAnalyzer(CSVAnalyzer):
+    def __init__(self, filenames):
+        super().__init__(filenames=filenames)
+
+    # Map is called to get for each simulation a data object (all the metadata of the simulations) and simulation object
+    def map(self, data, simulation):
+        # If there are 1 to many csv files, concatenate csv data columns into one dataframe
+        # filter the data by filenames (data could contains data/dataframe from other analyzers within the same AnalyzerManager)
+        concatenated_df = pd.concat([data[filename] for filename in self.filenames],
+                                    axis=0, ignore_index=True, sort=True)
+        return concatenated_df
+
+    # In reduce, we are printing and plotting the simulation and result data filtered in map
+    def reduce(self, all_data):
+        # Let's hope the first simulation is representative
+        first_sim = next(iter(all_data.keys()))  # Iterate over the dataframe keys
+        exp_id = str(first_sim.experiment.uid)  # Get the exp id from the first sim data
+
+        results = pd.concat(list(all_data.values()), axis=0,  # Combine a list of all the sims csv data column values
+                            keys=[str(k.uid) for k in all_data.keys()],  # Add a hierarchical index with the keys option
+                            names=['SimId'])  # Label the index keys you create with the names option
+
+        results.index = results.index.droplevel(1)  # Remove default index
+
+        # Make a directory labeled the exp id to write the csv results to
+        os.makedirs(exp_id, exist_ok=True)
+        # NOTE: If running twice with different filename, the output files will collide
+        results.to_csv(os.path.join(exp_id, self.__class__.__name__ + '.csv'))
+
+        channels = results.drop(['TimeStep'], axis=1).columns.tolist()
+        sims = results.index.unique().to_list()
+        # Create the sub plots
+        ncol = int(len(channels) / 2)
+        nrow = int(np.ceil(float(len(channels)) / ncol))
+        figsize = (max(10, min(10, 8 * ncol)), min(10, 6 * nrow))
+        fig, axs = plt.subplots(figsize=figsize, nrows=nrow, ncols=ncol, sharex=True)
+        flat_axes = [axs] if ncol * nrow == 1 else axs.flat
+
+        # Plot
+        for channel, ax in zip(channels, flat_axes):
+            ax.set_title(channel)
+            ax.set_xlabel("TimeStep")
+            for sim in sims:
+                ax.plot(results[results.index == sim]['TimeStep'], results[results.index == sim][channel])
+
+        # Create the legend
+        sims_label = [str(sim).split('-')[0] for sim in sims]
+        plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), fancybox=True, shadow=True, ncol=5,
+                   fontsize='xx-small', labels=sims_label)
+
+        # Save the figure
+        plt.savefig(os.path.join(exp_id, self.__class__.__name__ + '.png'))
+
+
+# Create a class for individual level analyzer
+class InfectiousnessCSVAnalyzer(CSVAnalyzer):
+    def __init__(self, filenames, channel="infectiousness", filter_by='is_infected'):
+        super().__init__(filenames=filenames)
+        self.channel = channel
+        self.filter_by = filter_by
+
+    # Map is called to get for each simulation a data object (all the metadata of the simulations) and simulation object
+    def map(self, data, simulation):
+        # Filter by filenames in this analyzer
+        my_data = [data[filename] for filename in self.filenames]
+        # If there are 1 to many csv files, concatenate csv data columns into one dataframe
+        # Collect the infectiousness data when is_infected is True only
+        concatenated_df = pd.concat([channel_data[channel_data[self.filter_by] == 1][self.channel] for
+                                     channel_data in my_data],
+                                    axis=0, ignore_index=True, sort=True)
+        return concatenated_df
+
+    # In reduce, we are printing and plotting the simulation and result data filtered in map
+    def reduce(self, all_data):
+        # Let's hope the first simulation is representative
+        first_sim = next(iter(all_data.keys()))  # Iterate over the dataframe keys
+        exp_id = str(first_sim.experiment.uid)  # Set the exp id from the first sim data
+
+        keys = []
+        for k in all_data.keys():
+            if 'simulation_name_tag' in k.tags:
+                k.tags.pop('simulation_name_tag')
+            keys.append(str(k.tags))
+        results = pd.concat(list(all_data.values()), axis=0,  # Combine a list of all the sims csv data column values
+                            keys=keys,  # Add a hierarchical index with the keys option
+                            names=['SimTags'])  # Using Simulation tags as the keys
+        results.index = results.index.droplevel(1)  # Remove default index
+
+        # Make a directory labeled the exp id to write the csv results to
+        os.makedirs(exp_id, exist_ok=True)
+        # NOTE: If running twice with different filename, the output files will collide
+        results.to_csv(os.path.join(exp_id, self.__class__.__name__ + '.csv'))
+
+        sims = results.index.unique().to_list()
+        fig, ax = plt.subplots()
+
+        # Plot
+        for sim in sims:
+            # Filter data by sim tags and then generate the kernel density estimation plot
+            results[results.index == sim].plot.kde(bw_method='scott', ax=ax, label=str(sim))
+            # sns.distplot() would generate a prettier kde plot but seaborn is not a required library in idmtools.
+            # import seaborn as sns
+            # sns.distplot(results[results.index==sim], kde=True, hist=False, ax=ax, label=str(sim))
+        plt.legend(loc=0, fontsize='xx-small')
+        ax.set_title(self.channel + "(kde)")
+        ax.set_ylim(bottom=0)
+        # Save the figure
+        plt.savefig(os.path.join(exp_id, self.__class__.__name__ + '.png'))

idmtools_test/inputs/python/ye_seir_model/Assets/python.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+export PYTHONPATH=$PWD/Assets/site-packages
+echo $PYTHONPATH
+python3 Assets/SEIR_model.py