emod-api 3.0.2__py3-none-any.whl

emod_api/channelreports/plot_icj_means.py

@@ -0,0 +1,149 @@
+import json
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+import sqlite3
+
+
+def collect(exp_id: str,
+            chan: str = "Infected",
+            tag: str = None,
+            smoothing: bool = True) -> dict:
+    """
+    Collect all the time series data for a given channel for a given experiment from InsetChart.json
+    files in local subdirectory that have been downoaded from COMPS, assuming following structure.
+
+        exp_id/
+           sim_id/
+               InsetChart.json
+
+    Args:
+        exp_id: Experiment Id that has had data downloaded to current working diretory.
+        chan:   Channel name
+        tag:    key=value. Using results.db (sqlite3, from emodpy), limit results to just where key=value.
+                If value is set to SWEEP, find all values for key and plot all values separately (but with mean/spread from other tags).
+
+    Returns:
+        Array of channel data for further processing.
+    """
+
+    chan_data = {}
+    groupby_values = {}
+    if tag:
+        if len(tag.split("=")) == 1:
+            raise ValueError("When passing tag, has to have key=value format.")
+
+        groupby_key = tag.split("=")[0]
+        groupby_value = tag.split("=")[1]
+        db = os.path.join("latest_experiment", "results.db")
+        con = sqlite3.connect(db)
+        cur = con.cursor()
+        if groupby_value == "SWEEP":
+            query = f"SELECT sim_id, {groupby_key} FROM results"
+            all_results = cur.execute(query)
+            for result in all_results:
+                sim_id = result[0]
+                groupby_value = result[1]
+                if groupby_value not in groupby_values:
+                    groupby_values[groupby_value] = list()
+                groupby_values[groupby_value].append(sim_id)
+        else: # select only sim_id's where gb key == value
+            query = f"SELECT sim_id FROM results where {groupby_key} = {groupby_value}"
+            all_results = cur.execute(query)
+            groupby_values["ref"] = list()
+            for result in all_results:
+                sim_id = result[0]
+                groupby_values["ref"].append(sim_id)
+    else:
+        groupby_values["ref"] = os.listdir(exp_id)
+        groupby_values["ref"].remove("results.db")
+
+    def moving_average(x, w=7):
+        return np.convolve(x, np.ones(w), 'valid') / w
+
+    max_len = 0
+    # poi = param of interest
+    for value in groupby_values:
+        simdirs = groupby_values[value]
+        for sim in simdirs:
+            thedir = os.path.join(exp_id, sim)
+
+            if value not in chan_data:
+                chan_data[value] = []
+            if not os.path.exists(thedir + "/InsetChart.json"):
+                continue
+            with open(thedir + "/InsetChart.json") as fp:
+                icj = json.loads(fp.read())
+            if chan not in icj["Channels"]:
+                raise ValueError(f"Can't find channel {chan} in file. Did find {icj['Channels'].keys()}.")
+            new_data = np.asarray(icj["Channels"][chan]["Data"])
+            if smoothing:
+                new_data = moving_average(new_data)
+            chan_data[value].append(new_data)
+            if len(new_data) > max_len:
+                max_len = len(new_data)
+    if max_len == 0:
+        raise ValueError(f"No InsetChart.json files with channel data for {chan} and experiment {exp_id}.")
+    """
+    If users run simulations that end when prevalence is zero, the length of the time series can vary
+    We need to get them all the same to calc the mean.
+    """
+    data_for_plotting = {}
+    for poi in chan_data:
+        data_for_plotting[poi] = []
+        for data in chan_data[poi]:
+            if len(data) < max_len:
+                data = np.pad(data, (0, max_len - len(data)))
+            data_for_plotting[poi].append(data)
+
+    return data_for_plotting
+
+
+def display(chan_data, save=False, chan_name="Infected", exp_id=None):
+    """
+    Plot mean and std dev of the array/list of time series-es in chan_data.
+    """
+    mean_chan_data = None
+    spread_chan_data = None
+    fig, ax = plt.subplots(1)
+    if exp_id:
+        plt.title(exp_id, loc="center")
+    for poi_chan_data in sorted(chan_data):
+        prev_list = chan_data[poi_chan_data]
+        if len(prev_list) == 0:
+            raise ValueError("Input channel data array seems to have no data.")
+        mean_chan_data = np.mean(np.array(prev_list), axis=0)
+        if len(chan_data) == 1 and save:
+            ref_json = {"Channels": {"Channel": {"Data": []}}}
+            ref_json["Channels"]["Channel"]["Data"] = list(mean_chan_data)
+            with open("mean_ref.json", "w") as fp:
+                json.dump(ref_json, fp, indent=4)
+        spread_chan_data = np.std(np.array(prev_list), axis=0)
+
+        t = np.arange(len(mean_chan_data))
+        ax.plot(t, mean_chan_data, label=poi_chan_data)
+        plt.xlim(0, len(mean_chan_data))
+        ax.fill_between(t, mean_chan_data + spread_chan_data, mean_chan_data - spread_chan_data, facecolor='yellow', alpha=0.5)
+    ax.set_xlabel("Simulation Time")
+    ax.set_ylabel(chan_name)
+    plt.legend()
+    plt.show()
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Mean 'InsetChart' Report Plotting")
+    parser.add_argument('-c', '--channel', action='store', default="Infected", help='channel(s) to display [Infected]')
+    parser.add_argument('-e', '--experiment_id', action='store', default=None, help='experiment id to plot, data assumed to be local')
+    parser.add_argument('-t', '--tag', action='store', default=None, help='key=value tag constraint')
+    args = parser.parse_args()
+    if not args.experiment_id:
+        with open("COMPS_ID") as fp:
+            args.experiment_id = fp.read()
+
+    # check that folder with name experiment_id exists
+    if not os.path.exists(str(args.experiment_id)):
+        raise ValueError(f"Don't see folder for {args.experiment_id}.")
+
+    chan_data = collect(args.experiment_id, args.channel, args.tag)
+    display(chan_data, False, args.channel, args.experiment_id)

emod_api/channelreports/plot_prop_report.py

@@ -0,0 +1,205 @@
+#! /usr/bin/env python3
+
+"""Command line utility for plotting property reports."""
+
+import argparse
+from functools import reduce
+import json
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from emod_api.channelreports.utils import read_json_file, get_report_channels, accumulate_channel_data, save_to_csv, plot_traces
+from emod_api.channelreports.utils import _validate_property_report_channels, _validate_property_report_ips
+
+
+def main(args: argparse.Namespace):
+
+    """
+    Plot specified property report with the given options.
+    """
+
+    json_data = read_json_file(args.filename)
+    channel_data = get_report_channels(json_data)
+    channel_keys = sorted(channel_data)
+
+    if args.verbose:
+        print("Channels:Pools-")
+        print(json.dumps(channel_keys, indent=4))
+
+    if args.list:
+        list_channels_and_ips(channel_keys)
+        return
+
+    _validate_property_report_channels(args.channels, channel_data)
+    _validate_property_report_ips(args.groupby, channel_data)
+
+    if args.normalize and ("Statistical Population" not in args.channels):
+        args.channels.append("Statistical Population")
+
+    trace_values = accumulate_channel_data(args.channels, args.verbose, args.groupby, channel_data)
+
+    if args.csv is None:
+        call_plot_traces(args, trace_values)
+    else:
+        save_to_csv(trace_values, args.csv, args.transpose)
+
+    return
+
+
+def list_channels_and_ips(channel_keys: list[str]) -> None:
+
+    """
+    List the channels and properties found in a property report from the
+    CHANNEL:IP:value,...,IP:value keys of the channel dictionary.
+    """
+
+    # keys look like "CHANNEL:IP:value,...,IP:value"
+    channels = sorted(set([key.split(":", 1)[0] for key in channel_keys]))
+
+    print("\nChannels:")
+    for channel in channels:
+        print(f"\t{channel}")
+
+    # Each channel _should_ have the same set of IPs, but we'll check them all
+    csvkvps = [key.split(":", 1)[1] for key in channel_keys]            # For each channel get a comma separated list of IP:value pairs (see format above)
+    kvplists = [csv.split(",") for csv in csvkvps]                      # For each CSV convert to actual list by splitting on ","
+    ips = [map(lambda t: t.split(":")[0], kvps) for kvps in kvplists]   # Convert each IP:value entry to just IP
+    properties = sorted(reduce(lambda s, e: s.union(e), ips, set()))    # Add all IPs to an initially empty set
+
+    print("\nIPs:")
+    for prop in properties:
+        print(f"\t{prop}")
+
+    print()
+
+    return
+
+
+def call_plot_traces(args: argparse.Namespace,
+                     trace_values: dict[str, np.ndarray]) -> None:
+
+    """
+    Call the internal `plot_traces` function and, optionally, save the results to disk.
+    """
+
+    if args.verbose:
+        print(sorted(trace_values))
+
+    if args.normalize:
+        stat_pop = "Statistical Population"
+        traces = {key: value for (key, value) in trace_values.items() if not key.startswith(stat_pop)}
+        # reduce the various statistical population traces to a single vector
+        norms = reduce(lambda x, y: np.array(y) + x, [value for (key, value) in trace_values.items() if key.startswith(stat_pop)], 0)
+    else:
+        traces = trace_values
+        norms = None
+
+    figure = plot_traces(traces, norms, args.overlay, args.channels, args.filename, args.legend)
+
+    if args.saveFigure:
+        print("Saving figure 'propertyReport.png'...")
+        figure.savefig('propertyReport.png')
+
+    plt.show()
+
+    return
+
+
+def prop_report_json_to_csv(output_path: str,
+                            channel_name: str = "Infected",
+                            groupby: str = "Geographic"):
+    """
+    Converts selected channel of PropertyReportXXX.json into a CSV file, rolled up into a single property.
+
+    Args:
+        output_path: Subdirectory in which to find a file called PropertyReportXXX.json.
+            XXX can be blank or a disease named like 'TB'.
+        channel_name: Name of the channel to process from the property report.
+            Defaults to "Infected".
+        groupby: Property to group by. Defaults to "Geographic".
+
+    Returns:
+
+    Raises:
+        ValueError: If no PropertyReportXXX.json file is found in the directory.
+    """
+
+    def find_file_starting_with(directory, prefix):
+        path = Path(directory)
+        for file in path.iterdir():
+            if file.name.startswith(prefix) and file.is_file() and file.name.endswith("json"):
+                return str(file)
+        return None
+
+    prop_report_path = find_file_starting_with(output_path, "PropertyReport")
+    if not prop_report_path:
+        raise ValueError(f"No json file starting with 'PropertyReport' found in '{output_path}'.")
+
+    # This class probably exists somewhere else. Maybe we can move it to a common utils.py file or getit from elsewhere.
+    class DynamicObject:
+        def __init__(self):
+            object.__setattr__(self, 'members', {})
+
+        def __setattr__(self, name, value):
+            self.members[name] = value
+
+        def __getattr__(self, name):
+            return self.members.get(name)
+
+    faux_args = DynamicObject()
+    faux_args.filename = prop_report_path
+    csv_out_name = "prop_report_" + channel_name.replace(' ', '_').lower() + ".csv"
+    faux_args.csv = csv_out_name
+    faux_args.channels = [channel_name]
+    faux_args.channels.append("Statistical Population")
+    faux_args.normalize = True
+    faux_args.groupby = [groupby]
+    main(faux_args)
+
+
+def process_cmd_line() -> argparse.Namespace:
+
+    """
+    Put command line processing here rather than in `if 'name' == '__main__'`.
+    """
+
+    parser = argparse.ArgumentParser(description='Property Report Plotting')
+    parser.add_argument('filename', nargs='?', default='PropertyReport.json', help='property report filename [PropertyReport.json]')
+    parser.add_argument('-c', '--channel', action='append', help='channel(s) to display [Infected]', metavar='channelName', dest='channels')
+    parser.add_argument('-g', '--groupby', action='append', help="IP(s) under which to aggregate other IP keys and values")
+    parser.add_argument('-n', '--normalize', help='plot channel(s) normalized by statistical population', action='store_true')
+    parser.add_argument('-o', '--overlay', help='overlay pools of the same channel', action='store_true')
+    parser.add_argument('-s', '--save', help="save figure to file 'propertyReport.png'", action='store_true', dest='saveFigure')
+    parser.add_argument('-v', '--verbose', action="store_true")
+    parser.add_argument('--no-legend', action="store_false", dest="legend")     # Note args.legend default to True, passing --no-legend sets args.legend to False
+    parser.add_argument('-l', '--list', action="store_true", help="List channels and IP keys found in the report. No plotting is performed with this option.")
+    parser.add_argument("--csv", type=Path, default=None, help="Write data for selected channel(s) to given file.")
+    parser.add_argument("-t", "--transpose", action="store_true", help="write channels as columns rather than rows (only in effect with '--csv' option)")
+
+    args = parser.parse_args()
+
+    if not args.channels:
+        args.channels = ['Infected']
+
+    if args.groupby is not None and len(args.groupby) == 1 and args.groupby[0].lower() == "all":
+        args.groupby = []
+
+    if not args.list:
+        print(f"Filename:              '{args.filename}'")
+        print(f"Channel(s):            {args.channels}")
+        print(f"Groupby:               {args.groupby}")
+        print(f"Normalize:             {args.normalize}")
+        print(f"Overlay:               {args.overlay}")
+        print(f"Save:                  {args.saveFigure}")
+        if args.csv:
+            print(f"CSV filename:          '{args.csv}'")
+        print(f"Transpose CSV:         {args.transpose}")
+
+    return args
+
+
+if __name__ == '__main__':
+
+    main(process_cmd_line())

emod_api/channelreports/utils.py

@@ -0,0 +1,326 @@
+"""
+Helper functions, primarily for property reports, which are channel reports.
+"""
+
+import json
+from pathlib import Path
+from typing import Union, Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from emod_api.channelreports.channels import ChannelReport
+
+__all__ = [
+    "property_report_to_csv",
+    "read_json_file",
+    "get_report_channels",
+    "_validate_property_report_channels",
+    "_validate_property_report_ips",
+    "accumulate_channel_data",
+    "__get_trace_name",
+    "save_to_csv",
+    "plot_traces",
+    "__index_for",
+    "__title_for"]
+
+
+def property_report_to_csv(source_file: Union[str, Path],
+                           csv_file: Union[str, Path],
+                           channels: Optional[list[str]] = None,
+                           groupby: Optional[list[str]] = None,
+                           transpose: bool = False) -> None:
+
+    """
+    Write a property report to a CSV formatted file.
+
+    Optionally selected a subset of available channels.
+    Optionally "rolling-up" IP:value sub-channels into a "parent" IP.
+
+    Args:
+        source_file: filename of property report
+        channels:    list of channels to output, None results in writing _all_ channels to output
+        groupby:     list of IPs into which to aggregate remaining IPs, None indicates no grouping, [] indicates _all_ aggregated
+        csv_file:    filename of CSV formatted result
+        transpose:   write channels as columns rather than rows
+    """
+
+    json_data = read_json_file(Path(source_file))
+    channel_data = get_report_channels(json_data)
+
+    if channels is None:
+        channels = sorted({key.split(":")[0] for key in channel_data})
+    elif isinstance(channels, str):
+        channels = [channels]
+
+    if isinstance(groupby, str):
+        groupby = [groupby]
+
+    _validate_property_report_channels(channels, channel_data)
+    _validate_property_report_ips(groupby, channel_data)
+
+    trace_values = accumulate_channel_data(channels, False, groupby, channel_data)
+
+    save_to_csv(trace_values, csv_file, transpose)
+
+    return
+
+
+def read_json_file(filename: Union[str, Path]) -> dict:
+
+    with Path(filename).open("r", encoding="utf-8") as file:
+        json_data = json.load(file)
+
+    return json_data
+
+
+def get_report_channels(json_data: dict) -> dict:
+
+    try:
+        channel_data = json_data['Channels']
+    except KeyError as exc:
+        raise KeyError("Didn't find 'Channels' in JSON data.") from exc
+
+    return channel_data
+
+
+def _validate_property_report_channels(channels, channel_data) -> None:
+
+    if channels:
+        keys = set(map(lambda name: name.split(":", 1)[0], channel_data))
+        not_found = [name for name in channels if name not in keys]
+        if not_found:
+            print("Valid channel names:")
+            print("\n".join(keys))
+            raise ValueError(f"Specified channel(s) - {not_found} - is/are not valid channel names.")
+
+    return
+
+
+def _validate_property_report_ips(groupby, channel_data) -> None:
+
+    if groupby:
+        first = next(iter(channel_data))
+        ip_string = first.split(":", 1)[1]
+        ips = [kvp.split(":")[0] for kvp in ip_string.split(",")]
+        not_found = [ip for ip in groupby if ip not in ips]
+        if not_found:
+            print("Valid IPs:")
+            print("\n".join(ips))
+            raise ValueError(f"Specified groupby IP(s) - {not_found} - is/are not valid IP names.")
+
+    return
+
+
+def accumulate_channel_data(channels: list[str], verbose: bool, groupby: list[str], channel_data: dict) -> dict[str, np.ndarray]:
+
+    """
+    Extract selected channel(s) from property report data.
+
+    Aggregate on groupby IP(s), if provided, otherwise on channel per unique
+    IP:value pair (e.g., "QualityOfCare:High"), per main channel (e.g., "Infected").
+
+    Args:
+        channels:       names of channels to plot
+        verbose:        output some "debugging"/progress information if true
+        groupby:        IP(s) under which to aggregate other IP:value pairs
+        channel_data:   data for channels keyed on channel name
+
+    Returns:
+        tuple of dictionary of aggregated data, keyed on channel name, and of Numpy array of normalization values
+    """
+
+    trace_values = {}
+    pool_keys = sorted(channel_data)
+
+    name_ip_pairs = map(lambda key: tuple(key.split(":", 1)), pool_keys)
+    name_ip_pairs_to_process = filter(lambda p: p[0] in channels, name_ip_pairs)
+    for (channel_title, key_value_pairs) in name_ip_pairs_to_process:
+
+        if verbose:
+            print(f"Processing channel '{channel_title}:{key_value_pairs}'")
+
+        key_value_pairs = key_value_pairs.split(',')
+        trace_name = __get_trace_name(channel_title, key_value_pairs, groupby)
+        trace_data = np.array(channel_data[f"{channel_title}:{','.join(key_value_pairs)}"]['Data'], dtype=np.float32)
+
+        if trace_name not in trace_values:
+            if verbose:
+                print(f"New trace: '{trace_name}'")
+            trace_values[trace_name] = trace_data
+        else:
+            if verbose:
+                print(f"Add to trace: '{trace_name}'")
+            trace_values[trace_name] += trace_data
+
+    return trace_values
+
+
+def __get_trace_name(channel_title: str, key_value_pairs: list[str], groupby: list[str]) -> str:
+
+    """
+    Return "canonical" trace name for a given channel, IP:value list, and groupby list.
+
+    Since we may be aggregating by IP values, trace name may not equal any particular channel name.
+
+    Example:
+        title = "Infected"
+        key_value_pairs = ["Age_Bin:Age_Bin_Property_From_0_To_20","QualityOfCare:High","QualityOfCare1:High","QualityOfCare2:High"]
+
+        groupby = None
+        return "Infected:Age_Bin:Age_Bin_Property_From_0_To_20,QualityOfCare:High,QualityOfCare1:High,QualityOfCare2:High"
+
+        groupby = ["Age_Bin"]
+        return = "Infected:Age_Bin:Age_Bin_Property_From_0_To_20"
+
+        groupby = ["Age_Bin", "QualityOfCare"]
+        return = "Infected:Age_Bin:Age_Bin_Property_From_0_To_20,QualityOfCare:High"
+
+        groupby = []
+        return = "Infected"
+    """
+
+    # trace name will have channel title and any property:value pairs
+    # which aren't being grouped
+
+    trace_name = channel_title + ':'
+
+    if groupby is None:
+        trace_name = f"{channel_title}:{','.join(key_value_pairs)}"
+    else:
+        if len(groupby) > 0:
+            kvps = filter(lambda pair: pair.split(":")[0] in groupby, key_value_pairs)
+            trace_name = f"{channel_title}:{','.join(kvps)}"
+        else:
+            trace_name = channel_title
+
+    return trace_name
+
+
+def save_to_csv(trace_values: dict[str, np.ndarray],
+                filename: Union[str, Path],
+                transpose: bool = False) -> None:
+
+    """
+    Save property report to CSV. Uses underlying ChannelReport.to_csv() function.
+
+    Args:
+        trace_values: full set of available channels, keyed on channel name
+        filename:     destination file for CSV data
+        transpose:    write channels as columns rather than rows
+    """
+
+    report = ChannelReport()
+
+    for channel, data in trace_values.items():
+        report.channels[channel] = data
+
+    report.to_csv(Path(filename), transpose=transpose)  # by default, use _all_ the channels we just added
+
+    return
+
+
+def plot_traces(trace_values: dict[str, np.ndarray],
+                norm_values: Union[int, np.ndarray, None],
+                overlay: bool,
+                channels: list[str],
+                title: str,
+                legend: bool) -> plt.Figure:
+
+    """
+    Plot trace data. One subplot per channel unless overlaying all variations of rolled-up IP(s) is requested.
+
+    A trace (like old-time pen and ink EKG) may represent the aggregation of
+    several IP values so trace may not equal any particular channel data.
+
+    Args:
+        trace_values: channel data, keyed on channel name
+        norm_values:  normalization data for channels
+        overlay:      whether or not to overlay all variations of a given channel on one subplot
+        channels:     selection of channel names to plot
+        title:        plot title
+        legend:       whether or not to include a legend on plots
+
+    Returns:
+        plt.Figure
+    """
+
+    if len(trace_values) == 0:
+        print("Didn't find requested channel(s) in property report.")
+        return
+
+    if not overlay:
+        plot_count = len(trace_values)
+    else:
+        plot_count = len(channels)
+
+    normalize = norm_values is not None
+    if normalize:
+        plot_count *= 2
+
+    figure = plt.figure(title, figsize=(16, 9), dpi=300)
+    trace_keys = sorted(trace_values)
+
+    # plotting here
+    for trace_name in trace_keys:
+        plot_index = __index_for(trace_name, channels, trace_keys, normalize, overlay)
+        plt.subplot(plot_count, 1, plot_index)
+        plt.plot(trace_values[trace_name], label=trace_name)
+        if normalize:
+            plt.subplot(plot_count, 1, plot_index + 1)
+            plt.ylim((0.0, 1.0))    # yes, this takes a tuple
+            plt.plot(trace_values[trace_name] / norm_values, label=trace_name)
+
+    # make it pretty
+    _ = plt.subplot(plot_count, 1, 1)
+    for trace_name in trace_keys:
+        plot_index = __index_for(trace_name, channels, trace_keys, normalize, overlay)
+        plot_title = __title_for(trace_name, channels, overlay)
+        plt.subplot(plot_count, 1, plot_index)
+        plt.title(plot_title)
+        if legend:
+            plt.legend()
+        if normalize:
+            plt.subplot(plot_count, 1, plot_index + 1)
+            plt.title(f"{plot_title} normalized by 'Statistical Population'")
+            if legend:
+                plt.legend()
+
+    plt.tight_layout()
+
+    return figure
+
+
+def __index_for(trace_name: str, channels: list[str], trace_keys: list[str], normalize: bool, overlay: bool) -> int:
+
+    if overlay:
+        # all pools of the same channel overlaid
+        index = 0
+        for channel in channels:
+            if channel in trace_name:
+                break
+            index += 1
+    else:
+        # each trace separate
+        index = trace_keys.index(trace_name)
+
+    # if we're normalizing, there's a normalized trace per regular trace
+    if normalize:
+        index *= 2
+
+    # matplotlib is 1-based (like MATLAB)
+    return index + 1
+
+
+def __title_for(trace_name: str, channels: list[str], overlay: bool):
+
+    # use channel name
+    if overlay:
+        for channel in channels:
+            if channel in trace_name:
+                title = channel
+                break
+    else:
+        title = trace_name
+
+    return title

emod_api/config/default_from_schema.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+from emod_api.config import default_from_schema_no_validation as old
+
+
+def write_default_from_schema(path_to_schema):
+    """
+    This module is deprecated. Please use default_from_schema_no_validation.
+    """
+    print("This module is deprecated. Please use default_from_schema_no_validation.")
+    return old.write_default_from_schema(path_to_schema)
+
+
+if __name__ == "__main__":
+    print("This module is deprecated. Please use default_from_schema_no_validation.")
+    old._do_main()