captest 0.12.1__py2.py3-none-any.whl → 0.13.0__py2.py3-none-any.whl

captest/io.py

@@ -1,4 +1,5 @@
 # this file is formatted with black
+import copy
 import dateutil
 import datetime
 from pathlib import Path
@@ -46,7 +47,7 @@ def load_excel_column_groups(path):
     dict
         Dictionary mapping column group names to lists of column names.
     """
-    df = pd.read_excel(path, header=None).fillna(method="ffill")
+    df = pd.read_excel(path, header=None).ffill(axis='index')
     return df.groupby(0)[1].apply(list).to_dict()
 
 
@@ -60,6 +61,9 @@ def load_pvsyst(
     """
     Load data from a PVsyst energy production model.
 
+    Will load day first or month first dates. Expects files that use a comma as a
+    separator rather than a semicolon.
+
     Parameters
     ----------
     path : str
@@ -72,7 +76,8 @@ def load_pvsyst(
         By default sets power to E_Grid, poa to GlobInc, t_amb to T Amb, and w_vel to
         WindVel. Set to False to not set regression columns on load.
     **kwargs
-        Use to pass additional kwargs to pandas read_csv.
+        Use to pass additional kwargs to pandas read_csv. Pass sep=';' to load files
+        that use semicolons instead of commas as the separator.
 
     Returns
     -------
@@ -109,8 +114,33 @@ def load_pvsyst(
             break
 
     pvraw.columns = pvraw.columns.droplevel(1)
-    dates = pvraw.loc[:, "date"]
     try:
+        dates = pvraw.loc[:, "date"]
+    except KeyError:
+        warnings.warn(
+            "No 'date' column found in the PVsyst data. This may be due to "
+            "the separator being a semicolon ';' rather than a comma ','. "
+            "If this is the case, try passing sep=';' when calling load_pvsyst. "
+            "Otherwise the date column may actually be missing. Exception:"
+        )
+        raise
+    # PVsyst creates dates like '01/01/90 00:00' i.e. January 1st, 1990.
+    # Opening the PVsyst output in excel will likely result in the dates modified to
+    # 1/1/1990 0:00. The strftime format specified won't load the excel modified dates
+    # so these are caught by checking for consistent length and reformatted
+    if not all(dates.str.len() == 14):
+        date_parts = dates.str.split(' ').str[0].str.split('/')
+        time_parts = dates.str.split(' ').str[1].str.split(':')
+        dates = (
+            date_parts.str[0].str.zfill(2) + '/' +
+            date_parts.str[1].str.zfill(2) + '/' +
+            '90 ' +
+            time_parts.str[0].str.zfill(2) + ':' +
+            time_parts.str[1]
+        )
+    try:
+        # mm/dd/yy hh:mm, lower case y gives
+        # Year without century as a zero-padded decimal number. e.g. 00, 01, …, 99
         dt_index = pd.to_datetime(dates, format="%m/%d/%y %H:%M")
     except ValueError:
         warnings.warn(
@@ -133,7 +163,6 @@ def load_pvsyst(
         cd.data["E_Grid"] = cd.data["E_Grid"] / egrid_unit_adj_factor
     cd.data_filtered = cd.data.copy()
     cd.column_groups = cg.group_columns(cd.data)
-    cd.trans_keys = list(cd.column_groups.keys())
     if set_regression_columns:
         cd.set_regression_cols(
             power="E_Grid", poa="GlobInc", t_amb="T_Amb", w_vel="WindVel"
@@ -260,7 +289,6 @@ class DataLoader:
             current_file, missing_intervals, freq_str = util.reindex_datetime(
                 file,
                 report=False,
-                add_index_col=True,
             )
             reindexed_dfs[name] = current_file
             file_frequencies.append(freq_str)
@@ -387,12 +415,18 @@ class DataLoader:
                         print(err)
                     failed_to_load_count += 1
                     continue
-            (
-                self.loaded_files,
-                self.common_freq,
-                self.file_frequencies,
-            ) = self._reindex_loaded_files()
-            data = self._join_files()
+            if len(self.loaded_files) == 0:
+                warnings.warn(
+                    "No files were loaded. Check that file_reader is working")
+            elif len(self.loaded_files) > 1:
+                (
+                    self.loaded_files,
+                    self.common_freq,
+                    self.file_frequencies,
+                ) = self._reindex_loaded_files()
+                data = self._join_files()
+            elif len(self.loaded_files) == 1:
+                data = list(self.loaded_files.values())[0]
             data.index.name = "Timestamp"
             self.data = data
         else:
@@ -463,13 +497,13 @@ def load_data(
         By default will create a new index for the data using the earliest datetime,
         latest datetime, and the most frequent time interval ensuring there are no
         missing intervals.
-    site : dict, default None
-        Pass a dictionary containing site data, which will be used to generate
-        modeled clear sky ghi and poa values. The clear sky irradiance values are
-        added to the data and the column_groups attribute is updated to include these
-        two irradiance columns. The site data dictionary should be
-        {sys: {system data}, loc: {location data}}. See the capdata.csky documentation
-        for the format of the system data and location data.
+    site : dict or str, default None
+        Pass a dictionary or path to a json or yaml file containing site data, which
+        will be used to generate modeled clear sky ghi and poa values. The clear sky
+        irradiance values are added to the data and the column_groups attribute is
+        updated to include these two irradiance columns. The site data dictionary should
+        be {sys: {system data}, loc: {location data}}. See the capdata.csky
+        documentation for the format of the system data and location data.
     column_groups_template : bool, default False
         If True, will call `CapData.data_columns_to_excel` to save a file to use to
         manually create column groupings at `path`.
@@ -508,12 +542,20 @@ def load_data(
         elif (p.suffix == '.xlsx') or (p.suffix == '.xls'):
             cd.column_groups = cg.ColumnGroups(load_excel_column_groups(group_columns))
     if site is not None:
-        cd.data = csky(cd.data, loc=site['loc'], sys=site['sys'])
-        cd.data_filtered = cd.data.copy()
-        cd.column_groups['irr-poa-clear_sky'] = ['poa_mod_csky']
-        cd.column_groups['irr-ghi-clear_sky'] = ['ghi_mod_csky']
+        if isinstance(site, str):
+            path_to_site = Path(site)
+            if path_to_site.is_file():
+                if path_to_site.suffix == ".json":
+                    site = util.read_json(site)
+                if (path_to_site.suffix == ".yaml") or (path_to_site.suffix == ".yml"):
+                    site = util.read_yaml(site)
+                cd.site = copy.deepcopy(site)
+        if isinstance(site, dict):
+            cd.data = csky(cd.data, loc=site['loc'], sys=site['sys'])
+            cd.data_filtered = cd.data.copy()
+            cd.column_groups['irr-poa-clear_sky'] = ['poa_mod_csky']
+            cd.column_groups['irr-ghi-clear_sky'] = ['ghi_mod_csky']
     cd.trans_keys = list(cd.column_groups.keys())
-    cd.set_plot_attributes()
     if column_groups_template:
         cd.data_columns_to_excel()
     return cd

captest/plotting.py

@@ -0,0 +1,492 @@
+from pathlib import Path
+import copy
+import json
+import warnings
+import itertools
+from functools import partial
+import numpy as np
+import pandas as pd
+import panel as pn
+from panel.interact import fixed
+import holoviews as hv
+from holoviews import opts
+import colorcet as cc
+from bokeh.models import NumeralTickFormatter
+
+from .util import tags_by_regex, append_tags, read_json
+
+# disable error messages for panel dashboard
+pn.config.console_output = 'disable'
+
+COMBINE = {
+    'poa_ghi': 'irr.*(poa|ghi)$',
+    'poa_csky': '(?=.*poa)(?=.*irr)',
+    'ghi_csky': '(?=.*ghi)(?=.*irr)',
+    'temp_amb_bom': '(?=.*temp)((?=.*amb)|(?=.*bom))',
+    'inv_sum_mtr_pwr': ['(?=.*real)(?=.*pwr)(?=.*mtr)', '(?=.*pwr)(?=.*agg)'],
+}
+
+DEFAULT_GROUPS = [
+    'inv_sum_mtr_pwr',
+    '(?=.*real)(?=.*pwr)(?=.*inv)',
+    '(?=.*real)(?=.*pwr)(?=.*mtr)',
+    'poa_ghi',
+    'poa_csky',
+    'ghi_csky',
+    'temp_amb_bom',
+]
+
+
+def find_default_groups(groups, default_groups):
+    """
+    Find the default groups in the list of groups.
+
+    Parameters
+    ----------
+    groups : list of str
+        The list of groups to search for the default groups.
+    default_groups : list of str
+        List of regex strings to use to identify default groups.
+
+    Returns
+    -------
+    list of str
+        The default groups found in the list of groups.
+    """
+    found_groups = []
+    for re_str in default_groups:
+        found_grp = tags_by_regex(groups, re_str)
+        if len(found_grp) == 1:
+            found_groups.append(found_grp[0])
+        elif len(found_grp) > 1:
+            warnings.warn(
+                f'More than one group found for regex string {re_str}. '
+                'Refine regex string to find only one group. '
+                f'Groups found: {found_grp}'
+
+            )
+    return found_groups
+
+
+def parse_combine(combine, column_groups=None, data=None, cd=None):
+    """
+    Parse regex strings for identifying groups of columns or tags to combine.
+
+    Parameters
+    ----------
+    combine : dict
+        Dictionary of group names and regex strings to use to identify groups from
+        column groups and individual tags (columns) to combine into new groups.
+        Keys should be strings for names of new groups. Values should be either a string
+        or a list of two strings. If a string, the string is used as a regex to identify
+        groups to combine. If a list, the first string is used to identify groups to
+        combine and the second is used to identify individual tags (columns) to combine.
+    column_groups : ColumnGroups, optional
+        The column groups object to add new groups to. Required if `cd` is not provided.
+    data : pd.DataFrame, optional
+        The data to use to identify groups and columns to combine. Required if `cd` is
+        not provided.
+    cd : captest.CapData, optional
+        The captest.CapData object with the `data` and `column_groups` attributes set.
+        Required if `columng_groups` and `data` are not provided.
+
+    Returns
+    -------
+        ColumnGroups
+            New column groups object with new groups added.
+    """
+    if cd is not None:
+        data = cd.data
+        column_groups = cd.column_groups
+    cg_out = copy.deepcopy(column_groups)
+    orig_groups = list(cg_out.keys())
+
+    tags = list(data.columns)
+
+    for grp_name, re_str in combine.items():
+        group_re = None
+        tag_re = None
+        tags_in_matched_groups = []
+        matched_tags = []
+        if isinstance(re_str, str):
+            group_re = re_str
+        elif isinstance(re_str, list):
+            if len(re_str) != 2:
+                warnings.warn(
+                    'When passing a list of regex. There should be two strings. One for '
+                    'identifying groups and one for identifying individual tags (columns).'
+                )
+                return None
+            else:
+                group_re = re_str[0]
+                tag_re = re_str[1]
+        if group_re is not None:
+            matched_groups = tags_by_regex(orig_groups, group_re)
+            if len(matched_groups) >= 1:
+                tags_in_matched_groups = list(
+                    itertools.chain(*[cg_out[grp] for grp in matched_groups])
+                )
+        if tag_re is not None:
+            matched_tags = tags_by_regex(tags, tag_re)
+        cg_out[grp_name] = tags_in_matched_groups + matched_tags
+    return cg_out
+
+
+def msel_from_column_groups(column_groups, groups=True):
+    """
+    Create a multi-select widget from a column groups object.
+
+    Parameters
+    ----------
+    column_groups : ColumnGroups
+        The column groups object.
+    groups : bool, default True
+        By default creates list of groups i.e. the keys of `column_groups`,
+        otherwise creates list of individual columns i.e. the values of `column_groups`
+        concatenated together.
+    """
+    if groups:
+        keys = list(column_groups.data.keys())
+        keys.sort()
+        options = {k: column_groups.data[k] for k in keys}
+        name = 'Groups'
+        value = column_groups.data[list(column_groups.keys())[0]]
+    else:
+        options = []
+        for columns in column_groups.values():
+            options += columns
+        options.sort()
+        name = 'Columns'
+        value = [options[0]]
+    return pn.widgets.MultiSelect(
+        name=name,
+        value=value,
+        options=options,
+        size=8,
+        height=400,
+        width=400
+    )
+
+
+def plot_tag(data, tag, width=1500, height=250):
+    if len(tag) == 1:
+        plot = hv.Curve(data[tag])
+    elif len(tag) > 1:
+        curves = {}
+        for column in tag:
+            try:
+                curves[column] = hv.Curve(data[column])
+            except KeyError:
+                continue
+        plot = hv.NdOverlay(curves)
+    elif len(tag) == 0:
+        plot = hv.Curve(pd.DataFrame(
+            {'no_data': [np.NaN] * data.shape[0]},
+            index=data.index
+        ))
+    plot.opts(
+        opts.Curve(
+            line_width=1,
+            width=width,
+            height=height,
+            muted_alpha=0,
+            tools=['hover'],
+            yformatter=NumeralTickFormatter(format='0,0'),
+        ),
+        opts.NdOverlay(width=width, height=height, legend_position='right')
+    )
+    return plot
+
+
+def group_tag_overlay(group_tags, column_tags):
+    """
+    Overlay curves of groups and individually selected columns.
+
+    Parameters
+    ----------
+    group_tags : list of str
+        The tags to plot from the groups selected.
+    column_tags : list of str
+        The tags to plot from the individually selected columns.
+    """
+    joined_tags = [t for tag_list in group_tags for t in tag_list] + column_tags
+    return joined_tags
+
+
+def plot_group_tag_overlay(data, group_tags, column_tags, width=1500, height=400):
+    """
+    Overlay curves of groups and individually selected columns.
+
+    Parameters
+    ----------
+    data : pd.DataFrame
+        The data to plot.
+    group_tags : list of str
+        The tags to plot from the groups selected.
+    column_tags : list of str
+        The tags to plot from the individually selected columns.
+    """
+    joined_tags = group_tag_overlay(group_tags, column_tags)
+    return plot_tag(data, joined_tags, width=width, height=height)
+
+
+def plot_tag_groups(data, tags_to_plot, width=1500, height=250):
+    """
+    Plot groups of tags, one of overlayed curves per group.
+
+    Parameters
+    ----------
+    data : pd.DataFrame
+        The data to plot.
+    tags_to_plot : list
+        List of lists of strings. One plot for each inner list.
+    """
+    group_plots = []
+    if len(tags_to_plot) == 0:
+        tags_to_plot = [[]]
+    for group in tags_to_plot:
+        plot = plot_tag(data, group, width=width, height=height)
+        group_plots.append(plot)
+    return hv.Layout(group_plots).cols(1)
+
+
+def filter_list(text_input, ms_to_filter, names, event=None):
+    """
+    Filter a multi-select widget by a regex string.
+
+    Parameters
+    ----------
+    text_input : pn.widgets.TextInput
+        The text input widget to get the regex string from.
+    ms_to_filter : pn.widgets.MultiSelect
+        The multi-select widget to update.
+    names : list of str
+        The list of names to filter.
+    event : pn.widgets.event, optional
+        Passed by the `param.watch` method. Not used.
+
+    Returns
+    -------
+    None
+    """
+    if text_input.value == '':
+        re_value = '.*'
+    else:
+        re_value = text_input.value
+    names_ = copy.deepcopy(names)
+    if isinstance(names_, dict):
+        selected_groups = tags_by_regex(list(names_.keys()), re_value)
+        selected_groups.sort()
+        options = {k: names_[k] for k in selected_groups}
+    else:
+        options = tags_by_regex(names_, re_value)
+        options.sort()
+    ms_to_filter.param.update(options=options)
+
+
+def scatter_dboard(data, **kwargs):
+    """
+    Create a dashboard to plot any two columns of data against each other.
+
+    Parameters
+    ----------
+    data : pd.DataFrame
+        The data to plot.
+    **kwargs : optional
+        Pass additional keyword arguments to the holoviews options of the scatter plot.
+
+    Returns
+    -------
+    pn.Column
+        The dashboard with a scatter plot of the data.
+    """
+    cols = list(data.columns)
+    cols.sort()
+    x = pn.widgets.Select(name='x', value=cols[0], options=cols)
+    y = pn.widgets.Select(name='y', value=cols[1], options=cols)
+    # slope = pn.widgets.Checkbox(name='Slope', value=False)
+
+    defaults = {
+        'width': 500,
+        'height': 500,
+        'fill_alpha': 0.4,
+        'line_alpha': 0,
+        'size': 4,
+        'yformatter': NumeralTickFormatter(format='0,0'),
+        'xformatter': NumeralTickFormatter(format='0,0'),
+    }
+    for opt, value in defaults.items():
+        kwargs.setdefault(opt, value)
+
+    def scatter(data, x, y, slope=True, **kwargs):
+        scatter_plot = hv.Scatter(data, x, y).opts(**kwargs)
+        # if slope:
+        #     slope_line = hv.Slope.from_scatter(scatter_plot).opts(
+        #         line_color='red',
+        #         line_width=1,
+        #         line_alpha=0.4,
+        #         line_dash=(5,3)
+        #     )
+        # if slope:
+        #     return scatter_plot * slope_line
+        # else:
+        return scatter_plot
+
+    # dboard = pn.Column(
+    #     pn.Row(x, y, slope),
+    #     pn.bind(scatter, data, x, y, slope=slope, **kwargs)
+    # )
+    dboard = pn.Column(
+        pn.Row(x, y),
+        pn.bind(scatter, data, x, y, **kwargs)
+    )
+    return dboard
+
+
+def plot(
+    cd=None,
+    cg=None,
+    data=None,
+    combine=COMBINE,
+    default_groups=DEFAULT_GROUPS,
+    group_width=1500,
+    group_height=250,
+    **kwargs,
+):
+    """
+    Create plotting dashboard.
+
+    NOTE: If a 'plot_defaults.json' file exists in the same directory as the file this
+    function is called from called, then the default groups will be read from that file
+    instead of using the `default_groups` argument. Delete or manually edit the file to
+    change the default groups. Use the `default_groups` or manually edit the file to
+    control the order of the plots.
+
+    Parameters
+    ----------
+    cd : captest.CapData, optional
+        The captest.CapData object.
+    cg : captest.ColumnGroups, optional
+        The captest.ColumnGroups object. `data` must also be provided.
+    data : pd.DataFrame, optional
+        The data to plot. `cg` must also be provided.
+    combine : dict, optional
+        Dictionary of group names and regex strings to use to identify groups from
+        column groups and individual tags (columns) to combine into new groups. See the
+        `parse_combine` function for more details.
+    default_groups : list of str, optional
+        List of regex strings to use to identify default groups to plot. See the
+        `find_default_groups` function for more details.
+    group_width : int, optional
+        The width of the plots on the Groups tab.
+    group_height : int, optional
+        The height of the plots on the Groups tab.
+    **kwargs : optional
+        Pass additional keyword arguments to the holoviews options of the scatter plot
+        on the 'Scatter' tab.
+    """
+    if cd is not None:
+        data = cd.data
+        cg = cd.column_groups
+    # make sure data is numeric
+    data = data.apply(pd.to_numeric, errors='coerce')
+    bool_columns = data.select_dtypes(include='bool').columns
+    data.loc[:, bool_columns] = data.loc[:, bool_columns].astype(int)
+    # setup custom plot for 'Custom' tab
+    groups = msel_from_column_groups(cg)
+    tags = msel_from_column_groups({'all_tags': list(data.columns)}, groups=False)
+    columns_re_input = pn.widgets.TextInput(name='Input regex to filter columns list')
+    groups_re_input = pn.widgets.TextInput(name='Input regex to filter groups list')
+
+    columns_re_input.param.watch(
+        partial(filter_list, columns_re_input, tags, tags.options),
+        'value'
+    )
+    groups_re_input.param.watch(
+        partial(filter_list, groups_re_input, groups, groups.options),
+        'value'
+    )
+
+    custom_plot_name = pn.widgets.TextInput()
+    update = pn.widgets.Button(name='Update')
+    width_custom = pn.widgets.IntInput(
+        name='Plot Width', value=1500, start=200, end=2800, step=100, width=200
+    )
+    height_custom = pn.widgets.IntInput(
+        name='Plot height', value=400, start=150, end=800, step=50, width=200
+    )
+    custom_plot = pn.Column(
+        pn.Row(custom_plot_name, update, width_custom, height_custom),
+        pn.Row(
+            pn.WidgetBox(groups_re_input, groups),
+            pn.WidgetBox(columns_re_input, tags),
+        ),
+        pn.Row(pn.bind(
+            plot_group_tag_overlay,
+            data,
+            groups,
+            tags,
+            width=width_custom,
+            height=height_custom,
+        ))
+    )
+
+    # setup group plotter for 'Main' tab
+    cg_layout = parse_combine(combine, column_groups=cg, data=data)
+    main_ms = msel_from_column_groups(cg_layout)
+
+    def add_custom_plot_group(event):
+        column_groups_ = copy.deepcopy(main_ms.options)
+        column_groups_ = add_custom_plot(
+            custom_plot_name.value,
+            column_groups_,
+            groups.value,
+            tags.value,
+        )
+        main_ms.options = column_groups_
+    update.on_click(add_custom_plot_group)
+    plots_to_layout = pn.widgets.Button(name='Set plots to current layout')
+    width_main = pn.widgets.IntInput(
+        name='Plot Width', value=1500, start=200, end=2800, step=100, width=200
+    )
+    height_main = pn.widgets.IntInput(
+        name='Plot height', value=250, start=150, end=800, step=50, width=200
+    )
+    main_plot = pn.Column(
+        pn.Row(pn.WidgetBox(plots_to_layout, main_ms, pn.Row(width_main, height_main))),
+        pn.Row(pn.bind(
+            plot_tag_groups, data, main_ms, width=width_main, height=height_main
+        )),
+    )
+
+    def set_defaults(event):
+        with open('./plot_defaults.json', 'w') as file:
+            json.dump(main_ms.value, file)
+    plots_to_layout.on_click(set_defaults)
+
+    # setup default groups
+    if Path('./plot_defaults.json').exists():
+        default_tags = read_json('./plot_defaults.json')
+    else:
+        default_groups = find_default_groups(list(cg_layout.keys()), default_groups)
+        default_tags = [cg_layout.get(grp, []) for grp in default_groups]
+
+    # layout dashboard
+    plotter = pn.Tabs(
+        ('Groups', plot_tag_groups(data, default_tags, width=group_width, height=group_height)),
+        ('Layout', main_plot),
+        ('Overlay', custom_plot),
+        ('Scatter', scatter_dboard(data, **kwargs)),
+    )
+    return plotter
+
+
+def add_custom_plot(name, column_groups, group_tags, column_tags):
+    """
+    Append a new custom group to column groups for plotting.
+
+    Parameters
+    ----------
+    """
+    column_groups[name] = group_tag_overlay(group_tags, column_tags)
+    return column_groups

captest/util.py

@@ -1,3 +1,4 @@
+import re
 import json
 import yaml
 import numpy as np
@@ -61,7 +62,7 @@ def get_common_timestep(data, units='m', string_output=True):
     else:
         return freq
 
-def reindex_datetime(data, report=False, add_index_col=True):
+def reindex_datetime(data, report=False):
     """
     Find dataframe index frequency and reindex to add any missing intervals.
 
@@ -86,10 +87,6 @@ def reindex_datetime(data, report=False, add_index_col=True):
     df_index_length = df.shape[0]
     missing_intervals = df_index_length - data_index_length
 
-    if add_index_col:
-        ix_ser = df.index.to_series()
-        df['index'] = ix_ser.apply(lambda x: x.strftime('%m/%d/%Y %H %M'))
-
     if report:
         print('Frequency determined to be ' + freq_str + ' minutes.')
         print('{:,} intervals added to index.'.format(missing_intervals))
@@ -136,3 +133,14 @@ def generate_irr_distribution(
         else:
             irr_values.append(next_val)
     return irr_values
+
+
+def tags_by_regex(tag_list, regex_str):
+    regex = re.compile(regex_str, re.IGNORECASE)
+    return [tag for tag in tag_list if regex.search(tag) is not None]
+
+
+def append_tags(sel_tags, tags, regex_str):
+    new_list = sel_tags.copy()
+    new_list.extend(tags_by_regex(tags, regex_str))
+    return new_list

{captest-0.12.1.dist-info → captest-0.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: captest
-Version: 0.12.1
+Version: 0.13.0
 Summary: Framework and methods to facilitate photovoltaic facility capacity testing following ASTM E2848.
 Home-page: http://github.com/bt-/pvcaptest
 Author: Ben Taylor