ooi-data-explorations 0.2.3__py3-none-any.whl

ooi_data_explorations/construct_urls.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@author Christopher Wingard
+@brief Creates the YAML file with the information needed to construct URLs for M2M requests
+"""
+import requests
+
+from ooi_data_explorations.common import AUTH, BASE_URL
+from ooi_data_explorations.common import list_nodes, list_sensors, list_methods, list_streams
+
+# Organize the nodes into assemblies common across all the arrays. Helps to better organize the data, taking all the
+# different names used in OOI and collapsing them down to a more coherent, and logical grouping. This list is
+# deliberately non-inclusive of all possible nodes, as some are either specific to control systems or have data not
+# readily accessed through the M2M system.
+ASSEMBLY = {
+    'buoy': ['SBD11', 'SBD12', 'SBD17'],
+    'midwater': ['PC01A', 'PC01B', 'PC03A', 'RID16', 'RID26', 'RID27', 'RII01', 'RII11', 'RIM01', 'RIS01'],
+    'seafloor': [
+        'LJ01A', 'LJ01B', 'LJ01C', 'LJ01D', 'LJ03A', 'MFD35', 'MFD37', 'MJ01A', 'MJ01B', 'MJ03A', 'MJ03B',
+        'MJ03C', 'MJ03D', 'MJ03E', 'MJ03F', 'PN03B'],
+    'glider': [
+        'GL247', 'GL276', 'GL311', 'GL312', 'GL319', 'GL320', 'GL326', 'GL327', 'GL335', 'GL336', 'GL339',
+        'GL340', 'GL361', 'GL362', 'GL363', 'GL364', 'GL365', 'GL374', 'GL375', 'GL376', 'GL379', 'GL380',
+        'GL381', 'GL382', 'GL383', 'GL384', 'GL386', 'GL387', 'GL388', 'GL389', 'GL453', 'GL469', 'GL470',
+        'GL477', 'GL478', 'GL484', 'GL485', 'GL486', 'GL493', 'GL494', 'GL495', 'GL496', 'GL514', 'GL523',
+        'GL524', 'GL525', 'GL537', 'GL538', 'GL559', 'GL560', 'GL561', 'PG514', 'PG515', 'PG528', 'PG562',
+        'PG563', 'PG564', 'PG565', 'PG566', 'PG575', 'PG578', 'PG580', 'PG583'
+    ],
+    'auv': ['A6263', 'A6264'],
+    'profiler': ['WFP01', 'WFP02', 'WFP03', 'SP001', 'SF01A', 'SF01B', 'SF03A', 'DP01A', 'DP01B', 'DP03A']
+}
+
+# Assemblies can be further categorized by a sub-assembly type if needed. Otherwise, the code and assembly type are 
+# the same. This helps to differentiate times where more than one assembly type is present at a site and we need to 
+# further distinguish between them. 
+SUBASSEMBLY = {
+    'nsif': ['RID16', 'RID26', 'RID27'],
+    'riser': ['RII01', 'RII11', 'RIM01'],
+    'sphere': ['RIS01'],
+    '200m': ['PC01A', 'PC01B', 'PC03A'],
+    'mfn': ['MFD35', 'MFD37'],
+    'bep': ['LJ01C', 'LJ01D'],
+    'low-power': ['LJ01A', 'LJ01B', 'LJ03A'],
+    'medium-power': ['MJ01A', 'MJ01B', 'MJ03A', 'MJ03B', 'MJ03C', 'MJ03D', 'MJ03E', 'MJ03F'],
+    'coastal-glider': [
+        'GL247', 'GL276', 'GL311', 'GL312', 'GL319', 'GL320', 'GL326', 'GL327', 'GL335', 'GL336', 'GL339', 'GL340',
+        'GL361', 'GL362', 'GL363', 'GL364', 'GL365', 'GL374', 'GL375', 'GL376', 'GL379', 'GL380', 'GL381', 'GL382',
+        'GL383', 'GL384', 'GL386', 'GL387', 'GL388', 'GL389', 'GL514',
+    ],
+    'global-glider': [
+        'GL453', 'GL469', 'GL470', 'GL477', 'GL478', 'GL484', 'GL485', 'GL486', 'GL493', 'GL494', 'GL495', 'GL496',
+        'GL523', 'GL524', 'GL525', 'GL537', 'GL538', 'GL559', 'GL560', 'GL561'
+    ],
+    'profiling-glider': [
+        'PG514', 'PG515', 'PG528', 'PG562', 'PG563', 'PG564', 'PG565', 'PG566', 'PG575', 'PG578', 'PG580', 'PG583'
+    ],
+    'cspp': ['SP001'],
+    'coastal-wfp': ['WFP01'],
+    'global-wfp': ['WFP02', 'WFP03'],
+    'cabled-wfp': ['DP01A', 'DP01B', 'DP03A'],
+    'shallow-profiler': ['SF01A', 'SF01B', 'SF03A']
+}
+
+# List of allowed methods (there are some called bad_*, ignoring those)
+METHODS = ['streamed', 'telemetered', 'recovered_host', 'recovered_inst', 'recovered_cspp', 'recovered_wfp']
+
+# Sensor codes to exclude. These are either not science sensors (e.g. 00-DCLENG000), or not really accessible from the
+# M2M system (e.g. cameras, hydrophones or seismometers).
+SENSOR_EXCLUDES = [
+    'AOABP', 'CAM', 'COVIS', 'ENG', 'FLOBN', 'HPIES', 'HYD', 'MASSP', 'OBS', 'OSMOI', 'OVRSR', 'PPSDNA', 'QNTSR',
+    'RASFL', 'SCPRA', 'SCTAA', 'ZPLSCB',
+]
+
+# Several streams were defined within OOI that are beyond core science needs. The terms below filter out those streams.
+STREAM_EXCLUDES = [
+    'metadata', 'diagnostics', 'abc_power', 'dcl_power', 'config', 'wave_burst', 'imodem_status',
+    'imodem_start_time', 'cspp_dark_instrument', 'instrument_blank', 'adcp_engineering', 'adcp_ancillary',
+    'adcp_transmit', 'dcl_fourier', 'dcl_mean_directional', 'dcl_motion', 'dcl_non_directional', 'echogram_data',
+    'calibration', 'status', 'settings', 'hardware', 'adcp_pd0_beam_parsed', 'identification_string', 'clock_data',
+    'battery_voltage', 'thermistor_voltage', 'dev1_data', 'data_record_cal', 'dark_sample', 'imodem_control',
+    'imodem_power', 'offset', 'adcp_velocity_glider', 'nutnr_b_dcl_dark', 'nutnr_b_metadata', 'nutnr_b_dark',
+    'prest_event_counter', 'prest_reference_oscillator', 'vel3d_b_engineering', 'tmpsf_engineering',
+    'vadcp_5thbeam_pd0_beam_parsed', 'nutnr_a_test'
+]
+
+
+# grab the entire OOI vocabulary dictionary
+def get_vocabulary():
+    """
+    Uses the an API call to grab the entire vocabulary dictionary for all OOI sites, nodes and sensors
+
+    :return: json object with the site-node-sensor specific vocabulary
+    """
+    r = requests.get(BASE_URL + '12586/vocab', auth=(AUTH[0], AUTH[2]))
+    if r.status_code == requests.codes.ok:
+        return r.json()
+    else:
+        return None
+
+
+VOCAB = get_vocabulary()
+
+
+def add_site(site, url_file):
+    """
+    For an OOI site, assemble a curated list of all the instruments and data streams that are available for data
+    explorations. This file will create a YAML file per site. Additional HITL work is required to further clean-up and
+    check the list, and pruning streams and methods down to the core set of science sensors. The results of this work
+    is used to create the m2m_urls.yml file used in this python toolbox.
+
+    :param site: OOI 8 character site designation (e.g. CE01ISSM)
+    :param url_file: data file to save the results in.
+    :return: None, creates the data file.
+    """
+    sp = '  '  # create an explicit two-space indent string for the YAML file
+    # open the YAML file to store the results
+    with open(url_file, 'w') as f:
+        # create the header portion of the YAML, getting the array and site names from the API
+        site = site.upper()
+        for site_vocab in VOCAB:
+            if site_vocab.get('refdes') == site:
+                break
+
+        f.write('---\n{site}:\n'.format(site=site))
+        f.write('{sp}array: {array}\n'.format(sp=sp*1, array=site_vocab['tocL1']))
+        f.write('{sp}name: {name}\n'.format(sp=sp*1, name=site_vocab['tocL2']))
+        f.write('{sp}assembly:\n'.format(sp=sp*1))
+
+        # create a list of nodes for this site
+        nodes = list_nodes(site)
+
+        # for each node, if it is one of interest as defined above, create an assembly entry
+        for k, v in ASSEMBLY.items():
+            # find the nodes that correspond to this assembly
+            assembly = k
+            node = sorted(set(v) & set(nodes))
+
+            # skip if we don't have any nodes in this assembly
+            if not node:
+                continue
+
+            # if we have nodes, create the assembly entry
+            for node_vocab in VOCAB:
+                if node_vocab.get('refdes') == site + '-' + node[0]:
+                    break
+
+            # for each node, create the list of sensors
+            for n in node:
+                f.write('{sp}- type: {assembly}\n'.format(sp=sp * 2, assembly=assembly))
+                f.write('{sp}name: {name}\n'.format(sp=sp * 3, name=node_vocab['tocL3']))
+                # if we need to further distinguish, add the assembly code name
+                for l, w in SUBASSEMBLY.items():
+                    if n in w:
+                        f.write('{sp}subassembly: {name}\n'.format(sp=sp * 3, name=l))
+                f.write('{sp}instrument:\n'.format(sp=sp * 3))
+                sensors = list_sensors(site, n)
+                sensors = filter_stream(sensors, SENSOR_EXCLUDES)   # remove sensors of no interest
+
+                if not sensors:
+                    continue
+
+                for sensor in sensors:
+                    for sensor_vocab in VOCAB:
+                        if sensor_vocab.get('refdes') == site + '-' + n + '-' + sensor:
+                            break
+
+                    instrument = (sensor[3:8]).lower()
+                    f.write('{sp}- class: {instrument}\n'.format(sp=sp*4, instrument=instrument))
+                    f.write('{sp}instrument_name: {name}\n'.format(sp=sp*5, name=sensor_vocab['instrument']))
+                    f.write('{sp}instrument_model: {model}\n'.format(sp=sp*5, model=sensor_vocab['model']))
+                    f.write('{sp}instrument_manufacturer: {manu}\n'.format(sp=sp*5, manu=sensor_vocab['manufacturer']))
+                    f.write('{sp}mindepth: {mindepth}\n'.format(sp=sp*5, mindepth=sensor_vocab['mindepth']))
+                    f.write('{sp}maxdepth: {maxdepth}\n'.format(sp=sp*5, maxdepth=sensor_vocab['maxdepth']))
+                    f.write('{sp}node: {node}\n'.format(sp=sp*5, node=n))
+                    f.write('{sp}sensor: {sensor}\n'.format(sp=sp*5, sensor=sensor))
+                    f.write('{sp}stream:\n'.format(sp=sp*5))
+
+                    methods = list_methods(site, n, sensor)
+                    if not methods:
+                        f.write('{sp}unknown: null\n'.format(sp=sp*6,))
+                        continue
+
+                    for method in methods:
+                        if method in METHODS:
+                            streams = list_streams(site, n, sensor, method)
+                            streams = filter_stream(streams, STREAM_EXCLUDES)
+                            if len(streams) == 1:
+                                f.write('{sp}{method}: {streams}\n'.format(sp=sp*6, method=method,
+                                                                           streams=streams[0]))
+                            else:
+                                f.write('{sp}{method}:\n'.format(sp=sp*6, method=method))
+                                for stream in streams:
+                                    f.write('{sp}- {stream}\n'.format(sp=sp*7, stream=stream))
+
+
+def filter_stream(streams, excludes):
+    """
+    Uses a list of keywords to remove sensors or streams from the list returned by OOI Net.
+
+    :param streams: list of sensor or streams returned from OOI Net
+    :param excludes: list of keywords to use in pruning the list
+    :return: a cleaned, pruned list
+    """
+    clean = []
+    for stream in streams:
+        if not any(sub in stream for sub in excludes):
+            clean.append(stream)
+
+    return clean

ooi_data_explorations/data_request.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import numpy as np
+import pytz
+import xarray as xr
+import dateutil.parser as parser
+
+from ooi_data_explorations import M2M_URLS
+from ooi_data_explorations.common import dr_inputs, m2m_request, m2m_collect, get_deployment_dates
+
+
+def filter_urls(site, assembly, instrument, method):
+    """
+    Takes the M2M_URLS dictionary and searches for the instrument of interest
+    as defined by the site code, assembly type, instrument class, and data
+    delivery method to return the OOI specific site, node and stream names
+    needed to request the data.
+
+    :param site: OOI eight letter site code (e.g. CE04OSPS for the Oregon
+        Offshore Shallow Profiler)
+    :param assembly: Assembly grouping name (e.g. midwater for the 200 m
+        Platform)
+    :param instrument: The instrument class name (e.g. phsen for the
+        SAMI2-pH sensor)
+    :param method: The data delivery method (e.g. streamed for cabled
+        streaming data)
+
+    :return node: The OOI specific node code(s) for the assembly
+    :return sensor: The OOI specific sensor code(s) for the instrument class
+    :return stream: The OOI specific stream name(s) for the site, node, sensor
+        and delivery method combination
+    """
+    # Use the site, assembly, instrument, and data delivery method to determine the OOI specific site, node, sensor,
+    # method and stream
+    node = []
+    sensor = []
+    stream = []
+
+    # pare the larger dictionary down to the site of interest and check if a valid site was used
+    m2m_urls = M2M_URLS.get(site.upper())
+    if not m2m_urls:
+        raise SyntaxError('Unknown site code: %s' % site)
+
+    # make sure the correct data delivery method was specified
+    if method not in ['streamed', 'telemetered', 'recovered_host', 'recovered_inst', 'recovered_cspp', 'recovered_wfp']:
+        raise SyntaxError('Unknown data delivery method: %s' % method)
+
+    # find the instrument(s) of interest in the assembly group
+    for grouping in m2m_urls.get('assembly'):
+        if grouping.get('type') == assembly or grouping.get('subassembly') == assembly:
+            for instrmt in grouping['instrument']:
+                if instrmt['class'] == instrument:
+                    node.append(instrmt.node)
+                    sensor.append(instrmt.sensor)
+                    stream.append(instrmt.stream.get(method))
+
+    # check to see if we were able to find the system of interest
+    if not stream:
+        raise RuntimeWarning('Instrument defined by %s-%s-%s-%s cannot be found.' % (site, assembly, instrument,
+                                                                                     method))
+
+    # return the OOI specific names for the node(s), sensor(s) and stream(s)
+    return node, sensor, stream
+
+
+def data_request(site, assembly, instrument, method, **kwargs):
+    """
+    Requests data via the OOI M2M API using the site code, assembly type,
+    instrument class and data delivery method as defined in the m2m_urls.yml to
+    construct the OOI specific data request.
+
+    :param site: OOI site code as an 8 character string
+    :param assembly: The assembly type where the instrument is located
+    :param instrument: the OOI instrument class name for the instrument of
+        interest
+    :param method: The data delivery method for the system of interest
+    :param kwargs: Takes the following optional keyword arguments:
+
+        start: Starting date/time for the data request in a dateutil.parser
+            recognizable form. If ``None``, the default, the beginning of the
+            data record will be used
+        stop: Ending date/time for the data request in a dateutil.parser
+            recognizable form If ``None``, the default, the end of the data
+            record will be used
+        deploy: Use the deployment number, entered as an integer, to set the
+            starting and ending dates If ``None``, the default, the starting
+            and ending dates are used. If you enter both, the deployment number
+            will take priority in setting the start and end dates
+        aggregate: In cases where more than one instance of an instrument class
+            is part of an assembly, will collect all of the data if the integer
+            value entered is ``0``, or the specific instance of the instrument
+            is requested if any value greater than ``0`` is used. If ``None``,
+            the default, the first instance of an instrument will be used.
+
+    :return data: Returns the request data as an xarray dataset for further analysis
+    """
+    # setup inputs to the function, make sure case is correct
+    site = site.upper()
+    assembly = assembly.lower()
+    instrument = instrument.lower()
+    method = method.lower()
+
+    # parse the keyword arguments
+    start = None
+    stop = None
+    deploy = None
+    aggregate = None
+    for key, value in kwargs.items():
+        if key not in ['start', 'stop', 'deploy', 'aggregate']:
+            raise KeyError('Unknown keyword (%s) argument.' % key)
+        else:
+            if key == 'start':
+                start = value
+            if key == 'stop':
+                stop = value
+            if key == 'deploy':
+                deploy = value
+            if key == 'aggregate':
+                aggregate = value
+
+    # use the assembly, instrument and data delivery methods to find the system of interest
+    node, sensor, stream = filter_urls(site, assembly, instrument, method)
+
+    # check the formatting of the start and end dates. We need to be able to parse and convert to an ISO format.
+    if start:
+        # check the formatting of the start date string and convert to the ISO format used by the M2M API
+        try:
+            start = parser.parse(start)
+            start = start.astimezone(pytz.utc)
+            start = start.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+        except parser.ParserError:
+            raise SyntaxError('Formatting of the starting date string needs to be in a recognizable format')
+
+    if stop:
+        # check the formatting of the stop date string and convert to the ISO format used by the M2M API
+        try:
+            stop = parser.parse(stop)
+            stop = stop.astimezone(pytz.utc)
+            stop = stop.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+        except parser.ParserError:
+            raise SyntaxError('Formatting of the ending date string needs to be in a recognizable format')
+
+    if deploy:
+        # Determine start and end dates based on the deployment number
+        start, stop = get_deployment_dates(site, node[0], sensor[0], deploy)
+        if not start or not stop:
+            exit_text = ('Deployment dates are unavailable for %s-%s-%s-%s, deployment %02d.' % (site.lower(), assembly,
+                                                                                                 instrument, method,
+                                                                                                 deploy))
+            raise RuntimeWarning(exit_text)
+
+    # for some cases, there maybe more than 1 stream, but in general, we only want the first one
+    if isinstance(stream[0], list):
+        stream = stream[0][0]
+    else:
+        stream = stream[0]
+
+    tag = ('.*{instrument}.*\\.nc$'.format(instrument=instrument.upper()))  # set regex tag to use when downloading
+    data = None     # setup the default data set
+
+    # check if there are multiple instances of this instrument class on the assembly
+    if len(node) > 1:
+        print('There are multiple instances of the instrument %s under %s-%s.' % (instrument, site.lower(), assembly))
+
+    # check if we are aggregating the multiple instruments into a single data set
+    if isinstance(aggregate, int):
+        if aggregate == 0:
+            # request all of the instruments associated with this site, assembly, instrument and method
+            print(('Requesting all %d instances of this instrument. Data sets will be concatenated\n'
+                   'and a new variable called `sensor_count` will be added to help distinguish the \n'
+                   'instruments for later processing.') % len(node))
+            for i in range(len(node)):
+                r = m2m_request(site, node[i], sensor[i], method, stream, start, stop)
+                if r:
+                    temp = m2m_collect(r, tag)
+                    temp['sensor_count'] = temp['deployment'] * 0 + i + 1
+                    if not data:
+                        data = temp
+                    else:
+                        data = xr.concat([data, temp], dim='time')
+        else:
+            # request a specific instrument of the multiple instruments associated with this site, assembly,
+            # instrument and method.
+            if aggregate > len(node):
+                raise SyntaxError('Only %d instruments available, you selected %d' % (len(node), aggregate))
+
+            print('Requesting instrument %d out of %d.' % (aggregate, len(node)))
+            i = aggregate - 1
+            r = m2m_request(site, node[i], sensor[i], method, stream, start, stop)
+            if r:
+                data = m2m_collect(r, tag)
+    else:
+        r = m2m_request(site, node[0], sensor[0], method, stream, start, stop)
+        if r:
+            data = m2m_collect(r, tag)
+
+    if not data:
+        raise RuntimeWarning('Data unavailable for %s-%s-%s-%s.' % (site.lower(), assembly, instrument, method))
+
+    # return the resulting data, which is an xarray.Dataset object
+    for v in data.variables:
+        # first convert strings with data types set as objects or S64 with binary encoding
+        if data[v].dtype == np.dtype('O') or data[v].dtype == np.dtype('S64'):
+            data[v] = data[v].astype(np.str)
+
+    return data
+
+
+def main(argv=None):
+    """
+    CLI interface to the data_request function. Requests data via the OOI M2M
+    API using the site code, assembly type, instrument class and data delivery
+    method as defined in the m2m_urls.yml to construct the OOI specific data
+    request.
+
+    :param argv: Command line inputs used by the function are:
+
+        site (-s, --site): OOI site code as an 8 character string (required)
+        assembly (-a, --assembly): Assembly or subassembly type where the
+            instrument is located (required)
+        instrument (-i, --instrument): OOI instrument class name for the
+            instrument of interest (required)
+        method (-m, --method): The data delivery method for the system of
+            interest (required)
+        outfile (-o, --outfile): An absolute or relative path and file name
+            for where the resulting data should be saved as a NetCDF file
+            (required)
+
+        start (-bt, --beginDT): Starting or beginning date/time for the data
+            request in a dateutil.parser recognizable form. If ``None``, the
+            default, the beginning of the data record will be used (optional)
+        stop (-et, --endDT): Ending date/time for the data request in a
+            dateutil.parser recognizable form If ``None``, the default, the end
+            date of the data record will be used (optional)
+        deploy (-dp, --deploy): Use the deployment number, entered as an
+            integer, to set the beginning and ending dates If ``None``, the
+            default, the beginning and ending dates are used. If you enter
+            both, the deployment number will take priority in setting the start
+            and end dates (optional)
+        aggregate (-ag, --aggregate): In cases where more than one instance of
+            an instrument class is part of an assembly, collect all of the data
+            if the integer value entered is ``0``, or the specific instance of
+            the instrument is requested if any value greater than ``0`` is
+            used. If ``None``, the default, the first instance of an instrument
+            will be requested. (optional)
+
+    :return: Saves the data to disk as a NetCDF file.
+    """
+    args = dr_inputs(argv)
+    site = args.site
+    assembly = args.assembly
+    instrument = args.instrument
+    method = args.method
+    outfile = os.path.abspath(args.outfile)
+    deploy = args.deploy
+    start = args.start
+    stop = args.stop
+    aggregate = args.aggregate
+
+    # request the data
+    data = data_request(site, assembly, instrument, method, start=start, stop=stop, deploy=deploy, aggregate=aggregate)
+
+    # save the data to disk
+    if not os.path.exists(os.path.dirname(outfile)):
+        os.makedirs(os.path.dirname(outfile))
+
+    data.to_netcdf(outfile, mode='w', format='NETCDF4', engine='h5netcdf')
+
+
+if __name__ == '__main__':
+    main()