PyThea 0.8.1__tar.gz → 0.9.0__tar.gz

{PyThea-0.8.1 → PyThea-0.9.0}/CHANGELOG.md

@@ -1,3 +1,19 @@
+# v0.9.0 (8-Feb-2024)
+
+## Features
+- Adds sample data methods for testing and document builds
+- Adds tests: test_get_horizons_coord, test_vso_search, test_hek_client, and test_parameter_fit_polynomial
+- Adds calibration for AIA, LASCO, and STEREO EUVI
+
+## Major Changes
+- Changes the fitting time input from map.date to map.date_average when exist (see #24)
+
+## Minor Changes
+- Improves the following utilities: make_figure, maps_process, plot_fitting_model
+- Improves the test_get_hek_flare and test_hek_client tests
+- Changes how best_fit_x is defined in the parameters_fit utility
+- Changes the pipeline of imaging data download and processing in the app
+
 # v0.8.1 (13-Jan-2024)
 
 ## Features

{PyThea-0.8.1/PyThea.egg-info → PyThea-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyThea
-Version: 0.8.1
+Version: 0.9.0
 Summary: PyThea: A software package to reconstruct the 3D structure of CMEs and shock waves
 Home-page: https://github.com/AthKouloumvakos/PyThea
 Author: Athanasios Kouloumvakos
@@ -23,11 +23,13 @@ License-File: LICENSE.md
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: scipy
+Requires-Dist: aiapy
 Requires-Dist: astropy
 Requires-Dist: astroquery
 Requires-Dist: numexpr
 Requires-Dist: sunpy==4.1.0
 Requires-Dist: parfive==1.5.1
+Requires-Dist: pooch
 Requires-Dist: matplotlib
 Requires-Dist: seaborn
 Requires-Dist: streamlit

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea/PyThea_app.py

@@ -33,8 +33,9 @@ from PyThea.modules import (date_and_event_selection, final_parameters_gmodel,
                             fitting_and_slider_options_container,
                             fitting_sliders, maps_clims)
 from PyThea.sunpy_dev.map.maputils import get_closest
-from PyThea.utils import (download_fits, make_figure, maps_process,
-                          model_fittings, plot_bodies, plot_fitting_model)
+from PyThea.utils import (download_fits, make_figure, model_fittings,
+                          plot_bodies, plot_fitting_model,
+                          single_imager_maps_process)
 from PyThea.version import version
 
 
@@ -66,6 +67,10 @@ def footer_text():
                    **Version**: {version} (latest release [![Version](https://img.shields.io/github/v/release/AthKouloumvakos/PyThea)](https://github.com/AthKouloumvakos/PyThea/releases))
                    """)
     left.image('https://github.com/AthKouloumvakos/PyThea/blob/master/docs/logo/pythea_logo.png?raw=true')
+    st.warning('''
+           ⚠️ **NOTE: From PyThea >0.8.1 the JSON fitting files will be slightly different from the old ones.** ⚠️
+           * Due to a change in the fitting time input, the new fitting files may have slightly different times for the same images (see further information [here](https://github.com/AthKouloumvakos/PyThea/discussions/24)).
+           ''')
     st.markdown('---')
 
 
@@ -165,9 +170,7 @@ def run():
                                                        options=selected_imagers.imager_dict.keys(),
                                                        default=['LC2', 'LC3', 'COR2A'],
                                                        key='imagers_list')
-        select_imagers_form.form_submit_button(label='Submit',
-                                               on_click=delete_from_state,
-                                               kwargs={'vars': ['map', ]})
+        select_imagers_form.form_submit_button(label='Submit')
         select_timerange_form = st.form(key='select_timerange_form', border=False)
         imaging_time_range = select_timerange_form.slider('Time Range [hours]',
                                                           min_value=-3., max_value=6.,
@@ -175,7 +178,7 @@ def run():
                                                           key='imaging_time_range')
         select_timerange_form.form_submit_button(label='Submit',
                                                  on_click=delete_from_state,
-                                                 kwargs={'vars': ['map', 'map_']})
+                                                 kwargs={'vars': ['map', 'map_', 'imagers_list_']})
 
     with st.sidebar.expander('Processing Options'):
         procoption_container = st.container()
@@ -184,7 +187,7 @@ def run():
             st.session_state['imagers_list_'] = []
         image_mode = procoption_container.selectbox('Map sequence processing',
                                                     options=['Running Diff.', 'Base Diff.', 'Plain'], key='image_mode',
-                                                    on_change=delete_from_state, kwargs={'vars': ['map', ]})
+                                                    on_change=delete_from_state, kwargs={'vars': ['map', 'imagers_list_']})
 
     with st.sidebar.expander('Plot/View Options'):
         plotviewopt_container = st.container()
@@ -198,22 +201,31 @@ def run():
     #############################################################
     # Download and Process the Images
     # This part runs only if the map_ doesn't exits or the session_state.map_ does not contain all the imagers requested
-    if 'map_' not in st.session_state or [False for lst in imagers_list if lst not in st.session_state.map_]:
+    imager_added = list(set(imagers_list) - set(st.session_state['imagers_list_']))
+    imager_removed = list(set(st.session_state['imagers_list_']) - set(imagers_list))
+    if 'map_' not in st.session_state or imager_added != []:
         st.session_state.map_ = {} if 'map_' not in st.session_state else st.session_state.map_
-        progress_bar = stqdm.stqdm(imagers_list, desc='Preparing to Download data')
+        st.session_state.map = {} if 'map' not in st.session_state else st.session_state.map
+        st.session_state['imagers_list_'] = imagers_list
+        progress_bar = stqdm.stqdm(imager_added, desc='Preparing to Download data')
         for imager in progress_bar:
-            if imager in st.session_state.map_:
-                pass
-            else:
-                progress_bar.desc = f'Downloaded {imager} images from VSO'
+            progress_bar.desc = f'Downloaded {imager} images from VSO'
+            if imager not in st.session_state.map_:
                 st.session_state.map_[imager] = download_fits(st.session_state.date_process,
                                                               imager, time_range=imaging_time_range)
+            processed_images = single_imager_maps_process(st.session_state.map_[imager],
+                                                          image_mode=image_mode,
+                                                          **selected_imagers.imager_dict[imager][1])
+            if processed_images != []:
+                st.session_state.map[imager] = processed_images
+            else:
+                st.session_state.imagers_list_.remove(imager)
+    if imager_removed != []:
+        st.session_state['imagers_list_'] = imagers_list
+        for imager in imager_removed:
+            st.session_state.map_.pop(imager, None)
 
-    if 'map' not in st.session_state:
-        st.session_state.map, st.session_state.imagers_list_ = maps_process(st.session_state.map_,
-                                                                            imagers_list,
-                                                                            image_mode)
-        maps_clims(st, imagers_list)
+    maps_clims(st, st.session_state.imagers_list_)
 
     if not st.session_state.imagers_list_:
         st.error('No images have been downloaded or processed.')  # TODO: Explain better
@@ -226,7 +238,7 @@ def run():
     imager_select = col1.selectbox('Select an imager',
                                    options=st.session_state.imagers_list_)
 
-    maps_date = [maps.date for maps in st.session_state.map[imager_select]]
+    maps_date = [getattr(maps, 'date_average', None) or getattr(maps, 'date', None) for maps in st.session_state.map[imager_select]]
     if len(maps_date) > 1:
         running_map_date = col2.select_slider('Slide to the image time',
                                               options=maps_date, value=maps_date[0],
@@ -261,12 +273,12 @@ def run():
         center = SkyCoord(np.sign(rcenter) * Spher_rep.to_cartesian(),
                           frame=frames.HeliographicCarrington,
                           observer='earth',
-                          obstime=running_map.date)
+                          obstime=running_map_date)
     elif st.session_state.coord_system == 'HGS':
         center = SkyCoord(np.sign(rcenter) * Spher_rep.to_cartesian(),
                           frame=frames.HeliographicStonyhurst,
                           observer='earth',
-                          obstime=running_map.date)
+                          obstime=running_map_date)
     st.session_state.center = center
 
     if st.session_state.geometrical_model == 'Spheroid':
@@ -373,29 +385,29 @@ def run():
 
         if plt_kinematics_select == 'All':
             col1, col2 = st.columns(2)
-            fig_ht = plot_fitting_model(st.session_state.model_fittings,
-                                        fit_args=fit_args_,
-                                        plt_type='HeightT')
+            fig_ht, axis_ht = plot_fitting_model(st.session_state.model_fittings,
+                                                 fit_args=fit_args_,
+                                                 plt_type='HeightT')
             col1.pyplot(fig_ht)
-            fig_vt = plot_fitting_model(st.session_state.model_fittings,
-                                        fit_args=fit_args_,
-                                        plt_type='SpeedT')
+            fig_vt, axis_vt = plot_fitting_model(st.session_state.model_fittings,
+                                                 fit_args=fit_args_,
+                                                 plt_type='SpeedT')
             col2.pyplot(fig_vt)
         elif plt_kinematics_select == 'Long-LatT':
             fit_args_['bounds'] = ([-np.inf, 0, -np.inf], [np.inf, np.inf, np.inf])
             col1, col2 = st.columns(2)
-            fig_loT = plot_fitting_model(st.session_state.model_fittings,
-                                         fit_args=fit_args_,
-                                         plt_type='LongT')
+            fig_loT, axis_loT = plot_fitting_model(st.session_state.model_fittings,
+                                                   fit_args=fit_args_,
+                                                   plt_type='LongT')
             col1.pyplot(fig_loT)
-            fig_laT = plot_fitting_model(st.session_state.model_fittings,
-                                         fit_args=fit_args_,
-                                         plt_type='LatT')
+            fig_laT, axis_laT = plot_fitting_model(st.session_state.model_fittings,
+                                                   fit_args=fit_args_,
+                                                   plt_type='LatT')
             col2.pyplot(fig_laT)
         else:
-            fig = plot_fitting_model(st.session_state.model_fittings,
-                                     fit_args=fit_args_,
-                                     plt_type=plt_kinematics_select)
+            fig, axis = plot_fitting_model(st.session_state.model_fittings,
+                                           fit_args=fit_args_,
+                                           plt_type=plt_kinematics_select)
             st.pyplot(fig)
         st.session_state.model_fittings.kinematics['fit_method'] = fit_args_
     else:

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.8.1'
-__version_tuple__ = version_tuple = (0, 8, 1)
+__version__ = version = '0.9.0'
+__version_tuple__ = version_tuple = (0, 9, 0)

PyThea-0.9.0/PyThea/data/sample_data.py

@@ -0,0 +1,15 @@
+import os
+from pathlib import Path
+
+import pooch
+
+database_dir = os.path.join(Path.home(), 'PyThea')
+github_main_url = 'https://github.com/AthKouloumvakos/PyThea-sample-data'
+
+aia_sample_data = pooch.create(
+    path=os.path.join(database_dir, 'sample_data'),  # The cache folder
+    base_url=f'{github_main_url}/raw/main/data/',  # The remote data url on Github
+    registry={
+        'aia_lev1_1600a_2012_06_06t04_07_29_12z_image_lev1_lowres.fits': 'sha256:396b9ef5cbc1cbe2f2af806758f449cfd36f352826b61295e6330859ac7f7652',
+    },
+)

PyThea-0.9.0/PyThea/sunpy_dev/extern/sunkit_instruments/lasco/utils.py

@@ -0,0 +1,19 @@
+from astropy.time import TimeDelta
+from sunpy.coordinates.ephemeris import get_horizons_coord
+
+__all__ = ['prep_lasco']
+
+
+def prep_lasco(map_sequence):
+    for map_ in map_sequence:
+        map_.meta['DATE-AVG'] = (map_.date + TimeDelta(map_.meta['exptime']/2, format='sec')).value
+
+    dates = [tmap.date_average for tmap in map_sequence]
+
+    observer = get_horizons_coord('SOHO', dates)
+    for map_, soho_ in zip(map_sequence, observer):
+        map_.meta['HGLN_OBS'] = soho_.lon.to('deg').value
+        map_.meta['HGLT_OBS'] = soho_.lat.to('deg').value
+        map_.meta['DSUN_OBS'] = soho_.radius.to('m').value
+
+    return map_sequence

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea/sunpy_dev/extern/sunkit_instruments/stereo/utils.py

@@ -1,9 +1,11 @@
+import copy
 import numpy as np
 import sunpy.map
+import scipy
 
 from PyThea.sunpy_dev.map import maputils
 
-__all__ = ['cor_polariz']
+__all__ = ['cor_polariz', 'euvi_prep']
 
 
 def cor_polariz(map_sequence):
@@ -58,11 +60,15 @@ def cor_polariz(map_sequence):
 
         t = [map_0.date, map_120.date, map_240.date]
         obs_time = min(t) + (max(t)-min(t))/2
+        t = [map_0.date_average, map_120.date_average, map_240.date_average]
+        time_avg = min(t) + (max(t)-min(t))/2
+
         crota = np.mean([map_0.meta['crota'], map_120.meta['crota'], map_240.meta['crota']])
 
         new_map_meta = map_120.meta
         # TODO: Check if any other keys need to change
         new_map_meta['date-obs'] = obs_time.strftime('%Y-%m-%dT%H:%M:%S.%f')
+        new_map_meta['date-avg'] = time_avg.strftime('%Y-%m-%dT%H:%M:%S.%f')
         new_map_meta['polar'] = '1001'
         new_map_meta['crval1'] = np.mean([map_0.meta['crval1'], map_120.meta['crval1'], map_240.meta['crval1']])
         new_map_meta['crval2'] = np.mean([map_0.meta['crval2'], map_120.meta['crval2'], map_240.meta['crval2']])
@@ -78,3 +84,60 @@ def cor_polariz(map_sequence):
         smap.append(map_)
 
     return sunpy.map.Map(smap, sequence=True)
+
+
+def euvi_prep(map_sequence):
+    smap = []
+    for map_ in map_sequence:
+
+        image = map_.data
+        new_map_meta = copy.deepcopy(map_.meta)
+
+        # Bias Subtraction
+        if map_.meta['offsetcr'] == 0:
+            bias = map_.meta['BIASMEAN']
+            if map_.meta['ipsum'] > 1:
+                bias *= ((2.0**(map_.meta['ipsum'] - 1))**2.0)
+            if map_.meta['IP_PROG3'] == 95:  # Added manually
+                bias *= 1.995  # Added manually
+            image = image - bias
+
+        # Normalize to Open filter position
+        if map_.meta['FILTER'] == 'OPEN':
+            filter_factor = 1  # no filter
+        elif map_.meta['FILTER'] == 'S1':
+            filter_factor = 0.49  # 1500A filter
+        elif map_.meta['FILTER'] == 'S2':
+            filter_factor = 0.49  # 1500A filter
+        elif map_.meta['FILTER'] == 'DBL':
+            filter_factor = 0.41  # 3000A filter
+        image = image / filter_factor
+
+        # Image Dejitter
+        offset = [0., 0.]
+        nsum = 2**(map_.meta['summed']-1)
+
+        scale = map_.dimensions[0].value/2048
+
+        offset[0] = scale * round(map_.meta['fpsoffz'] / 38.) / nsum
+        offset[1] = scale * round(map_.meta['fpsoffy'] / 38.) / nsum
+
+        if map_.meta['obsrvtry'] == 'STEREO_B':
+            offset = [-offset[0], -offset[1]]
+
+        # TODO: if hdr.date_obs lt '2015-05-19' then offset = -offset
+
+        new_map_meta['crpix1'] += offset[0]
+        new_map_meta['crpix2'] += offset[1]
+        new_map_meta['crpix1a'] += offset[0]
+        new_map_meta['crpix2a'] += offset[1]
+        new_map_meta['xcen'] -= map_.meta['cdelt1'] * offset[0]
+        new_map_meta['ycen'] -= map_.meta['cdelt2'] * offset[1]
+
+        image = scipy.ndimage.interpolation.affine_transform(
+            np.nan_to_num(image).T, [[1, 0], [0, 1]], offset=[-offset[0], -offset[1]], order=3,
+            mode='constant', cval=0.0).T
+
+        smap.append(sunpy.map.Map(image, new_map_meta))
+
+    return sunpy.map.Map(smap, sequence=True)

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea/sunpy_dev/map/maputils.py

@@ -8,7 +8,9 @@ import warnings
 import astropy.units as u
 import numpy as np
 import sunpy.map
+from aiapy.calibrate import fix_observer_location, update_pointing
 
+import PyThea.sunpy_dev.extern.sunkit_instruments.lasco.utils  # noqa
 import PyThea.sunpy_dev.extern.sunkit_instruments.stereo.utils  # noqa
 
 __all__ = ['maps_sequence_processing', 'get_closest', 'normalize_exposure',
@@ -76,7 +78,7 @@ def get_closest(smap, date):
     `~sunpy.map.GenericMap`
         A SunPy map.
     """
-    delta_time = [abs(map_.date-date) for map_ in smap]
+    delta_time = [abs(map_.date_average-date) for map_ in smap]
     map_closest = smap[delta_time.index(min(delta_time))]
     return map_closest
 
@@ -108,7 +110,7 @@ def normalize_exposure(smap):
     return smap_new
 
 
-def filter_maps(map_sequence, extra):
+def filter_maps(map_sequence, **kwargs):
     '''
     Returns filtered maps.
 
@@ -121,7 +123,7 @@ def filter_maps(map_sequence, extra):
     map_sequence : `~sunpy.map.GenericMap`
         A SunPy map.
 
-    extra : A list with with the preparation arguments
+    kwargs : A list with with the preparation arguments
 
     Returns
     -------
@@ -130,14 +132,14 @@ def filter_maps(map_sequence, extra):
 
     '''
 
-    if 'exposure' in extra:
-        map_sequence = [tmap for tmap in map_sequence if tmap.exposure_time > extra['exposure']*u.second]
+    if 'exposure' in kwargs.keys():
+        map_sequence = [tmap for tmap in map_sequence if tmap.exposure_time > kwargs['exposure']*u.second]
 
-    if 'dimensions' in extra:
-        map_sequence = [tmap for tmap in map_sequence if (tmap.dimensions[0], tmap.dimensions[1]) == extra['dimensions']]
+    if 'dimensions' in kwargs.keys():
+        map_sequence = [tmap for tmap in map_sequence if (tmap.dimensions[0], tmap.dimensions[1]) == kwargs['dimensions']]
 
-    if 'polar' in extra:
-        map_sequence = [tmap for tmap in map_sequence if tmap.meta['polar'] == extra['polar']]
+    if 'polar' in kwargs.keys():
+        map_sequence = [tmap for tmap in map_sequence if tmap.meta['polar'] == kwargs['polar']]
 
     if len(map_sequence) != 0:
         sequence_final = sunpy.map.Map(map_sequence, sequence=True)
@@ -146,7 +148,7 @@ def filter_maps(map_sequence, extra):
     return sequence_final
 
 
-def prepare_maps(map_sequence, extra):
+def prepare_maps(map_sequence, **kwargs):
     '''
     Returns prepared maps.
 
@@ -159,7 +161,7 @@ def prepare_maps(map_sequence, extra):
     map_sequence : `~sunpy.map.GenericMap`
         A SunPy map.
 
-    extra : A list with with the preparation arguments
+    kwargs : A list with with the preparation arguments
 
     Returns
     -------
@@ -167,26 +169,36 @@ def prepare_maps(map_sequence, extra):
         A SunPy map.
 
     '''
+    detector = map_sequence[0].detector
+    print(f'Preparing image sequence for {detector}.')
+
     if len(map_sequence) == 0:
         return []
 
+    # Prepare the maps before anything else
+    if detector == 'AIA':
+        map_sequence = [update_pointing(tmap) for tmap in map_sequence]
+        map_sequence = [fix_observer_location(tmap) for tmap in map_sequence]
+    elif detector == 'COR1':
+        if 'polar' not in kwargs.keys():
+            map_sequence = PyThea.sunpy_dev.extern.sunkit_instruments.stereo.utils.cor_polariz(map_sequence)
+    elif detector == 'EUVI':
+        map_sequence = PyThea.sunpy_dev.extern.sunkit_instruments.stereo.utils.euvi_prep(map_sequence)
+    elif detector in ['C2', 'C3']:
+        map_sequence = PyThea.sunpy_dev.extern.sunkit_instruments.lasco.utils.prep_lasco(map_sequence)
+
     map_sequence = [normalize_exposure(tmap) for tmap in map_sequence]
 
     map_sequence = [mask_occulter(tmap) for tmap in map_sequence]
 
-    if 'superpixel' in extra:
-        nsuper = extra['superpixel']
+    if 'superpixel' in kwargs.keys():
+        nsuper = kwargs['superpixel']
         super_dim = u.Quantity([nsuper, nsuper] * u.pixel)
         map_sequence = [tmap.superpixel(super_dim) for tmap in map_sequence]
 
     # map_sequence = [tmap.rotate(recenter=True) for tmap in map_sequence]
 
-    if map_sequence[0].detector == 'COR1':
-        if 'polar' not in extra:
-            sequence_final = PyThea.sunpy_dev.extern.sunkit_instruments.stereo.utils.cor_polariz(map_sequence)
-    else:
-        sequence_final = map_sequence
-    return sequence_final
+    return map_sequence
 
 
 def difference_maps(smapi, smapm):

PyThea-0.9.0/PyThea/test/test_remote_clients.py

@@ -0,0 +1,88 @@
+"""
+Test the remote clients
+"""
+
+import numpy as np
+import pytest
+from astropy.time import Time
+from sunpy.net import Fido
+from sunpy.net import attrs as a
+from sunpy.net import hek
+
+from PyThea.config.selected_imagers import imager_dict
+
+
+def time_query_hek():
+    for query in [a.Time('2011/08/09 07:23:56', '2011/08/09 12:40:29'),
+                  a.Time('2012/09/10 08:24:57', '2012/09/10 13:41:30'),
+                  a.Time('2013/10/11 09:25:58', '2013/10/11 14:42:31'), ]:
+        yield query
+
+
+@pytest.mark.remote_data
+@pytest.mark.parametrize('time_query', time_query_hek())
+def test_hek_client(time_query):
+    """
+    Tests that the HEK client returns a HEKTable and the hek_query results for EventType('FL') have consistent format
+    """
+    h = hek.HEKClient()
+    hek_query = h.search(time_query, a.hek.EventType('FL'))
+    assert type(hek_query) == hek.hek.HEKTable
+
+    assert isinstance(hek_query['event_starttime'], Time)
+    assert isinstance(hek_query['event_peaktime'], Time)
+    assert isinstance(hek_query['event_endtime'], Time)
+    assert all(isinstance(h, np.str_) for h in hek_query['fl_goescls'])
+
+
+@pytest.mark.remote_data
+def test_hek_client_flare_data():
+    """
+    Tests that the HEK client returns always the same results (flare class, ...)
+    """
+
+    startTime = '2011/08/09 07:23:56'
+    endTime = '2011/08/09 12:40:29'
+    eventType = 'FL'
+
+    hekTime = a.Time(startTime, endTime)
+    hekEvent = a.hek.EventType(eventType)
+
+    h = hek.HEKClient()
+
+    hek_query = h.search(hekTime, hekEvent, a.hek.FL.GOESCls > 'B1.0', a.hek.OBS.Observatory == 'GOES')
+    assert all(hek_query['fl_goescls'] == ['C1.4', 'X6.9'])
+    assert all(hek_query['event_starttime'] == Time(['2011-08-09 07:19:00.000', '2011-08-09 07:48:00.000']))
+    assert all(hek_query['event_peaktime'] == Time(['2011-08-09 07:23:00.000', '2011-08-09 08:05:00.000']))
+    assert all(hek_query['event_endtime'] == Time(['2011-08-09 07:27:00.000', '2011-08-09 08:08:00.000']))
+
+
+def imager_query():
+    for imager in imager_dict.keys():
+        yield imager
+
+
+@pytest.mark.remote_data
+@pytest.mark.parametrize('imager', imager_query())
+def test_vso_search(imager):
+    """
+    Tests that the Fido.search returns always the same number of files from VSO.
+    """
+    file_num = {'AIA': [60, 61],
+                'LC2': [5], 'LC3': [5],
+                'COR2A': [7], 'COR2B': [7],
+                'EUVIA': [11], 'EUVIB': [11],
+                'COR1A': [18, 34], 'COR1B': [17, 66],
+                'HI1A': [1, 2, 3], 'HI1B': [1, 2],
+                'HI2A': [1, 2], 'HI2B': [1], }
+
+    for query in [a.Time('2011/01/01 00:00:00', '2011/01/01 01:00:00'),
+                  a.Time('2012/01/01 00:00:00', '2012/01/01 01:00:00'),
+                  a.Time('2013/01/01 00:00:00', '2013/01/01 01:00:00'), ]:
+        args = imager_dict[imager][0]
+        result = Fido.search(query, *args)
+
+        assert len(result) == 1
+        assert 'vso' in result.keys()
+
+        assert result.file_num in file_num[imager]

PyThea-0.9.0/PyThea/test/test_utils.py

@@ -0,0 +1,78 @@
+"""
+Test the utilities
+"""
+
+from datetime import datetime
+
+import matplotlib.dates as mdates
+from astropy.coordinates import SkyCoord
+from astropy.tests.helper import assert_quantity_allclose
+from sunpy.coordinates import get_horizons_coord
+from sunpy.coordinates.frames import HeliographicStonyhurst
+from sunpy.net import hek
+
+from PyThea.config.selected_bodies import bodies_dict
+from PyThea.utils import get_hek_flare, parameter_fit
+
+
+def test_get_hek_flare():
+    """
+    Test that the get_hek_flare returns a hek.hek.HEKTable and the results are formated correct.
+    """
+    # This test requests the flare info for a date that returns a list of flares and compares against previous results
+    selectbox_list, flare_list = get_hek_flare(datetime(2017, 9, 10))
+    assert type(flare_list) == hek.hek.HEKTable
+    assert selectbox_list == ['FLM1.1|2017-09-09T23:53:00', 'FLC9.0|2017-09-10T03:09:00', 'FLC2.9|2017-09-10T09:20:00',
+                              'FLC1.6|2017-09-10T14:23:00', 'FLC1.0|2017-09-10T15:26:00', 'FLX8.2|2017-09-10T16:06:00']
+
+    # This test requests the flare info for a date that returns an empty list
+    selectbox_list, flare_list = get_hek_flare(datetime(2117, 9, 10))
+    assert flare_list == []
+    assert selectbox_list == ['No events returned']
+
+
+def test_get_horizons_coord():
+    """
+    Test that get_horizons_coord returns a Skycoord instance, in HeliographicStonyhurst frame, and returns the same coordinates every time.
+    """
+    bodies_coord = []
+    for body in bodies_dict:
+        bodies_coord.append(get_horizons_coord(bodies_dict[body][0], '2022-01-01T00:00:00'))
+
+    coords = [[-106.56704928, 1.45508574, 0.3631443],
+              [-4.65750614, -1.34575135, 0.71937524],
+              [2.55987423e-06, -2.99926774, 0.98335562],
+              [135.77581107, -2.68402546, 1.53663218],
+              [-121.16524996, 6.08953762, 4.99274907],
+              [32.43818351, -5.96794509, 1.0004962],
+              [-34.99771814, 1.28725362, 0.96437702],
+              [-11.53530598, -0.39313636, 0.99738376],
+              [-138.81147107, 3.52846474, 0.74828612],
+              [144.12380481, -2.64432759, 0.67510736]
+              ]
+
+    for i, body_coord in enumerate(bodies_coord):
+        assert isinstance(body_coord, SkyCoord)
+        assert isinstance(body_coord.frame, HeliographicStonyhurst)
+        assert_quantity_allclose([body_coord.lon.value, body_coord.lat.value, body_coord.radius.value], coords[i], rtol=1e-5, atol=1e-4)
+
+
+def test_parameter_fit_polynomial():
+    x = [datetime(2020, 1, 1, 0, 0, 0), datetime(2020, 1, 1, 0, 10, 0), datetime(2020, 1, 1, 0, 20, 0), datetime(2020, 1, 1, 0, 30, 0)]
+    xx = (mdates.date2num(x) - mdates.date2num(x[0]))
+
+    y = [0, 1, 2, 3]
+
+    fit = parameter_fit(x, y, {'type': 'polynomial', 'order': 1})
+    assert_quantity_allclose([fit['popt'][0]*xx[1], fit['popt'][1]], [1, 0], atol=1e-7)
+
+    fit = parameter_fit(x, y, {'type': 'polynomial', 'order': 2})
+    assert_quantity_allclose([fit['popt'][0], fit['popt'][1]*xx[1], fit['popt'][2]], [0, 1, 0], atol=1e-5)
+
+    y = [0, 2, 4, 6]
+
+    fit = parameter_fit(x, y, {'type': 'polynomial', 'order': 1})
+    assert_quantity_allclose([fit['popt'][0]*xx[1], fit['popt'][1]], [2, 0], atol=1e-7)
+
+    fit = parameter_fit(x, y, {'type': 'polynomial', 'order': 2})
+    assert_quantity_allclose([fit['popt'][0], fit['popt'][1]*xx[1], fit['popt'][2]], [0, 2, 0], atol=1e-5)

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea/utils.py

@@ -22,6 +22,7 @@ import datetime
 import io
 import json
 import os
+import re
 from copy import copy
 from pathlib import Path
 
@@ -31,6 +32,7 @@ import matplotlib.dates as mdates
 import matplotlib.pyplot as plt
 import numexpr
 import numpy as np
+import pandas as pd
 import seaborn as sns
 import sunpy.map
 from scipy.interpolate import UnivariateSpline
@@ -39,6 +41,7 @@ from sunpy.coordinates import get_horizons_coord
 from sunpy.map.maputils import contains_coordinate
 from sunpy.net import Fido
 from sunpy.net import attrs as a
+from sunpy.time import parse_time
 
 from PyThea.config.selected_bodies import bodies_dict
 from PyThea.config.selected_imagers import imager_dict
@@ -74,22 +77,26 @@ def get_hek_flare(day):
     return selectbox_list, flare_list_
 
 
-def make_figure(map, image_mode, clim=[-20, 20], clip_model=True):
+def make_figure(map, image_mode, clim=[-20, 20], clip_model=True, **kwargs):
     '''
     Makes the main imager figure and returns the figure and axis handle.
     '''
-    fig = plt.figure()
-    axis = plt.subplot(projection=map)
-    # TODO: For plain images or when EUVIA-B are used, this does not work very well.
+    fig = kwargs.get('fig', plt.figure())
+    axis = kwargs.get('axis', plt.subplot(projection=map))
+
     if image_mode == 'Plain':
+        # TODO: For plain images or when EUVIA-B are used, this does not work very well.
         map.plot()
     else:
         map.plot(cmap='Greys_r',
                  norm=colors.Normalize(vmin=clim[0], vmax=clim[1]))
-    map.draw_limb(resolution=180)
+
+    map.draw_limb(resolution=90)
     # map.draw_grid(linewidth=2, color='red') # TODO: This takes too much computation time. Maybe for AIA or EUVI?
+
     yax = axis.coords[1]
     yax.set_ticklabel(rotation=90)
+
     if clip_model:
         axis.set_xlim([0, map.data.shape[0]])
         axis.set_ylim([0, map.data.shape[1]])
@@ -102,6 +109,10 @@ def make_figure(map, image_mode, clim=[-20, 20], clip_model=True):
     if cref.Ty > 0:
         axis.invert_yaxis()
 
+    axis.set_title(re.sub(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}',
+                          ' $T_{AGV}:$' + parse_time(map.date_average).strftime('%Y-%m-%d %H:%M:%S'),
+                          map.latex_name), fontsize=10, pad=8)
+
     return fig, axis
 
 
@@ -110,7 +121,7 @@ def plot_bodies(axis, bodies_list, smap):
     Plots in the images the possition of the pre-configured bodies (Earth, STA, Venus etc.)
     '''
     for body in bodies_list:
-        body_coo = get_horizons_coord(bodies_dict[body][0], smap.date)
+        body_coo = get_horizons_coord(bodies_dict[body][0], smap.date_average)
         if contains_coordinate(smap, body_coo):
             axis.plot_coord(body_coo, 'o', color=bodies_dict[body][1],
                             fillstyle='none', markersize=6, label=body)
@@ -143,28 +154,44 @@ def download_fits(date_process, imager, time_range=[-1, 1]):
     return map_
 
 
-def maps_process(ninstr_map_in, imagers_list_in, image_mode):
+def maps_process(maps_dict_in, imagers_list_in, image_mode, **kwargs):
     '''
-    Process the images for the selected imagers and return the final maps.
+    Process the images for the selected imagers and return the final maps and the list of imagers loaded.
 
     Note
     ----
-    Here the ninstr_map_in is the session_state.map_ when used from the application.
+    Here the maps_dict_in is the session_state.map_ when used from the application.
     '''
-    ninstr_map_out = {}
+    maps_dict_out = {}
     imagers_list_out = []
 
     for imager in imagers_list_in:
-        extra = imager_dict[imager][1]
-        if imager in ninstr_map_in and ninstr_map_in[imager] != []:
-            ninstr_map_out[imager] = filter_maps(ninstr_map_in[imager], extra)
-            ninstr_map_out[imager] = prepare_maps(ninstr_map_out[imager], extra)
-            ninstr_map_out[imager] = maps_sequence_processing(ninstr_map_out[imager],
-                                                              seq_type=image_mode)
-            if ninstr_map_out[imager] != []:
+        if imager in maps_dict_in and maps_dict_in[imager] != []:
+            if not kwargs:
+                extras = imager_dict[imager][1]
+            else:
+                if imager in kwargs:
+                    extras = kwargs[imager]
+                else:
+                    print(f'Warning [maps_process]: No extras provided for {imager}.')
+            maps_dict_out[imager] = single_imager_maps_process(maps_dict_in[imager],
+                                                               image_mode=image_mode,
+                                                               **extras)
+            if maps_dict_out[imager] != []:
                 imagers_list_out.append(imager)
 
-    return ninstr_map_out, imagers_list_out
+    return maps_dict_out, imagers_list_out
+
+
+def single_imager_maps_process(map_list_in, image_mode='Plain', **kwargs):
+    '''
+    Process the images for a single imager and return the final maps.
+    '''
+    map_list_out = filter_maps(map_list_in, **kwargs)
+    map_list_out = prepare_maps(map_list_out, **kwargs)
+    map_list_out = maps_sequence_processing(map_list_out, seq_type=image_mode)
+
+    return map_list_out
 
 
 # TODO: Implement units here
@@ -232,29 +259,28 @@ def plot_fitting_model(model, fit_args, plt_type='HeightT'):
     }
     parameters = parameters[model.geometrical_model]
     # Height vs time
-    fig = plt.figure(figsize=(5.5, 5.5), tight_layout=True)
-    axis = plt.subplot()
+    fig, axis = plt.subplots(figsize=(5.5, 5.5), tight_layout=True)
     if plt_type == 'HeightT':
         for p in parameters.keys():
-            plt.plot(model.parameters.index,
-                     model.parameters[p],
-                     marker=parameters[p][0],
-                     linestyle=parameters[p][1],
-                     color=parameters[p][2],
-                     label=parameters[p][3])  # label='fit: a=%5.3f, b=%5.3f, c=%5.3f' % tuple(popt)
+            axis.plot(model.parameters.index,
+                      model.parameters[p],
+                      marker=parameters[p][0],
+                      linestyle=parameters[p][1],
+                      color=parameters[p][2],
+                      label=parameters[p][3])  # label='fit: a=%5.3f, b=%5.3f, c=%5.3f' % tuple(popt)
             if len(model.parameters[p])-1 > fit_args['order']:
                 # How to get confidence intervals from curve_fit?
                 fit = parameter_fit(model.parameters.index, model.parameters[p], fit_args)
-                plt.plot(fit['best_fit_x'], fit['best_fit_y'], '-', color=parameters[p][2])
-                plt.fill_between(fit['best_fit_x'], fit['sigma_bounds']['up'], fit['sigma_bounds']['low'],
-                                 color=parameters[p][2], alpha=0.20)
+                axis.plot(fit['best_fit_x'], fit['best_fit_y'], '-', color=parameters[p][2])
+                axis.fill_between(fit['best_fit_x'], fit['sigma_bounds']['up'], fit['sigma_bounds']['low'],
+                                  color=parameters[p][2], alpha=0.20)
                 if fit_args['type'] == 'spline':
-                    plt.fill_between(fit['best_fit_x'], fit['sigv_bounds']['up'], fit['sigv_bounds']['low'],
-                                     color=parameters[p][2], alpha=0.05)
+                    axis.fill_between(fit['best_fit_x'], fit['sigv_bounds']['up'], fit['sigv_bounds']['low'],
+                                      color=parameters[p][2], alpha=0.05)
             else:
-                plt.plot(model.parameters.index, model.parameters[p], '--', color=parameters[p][2])
-        ylabel = 'Height or Length [Rsun]'
-        plt.gca().set_ylim(bottom=0)
+                axis.plot(model.parameters.index, model.parameters[p], '--', color=parameters[p][2])
+        ylabel = 'Height of apex and length of flanks [Rsun]'
+        axis.set_ylim(bottom=0)
     elif plt_type == 'SpeedT':
         for p in parameters.keys():
             if len(model.parameters[p])-1 > fit_args['order']:
@@ -265,34 +291,34 @@ def plot_fitting_model(model, fit_args, plt_type='HeightT'):
                 speed_best_fit = (Rs2km/sec) * np.gradient(fit['best_fit_y'], fit['best_fit_x_num'])
                 speed_bound_upper = (Rs2km/sec) * np.gradient(fit['sigma_bounds']['up'], fit['best_fit_x_num'])
                 speed_bound_lower = (Rs2km/sec) * np.gradient(fit['sigma_bounds']['low'], fit['best_fit_x_num'])
-                plt.plot(fit['best_fit_x'], speed_best_fit, '-', color=parameters[p][2], label=parameters[p][3])
-                plt.fill_between(fit['best_fit_x'], speed_bound_lower, speed_bound_upper,
-                                 color=parameters[p][2], alpha=0.20)
+                axis.plot(fit['best_fit_x'], speed_best_fit, '-', color=parameters[p][2], label=parameters[p][3])
+                axis.fill_between(fit['best_fit_x'], speed_bound_lower, speed_bound_upper,
+                                  color=parameters[p][2], alpha=0.20)
                 if fit_args['type'] == 'spline':
-                    plt.fill_between(fit['best_fit_x'], (Rs2km/sec) * fit['sigv_bounds']['dlow'], (Rs2km/sec) * fit['sigv_bounds']['dup'],
-                                     color=parameters[p][2], alpha=0.05)
+                    axis.fill_between(fit['best_fit_x'], (Rs2km/sec) * fit['sigv_bounds']['dlow'], (Rs2km/sec) * fit['sigv_bounds']['dup'],
+                                      color=parameters[p][2], alpha=0.05)
             else:
                 pass
         ylabel = 'Speed [km/s]'
-        plt.gca().set_ylim(bottom=0)
+        axis.set_ylim(bottom=0)
     if plt_type == 'LongT' or plt_type == 'LatT':
         parameters = {'LongT': ['hgln', '+', palete[3], 'Longitude'],
                       'LatT': ['hglt', '+', palete[2], 'Latitude']}
-        plt.plot(model.parameters.index,
-                 model.parameters[parameters[plt_type][0]],
-                 marker=parameters[plt_type][1],
-                 linestyle='',
-                 color=parameters[plt_type][2],
-                 label=parameters[plt_type][3])
+        axis.plot(model.parameters.index,
+                  model.parameters[parameters[plt_type][0]],
+                  marker=parameters[plt_type][1],
+                  linestyle='',
+                  color=parameters[plt_type][2],
+                  label=parameters[plt_type][3])
         if len(model.parameters[parameters[plt_type][0]])-1 > 3:
             fit = parameter_fit(model.parameters.index, model.parameters[parameters[plt_type][0]], fit_args)
-            plt.plot(fit['best_fit_x'], fit['best_fit_y'], '-', color=parameters[plt_type][2])
-            plt.fill_between(fit['best_fit_x'], fit['sigma_bounds']['up'], fit['sigma_bounds']['low'],
-                             color=parameters[plt_type][2], alpha=0.05)
+            axis.plot(fit['best_fit_x'], fit['best_fit_y'], '-', color=parameters[plt_type][2])
+            axis.fill_between(fit['best_fit_x'], fit['sigma_bounds']['up'], fit['sigma_bounds']['low'],
+                              color=parameters[plt_type][2], alpha=0.05)
         ylabel = parameters[plt_type][3] + ' [degrees]'
 
-    plt.xlabel('Time [UT]')
-    plt.ylabel(ylabel)
+    axis.set_xlabel('Time [UTC]')
+    axis.set_ylabel(ylabel)
     if fit_args['type'] == 'polynomial':
         title = 'Event: ' + model.event_selected + ' | ' + fit_args['type'] + str(fit_args['order'])
     elif fit_args['type'] == 'spline':
@@ -310,18 +336,20 @@ def plot_fitting_model(model, fit_args, plt_type='HeightT'):
     if xlim[1] - xlim[0] <= 0.5:
         axis.xaxis.set_minor_locator(mdates.MinuteLocator(byminute=[0, 10, 20, 30, 40, 50]))
     fig.autofmt_xdate(bottom=0, rotation=0, ha='center')
-    plt.legend(loc='lower right')
+    axis.legend(loc='lower right')
 
-    return fig
+    return fig, axis
 
 
 def parameter_fit(x, y, fit_args):
     '''
     Performs a polynomial or spline fit to a set of x, y parameters.
     '''
-    xx = (mdates.date2num(x) - mdates.date2num(x[0]))
-    xxx = np.linspace(xx.min(), xx.max(), 120)
-    dd = mdates.num2date(xxx + mdates.date2num(x[0]))
+    minx = x[0].replace(second=0, microsecond=0)
+    xx = (mdates.date2num(x) - mdates.date2num(minx))  # fractional days from minx
+    step = 1/(60*24)  # one minute timestep in units of xx (fractional days)
+    xxx = np.arange(0, xx.max()+step, step)  # Makes an array with a timestep of one minute in fractional days
+    dd = pd.DatetimeIndex(mdates.num2date(xxx + mdates.date2num(minx)))
 
     if fit_args['type'] == 'polynomial':
         # scipy.optimize.curve_fit and numpy.polyfit

{PyThea-0.8.1 → PyThea-0.9.0/PyThea.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PyThea
-Version: 0.8.1
+Version: 0.9.0
 Summary: PyThea: A software package to reconstruct the 3D structure of CMEs and shock waves
 Home-page: https://github.com/AthKouloumvakos/PyThea
 Author: Athanasios Kouloumvakos
@@ -23,11 +23,13 @@ License-File: LICENSE.md
 Requires-Dist: numpy
 Requires-Dist: pandas
 Requires-Dist: scipy
+Requires-Dist: aiapy
 Requires-Dist: astropy
 Requires-Dist: astroquery
 Requires-Dist: numexpr
 Requires-Dist: sunpy==4.1.0
 Requires-Dist: parfive==1.5.1
+Requires-Dist: pooch
 Requires-Dist: matplotlib
 Requires-Dist: seaborn
 Requires-Dist: streamlit

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea.egg-info/SOURCES.txt

@@ -30,12 +30,16 @@ PyThea/config/app_styles.py
 PyThea/config/config_sliders.py
 PyThea/config/selected_bodies.py
 PyThea/config/selected_imagers.py
+PyThea/data/__init__.py
+PyThea/data/sample_data.py
 PyThea/extensions/LICENSE_gcs_python.md
 PyThea/extensions/__init__.py
 PyThea/extensions/buttons.py
 PyThea/sunpy_dev/__init__.py
 PyThea/sunpy_dev/extern/__init__.py
 PyThea/sunpy_dev/extern/sunkit_instruments/__init__.py
+PyThea/sunpy_dev/extern/sunkit_instruments/lasco/__init__.py
+PyThea/sunpy_dev/extern/sunkit_instruments/lasco/utils.py
 PyThea/sunpy_dev/extern/sunkit_instruments/stereo/__init__.py
 PyThea/sunpy_dev/extern/sunkit_instruments/stereo/utils.py
 PyThea/sunpy_dev/map/__init__.py

{PyThea-0.8.1 → PyThea-0.9.0}/PyThea.egg-info/requires.txt

@@ -1,11 +1,13 @@
 numpy
 pandas
 scipy
+aiapy
 astropy
 astroquery
 numexpr
 sunpy==4.1.0
 parfive==1.5.1
+pooch
 matplotlib
 seaborn
 streamlit

{PyThea-0.8.1 → PyThea-0.9.0}/environment.yml

@@ -10,6 +10,7 @@ dependencies:
   - pandas
   - scipy
   - matplotlib
+  - aiapy
   - astropy
   - astroquery
   - numexpr
@@ -23,4 +24,5 @@ dependencies:
   - pytest-sugar
   - pip
   - pip:
+      - pooch
       - stqdm

{PyThea-0.8.1 → PyThea-0.9.0}/requirements.txt

@@ -1,11 +1,13 @@
 numpy
 pandas
 scipy
+aiapy
 astropy
 astroquery
 numexpr
 sunpy==4.1.0
 parfive==1.5.1
+pooch
 matplotlib
 seaborn
 streamlit

PyThea-0.8.1/PyThea/test/test_remote_clients.py

@@ -1,25 +0,0 @@
-"""
-Test the remote clients
-"""
-
-import pytest
-from sunpy.net import attrs, hek
-
-
-@pytest.mark.remote_data
-def test_hek_client():
-    """
-    Tests that the HEK client returns a HEKTable
-    """
-    startTime = '2011/08/09 07:23:56'
-    endTime = '2011/08/09 12:40:29'
-    eventType = 'FL'
-
-    hekTime = attrs.Time(startTime, endTime)
-    hekEvent = attrs.hek.EventType(eventType)
-
-    h = hek.HEKClient()
-    hek_query = h.search(hekTime, hekEvent)
-    assert type(hek_query) == hek.hek.HEKTable
-
-    # Test that the hek_query['event_peaktime'] is <class 'astropy.time.core.Time'>

PyThea-0.8.1/PyThea/test/test_utils.py

@@ -1,20 +0,0 @@
-"""
-Test the utilities
-"""
-
-from sunpy.net import hek
-
-from PyThea.utils import get_hek_flare
-
-
-# ------------------
-# Test utilities
-def test_get_hek_flare_HEKTable():
-    """
-    Test that the get_hek_flare returns a hek.hek.HEKTable
-    """
-    from datetime import datetime
-    day = datetime(2017, 9, 10)
-    _, flare_list_ = get_hek_flare(day)
-
-    assert type(flare_list_) == hek.hek.HEKTable