ocf-data-sampler 0.1.10__py3-none-any.whl → 0.1.16__py3-none-any.whl

tests/select/test_dropout.py

@@ -1,75 +0,0 @@
-from ocf_data_sampler.select.dropout import draw_dropout_time, apply_dropout_time
-
-import numpy as np
-import pandas as pd
-import xarray as xr
-
-import pytest
-
-
-@pytest.fixture(scope="module")
-def da_sample():
-    """Create dummy data which looks like satellite data"""
-
-    datetimes = pd.date_range("2024-01-01 12:00", "2024-01-01 13:00", freq="5min")
-
-    da_sat = xr.DataArray(
-        np.random.normal(size=(len(datetimes),)),
-        coords=dict(
-            time_utc=(["time_utc"], datetimes),
-        )
-    )
-    return da_sat
-
-
-def test_draw_dropout_time():
-    t0 = pd.Timestamp("2021-01-01 04:00:00")
-
-    dropout_timedeltas = pd.to_timedelta([-30, -60], unit="min")
-    dropout_time = draw_dropout_time(t0, dropout_timedeltas, dropout_frac=1)
-
-    assert isinstance(dropout_time, pd.Timestamp)
-    assert dropout_time-t0 in dropout_timedeltas
-
-
-def test_draw_dropout_time_partial():
-    t0 = pd.Timestamp("2021-01-01 04:00:00")
-
-    dropout_timedeltas = pd.to_timedelta([-30, -60], unit="min")
-
-    dropouts = set()
-
-    # Loop over 1000 to have very high probability of seeing all dropouts
-    # The chances of this failing by chance are approx ((2/3)^100)*3 = 7e-18
-    for _ in range(100):
-        dropouts.add(draw_dropout_time(t0, dropout_timedeltas, dropout_frac=2/3))
-
-    # Check all expected dropouts are present
-    dropouts == {None} | set(t0 + dt for dt in dropout_timedeltas)
-
-
-def test_draw_dropout_time_none():
-    t0 = pd.Timestamp("2021-01-01 04:00:00")
-
-    # No dropout timedeltas
-    dropout_time = draw_dropout_time(t0, dropout_timedeltas=None, dropout_frac=1)
-    assert dropout_time is None
-
-    # Dropout fraction is 0
-    dropout_timedeltas = [pd.Timedelta(-30, "min")]
-    dropout_time = draw_dropout_time(t0, dropout_timedeltas=dropout_timedeltas, dropout_frac=0)
-    assert dropout_time is None
-
-    # No dropout timedeltas and dropout fraction is 0
-    dropout_time = draw_dropout_time(t0, dropout_timedeltas=None, dropout_frac=0)
-    assert dropout_time is None
-
-
-@pytest.mark.parametrize("t0_str", ["12:00", "12:30", "13:00"])
-def test_apply_dropout_time(da_sample, t0_str):
-    dropout_time = pd.Timestamp(f"2024-01-01 {t0_str}")
-
-    da_dropout = apply_dropout_time(da_sample, dropout_time)
-
-    assert da_dropout.sel(time_utc=slice(None, dropout_time)).notnull().all()
-    assert da_dropout.sel(time_utc=slice(dropout_time+pd.Timedelta(5, "min"), None)).isnull().all()

tests/select/test_fill_time_periods.py

@@ -1,28 +0,0 @@
-import pandas as pd
-
-from ocf_data_sampler.select.fill_time_periods import fill_time_periods
-
-def test_fill_time_periods():
-    time_periods = pd.DataFrame(
-        {
-            "start_dt": [
-                "2021-01-01 04:10:00", "2021-01-01 09:00:00",
-                "2021-01-01 09:15:00", "2021-01-01 12:00:00"
-            ],
-            "end_dt": [
-                "2021-01-01 06:00:00", "2021-01-01 09:00:00", 
-                "2021-01-01 09:20:00", "2021-01-01 14:45:00"
-            ],
-        }
-    )
-    freq = pd.Timedelta("30min")
-    filled_time_periods = fill_time_periods(time_periods, freq)
-
-    expected_times = [
-        "04:30", "05:00", "05:30", "06:00", "09:00", "12:00", 
-        "12:30", "13:00", "13:30", "14:00", "14:30"
-    ]
-
-    expected_times = pd.DatetimeIndex([f"2021-01-01 {t}" for t in expected_times])
-
-    pd.testing.assert_index_equal(filled_time_periods, expected_times)

tests/select/test_find_contiguous_time_periods.py

@@ -1,202 +0,0 @@
-import pandas as pd
-
-from ocf_data_sampler.select.find_contiguous_time_periods import (
-    find_contiguous_t0_periods, find_contiguous_t0_periods_nwp, 
-    intersection_of_multiple_dataframes_of_periods,
-)
-
-
-
-def test_find_contiguous_t0_periods():
-
-    # Create 5-minutely data timestamps
-    freq = pd.Timedelta(5, "min")
-    interval_start = pd.Timedelta(-60, "min")
-    interval_end = pd.Timedelta(15, "min")
-
-    datetimes = (
-        pd.date_range("2023-01-01 12:00", "2023-01-01 17:00", freq=freq)
-        .delete([5, 6, 30])
-    )
-
-    periods = find_contiguous_t0_periods(
-        datetimes=datetimes,
-        interval_start=interval_start,
-        interval_end=interval_end,
-        sample_period_duration=freq,
-    )
-
-    expected_results = pd.DataFrame(
-        {
-            "start_dt": pd.to_datetime(
-                [
-                    "2023-01-01 13:35",
-                    "2023-01-01 15:35",
-                ]
-            ),
-            "end_dt": pd.to_datetime(
-                [
-                    "2023-01-01 14:10",
-                    "2023-01-01 16:45",
-                ]
-            ),
-        },
-    )
-
-    assert periods.equals(expected_results)
-
-
-def test_find_contiguous_t0_periods_nwp():
-
-    # These are the expected results of the test
-    expected_results = [
-        pd.DataFrame(
-            {
-                "start_dt": pd.to_datetime(["2023-01-01 03:00", "2023-01-02 03:00"]),
-                "end_dt": pd.to_datetime(["2023-01-01 21:00", "2023-01-03 06:00"]),
-            },
-        ),
-        pd.DataFrame(
-            {
-                "start_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 05:00",
-                        "2023-01-02 05:00",
-                    ]
-                ),
-                "end_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 21:00",
-                        "2023-01-03 06:00",
-                    ]
-                ),
-            },
-        ),
-        pd.DataFrame(
-            {
-                "start_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 05:00",
-                        "2023-01-02 05:00",
-                        "2023-01-02 14:00",
-                    ]
-                ),
-                "end_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 18:00",
-                        "2023-01-02 09:00",
-                        "2023-01-03 03:00",
-                    ]
-                ),
-            },
-        ),
-        pd.DataFrame(
-            {
-                "start_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 05:00",
-                        "2023-01-01 11:00",
-                        "2023-01-02 05:00",
-                        "2023-01-02 14:00",
-                    ]
-                ),
-                "end_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 06:00",
-                        "2023-01-01 15:00",
-                        "2023-01-02 06:00",
-                        "2023-01-03 00:00",
-                    ]
-                ),
-            },
-        ),
-        pd.DataFrame(
-            {
-                "start_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 06:00",
-                        "2023-01-01 12:00",
-                        "2023-01-02 06:00",
-                        "2023-01-02 15:00",
-                    ]
-                ),
-                "end_dt": pd.to_datetime(
-                    [
-                        "2023-01-01 09:00",
-                        "2023-01-01 18:00",
-                        "2023-01-02 09:00",
-                        "2023-01-03 03:00",
-                    ]
-                ),
-            },
-        ),
-    ]
-
-    # Create 3-hourly init times with a few time stamps missing
-    freq = pd.Timedelta(3, "h")
-
-    init_times = (
-        pd.date_range("2023-01-01 03:00", "2023-01-02 21:00", freq=freq)
-        .delete([1, 4, 5, 6, 7, 9, 10])
-    )
-
-    # Choose some history durations and max stalenesses
-    history_durations_hr = [0, 2, 2, 2, 2]
-    max_stalenesses_hr = [9, 9, 6, 3, 6]
-    max_dropouts_hr = [0, 0, 0, 0, 3]
-
-    for i in range(len(expected_results)):
-        interval_start = pd.Timedelta(-history_durations_hr[i], "h")
-        max_staleness = pd.Timedelta(max_stalenesses_hr[i], "h")
-        max_dropout = pd.Timedelta(max_dropouts_hr[i], "h")
-
-        time_periods = find_contiguous_t0_periods_nwp(
-            init_times=init_times,
-            interval_start=interval_start,
-            max_staleness=max_staleness,
-            max_dropout=max_dropout,
-        )
-
-        # Check if results are as expected
-        assert time_periods.equals(expected_results[i])
-
-
-def test_intersection_of_multiple_dataframes_of_periods():
-    periods_1 = pd.DataFrame(
-        {
-            "start_dt": pd.to_datetime(["2023-01-01 05:00", "2023-01-01 14:10"]),
-            "end_dt": pd.to_datetime(["2023-01-01 13:35", "2023-01-01 18:00"]),
-        },
-    )
-
-    periods_2 = pd.DataFrame(
-        {
-            "start_dt": pd.to_datetime(["2023-01-01 12:00"]),
-            "end_dt": pd.to_datetime(["2023-01-02 00:00"]),
-        },
-    )
-
-    periods_3 = pd.DataFrame(
-        {
-            "start_dt": pd.to_datetime(["2023-01-01 00:00", "2023-01-01 13:00"]),
-            "end_dt": pd.to_datetime(["2023-01-01 12:30", "2023-01-01 23:00"]),
-        },
-    )
-
-    expected_result = pd.DataFrame(
-        {
-            "start_dt": pd.to_datetime(
-                ["2023-01-01 12:00", "2023-01-01 13:00", "2023-01-01 14:10"]
-            ),
-            "end_dt": pd.to_datetime([
-                "2023-01-01 12:30", "2023-01-01 13:35", "2023-01-01 18:00"]
-            ),
-        },
-    )
-
-    overlaping_periods = intersection_of_multiple_dataframes_of_periods(
-        [periods_1, periods_2, periods_3]
-    )   
-
-    # Check if results are as expected
-    assert overlaping_periods.equals(expected_result)

tests/select/test_location.py

@@ -1,67 +0,0 @@
-from ocf_data_sampler.select.location import Location
-import pytest
-
-
-def test_make_valid_location_object_with_default_coordinate_system():
-    x, y = -1000.5, 50000
-    location = Location(x=x, y=y)
-    assert location.x == x, "location.x value not set correctly"
-    assert location.y == y, "location.x value not set correctly"
-    assert (
-        location.coordinate_system == "osgb"
-    ), "location.coordinate_system value not set correctly"
-
-
-def test_make_valid_location_object_with_osgb_coordinate_system():
-    x, y, coordinate_system = 1.2, 22.9, "osgb"
-    location = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert location.x == x, "location.x value not set correctly"
-    assert location.y == y, "location.x value not set correctly"
-    assert (
-        location.coordinate_system == coordinate_system
-    ), "location.coordinate_system value not set correctly"
-
-
-def test_make_valid_location_object_with_lon_lat_coordinate_system():
-    x, y, coordinate_system = 1.2, 1.2, "lon_lat"
-    location = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert location.x == x, "location.x value not set correctly"
-    assert location.y == y, "location.x value not set correctly"
-    assert (
-        location.coordinate_system == coordinate_system
-    ), "location.coordinate_system value not set correctly"
-
-
-def test_make_invalid_location_object_with_invalid_osgb_x():
-    x, y, coordinate_system = 10000000, 1.2, "osgb"
-    with pytest.raises(ValueError) as err:
-        _ = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert err.typename == "ValidationError"
-
-
-def test_make_invalid_location_object_with_invalid_osgb_y():
-    x, y, coordinate_system = 2.5, 10000000, "osgb"
-    with pytest.raises(ValueError) as err:
-        _ = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert err.typename == "ValidationError"
-
-
-def test_make_invalid_location_object_with_invalid_lon_lat_x():
-    x, y, coordinate_system = 200, 1.2, "lon_lat"
-    with pytest.raises(ValueError) as err:
-        _ = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert err.typename == "ValidationError"
-
-
-def test_make_invalid_location_object_with_invalid_lon_lat_y():
-    x, y, coordinate_system = 2.5, -200, "lon_lat"
-    with pytest.raises(ValueError) as err:
-        _ = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert err.typename == "ValidationError"
-
-
-def test_make_invalid_location_object_with_invalid_coordinate_system():
-    x, y, coordinate_system = 2.5, 1000, "abcd"
-    with pytest.raises(ValueError) as err:
-        _ = Location(x=x, y=y, coordinate_system=coordinate_system)
-    assert err.typename == "ValidationError"

tests/select/test_select_spatial_slice.py

@@ -1,154 +0,0 @@
-import numpy as np
-import xarray as xr
-from ocf_data_sampler.select.location import Location
-import pytest
-
-from ocf_data_sampler.select.select_spatial_slice import (
-    select_spatial_slice_pixels, _get_idx_of_pixel_closest_to_poi
-)
-
-@pytest.fixture(scope="module")
-def da():
-    # Create dummy data
-    x = np.arange(-100, 100)
-    y = np.arange(-100, 100)
-
-    da = xr.DataArray(
-        np.random.normal(size=(len(x), len(y))),
-        coords=dict(
-            x_osgb=(["x_osgb"], x),
-            y_osgb=(["y_osgb"], y),
-        )
-    )
-    return da
-
-
-def test_get_idx_of_pixel_closest_to_poi(da):
-    
-    idx_location  = _get_idx_of_pixel_closest_to_poi(
-        da,
-        location=Location(x=10, y=10, coordinate_system="osgb"),
-    )
-
-    assert idx_location.coordinate_system == "idx"
-    assert idx_location.x == 110
-    assert idx_location.y == 110
-
-
-
-
-def test_select_spatial_slice_pixels(da):
-
-    # Select window which lies within x-y bounds of the data
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=-90, y=-80, coordinate_system="osgb"),
-        width_pixels=10,
-        height_pixels=10,
-        allow_partial_slice=True,
-    )
-
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(-95, -85)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-85, -75)).all()
-    # No padding in this case so no NaNs
-    assert not da_sliced.isnull().any()
-
-
-    # Select window where the edge of the window lies right on the edge of the data
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=-90, y=-80, coordinate_system="osgb"),
-        width_pixels=20,
-        height_pixels=20,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(-100, -80)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-90, -70)).all()
-    # No padding in this case so no NaNs
-    assert not da_sliced.isnull().any()
-
-    # Select window which is partially outside the boundary of the data - padded on left
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=-90, y=-80, coordinate_system="osgb"),
-        width_pixels=30,
-        height_pixels=30,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(-105, -75)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-95, -65)).all()
-    # Data has been padded on left by 5 NaN pixels
-    assert da_sliced.isnull().sum() == 5*len(da_sliced.y_osgb)
-
-
-    # Select window which is partially outside the boundary of the data - padded on right
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=90, y=-80, coordinate_system="osgb"),
-        width_pixels=30,
-        height_pixels=30,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(75, 105)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-95, -65)).all()
-    # Data has been padded on right by 5 NaN pixels
-    assert da_sliced.isnull().sum() == 5*len(da_sliced.y_osgb)
-
-
-    location = Location(x=-90, y=-0, coordinate_system="osgb")
-
-    # Select window which is partially outside the boundary of the data - padded on top
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=-90, y=95, coordinate_system="osgb"),
-        width_pixels=20,
-        height_pixels=20,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(-100, -80)).all()
-    assert (da_sliced.y_osgb.values == np.arange(85, 105)).all()
-    # Data has been padded on top by 5 NaN pixels
-    assert da_sliced.isnull().sum() == 5*len(da_sliced.x_osgb)
-
-    # Select window which is partially outside the boundary of the data - padded on bottom
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=-90, y=-95, coordinate_system="osgb"),
-        width_pixels=20,
-        height_pixels=20,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(-100, -80)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-105, -85)).all()
-    # Data has been padded on bottom by 5 NaN pixels
-    assert da_sliced.isnull().sum() == 5*len(da_sliced.x_osgb)
-
-    # Select window which is partially outside the boundary of the data - padded right and bottom
-    da_sliced = select_spatial_slice_pixels(
-        da,
-        location=Location(x=90, y=-80, coordinate_system="osgb"),
-        width_pixels=50,
-        height_pixels=50,
-        allow_partial_slice=True,
-    )
-
-    assert isinstance(da_sliced, xr.DataArray)
-    assert (da_sliced.x_osgb.values == np.arange(65, 115)).all()
-    assert (da_sliced.y_osgb.values == np.arange(-105, -55)).all()
-    # Data has been padded on right by 15 pixels and bottom by 5 NaN pixels
-    assert da_sliced.isnull().sum() == 15*len(da_sliced.y_osgb) + 5*len(da_sliced.x_osgb) - 15*5
-
-
-