ebtools 0.1.0__py3-none-any.whl

ebtools/__init__.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon May 17 15:48:46 2021
+
+@author: Eamonn.Bell
+"""
+
+
+
+from ebtools.general import (
+
+    # HELPER functions
+    get_file_type,
+    force_to_list,
+    force_to_np_array,
+    sort_dict_by_keys,
+    adjust_for_clock_change,
+    base_df,
+    df_filter,
+    df_cutoff,
+    drop_empty_df_cols,
+    melt_df,
+    unique_element_count,
+    non_numeric_pivot,
+    df_round_to_decimal,
+    make_dict_keys_from_list,
+    remove_df_index_name,
+    remove_df_column_name,
+    correct_str_elements_float,
+    correct_str_elements_int,
+
+    # DF TOOLS - Create df from JSON
+    create_df_from_json,
+    create_df_from_json_from_file,
+
+    # READ data - general file_types
+    read_from_csv,
+    read_from_xlsx,
+    read_from_parquet,
+    
+    # READ data - HH Data into a dataframe
+    load_hh_data,
+
+    # SAVE data
+    save_to_csv,
+    save_to_xlsx,
+    save_to_parquet,
+    
+    # INSPECT functions
+    get_call_func_name,
+    get_function_arguments,
+    
+    # FILE READING tools
+    does_file_exist,
+    list_all_files_in_folder,
+    zip_checker,
+    
+    # DATETIME - Helper funcs
+    is_year_leap_year,
+    get_daylight_savings_dates,
+    bst_date_range,
+    convert_single_datetime_to_sp_start,
+    convert_single_datetime_to_sp_end,
+    
+    # DATETIME - Convert Datetime
+    convert_date_to_standard_date_str,
+    convert_date_to_datetime,
+    convert_datetime_to_date,
+    convert_date_to_datetime_str,
+    convert_date_to_first_of_month,
+    convert_date_to_last_of_month,
+    convert_datetime_remove_tz_aware,
+    convert_datetime_by_tz,
+    convert_datetime_sp_start,
+    convert_datetime_sp_end,
+    
+    # DATETIME - Check Datetime
+    check_datetime_formats,
+    check_datetime_formats_tz,
+    check_string_datetime_formats,
+    
+    # DATETIME - Adjust Dates
+    start_of_delivery_year,
+    roll_back_by_month_str,
+    roll_back_by_month_df,
+    
+    # DATETIME - Add first-of-month-col
+    add_first_of_month_col_from_int_cols,
+    add_first_of_month_col_from_date_col,
+
+    # DATETIME - Month-start and Month-end
+    move_date_to_start_of_month,
+    move_date_to_end_of_month,
+    
+    # SP_EFA
+    sp_to_efa,
+    hour_to_efa,
+    current_sp,
+    current_efa,
+    make_sp_efa_table,
+    convert_date_to_efa_datetime,
+    convert_datetime_to_date_and_efa,
+    make_sp_from_string,
+    make_efa_from_string,
+    add_sp_from_datetime_col,
+    add_sp_start_end_datetimes,
+    add_sp_to_efa_column,
+    add_efa_datetime_col,
+    add_date_efa_cols,
+    add_sp_and_efa_from_datetime,
+    
+    # DATAFRAME SELECTION - Dataframe Augmentation
+    add_columns,
+    add_buffer_days,
+    
+    # Dataframe Date Range Selection
+    select_year,
+    select_weeks,
+    select_days,
+    
+    # Analysis and Plotting dataframes
+    make_full_year_pivot,
+    make_full_plotting_df,
+    make_pivot_calendar,
+    
+    # DATAFRAME SELECTION - STRING date functions
+    week_starting,
+    week_number,
+    weekday_number,
+    day_of_the_year_number,
+    
+    # ARRAY GENERAL functions
+    add_newaxis,
+    
+    # ARRAY SELECTION functions 
+    n_smallest_values_per_row,
+    n_largest_values_per_row,
+
+    # ONLINE tools
+    is_online
+
+    )
+
+
+from ebtools.data import (
+    
+    # Date lists for quick formatting
+    format_list_standard,
+    format_list_tz,
+    
+    )
+
+
+import ebtools.data as data
+
+
+from ebtools._version import __version__
+
+
+# Module level doc_string
+__doc__ = """
+
+Write a docstring here which sets out what each high level component of the 
+module does.
+
+State the the docstrings of each high level component will suggest the best
+ways to use that module, the most useful functions, and the right order in 
+which to use them. 
+"""

ebtools/_version.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Oct 25 00:02:20 2023
+
+@author: eamonnbell
+"""
+
+
+__version__ = '0.1.0'
+
+__doc__ = """
+
+0.1.0   Initial release. Module refactored to provide consistency:
+            -- NumPy style docstrings throughout.
+            -- Type hints for all existing functions.
+            -- Usage examples for all functions.
+            -- Tests added for all public functions. 
+            -- Functions broken out into smaller helper private funcs where
+                this is helpful for clarity and maintenance.
+
+0.0.2   Module general update:
+            -- ebtools.ebdicts RENAMED as ebtools.data
+            -- df_tools funcs (and others) rearranged.
+
+0.0.1   Module rearanged so that general functions are spread over multiple
+            different sub-files and that general becomes a sub-module.
+
+0.0.1   Module created. Module containing helper functions for use within 
+            other modules, so that the same helper functions do not need to be
+            re-written time and again. 
+
+"""

ebtools/data/__init__.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Mar 24 16:33:39 2023
+
+@author: Eamonn.Bell
+"""
+
+from ebtools.data.dno import (
+    # DNO
+    dno_codes,
+    dno_names,
+    dno_to_gsp_group
+    )
+
+from ebtools.data.powerenergy import (
+    # Power and Energy
+    power_conversion
+    )
+
+from ebtools.data.ukbankholidays import (
+    uk_bank_hols
+    )
+
+from ebtools.data.ukclockchangedays import (
+    uk_clock_change_forward,
+    uk_clock_change_backward
+    )
+
+from ebtools.data.units import (
+    unit_dict
+    )
+
+from ebtools.data.dateformats import (
+    
+    format_list_standard,
+    format_list_tz
+    
+    )

ebtools/data/dateformats.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed May 26 17:24:12 2021
+
+@author: Eamonn.Bell
+"""
+
+"""
+These are standard lists of DATETIME FORMAT codes to check when importing
+data from sources with possibly different datetime formats.
+
+=============
+Format Codes:
+=============
+The user may wish to compile their own format code list for usage in 
+specific situations. In that case, the user should build a list of 
+FORMAT strings based on the set of acceptable python FORMAT CODES to be 
+found at the following link under the heading "strftime() and strptime()
+Format Codes":
+https://docs.python.org/3/library/datetime.html
+
+"""
+
+# Non-timezone-specific dates: the most common datetime occurance
+format_list_standard = [
+    
+    # Year-first date formats
+    '%Y-%m-%d',
+    '%y-%m-%d',
+    
+    # Year-first with TIME formats
+    '%Y-%m-%dT%H:%M:%S',
+    '%Y-%m-%d %H:%M:%S',
+    '%Y-%m-%d %H:%M',    
+    
+    # Day-first formats
+    '%d-%m-%y',
+    '%d-%b-%y',
+    '%d-%b-%Y',
+    '%d-%m-%Y', 
+    '%d/%m/%Y',
+    '%d/%m/%y',
+    
+    # Datetime formats
+    '%d/%m/%Y %H:%M',
+    '%d/%m/%Y %H:%M:%S'
+    
+    ]
+
+# Timezone-specific dates: when there is a timezone element in the datetime
+format_list_tz = [
+    
+    # Format code strings registering ZULU time (GMT)
+    '%Y-%m-%dT%H:%M:%SZ',
+    '%Y-%m-%dT%H:%M:%S Z',
+    '%Y-%m-%d %H:%M:%SZ',
+    '%Y-%m-%d %H:%M:%S Z',
+    
+    # Format code strings with a UTC OFFSET: +01:00 / -0600 / etc
+    '%Y-%m-%dT%H:%M:%S%z',
+    '%Y-%m-%dT%H:%M:%S %z',
+    '%Y-%m-%d %H:%M:%S%z',
+    '%Y-%m-%d %H:%M:%S %z'
+    ]
+

ebtools/data/dno.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Nov 19 14:09:47 2018
+
+@author: Eamonn_Bell
+This file contains the set of lists and dictionaries which may be useful.
+
+"""
+
+
+
+"""
+******************************************************************************
+******************************************************************************
+
+# ---- DNO data
+                
+                 Region Names & GSP Group look-up codes
+
+******************************************************************************
+******************************************************************************
+"""
+
+
+# Sources for the names and MPAS Operator IDs for each Licenced Distribution
+# Network Owner (LDNO) come from:
+#     -- https://en.wikipedia.org/wiki/Distribution_network_operator
+#     -- https://www.elexonportal.co.uk/svallf
+dno_codes = {
+    10 : 'EELC',
+    11 : 'EMEB',
+    12 : 'LOND',
+    13 : 'MANW',
+    14 : 'MIDE',
+    15 : 'NEEB',
+    16 : 'NORW',
+    17 : 'HYDE',
+    18 : 'SPOW',
+    19 : 'SEEB',
+    20 : 'SOUT',
+    21 : 'SWAE',
+    22 : 'SWEB',
+    23 : 'YELG',
+    24 : 'IPNL',
+    25 : 'LENG',
+    26 : 'GUCL',
+    27 : 'ETCL',
+    28 : 'EDFI',
+    29 : 'HARL',
+    30 : 'PENL',
+    31 : 'UKPD',
+    32 : 'UDNL',
+    33 : 'GGEN',
+    34 : 'MPDL',
+    35 : 'FEAL',
+    36 : 'VATT',
+    37 : 'FORB',
+    38 : 'INDI'
+    }
+
+dno_names = {
+    10 : 'Eastern England',
+    11 : 'East Midlands',
+    12 : 'London',
+    13 : 'Merseyside and Northern Wales',
+    14 : 'West Midlands',
+    15 : 'North Eastern England',
+    16 : 'North Western_England',
+    17 : 'Northern Scotland',
+    18 : 'Southern Scotland',
+    19 : 'South Eastern England',
+    20 : 'Southern England',
+    21 : 'Southern_Wales',
+    22 : 'South Western England',
+    23 : 'Yorkshire'
+    }
+
+dno_to_gsp_group = {
+    10 : '_A',
+    11 : '_B',
+    12 : '_C',
+    13 : '_D',
+    14 : '_E',
+    15 : '_F',
+    16 : '_G',
+    17 : '_P',
+    18 : '_N',
+    19 : '_J',
+    20 : '_H',
+    21 : '_K',
+    22 : '_L',
+    23 : '_M'
+    }
+
+
+
+
+
+
+
+
+
+

ebtools/data/powerenergy.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr  4 17:40:57 2023
+
+@author: Eamonn.Bell
+"""
+
+
+# =============================================================================
+# ---- Power conversion factors
+# =============================================================================
+
+# Dictionary to convert power units from one scale to another
+power_conversion = {
+    'W'  : 1,
+    'kW' : 10**3,
+    'MW' : 10**6,
+    'GW' : 10**9,
+    }

ebtools/data/ukbankholidays.py

@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Apr  4 17:37:15 2023
+
+@author: Eamonn.Bell
+"""
+
+# =============================================================================
+# ---- UK Bank Holidays
+# =============================================================================
+
+# Maintain a list of UK Bank Holidays here.
+#
+# The official source for UK Bank Holidays is:
+# https://www.gov.uk/bank-holidays
+# At this gov.uk site the list of UK bank holidays is available from 2016
+# to one year ahead of the current year. 
+#
+# For Bank Holidays ahead of the current and next year, dates are gathered
+# from the Gottex multi-year calendar. A version is available here:
+# https://vdocuments.site/gottex-calendar.html
+# These dates should be indicated as being 'Unconfirmed' and should be 
+# confirmed closer to realtime via the gov.uk website above.
+
+uk_bank_hols = [
+    # Confirmed
+    '2016-01-01',
+    '2016-03-25',
+    '2016-03-28',
+    '2016-05-02',
+    '2016-05-30',
+    '2016-08-29',
+    '2016-12-26',
+    '2016-12-27',
+    
+    # Confirmed
+    '2017-01-02',
+    '2017-04-14',
+    '2017-04-17',
+    '2017-05-01',
+    '2017-05-29',
+    '2017-08-28',
+    '2017-12-25',
+    '2017-12-26',    
+    
+    # Confirmed
+    '2018-01-01',
+    '2018-03-30',
+    '2018-04-02',
+    '2018-05-07',
+    '2018-05-28',
+    '2018-08-27',
+    '2018-12-25',
+    '2018-12-26',
+    
+    # Confirmed
+    '2019-01-01',
+    '2019-04-19',
+    '2019-04-22',
+    '2019-05-06',
+    '2019-05-27',
+    '2019-08-26',
+    '2019-12-25',
+    '2019-12-26',
+    
+    # Confirmed
+    '2020-01-01',
+    '2020-04-10',
+    '2020-04-13',
+    '2020-05-08',
+    '2020-05-25',
+    '2020-08-31',
+    '2020-12-25',
+    '2020-12-28',
+    
+    # Confirmed
+    # 2021 Bank Holiday days with description to compare with 2022.
+    '2021-01-01',  # New Year's Day
+    '2021-04-02',  # Good Friday
+    '2021-04-05',  # Easter Monday
+    '2021-05-03',  # Early May Bank Holiday
+    '2021-05-31',  # Spring Bank Holiday
+    '2021-08-30',  # Summer Bank Holiday
+    '2021-12-27',  # Christmas Day (substitute day)
+    '2021-12-28',  # Boxing Day (substitute day)
+    
+    # Confirmed
+    # 2022 will have TWO EXTRA BANK HOLIDAYS - Queen's Platinum Jubilee Year
+    '2022-01-03',  # New Year's Day  
+    '2022-04-15',  # Good Friday
+    '2022-04-18',  # Easter Monday
+    '2022-05-02',  # Early May Bank Holiday
+    '2022-06-02',  # Spring Bank Holiday
+    '2022-06-03',  # Platinum Jubilee Bank Holiday  <--- EXTRA BANK HOLIDAY
+    '2022-08-29',  # Summer Bank Holiday
+    '2022-09-19',  # Queen's State Funeral          <--- EXTRA BANK HOLIDAY
+    '2022-12-26',  # Boxing Day
+    '2022-12-27',  # Christmas Day (substitute day)
+    
+    # Confirmed
+    # 2023 will have AN EXTRA BANK HOLIDAY - Conronation of Charles III
+    '2023-01-02',  # New Year's Day
+    '2023-04-07',  # Good Friday
+    '2023-04-10',  # Easter Monday
+    '2023-05-01',  # Early May Bank Holiday
+    '2023-05-08',  # Bank Holiday for the coronation of King Charles III <--- EXTRA BANK HOLIDAY
+    '2023-05-29',  # Spring Bank Holiday
+    '2023-08-28',  # Summer Bank Holiday
+    '2023-12-25',  # Christmas Day
+    '2023-12-26',  # Boxing Day
+
+    # Confirmed
+    '2024-01-01',  # New Year's Day
+    '2024-03-29',  # Good Friday
+    '2024-04-01',  # Easter Monday
+    '2024-05-06',  # Early May Bank Holiday
+    '2024-05-27',  # Spring Bank Holiday
+    '2024-08-26',  # Summer Bank Holiday
+    '2024-12-25',  # Christmas Day
+    '2024-12-26',  # Boxing Day
+
+    # Confirmed
+    '2025-01-01',  # New Year's Day
+    '2025-04-18',  # Good Friday
+    '2025-04-21',  # Easter Monday
+    '2025-05-05',  # Early May Bank Holiday
+    '2025-05-26',  # Spring Bank Holiday
+    '2025-08-25',  # Summer Bank Holiday
+    '2025-12-25',  # Christmas Day
+    '2025-12-26',  # Boxing Day
+
+    ## Unconfirmed
+    '2026-01-01',
+    '2026-04-03',
+    '2026-04-06',
+    '2026-05-04',
+    '2026-05-25',
+    '2026-08-31',
+    '2026-12-25',
+    '2026-12-28',
+
+    ## Unconfirmed
+    '2027-01-01',
+    '2027-03-26',
+    '2027-03-29',
+    '2027-05-03',
+    '2027-05-21',
+    '2027-08-30',
+    '2027-12-27',
+    '2027-12-28',
+
+    ## Unconfirmed
+    '2028-01-03',
+    '2028-04-14',
+    '2028-04-17',
+    '2028-05-01',
+    '2028-05-29',
+    '2028-08-28',
+    '2028-12-25',
+    '2028-12-26',
+
+    ## Unconfirmed
+    '2029-01-01',
+    '2029-03-30',
+    '2029-04-02',
+    '2029-05-07',
+    '2029-05-28',
+    '2029-08-27',
+    '2029-12-25',
+    '2029-12-26',
+
+    ## Unconfirmed
+    '2030-01-01',
+    '2030-04-19',
+    '2030-04-22',
+    '2030-05-06',
+    '2030-05-27',
+    '2030-08-26',
+    '2030-12-25',
+    '2030-12-26'
+    ]