ebm 0.99.3__py3-none-any.whl

ebm/__main__.py

@@ -0,0 +1,152 @@
+"""EBM start from where when running as a script or module"""
+import os
+import pathlib
+import sys
+
+import pandas as pd
+from loguru import logger
+
+from ebm.cmd import prepare_main
+from ebm.cmd.helpers import configure_json_log, configure_loglevel, load_environment_from_dotenv
+from ebm.cmd.initialize import create_output_directory, init
+from ebm.cmd.migrate import migrate_directories
+from ebm.cmd.pipeline import export_energy_model_reports
+from ebm.cmd.result_handler import EbmDefaultHandler, append_result, transform_model_to_horizontal
+from ebm.cmd.run_calculation import validate_years
+from ebm.model.building_category import BuildingCategory
+from ebm.model.database_manager import DatabaseManager
+from ebm.model.enums import ReturnCode
+from ebm.model.file_handler import FileHandler
+
+df = None
+
+
+def main() -> tuple[ReturnCode, pd.DataFrame | None]:
+    """
+    Execute the EBM module as a script.
+
+    This function serves as the entry point for the script. It handles argument parsing,
+    initializes necessary components, and orchestrates the main workflow of the script.
+
+    Returns
+    -------
+    exit code : tuple[ReturnCode, pd.DataFrame]
+        zero when the program exits gracefully
+
+    """
+    load_environment_from_dotenv()
+    configure_loglevel(log_format=os.environ.get('LOG_FORMAT', '{level.icon} <level>{message}</level>'))
+    configure_json_log()
+
+    logger.debug(f'Starting {sys.executable} {__file__}')
+
+    program_name = 'ebm'
+    default_path = pathlib.Path('output/ebm_output.xlsx')
+
+    arguments = prepare_main.make_arguments(program_name, default_path)
+
+    # Make local variable from arguments for clarity
+    building_categories = [BuildingCategory.from_string(b_c) for b_c in arguments.categories]
+    if not building_categories:
+        building_categories = list(BuildingCategory)
+
+    # `;` Will normally be interpreted as line end when typed in a shell. If the
+    # delimiter is empty make the assumption that the user used ;. An empty delimiter is not valid anyway.
+    csv_delimiter = arguments.csv_delimiter if arguments.csv_delimiter else ';'
+
+    # Make sure everything is working as expected
+    model_years = validate_years(start_year=arguments.start_year, end_year=arguments.end_year)
+
+    input_directory = arguments.input
+    logger.info(f'Using data from "{input_directory}"')
+    database_manager = DatabaseManager(file_handler=FileHandler(directory=input_directory))
+
+    # Create input directory if requested
+    if arguments.create_input:
+        if init(database_manager.file_handler):
+            logger.success('Finished creating input files in {input_directory}',
+                           input_directory=database_manager.file_handler.input_directory)
+            return ReturnCode.OK, None
+        # Exit with 0 for success. The assumption is that the user would like to review the input before proceeding.
+        return ReturnCode.MISSING_INPUT_FILES, None
+    if arguments.migrate:
+        migrate_directories([database_manager.file_handler.input_directory])
+        logger.success('Finished migration')
+        return ReturnCode.OK, None
+
+    missing_input_error = f"""
+Use `<program name> --create-input --input={input_directory}` to create an input directory with the default input files
+""".strip().replace('\n',  ' ')
+
+    # Make sure all required files exists
+    try:
+        missing_files = database_manager.file_handler.check_for_missing_files()
+        if missing_files:
+            print(missing_input_error, file=sys.stderr)
+            return ReturnCode.MISSING_INPUT_FILES, None
+    except FileNotFoundError as file_not_found:
+        if str(file_not_found).startswith('Input Directory Not Found'):
+            logger.error(f'Input Directory "{input_directory}" Not Found')
+            print(missing_input_error, file=sys.stderr)
+            return ReturnCode.FILE_NOT_ACCESSIBLE, None
+
+    database_manager.file_handler.validate_input_files()
+
+    output_file = arguments.output_file
+    create_output_directory(filename=output_file)
+
+    output_file_return_code = prepare_main.check_output_file_status(output_file, arguments.force, default_path,
+                                                                    program_name)
+    if output_file_return_code!= ReturnCode.OK:
+        return output_file_return_code, None
+
+    step_choice = arguments.step
+
+    convert_result_to_horizontal: bool = arguments.horizontal_years
+
+    default_handler = EbmDefaultHandler()
+
+    model = None
+
+    files_to_open = [output_file]
+
+    if step_choice == 'energy-use':
+        output_directory = output_file if output_file.is_dir() else output_file.parent
+        files_to_open = export_energy_model_reports(model_years, database_manager, output_directory)
+    else:
+        model = default_handler.extract_model(model_years, building_categories, database_manager, step_choice)
+
+        if convert_result_to_horizontal and (step_choice in ['area-forecast', 'energy-requirements']) and output_file.suffix=='.xlsx':
+            sheet_name_prefix = 'area' if step_choice == 'area-forecast' else 'energy'
+            logger.debug(f'Transform heating {step_choice}')
+
+            df = transform_model_to_horizontal(model.reset_index())
+            append_result(output_file, df, f'{sheet_name_prefix} condition')
+
+            model = model.reset_index()
+            # Demolition should not be summed any further
+            model = model[model.building_condition!='demolition']
+            model['building_condition'] = 'all'
+            df = transform_model_to_horizontal(model)
+            append_result(output_file, df, f'{sheet_name_prefix} TEK')
+
+            model['building_code'] = 'all'
+            df = transform_model_to_horizontal(model)
+            append_result(output_file, df, f'{sheet_name_prefix} category')
+            logger.success('Wrote {filename}', filename=output_file)
+        else:
+            default_handler.write_tqdm_result(output_file, model, csv_delimiter)
+
+    for file_to_open in files_to_open:
+        if arguments.open or os.environ.get('EBM_ALWAYS_OPEN', 'FALSE').upper() == 'TRUE':
+            logger.info(f'Open {file_to_open}')
+            os.startfile(file_to_open, 'open')
+        else:
+            logger.debug(f'Finished {file_to_open}')
+
+    return ReturnCode.OK, model
+
+
+if __name__ == '__main__':
+    exit_code, result = main()
+    df = result

ebm/__version__.py

@@ -0,0 +1 @@
+version = "0.99.3"

ebm/cmd/calibrate.py

@@ -0,0 +1,83 @@
+import pathlib
+
+from loguru import logger
+import pandas as pd
+
+from dotenv import load_dotenv
+
+from ebm.model.bema import map_sort_order
+
+from ebm.model.calibrate_heating_systems import extract_area_forecast, extract_energy_requirements, \
+    extract_heating_systems
+from ebm.model.data_classes import YearRange
+from ebm.services.files import make_unique_path
+
+CALIBRATION_YEAR = 2023
+
+model_period = YearRange(2020, 2050)
+start_year = model_period.start
+end_year = model_period.end
+
+
+def run_calibration(database_manager,
+                    calibration_year,
+                    area_forecast: pd.DataFrame = None,
+                    write_to_output = False):
+    """
+
+    Parameters
+    ----------
+    database_manager : ebm.model.database_manager.DatabaseManager
+
+    Returns
+    -------
+    pandas.core.frame.DataFrame
+    """
+    load_dotenv(pathlib.Path('.env'))
+
+    input_directory = database_manager.file_handler.input_directory
+
+    logger.info(f'Using input directory "{input_directory}"')
+    logger.info('Extract area forecast')
+    area_forecast = extract_area_forecast(database_manager) if area_forecast is None else area_forecast
+    if write_to_output:
+        write_dataframe(area_forecast[area_forecast.year == calibration_year], 'area_forecast')
+
+    logger.info('Extract energy requirements')
+    energy_requirements = extract_energy_requirements(area_forecast, database_manager)
+    if write_to_output:
+        en_req = energy_requirements.xs(2023, level='year').reset_index().sort_values(
+            by='building_category', key=lambda x: x.map(map_sort_order))
+        write_dataframe(en_req, 'energy_requirements')
+        grouped = en_req[['building_category', 'm2', 'kwh_m2', 'energy_requirement']].groupby(
+            by=['building_category'], as_index=False).agg({'m2': 'first', 'kwh_m2': 'first', 'energy_requirement': 'sum'})
+        grouped = grouped.sort_values(by='building_category', key=lambda x: x.map(map_sort_order))
+        write_dataframe(grouped, 'energy_requirements_sum', sheet_name='sum')
+
+    logger.info('Extract heating systems')
+    heating_systems = extract_heating_systems(energy_requirements, database_manager)
+    if write_to_output:
+        write_dataframe(heating_systems.xs(2023, level='year'), 'heating_systems')
+
+
+    return heating_systems
+
+
+def write_dataframe(df, name='dataframe', sheet_name='Sheet1'):
+    output_directory = pathlib.Path('output')
+    if output_directory.is_dir():
+        logger.debug(f'Writing {name} to file')
+        output_file = output_directory / f'{name}.xlsx'
+        output_file = make_unique_path(output_file)
+        df.to_excel(output_file, merge_cells=False, sheet_name=sheet_name)
+        logger.info(f'Wrote {name} to {output_file} ! {sheet_name if sheet_name!="Sheet1" else ""}')
+    else:
+        logger.warning(f'Cannot write to {output_directory}. Directory does not exists')
+
+
+def main():
+    raise NotImplementedError('Running calibrate as a script is not supported')
+
+
+if __name__ == '__main__':
+    main()

ebm/cmd/calibrate_excel_com_io.py

@@ -0,0 +1,128 @@
+import os
+import pathlib
+import time
+
+import pandas as pd
+from dotenv import load_dotenv
+from loguru import logger
+
+from ebm.cmd.calibrate import run_calibration, write_dataframe
+from ebm.cmd.helpers import configure_loglevel
+from ebm.model.file_handler import FileHandler
+from ebm.model.database_manager import DatabaseManager
+from ebm.model.calibrate_energy_requirements import EnergyRequirementCalibrationWriter, \
+    EnergyConsumptionCalibrationWriter
+from ebm.model.calibrate_heating_systems import DistributionOfHeatingSystems, group_heating_systems_by_energy_carrier
+from ebm.services.calibration_writer import ComCalibrationReader, ExcelComCalibrationResultWriter
+
+LOG_FORMAT = """
+<green>{time:HH:mm:ss.SSS}</green> | <blue>{elapsed}</blue> | <level>{level: <8}</level> | <cyan>{function: <20}</cyan>:<cyan>{line: <3}</cyan> - <level>{message}</level>
+""".strip()
+
+
+def heatpump_filter(df):
+    vannbasert = [n for n in df.index.get_level_values('heating_systems').unique() if
+                  n.startswith('HP Central heating')]
+    elektrisk = [n for n in df.index.get_level_values('heating_systems').unique() if
+                 n.startswith('HP') and n not in vannbasert]
+    el_slice = (slice(None), ['original_condition'], ['heating_rv'], ['TEK07'], slice(None), elektrisk + vannbasert)
+    df = df.loc[el_slice]  # luftluft
+    return df
+
+
+def main():
+    start_time = time.time()
+    load_dotenv(pathlib.Path('.env'))
+    configure_loglevel(log_format=LOG_FORMAT)
+
+    write_to_disk = os.environ.get('EBM_WRITE_TO_DISK', 'False').upper() == 'TRUE'
+    calibration_year = int(os.environ.get('EBM_CALIBRATION_YEAR', 2023))
+    calibration_spreadsheet_name = os.environ.get("EBM_CALIBRATION_OUT", "Kalibreringsark.xlsx!Ut")
+    calibration_sheet = os.environ.get("EBM_CALIBRATION_SHEET", "Kalibreringsark.xlsx!Kalibreringsfaktorer")
+    energy_requirements_calibration_file = os.environ.get('EBM_CALIBRATION_ENERGY_REQUIREMENT',
+                                                          f'kalibrering/{FileHandler.CALIBRATE_ENERGY_REQUIREMENT}')
+    energy_consumption_calibration_file = os.environ.get('EBM_CALIBRATION_ENERGY_CONSUMPTION',
+                                                          f'kalibrering/{FileHandler.CALIBRATE_ENERGY_CONSUMPTION}')
+
+    energy_source_target_cells = os.environ.get('EBM_CALIBRATION_ENERGY_SOURCE_USAGE', 'C64:E68')
+    ebm_calibration_energy_heating_pump = os.environ.get('EBM_CALIBRATION_ENERGY_HEATING_PUMP', 'C72:E74')
+    hs_distribution_cells = os.environ.get('EBM_CALIBRATION_ENERGY_HEATING_SYSTEMS_DISTRIBUTION', 'C32:F44')
+
+    output_directory = pathlib.Path('output')
+
+    logger.info(f'Loading {calibration_sheet}')
+    workbook_name = calibration_sheet.split('!')[0]
+    sheet_name = calibration_sheet.split('!')[1] if '!' in calibration_sheet else 'Kalibreringsfaktorer'
+
+    com_calibration_reader = ComCalibrationReader(workbook_name, sheet_name)
+    calibration = com_calibration_reader.extract()
+    logger.info(f'Make {calibration_sheet} compatible with ebm')
+    energy_source_by_building_group = com_calibration_reader.transform(calibration)
+
+    logger.info('Write calibration to ebm')
+    eq_calibration_writer = EnergyRequirementCalibrationWriter()
+    eq_calibration_writer.load(energy_source_by_building_group, energy_requirements_calibration_file)
+
+    ec_calibration_writer = EnergyConsumptionCalibrationWriter()
+    ec_calibration = ec_calibration_writer.transform(energy_source_by_building_group)
+    ec_calibration_writer.load(ec_calibration, energy_consumption_calibration_file)
+
+    logger.info('Calculate calibrated energy use')
+    area_forecast = None
+    area_forecast_file = pathlib.Path('kalibrert/area_forecast.csv')
+    if area_forecast_file.is_file():
+        logger.info(f'  Using {area_forecast_file}')
+        area_forecast = pd.read_csv(area_forecast_file)
+
+    database_manager = DatabaseManager(FileHandler(directory='kalibrert'))
+
+    df = run_calibration(database_manager, calibration_year=2023,
+                         area_forecast=area_forecast, write_to_output=write_to_disk)
+
+    # df = heatpump_filter(df)
+
+    logger.info('Transform heating systems')
+
+    energy_source_by_building_group = group_heating_systems_by_energy_carrier(df)
+    energy_source_by_building_group = energy_source_by_building_group.xs(2023, level='year')
+
+    if write_to_disk:
+        if not output_directory.is_dir():
+            output_directory.mkdir()
+        write_dataframe(energy_source_by_building_group, 'energy_source_by_building_group')
+
+    energy_source_by_building_group = energy_source_by_building_group.fillna(0)
+
+    logger.info(f'Writing heating systems distribution to {calibration_spreadsheet_name}')
+    hs_distribution_writer = ExcelComCalibrationResultWriter(excel_filename=calibration_spreadsheet_name,
+                                                             target_cells=hs_distribution_cells)
+
+    distribution_of_heating_systems = DistributionOfHeatingSystems()
+    shares_start_year = distribution_of_heating_systems.extract(database_manager)
+    heating_systems_distribution = distribution_of_heating_systems.transform(shares_start_year)
+
+    hs_distribution_writer.extract()
+    hs_distribution_writer.transform(heating_systems_distribution)
+    hs_distribution_writer.load()
+
+    logger.info(f'Writing energy_source using writer to {calibration_spreadsheet_name}')
+    energy_source_excel_com_writer = ExcelComCalibrationResultWriter(
+        excel_filename=calibration_spreadsheet_name, target_cells=energy_source_target_cells)
+
+    energy_source_excel_com_writer.extract()
+    energy_source_excel_com_writer.transform(energy_source_by_building_group)
+    energy_source_excel_com_writer.load()
+
+    logger.info(f'Writing calculated energy pump use to {calibration_spreadsheet_name}')
+    heatpump_excel_com_writer = ExcelComCalibrationResultWriter(
+        excel_filename=calibration_spreadsheet_name, target_cells=ebm_calibration_energy_heating_pump)
+
+    heatpump_excel_com_writer.extract()
+    heatpump_excel_com_writer.transform(energy_source_by_building_group)
+    heatpump_excel_com_writer.load()
+
+    logger.info(f'Calibrated {calibration_spreadsheet_name} in {round(time.time() - start_time, 2)} seconds')
+
+
+if __name__ == '__main__':
+    main()

ebm/cmd/heating_systems_by_year.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+from ebm.model.calibrate_heating_systems import group_heating_systems_by_energy_carrier
+from ebm.model.data_classes import YearRange
+
+
+def group_heating_systems_energy_source_by_year(hs: pd.DataFrame) -> pd.DataFrame:
+    df = hs.set_index(['building_category', 'building_condition', 'purpose', 'building_code', 'year', 'heating_systems'])
+
+    return group_heating_systems_by_energy_carrier(df)
+
+
+
+
+def group_heating_systems_energy_source_by_year_horizontal(hs: pd.DataFrame, year_range: YearRange=None) -> pd.DataFrame:
+    df = group_heating_systems_energy_source_by_year(hs)
+    return df.reset_index().pivot(columns=['year'], index=['building_category', 'energy_source'], values=['energy_use'])
+

ebm/cmd/helpers.py

@@ -0,0 +1,134 @@
+import os
+import pathlib
+import sys
+from datetime import datetime
+
+from dotenv import find_dotenv, load_dotenv
+from loguru import logger
+
+
+def load_environment_from_dotenv() -> None:
+    """
+    Load environment variables from a .env file located in the current working directory.
+
+    If a .env file is found, its contents are loaded into the environment.
+    """
+    env_file = pathlib.Path(find_dotenv(usecwd=True))
+    if env_file.is_file():
+        logger.trace('Loading environment from {env_file}', env_file=env_file)
+        load_dotenv(env_file)
+    else:
+        logger.trace(f'.env not found in {env_file}', env_file=env_file.absolute())
+
+
+def configure_json_log(log_directory: str|bool=False) -> None:
+    """
+    Configure JSON logging using the `loguru` logger.
+
+    This function sets up structured JSON logging to a file, with the log file path
+    determined by the `LOG_DIRECTORY` environment variable or the provided `log_directory` argument.
+    If `LOG_DIRECTORY` is set to 'TRUE', the default directory 'log' is used.
+    If it is set to 'FALSE', logging is skipped.
+
+    Parameters
+    ----------
+    log_directory : str or bool, optional
+        The directory where the log file should be saved. If set to `False`, logging is disabled
+        unless overridden by the `LOG_DIRECTORY` environment variable.
+
+    Environment Variables
+    ---------------------
+    LOG_DIRECTORY : str
+        Overrides the `log_directory` argument when set. Special values:
+        - 'TRUE': uses default directory 'log'
+        - 'FALSE': disables logging
+
+    Notes
+    -----
+    - The log file is named using the current timestamp in ISO format (without colons).
+    - The log file is serialized in JSON format.
+    - The directory is created if it does not exist.
+
+    Examples
+    --------
+    >>> configure_json_log("logs")
+    >>> os.environ["LOG_DIRECTORY"] = "TRUE"
+
+    >>> configure_json_log(False)
+
+    """
+    if not log_directory:
+        return
+
+    script_name = pathlib.Path(pathlib.Path(sys.argv[0]))
+    file_stem = script_name.stem if script_name.stem!='__main__' else script_name.parent.name + script_name.stem
+    if 'PYTEST_CURRENT_TEST' in os.environ and os.environ.get('PYTEST_CURRENT_TEST'):
+        pytest_current_test = os.environ.get('PYTEST_CURRENT_TEST').split('::')
+        file_stem = pathlib.Path(pytest_current_test[0]).stem + pytest_current_test[1].replace('(call)', '').strip()
+
+    env_log_directory = os.environ.get('LOG_DIRECTORY', log_directory)
+    if isinstance(env_log_directory, bool):
+        env_log_directory = pathlib.Path.cwd() / 'log'
+    log_to_json = str(env_log_directory).upper().strip()!='FALSE'
+    env_log_directory = env_log_directory if log_to_json and str(env_log_directory).upper().strip() != 'TRUE' else 'log'
+
+    if log_to_json:
+        log_directory = pathlib.Path(env_log_directory if env_log_directory else log_directory)
+        if log_directory.is_file():
+            logger.warning(f'LOG_DIRECTORY={log_directory} is a file. Skipping json logging')
+            return
+        log_directory.mkdir(exist_ok=True)
+
+        log_start = datetime.now()
+        timestamp = log_start.isoformat(timespec='seconds').replace(':', '')
+        log_filename = log_directory / f'{file_stem}-{timestamp}.json'
+        if log_filename.is_file():
+            log_start_milliseconds = log_start.isoformat(timespec='milliseconds').replace(':', '')
+            log_filename = log_filename.with_stem(f'{file_stem}-{log_start_milliseconds}')
+
+        logger.debug(f'Logging json to {log_filename}')
+        logger.add(log_filename, level=os.environ.get('LOG_LEVEL_JSON', 'TRACE'), serialize=True)
+        if len(sys.argv) > 1:
+            logger.info(f'argv={sys.argv[1:]}')
+    else:
+        logger.debug('Skipping json log. LOG_DIRECTORY is undefined.')
+
+
+def configure_loglevel(log_format: str | None = None, level: str = 'INFO') -> None:
+    """
+    Configure the loguru logger with a specified log level and format.
+
+    By default, sets the log level to INFO unless either:
+    - The '--debug' flag is present in the command-line arguments (`sys.argv`), or
+    - The environment variable DEBUG is set to 'TRUE' (case-insensitive).
+
+    If debug mode is enabled, the log level is set to DEBUG and a filter is applied
+    to suppress DEBUG logs from the 'ebm.model.file_handler' logger.
+
+    Parameters
+    ----------
+    log_format : str, optional
+        Custom format string for log messages. If not provided, the default format is used.
+    level : str, optional
+        Default log level to use when debug mode is not active. Defaults to 'INFO'.
+
+    Returns
+    -------
+    None
+
+    """
+    logger.remove()
+    options = {'level': level}
+    if log_format:
+        options['format'] = log_format
+
+    # Accessing sys.argv directly since we want to figure out the log level before loading arguments with arg_parser.
+    # Debug level may also be conveyed through environment variables, so read that from environ as well.
+    if '--debug' in sys.argv or os.environ.get('DEBUG', '').upper() == 'TRUE':
+        options['level'] = 'DEBUG'
+
+    logger.add(sys.stderr,
+               filter=lambda f: not (f['name'] == 'ebm.model.file_handler' and f['level'].name == 'DEBUG'),
+               **options)
+
+