tariff_fetch 0.1__py3-none-any.whl

tariff_fetch/__init__.py

@@ -0,0 +1,3 @@
+"""This is a repository for modern data science projects.."""
+
+__version__ = "0.1.0"

tariff_fetch/_cli/__init__.py

@@ -0,0 +1,33 @@
+import os
+from datetime import datetime
+from pathlib import Path
+
+import questionary
+from pathvalidate import sanitize_filename
+from rich.console import Console
+
+console = Console()
+
+
+def prompt_filename(output_folder: Path, suggested_filename: str, extension: str) -> Path:
+    date_str = datetime.now().strftime("%Y-%m-%d")
+    suggested_filename = sanitize_filename(f"{suggested_filename}_{date_str}")
+    if output_folder.exists():
+        existing_filenames = set(output_folder.iterdir())
+        filepath = next(
+            _
+            for i in range(0xFFFFFF)
+            if (_ := output_folder.joinpath(f"{suggested_filename}-{i}{os.extsep}{extension}"))
+            not in existing_filenames
+        )
+    else:
+        filepath = output_folder.joinpath(f"{suggested_filename}-0{os.extsep}{extension}")
+
+    return Path(
+        questionary.path(
+            message="Path to save the results",
+            default=filepath.as_posix(),
+            file_filter=lambda _: Path(_).suffix == extension,
+            validate=lambda _: (not os.path.exists(_)) or "A file with that name already exists",
+        ).ask()
+    )

tariff_fetch/_cli/genability.py

@@ -0,0 +1,152 @@
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import questionary
+from dotenv import load_dotenv
+
+from tariff_fetch.genability.lse import get_lses_page
+from tariff_fetch.genability.tariffs import CustomerClass, TariffType, tariffs_paginate
+
+from . import console, prompt_filename
+from .types import Utility
+
+
+def _find_utility_lse_id(utility: Utility) -> int | None:
+    with console.status("Fetching lses..."):
+        lses = get_lses_page(
+            fields="min",
+            searchOn=["code"],
+            search=str(utility.eia_id),
+            startsWith=True,
+            endsWith=True,
+        )["results"]
+    if len(lses) == 0:
+        # No utilities found with this eia id
+        console.print(
+            f'Utility "{utility.name}" with EIA Id {utility.eia_id} not found in arcadia database', style="bold red"
+        )
+        return None
+    if len(lses) == 1:
+        # Found one utility
+        utility_lse_id = lses[0]["lseId"]
+        return utility_lse_id
+    else:
+        # Nothing found; this should *theoretically* never happen but let's keep it just in case
+        choices = [questionary.Choice(title=_["name"], value=_["lseId"]) for _ in lses]
+        choices.append(questionary.Separator())
+        choices.append(questionary.Choice(title="None of these", value=None))
+        utility_lse_id: int | None = questionary.select(
+            message=f"Found multiple utilities with lse id = {utility.eia_id}. Select one.", choices=choices
+        ).ask()
+        if utility_lse_id is None:
+            console.print("No utility chosen")
+            return None
+        return utility_lse_id
+
+
+def _select_tariffs(
+    lse_id: int, customer_classes: list[CustomerClass], tariff_types: list[TariffType]
+) -> list[tuple[str, int]]:
+    with console.status("Fetching tariffs..."):
+        tariffs = list(
+            tariffs_paginate(
+                lseId=lse_id,
+                fields="min",
+                effectiveOn=datetime.now(timezone.utc),
+                customerClasses=customer_classes,
+                tariffTypes=tariff_types,
+            )
+        )
+    if not tariffs:
+        return []
+    return questionary.checkbox(
+        message="Select tariffs",
+        choices=[
+            questionary.Choice(title=_["tariffName"], value=(_["tariffName"], _["masterTariffId"]), checked=True)
+            for _ in tariffs
+        ],
+        use_search_filter=True,
+        use_jk_keys=False,
+    ).ask()
+
+
+def _select_customer_classes() -> list[CustomerClass]:
+    return questionary.checkbox(
+        message="Select customer classes",
+        choices=[
+            questionary.Choice(title="Residential", value="RESIDENTIAL"),
+            questionary.Choice(title="General", value="GENERAL"),
+            questionary.Choice(title="Special Use", value="SPECIAL_USE"),
+        ],
+        validate=lambda _: True if _ else "Select at least one customer class",
+    ).ask()
+
+
+def _select_tariff_types() -> list[TariffType]:
+    return questionary.checkbox(
+        message="Select tariff types",
+        choices=[
+            questionary.Choice(title="Default", value="DEFAULT"),
+            questionary.Choice(title="Alternative", value="ALTERNATIVE"),
+            questionary.Choice(title="Optional extra", value="OPTIONAL_EXTRA"),
+            questionary.Choice(title="Rider", value="RIDER"),
+        ],
+        validate=lambda _: bool(_) or "Select at least one tariff type",
+    ).ask()
+
+
+def _fetch_tariffs(tariffs: list[tuple[str, int]]):
+    result = []
+    with console.status("Fetching tariffs..."):
+        for name, id_ in tariffs:
+            console.print(f"Tariff id: {name}")
+            page = list(
+                tariffs_paginate(
+                    masterTariffId=id_,
+                    effectiveOn=datetime.now(timezone.utc),
+                    fields="ext",
+                    populateProperties=True,
+                    populateRates=True,
+                )
+            )
+            result.extend(page)
+    return result
+
+
+def process_genability(utility: Utility, output_folder: Path):
+    load_dotenv()
+    if not os.getenv("ARCADIA_APP_ID"):
+        console.print("[b]ARCADIA_APP_ID[/] environment variable is not set.")
+    if not os.getenv("ARCADIA_APP_KEY"):
+        console.print("[b]ARCADIA_APP_KEY[/] environment variable is not set.")
+    if not (os.getenv("ARCADIA_APP_ID") and os.getenv("ARCADIA_APP_KEY")):
+        console.print("Cannot use Arcadia API due to missing credentials")
+        console.input("Press enter to proceed...")
+        return
+
+    lse_id = _find_utility_lse_id(utility)
+    if lse_id is None:
+        return
+
+    if not (customer_classes := _select_customer_classes()):
+        return
+
+    if not (tariff_types := _select_tariff_types()):
+        return
+
+    if not (tariffs := _select_tariffs(lse_id, customer_classes, tariff_types)):
+        console.print("[red]No tariffs found[/]")
+        console.input("Press enter to proceed...")
+        return
+
+    results = _fetch_tariffs(tariffs)
+    suggested_filename = f"arcadia_{utility.name}"
+
+    if not (filename := prompt_filename(output_folder, suggested_filename, "json")):
+        return
+
+    filename.parent.mkdir(exist_ok=True)
+    filename.write_text(json.dumps(results, indent=2))
+    console.print(f"Wrote [blue]{len(results)}[/] records to {filename}")

tariff_fetch/_cli/openei.py

@@ -0,0 +1,89 @@
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+import questionary
+from dotenv import load_dotenv
+
+from tariff_fetch.openei.utility_rates import UtilityRateSector, UtilityRatesResponseItem, iter_utility_rates
+
+from . import console, prompt_filename
+from .types import Utility
+
+
+def _prompt_sector() -> UtilityRateSector:
+    return questionary.select(
+        message="Select sector",
+        choices=[
+            "Residential",
+            "Commercial",
+            "Industrial",
+            "Lighting",
+        ],
+    ).ask()
+
+
+def _prompt_detail_level() -> Literal["full", "minimal"]:
+    return questionary.select(
+        message="Select level of detail",
+        choices=["full", "minimal"],
+    ).ask()
+
+
+def _get_tariffs(
+    eia_id: int, sector: UtilityRateSector, detail: Literal["full", "minimal"]
+) -> list[UtilityRatesResponseItem]:
+    api_key = os.getenv("OPENEI_API_KEY")
+    if not api_key:
+        raise ValueError("API Key is not set (via OPENEI_API_KEY variable)")
+    with console.status("Fetching rates..."):
+        iterator = iter_utility_rates(
+            api_key,
+            effective_on_date=datetime.now(timezone.utc),
+            sector=sector,
+            detail=detail,
+            eia=eia_id,
+        )
+        return list(iterator)
+
+
+def _prompt_tariffs(tariffs: list[UtilityRatesResponseItem]) -> list[UtilityRatesResponseItem]:
+    return questionary.checkbox(
+        message="Select tariffs to include",
+        choices=[questionary.Choice(title=_["name"], value=_, checked=True) for _ in tariffs],
+    ).ask()
+
+
+def process_openei(utility: Utility, output_folder: Path):
+    load_dotenv()
+    if not os.getenv("OPENEI_API_KEY"):
+        console.print("[b]OPENEI_API_KEY[/] environment variable is not set")
+        console.print("Cannot use OpenEI API due to missing credentials")
+        console.input("Press enter to proceed...")
+        return
+
+    if not (sector := _prompt_sector()):
+        return
+    if not (detail_level := _prompt_detail_level()):
+        return
+    tariffs = _get_tariffs(utility.eia_id, sector, detail_level)
+    if not tariffs:
+        console.print("[red]No tariffs found[/]")
+        console.input("Press enter to proceed...")
+        return
+    tariffs = _prompt_tariffs(tariffs)
+    if not tariffs:
+        console.print("[red]No tariffs selected[/]")
+        console.input("Press enter to proceed...")
+        return
+
+    suggested_filename = f"openei_{utility.name}_{sector}_{detail_level}"
+    if not (filepath := prompt_filename(output_folder, suggested_filename, "json")):
+        return
+
+    filepath.parent.mkdir(exist_ok=True)
+    print(filepath)
+    filepath.write_text(json.dumps(tariffs, indent=2))
+    console.print(f"Wrote [blue]{len(tariffs)}[/] items to {filepath}")

tariff_fetch/_cli/rateacuity.py

@@ -0,0 +1,164 @@
+import json
+import os
+from pathlib import Path
+
+import questionary
+import tenacity
+from dotenv import load_dotenv
+from fuzzywuzzy import fuzz
+from selenium.common.exceptions import WebDriverException
+
+from tariff_fetch._cli.types import Utility
+from tariff_fetch.rateacuity import LoginState, create_context
+
+from . import console, prompt_filename
+
+
+def process_rateacuity_gas(output_folder: Path, state: str):
+    load_dotenv()
+    if not (username := os.getenv("RATEACUITY_USERNAME")):
+        console.print("[b]RATEACUITY_USERNAME[/] environment variable is not set")
+    if not (password := os.getenv("RATEACUITY_PASSWORD")):
+        console.print("[b]RATEACUITY_PASSWORD[/] environment variable is not set")
+    if not (username and password):
+        console.print("Cannot use RateAcuity due to missing credentials")
+        console.input("Press enter to proceed...")
+        return
+
+    selected_utility = None
+    tariffs_to_include = None
+    results = []
+
+    for attempt in tenacity.Retrying(
+        stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_exception_type(WebDriverException)
+    ):
+        with attempt, create_context() as context:
+            with console.status("Fetching list of utilities..."):
+                scraping_state = (
+                    LoginState(context).login(username, password).gas().benchmark_all().select_state(state.upper())
+                )
+                utilities = [_ for _ in scraping_state.get_utilities() if _]
+
+            if not utilities:
+                raise RuntimeError(f"Something's wrong: rateacuity shows no utilities for this state ({state})")
+
+            if selected_utility is None:
+                selected_utility = questionary.select(
+                    message="Select a utility from available choices",
+                    choices=utilities,
+                    use_jk_keys=False,
+                    use_search_filter=True,
+                    use_shortcuts=False,
+                ).ask()
+                if not selected_utility:
+                    return
+
+            with console.status("Fetching list of tariffs..."):
+                scraping_state = scraping_state.select_utility(selected_utility)
+                tariffs = [_ for _ in scraping_state.get_schedules() if _]
+
+            if tariffs_to_include is None:
+                tariffs_to_include = questionary.checkbox(
+                    message="Select tariffs to include",
+                    choices=tariffs,
+                    use_jk_keys=False,
+                    use_search_filter=True,
+                    validate=lambda _: bool(_) or "Select at least one tariff",
+                ).ask()
+
+            if not tariffs_to_include:
+                console.print("[red]No tariffs selected[/]")
+                console.input("Press enter to proceed...")
+                return
+
+            with console.status("Fetching tariffs..."):
+                while tariffs_to_include:
+                    tariff = tariffs_to_include.pop(0)
+                    console.log(f"Fetching {tariff}")
+                    scraping_state = scraping_state.select_schedule(tariff)
+                    sections = scraping_state.as_sections()
+                    results.append({"schedule": tariff, "sections": sections})
+                    scraping_state = scraping_state.back_to_selections()
+
+    assert selected_utility
+    suggested_filename = f"gas_rateacuity_{selected_utility}"
+    filename = prompt_filename(output_folder, suggested_filename, "json")
+    filename.parent.mkdir(exist_ok=True)
+    filename.write_text(json.dumps(results, indent=2))
+
+
+def process_rateacuity(output_folder: Path, state: str, utility: Utility):
+    load_dotenv()
+    if not (username := os.getenv("RATEACUITY_USERNAME")):
+        console.print("[b]RATEACUITY_USERNAME[/] environment variable is not set")
+    if not (password := os.getenv("RATEACUITY_PASSWORD")):
+        console.print("[b]RATEACUITY_PASSWORD[/] environment variable is not set")
+    if not (username and password):
+        console.print("Cannot use RateAcuity due to missing credentials")
+        console.input("Press enter to proceed...")
+        return
+
+    selected_utility = None
+    tariffs_to_include = None
+    results = []
+
+    for attempt in tenacity.Retrying(
+        stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_exception_type(WebDriverException)
+    ):
+        with attempt, create_context() as context:
+            with console.status("Fetching list of utilities..."):
+                scraping_state = (
+                    LoginState(context).login(username, password).electric().benchmark_all().select_state(state.upper())
+                )
+                utilities = [_ for _ in scraping_state.get_utilities() if _]
+
+            if not utilities:
+                raise RuntimeError(f"Something's wrong: rateacuity shows no utilities for this state ({state})")
+
+            if selected_utility is None:
+                utilities_scored = sorted(utilities, key=lambda _: fuzz.ratio(utility.name, _), reverse=True)
+                selected_utility = utilities_scored.pop(0)
+                if not questionary.confirm(f"Is this the correct utility: {selected_utility} ?").ask():
+                    selected_utility = questionary.select(
+                        message="Select a utility from available choices",
+                        choices=utilities_scored,
+                        use_jk_keys=False,
+                        use_search_filter=True,
+                        use_shortcuts=False,
+                    ).ask()
+                if not selected_utility:
+                    return
+
+            with console.status("Fetching list of tariffs..."):
+                scraping_state = scraping_state.select_utility(selected_utility)
+                tariffs = [_ for _ in scraping_state.get_schedules() if _]
+
+            if tariffs_to_include is None:
+                tariffs_to_include = questionary.checkbox(
+                    message="Select tariffs to include",
+                    choices=tariffs,
+                    use_jk_keys=False,
+                    use_search_filter=True,
+                    validate=lambda _: bool(_) or "Select at least one tariff",
+                ).ask()
+
+            if not tariffs_to_include:
+                console.print("[red]No tariffs selected[/]")
+                console.input("Press enter to proceed...")
+                return
+
+            with console.status("Fetching tariffs..."):
+                while tariffs_to_include:
+                    tariff = tariffs_to_include.pop(0)
+                    console.log(f"Fetching {tariff}")
+                    scraping_state = scraping_state.select_schedule(tariff)
+                    sections = scraping_state.as_sections()
+                    results.append({"schedule": tariff, "sections": sections})
+                    scraping_state = scraping_state.back_to_selections()
+
+    assert selected_utility
+    suggested_filename = f"rateacuity_{selected_utility}"
+    if not (filename := prompt_filename(output_folder, suggested_filename, "json")):
+        return
+    filename.parent.mkdir(exist_ok=True)
+    filename.write_text(json.dumps(results, indent=2))

tariff_fetch/_cli/rateacuity_gas_urdb.py

@@ -0,0 +1,138 @@
+import json
+import os
+from collections.abc import Collection
+from datetime import date
+from pathlib import Path
+from statistics import mean
+from typing import cast
+
+import questionary
+import tenacity
+from dotenv import load_dotenv
+from rich.prompt import Confirm
+from selenium.common.exceptions import WebDriverException
+
+from tariff_fetch.rateacuity import LoginState, create_context
+from tariff_fetch.urdb.rateacuity_history_gas import (
+    build_urdb,
+)
+from tariff_fetch.urdb.rateacuity_history_gas.history_data import HistoryData, PercentageRow, Row
+from tariff_fetch.urdb.schema import URDBRate
+
+from . import console, prompt_filename
+
+# TODO: This is ungodly ugly but it works
+
+
+def process_rateacuity_gas_urdb(output_folder: Path, state: str, year: int):
+    _ = load_dotenv()
+    if not (username := os.getenv("RATEACUITY_USERNAME")):
+        console.print("[b]RATEACUITY_USERNAME[/] environment variable is not set")
+    if not (password := os.getenv("RATEACUITY_PASSWORD")):
+        console.print("[b]RATEACUITY_PASSWORD[/] environment variable is not set")
+    if not (username and password):
+        console.print("Cannot use RateAcuity due to missing credentials")
+        _ = console.input("Press enter to proceed...")
+        return
+
+    selected_utility = None
+    tariffs_to_include = None
+    result: list[URDBRate] = []
+    for attempt in tenacity.Retrying(
+        stop=tenacity.stop_after_attempt(3), retry=tenacity.retry_if_exception_type(WebDriverException)
+    ):
+        with attempt, create_context() as context:
+            with console.status("Fetching list of utilities..."):
+                scraping_state = (
+                    LoginState(context).login(username, password).gas().history().select_state(state.upper())
+                )
+                utilities = [_ for _ in scraping_state.get_utilities() if _]
+            if selected_utility is None:
+                selected_utility = cast(
+                    str,
+                    questionary.select(
+                        message="Select a utility from available choices",
+                        choices=utilities,
+                        use_jk_keys=False,
+                        use_search_filter=True,
+                        use_shortcuts=False,
+                    ).ask(),
+                )
+                if not selected_utility:
+                    return
+            with console.status("Fetching list of tariffs..."):
+                scraping_state = scraping_state.select_utility(selected_utility)
+                tariffs = [_ for _ in scraping_state.get_schedules() if _]
+            if tariffs_to_include is None:
+                tariffs_to_include = cast(
+                    list[str],
+                    questionary.checkbox(
+                        message="Select tariffs to include",
+                        choices=tariffs,
+                        use_jk_keys=False,
+                        use_search_filter=True,
+                        validate=lambda _: bool(_) or "Select at least one tariff",
+                    ).ask(),
+                )
+
+            if not tariffs_to_include:
+                console.print("[red]No tariffs selected[/]")
+                _ = console.input("Press enter to proceed...")
+                return
+            console.print("Fetching tariffs")
+            while tariffs_to_include:
+                tariff = tariffs_to_include.pop(0)
+                console.log(f"Fetching {tariff}")
+                scraping_state = (
+                    scraping_state.select_schedule(tariff)
+                    .set_enddate(date(year, 12, 1))
+                    .set_number_of_comparisons(12)
+                    .set_frequency(1)
+                )
+                df = scraping_state.as_dataframe()
+                hd = HistoryData(df)
+                validation_errors = hd.validate_rows()
+                proceed = True
+                if validation_errors:
+                    console.print("Following rows cannot be processed and will be ignored:")
+                    for error in validation_errors:
+                        console.print(f" - {error.row}")
+                    proceed = Confirm.ask("Proceed?", console=console)
+
+                if proceed:
+                    apply_percentages = False
+                    rows = list(hd.rows())
+                    if percentage_columns := _get_percentage_columns(rows):
+                        percentage_columns_strings = [
+                            f"- {c[0]} ({c[1]}): {c[2]}" if c[1] else f"- {c[0]}: {c[2]}" for c in percentage_columns
+                        ]
+                        console.print("Found following percentage columns (values are averages over 12 months):")
+                        console.print("\n".join(percentage_columns_strings))
+                        console.print("It is impossible to tell which percentages apply to which specific rates.")
+                        console.print("Percentages will be applied to the final result as is")
+                        apply_percentages = Confirm.ask("Apply percentages? (otherwise percentages will be ignored)")
+
+                    urdb = build_urdb(rows, apply_percentages)
+                    urdb["utility"] = selected_utility
+                    urdb["name"] = tariff
+                    result.append(urdb)
+
+                scraping_state = (
+                    scraping_state.back_to_selections()
+                    .history()
+                    .select_state(state.upper())
+                    .select_utility(selected_utility)
+                )
+    suggested_filename = f"rateacuity_{selected_utility}.urdb.{year}."
+    if not (filename := prompt_filename(output_folder, suggested_filename, "json")):
+        return
+    filename.parent.mkdir(exist_ok=True)
+    _ = filename.write_text(json.dumps(result, indent=2))
+
+
+def _get_percentage_columns(rows: Collection[Row]) -> list[tuple[str, str | None, float]]:
+    return [
+        (row.rate, row.location, mean(row.month_value_float(month) for month in range(0, 12)))
+        for row in rows
+        if isinstance(row, PercentageRow)
+    ]

tariff_fetch/_cli/types.py

@@ -0,0 +1,67 @@
+from enum import Enum
+from typing import NamedTuple
+
+
+class Provider(str, Enum):
+    GENABILITY = "Genability"
+    OPENEI = "OpenEI"
+    RATEACUITY = "RateAcuity"
+
+
+class StateCode(str, Enum):
+    AL = "al"
+    AK = "ak"
+    AZ = "az"
+    AR = "ar"
+    CA = "ca"
+    CO = "co"
+    CT = "ct"
+    DE = "de"
+    FL = "fl"
+    GA = "ga"
+    HI = "hi"
+    ID = "id"
+    IL = "il"
+    IN = "in"
+    IA = "ia"
+    KS = "ks"
+    KY = "ky"
+    LA = "la"
+    ME = "me"
+    MD = "md"
+    MA = "ma"
+    MI = "mi"
+    MN = "mn"
+    MS = "ms"
+    MO = "mo"
+    MT = "mt"
+    NE = "ne"
+    NV = "nv"
+    NH = "nh"
+    NJ = "nj"
+    NM = "nm"
+    NY = "ny"
+    NC = "nc"
+    ND = "nd"
+    OH = "oh"
+    OK = "ok"
+    OR = "or"
+    PA = "pa"
+    RI = "ri"
+    SC = "sc"
+    SD = "sd"
+    TN = "tn"
+    TX = "tx"
+    UT = "ut"
+    VT = "vt"
+    VA = "va"
+    WA = "wa"
+    WV = "wv"
+    WI = "wi"
+    WY = "wy"
+    DC = "dc"
+
+
+class Utility(NamedTuple):
+    eia_id: int
+    name: str