chellow 1704969359.0.0__tar.gz → 1705585731.0.0__tar.gz

{chellow-1704969359.0.0 → chellow-1705585731.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1704969359.0.0
+Version: 1705585731.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1705585731.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1705585731.0.0"

{chellow-1704969359.0.0 → chellow-1705585731.0.0}/chellow/__init__.py

@@ -54,6 +54,7 @@ def get_importer_modules():
         chellow.e.bsuos,
         chellow.e.system_price,
         chellow.e.hh_importer,
+        chellow.e.lcc,
         chellow.testing,
         chellow.e.tlms,
         chellow.bank_holidays,

chellow-1705585731.0.0/chellow/e/cfd.py

@@ -0,0 +1,305 @@
+from datetime import datetime as Datetime
+
+from dateutil.relativedelta import relativedelta
+from sqlalchemy import null, or_, select
+
+from chellow.e.lcc import api_search, api_sql
+from chellow.models import Contract, RateScript
+from chellow.utils import ct_datetime, hh_format, to_ct, to_utc
+
+
+def hh(data_source):
+    try:
+        cfd_cache = data_source.caches["cfd"]
+    except KeyError:
+        cfd_cache = data_source.caches["cfd"] = {}
+
+    for h in data_source.hh_data:
+        try:
+            h["cfd-rate"] = cfd_cache[h["start-date"]]
+        except KeyError:
+            h_start = h["start-date"]
+            daily_contract = Contract.get_non_core_by_name(
+                data_source.sess, "cfd_reconciled_daily_levy_rates"
+            )
+            daily_rs = data_source.sess.scalars(
+                select(RateScript).where(
+                    RateScript.contract == daily_contract,
+                    RateScript.start_date <= h_start,
+                    or_(
+                        RateScript.finish_date == null(),
+                        RateScript.finish_date >= h_start,
+                    ),
+                )
+            ).one_or_none()
+            if daily_rs.finish_date is None and h_start >= to_utc(
+                to_ct(daily_rs.start_date) + relativedelta(months=3)
+            ):
+                base_rate = data_source.non_core_rate(
+                    "cfd_in_period_tracking", h_start
+                )["rate_gbp_per_kwh"]
+            else:
+                base_rate = data_source.non_core_rate(
+                    "cfd_reconciled_daily_levy_rates", h_start
+                )["rate_gbp_per_kwh"]
+
+            effective_ocl_rate = data_source.non_core_rate(
+                "cfd_operational_costs_levy", h["start-date"]
+            )["record"]["Effective_OCL_Rate_GBP_Per_MWh"]
+            if effective_ocl_rate == "":
+                levy_rate_str = data_source.non_core_rate(
+                    "cfd_operational_costs_levy", h["start-date"]
+                )["record"]["OCL_Rate_GBP_Per_MWh"]
+            else:
+                levy_rate_str = effective_ocl_rate
+            levy_rate = float(levy_rate_str) / 1000
+            h["cfd-rate"] = cfd_cache[h["start-date"]] = float(base_rate) + levy_rate
+
+
+def lcc_import(sess, log, set_progress, s):
+    import_in_period_tracking(sess, log, set_progress, s)
+    import_operational_costs_levy(sess, log, set_progress, s)
+    import_reconciled_daily_levy_rates(sess, log, set_progress, s)
+
+
+def _quarters(s):
+    quarter = {}
+    res_j = api_search(
+        s, "2fc2fad9-ad57-4901-982a-f92d4ef6c622", sort="Settlement_Date"
+    )
+    for record in res_j["result"]["records"]:
+        settlement_date_str = record["Settlement_Date"]
+        settlement_date_ct = to_ct(
+            Datetime.strptime(settlement_date_str[:10], "%Y-%m-%d")
+        )
+        settlement_date = to_utc(settlement_date_ct)
+
+        quarter[settlement_date] = record
+        settlement_next_ct = settlement_date_ct + relativedelta(days=1)
+
+        if settlement_next_ct.month in (1, 4, 7, 10) and settlement_next_ct.day == 1:
+            yield quarter
+            quarter = {}
+
+
+def _parse_number(num_str):
+    if num_str == "":
+        return 0
+    else:
+        return float(num_str)
+
+
+def _parse_date(date_str):
+    return to_utc(to_ct(Datetime.strptime(date_str[:10], "%Y-%m-%d")))
+
+
+def import_in_period_tracking(sess, log, set_progress, s):
+    log("Starting to check for new LCC CfD In-Period Tracking")
+
+    contract_name = "cfd_in_period_tracking"
+    contract = Contract.find_non_core_by_name(sess, contract_name)
+    if contract is None:
+        contract = Contract.insert_non_core(
+            sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
+        )
+
+    for quarter in _quarters(s):
+        quarter_start = sorted(quarter.keys())[0]
+        rs = sess.execute(
+            select(RateScript).where(
+                RateScript.contract == contract,
+                RateScript.start_date == quarter_start,
+            )
+        ).scalar_one_or_none()
+        if rs is None:
+            rs = contract.insert_rate_script(sess, quarter_start, {"records": {}})
+
+        gbp = 0
+        kwh = 0
+        for record in quarter.values():
+            gbp += _parse_number(record["Actual_CFD_Payments_GBP"])
+            gbp += _parse_number(record["Expected_CFD_Payments_GBP"])
+            kwh += _parse_number(record["Actual_Eligible_Demand_MWh"]) * 1000
+            kwh += _parse_number(record["Expected_Eligible_Demand_MWh"]) * 1000
+
+        rate = gbp / kwh
+
+        rs_script = rs.make_script()
+        records = rs_script["records"]
+        for k, v in sorted(quarter.items()):
+            records[hh_format(k)] = v
+        rs_script["rate_gbp_per_kwh"] = rate
+        rs.update(rs_script)
+        sess.commit()
+    log("Finished LCC CfD In-Period Tracking")
+
+
+def import_operational_costs_levy(sess, log, set_progress, s):
+    log("Starting to check for new LCC CfD Operational Costs Levy")
+
+    contract_name = "cfd_operational_costs_levy"
+    contract = Contract.find_non_core_by_name(sess, contract_name)
+    if contract is None:
+        contract = Contract.insert_non_core(
+            sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
+        )
+
+    res_j = api_search(s, "44f41eac-61b3-4e8d-8c52-3eda7b8e8517", sort="Period_Start")
+    for record in res_j["result"]["records"][1:]:  # skip title row
+        period_start_str = record["Period_Start"]
+        period_start = to_utc(
+            to_ct(Datetime.strptime(period_start_str[:10], "%Y-%m-%d"))
+        )
+
+        rs = sess.execute(
+            select(RateScript).where(
+                RateScript.contract == contract,
+                RateScript.start_date == period_start,
+            )
+        ).scalar_one_or_none()
+        if rs is None:
+            rs = contract.insert_rate_script(sess, period_start, {})
+
+        rs_script = rs.make_script()
+        rs_script["record"] = record
+        rs.update(rs_script)
+        sess.commit()
+
+
+RUN_TYPES = ("II", "SF", "R1", "R2", "R3", "RF", "DF")
+
+
+def _reconciled_days(s, search_from):
+    runs = {}
+    res_j = api_sql(
+        s,
+        f"""SELECT * from "e0e163cb-ba36-416d-83fe-976992d61516"
+        WHERE "Settlement_Date" > '{search_from}' ORDER BY "Settlement_Date"
+        """,
+    )
+
+    records = res_j["result"]["records"]
+    if len(records) > 0:
+        prev_settlement_date = _parse_date(records[0]["Settlement_Date"])
+        runs = {}
+        for record in records:
+            settlement_date = _parse_date(record["Settlement_Date"])
+            if settlement_date != prev_settlement_date:
+                yield prev_settlement_date, runs
+                runs = {}
+
+            settlement_run_type = record["Settlement_Run_Type"]
+            runs[settlement_run_type] = record
+            prev_settlement_date = settlement_date
+
+        yield settlement_date, runs
+
+
+def _reconciled_daily_quarters(s, search_from):
+    quarter = {}
+    for settlement_date, lcc_runs in _reconciled_days(s, search_from):
+        try:
+            runs = quarter[settlement_date]
+        except KeyError:
+            runs = quarter[settlement_date] = {}
+
+        for run_type, record in lcc_runs.items():
+            runs[run_type] = record
+
+        settlement_next_ct = to_ct(settlement_date) + relativedelta(days=1)
+
+        if settlement_next_ct.month in (1, 4, 7, 10) and settlement_next_ct.day == 1:
+            yield quarter
+            quarter = {}
+
+
+def import_reconciled_daily_levy_rates(sess, log, set_progress, s):
+    log("Starting to check for new LCC CfD Reconciled Daily Levy Rates")
+
+    contract_name = "cfd_reconciled_daily_levy_rates"
+    contract = Contract.find_non_core_by_name(sess, contract_name)
+    if contract is None:
+        contract = Contract.insert_non_core(
+            sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
+        )
+
+    search_from = "1996-04-01"
+    for rs in sess.scalars(
+        select(RateScript)
+        .where(RateScript.contract == contract)
+        .order_by(RateScript.start_date.desc())
+    ):
+        script = rs.make_script()
+        try:
+            records = script["records"]
+        except KeyError:
+            break
+
+        complete = True
+        rs_start_ct = to_ct(rs.start_date)
+        quarter_finish_ct = (
+            rs_start_ct + relativedelta(months=3) - relativedelta(days=1)
+        )
+        day_ct = rs_start_ct
+        while day_ct <= quarter_finish_ct:
+            try:
+                records[hh_format(to_utc(day_ct))]["DF"]
+            except KeyError:
+                complete = False
+                break
+
+            day_ct += relativedelta(days=1)
+
+        if complete:
+            search_from = quarter_finish_ct.strftime("%Y-%m-%d")
+        else:
+            break
+
+    for quarter in _reconciled_daily_quarters(s, search_from):
+        quarter_start = sorted(quarter.keys())[0]
+        rs = sess.execute(
+            select(RateScript).where(
+                RateScript.contract == contract,
+                RateScript.start_date == quarter_start,
+            )
+        ).scalar_one_or_none()
+        if rs is None:
+            rs = contract.insert_rate_script(sess, quarter_start, {"records": {}})
+
+        gbp = 0
+        kwh = 0
+        run_types_reverse = list(reversed(RUN_TYPES))
+        for runs in quarter.values():
+            top_run = None
+            for run_type in run_types_reverse:
+                if run_type in runs:
+                    top_run = runs[run_type]
+                    break
+
+            eligible_mwh = _parse_number(top_run["Reconciled_Eligible_Demand_MWh"])
+            gbp += (
+                _parse_number(top_run["Reconciled_Daily_Levy_Rate_GBP_Per_MWh"])
+                * eligible_mwh
+            )
+            kwh += eligible_mwh * 1000
+
+        rate = gbp / kwh
+
+        rs_script = rs.make_script()
+        records = rs_script["records"]
+        for dt, runs in sorted(quarter.items()):
+            date_str = hh_format(dt)
+            try:
+                rs_runs = records[date_str]
+            except KeyError:
+                rs_runs = records[date_str] = {}
+
+            for run_type, record in runs.items():
+                rs_runs[run_type] = record
+
+        rs_script["rate_gbp_per_kwh"] = rate
+        rs.update(rs_script)
+        sess.commit()
+
+    log("Finished LCC CfD Reconciled Daily Levy Rates")
+    sess.commit()

chellow-1705585731.0.0/chellow/e/lcc.py

@@ -0,0 +1,164 @@
+import atexit
+import collections
+import threading
+import traceback
+from datetime import timedelta
+from importlib import import_module
+
+import requests
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.models import (
+    Contract,
+    Session,
+)
+from chellow.utils import ct_datetime_now, hh_format, utc_datetime_now
+
+
+importer = None
+
+
+def api_sql(s, sql):
+    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search_sql"
+    params = {"sql": sql}
+    res = s.get(url, params=params)
+    try:
+        res_j = res.json()
+    except requests.exceptions.JSONDecodeError as e:
+        raise BadRequest(
+            f"Couldn't parse as JSON the content from {url} with error {e}: "
+            f"{res.text}"
+        )
+
+    if "success" in res_j and not res_j["success"]:
+        raise BadRequest(res_j)
+
+    return res_j
+
+
+def api_search(s, resource_id, sort=None):
+    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search"
+    params = {"resource_id": resource_id}
+    if sort is not None:
+        params["sort"] = sort
+    res = s.get(url, params=params)
+    try:
+        res_j = res.json()
+    except requests.exceptions.JSONDecodeError as e:
+        raise BadRequest(
+            f"Couldn't parse as JSON the content from {url} with error {e}: "
+            f"{res.text}"
+        )
+
+    if "success" in res_j and not res_j["success"]:
+        raise BadRequest(res_j)
+
+    return res_j
+
+
+def run_import(sess, log, set_progress):
+    log("Starting to import data from the Low Carbon Contracts Company")
+    s = requests.Session()
+    s.verify = False
+
+    for mod_name in ("chellow.e.cfd",):
+        mod = import_module(mod_name)
+        mod.lcc_import(sess, log, set_progress, s)
+
+
+LAST_RUN_KEY = "last_run"
+GLOBAL_ALERT = (
+    "There's a problem with a <a href='/lcc'>Low Carbon Contracts import</a>."
+)
+LCC_STATE_KEY = "lcc"
+
+
+class LowCarbonContracts(threading.Thread):
+    def __init__(self):
+        super().__init__(name="Low Carbon Contracts")
+        self.messages = collections.deque(maxlen=500)
+        self.progress = ""
+        self.stopped = threading.Event()
+        self.going = threading.Event()
+        self.global_alert = None
+
+    def stop(self):
+        self.stopped.set()
+        self.going.set()
+        self.join()
+
+    def go(self):
+        self.going.set()
+
+    def log(self, message):
+        self.messages.appendleft(
+            f"{ct_datetime_now().strftime('%Y-%m-%d %H:%M:%S')} - {message}"
+        )
+
+    def set_progress(self, progress):
+        self.progress = progress
+
+    def run(self):
+        while not self.stopped.is_set():
+            with Session() as sess:
+                try:
+                    config = Contract.get_non_core_by_name(sess, "configuration")
+                    state = config.make_state()
+                    lcc_state = state.get(LCC_STATE_KEY, {})
+                except BaseException as e:
+                    msg = f"{e.description} " if isinstance(e, BadRequest) else ""
+                    self.log(f"{msg}{traceback.format_exc()}")
+                    self.global_alert = GLOBAL_ALERT
+                    sess.rollback()
+
+            last_run = lcc_state.get(LAST_RUN_KEY)
+            if last_run is None or utc_datetime_now() - last_run > timedelta(days=1):
+                self.going.set()
+
+            if self.going.is_set():
+                self.global_alert = None
+                with Session() as sess:
+                    try:
+                        config = Contract.get_non_core_by_name(sess, "configuration")
+                        state = config.make_state()
+                        try:
+                            lcc_state = state[LCC_STATE_KEY]
+                        except KeyError:
+                            lcc_state = state[LCC_STATE_KEY] = {}
+
+                        lcc_state[LAST_RUN_KEY] = utc_datetime_now()
+                        config.update_state(state)
+                        sess.commit()
+                        run_import(sess, self.log, self.set_progress)
+                    except BaseException as e:
+                        msg = f"{e.description} " if isinstance(e, BadRequest) else ""
+                        self.log(f"{msg}{traceback.format_exc()}")
+                        self.global_alert = GLOBAL_ALERT
+                        sess.rollback()
+                    finally:
+                        self.going.clear()
+                        self.log("Finished importing Low Carbon Contracts data.")
+
+            else:
+                self.log(
+                    f"The importer was last run at {hh_format(last_run)}. There will "
+                    f"be another import when 24 hours have elapsed since the last run."
+                )
+                self.going.wait(60 * 60)
+
+
+def get_importer():
+    return importer
+
+
+def startup():
+    global importer
+    importer = LowCarbonContracts()
+    importer.start()
+
+
+@atexit.register
+def shutdown():
+    if importer is not None:
+        importer.stop()

{chellow-1704969359.0.0 → chellow-1705585731.0.0}/chellow/e/views.py

@@ -29,6 +29,7 @@ from werkzeug.exceptions import BadRequest
 from zish import dumps, loads
 
 import chellow.e.dno_rate_parser
+import chellow.e.lcc
 from chellow.e.computer import SupplySource, contract_func, forecast_date
 from chellow.e.energy_management import totals_runner
 from chellow.models import (
@@ -2330,6 +2331,23 @@ def lafs_get():
     return render_template("lafs.html", dno=dno, llfc=llfc, lafs=lafs)
 
 
+@e.route("/lcc")
+def lcc_get():
+    importer = chellow.e.lcc.importer
+
+    return render_template(
+        "lcc.html",
+        importer=importer,
+    )
+
+
+@e.route("/lcc", methods=["POST"])
+def lcc_post():
+    importer = chellow.e.lcc.importer
+    importer.go()
+    return chellow_redirect("/lcc", 303)
+
+
 @e.route("/llfcs")
 def llfcs_get():
     dno_id = req_int("dno_id")

chellow-1705585731.0.0/chellow/templates/e/lcc.html

@@ -0,0 +1,55 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» Low Carbon Contracts Importer
+{% endblock %}
+
+{% block nav %}
+	Low Carbon Contracts Importer
+{% endblock %}
+
+{% block content %}
+	{% if importer %}
+		<ul>
+			<li><a href="#controls">Controls</a></li>
+			<li><a href="#log">Log</a></li>
+		</ul>
+
+		<h4 id="controls">Controls</h4>
+
+		<table>
+			<tr>
+				<th>Is Going?</th>
+				<td>{{importer.going.is_set()}}</td>
+			</tr>
+			<tr>
+				<th>Import Now</th>
+				<td>
+					<form method="post" action="/e/lcc">
+						<fieldset {% if importer.going.is_set() %}disabled{% endif %}>
+							<input type="submit" value="Import" name="import">
+						</fieldset>
+					</form>
+				</td>
+			</tr>
+			{% if importer.going.is_set() %}
+				<tr>
+					<th>Progress</th>
+					<td>{{importer.progress}}</td>
+				</tr>
+			{% endif %}
+		</table>
+
+
+		<h4 id="log">Log</h4>
+
+		<ul>
+			{% for message in importer.messages %}
+				<li>{{message}}</li>
+			{% endfor %}
+		</ul>
+
+	{% else %}
+		<p>Importer not present.</p>
+	{% endif %}
+{% endblock %}

{chellow-1704969359.0.0 → chellow-1705585731.0.0}/chellow/templates/home.html

@@ -213,6 +213,7 @@
 					<li><a href="/api/v1" >API</a></li>
 					<li><a href="/rate_server">Automatic Importer: Rate Server</a></li>
 					<li><a href="/national_grid">Automatic Importer: National Grid</a></li>
+					<li><a href="/e/lcc">Automatic Importer: Low Carbon Contracts</a></li>
 				</ul>
 			
 				<ul>

{chellow-1704969359.0.0 → chellow-1705585731.0.0}/chellow/views.py

@@ -61,6 +61,7 @@ import chellow.e.bsuos
 import chellow.e.computer
 import chellow.e.hh_importer
 import chellow.e.laf_import
+import chellow.e.lcc
 import chellow.e.mdd_importer
 import chellow.e.rcrc
 import chellow.e.system_price