chellow 1744710468.0.0__tar.gz → 1744720514.0.0__tar.gz

{chellow-1744710468.0.0 → chellow-1744720514.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1744710468.0.0
+Version: 1744720514.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1744720514.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1744720514.0.0"

{chellow-1744710468.0.0 → chellow-1744720514.0.0}/chellow/__init__.py

@@ -22,6 +22,7 @@ import chellow.e.hh_importer
 import chellow.e.rcrc
 import chellow.e.system_price
 import chellow.e.views
+import chellow.fake_batch_updater
 import chellow.gas.cv
 import chellow.gas.views
 import chellow.national_grid
@@ -62,6 +63,7 @@ def get_importer_modules():
         chellow.national_grid,
         chellow.rate_server,
         chellow.rrun,
+        chellow.fake_batch_updater,
     )
 
 

chellow-1744720514.0.0/chellow/fake_batch_updater.py

@@ -0,0 +1,305 @@
+import atexit
+import collections
+import threading
+import traceback
+from datetime import timedelta
+
+from sqlalchemy import false, null, select
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.e.computer import contract_func
+from chellow.gas.engine import g_contract_func
+from chellow.models import (
+    Batch,
+    BillType,
+    Contract,
+    GBatch,
+    GBill,
+    GContract,
+    GRateScript,
+    GReadType,
+    GSupply,
+    GUnit,
+    RateScript,
+    ReadType,
+    Session,
+    Supply,
+    Tpr,
+)
+from chellow.utils import (
+    c_months_u,
+    ct_datetime_now,
+    hh_format,
+    keydefaultdict,
+    utc_datetime_now,
+)
+
+
+importer = None
+
+
+def run_import(sess, log, set_progress):
+    log("Starting to update the fake batches")
+    caches = {}
+
+    now_ct = ct_datetime_now()
+    (last_month_start, last_month_finish), (
+        current_month_start,
+        current_month_finish,
+    ) = list(c_months_u(finish_year=now_ct.year, finish_month=now_ct.month, months=2))
+
+    for last_rate_script in sess.scalars(
+        select(RateScript)
+        .join(Contract, RateScript.contract_id == Contract.id)
+        .where(RateScript.finish_date == null())
+    ):
+        contract = last_rate_script.contract
+        fb_func = contract_func(caches, contract, "make_fake_bills")
+        if fb_func is None:
+            continue
+
+        fake_batch_name = f"fake_e_batch_{contract.id}"
+
+        fake_batch = sess.scalar_one_or_none(
+            select(Batch).where(
+                Batch.contract == contract,
+                Batch.reference == fake_batch_name,
+            )
+        )
+
+        if fake_batch is not None and fake_batch.start_date < current_month_start:
+            fake_batch.delete(sess)
+            sess.flush()
+            fake_batch = None
+
+        if fake_batch is None:
+            fake_batch = contract.insert_batch(sess, fake_batch_name, "Fake Batch")
+            bill_types = keydefaultdict(lambda k: BillType.get_by_code(sess, k))
+
+            tprs = keydefaultdict(
+                lambda k: None if k is None else Tpr.get_by_code(sess, k)
+            )
+
+            read_types = keydefaultdict(lambda k: ReadType.get_by_code(sess, k))
+            for raw_bill in fb_func(
+                sess,
+                log,
+                last_month_start,
+                last_month_finish,
+                current_month_start,
+                current_month_finish,
+            ):
+                mpan_core = raw_bill["mpan_core"]
+                supply = Supply.get_by_mpan_core(sess, mpan_core)
+                bill = fake_batch.insert_bill(
+                    sess,
+                    raw_bill["account"],
+                    raw_bill["reference"],
+                    raw_bill["issue_date"],
+                    raw_bill["start_date"],
+                    raw_bill["finish_date"],
+                    raw_bill["kwh"],
+                    raw_bill["net"],
+                    raw_bill["vat"],
+                    raw_bill["gross"],
+                    bill_types[raw_bill["bill_type_code"]],
+                    raw_bill["breakdown"],
+                    supply,
+                )
+                for raw_read in raw_bill["reads"]:
+                    bill.insert_read(
+                        sess,
+                        tprs[raw_read["tpr_code"]],
+                        raw_read["coefficient"],
+                        raw_read["units"],
+                        raw_read["msn"],
+                        raw_read["mpan"],
+                        raw_read["prev_date"],
+                        raw_read["prev_value"],
+                        read_types[raw_read["prev_type_code"]],
+                        raw_read["pres_date"],
+                        raw_read["pres_value"],
+                        read_types[raw_read["pres_type_code"]],
+                    )
+    for last_rate_script in sess.scalars(
+        select(GRateScript)
+        .join(GContract, GRateScript.g_contract_id == GContract.id)
+        .where(GContract.is_industry == false(), GRateScript.finish_date == null())
+    ):
+        g_contract = last_rate_script.g_contract
+        log(f"Looking at gas contract {g_contract.name}")
+        fb_func = g_contract_func(caches, g_contract, "make_fake_bills")
+        if fb_func is None:
+            log("Doesn't have a make_fake_bills function so skipping")
+            continue
+
+        fake_batch_name = f"fake_g_batch_{g_contract.id}"
+
+        fake_batch = sess.scalars(
+            select(GBatch).where(
+                GBatch.g_contract == g_contract,
+                GBatch.reference == fake_batch_name,
+            )
+        ).one_or_none()
+
+        if fake_batch is not None:
+            first_fake_bill = sess.scalars(
+                select(GBill)
+                .where(GBill.g_batch == fake_batch)
+                .order_by(GBill.start_date)
+            ).first()
+            if (
+                first_fake_bill is None
+                or first_fake_bill.start_date < current_month_start
+            ):
+                fake_batch.delete(sess)
+                sess.flush()
+                fake_batch = None
+
+        if fake_batch is None:
+            fake_batch = g_contract.insert_g_batch(sess, fake_batch_name, "Fake Batch")
+            raw_bills = fb_func(
+                sess,
+                log,
+                last_month_start,
+                last_month_finish,
+                current_month_start,
+                current_month_finish,
+            )
+            if raw_bills is not None and len(raw_bills) > 0:
+                for raw_bill in raw_bills:
+                    bill_type = BillType.get_by_code(sess, raw_bill["bill_type_code"])
+                    g_supply = GSupply.get_by_mprn(sess, raw_bill["mprn"])
+                    g_bill = fake_batch.insert_g_bill(
+                        sess,
+                        g_supply,
+                        bill_type,
+                        raw_bill["reference"],
+                        raw_bill["account"],
+                        raw_bill["issue_date"],
+                        raw_bill["start_date"],
+                        raw_bill["finish_date"],
+                        raw_bill["kwh"],
+                        raw_bill["net_gbp"],
+                        raw_bill["vat_gbp"],
+                        raw_bill["gross_gbp"],
+                        raw_bill["raw_lines"],
+                        raw_bill["breakdown"],
+                    )
+                    sess.flush()
+                    for raw_read in raw_bill["reads"]:
+                        prev_type = GReadType.get_by_code(
+                            sess, raw_read["prev_type_code"]
+                        )
+                        pres_type = GReadType.get_by_code(
+                            sess, raw_read["pres_type_code"]
+                        )
+                        g_unit = GUnit.get_by_code(sess, raw_read["unit"])
+                        g_bill.insert_g_read(
+                            sess,
+                            raw_read["msn"],
+                            g_unit,
+                            raw_read["correction_factor"],
+                            raw_read["calorific_value"],
+                            raw_read["prev_value"],
+                            raw_read["prev_date"],
+                            prev_type,
+                            raw_read["pres_value"],
+                            raw_read["pres_date"],
+                            pres_type,
+                        )
+                    sess.commit()
+
+
+LAST_RUN_KEY = "fake_batch_updater_last_run"
+
+
+class FakeBatchUpdater(threading.Thread):
+    def __init__(self):
+        super().__init__(name="Fake Batch Updater")
+        self.messages = collections.deque(maxlen=500)
+        self.progress = ""
+        self.stopped = threading.Event()
+        self.going = threading.Event()
+        self.global_alert = None
+
+    def stop(self):
+        self.stopped.set()
+        self.going.set()
+        self.join()
+
+    def go(self):
+        self.going.set()
+
+    def log(self, message):
+        self.messages.appendleft(
+            f"{ct_datetime_now().strftime('%Y-%m-%d %H:%M:%S')} - {message}"
+        )
+
+    def set_progress(self, progress):
+        self.progress = progress
+
+    def run(self):
+        while not self.stopped.is_set():
+            with Session() as sess:
+                try:
+                    config = Contract.get_non_core_by_name(sess, "configuration")
+                    state = config.make_state()
+                except BaseException as e:
+                    msg = f"{e.description} " if isinstance(e, BadRequest) else ""
+                    self.log(f"{msg}{traceback.format_exc()}")
+                    self.global_alert = (
+                        "There's a problem with a <a href='/fake_batch_updater'>"
+                        "Fake Batch Updater</a>."
+                    )
+                    sess.rollback()
+
+            last_run = state.get(LAST_RUN_KEY)
+            if last_run is None or utc_datetime_now() - last_run > timedelta(days=1):
+                self.going.set()
+
+            if self.going.is_set():
+                self.global_alert = None
+                with Session() as sess:
+                    try:
+                        config = Contract.get_non_core_by_name(sess, "configuration")
+                        state = config.make_state()
+                        state[LAST_RUN_KEY] = utc_datetime_now()
+                        config.update_state(state)
+                        sess.commit()
+                        run_import(sess, self.log, self.set_progress)
+                    except BaseException as e:
+                        msg = f"{e.description} " if isinstance(e, BadRequest) else ""
+                        self.log(f"{msg}{traceback.format_exc()}")
+                        self.global_alert = (
+                            "There's a problem with a "
+                            "<a href='/fake_batch_updater'>Fake Batch Updater</a>."
+                        )
+                        sess.rollback()
+                    finally:
+                        self.going.clear()
+                        self.log("Finished updating fake batches.")
+
+            else:
+                self.log(
+                    f"The updater was last run at {hh_format(last_run)}. There will "
+                    f"be another update when a hour has elapsed since the last run."
+                )
+                self.going.wait(60 * 60)
+
+
+def get_importer():
+    return importer
+
+
+def startup():
+    global importer
+    importer = FakeBatchUpdater()
+    importer.start()
+
+
+@atexit.register
+def shutdown():
+    if importer is not None:
+        importer.stop()

chellow-1744720514.0.0/chellow/templates/fake_batch_updater.html

@@ -0,0 +1,99 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» Fake Batch Updater
+{% endblock %}
+
+{% block nav %}
+	Fake Batch Updater
+{% endblock %}
+
+{% block content %}
+	<p>
+	{% if importer %}
+		<ul>
+			<li><a href="#controls">Controls</a></li>
+			<li><a href="#log">Log</a></li>
+			<li><a href="#batches">Fake Batches</a></li>
+		</ul>
+
+		<h4 id="controls">Controls</h4>
+
+		<table>
+			<tr>
+				<th>Is Going?</th>
+				<td>{{importer.going.is_set()}}</td>
+			</tr>
+			<tr>
+				<th>Import Now</th>
+				<td>
+					<form method="post">
+						<fieldset {% if importer.going.is_set() %}disabled{% endif %}>
+							<input type="submit" value="Import" name="import">
+						</fieldset>
+					</form>
+				</td>
+			</tr>
+			{% if importer.going.is_set() %}
+				<tr>
+					<th>Progress</th>
+					<td>{{importer.progress}}</td>
+				</tr>
+			{% endif %}
+		</table>
+
+
+		<h4 id="log">Log</h4>
+
+		<ul>
+			{% for message in importer.messages %}
+				<li>{{message}}</li>
+			{% endfor %}
+		</ul>
+
+		<h4 id="batches">Fake Batches</h4>
+
+		<table>
+			<caption>Electricity</caption>
+			<thead>
+				<tr>
+					<th>View</th>
+					<th>Reference</th>
+					<th>Contract Name</th>
+				</tr>
+			</thead>
+			<tbody>
+				{% for batch in e_fake_batches %}
+					<tr>
+						<th><a href="/e/supplier_batches/{{batch.id}}">View</a></th>
+						<td>{{ batch.reference }}</td>
+						<td>{{ batch.contract.name }}</td>
+					</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+
+		<table>
+			<caption>Gas</caption>
+			<thead>
+				<tr>
+					<th>View</th>
+					<th>Reference</th>
+					<th>Contract Name</th>
+				</tr>
+			</thead>
+			<tbody>
+				{% for batch in g_fake_batches %}
+					<tr>
+						<th><a href="/g/batches/{{batch.id}}">View</a></th>
+						<td>{{ batch.reference }}</td>
+						<td>{{ batch.g_contract.name }}</td>
+					</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+
+	{% else %}
+		<p>Importer not present.</p>
+	{% endif %}
+{% endblock %}

{chellow-1744710468.0.0 → chellow-1744720514.0.0}/chellow/templates/home.html

@@ -214,6 +214,7 @@
 					<li><a href="/national_grid">Automatic Importer: National Grid</a></li>
 					<li><a href="/e/lcc">Automatic Importer: Low Carbon Contracts</a></li>
 					<li><a href="/e/elexon">Automatic Importer: Elexon</a></li>
+					<li><a href="/fake_batch_updater">Automatic Fake Batch Updater</a></li>
 				</ul>
 			
 				<ul>

{chellow-1744710468.0.0 → chellow-1744720514.0.0}/chellow/views.py

@@ -72,6 +72,7 @@ import chellow.national_grid
 import chellow.rate_server
 from chellow.edi_lib import SEGMENTS, parse_edi
 from chellow.models import (
+    Batch,
     BillType,
     Channel,
     Comm,
@@ -79,6 +80,7 @@ from chellow.models import (
     Cop,
     EnergisationStatus,
     Era,
+    GBatch,
     GContract,
     GEra,
     GExitZone,
@@ -156,6 +158,41 @@ def configuration():
     return redirect(f"/non_core_contracts/{config.id}")
 
 
+@home.route("/fake_batch_updater")
+def fake_batch_updater_get():
+    importer = chellow.fake_batch_updater.importer
+    config = Contract.get_non_core_by_name(g.sess, "configuration")
+    props = config.make_properties()
+
+    e_fake_batches = g.sess.scalars(
+        select(Batch).where(
+            Batch.reference.startswith("e_fake_batch_"),
+        )
+    )
+
+    g_fake_batches = g.sess.scalars(
+        select(GBatch).where(
+            GBatch.reference.startswith("g_fake_batch_"),
+        )
+    )
+
+    return render_template(
+        "fake_batch_updater.html",
+        importer=importer,
+        config_state=config.make_state(),
+        config_properties=props.get("fake_batch_updater", {}),
+        e_fake_batches=e_fake_batches,
+        g_fake_batches=g_fake_batches,
+    )
+
+
+@home.route("/fake_batch_updater", methods=["POST"])
+def fake_batch_updater_post():
+    importer = chellow.fake_batch_updater.importer
+    importer.go()
+    return redirect("/fake_batch_updater", 303)
+
+
 @home.route("/health")
 def health():
     return Response("healthy\n", mimetype="text/plain")

chellow-1744720514.0.0/test/test_fake_batch_updater.py

@@ -0,0 +1,74 @@
+from decimal import Decimal
+
+from chellow.fake_batch_updater import run_import
+from chellow.models import Contract, GContract, MarketRole, Participant
+from chellow.utils import ct_datetime, to_utc, utc_datetime
+
+
+def test_run_import_existing_batch_in_current_month(mocker, sess):
+    vf = to_utc(ct_datetime(1996, 4, 1))
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    Contract.insert_non_core(sess, "configuration", "", {}, vf, None, {})
+    g_cv_rate_script = {
+        "cvs": {
+            "EA": {
+                1: {"applicable_at": utc_datetime(2020, 10, 3), "cv": 39.2000},
+            }
+        }
+    }
+    GContract.insert_industry(sess, "cv", "", {}, vf, None, g_cv_rate_script)
+    ug_rate_script = {
+        "ug_gbp_per_kwh": {"EA1": Decimal("40.1")},
+    }
+    GContract.insert_industry(sess, "ug", "", {}, vf, None, ug_rate_script)
+    ccl_rate_script = {"ccl_gbp_per_kwh": Decimal("0.00339")}
+    GContract.insert_industry(sess, "ccl", "", {}, vf, None, ccl_rate_script)
+    bank_holiday_rate_script = {"bank_holidays": []}
+    Contract.insert_non_core(
+        sess,
+        "bank_holidays",
+        "",
+        {},
+        vf,
+        None,
+        bank_holiday_rate_script,
+    )
+    charge_script = """
+import chellow.gas.ccl
+from chellow.gas.engine import g_rates
+from chellow.utils import reduce_bill_hhs
+
+
+def make_fake_bills(
+        sess, log, last_month_start, last_month_finish, current_month_start,
+        current_months_finish
+):
+    pass
+"""
+    g_contract_rate_script = {
+        "gas_rate": 0.1,
+        "standing_rate": 0.1,
+    }
+    g_contract = GContract.insert(
+        sess,
+        False,
+        "Fusion 2020",
+        charge_script,
+        {},
+        utc_datetime(2000, 1, 1),
+        None,
+        g_contract_rate_script,
+    )
+    g_contract.insert_g_batch(sess, f"fake_g_batch_{g_contract.id}", "Fake Batch")
+    sess.commit()
+    messages = []
+
+    def log(message):
+        messages.append(message)
+
+    def set_progress(progress):
+        pass
+
+    run_import(sess, log, set_progress)

chellow-1744710468.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1744710468.0.0"