chellow 1706260266.0.0__tar.gz → 1706691842.0.0__tar.gz

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1706260266.0.0
+Version: 1706691842.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -55,6 +55,7 @@ Chellow is a Python web application that uses the PostgreSQL database. To instal
 Chellow, follow these steps:
 
 * Install [PostgreSQL](http://www.postgresql.org/) 12
+  * Set the PostgreSQL time zone to 'UTC'.
 * Create a PostgreSQL database: `createdb --encoding=UTF8 chellow`
 * Set up the following environment variables to configure Chellow:
 

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/README.md

@@ -26,6 +26,7 @@ Chellow is a Python web application that uses the PostgreSQL database. To instal
 Chellow, follow these steps:
 
 * Install [PostgreSQL](http://www.postgresql.org/) 12
+  * Set the PostgreSQL time zone to 'UTC'.
 * Create a PostgreSQL database: `createdb --encoding=UTF8 chellow`
 * Set up the following environment variables to configure Chellow:
 

chellow-1706691842.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1706691842.0.0"

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/e/elexon.py

@@ -57,16 +57,17 @@ def api_search(s, resource_id, sort=None):
     return res_j
 
 
-def run_import(sess, log, set_progress):
+def run_import(sess, log, set_progress, scripting_key):
     log("Starting to import data from Elexon")
     s = requests.Session()
     s.verify = False
 
     for mod_name in ("chellow.e.system_price", "chellow.e.tlms", "chellow.e.rcrc"):
         mod = import_module(mod_name)
-        mod.elexon_import(sess, log, set_progress, s)
+        mod.elexon_import(sess, log, set_progress, s, scripting_key)
 
 
+ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
 LAST_RUN_KEY = "last_run"
 GLOBAL_ALERT = "There's a problem with an <a href='/e/elexon'>Elexon import</a>."
 ELEXON_STATE_KEY = "elexon"
@@ -119,6 +120,13 @@ class Elexon(threading.Thread):
                 with Session() as sess:
                     try:
                         config = Contract.get_non_core_by_name(sess, "configuration")
+                        props = config.make_properties()
+                        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
+                        if scripting_key is None:
+                            raise BadRequest(
+                                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} "
+                                f"cannot be found in the configuration properties."
+                            )
                         state = config.make_state()
                         try:
                             elexon_state = state[ELEXON_STATE_KEY]
@@ -128,7 +136,7 @@ class Elexon(threading.Thread):
                         elexon_state[LAST_RUN_KEY] = utc_datetime_now()
                         config.update_state(state)
                         sess.commit()
-                        run_import(sess, self.log, self.set_progress)
+                        run_import(sess, self.log, self.set_progress, scripting_key)
                     except BaseException as e:
                         msg = f"{e.description} " if isinstance(e, BadRequest) else ""
                         self.log(f"{msg}{traceback.format_exc()}")

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/e/rcrc.py

@@ -63,14 +63,23 @@ def _find_month(lines, month_start, month_finish):
     next(parser)
     month_rcrcs = {}
     for values in parser:
-        hh_date = to_utc(ct_datetime_parse(values[0], "%d/%m/%Y"))
+        if len(values) == 0:
+            continue
+        date_str = values[0]
+        if "/" in date_str:
+            pattern = "%d/%m/%Y"
+        elif " " in date_str:
+            pattern = "%d %b %Y"
+        else:
+            raise BadRequest(f"Date format {date_str} not recognized.")
+        hh_date = to_utc(ct_datetime_parse(date_str, pattern))
         hh_date += relativedelta(minutes=30 * int(values[2]))
         if month_start <= hh_date <= month_finish:
             month_rcrcs[key_format(hh_date)] = Decimal(values[3])
     return month_rcrcs
 
 
-def elexon_import(sess, log, set_progress, s):
+def elexon_import(sess, log, set_progress, s, scripting_key):
     log("Starting to check RCRCs.")
     contract = Contract.get_non_core_by_name(sess, "rcrc")
     latest_rs = (
@@ -90,25 +99,15 @@ def elexon_import(sess, log, set_progress, s):
     month_start, month_finish = months[1]
     now = utc_datetime_now()
     if now > month_finish:
-        config = Contract.get_non_core_by_name(sess, "configuration")
-        props = config.make_properties()
-
-        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-        if scripting_key is None:
-            raise BadRequest(
-                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be found in "
-                f"the configuration properties."
-            )
-
-        contract_props = contract.make_properties()
-        url_str = f"{contract_props['url']}file/download/RCRC_FILE?key={scripting_key}"
+        params = {"key": scripting_key}
+        url_str = "https://downloads.elexonportal.co.uk/file/download/RCRC_FILE"
         log(
-            f"Downloading {url_str} to see if data is available from "
-            f"{hh_format(month_start)} to {hh_format(month_finish)}."
+            f"Downloading {url_str}?key={scripting_key} to see if data is available "
+            f"from {hh_format(month_start)} to {hh_format(month_finish)}."
         )
 
         sess.rollback()  # Avoid long-running transaction
-        r = s.get(url_str, timeout=60)
+        r = s.get(url_str, timeout=120, params=params)
         month_rcrcs = _find_month(
             (x.decode() for x in r.iter_lines()), month_start, month_finish
         )

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/e/system_price.py

@@ -2,6 +2,8 @@ import datetime
 import traceback
 from datetime import timedelta as Timedelta
 
+from sqlalchemy import select
+
 from werkzeug.exceptions import BadRequest
 
 import xlrd
@@ -12,9 +14,6 @@ from chellow.models import Contract, RateScript
 from chellow.utils import HH, hh_format, to_ct, to_utc
 
 
-ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
-
-
 def key_format(dt):
     return dt.strftime("%d %H:%M")
 
@@ -61,7 +60,7 @@ def hh(data_source):
         h["ssp-gbp"] = h["nbp-kwh"] * ssp
 
 
-def elexon_import(sess, log, set_progress, s):
+def elexon_import(sess, log, set_progress, s, scripting_key):
     contract = Contract.get_non_core_by_name(sess, "system_price")
     contract_props = contract.make_properties()
 
@@ -74,9 +73,9 @@ def elexon_import(sess, log, set_progress, s):
 
     log("Starting to check System Prices.")
 
-    for rscript in (
-        sess.query(RateScript)
-        .filter(RateScript.contract == contract)
+    for rscript in sess.scalars(
+        select(RateScript)
+        .where(RateScript.contract == contract)
         .order_by(RateScript.start_date.desc())
     ):
         ns = loads(rscript.script)
@@ -87,24 +86,16 @@ def elexon_import(sess, log, set_progress, s):
         elif rates[key_format(rscript.finish_date)]["run"] == "DF":
             fill_start = rscript.finish_date + HH
             break
+    params = {"key": scripting_key}
+    url = "https://downloads.elexonportal.co.uk/file/download/BESTVIEWPRICES_FILE"
 
-    config = Contract.get_non_core_by_name(sess, "configuration")
-    config_props = config.make_properties()
-
-    scripting_key = config_props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-    if scripting_key is None:
-        raise BadRequest(
-            f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be found in "
-            f"the configuration properties."
-        )
-    url = (
-        f"{contract_props['url']}file/download/BESTVIEWPRICES_FILE?key={scripting_key}"
+    log(
+        f"Downloading from {url}?key={scripting_key} and extracting data from "
+        f"{hh_format(fill_start)}"
     )
 
-    log(f"Downloading from {url} and extracting data from {hh_format(fill_start)}")
-
     sess.rollback()  # Avoid long-running transactions
-    res = s.get(url)
+    res = s.get(url, params=params)
     log(f"Received {res.status_code} {res.reason}")
     data = res.content
     book = xlrd.open_workbook(file_contents=data)

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/e/tlms.py

@@ -4,7 +4,7 @@ from decimal import Decimal, InvalidOperation
 
 from dateutil.relativedelta import relativedelta
 
-from sqlalchemy import or_
+from sqlalchemy import or_, select
 from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
@@ -12,11 +12,9 @@ from werkzeug.exceptions import BadRequest
 from zish import dumps, loads
 
 from chellow.models import Contract, RateScript
-from chellow.utils import HH, hh_format, to_ct, to_utc
+from chellow.utils import HH, hh_format, hh_range, to_ct, to_utc
 
 
-ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
-
 RUNS = ["DF", "RF", "R3", "R2", "R1", "SF", "II"]
 
 
@@ -77,51 +75,76 @@ def hh(data_source, run="DF"):
         h["nbp-kwh"] = h["gsp-kwh"] * tlm
 
 
-def _save_cache(sess, cache):
-    for yr, yr_cache in cache.items():
-        for month, (rs, rates, rts) in tuple(yr_cache.items()):
-            rs.script = dumps(rates)
-            sess.commit()
-            del yr_cache[month]
+def _find_complete_date(caches, sess, contract, cache):
+    for rs in sess.scalars(
+        select(RateScript)
+        .where(RateScript.contract == contract, RateScript.finish_date != null())
+        .order_by(RateScript.start_date.desc())
+    ):
+        rates = rs.make_script()
+        rates["id"] = rs.id
+        cache["rate_scripts"].append(rates)
+        timestamps = cache["timestamps"]
+        tlms = rates["tlms"]
+        complete = True
+        for dt in hh_range(caches, rs.start_date, rs.finish_date):
+            timestamps[dt] = rates
+            key = key_format(dt)
+            if key in tlms:
+                gps = tlms[key]
+                for group in GSP_GROUP_LOOKUP.values():
+                    if not (group in gps and "DF" in gps[group]):
+                        complete = False
+            else:
+                complete = False
+
+        if complete:
+            return rs.finish_date
 
 
-def elexon_import(sess, log, set_progress, s):
-    cache = {}
+def elexon_import(sess, log, set_progress, s, scripting_key):
+    cache = {"rate_scripts": [], "timestamps": {}}
+    caches = {}
     log("Starting to check TLMs.")
     contract = Contract.get_non_core_by_name(sess, "tlms")
     contract_props = contract.make_properties()
     if contract_props.get("enabled", False):
-        config = Contract.get_non_core_by_name(sess, "configuration")
-        props = config.make_properties()
-        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-        if scripting_key is None:
-            raise BadRequest(
-                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be "
-                f"found in the configuration properties."
-            )
 
-        url_str = f"{contract_props['url']}file/download/TLM_FILE?key={scripting_key}"
+        params = {"key": scripting_key}
+        url_str = "https://downloads.elexonportal.co.uk/file/download/TLM_FILE"
+        complete_date = _find_complete_date(caches, sess, contract, cache)
+        log(f"Found complete up to {complete_date}")
 
         sess.rollback()  # Avoid long-running transaction
-        r = s.get(url_str)
+        r = s.get(url_str, params=params, timeout=120)
         parser = csv.reader(
             (x.decode() for x in r.iter_lines()), delimiter=",", quotechar='"'
         )
-        log(f"Opened {url_str}.")
+        log(f"Opened {url_str}?key={scripting_key}.")
 
         next(parser, None)
         for values in parser:
             if len(values) == 0:
                 continue
 
-            elif values[3] == "":
+            set_progress(values[0])
+
+            if values[3] == "":
                 for zone in GSP_GROUP_LOOKUP.keys():
                     values[3] = zone
-                    _process_line(cache, sess, contract, log, values)
+                    _process_line(
+                        cache, sess, contract, log, values, complete_date, caches
+                    )
             else:
-                _process_line(cache, sess, contract, log, values)
+                _process_line(cache, sess, contract, log, values, complete_date, caches)
+
+        for rscript in cache["rate_scripts"]:
+            rs = sess.scalars(
+                select(RateScript).where(RateScript.id == rscript["id"])
+            ).one()
+            rs.script = dumps({"tlms": rscript["tlms"]})
+            sess.commit()
 
-        _save_cache(sess, cache)
     else:
         log(
             "The importer is disabled. Set 'enabled' to 'true' in the "
@@ -148,10 +171,12 @@ GSP_GROUP_LOOKUP = {
 }
 
 
-def _process_line(cache, sess, contract, log_func, values):
+def _process_line(cache, sess, contract, log_func, values, complete_date, caches):
     hh_date_ct = to_ct(Datetime.strptime(values[0], "%d/%m/%Y"))
     hh_date = to_utc(hh_date_ct)
     hh_date += relativedelta(minutes=30 * (int(values[2]) - 1))
+    if complete_date is not None and hh_date <= complete_date:
+        return
     run = values[1]
     gsp_group_code = GSP_GROUP_LOOKUP[values[3]]
     off_taking_str = values[4]
@@ -166,45 +191,13 @@ def _process_line(cache, sess, contract, log_func, values):
 
     delivering = Decimal(values[5])
 
+    key = key_format(hh_date)
     try:
-        rs, rates, rts = cache[hh_date.year][hh_date.month]
+        cache["timestamps"][hh_date]["tlms"][key][gsp_group_code][run]
     except KeyError:
-        _save_cache(sess, cache)
         try:
-            yr_cache = cache[hh_date.year]
+            rate = cache["timestamps"][hh_date]
         except KeyError:
-            yr_cache = cache[hh_date.year] = {}
-
-        rs = (
-            sess.query(RateScript)
-            .filter(
-                RateScript.contract == contract,
-                RateScript.start_date <= hh_date,
-                or_(
-                    RateScript.finish_date == null(), RateScript.finish_date >= hh_date
-                ),
-            )
-            .first()
-        )
-        while rs is None:
-            log_func(f"There's no rate script at {hh_format(hh_date)}.")
-            latest_rs = (
-                sess.query(RateScript)
-                .filter(RateScript.contract == contract)
-                .order_by(RateScript.start_date.desc())
-                .first()
-            )
-            contract.update_rate_script(
-                sess,
-                latest_rs,
-                latest_rs.start_date,
-                latest_rs.start_date + relativedelta(months=2) - HH,
-                loads(latest_rs.script),
-            )
-            new_rs_start = latest_rs.start_date + relativedelta(months=1)
-            contract.insert_rate_script(sess, new_rs_start, {})
-            sess.commit()
-            log_func(f"Added a rate script starting at {hh_format(new_rs_start)}.")
 
             rs = (
                 sess.query(RateScript)
@@ -218,29 +211,56 @@ def _process_line(cache, sess, contract, log_func, values):
                 )
                 .first()
             )
+            assert rs is None
+            while rs is None:
+                log_func(f"There's no rate script at {hh_format(hh_date)}.")
+                latest_rs = (
+                    sess.query(RateScript)
+                    .filter(RateScript.contract == contract)
+                    .order_by(RateScript.start_date.desc())
+                    .first()
+                )
+                contract.update_rate_script(
+                    sess,
+                    latest_rs,
+                    latest_rs.start_date,
+                    latest_rs.start_date + relativedelta(months=2) - HH,
+                    loads(latest_rs.script),
+                )
+                new_rs_start = latest_rs.start_date + relativedelta(months=1)
+                new_rs = contract.insert_rate_script(sess, new_rs_start, {})
+                rt = {"tlms": {}, "id": new_rs.id}
+                cache["rate_scripts"].append(rt)
+                for h in hh_range(caches, new_rs.start_date, new_rs.finish_date):
+                    cache["timestamps"][h] = rt
+                sess.commit()
+                log_func(f"Added a rate script starting at {hh_format(new_rs_start)}.")
+
+                rs = (
+                    sess.query(RateScript)
+                    .filter(
+                        RateScript.contract == contract,
+                        RateScript.start_date <= hh_date,
+                        or_(
+                            RateScript.finish_date == null(),
+                            RateScript.finish_date >= hh_date,
+                        ),
+                    )
+                    .first()
+                )
 
-        rates = loads(rs.script)
+            rate = cache["timestamps"][hh_date]
 
         try:
-            rts = rates["tlms"]
+            existing = rate["tlms"][key]
         except KeyError:
-            rts = rates["tlms"] = {}
-
-        yr_cache[hh_date.month] = rs, rates, rts
-        sess.rollback()
-
-    key = key_format(hh_date)
-    try:
-        existing = rts[key]
-    except KeyError:
-        existing = rts[key] = {}
+            existing = rate["tlms"][key] = {}
 
-    try:
-        group = existing[gsp_group_code]
-    except KeyError:
-        group = existing[gsp_group_code] = {}
+        try:
+            group = existing[gsp_group_code]
+        except KeyError:
+            group = existing[gsp_group_code] = {}
 
-    if run not in group:
         group[run] = {"off_taking": off_taking, "delivering": delivering}
 
         log_func(

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/templates/non_core_contract.html

@@ -10,8 +10,7 @@
 {% endblock %}
 
 {% block content %}
-	{% if contract.name in [
-	    'bank_holidays', 'bmarketidx', 'rcrc', 'system_price', 'tlms'] %}
+	{% if contract.name in ['bank_holidays', 'bmarketidx'] %}
 		<p>
 			<a href="/non_core_contracts/{{contract.id}}/auto_importer">Automatic Importer</a>
 		</p>
@@ -34,6 +33,10 @@
 				<li><a href="/national_grid">Forecast Rates</a></li>
 			</ul>
 		</p>
+	{% elif contract.name in ['rcrc', 'system_price', 'tlms'] %}
+		<p>
+			<a href="/e/elexon">Automatic Importer</a>
+		</p>
 	{% endif %}
 
 

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/chellow/templates/non_core_contract_edit.html

@@ -68,8 +68,8 @@ to the 'urls' list. */
 		{% elif contract.name == 'tlms' %}
 			<h4>Example</h4>
 			<pre>{
-"enabled": true,
-"url": "https://downloads.elexonportal.co.uk/"}
+  "enabled": true,
+}
 
 /* Requires the 'elexonportal_scripting_key' to be set in the 'configuration'
 non-core contract */

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/test/e/system_price/test_system_price.py

@@ -10,7 +10,6 @@ def test_process(sess, mocker):
     participant.insert_party(sess, market_role_Z, "non-core", from_date, None, None)
     system_price_properties = {
         "enabled": True,
-        "url": "https://example.com",
     }
     rate_script_properties = {"gbp_per_nbp_mwh": {}}
     Contract.insert_non_core(
@@ -47,11 +46,13 @@ def test_process(sess, mocker):
     mock_response.reason = "OK"
 
     mock_set_progress = mocker.Mock()
-    elexon_import(sess, log_f, mock_set_progress, mock_s)
+    scripting_key = "xxx"
+    elexon_import(sess, log_f, mock_set_progress, mock_s, scripting_key)
 
     assert log == [
         "Starting to check System Prices.",
-        "Downloading from https://example.comfile/download/BESTVIEWPRICES_FILE?"
+        "Downloading from "
+        "https://downloads.elexonportal.co.uk/file/download/BESTVIEWPRICES_FILE?"
         "key=xxx and extracting data from 2001-04-01 01:00",
         "Received 200 OK",
         "Successfully extracted data.",

{chellow-1706260266.0.0 → chellow-1706691842.0.0}/test/e/test_tlms.py

@@ -214,5 +214,6 @@ def test_elexon_import(sess, mocker):
     lines = [b"", b""]
     mock_response.iter_lines = mocker.Mock(return_value=lines)
     s.get = mocker.Mock(return_value=mock_response)
+    scripting_key = mocker.Mock()
 
-    elexon_import(sess, log, set_progress, s)
+    elexon_import(sess, log, set_progress, s, scripting_key)

chellow-1706260266.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1706260266.0.0"