chellow 1706265605.0.0__py3-none-any.whl → 1706691842.0.0__py3-none-any.whl

chellow/e/elexon.py

@@ -57,16 +57,17 @@ def api_search(s, resource_id, sort=None):
     return res_j
 
 
-def run_import(sess, log, set_progress):
+def run_import(sess, log, set_progress, scripting_key):
     log("Starting to import data from Elexon")
     s = requests.Session()
     s.verify = False
 
     for mod_name in ("chellow.e.system_price", "chellow.e.tlms", "chellow.e.rcrc"):
         mod = import_module(mod_name)
-        mod.elexon_import(sess, log, set_progress, s)
+        mod.elexon_import(sess, log, set_progress, s, scripting_key)
 
 
+ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
 LAST_RUN_KEY = "last_run"
 GLOBAL_ALERT = "There's a problem with an <a href='/e/elexon'>Elexon import</a>."
 ELEXON_STATE_KEY = "elexon"
@@ -119,6 +120,13 @@ class Elexon(threading.Thread):
                 with Session() as sess:
                     try:
                         config = Contract.get_non_core_by_name(sess, "configuration")
+                        props = config.make_properties()
+                        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
+                        if scripting_key is None:
+                            raise BadRequest(
+                                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} "
+                                f"cannot be found in the configuration properties."
+                            )
                         state = config.make_state()
                         try:
                             elexon_state = state[ELEXON_STATE_KEY]
@@ -128,7 +136,7 @@ class Elexon(threading.Thread):
                         elexon_state[LAST_RUN_KEY] = utc_datetime_now()
                         config.update_state(state)
                         sess.commit()
-                        run_import(sess, self.log, self.set_progress)
+                        run_import(sess, self.log, self.set_progress, scripting_key)
                     except BaseException as e:
                         msg = f"{e.description} " if isinstance(e, BadRequest) else ""
                         self.log(f"{msg}{traceback.format_exc()}")

chellow/e/rcrc.py

@@ -63,14 +63,23 @@ def _find_month(lines, month_start, month_finish):
     next(parser)
     month_rcrcs = {}
     for values in parser:
-        hh_date = to_utc(ct_datetime_parse(values[0], "%d/%m/%Y"))
+        if len(values) == 0:
+            continue
+        date_str = values[0]
+        if "/" in date_str:
+            pattern = "%d/%m/%Y"
+        elif " " in date_str:
+            pattern = "%d %b %Y"
+        else:
+            raise BadRequest(f"Date format {date_str} not recognized.")
+        hh_date = to_utc(ct_datetime_parse(date_str, pattern))
         hh_date += relativedelta(minutes=30 * int(values[2]))
         if month_start <= hh_date <= month_finish:
             month_rcrcs[key_format(hh_date)] = Decimal(values[3])
     return month_rcrcs
 
 
-def elexon_import(sess, log, set_progress, s):
+def elexon_import(sess, log, set_progress, s, scripting_key):
     log("Starting to check RCRCs.")
     contract = Contract.get_non_core_by_name(sess, "rcrc")
     latest_rs = (
@@ -90,25 +99,15 @@ def elexon_import(sess, log, set_progress, s):
     month_start, month_finish = months[1]
     now = utc_datetime_now()
     if now > month_finish:
-        config = Contract.get_non_core_by_name(sess, "configuration")
-        props = config.make_properties()
-
-        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-        if scripting_key is None:
-            raise BadRequest(
-                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be found in "
-                f"the configuration properties."
-            )
-
-        contract_props = contract.make_properties()
-        url_str = f"{contract_props['url']}file/download/RCRC_FILE?key={scripting_key}"
+        params = {"key": scripting_key}
+        url_str = "https://downloads.elexonportal.co.uk/file/download/RCRC_FILE"
         log(
-            f"Downloading {url_str} to see if data is available from "
-            f"{hh_format(month_start)} to {hh_format(month_finish)}."
+            f"Downloading {url_str}?key={scripting_key} to see if data is available "
+            f"from {hh_format(month_start)} to {hh_format(month_finish)}."
         )
 
         sess.rollback()  # Avoid long-running transaction
-        r = s.get(url_str, timeout=60)
+        r = s.get(url_str, timeout=120, params=params)
         month_rcrcs = _find_month(
             (x.decode() for x in r.iter_lines()), month_start, month_finish
         )

chellow/e/system_price.py

@@ -2,6 +2,8 @@ import datetime
 import traceback
 from datetime import timedelta as Timedelta
 
+from sqlalchemy import select
+
 from werkzeug.exceptions import BadRequest
 
 import xlrd
@@ -12,9 +14,6 @@ from chellow.models import Contract, RateScript
 from chellow.utils import HH, hh_format, to_ct, to_utc
 
 
-ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
-
-
 def key_format(dt):
     return dt.strftime("%d %H:%M")
 
@@ -61,7 +60,7 @@ def hh(data_source):
         h["ssp-gbp"] = h["nbp-kwh"] * ssp
 
 
-def elexon_import(sess, log, set_progress, s):
+def elexon_import(sess, log, set_progress, s, scripting_key):
     contract = Contract.get_non_core_by_name(sess, "system_price")
     contract_props = contract.make_properties()
 
@@ -74,9 +73,9 @@ def elexon_import(sess, log, set_progress, s):
 
     log("Starting to check System Prices.")
 
-    for rscript in (
-        sess.query(RateScript)
-        .filter(RateScript.contract == contract)
+    for rscript in sess.scalars(
+        select(RateScript)
+        .where(RateScript.contract == contract)
         .order_by(RateScript.start_date.desc())
     ):
         ns = loads(rscript.script)
@@ -87,24 +86,16 @@ def elexon_import(sess, log, set_progress, s):
         elif rates[key_format(rscript.finish_date)]["run"] == "DF":
             fill_start = rscript.finish_date + HH
             break
+    params = {"key": scripting_key}
+    url = "https://downloads.elexonportal.co.uk/file/download/BESTVIEWPRICES_FILE"
 
-    config = Contract.get_non_core_by_name(sess, "configuration")
-    config_props = config.make_properties()
-
-    scripting_key = config_props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-    if scripting_key is None:
-        raise BadRequest(
-            f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be found in "
-            f"the configuration properties."
-        )
-    url = (
-        f"{contract_props['url']}file/download/BESTVIEWPRICES_FILE?key={scripting_key}"
+    log(
+        f"Downloading from {url}?key={scripting_key} and extracting data from "
+        f"{hh_format(fill_start)}"
     )
 
-    log(f"Downloading from {url} and extracting data from {hh_format(fill_start)}")
-
     sess.rollback()  # Avoid long-running transactions
-    res = s.get(url)
+    res = s.get(url, params=params)
     log(f"Received {res.status_code} {res.reason}")
     data = res.content
     book = xlrd.open_workbook(file_contents=data)

chellow/e/tlms.py

@@ -4,7 +4,7 @@ from decimal import Decimal, InvalidOperation
 
 from dateutil.relativedelta import relativedelta
 
-from sqlalchemy import or_
+from sqlalchemy import or_, select
 from sqlalchemy.sql.expression import null
 
 from werkzeug.exceptions import BadRequest
@@ -12,11 +12,9 @@ from werkzeug.exceptions import BadRequest
 from zish import dumps, loads
 
 from chellow.models import Contract, RateScript
-from chellow.utils import HH, hh_format, to_ct, to_utc
+from chellow.utils import HH, hh_format, hh_range, to_ct, to_utc
 
 
-ELEXON_PORTAL_SCRIPTING_KEY_KEY = "elexonportal_scripting_key"
-
 RUNS = ["DF", "RF", "R3", "R2", "R1", "SF", "II"]
 
 
@@ -77,51 +75,76 @@ def hh(data_source, run="DF"):
         h["nbp-kwh"] = h["gsp-kwh"] * tlm
 
 
-def _save_cache(sess, cache):
-    for yr, yr_cache in cache.items():
-        for month, (rs, rates, rts) in tuple(yr_cache.items()):
-            rs.script = dumps(rates)
-            sess.commit()
-            del yr_cache[month]
+def _find_complete_date(caches, sess, contract, cache):
+    for rs in sess.scalars(
+        select(RateScript)
+        .where(RateScript.contract == contract, RateScript.finish_date != null())
+        .order_by(RateScript.start_date.desc())
+    ):
+        rates = rs.make_script()
+        rates["id"] = rs.id
+        cache["rate_scripts"].append(rates)
+        timestamps = cache["timestamps"]
+        tlms = rates["tlms"]
+        complete = True
+        for dt in hh_range(caches, rs.start_date, rs.finish_date):
+            timestamps[dt] = rates
+            key = key_format(dt)
+            if key in tlms:
+                gps = tlms[key]
+                for group in GSP_GROUP_LOOKUP.values():
+                    if not (group in gps and "DF" in gps[group]):
+                        complete = False
+            else:
+                complete = False
+
+        if complete:
+            return rs.finish_date
 
 
-def elexon_import(sess, log, set_progress, s):
-    cache = {}
+def elexon_import(sess, log, set_progress, s, scripting_key):
+    cache = {"rate_scripts": [], "timestamps": {}}
+    caches = {}
     log("Starting to check TLMs.")
     contract = Contract.get_non_core_by_name(sess, "tlms")
     contract_props = contract.make_properties()
     if contract_props.get("enabled", False):
-        config = Contract.get_non_core_by_name(sess, "configuration")
-        props = config.make_properties()
-        scripting_key = props.get(ELEXON_PORTAL_SCRIPTING_KEY_KEY)
-        if scripting_key is None:
-            raise BadRequest(
-                f"The property {ELEXON_PORTAL_SCRIPTING_KEY_KEY} cannot be "
-                f"found in the configuration properties."
-            )
 
-        url_str = f"{contract_props['url']}file/download/TLM_FILE?key={scripting_key}"
+        params = {"key": scripting_key}
+        url_str = "https://downloads.elexonportal.co.uk/file/download/TLM_FILE"
+        complete_date = _find_complete_date(caches, sess, contract, cache)
+        log(f"Found complete up to {complete_date}")
 
         sess.rollback()  # Avoid long-running transaction
-        r = s.get(url_str)
+        r = s.get(url_str, params=params, timeout=120)
         parser = csv.reader(
             (x.decode() for x in r.iter_lines()), delimiter=",", quotechar='"'
         )
-        log(f"Opened {url_str}.")
+        log(f"Opened {url_str}?key={scripting_key}.")
 
         next(parser, None)
         for values in parser:
             if len(values) == 0:
                 continue
 
-            elif values[3] == "":
+            set_progress(values[0])
+
+            if values[3] == "":
                 for zone in GSP_GROUP_LOOKUP.keys():
                     values[3] = zone
-                    _process_line(cache, sess, contract, log, values)
+                    _process_line(
+                        cache, sess, contract, log, values, complete_date, caches
+                    )
             else:
-                _process_line(cache, sess, contract, log, values)
+                _process_line(cache, sess, contract, log, values, complete_date, caches)
+
+        for rscript in cache["rate_scripts"]:
+            rs = sess.scalars(
+                select(RateScript).where(RateScript.id == rscript["id"])
+            ).one()
+            rs.script = dumps({"tlms": rscript["tlms"]})
+            sess.commit()
 
-        _save_cache(sess, cache)
     else:
         log(
             "The importer is disabled. Set 'enabled' to 'true' in the "
@@ -148,10 +171,12 @@ GSP_GROUP_LOOKUP = {
 }
 
 
-def _process_line(cache, sess, contract, log_func, values):
+def _process_line(cache, sess, contract, log_func, values, complete_date, caches):
     hh_date_ct = to_ct(Datetime.strptime(values[0], "%d/%m/%Y"))
     hh_date = to_utc(hh_date_ct)
     hh_date += relativedelta(minutes=30 * (int(values[2]) - 1))
+    if complete_date is not None and hh_date <= complete_date:
+        return
     run = values[1]
     gsp_group_code = GSP_GROUP_LOOKUP[values[3]]
     off_taking_str = values[4]
@@ -166,45 +191,13 @@ def _process_line(cache, sess, contract, log_func, values):
 
     delivering = Decimal(values[5])
 
+    key = key_format(hh_date)
     try:
-        rs, rates, rts = cache[hh_date.year][hh_date.month]
+        cache["timestamps"][hh_date]["tlms"][key][gsp_group_code][run]
     except KeyError:
-        _save_cache(sess, cache)
         try:
-            yr_cache = cache[hh_date.year]
+            rate = cache["timestamps"][hh_date]
         except KeyError:
-            yr_cache = cache[hh_date.year] = {}
-
-        rs = (
-            sess.query(RateScript)
-            .filter(
-                RateScript.contract == contract,
-                RateScript.start_date <= hh_date,
-                or_(
-                    RateScript.finish_date == null(), RateScript.finish_date >= hh_date
-                ),
-            )
-            .first()
-        )
-        while rs is None:
-            log_func(f"There's no rate script at {hh_format(hh_date)}.")
-            latest_rs = (
-                sess.query(RateScript)
-                .filter(RateScript.contract == contract)
-                .order_by(RateScript.start_date.desc())
-                .first()
-            )
-            contract.update_rate_script(
-                sess,
-                latest_rs,
-                latest_rs.start_date,
-                latest_rs.start_date + relativedelta(months=2) - HH,
-                loads(latest_rs.script),
-            )
-            new_rs_start = latest_rs.start_date + relativedelta(months=1)
-            contract.insert_rate_script(sess, new_rs_start, {})
-            sess.commit()
-            log_func(f"Added a rate script starting at {hh_format(new_rs_start)}.")
 
             rs = (
                 sess.query(RateScript)
@@ -218,29 +211,56 @@ def _process_line(cache, sess, contract, log_func, values):
                 )
                 .first()
             )
+            assert rs is None
+            while rs is None:
+                log_func(f"There's no rate script at {hh_format(hh_date)}.")
+                latest_rs = (
+                    sess.query(RateScript)
+                    .filter(RateScript.contract == contract)
+                    .order_by(RateScript.start_date.desc())
+                    .first()
+                )
+                contract.update_rate_script(
+                    sess,
+                    latest_rs,
+                    latest_rs.start_date,
+                    latest_rs.start_date + relativedelta(months=2) - HH,
+                    loads(latest_rs.script),
+                )
+                new_rs_start = latest_rs.start_date + relativedelta(months=1)
+                new_rs = contract.insert_rate_script(sess, new_rs_start, {})
+                rt = {"tlms": {}, "id": new_rs.id}
+                cache["rate_scripts"].append(rt)
+                for h in hh_range(caches, new_rs.start_date, new_rs.finish_date):
+                    cache["timestamps"][h] = rt
+                sess.commit()
+                log_func(f"Added a rate script starting at {hh_format(new_rs_start)}.")
+
+                rs = (
+                    sess.query(RateScript)
+                    .filter(
+                        RateScript.contract == contract,
+                        RateScript.start_date <= hh_date,
+                        or_(
+                            RateScript.finish_date == null(),
+                            RateScript.finish_date >= hh_date,
+                        ),
+                    )
+                    .first()
+                )
 
-        rates = loads(rs.script)
+            rate = cache["timestamps"][hh_date]
 
         try:
-            rts = rates["tlms"]
+            existing = rate["tlms"][key]
         except KeyError:
-            rts = rates["tlms"] = {}
-
-        yr_cache[hh_date.month] = rs, rates, rts
-        sess.rollback()
-
-    key = key_format(hh_date)
-    try:
-        existing = rts[key]
-    except KeyError:
-        existing = rts[key] = {}
+            existing = rate["tlms"][key] = {}
 
-    try:
-        group = existing[gsp_group_code]
-    except KeyError:
-        group = existing[gsp_group_code] = {}
+        try:
+            group = existing[gsp_group_code]
+        except KeyError:
+            group = existing[gsp_group_code] = {}
 
-    if run not in group:
         group[run] = {"off_taking": off_taking, "delivering": delivering}
 
         log_func(

chellow/templates/non_core_contract_edit.html

@@ -68,8 +68,8 @@ to the 'urls' list. */
 		{% elif contract.name == 'tlms' %}
 			<h4>Example</h4>
 			<pre>{
-"enabled": true,
-"url": "https://downloads.elexonportal.co.uk/"}
+  "enabled": true,
+}
 
 /* Requires the 'elexonportal_scripting_key' to be set in the 'configuration'
 non-core contract */

{chellow-1706265605.0.0.dist-info → chellow-1706691842.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1706265605.0.0
+Version: 1706691842.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -55,6 +55,7 @@ Chellow is a Python web application that uses the PostgreSQL database. To instal
 Chellow, follow these steps:
 
 * Install [PostgreSQL](http://www.postgresql.org/) 12
+  * Set the PostgreSQL time zone to 'UTC'.
 * Create a PostgreSQL database: `createdb --encoding=UTF8 chellow`
 * Set up the following environment variables to configure Chellow:
 

{chellow-1706265605.0.0.dist-info → chellow-1706691842.0.0.dist-info}/RECORD

@@ -22,7 +22,7 @@ chellow/e/cfd.py,sha256=3yRchycC-WZwgqryGQK3B59OcMGVIWgbxrkekUOy-9k,14914
 chellow/e/computer.py,sha256=J8VvYjcSeJNrG2Ak8Zi2t2bb_8tMrkHWny4JQlgIyy8,67032
 chellow/e/dno_rate_parser.py,sha256=5MYEbcYngh7n6Szn2h9zLTwQKXH-WHKy6c80PYwNLLQ,20983
 chellow/e/duos.py,sha256=ISTcNqe9KNjVNM2Qs8IBoQxnmSXOt5W_G7tZxp4T78M,28870
-chellow/e/elexon.py,sha256=I0PislGIm1Kgq0TCLki7hrN2A9cWGXnpIcG-c-deaag,4946
+chellow/e/elexon.py,sha256=s8QdLUQMeylLhghxvvhLyOL3-0ifUnMfmo3vcbot_DY,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
 chellow/e/hh_importer.py,sha256=G1CK3AQQuJaNXCxGEY3UU_CjqOzfVZT-PDR67b3zpA0,20362
 chellow/e/hh_parser_bg_csv.py,sha256=IOjM-QliKzvSXyhslmxz504XWnsGkVKEaCQ48SNeNeQ,2423
@@ -31,11 +31,11 @@ chellow/e/hh_parser_simple_csv.py,sha256=lVRprIEjDpTQdeXm2xcLmRAyre2LWec8ueVwcCI
 chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
 chellow/e/lcc.py,sha256=q5iGbeYZdnozkIdbGuxXoJO-1KDO7AHqYOXEGBIaNHI,4963
 chellow/e/mdd_importer.py,sha256=9GN-at0DC3vOjd-0N2U856O3hInnjkMWWcRfxzQ5DjA,32042
-chellow/e/rcrc.py,sha256=UH9_eM-4sPuEkEyVFGubr5VCXnOJW_kMAjnyTiToxKI,4350
+chellow/e/rcrc.py,sha256=92CA1uIotIHd1epQ_jEPdJKzXqDFV-AoJOJeRO6MEyA,4274
 chellow/e/ro.py,sha256=dZKZv_9wXSWuwcb3jiKavoD_9ot-PZseNVeEEe0siLo,596
 chellow/e/scenario.py,sha256=1tUxnvwTzr6cKqiw2wphdv5XDzV6JO6UVYkyQa67vHs,23263
-chellow/e/system_price.py,sha256=UPgUts6p7ETdclnO_MEC1mMqSWBg2bbSylDuj452OyY,6152
-chellow/e/tlms.py,sha256=BvHfe9bhiGvA_qeZ7iFwtOh_bmE_es3xqro1213ipDQ,7311
+chellow/e/system_price.py,sha256=3cPWohHmmBI9v7fENLBzXQkpAeC3r0s5xV29Nmr6k_E,5839
+chellow/e/tlms.py,sha256=kHAy2A2d1Mma7x4GdNDyrzscJd5DpJtDBYiZEg241Dw,8538
 chellow/e/tnuos.py,sha256=qybrPCVukQ2l0RIdqn8XtHEbAU0izGoqUrFLoLVUoVI,4126
 chellow/e/triad.py,sha256=S6LEMHvUKhAZe0-yfLIRciYDZ8IKMn1jh1TmmsbQD3s,13588
 chellow/e/views.py,sha256=aZecpfM1sVn3xMwlKuiw9pE2f6s4uyd98obQXToKtqA,212466
@@ -125,7 +125,7 @@ chellow/templates/macros.html,sha256=Koc8dT2pZaO2-iVm9RQiR_V3qymSZ_3iaKe2rCyrvgk
 chellow/templates/national_grid.html,sha256=8W92tsjlqPSB0J--nyFIi-wzFae9CyDr6fODXLZp8ic,991
 chellow/templates/non_core_auto_importer.html,sha256=s9SluRN1bHTHjd8GP6uFhk6LrZYG8dqBG3y94zUKe5Q,944
 chellow/templates/non_core_contract.html,sha256=BdGtxErTvw4DwXLb6B2vimuU6ode3fFA-90kBmHCwYc,1754
-chellow/templates/non_core_contract_edit.html,sha256=_EWwJdrez2uI2mA58Jf5WIYs7UfBw1xWV-Bpgliyurw,3005
+chellow/templates/non_core_contract_edit.html,sha256=rjY174SH_wXFU345VRIfN_yjd-jc9QQjzhyrJAf3VEk,2961
 chellow/templates/non_core_contracts.html,sha256=vIrVdn4NUj58Uy17REs_fy2JBQzPkjcg63bg7q6kELg,661
 chellow/templates/non_core_rate_script.html,sha256=CqGnuWdzhk_o_2xNFcF8rgk0p7SNh0B0c3k1JzOtn98,1283
 chellow/templates/non_core_rate_script_add.html,sha256=Qx8_cYFRQZrXSyR3uf_8OxUAUz3PqyYc4V11lTU18sE,690
@@ -363,6 +363,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1706265605.0.0.dist-info/METADATA,sha256=4kU_QU4coCbZe3Ljngxxa-zhWlIlwGsGneoNNqrhRw0,12160
-chellow-1706265605.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-chellow-1706265605.0.0.dist-info/RECORD,,
+chellow-1706691842.0.0.dist-info/METADATA,sha256=L3rxfmbWGPmkWihil72bLkI0vYaKsC3jxz5YPxnmbZ0,12203
+chellow-1706691842.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+chellow-1706691842.0.0.dist-info/RECORD,,