chellow 1682682992.0.0__tar.gz → 1682692302.0.0__tar.gz

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1682682992.0.0
+Version: 1682692302.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1682692302.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1682692302.0.0"

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/e/bsuos.py

@@ -220,7 +220,8 @@ def _process_url(logger, sess, url, contract, s):
 
     for row in reader:
         date_str = row[0]
-        date = Datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S")
+        date_format = "%Y-%m-%dT%H:%M:%S" if len(date_str) == 19 else "%d-%m-%y"
+        date = Datetime.strptime(date_str, date_format)
         period_str = row[1]
         period = int(period_str)
         price_str = row[2]

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/e/tnuos.py

@@ -1,7 +1,10 @@
+from datetime import datetime as Datetime, timedelta as Timedelta
 from decimal import Decimal, InvalidOperation
+from io import BytesIO
 
 from dateutil.relativedelta import relativedelta
 
+from pypdf import PdfReader
 
 from sqlalchemy import null, or_, select
 
@@ -11,10 +14,12 @@ import chellow.e.computer
 import chellow.e.duos
 from chellow.models import Contract, RateScript
 from chellow.national_grid import api_get
+from chellow.rate_server import download
 from chellow.utils import (
     c_months_u,
     ct_datetime,
     hh_after,
+    hh_format,
     hh_min,
     to_ct,
     to_utc,
@@ -514,3 +519,66 @@ def ng_import_triad(sess, log, set_progress, s):
 
     log("Finished TNUoS TRIAD Tariffs")
     sess.commit()
+
+
+def _find_triad_dates(file_name, file_like):
+    dates = []
+    rate_script = {"a_file_name": file_name, "triad_dates": dates}
+
+    reader = PdfReader(file_like)
+    in_table = False
+    for page in reader.pages:
+        for line in page.extract_text().splitlines():
+            if in_table:
+                date_str, period_str, _ = line.split()
+                date = Datetime.strptime(date_str, "%d/%m/%Y")
+                delta = Timedelta(minutes=30) * (int(period_str) - 1)
+                dates.append(to_utc(to_ct(date + delta)))
+                if len(dates) == 3:
+                    in_table = False
+            else:
+                if "Demand (MW)" in line:
+                    in_table = True
+
+    return rate_script
+
+
+def rate_server_import(sess, log, set_progress, s, paths):
+    log("Starting to check for new TNUoS triad date PDFs")
+
+    year_entries = {}
+    for path, url in paths:
+        if len(path) == 4:
+            year, utility, rate_type, file_name = path
+            if utility == "electricity" and rate_type == "tnuos":
+                try:
+                    fl_entries = year_entries[year]
+                except KeyError:
+                    fl_entries = year_entries[year] = {}
+
+                fl_entries[file_name] = url
+
+    for year, year_pdfs in sorted(year_entries.items()):
+        year_start = to_utc(ct_datetime(year, 4, 1))
+        contract = Contract.get_non_core_by_name(sess, "triad_dates")
+        if year_start < contract.start_rate_script.start_date:
+            continue
+        rs = sess.execute(
+            select(RateScript).where(
+                RateScript.contract == contract,
+                RateScript.start_date == year_start,
+            )
+        ).scalar_one_or_none()
+        if rs is None:
+            rs = contract.insert_rate_script(sess, year_start, {})
+
+        if len(year_pdfs) > 0:
+            file_name, url = sorted(year_pdfs.items())[-1]
+
+            rs_script = rs.make_script()
+            if rs_script.get("a_file_name") != file_name:
+                rs.update(_find_triad_dates(file_name, BytesIO(download(s, url))))
+                log(f"Updated triad dates rate script for {hh_format(year_start)}")
+
+    log("Finished TNUoS triad dates PDFs")
+    sess.commit()

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/rate_server.py

@@ -72,8 +72,13 @@ def run_import(sess, log, set_progress):
     paths_list = []
     for sub_entry in tree_entry["tree"]:
         path = sub_entry["path"].split("/")
-        if path[-1] != "README.md":
-            paths_list.append((path, sub_entry["url"]))
+        if path[-1] == "README.md":
+            continue
+        if len(path) == 1 and path[0] == "LICENSE":
+            continue
+
+        path[0] = int(path[0])
+        paths_list.append((tuple(path), sub_entry["url"]))
 
     paths = tuple(paths_list)
 
@@ -82,6 +87,7 @@ def run_import(sess, log, set_progress):
         "chellow.e.dno_rate_parser",
         "chellow.e.laf_import",
         "chellow.e.mdd_importer",
+        "chellow.e.tnuos",
         "chellow.gas.dn_rate_parser",
     ):
         mod = import_module(mod_name)

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/templates/non_core_contract.html

@@ -19,6 +19,10 @@
 		<p>
 			<a href="/national_grid">Automatic Importer</a>
 		</p>
+	{% elif contract.name == 'triad_dates' %}
+		<p>
+			<a href="/rate_server">Automatic Importer</a>
+		</p>
 	{% endif %}
 
 

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/templates/rate_server.html

@@ -100,7 +100,7 @@
 					</td>
 				</tr>
 				<tr>
-					<th>TNUoS Spreadsheet</th>
+					<th>TRIAD Dates PDFs</th>
 					<td>
 						<table>
 							<thead>
@@ -110,7 +110,7 @@
 								</tr>
 							</thead>
 							<tbody>
-								{% for rs in tnuos_rs %}
+								{% for rs in triad_dates_rs %}
 									{% set script = rs.make_script() %}
 									<tr>
 										<td>{{rs.start_date|hh_format}}</td>

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/chellow/views.py

@@ -2176,17 +2176,6 @@ def rate_server_get():
         .where(MarketRole.code == "R", RateScript.start_date >= fy_start)
         .order_by(Contract.name, RateScript.start_date.desc())
     ).scalars()
-    tnuos_rs = g.sess.execute(
-        select(RateScript)
-        .join(RateScript.contract)
-        .join(MarketRole)
-        .where(
-            MarketRole.code == "Z",
-            RateScript.start_date >= fy_start,
-            Contract.name == "tnuos",
-        )
-        .order_by(RateScript.start_date.desc())
-    ).scalars()
     nts_rs = g.sess.execute(
         select(GRateScript)
         .join(GRateScript.g_contract)
@@ -2218,6 +2207,17 @@ def rate_server_get():
         )
         .order_by(RateScript.start_date.desc())
     ).scalars()
+    triad_dates_rs = g.sess.execute(
+        select(RateScript)
+        .join(RateScript.contract)
+        .join(MarketRole)
+        .where(
+            MarketRole.code == "Z",
+            RateScript.start_date >= fy_start,
+            Contract.name == "triad_dates",
+        )
+        .order_by(RateScript.start_date.desc())
+    ).scalars()
 
     return render_template(
         "rate_server.html",
@@ -2225,10 +2225,10 @@ def rate_server_get():
         config_state=config.make_state(),
         config_properties=props.get("rate_server", {}),
         dno_rs=dno_rs,
-        tnuos_rs=tnuos_rs,
         nts_rs=nts_rs,
         dn_rs=dn_rs,
         bsuos_rs=bsuos_rs,
+        triad_dates_rs=triad_dates_rs,
     )
 
 

{chellow-1682682992.0.0 → chellow-1682692302.0.0}/test/e/test_tnuos.py

@@ -1,7 +1,7 @@
 from decimal import Decimal
 
 from chellow.e.computer import SupplySource
-from chellow.e.tnuos import _process_banded_hh, _process_triad_hh
+from chellow.e.tnuos import _find_triad_dates, _process_banded_hh, _process_triad_hh
 from chellow.models import (
     Comm,
     Contract,
@@ -407,3 +407,34 @@ def test_process_triad_hh(sess):
         "triad-estimate-months": 1,
         "triad-estimate-rate": 0.0,
     }
+
+
+def test_find_triad_dates(mocker):
+    file_name = "nail_file.pdf"
+    file_like = mocker.Mock()
+    mock_reader = mocker.Mock()
+    lines = [
+        "here is the table: ",
+        "Date  Settlement ",
+        "Period  Net System ",
+        "Demand (MW)  ",
+        "15/12/2022  35 44,561 ",
+        "17/01/2023  35 42,022 ",
+        "02/12/2022  36 39,573 ",
+        "",
+    ]
+    page = mocker.Mock()
+    page.extract_text = mocker.Mock(return_value="\n".join(lines))
+    mock_reader.pages = [page]
+
+    mocker.patch("chellow.e.tnuos.PdfReader", return_value=mock_reader)
+    actual = _find_triad_dates(file_name, file_like)
+    expected = {
+        "a_file_name": "nail_file.pdf",
+        "triad_dates": [
+            utc_datetime(2022, 12, 15, 17, 0),
+            utc_datetime(2023, 1, 17, 17, 0),
+            utc_datetime(2022, 12, 2, 17, 30),
+        ],
+    }
+    assert actual == expected

chellow-1682682992.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1682682992.0.0"