chellow 1753106469.0.0__py3-none-any.whl → 1753180539.0.0__py3-none-any.whl

chellow/e/elexon.py

@@ -62,7 +62,12 @@ def run_import(sess, log, set_progress, scripting_key):
     s = requests.Session()
     s.verify = False
 
-    for mod_name in ("chellow.e.system_price", "chellow.e.tlms", "chellow.e.rcrc"):
+    for mod_name in (
+        "chellow.e.system_price",
+        "chellow.e.tlms",
+        "chellow.e.rcrc",
+        "chellow.e.lafs",
+    ):
         mod = import_module(mod_name)
         mod.elexon_import(sess, log, set_progress, s, scripting_key)
 

chellow/e/{laf_import.py → lafs.py}

@@ -4,10 +4,10 @@ from decimal import Decimal
 from io import BytesIO, StringIO
 from zipfile import ZipFile
 
-# from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import text
 
 from werkzeug.exceptions import BadRequest
+from werkzeug.http import parse_options_header
 
 from chellow.models import Contract, Party
 from chellow.rate_server import download
@@ -30,7 +30,12 @@ DO UPDATE SET (llfc_id, timestamp, value) =
     )
     fname = name_list[0]
     csv_file = StringIO(zip_file.read(fname).decode("utf-8"))
-    csv_dt = to_utc(to_ct(Datetime.strptime(fname[-12:-4], "%Y%m%d")))
+    try:
+        csv_dt = to_utc(to_ct(Datetime.strptime(fname[-12:-4], "%Y%m%d")))
+    except ValueError as e:
+        log(f"Can't parse the date in the file name {fname} {e}")
+        return
+
     for llfc_ids, timestamps, values in laf_days(
         sess, log, set_progress, csv_file, csv_dt
     ):
@@ -110,6 +115,67 @@ LAF_START = "llf"
 LAF_END = "ptf.zip"
 
 
+def elexon_import(sess, log, set_progress, s, scripting_key):
+    log(
+        "Starting to check for new LAF files for participant codes in "
+        "configuration.properties.laf_importer.participant_codes"
+    )
+    conf = Contract.get_non_core_by_name(sess, "configuration")
+    props = conf.make_properties()
+    state = conf.make_state()
+
+    try:
+        laf_state = state["laf_importer"]
+    except KeyError:
+        laf_state = state["laf_importer"] = {}
+
+    url = "https://downloads.elexonportal.co.uk/svallf/download"
+
+    laf_props = props.get("laf_importer", {})
+    for participant_code in laf_props.get("participant_codes", []):
+        params = {"key": scripting_key, "ldso": participant_code, "format": "PTF"}
+
+        log(
+            f"Downloading from {url}?key={scripting_key}&ldso={participant_code}&"
+            f"format=PTF"
+        )
+        sess.rollback()  # Avoid long-running transactions
+        res = s.get(url, params=params)
+        log(f"Received {res.status_code} {res.reason}")
+        res.raise_for_status()
+        try:
+            cd_header = res.headers["Content-Disposition"]
+        except KeyError:
+            log("No Content-Disposition header found, so skipping.")
+            continue
+
+        cd = parse_options_header(cd_header)
+        file_name = cd[1]["filename"]
+
+        # File name llfetcl20220401ptf.zip
+        if not file_name.startswith(LAF_START):
+            raise BadRequest(f"A laf file must begin with '{LAF_START}'")
+        if not file_name.endswith(LAF_END):
+            raise BadRequest(f"A laf file must end with '{LAF_END}'")
+
+        participant_code = file_name[3:7]
+        timestamp = file_name[7:15]
+        log(f"File name {file_name}")
+
+        if timestamp > laf_state.get(participant_code, ""):
+            data = res.content
+            _process(sess, log, set_progress, BytesIO(data))
+            laf_state[participant_code] = timestamp
+            conf.update_state(state)
+            sess.commit()
+            log(f"Processed file {file_name}.")
+        else:
+            log(f"Already loaded {file_name}.")
+
+    log("Finished LAF files")
+    sess.commit()
+
+
 def find_participant_entries(paths, laf_state):
     participant_entries = {}
     for path, url in paths:

chellow/e/views.py

@@ -2344,30 +2344,29 @@ def hh_datum_edit_post(datum_id):
 @e.route("/lafs")
 def lafs_get():
     llfc_id = req_int("llfc_id")
-    year = req_int("year")
-    month = req_int("month")
-
+    month_start = req_date("timestamp", resolution="month")
+    month_start_ct = to_ct(month_start)
     llfc = Llfc.get_by_id(g.sess, llfc_id)
     dno = llfc.dno
 
-    start_date, finish_date = next(
-        c_months_u(start_year=year, start_month=month, months=1)
-    )
-    lafs = (
-        g.sess.execute(
-            select(Laf)
-            .where(
-                Laf.llfc == llfc,
-                Laf.timestamp >= start_date,
-                Laf.timestamp <= finish_date,
-            )
-            .order_by(Laf.timestamp)
+    month_start, month_finish = next(
+        c_months_u(
+            start_year=month_start_ct.year, start_month=month_start_ct.month, months=1
         )
-        .scalars()
-        .all()
     )
+    lafs = g.sess.execute(
+        select(Laf)
+        .where(
+            Laf.llfc == llfc,
+            Laf.timestamp >= month_start,
+            Laf.timestamp <= month_finish,
+        )
+        .order_by(Laf.timestamp)
+    ).scalars()
 
-    return render_template("lafs.html", dno=dno, llfc=llfc, lafs=lafs)
+    return render_template(
+        "lafs.html", dno=dno, llfc=llfc, lafs=lafs, month_start=month_start
+    )
 
 
 @e.route("/lcc")

chellow/gas/bill_parser_bgs_xlsx.py

@@ -5,12 +5,13 @@ from decimal import Decimal, InvalidOperation
 from enum import Enum, auto
 from io import BytesIO
 
+from dateutil.relativedelta import relativedelta
 
 from openpyxl import load_workbook
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import c_months_u, to_ct, to_utc
+from chellow.utils import to_ct, to_utc
 
 
 class Title(Enum):
@@ -151,13 +152,13 @@ def _parse_row(bills, sheet, row, title_row):
     column_map = make_column_map(title_row)
     mprn_raw = get_value(sheet, row, column_map[Title.MPRN])
     mprn = str(mprn_raw)
-    reference = get_value(sheet, row, column_map[Title.BILL])
+    reference = str(get_value(sheet, row, column_map[Title.BILL]))
     account = get_value(sheet, row, column_map[Title.ACCOUNT])
     issue_date = get_date(sheet, row, column_map[Title.BILL_DATE])
-    period_naive = get_date_naive(sheet, row, column_map[Title.BILLING_PERIOD])
-    start_date, finish_date = list(
-        c_months_u(start_year=period_naive.year, start_month=period_naive.month)
-    )[0]
+    charge_period_from = get_date(sheet, row, column_map[Title.CHARGE_PERIOD_FROM])
+    charge_period_end_naive = get_date_naive(
+        sheet, row, column_map[Title.CHARGE_PERIOD_END]
+    )
 
     try:
         mprn_values = bills[mprn]
@@ -165,16 +166,18 @@ def _parse_row(bills, sheet, row, title_row):
         mprn_values = bills[mprn] = {}
 
     try:
-        bill = mprn_values[period_naive]
+        bill = mprn_values[reference]
     except KeyError:
-        bill = mprn_values[period_naive] = {
-            "bill_type_code": "N",
+        bill = mprn_values[reference] = {
+            "bill_type_code": "W" if reference.startswith("CR") else "N",
             "mprn": mprn,
             "reference": reference,
             "account": account,
             "issue_date": issue_date,
-            "start_date": start_date,
-            "finish_date": finish_date,
+            "start_date": charge_period_from,
+            "finish_date": to_utc(
+                to_ct(charge_period_end_naive) + relativedelta(hours=23, minutes=30)
+            ),
             "kwh": Decimal("0"),
             "breakdown": defaultdict(int, {}),
             "net_gbp": Decimal("0.00"),
@@ -190,6 +193,8 @@ def _parse_row(bills, sheet, row, title_row):
     qunit = get_value(sheet, row, column_map[Title.QUNIT])
     charge = get_dec(sheet, row, column_map[Title.CHARGE]) / Decimal("100")
     total = get_dec(sheet, row, column_map[Title.TOTAL])
+    if bill["bill_type_code"] == "W":
+        quantity = quantity * -1
     if element_desc == "VAT":
         bill["net_gbp"] += quantity
         bill["vat_gbp"] += total

chellow/rate_server.py

@@ -87,8 +87,8 @@ def run_import(sess, log, set_progress):
         "chellow.e.bsuos",
         "chellow.e.ccl",
         "chellow.e.dno_rate_parser",
-        "chellow.e.laf_import",
         "chellow.e.triad",
+        "chellow.e.lafs",
         "chellow.gas.ccl",
         "chellow.gas.dn_rate_parser",
     ):

chellow/templates/e/lafs.html

@@ -17,7 +17,7 @@
 		<fieldset>
 			<legend>Show data</legend>	
 			<input type="hidden" name="llfc_id" value="{{llfc.id}}">
-			<label>Month</label> {{input_date(None, None, 'month')}}
+			<label>Month</label> {{input_date(resolution='month')}}
 			<input type="submit" value="Show">
 		</fieldset>
 	</form>

chellow/templates/e/llfc.html

@@ -45,7 +45,7 @@
 		<tr>
 			<th>Loss Adjustment Factors</th>
 			<td>
-				<a href="/e/lafs?llfc_id={{llfc.id}}&amp;year={{now.year}}&amp;month={{now.month}}">LAFs</a>
+				<a href="/e/lafs?llfc_id={{llfc.id}}&amp;timestamp_year={{now.year}}&amp;timestamp_month={{now.month}}">LAFs</a>
 			</td>
 		</tr>
 	</table>

chellow/views.py

@@ -61,7 +61,6 @@ import chellow.e.bill_importer
 import chellow.e.bsuos
 import chellow.e.computer
 import chellow.e.hh_importer
-import chellow.e.laf_import
 import chellow.e.lcc
 import chellow.e.mdd_importer
 import chellow.e.rcrc

{chellow-1753106469.0.0.dist-info → chellow-1753180539.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1753106469.0.0
+Version: 1753180539.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1753106469.0.0.dist-info → chellow-1753180539.0.0.dist-info}/RECORD

@@ -9,11 +9,11 @@ chellow/general_import.py,sha256=bm8FoaC9xUajGvJYShuS5GEwPwcL5eCF9D9g6o_AkB0,680
 chellow/models.py,sha256=n6vjsY6jgY98gOY5MGzKuHEEhOs73Pr3XO-auLeQ3es,244656
 chellow/national_grid.py,sha256=-c_vqNRtpNIQOcm0F1NDhS3_QUiOaLgEJYWzysSNc5Y,4369
 chellow/proxy.py,sha256=cVXIktPlX3tQ1BYcwxq0nJXKE6r3DtFTtfFHPq55HaM,1351
-chellow/rate_server.py,sha256=fg-Pf_9Hk3bXmC9riPQNGQxBvLvBa_WtNYdwDCjnCSg,5678
+chellow/rate_server.py,sha256=RwJo-AzBIdzxx7PAtboZEUH1nUjAeJckw0bk06-9oyM,5672
 chellow/rrun.py,sha256=1Kt2q_K9UoDG_nsZz-Q6XJiMNKroWqlqFdxn2M6Q8CA,2088
 chellow/testing.py,sha256=Dj2c1NX8lVlygueOrh2eyYawLW6qKEHxNhXVVUaNRO0,3637
 chellow/utils.py,sha256=i3GQK9MIcweosZk2gi-nX_IFq2DxURAJDyNoLBg6YwM,19421
-chellow/views.py,sha256=fPCuYTWXAAWnu1Cwt4Dk3k7PHNj9gw9HeQRiXgRIn60,85459
+chellow/views.py,sha256=oVZTSQ7k-VCwxzeDpj9sAfERtFw0rEHc58IxtvrhBi0,85431
 chellow/e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/aahedc.py,sha256=d2usudp7KYWpU6Pk3fal5EQ47EbvkvKeaFGylnb3NWw,606
 chellow/e/bill_importer.py,sha256=7UcnqNlKbJc2GhW9gy8sDp9GuqambJVpZLvbafOZztA,7411
@@ -24,7 +24,7 @@ chellow/e/cfd.py,sha256=CWLdYeNjgqT6Ro8YRf4vhwXIAJ2aV4Wi6HLNClVSeaQ,14260
 chellow/e/computer.py,sha256=lt4Zlr8EOClL1acuuvKA-tNPCI5RRHnAZbsaGQnB9j4,68835
 chellow/e/dno_rate_parser.py,sha256=NOVfS9HRDsc0rO282hU-IdrcuvMlC7VE8RySqd_5eT0,21762
 chellow/e/duos.py,sha256=RHrn93I1ASO2uYkuF18qlhG4p-jpuJhd_g3o69wtP4U,31004
-chellow/e/elexon.py,sha256=ALhXS9Es7PV0z9ukPbIramn3cf3iLyFi-PMWPSm5iOs,5487
+chellow/e/elexon.py,sha256=eOmX_wpxj6Ty6eaEgGAgXX-MgyJkYQSAT6DtJeP1rUE,5544
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
 chellow/e/hh_importer.py,sha256=adfogJgPWsJurPOVSBLLQ7m68s1LhfQmDYpAFnGm0eM,21510
 chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
@@ -34,7 +34,7 @@ chellow/e/hh_parser_schneider_csv.py,sha256=m8FHwXp1Tbas91RPS7TriCskSNRuSaFf1SD9
 chellow/e/hh_parser_schneider_xlsx.py,sha256=Vtq0TNz-oojoKJm4PeH4ZwBp2I-mjArB9-FL71ctCHs,3590
 chellow/e/hh_parser_simple_csv.py,sha256=lJx9tw9BWFSoBmns1Cws_vY-OIn90LPt2yvIN_CFcTE,2177
 chellow/e/hh_parser_vital_xlsx.py,sha256=g9-CElfH1PPfwpuUcVvD6WQpBlNxCo8j9pq_0Yza0ZM,4125
-chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
+chellow/e/lafs.py,sha256=SUUFtvn_IQQTrZue1zefCYgzuscA0FVX2ySZ9u8BDPw,7776
 chellow/e/lcc.py,sha256=OkpynN8_iAdHRlu-yyU6BhRUqYYOZsUnl0HbHULYo_4,4670
 chellow/e/mdd_importer.py,sha256=NugJr2JhuzkPTsEMl_5UdQuw5K2p8lVJ-hyz4MK6Hfg,35762
 chellow/e/rcrc.py,sha256=92CA1uIotIHd1epQ_jEPdJKzXqDFV-AoJOJeRO6MEyA,4274
@@ -44,7 +44,7 @@ chellow/e/system_price.py,sha256=6w5J7bzwFAZubE2zdOFRiS8IIrVP8hkoIOaG2yCt-Ic,623
 chellow/e/tlms.py,sha256=K2n-dF6qNW7VQp4m_sVK1HWnXarmAIzpotWXYqWQacY,9363
 chellow/e/tnuos.py,sha256=NBmc-f3oezrl4gviAKobljHfICTpBKxxxEGBGJi_lRk,4927
 chellow/e/triad.py,sha256=uQIngSrz8irBXQ0Rp_s8nAUzu-y2Ms7aj4B38_Ff8y8,13720
-chellow/e/views.py,sha256=tCzoVreBLEcNmBDbWWqhQ3qIp-0v93dBhF8yoDeT9mI,220738
+chellow/e/views.py,sha256=pWLbcG04t8mZB6y9Dp8bkcJpnWrfF1x10eCGagMbDl8,220804
 chellow/e/bill_parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/bill_parsers/activity_mop_stark_xlsx.py,sha256=opjXRrqrgBTbSKzL0JfTLP0fnz3DL3oRZZ4P0DifQ3I,4119
 chellow/e/bill_parsers/annual_mop_stark_xlsx.py,sha256=-HMoIfa_utXYKA44RuC0Xqv3vd2HLeQU_4P0iBUd3WA,4219
@@ -67,7 +67,7 @@ chellow/e/bill_parsers/settlement_dc_stark_xlsx.py,sha256=osCpUYUdLcPtlo7ngXWGw0
 chellow/e/bill_parsers/sse_edi.py,sha256=L85DOfNkqexeEIEr8pCBn_2sHJI-zEaw6cogpE3YyYM,15204
 chellow/e/bill_parsers/sww_xls.py,sha256=QEjiuvwvr5FuWCfqqVw8LaA_vZyAKsvRAS5fw3xtFhM,7533
 chellow/gas/bill_import.py,sha256=w0lPgK_Drzh8rtnEBQe3qFuxrgzZ6qQSgpaGrrGznMU,6549
-chellow/gas/bill_parser_bgs_xlsx.py,sha256=ySYAa6fq990DW01bQMk8Ix_sziWfm_5d2Il7IrYz9Ys,7522
+chellow/gas/bill_parser_bgs_xlsx.py,sha256=ZuLts8Y31gxtCQoXJq2CQRRMMpYbAWqsbfMB1S6fPqU,7757
 chellow/gas/bill_parser_csv.py,sha256=Ecdy-apFT-mWAxddAsM4k1s-9-FpIaOfjP0oFc0rdQg,5557
 chellow/gas/bill_parser_engie_edi.py,sha256=Ko0vZP-QdVQ1uuhS_5cdrii60_cM4b_LFJMoY0pZqnk,8950
 chellow/gas/bill_parser_total_edi.py,sha256=8HZH5Le24bVNFDc7vaKbauMaYR-n9P6u0ZG7gDdqbIA,11325
@@ -239,9 +239,9 @@ chellow/templates/e/generator_types.html,sha256=672cXDYewLAk32pzp6c8fi2P48FikxRr
 chellow/templates/e/gsp_group.html,sha256=yzLHZHQPHZAQtjLBszWEc-htw8-MEhSu3kLo-KmW5n8,405
 chellow/templates/e/gsp_groups.html,sha256=HagDJBJ7YzZDNhDq9ThI2uMrsW_viSuBapB2X4yC3Bk,453
 chellow/templates/e/hh_datum_edit.html,sha256=TJPvQ7FHw1yjkBwpXul0ZBs8UQtZR98wJ3fLjuQc2XA,1824
-chellow/templates/e/lafs.html,sha256=7MQvmoGCY-JCSDDos7vJVRKWAvqqM0eqhA0-hvaO1_o,996
+chellow/templates/e/lafs.html,sha256=QcQnkK_Nk-6cCLOApWQSP4dN0mfuFxIB_K4lBtr0Yv4,995
 chellow/templates/e/lcc.html,sha256=LLWgB9iNG2jee3p1DrbeDtUR3CPQIXlyDvGQDUsWKEs,1021
-chellow/templates/e/llfc.html,sha256=B0tw7PHvEEvxJWhAGe9dZji_J9apAz-TpNlkqvYFKfE,1168
+chellow/templates/e/llfc.html,sha256=vrGoR_7yWli34Ul6xJO1z8FvQU2rjOrOaWlyx-2irus,1188
 chellow/templates/e/llfc_edit.html,sha256=YwtEf8DwoX_t3RHNxPbZYFzikBL-Ly7UyjHs-YuJ3bk,1038
 chellow/templates/e/llfcs.html,sha256=OlY9x3XKs0OAoa4ZOZoRokYEjIZ_isfoBtn3DGL-u6o,1228
 chellow/templates/e/market_role.html,sha256=hQ62PwzByIKDfYRWMLsg8uuA2Yh6u_X9iaee5u31p80,1025
@@ -385,6 +385,6 @@ chellow/templates/g/supply_note_edit.html,sha256=b8mB6_ucBwoljp03iy6AgVaZUhGw3-1
 chellow/templates/g/supply_notes.html,sha256=6epNmZ3NKdXZz27fvmRUGeffg_oc1kmwuBeyRzQe3Rg,854
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1753106469.0.0.dist-info/METADATA,sha256=YMm8-MduiWKqPWeENv_u33zI391ioRp-tDbKrowYsVo,12585
-chellow-1753106469.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-chellow-1753106469.0.0.dist-info/RECORD,,
+chellow-1753180539.0.0.dist-info/METADATA,sha256=R7snYgxFJSS1VK-dHXFKW_4rRtTgBe4v6gzOXFUMX6A,12585
+chellow-1753180539.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+chellow-1753180539.0.0.dist-info/RECORD,,