chellow 1709805150.0.0__py3-none-any.whl → 1709887733.0.0__py3-none-any.whl

chellow/e/cfd.py

@@ -7,7 +7,7 @@ from sqlalchemy import null, or_, select
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.e.lcc import api_search, api_sql
+from chellow.e.lcc import api_records
 from chellow.models import Contract, RateScript
 from chellow.utils import c_months_u, ct_datetime, hh_format, to_ct, to_utc
 
@@ -114,12 +114,9 @@ def lcc_import(sess, log, set_progress, s):
     import_advanced_forecast_ilr_tra(sess, log, set_progress, s)
 
 
-def _quarters(s):
+def _quarters(log, s):
     quarter = {}
-    res_j = api_search(
-        s, "2fc2fad9-ad57-4901-982a-f92d4ef6c622", sort="Settlement_Date"
-    )
-    for record in res_j["result"]["records"]:
+    for record in api_records(log, s, "2fc2fad9-ad57-4901-982a-f92d4ef6c622"):
         settlement_date_str = record["Settlement_Date"]
         settlement_date_ct = to_ct(
             Datetime.strptime(settlement_date_str[:10], "%Y-%m-%d")
@@ -165,7 +162,7 @@ def import_in_period_tracking(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    for quarter in _quarters(s):
+    for quarter in _quarters(log, s):
         quarter_start = sorted(quarter.keys())[0]
         rs = sess.execute(
             select(RateScript).where(
@@ -206,12 +203,8 @@ def import_operational_costs_levy(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "44f41eac-61b3-4e8d-8c52-3eda7b8e8517", sort="Period_Start")
-    for record in res_j["result"]["records"][1:]:  # skip title row
-        period_start_str = record["Period_Start"]
-        period_start = to_utc(
-            to_ct(Datetime.strptime(period_start_str[:10], "%Y-%m-%d"))
-        )
+    for record in api_records(log, s, "44f41eac-61b3-4e8d-8c52-3eda7b8e8517", skip=1):
+        period_start = _parse_date(record["Period_Start"])
 
         rs = sess.execute(
             select(RateScript).where(
@@ -231,16 +224,14 @@ def import_operational_costs_levy(sess, log, set_progress, s):
 RUN_TYPES = ("II", "SF", "R1", "R2", "R3", "RF", "DF")
 
 
-def _reconciled_days(s, search_from):
+def _reconciled_days(log, s, search_from):
     runs = {}
-    res_j = api_sql(
-        s,
-        f"""SELECT * from "e0e163cb-ba36-416d-83fe-976992d61516"
-        WHERE "Settlement_Date" > '{search_from}' ORDER BY "Settlement_Date"
-        """,
-    )
-
-    records = res_j["result"]["records"]
+
+    records = [
+        r
+        for r in api_records(log, s, "e0e163cb-ba36-416d-83fe-976992d61516")
+        if r["Settlement_Date"] > search_from
+    ]
     if len(records) > 0:
         prev_settlement_date = _parse_date(records[0]["Settlement_Date"])
         runs = {}
@@ -257,9 +248,9 @@ def _reconciled_days(s, search_from):
         yield settlement_date, runs
 
 
-def _reconciled_daily_quarters(s, search_from):
+def _reconciled_daily_quarters(log, s, search_from):
     quarter = {}
-    for settlement_date, lcc_runs in _reconciled_days(s, search_from):
+    for settlement_date, lcc_runs in _reconciled_days(log, s, search_from):
         try:
             runs = quarter[settlement_date]
         except KeyError:
@@ -317,7 +308,7 @@ def import_reconciled_daily_levy_rates(sess, log, set_progress, s):
         else:
             break
 
-    for quarter in _reconciled_daily_quarters(s, search_from):
+    for quarter in _reconciled_daily_quarters(log, s, search_from):
         quarter_start = sorted(quarter.keys())[0]
         rs = sess.execute(
             select(RateScript).where(
@@ -377,10 +368,8 @@ def import_forecast_ilr_tra(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "fbece4ce-7cfc-42b7-8fb2-387cf59a3c32", sort="Period_Start")
-    for record in res_j["result"]["records"]:
-        period_start_str = record["Period_Start"]
-        period_start = _parse_varying_date(period_start_str)
+    for record in api_records(log, s, "fbece4ce-7cfc-42b7-8fb2-387cf59a3c32"):
+        period_start = _parse_varying_date(record["Period_Start"])
 
         rs = sess.execute(
             select(RateScript).where(
@@ -408,10 +397,8 @@ def import_advanced_forecast_ilr_tra(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "e3ad6876-c1e9-46f9-b557-cb9bdae53885", sort="Period Start")
-    for record in res_j["result"]["records"]:
-        period_start_str = record["Period Start"]
-        period_start = _parse_varying_date(period_start_str)
+    for record in api_records(log, s, "e3ad6876-c1e9-46f9-b557-cb9bdae53885"):
+        period_start = _parse_varying_date(record["Period Start"])
 
         rs = sess.execute(
             select(RateScript).where(

chellow/e/lcc.py

@@ -19,10 +19,11 @@ from chellow.utils import ct_datetime_now, hh_format, utc_datetime_now
 importer = None
 
 
-def api_sql(s, sql):
-    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search_sql"
-    params = {"sql": sql}
+def api_records(log, s, resource_id, skip=0):
+    url = f"https://dp.lowcarboncontracts.uk/datastore/dump/{resource_id}"
+    params = {"format": "json"}
     res = s.get(url, params=params)
+    log(f"Requested URL {res.url}")
     try:
         res_j = res.json()
     except requests.exceptions.JSONDecodeError as e:
@@ -34,27 +35,10 @@ def api_sql(s, sql):
     if "success" in res_j and not res_j["success"]:
         raise BadRequest(res_j)
 
-    return res_j
+    field_titles = [f["id"] for f in res_j["fields"]]
 
-
-def api_search(s, resource_id, sort=None):
-    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search"
-    params = {"resource_id": resource_id}
-    if sort is not None:
-        params["sort"] = sort
-    res = s.get(url, params=params)
-    try:
-        res_j = res.json()
-    except requests.exceptions.JSONDecodeError as e:
-        raise BadRequest(
-            f"Couldn't parse as JSON the content from {url} with error {e}: "
-            f"{res.text}"
-        )
-
-    if "success" in res_j and not res_j["success"]:
-        raise BadRequest(res_j)
-
-    return res_j
+    for record in res_j["records"][skip:]:
+        yield {k: v for k, v in zip(field_titles, record)}
 
 
 def run_import(sess, log, set_progress):
@@ -72,6 +56,7 @@ GLOBAL_ALERT = (
     "There's a problem with a <a href='/e/lcc'>Low Carbon Contracts import</a>."
 )
 LCC_STATE_KEY = "lcc"
+DELAY_DAYS = 7
 
 
 class LowCarbonContracts(threading.Thread):
@@ -113,7 +98,9 @@ class LowCarbonContracts(threading.Thread):
                     sess.rollback()
 
             last_run = lcc_state.get(LAST_RUN_KEY)
-            if last_run is None or utc_datetime_now() - last_run > timedelta(days=1):
+            if last_run is None or utc_datetime_now() - last_run > timedelta(
+                days=DELAY_DAYS
+            ):
                 self.going.set()
 
             if self.going.is_set():
@@ -143,7 +130,8 @@ class LowCarbonContracts(threading.Thread):
             else:
                 self.log(
                     f"The importer was last run at {hh_format(last_run)}. There will "
-                    f"be another import when 24 hours have elapsed since the last run."
+                    f"be another import when {DELAY_DAYS} days have elapsed since the "
+                    f"last run."
                 )
                 self.going.wait(60 * 60)
 

{chellow-1709805150.0.0.dist-info → chellow-1709887733.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1709805150.0.0
+Version: 1709887733.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1709805150.0.0.dist-info → chellow-1709887733.0.0.dist-info}/RECORD

@@ -18,7 +18,7 @@ chellow/e/bill_importer.py,sha256=y1bpn49xDwltj0PRFowWsjAVamcD8eyxUbCTdGxEZiE,73
 chellow/e/bmarketidx.py,sha256=0JCBBnP3xfLq2eFjXO_u-gzHyTizab4Pp6PRK6OZLcw,7134
 chellow/e/bsuos.py,sha256=aqteIr0YdkWDLlfJIQ8bW4mDsUxffi5zhsUAzykkqtk,15442
 chellow/e/ccl.py,sha256=CYeFrg0TNF7Av49nqxa0guA8CMcJhdMpBRmOq8PH6G0,564
-chellow/e/cfd.py,sha256=3yRchycC-WZwgqryGQK3B59OcMGVIWgbxrkekUOy-9k,14914
+chellow/e/cfd.py,sha256=WdrnV15AX34bj__u66cLgq_LgzXBYjSzpHsGlsXfRKI,14418
 chellow/e/computer.py,sha256=YL1JFUYfFW-i9DODfvSnp6gTL_MwIfwoWj1TWtOsxPg,67105
 chellow/e/dno_rate_parser.py,sha256=5MYEbcYngh7n6Szn2h9zLTwQKXH-WHKy6c80PYwNLLQ,20983
 chellow/e/duos.py,sha256=ISTcNqe9KNjVNM2Qs8IBoQxnmSXOt5W_G7tZxp4T78M,28870
@@ -29,7 +29,7 @@ chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw
 chellow/e/hh_parser_df2.py,sha256=ynNNRMpLm9qs23tJjfm0vGSaAduXfPMw_7-WfQHXliQ,4209
 chellow/e/hh_parser_simple_csv.py,sha256=RN4QOLvTQeoPrpvXvQ9hkOBZnR5piybLfjCiSJdjpjs,2112
 chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
-chellow/e/lcc.py,sha256=K84sKH4eEZNDQysT1KmsL833X2OYhPxjGuz3EKYXgmw,4965
+chellow/e/lcc.py,sha256=OkpynN8_iAdHRlu-yyU6BhRUqYYOZsUnl0HbHULYo_4,4670
 chellow/e/mdd_importer.py,sha256=9GN-at0DC3vOjd-0N2U856O3hInnjkMWWcRfxzQ5DjA,32042
 chellow/e/rcrc.py,sha256=92CA1uIotIHd1epQ_jEPdJKzXqDFV-AoJOJeRO6MEyA,4274
 chellow/e/ro.py,sha256=dZKZv_9wXSWuwcb3jiKavoD_9ot-PZseNVeEEe0siLo,596
@@ -363,6 +363,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1709805150.0.0.dist-info/METADATA,sha256=HEpyPr9kzZdaC_dUL5VE8TgT4fb4A6pCzEFebhseQsU,12203
-chellow-1709805150.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-chellow-1709805150.0.0.dist-info/RECORD,,
+chellow-1709887733.0.0.dist-info/METADATA,sha256=KWA1a2icFUtb2cVmZqyg5vJtVMw9VCqogcAguScl6iQ,12203
+chellow-1709887733.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+chellow-1709887733.0.0.dist-info/RECORD,,