chellow 1709657307.0.0__tar.gz → 1709887733.0.0__tar.gz

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1709657307.0.0
+Version: 1709887733.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1709887733.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1709887733.0.0"

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/chellow/e/cfd.py

@@ -7,7 +7,7 @@ from sqlalchemy import null, or_, select
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.e.lcc import api_search, api_sql
+from chellow.e.lcc import api_records
 from chellow.models import Contract, RateScript
 from chellow.utils import c_months_u, ct_datetime, hh_format, to_ct, to_utc
 
@@ -114,12 +114,9 @@ def lcc_import(sess, log, set_progress, s):
     import_advanced_forecast_ilr_tra(sess, log, set_progress, s)
 
 
-def _quarters(s):
+def _quarters(log, s):
     quarter = {}
-    res_j = api_search(
-        s, "2fc2fad9-ad57-4901-982a-f92d4ef6c622", sort="Settlement_Date"
-    )
-    for record in res_j["result"]["records"]:
+    for record in api_records(log, s, "2fc2fad9-ad57-4901-982a-f92d4ef6c622"):
         settlement_date_str = record["Settlement_Date"]
         settlement_date_ct = to_ct(
             Datetime.strptime(settlement_date_str[:10], "%Y-%m-%d")
@@ -165,7 +162,7 @@ def import_in_period_tracking(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    for quarter in _quarters(s):
+    for quarter in _quarters(log, s):
         quarter_start = sorted(quarter.keys())[0]
         rs = sess.execute(
             select(RateScript).where(
@@ -206,12 +203,8 @@ def import_operational_costs_levy(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "44f41eac-61b3-4e8d-8c52-3eda7b8e8517", sort="Period_Start")
-    for record in res_j["result"]["records"][1:]:  # skip title row
-        period_start_str = record["Period_Start"]
-        period_start = to_utc(
-            to_ct(Datetime.strptime(period_start_str[:10], "%Y-%m-%d"))
-        )
+    for record in api_records(log, s, "44f41eac-61b3-4e8d-8c52-3eda7b8e8517", skip=1):
+        period_start = _parse_date(record["Period_Start"])
 
         rs = sess.execute(
             select(RateScript).where(
@@ -231,16 +224,14 @@ def import_operational_costs_levy(sess, log, set_progress, s):
 RUN_TYPES = ("II", "SF", "R1", "R2", "R3", "RF", "DF")
 
 
-def _reconciled_days(s, search_from):
+def _reconciled_days(log, s, search_from):
     runs = {}
-    res_j = api_sql(
-        s,
-        f"""SELECT * from "e0e163cb-ba36-416d-83fe-976992d61516"
-        WHERE "Settlement_Date" > '{search_from}' ORDER BY "Settlement_Date"
-        """,
-    )
-
-    records = res_j["result"]["records"]
+
+    records = [
+        r
+        for r in api_records(log, s, "e0e163cb-ba36-416d-83fe-976992d61516")
+        if r["Settlement_Date"] > search_from
+    ]
     if len(records) > 0:
         prev_settlement_date = _parse_date(records[0]["Settlement_Date"])
         runs = {}
@@ -257,9 +248,9 @@ def _reconciled_days(s, search_from):
         yield settlement_date, runs
 
 
-def _reconciled_daily_quarters(s, search_from):
+def _reconciled_daily_quarters(log, s, search_from):
     quarter = {}
-    for settlement_date, lcc_runs in _reconciled_days(s, search_from):
+    for settlement_date, lcc_runs in _reconciled_days(log, s, search_from):
         try:
             runs = quarter[settlement_date]
         except KeyError:
@@ -317,7 +308,7 @@ def import_reconciled_daily_levy_rates(sess, log, set_progress, s):
         else:
             break
 
-    for quarter in _reconciled_daily_quarters(s, search_from):
+    for quarter in _reconciled_daily_quarters(log, s, search_from):
         quarter_start = sorted(quarter.keys())[0]
         rs = sess.execute(
             select(RateScript).where(
@@ -377,10 +368,8 @@ def import_forecast_ilr_tra(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "fbece4ce-7cfc-42b7-8fb2-387cf59a3c32", sort="Period_Start")
-    for record in res_j["result"]["records"]:
-        period_start_str = record["Period_Start"]
-        period_start = _parse_varying_date(period_start_str)
+    for record in api_records(log, s, "fbece4ce-7cfc-42b7-8fb2-387cf59a3c32"):
+        period_start = _parse_varying_date(record["Period_Start"])
 
         rs = sess.execute(
             select(RateScript).where(
@@ -408,10 +397,8 @@ def import_advanced_forecast_ilr_tra(sess, log, set_progress, s):
             sess, contract_name, "", {}, to_utc(ct_datetime(1996, 4, 1)), None, {}
         )
 
-    res_j = api_search(s, "e3ad6876-c1e9-46f9-b557-cb9bdae53885", sort="Period Start")
-    for record in res_j["result"]["records"]:
-        period_start_str = record["Period Start"]
-        period_start = _parse_varying_date(period_start_str)
+    for record in api_records(log, s, "e3ad6876-c1e9-46f9-b557-cb9bdae53885"):
+        period_start = _parse_varying_date(record["Period Start"])
 
         rs = sess.execute(
             select(RateScript).where(

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/chellow/e/lcc.py

@@ -19,10 +19,11 @@ from chellow.utils import ct_datetime_now, hh_format, utc_datetime_now
 importer = None
 
 
-def api_sql(s, sql):
-    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search_sql"
-    params = {"sql": sql}
+def api_records(log, s, resource_id, skip=0):
+    url = f"https://dp.lowcarboncontracts.uk/datastore/dump/{resource_id}"
+    params = {"format": "json"}
     res = s.get(url, params=params)
+    log(f"Requested URL {res.url}")
     try:
         res_j = res.json()
     except requests.exceptions.JSONDecodeError as e:
@@ -34,27 +35,10 @@ def api_sql(s, sql):
     if "success" in res_j and not res_j["success"]:
         raise BadRequest(res_j)
 
-    return res_j
+    field_titles = [f["id"] for f in res_j["fields"]]
 
-
-def api_search(s, resource_id, sort=None):
-    url = "https://dp.lowcarboncontracts.uk/api/3/action/datastore_search"
-    params = {"resource_id": resource_id}
-    if sort is not None:
-        params["sort"] = sort
-    res = s.get(url, params=params)
-    try:
-        res_j = res.json()
-    except requests.exceptions.JSONDecodeError as e:
-        raise BadRequest(
-            f"Couldn't parse as JSON the content from {url} with error {e}: "
-            f"{res.text}"
-        )
-
-    if "success" in res_j and not res_j["success"]:
-        raise BadRequest(res_j)
-
-    return res_j
+    for record in res_j["records"][skip:]:
+        yield {k: v for k, v in zip(field_titles, record)}
 
 
 def run_import(sess, log, set_progress):
@@ -69,9 +53,10 @@ def run_import(sess, log, set_progress):
 
 LAST_RUN_KEY = "last_run"
 GLOBAL_ALERT = (
-    "There's a problem with a <a href='/lcc'>Low Carbon Contracts import</a>."
+    "There's a problem with a <a href='/e/lcc'>Low Carbon Contracts import</a>."
 )
 LCC_STATE_KEY = "lcc"
+DELAY_DAYS = 7
 
 
 class LowCarbonContracts(threading.Thread):
@@ -113,7 +98,9 @@ class LowCarbonContracts(threading.Thread):
                     sess.rollback()
 
             last_run = lcc_state.get(LAST_RUN_KEY)
-            if last_run is None or utc_datetime_now() - last_run > timedelta(days=1):
+            if last_run is None or utc_datetime_now() - last_run > timedelta(
+                days=DELAY_DAYS
+            ):
                 self.going.set()
 
             if self.going.is_set():
@@ -143,7 +130,8 @@ class LowCarbonContracts(threading.Thread):
             else:
                 self.log(
                     f"The importer was last run at {hh_format(last_run)}. There will "
-                    f"be another import when 24 hours have elapsed since the last run."
+                    f"be another import when {DELAY_DAYS} days have elapsed since the "
+                    f"last run."
                 )
                 self.going.wait(60 * 60)
 

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/chellow/models.py

@@ -1683,8 +1683,17 @@ class Contract(Base, PersistentClass):
         batch = Batch(sess, self, reference, description)
         try:
             sess.add(batch)
-        except ProgrammingError:
-            raise BadRequest("There's already a batch with that reference.")
+            sess.flush()
+        except IntegrityError as e:
+            if (
+                'duplicate key value violates unique constraint "batch_reference_key"'
+                in str(e)
+            ):
+                raise BadRequest(
+                    f"There's already a batch with the reference '{reference}'."
+                ) from e
+            else:
+                raise e
         return batch
 
     def make_properties(self):
@@ -7309,6 +7318,21 @@ def db_upgrade_44_to_45(sess, root_path):
     sess.execute(text("ALTER TABLE g_era ALTER soq SET NOT NULL;"))
 
 
+def db_upgrade_45_to_46(sess, root_path):
+    sess.execute(
+        text(
+            "ALTER TABLE batch DROP CONSTRAINT IF EXISTS "
+            "batch_contract_id_reference_key;"
+        )
+    )
+    sess.execute(
+        text(
+            "CREATE UNIQUE INDEX IF NOT EXISTS batch_reference_key ON batch "
+            "(reference);"
+        )
+    )
+
+
 upgrade_funcs = [None] * 18
 upgrade_funcs.extend(
     [
@@ -7339,6 +7363,7 @@ upgrade_funcs.extend(
         db_upgrade_42_to_43,
         db_upgrade_43_to_44,
         db_upgrade_44_to_45,
+        db_upgrade_45_to_46,
     ]
 )
 

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/test/e/test_cfd.py

@@ -1,4 +1,9 @@
-from chellow.e.cfd import _find_quarter_rs, _reconciled_days, hh
+from chellow.e.cfd import (
+    _find_quarter_rs,
+    _reconciled_days,
+    hh,
+    import_forecast_ilr_tra,
+)
 from chellow.e.computer import SupplySource
 from chellow.models import (
     Comm,
@@ -28,19 +33,16 @@ from chellow.utils import ct_datetime, to_utc, utc_datetime
 
 def test_reconciled_days(mocker):
     s = mocker.Mock()
-    search_from = mocker.Mock()
-    result = {
-        "result": {
-            "records": [
-                {"Settlement_Date": "2023-03-30", "Settlement_Run_Type": "DF"},
-                {"Settlement_Date": "2023-03-30", "Settlement_Run_Type": "RF"},
-                {"Settlement_Date": "2023-03-31", "Settlement_Run_Type": "DF"},
-                {"Settlement_Date": "2023-03-31", "Settlement_Run_Type": "RF"},
-            ]
-        }
-    }
-    mocker.patch("chellow.e.cfd.api_sql", return_value=result)
-    actual = list(_reconciled_days(s, search_from))
+    search_from = "2023-01-01"
+    log = mocker.Mock()
+    result = [
+        {"Settlement_Date": "2023-03-30", "Settlement_Run_Type": "DF"},
+        {"Settlement_Date": "2023-03-30", "Settlement_Run_Type": "RF"},
+        {"Settlement_Date": "2023-03-31", "Settlement_Run_Type": "DF"},
+        {"Settlement_Date": "2023-03-31", "Settlement_Run_Type": "RF"},
+    ]
+    mocker.patch("chellow.e.cfd.api_records", return_value=result)
+    actual = list(_reconciled_days(log, s, search_from))
     expected = [
         (
             utc_datetime(2023, 3, 29, 23, 0),
@@ -81,6 +83,27 @@ def test_find_quarter_rs(sess):
     assert actual is None
 
 
+def test_import_forecast_ilr_tra(sess, mocker):
+    vf = to_utc(ct_datetime(1996, 1, 1))
+    participant = Participant.insert(sess, "CALB", "Calb")
+    market_role = MarketRole.insert(sess, "Z", "Non-core")
+    participant.insert_party(sess, market_role, "None core", vf, None, None)
+    sess.commit()
+
+    s = mocker.Mock()
+    mock_request = mocker.Mock()
+    req_j = {
+        "success": True,
+        "fields": [],
+        "records": [],
+    }
+    mock_request.json = mocker.Mock(return_value=req_j)
+    s.get = mocker.Mock(return_value=mock_request)
+    log = mocker.Mock()
+    set_progress = mocker.Mock()
+    import_forecast_ilr_tra(sess, log, set_progress, s)
+
+
 def test_hh(sess, mocker):
     vf = to_utc(ct_datetime(1996, 1, 1))
     site = Site.insert(sess, "CI017", "Water Works")

{chellow-1709657307.0.0 → chellow-1709887733.0.0}/test/test_models.py

@@ -916,6 +916,26 @@ def test_Contract_get_next_batch_details__no_suffix(mocker):
     assert desc == batch_description
 
 
+def test_Contract_insert_batch_duplicate(sess):
+    vf = to_utc(ct_datetime(1996, 1, 1))
+
+    market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    market_role_M = MarketRole.insert(sess, "M", "Mop")
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+
+    contract = Contract.insert_mop(sess, "Fusion", participant, "", {}, vf, None, {})
+    contract.insert_batch(sess, "a", "a ref")
+    sess.commit()
+
+    with pytest.raises(
+        BadRequest,
+        match="There's already a batch with the reference 'a'.",
+    ):
+        contract.insert_batch(sess, "a", "a ref")
+
+
 def test_Tpr_get_by_code_not_found(sess):
     with pytest.raises(
         BadRequest,

chellow-1709657307.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1709657307.0.0"