chellow 1724868336.0.0__tar.gz → 1725528079.0.0__tar.gz

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/PKG-INFO

@@ -1,24 +1,23 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1724868336.0.0
+Version: 1725528079.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
-Requires-Dist: cryptography==42.0.8
 Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
 Requires-Dist: odio==0.0.23
 Requires-Dist: openpyxl==3.1.2
-Requires-Dist: paramiko==3.4.0
+Requires-Dist: paramiko==3.4.1
 Requires-Dist: pep3143daemon==0.0.6
 Requires-Dist: pg8000==1.31.1
 Requires-Dist: pip>=9.0.1
 Requires-Dist: psutil==5.9.5
 Requires-Dist: pympler==1.0.1
-Requires-Dist: pypdf==3.17.0
+Requires-Dist: pypdf==4.3.1
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.32.2

chellow-1725528079.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1725528079.0.0"

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/chellow/e/mdd_importer.py

@@ -247,8 +247,6 @@ def _import_Market_Participant_Role(sess, rows, ctx):
         name = values[4]
         dno_code_str = values[14]
         dno_code = None if len(dno_code_str) == 0 else dno_code_str
-        if dno_code == "99":
-            continue
 
         if party is None:
             participant.insert_party(
@@ -324,7 +322,7 @@ def _import_Meter_Timeswitch_Class(sess, rows, ctx):
             has_comms_str = values[8]  # MTC Communication Indicator
             has_comms = parse_bool(has_comms_str)
             is_hh_str = values[9]  # MTC Type Indicator
-            is_hh = parse_bool(is_hh_str)
+            is_hh = is_hh_str == "H"
             tpr_count_str = values[10]  # TPR Count
             tpr_count = None if tpr_count_str == "" else int(tpr_count_str)
 
@@ -837,7 +835,101 @@ def _import_Valid_MTC_LLFC_SSC_PC_Combination(sess, rows, ctx):
             )
 
 
-def rate_server_import(sess, log, set_progress, s, paths):
+def add_88_99(sess, ctx):
+    rows = [
+        [
+            "CIDC",
+            "R",
+            "01/01/2000",
+            "",
+            "Virtual",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "88",
+        ],
+        [
+            "CROW",
+            "R",
+            "01/04/1996",
+            "",
+            "Non-settlement",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "99",
+        ],
+    ]
+    _import_Market_Participant_Role(sess, rows, ctx)
+    rows = [
+        ["845", "01/04/1996", "CIDC", "01/04/1996", ""],
+        ["845", "01/04/1996", "CROW", "01/04/1996", ""],
+    ]
+    _import_MTC_in_PES_Area(sess, rows, ctx)
+    rows = [
+        ["CIDC", "", "", "005", "01/04/2010", "EG_HV_Intermittent", "C", ""],
+        ["CIDC", "", "", "006", "01/04/2010", "EG_HV_Non_Intermittent", "C", ""],
+        ["CIDC", "", "", "110", "01/01/2000", "Profile 3 Unrestricted", "A", ""],
+        ["CIDC", "", "", "210", "01/01/2000", "Profile 4 Economy 7", "A", ""],
+        ["CIDC", "", "", "510", "01/01/2000", "PC 5-8 & HH HV", "A", ""],
+        ["CIDC", "", "", "521", "01/01/2000", "Export (HV)", "C", ""],
+        [
+            "CIDC",
+            "",
+            "",
+            "524",
+            "01/04/2010",
+            "HV Generation Non intermittent",
+            "C",
+            "",
+        ],
+        ["CIDC", "", "", "570", "01/01/2000", "PC 5-8 & HH LV", "A", ""],
+        ["CIDC", "", "", "581", "01/01/2000", "Export (LV)", "C", ""],
+        ["CIDC", "", "", "658", "01/04/1996", "HV HH metered", "A", ""],
+        ["CROW", "", "", "510", "01/04/1996", "PC 5-8 & HH HV", "A", ""],
+        ["CROW", "", "", "521", "01/04/1996", "Export (HV)", "C", ""],
+        ["CROW", "", "", "570", "01/04/1996", "PC 5-8 & HH LV", "A", ""],
+        ["CROW", "", "", "581", "01/04/1996", "Export (LV)", "C", ""],
+    ]
+
+    _import_Line_Loss_Factor_Class(sess, rows, ctx)
+    rows = [
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "005", "01/04/2010", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "006", "01/04/2010", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "110", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "210", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "510", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "521", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "524", "01/04/2010", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "570", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "581", "01/01/2000", ""],
+        ["845", "01/04/1996", "CIDC", "01/04/1996", "658", "01/01/2000", ""],
+        ["845", "01/04/1996", "CROW", "01/04/1996", "510", "01/04/1996", ""],
+        ["845", "01/04/1996", "CROW", "01/04/1996", "521", "01/04/1996", ""],
+        ["845", "01/04/1996", "CROW", "01/04/1996", "570", "01/04/1996", ""],
+        ["845", "01/04/1996", "CROW", "01/04/1996", "581", "01/04/1996", ""],
+    ]
+    _import_Valid_MTC_LLFC_Combination(sess, rows, ctx)
+
+
+def _get_csv_reader(s, url):
+    csv_file = StringIO(download(s, url).decode("utf8"))
+    return csv.reader(csv_file)
+
+
+def rate_server_import(sess, log, set_progress, s, paths, with_88_99=True):
     log("Starting to check for a new MDD version")
     mdd_entries = {}
     for path, url in paths:
@@ -872,8 +964,7 @@ def rate_server_import(sess, log, set_progress, s, paths):
     version = None
 
     for file_name, url in fl_entries.items():
-        csv_file = StringIO(download(s, url).decode("utf8"))
-        csv_reader = iter(csv.reader(csv_file))
+        csv_reader = iter(_get_csv_reader(s, url))
         next(csv_reader)  # Skip titles
 
         table_name_elements = file_name.split("_")
@@ -929,7 +1020,10 @@ def rate_server_import(sess, log, set_progress, s, paths):
             func(sess, gnames[tname], ctx)
         else:
             raise BadRequest(f"Can't find {tname} on the rate server.")
-
+    if with_88_99:
+        log("Starting to add MDD data for 88 and 99 DNOs")
+        add_88_99(sess, ctx)
+        log("Finished adding MDD data for 88 and 99 DNOs")
     config = Contract.get_non_core_by_name(sess, "configuration")
     state = config.make_state()
     state["mdd_version"] = version

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/chellow/e/views.py

@@ -4354,29 +4354,15 @@ def site_add_e_supply_form_get(site_id):
             EnergisationStatus.code
         )
         default_energisation_status = EnergisationStatus.get_by_code(g.sess, "E")
-
+        gsp_groups_q = select(GspGroup).order_by(GspGroup.code)
+        gsp_groups = g.sess.scalars(gsp_groups_q)
+        RateScriptAliasStart = aliased(RateScript)
+        RateScriptAliasFinish = aliased(RateScript)
         if "start_year" in request.values:
             start_date = req_date("start")
         else:
             start_date = to_utc(ct_datetime(ct_now.year, ct_now.month, ct_now.day))
 
-        site = Site.get_by_id(g.sess, site_id)
-        sources = g.sess.scalars(select(Source).order_by(Source.code))
-        source_id = req_int_none("source_id")
-        if source_id is None:
-            source = Source.get_by_code(g.sess, "grid")
-        else:
-            source = Source.get_by_id(g.sess, source_id)
-        generator_types = g.sess.query(GeneratorType).order_by(GeneratorType.code)
-        gsp_groups = g.sess.query(GspGroup).order_by(GspGroup.code)
-        eras = (
-            g.sess.query(Era)
-            .join(SiteEra)
-            .filter(SiteEra.site == site)
-            .order_by(Era.start_date.desc())
-        )
-        RateScriptAliasStart = aliased(RateScript)
-        RateScriptAliasFinish = aliased(RateScript)
         mop_contracts = g.sess.scalars(
             select(Contract)
             .join(MarketRole)
@@ -4431,12 +4417,15 @@ def site_add_e_supply_form_get(site_id):
             )
             .order_by(Contract.name)
         ).all()
-        pcs = g.sess.query(Pc).order_by(Pc.code)
-        pc_id = req_int_none("pc_id")
-        if pc_id is None:
-            pc = Pc.get_by_code(g.sess, "00")
+
+        site = Site.get_by_id(g.sess, site_id)
+        sources = g.sess.scalars(select(Source).order_by(Source.code))
+        source_id = req_int_none("source_id")
+        if source_id is None:
+            source = Source.get_by_code(g.sess, "grid")
         else:
-            pc = Pc.get_by_id(g.sess, pc_id)
+            source = Source.get_by_id(g.sess, source_id)
+        generator_types = g.sess.query(GeneratorType).order_by(GeneratorType.code)
 
         dnos = g.sess.scalars(
             select(Party)
@@ -4444,12 +4433,36 @@ def site_add_e_supply_form_get(site_id):
             .where(MarketRole.code == "R")
             .order_by(Party.dno_code)
         ).all()
+        pcs = g.sess.query(Pc).order_by(Pc.code)
         dno_id = req_int_none("dno_id")
         if dno_id is None:
             dno = dnos[0]
         else:
             dno = Party.get_by_id(g.sess, dno_id)
 
+        dno_contract = Contract.get_dno_by_name(g.sess, dno.dno_code)
+        dno_rate_script = dno_contract.find_rate_script_at(g.sess, start_date)
+        dno_props = dno_rate_script.make_script()
+        allowed_gsp_group_codes = list(dno_props.keys())
+
+        gsp_groups = g.sess.scalars(
+            select(GspGroup)
+            .where(GspGroup.code.in_(allowed_gsp_group_codes))
+            .order_by(GspGroup.code)
+        )
+
+        eras = (
+            g.sess.query(Era)
+            .join(SiteEra)
+            .filter(SiteEra.site == site)
+            .order_by(Era.start_date.desc())
+        )
+        pc_id = req_int_none("pc_id")
+        if pc_id is None:
+            pc = Pc.get_by_code(g.sess, "00")
+        else:
+            pc = Pc.get_by_id(g.sess, pc_id)
+
         participant = dno.participant
 
         if pc.code == "00":

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/chellow/models.py

@@ -1598,15 +1598,13 @@ class Contract(Base, PersistentClass):
         sess.delete(self)
 
     def find_rate_script_at(self, sess, date):
-        return (
-            sess.query(RateScript)
-            .filter(
+        return sess.scalars(
+            select(RateScript).where(
                 RateScript.contract == self,
                 RateScript.start_date <= date,
                 or_(RateScript.finish_date == null(), RateScript.finish_date >= date),
             )
-            .first()
-        )
+        ).first()
 
     def start_date(self):
         return self.start_rate_script.start_date

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/chellow/templates/e/dno.html

@@ -28,12 +28,12 @@
 				</td>
 			</tr>
 			<tr>
-				<td>Start Date</td>
-				<td>{{dno_contract.start_rate_script.start_date|hh_format }}</td>
+				<td>Valid From</td>
+				<td>{{dno.valid_from|hh_format }}</td>
 			</tr>
 			<tr>
-				<td>Finish Date</td>
-				<td>{{dno_contract.finish_rate_script.finish_date|hh_format }}</td>
+				<td>Valid To</td>
+				<td>{{dno.valid_to|hh_format }}</td>
 			</tr>
 			<tr>
 				<td>Custom Reports</td>

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/chellow/templates/e/site_add_e_supply_form.html

@@ -32,6 +32,14 @@
 		</select>
 	{% endif %}
 	<label>Meter Serial Number</label> {{input_text('msn')}}
+
+	<label>DNO</label>
+	<select name="dno_id">
+		{% for dno in dnos %}
+			{{input_option( 'dno_id', dno.id, dno.dno_code + ' ' + dno.name)}}
+		{% endfor %}
+	</select>
+
 	<label>GSP Group</label>
 	<select name="gsp_group_id">
 		{% for gsp_group in gsp_groups %}
@@ -90,13 +98,6 @@
 		{% endfor %}
 	</select>
 
-	<label>DNO</label>
-	<select name="dno_id">
-		{% for dno in dnos %}
-			{{input_option( 'dno_id', dno.id, dno.dno_code + ' ' + dno.name)}}
-		{% endfor %}
-	</select>
-
 	{% if pc.code == '00' %}
 		<input type="hidden" name="ssc_id" value="">
 	{% else %}

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
 ]
 dependencies = [
 	"odio==0.0.23",
-	"pypdf==3.17.0",
+	"pypdf==4.3.1",
 	"pg8000==1.31.1",
 	"Flask==2.3.3",
 	"SQLAlchemy==2.0.30",
@@ -27,8 +27,7 @@ dependencies = [
 	"waitress==2.1.2",
 	"pep3143daemon==0.0.6",
 	"pip>=9.0.1",
-	"paramiko==3.4.0",
-  "cryptography==42.0.8", # Remove this when paramiko upgraded
+	"paramiko==3.4.1",
 	"pympler==1.0.1",
 	"psutil==5.9.5",
 	"xlrd==2.0.1",

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/test/e/test_e_views.py

@@ -2724,6 +2724,7 @@ def test_site_add_e_supply_form_get(client, sess):
         sess, "Fusion Supplier 2000", participant, "", {}, vf, None, {}
     )
     dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    Contract.insert_dno(sess, dno.dno_code, participant, "", {}, vf, None, {})
     meter_type = MeterType.insert(sess, "C5", "COP 1-5", utc_datetime(2000, 1, 1), None)
     meter_payment_type = MeterPaymentType.insert(sess, "CR", "Credit", vf, None)
     mtc = Mtc.insert(sess, "845", False, True, vf, None)

{chellow-1724868336.0.0 → chellow-1725528079.0.0}/test/e/test_mdd_importer.py

@@ -1,5 +1,7 @@
 from sqlalchemy import select
 
+from utils import match
+
 from chellow.e.mdd_importer import (
     _import_MTC_in_PES_Area,
     _import_Market_Participant_Role,
@@ -23,10 +25,14 @@ from chellow.models import (
     MtcParticipant,
     MtcSsc,
     Participant,
+    Party,
     Pc,
+    Site,
     Ssc,
     Tpr,
     VoltageLevel,
+    insert_energisation_statuses,
+    insert_sources,
     insert_voltage_levels,
 )
 from chellow.utils import ct_datetime, to_utc, utc_datetime
@@ -452,7 +458,7 @@ def test_import_mdd(mocker, sess):
     paths = tuple((p, "") for p in path_list)
     s = mocker.Mock()
 
-    rate_server_import(sess, log, set_progress, s, paths)
+    rate_server_import(sess, log, set_progress, s, paths, with_88_99=False)
 
 
 def test_import_mdd_two_versions(mocker, sess):
@@ -505,4 +511,134 @@ def test_import_mdd_two_versions(mocker, sess):
     log = mocker.Mock()
     set_progress = mocker.Mock()
     s = mocker.Mock()
-    rate_server_import(sess, log, set_progress, s, paths)
+    rate_server_import(sess, log, set_progress, s, paths, with_88_99=False)
+
+
+def test_88(mocker, sess, client):
+    site = Site.insert(sess, "CI017", "Water Works")
+    insert_energisation_statuses(sess)
+    insert_sources(sess),
+    insert_voltage_levels(sess)
+    sess.commit()
+
+    log = mocker.Mock()
+    set_progress = mocker.Mock()
+
+    files = {
+        "2022/electricity/mdd/50/Market_Participant_Role_50.csv": [[]],
+        "2022/electricity/mdd/50/Clock_Interval_50.csv": [[]],
+        "2022/electricity/mdd/50/GSP_Group_50.csv": [[]],
+        "2022/electricity/mdd/50/Line_Loss_Factor_Class_50.csv": [[]],
+        "2022/electricity/mdd/50/Market_Participant_50.csv": [
+            [],
+            ["CIDC", "Independent"],
+            ["CROW", "Riddle"],
+        ],
+        "2022/electricity/mdd/50/Market_Role_50.csv": [[], ["R", "Distribution"]],
+        "2022/electricity/mdd/50/Measurement_Requirement_50.csv": [[]],
+        "2022/electricity/mdd/50/Meter_Timeswitch_Class_50.csv": [
+            [],
+            [
+                "845",
+                "01/04/1996",
+                "",
+                "HH COP5 And Above With Comms",
+                "T",
+                "F",
+                "C5",
+                "CR",
+                "F",
+                "F",
+                "",
+            ],
+        ],
+        "2022/electricity/mdd/50/MTC_in_PES_Area_50.csv": [[]],
+        "2022/electricity/mdd/50/MTC_Meter_Type_50.csv": [
+            [],
+            ["C5", "5th", "01/04/1996", ""],
+        ],
+        "2022/electricity/mdd/50/MTC_Payment_Type_50.csv": [
+            [],
+            ["CR", "Credit", "01/04/1996", ""],
+        ],
+        "2022/electricity/mdd/50/Profile_Class_50.csv": [
+            [],
+            ["00", "01/04/1996", "Half-hourly", "", ""],
+        ],
+        "2022/electricity/mdd/50/Standard_Settlement_Configuration_50.csv": [[]],
+        "2022/electricity/mdd/50/Time_Pattern_Regime_50.csv": [[]],
+        "2022/electricity/mdd/50/Valid_MTC_LLFC_Combination_50.csv": [[]],
+        "2022/electricity/mdd/50/Valid_MTC_SSC_Combination_50.csv": [[]],
+        "2022/electricity/mdd/50/Valid_MTC_LLFC_SSC_Combination_50.csv": [[]],
+        "2022/electricity/mdd/50/Valid_MTC_LLFC_SSC_PC_Combination_50.csv": [[]],
+    }
+
+    def mock_get_csv_reader(s, url):
+        return files[url]
+
+    mocker.patch(
+        "chellow.e.mdd_importer._get_csv_reader", side_effect=mock_get_csv_reader
+    )
+
+    path_list = (
+        ("2022", "electricity", "mdd", "50", "Market_Participant_Role_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Clock_Interval_50.csv"),
+        ("2022", "electricity", "mdd", "50", "GSP_Group_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Line_Loss_Factor_Class_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Market_Participant_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Market_Role_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Measurement_Requirement_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Meter_Timeswitch_Class_50.csv"),
+        ("2022", "electricity", "mdd", "50", "MTC_in_PES_Area_50.csv"),
+        ("2022", "electricity", "mdd", "50", "MTC_Meter_Type_50.csv"),
+        ("2022", "electricity", "mdd", "50", "MTC_Payment_Type_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Profile_Class_50.csv"),
+        (
+            "2022",
+            "electricity",
+            "mdd",
+            "50",
+            "Standard_Settlement_Configuration_50.csv",
+        ),
+        ("2022", "electricity", "mdd", "50", "Time_Pattern_Regime_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Valid_MTC_LLFC_Combination_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Valid_MTC_SSC_Combination_50.csv"),
+        ("2022", "electricity", "mdd", "50", "Valid_MTC_LLFC_SSC_Combination_50.csv"),
+        (
+            "2022",
+            "electricity",
+            "mdd",
+            "50",
+            "Valid_MTC_LLFC_SSC_PC_Combination_50.csv",
+        ),
+    )
+    paths = tuple((p, "/".join(p)) for p in path_list)
+    s = mocker.Mock()
+
+    rate_server_import(sess, log, set_progress, s, paths, with_88_99=True)
+    dno_88 = Party.get_dno_by_code(sess, "88", to_utc(ct_datetime(2020, 1, 1)))
+    dno_99 = Party.get_dno_by_code(sess, "99", to_utc(ct_datetime(2020, 1, 1)))
+    sess.commit()
+
+    query_string = {
+        "dno_id": dno_88.id,
+        "source_id": "",
+        "pc_id": "",
+        "mtc_participant_id": "",
+    }
+    response = client.get(
+        f"/e/sites/{site.id}/add_e_supply/form", query_string=query_string
+    )
+    patterns = []
+    match(response, 200, *patterns)
+    query_string = {
+        "dno_id": dno_99.id,
+        "source_id": "",
+        "pc_id": "",
+        "mtc_participant_id": "",
+    }
+    response = client.get(
+        f"/e/sites/{site.id}/add_e_supply/form", query_string=query_string
+    )
+    patterns = []
+    match(response, 200, *patterns)

chellow-1724868336.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1724868336.0.0"