chellow 1707396281.0.0__py3-none-any.whl → 1708358861.0.0__py3-none-any.whl

chellow/e/hh_importer.py

@@ -48,6 +48,7 @@ class HhDataImportProcess(threading.Thread):
         super().__init__(name=f"HH Manual Import: contract {dc_contract_id}")
 
         self.messages = []
+        self.error = None
         self.istream = istream
         self.dc_contract_id = dc_contract_id
         self.id = process_id
@@ -74,14 +75,16 @@ class HhDataImportProcess(threading.Thread):
                 parser_module = importlib.import_module(
                     f"chellow.e.hh_parser_{mod_name}"
                 )
-                self.converter = parser_module.create_parser(self.istream, mpan_map)
+                self.converter = parser_module.create_parser(
+                    self.istream, mpan_map, self.messages
+                )
                 sess.rollback()
                 HhDatum.insert(sess, self.converter, contract)
                 sess.commit()
             except BadRequest as e:
-                self.messages.append(e.description)
+                self.error = e.description
             except BaseException:
-                self.messages.append(f"Outer problem {traceback.format_exc()}")
+                self.error = f"Outer problem {traceback.format_exc()}"
 
     def get_status(self):
         return (
@@ -281,8 +284,8 @@ class HhImportTask(threading.Thread):
             for message in self.importer.messages:
                 self.log(message)
 
-            if len(self.importer.messages) > 0:
-                raise BadRequest("Problem loading file.")
+            if self.importer.error is not None:
+                raise BadRequest(f"Problem loading file. {self.importer.error}")
 
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)
@@ -384,12 +387,12 @@ class HhImportTask(threading.Thread):
 
             self.importer.run()
             messages = self.importer.messages
-            self.importer = None
             for message in messages:
                 self.log(message)
 
-            if len(messages) > 0:
-                raise BadRequest("Problem loading file.")
+            if self.importer.error is not None:
+                raise BadRequest(f"Problem loading file. {self.importer.error}")
+            self.importer = None
 
             contract = Contract.get_dc_by_id(sess, self.contract_id)
             contract.update_state(state)

chellow/e/hh_parser_bg_csv.py

@@ -8,12 +8,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import parse_mpan_core, utc_datetime
 
 
-def create_parser(reader, mpan_map):
-    return HhParserBglobal(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return HhParserBglobal(reader, mpan_map, messages)
 
 
 class HhParserBglobal:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.shredder = zip(itertools.count(1), csv.reader(reader))
         self.line_number, self.values = next(self.shredder)
         self.mpan_map = mpan_map

chellow/e/hh_parser_df2.py

@@ -6,12 +6,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import HH, parse_mpan_core, utc_datetime
 
 
-def create_parser(reader, mpan_map):
-    return StarkDf2HhParser(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return StarkDf2HhParser(reader, mpan_map, messages)
 
 
 class StarkDf2HhParser:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.line = self.line_number = None
         self.reader = zip(itertools.count(1), reader)
 
@@ -25,6 +25,9 @@ class StarkDf2HhParser:
             4: "REACTIVE_EXP",
         }
 
+        self.mpan_map = mpan_map
+        self.messages = messages
+
     def __iter__(self):
         return self
 
@@ -36,6 +39,15 @@ class StarkDf2HhParser:
                 lline = self.line.strip().upper()
                 if lline.startswith("#O"):
                     self.core = parse_mpan_core(lline[2:])
+                    if self.core in self.mpan_map and self.mpan_map[self.core] is None:
+                        msg = f"The MPAN core {self.core} has been ignored"
+                        if len(self.messages) == 0 or self.messages[-1] != msg:
+                            self.messages.append(
+                                f"The MPAN core {self.core} has been ignored"
+                            )
+                        self.core = None
+                        continue
+
                 elif lline.startswith("#S"):
                     sensor = int(lline[2:].strip())
                     try:
@@ -72,6 +84,10 @@ class StarkDf2HhParser:
                     except ValueError:
                         raise BadRequest("Problem parsing the value: " + fields[2])
                     status = fields[3][-1]
+
+                    if self.core is None:
+                        continue
+
                     local_datum = {
                         "mpan_core": self.core,
                         "channel_type": self.channel_type,

chellow/e/hh_parser_simple_csv.py

@@ -8,12 +8,12 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import parse_channel_type, parse_mpan_core, to_utc, validate_hh_start
 
 
-def create_parser(reader, mpan_map):
-    return HhParserCsvSimple(reader, mpan_map)
+def create_parser(reader, mpan_map, messages):
+    return HhParserCsvSimple(reader, mpan_map, messages)
 
 
 class HhParserCsvSimple:
-    def __init__(self, reader, mpan_map):
+    def __init__(self, reader, mpan_map, messages):
         self.shredder = zip(itertools.count(1), csv.reader(reader))
         next(self.shredder)  # skip the title line
         self.values = None

chellow/reports/report_233.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -10,20 +9,28 @@ from flask import g
 
 from sqlalchemy.sql.expression import true
 
-import chellow.dloads
-from chellow.models import Channel, Contract, Era, Session, Site, SiteEra, Snag, Supply
+from chellow.dloads import open_file
+from chellow.models import (
+    Channel,
+    Contract,
+    Era,
+    Session,
+    Site,
+    SiteEra,
+    Snag,
+    Supply,
+    User,
+)
 from chellow.utils import csv_make_val, hh_before, req_int, utc_datetime_now
 from chellow.views import chellow_redirect
 
 
-def content(contract_id, days_hidden, user):
+def content(contract_id, days_hidden, user_id):
     f = writer = None
     try:
         with Session() as sess:
-            running_name, finished_name = chellow.dloads.make_names(
-                "channel_snags.csv", user
-            )
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("channel_snags.csv", user, mode="w", newline="")
             writer = csv.writer(f, lineterminator="\n")
             titles = (
                 "Hidden Days",
@@ -111,13 +118,12 @@ def content(contract_id, days_hidden, user):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
 
 
 def do_get(sess):
     contract_id = req_int("dc_contract_id")
     days_hidden = req_int("days_hidden")
 
-    args = (contract_id, days_hidden, g.user)
+    args = contract_id, days_hidden, g.user.id
     threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

chellow/reports/report_33.py

@@ -1,5 +1,4 @@
 import csv
-import os
 import sys
 import threading
 import traceback
@@ -12,7 +11,12 @@ from sqlalchemy import or_, text
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql.expression import func, null
 
-import chellow.dloads
+from werkzeug.exceptions import BadRequest
+
+import chellow.e.duos
+import chellow.e.tnuos
+from chellow.dloads import open_file
+from chellow.e.computer import SupplySource
 from chellow.models import (
     Batch,
     Bill,
@@ -25,6 +29,7 @@ from chellow.models import (
     Session,
     SiteEra,
     Supply,
+    User,
 )
 from chellow.utils import (
     CHANNEL_TYPES,
@@ -40,10 +45,11 @@ from chellow.utils import (
 from chellow.views import chellow_redirect
 
 
-def content(running_name, finished_name, date, supply_id, mpan_cores):
+def content(user_id, date, supply_id, mpan_cores):
     try:
         with Session() as sess:
-            f = open(running_name, mode="w", newline="")
+            user = User.get_by_id(sess, user_id)
+            f = open_file("supplies_snapshot.csv", user, mode="w", newline="")
             _process(sess, f, date, supply_id, mpan_cores)
     except BaseException:
         msg = traceback.format_exc()
@@ -52,7 +58,354 @@ def content(running_name, finished_name, date, supply_id, mpan_cores):
     finally:
         if f is not None:
             f.close()
-            os.rename(running_name, finished_name)
+
+
+def _process_era(caches, sess, era_id, date, year_start):
+    NORMAL_READ_TYPES = ("N", "C", "N3")
+
+    era, supply, generator_type = (
+        sess.query(Era, Supply, GeneratorType)
+        .join(Supply, Era.supply_id == Supply.id)
+        .outerjoin(GeneratorType, Supply.generator_type_id == GeneratorType.id)
+        .filter(Era.id == era_id)
+        .options(
+            joinedload(Era.channels),
+            joinedload(Era.cop),
+            joinedload(Era.comm),
+            joinedload(Era.dc_contract),
+            joinedload(Era.exp_llfc),
+            joinedload(Era.exp_supplier_contract),
+            joinedload(Era.imp_llfc),
+            joinedload(Era.imp_supplier_contract),
+            joinedload(Era.mop_contract),
+            joinedload(Era.mtc_participant).joinedload(MtcParticipant.mtc),
+            joinedload(Era.mtc_participant).joinedload(MtcParticipant.meter_type),
+            joinedload(Era.pc),
+            joinedload(Era.site_eras).joinedload(SiteEra.site),
+            joinedload(Era.ssc),
+            joinedload(Era.energisation_status),
+            joinedload(Era.supply).joinedload(Supply.source),
+            joinedload(Era.supply).joinedload(Supply.gsp_group),
+            joinedload(Era.supply).joinedload(Supply.dno),
+        )
+        .one()
+    )
+
+    site_codes = []
+    site_names = []
+    for site_era in era.site_eras:
+        if site_era.is_physical:
+            physical_site = site_era.site
+        else:
+            site = site_era.site
+            site_codes.append(site.code)
+            site_names.append(site.name)
+
+    sup_eras = (
+        sess.query(Era).filter(Era.supply == supply).order_by(Era.start_date).all()
+    )
+    supply_start_date = sup_eras[0].start_date
+    supply_finish_date = sup_eras[-1].finish_date
+
+    if era.imp_mpan_core is None:
+        voltage_level_code = era.exp_llfc.voltage_level.code
+        is_substation = era.exp_llfc.is_substation
+    else:
+        voltage_level_code = era.imp_llfc.voltage_level.code
+        is_substation = era.imp_llfc.is_substation
+
+    if generator_type is None:
+        generator_type_str = ""
+    else:
+        generator_type_str = generator_type.code
+
+    metering_type = era.meter_category
+
+    if metering_type in ("nhh", "amr"):
+        latest_prev_normal_read = (
+            sess.query(RegisterRead)
+            .join(Bill)
+            .join(RegisterRead.previous_type)
+            .filter(
+                ReadType.code.in_(NORMAL_READ_TYPES),
+                RegisterRead.previous_date <= date,
+                Bill.supply_id == supply.id,
+            )
+            .order_by(RegisterRead.previous_date.desc())
+            .options(joinedload(RegisterRead.previous_type))
+            .first()
+        )
+
+        latest_pres_normal_read = (
+            sess.query(RegisterRead)
+            .join(Bill)
+            .join(RegisterRead.present_type)
+            .filter(
+                ReadType.code.in_(NORMAL_READ_TYPES),
+                RegisterRead.present_date <= date,
+                Bill.supply == supply,
+            )
+            .order_by(RegisterRead.present_date.desc())
+            .options(joinedload(RegisterRead.present_type))
+            .first()
+        )
+
+        if latest_prev_normal_read is None and latest_pres_normal_read is None:
+            latest_normal_read_date = None
+            latest_normal_read_type = None
+        elif latest_pres_normal_read is not None and latest_prev_normal_read is None:
+            latest_normal_read_date = latest_pres_normal_read.present_date
+            latest_normal_read_type = latest_pres_normal_read.present_type.code
+        elif latest_pres_normal_read is None and latest_prev_normal_read is not None:
+            latest_normal_read_date = latest_prev_normal_read.previous_date
+            latest_normal_read_type = latest_prev_normal_read.previous_type.code
+        elif (
+            latest_pres_normal_read.present_date > latest_prev_normal_read.previous_date
+        ):
+            latest_normal_read_date = latest_pres_normal_read.present_date
+            latest_normal_read_type = latest_pres_normal_read.present_type.code
+        else:
+            latest_normal_read_date = latest_prev_normal_read.previous_date
+            latest_normal_read_type = latest_prev_normal_read.previous_type.code
+        if latest_normal_read_date is not None:
+            latest_normal_read_date = hh_format(latest_normal_read_date)
+
+    else:
+        latest_normal_read_date = metering_type
+        latest_normal_read_type = None
+
+    mop_contract = era.mop_contract
+    mop_contract_name = mop_contract.name
+    mop_account = era.mop_account
+    latest_mop_bill_date = (
+        sess.query(Bill.finish_date)
+        .join(Batch)
+        .filter(
+            Bill.start_date <= date,
+            Bill.supply == supply,
+            Batch.contract == mop_contract,
+        )
+        .order_by(Bill.finish_date.desc())
+        .first()
+    )
+
+    if latest_mop_bill_date is not None:
+        latest_mop_bill_date = hh_format(latest_mop_bill_date[0])
+
+    dc_contract = era.dc_contract
+    dc_contract_name = dc_contract.name
+    dc_account = era.dc_account
+    latest_dc_bill_date = (
+        sess.query(Bill.finish_date)
+        .join(Batch)
+        .filter(
+            Bill.start_date <= date,
+            Bill.supply == supply,
+            Batch.contract == dc_contract,
+        )
+        .order_by(Bill.finish_date.desc())
+        .first()
+    )
+
+    if latest_dc_bill_date is not None:
+        latest_dc_bill_date = hh_format(latest_dc_bill_date[0])
+
+    channel_values = []
+    for imp_related in [True, False]:
+        for channel_type in CHANNEL_TYPES:
+            if era.find_channel(sess, imp_related, channel_type) is None:
+                channel_values.append("false")
+            else:
+                channel_values.append("true")
+
+    imp_avg_months = None
+    exp_avg_months = None
+    for is_import in [True, False]:
+        if metering_type == "nhh":
+            continue
+
+        params = {
+            "supply_id": supply.id,
+            "year_start": year_start,
+            "year_finish": date,
+            "is_import": is_import,
+        }
+        month_mds = tuple(
+            md[0] * 2
+            for md in sess.execute(
+                text(
+                    """
+
+select max(hh_datum.value) as md
+from hh_datum join channel on (hh_datum.channel_id = channel.id)
+join era on (channel.era_id = era.id)
+where era.supply_id = :supply_id and hh_datum.start_date >= :year_start
+and hh_datum.start_date <= :year_finish
+and channel.channel_type = 'ACTIVE'
+and channel.imp_related = :is_import
+group by extract(month from (hh_datum.start_date at time zone 'utc'))
+order by md desc
+limit 3
+
+"""
+                ),
+                params=params,
+            )
+        )
+
+        avg_months = sum(month_mds)
+        if len(month_mds) > 0:
+            avg_months /= len(month_mds)
+            if is_import:
+                imp_avg_months = avg_months
+            else:
+                exp_avg_months = avg_months
+
+    if (imp_avg_months is not None and imp_avg_months > 100) or (
+        exp_avg_months is not None and exp_avg_months > 100
+    ):
+        mandatory_hh = "yes"
+    else:
+        mandatory_hh = "no"
+
+    imp_latest_supplier_bill_date = None
+    exp_latest_supplier_bill_date = None
+    imp_tnuos_band = None
+    for is_import in (True, False):
+        for er in (
+            sess.query(Era)
+            .filter(Era.supply == era.supply, Era.start_date <= date)
+            .order_by(Era.start_date.desc())
+        ):
+            if is_import:
+                if er.imp_mpan_core is None:
+                    break
+                else:
+                    supplier_contract = er.imp_supplier_contract
+                    if era.energisation_status.code == "E":
+                        imp_ss = SupplySource(
+                            sess, date, date, date + HH, era, True, caches
+                        )
+                        chellow.e.duos.duos_vb(imp_ss)
+                        hh = imp_ss.hh_data[0]
+                        if hh["start-date"] >= chellow.e.tnuos.BANDED_START:
+                            imp_tnuos_band = chellow.e.tnuos.BAND_LOOKUP[
+                                hh["duos-description"]
+                            ]
+            else:
+                if er.exp_mpan_core is None:
+                    break
+                else:
+                    supplier_contract = er.exp_supplier_contract
+
+            latest_bill_date = (
+                sess.query(Bill.finish_date)
+                .join(Batch)
+                .filter(
+                    Bill.finish_date >= er.start_date,
+                    Bill.finish_date <= hh_min(er.finish_date, date),
+                    Bill.supply == supply,
+                    Batch.contract == supplier_contract,
+                )
+                .order_by(Bill.finish_date.desc())
+                .first()
+            )
+
+            if latest_bill_date is not None:
+                latest_bill_date = hh_format(latest_bill_date[0])
+
+                if is_import:
+                    imp_latest_supplier_bill_date = latest_bill_date
+                else:
+                    exp_latest_supplier_bill_date = latest_bill_date
+                break
+
+    meter_installation_date = (
+        sess.query(func.min(Era.start_date))
+        .filter(Era.supply == era.supply, Era.msn == era.msn)
+        .one()[0]
+    )
+
+    ssc = era.ssc
+    if ssc is None:
+        ssc_code = ssc_description = num_registers = None
+    else:
+        ssc_code, ssc_description = ssc.code, ssc.description
+        num_registers = (
+            sess.query(MeasurementRequirement)
+            .filter(MeasurementRequirement.ssc == ssc)
+            .count()
+        )
+
+    return (
+        [
+            date,
+            era.imp_mpan_core,
+            era.exp_mpan_core,
+            physical_site.code,
+            physical_site.name,
+            ", ".join(site_codes),
+            ", ".join(site_names),
+            supply.id,
+            supply.source.code,
+            generator_type_str,
+            supply.gsp_group.code,
+            supply.dno.dno_code,
+            voltage_level_code,
+            is_substation,
+            metering_type,
+            mandatory_hh,
+            era.pc.code,
+            era.mtc_participant.mtc.code,
+            era.cop.code,
+            era.comm.code,
+            ssc_code,
+            ssc_description,
+            era.energisation_status.code,
+            num_registers,
+            mop_contract_name,
+            mop_account,
+            dc_contract_name,
+            dc_account,
+            era.msn,
+            meter_installation_date,
+            latest_normal_read_date,
+            latest_normal_read_type,
+            latest_dc_bill_date,
+            latest_mop_bill_date,
+            supply_start_date,
+            supply_finish_date,
+            era.properties,
+            imp_tnuos_band,
+        ]
+        + channel_values
+        + [
+            era.imp_sc,
+            None if era.imp_llfc is None else era.imp_llfc.code,
+            None if era.imp_llfc is None else era.imp_llfc.description,
+            (
+                None
+                if era.imp_supplier_contract is None
+                else era.imp_supplier_contract.name
+            ),
+            era.imp_supplier_account,
+            imp_avg_months,
+            imp_latest_supplier_bill_date,
+        ]
+        + [
+            era.exp_sc,
+            None if era.exp_llfc is None else era.exp_llfc.code,
+            None if era.exp_llfc is None else era.exp_llfc.description,
+            (
+                None
+                if era.exp_supplier_contract is None
+                else era.exp_supplier_contract.name
+            ),
+            era.exp_supplier_account,
+            exp_avg_months,
+            exp_latest_supplier_bill_date,
+        ]
+    )
 
 
 def _process(sess, f, date, supply_id, mpan_cores):
@@ -95,6 +448,7 @@ def _process(sess, f, date, supply_id, mpan_cores):
         "Supply Start Date",
         "Supply Finish Date",
         "Properties",
+        "tnuos_band",
         "Import ACTIVE?",
         "Import REACTIVE_IMPORT?",
         "Import REACTIVE_EXPORT?",
@@ -118,8 +472,8 @@ def _process(sess, f, date, supply_id, mpan_cores):
     )
     writer.writerow(titles)
 
-    NORMAL_READ_TYPES = ("N", "C", "N3")
     year_start = date + HH - relativedelta(years=1)
+    caches = {}
 
     era_ids = (
         sess.query(Era.id)
@@ -141,342 +495,12 @@ def _process(sess, f, date, supply_id, mpan_cores):
         )
 
     for (era_id,) in era_ids:
-        era, supply, generator_type = (
-            sess.query(Era, Supply, GeneratorType)
-            .join(Supply, Era.supply_id == Supply.id)
-            .outerjoin(GeneratorType, Supply.generator_type_id == GeneratorType.id)
-            .filter(Era.id == era_id)
-            .options(
-                joinedload(Era.channels),
-                joinedload(Era.cop),
-                joinedload(Era.comm),
-                joinedload(Era.dc_contract),
-                joinedload(Era.exp_llfc),
-                joinedload(Era.exp_supplier_contract),
-                joinedload(Era.imp_llfc),
-                joinedload(Era.imp_supplier_contract),
-                joinedload(Era.mop_contract),
-                joinedload(Era.mtc_participant).joinedload(MtcParticipant.mtc),
-                joinedload(Era.mtc_participant).joinedload(MtcParticipant.meter_type),
-                joinedload(Era.pc),
-                joinedload(Era.site_eras).joinedload(SiteEra.site),
-                joinedload(Era.ssc),
-                joinedload(Era.energisation_status),
-                joinedload(Era.supply).joinedload(Supply.source),
-                joinedload(Era.supply).joinedload(Supply.gsp_group),
-                joinedload(Era.supply).joinedload(Supply.dno),
-            )
-            .one()
-        )
-
-        site_codes = []
-        site_names = []
-        for site_era in era.site_eras:
-            if site_era.is_physical:
-                physical_site = site_era.site
-            else:
-                site = site_era.site
-                site_codes.append(site.code)
-                site_names.append(site.name)
-
-        sup_eras = (
-            sess.query(Era).filter(Era.supply == supply).order_by(Era.start_date).all()
-        )
-        supply_start_date = sup_eras[0].start_date
-        supply_finish_date = sup_eras[-1].finish_date
-
-        if era.imp_mpan_core is None:
-            voltage_level_code = era.exp_llfc.voltage_level.code
-            is_substation = era.exp_llfc.is_substation
-        else:
-            voltage_level_code = era.imp_llfc.voltage_level.code
-            is_substation = era.imp_llfc.is_substation
-
-        if generator_type is None:
-            generator_type_str = ""
-        else:
-            generator_type_str = generator_type.code
-
-        metering_type = era.meter_category
-
-        if metering_type in ("nhh", "amr"):
-            latest_prev_normal_read = (
-                sess.query(RegisterRead)
-                .join(Bill)
-                .join(RegisterRead.previous_type)
-                .filter(
-                    ReadType.code.in_(NORMAL_READ_TYPES),
-                    RegisterRead.previous_date <= date,
-                    Bill.supply_id == supply.id,
-                )
-                .order_by(RegisterRead.previous_date.desc())
-                .options(joinedload(RegisterRead.previous_type))
-                .first()
-            )
-
-            latest_pres_normal_read = (
-                sess.query(RegisterRead)
-                .join(Bill)
-                .join(RegisterRead.present_type)
-                .filter(
-                    ReadType.code.in_(NORMAL_READ_TYPES),
-                    RegisterRead.present_date <= date,
-                    Bill.supply == supply,
-                )
-                .order_by(RegisterRead.present_date.desc())
-                .options(joinedload(RegisterRead.present_type))
-                .first()
-            )
-
-            if latest_prev_normal_read is None and latest_pres_normal_read is None:
-                latest_normal_read_date = None
-                latest_normal_read_type = None
-            elif (
-                latest_pres_normal_read is not None and latest_prev_normal_read is None
-            ):
-                latest_normal_read_date = latest_pres_normal_read.present_date
-                latest_normal_read_type = latest_pres_normal_read.present_type.code
-            elif (
-                latest_pres_normal_read is None and latest_prev_normal_read is not None
-            ):
-                latest_normal_read_date = latest_prev_normal_read.previous_date
-                latest_normal_read_type = latest_prev_normal_read.previous_type.code
-            elif (
-                latest_pres_normal_read.present_date
-                > latest_prev_normal_read.previous_date
-            ):
-                latest_normal_read_date = latest_pres_normal_read.present_date
-                latest_normal_read_type = latest_pres_normal_read.present_type.code
-            else:
-                latest_normal_read_date = latest_prev_normal_read.previous_date
-                latest_normal_read_type = latest_prev_normal_read.previous_type.code
-            if latest_normal_read_date is not None:
-                latest_normal_read_date = hh_format(latest_normal_read_date)
-
-        else:
-            latest_normal_read_date = metering_type
-            latest_normal_read_type = None
-
-        mop_contract = era.mop_contract
-        mop_contract_name = mop_contract.name
-        mop_account = era.mop_account
-        latest_mop_bill_date = (
-            sess.query(Bill.finish_date)
-            .join(Batch)
-            .filter(
-                Bill.start_date <= date,
-                Bill.supply == supply,
-                Batch.contract == mop_contract,
-            )
-            .order_by(Bill.finish_date.desc())
-            .first()
-        )
-
-        if latest_mop_bill_date is not None:
-            latest_mop_bill_date = hh_format(latest_mop_bill_date[0])
-
-        dc_contract = era.dc_contract
-        dc_contract_name = dc_contract.name
-        dc_account = era.dc_account
-        latest_dc_bill_date = (
-            sess.query(Bill.finish_date)
-            .join(Batch)
-            .filter(
-                Bill.start_date <= date,
-                Bill.supply == supply,
-                Batch.contract == dc_contract,
-            )
-            .order_by(Bill.finish_date.desc())
-            .first()
-        )
-
-        if latest_dc_bill_date is not None:
-            latest_dc_bill_date = hh_format(latest_dc_bill_date[0])
-
-        channel_values = []
-        for imp_related in [True, False]:
-            for channel_type in CHANNEL_TYPES:
-                if era.find_channel(sess, imp_related, channel_type) is None:
-                    channel_values.append("false")
-                else:
-                    channel_values.append("true")
-
-        imp_avg_months = None
-        exp_avg_months = None
-        for is_import in [True, False]:
-            if metering_type == "nhh":
-                continue
-
-            params = {
-                "supply_id": supply.id,
-                "year_start": year_start,
-                "year_finish": date,
-                "is_import": is_import,
-            }
-            month_mds = tuple(
-                md[0] * 2
-                for md in sess.execute(
-                    text(
-                        """
-
-select max(hh_datum.value) as md
-from hh_datum join channel on (hh_datum.channel_id = channel.id)
-    join era on (channel.era_id = era.id)
-where era.supply_id = :supply_id and hh_datum.start_date >= :year_start
-    and hh_datum.start_date <= :year_finish
-    and channel.channel_type = 'ACTIVE'
-    and channel.imp_related = :is_import
-group by extract(month from (hh_datum.start_date at time zone 'utc'))
-order by md desc
-limit 3
-
-"""
-                    ),
-                    params=params,
-                )
-            )
-
-            avg_months = sum(month_mds)
-            if len(month_mds) > 0:
-                avg_months /= len(month_mds)
-                if is_import:
-                    imp_avg_months = avg_months
-                else:
-                    exp_avg_months = avg_months
-
-        if (imp_avg_months is not None and imp_avg_months > 100) or (
-            exp_avg_months is not None and exp_avg_months > 100
-        ):
-            mandatory_hh = "yes"
-        else:
-            mandatory_hh = "no"
-
-        imp_latest_supplier_bill_date = None
-        exp_latest_supplier_bill_date = None
-        for is_import in (True, False):
-            for er in (
-                sess.query(Era)
-                .filter(Era.supply == era.supply, Era.start_date <= date)
-                .order_by(Era.start_date.desc())
-            ):
-                if is_import:
-                    if er.imp_mpan_core is None:
-                        break
-                    else:
-                        supplier_contract = er.imp_supplier_contract
-                else:
-                    if er.exp_mpan_core is None:
-                        break
-                    else:
-                        supplier_contract = er.exp_supplier_contract
-
-                latest_bill_date = (
-                    sess.query(Bill.finish_date)
-                    .join(Batch)
-                    .filter(
-                        Bill.finish_date >= er.start_date,
-                        Bill.finish_date <= hh_min(er.finish_date, date),
-                        Bill.supply == supply,
-                        Batch.contract == supplier_contract,
-                    )
-                    .order_by(Bill.finish_date.desc())
-                    .first()
-                )
-
-                if latest_bill_date is not None:
-                    latest_bill_date = hh_format(latest_bill_date[0])
-
-                    if is_import:
-                        imp_latest_supplier_bill_date = latest_bill_date
-                    else:
-                        exp_latest_supplier_bill_date = latest_bill_date
-                    break
-
-        meter_installation_date = (
-            sess.query(func.min(Era.start_date))
-            .filter(Era.supply == era.supply, Era.msn == era.msn)
-            .one()[0]
-        )
-
-        ssc = era.ssc
-        if ssc is None:
-            ssc_code = ssc_description = num_registers = None
-        else:
-            ssc_code, ssc_description = ssc.code, ssc.description
-            num_registers = (
-                sess.query(MeasurementRequirement)
-                .filter(MeasurementRequirement.ssc == ssc)
-                .count()
-            )
-
-        vals = (
-            [
-                date,
-                era.imp_mpan_core,
-                era.exp_mpan_core,
-                physical_site.code,
-                physical_site.name,
-                ", ".join(site_codes),
-                ", ".join(site_names),
-                supply.id,
-                supply.source.code,
-                generator_type_str,
-                supply.gsp_group.code,
-                supply.dno.dno_code,
-                voltage_level_code,
-                is_substation,
-                metering_type,
-                mandatory_hh,
-                era.pc.code,
-                era.mtc_participant.mtc.code,
-                era.cop.code,
-                era.comm.code,
-                ssc_code,
-                ssc_description,
-                era.energisation_status.code,
-                num_registers,
-                mop_contract_name,
-                mop_account,
-                dc_contract_name,
-                dc_account,
-                era.msn,
-                meter_installation_date,
-                latest_normal_read_date,
-                latest_normal_read_type,
-                latest_dc_bill_date,
-                latest_mop_bill_date,
-                supply_start_date,
-                supply_finish_date,
-                era.properties,
-            ]
-            + channel_values
-            + [
-                era.imp_sc,
-                None if era.imp_llfc is None else era.imp_llfc.code,
-                None if era.imp_llfc is None else era.imp_llfc.description,
-                (
-                    None
-                    if era.imp_supplier_contract is None
-                    else era.imp_supplier_contract.name
-                ),
-                era.imp_supplier_account,
-                imp_avg_months,
-                imp_latest_supplier_bill_date,
-            ]
-            + [
-                era.exp_sc,
-                None if era.exp_llfc is None else era.exp_llfc.code,
-                None if era.exp_llfc is None else era.exp_llfc.description,
-                (
-                    None
-                    if era.exp_supplier_contract is None
-                    else era.exp_supplier_contract.name
-                ),
-                era.exp_supplier_account,
-                exp_avg_months,
-                exp_latest_supplier_bill_date,
-            ]
-        )
+        try:
+            vals = _process_era(caches, sess, era_id, date, year_start)
+        except BaseException as e:
+            raise BadRequest(f"Problem with era {era_id}: {e}") from e
+        except BadRequest as e:
+            raise BadRequest(f"Problem with era {era_id}: {e.description}")
         writer.writerow([csv_make_val(v) for v in vals])
 
         # Avoid a long-running transaction
@@ -502,10 +526,6 @@ def do_get(session):
     else:
         mpan_cores = None
 
-    running_name, finished_name = chellow.dloads.make_names(
-        "supplies_snapshot.csv", user
-    )
-
-    args = (running_name, finished_name, date, supply_id, mpan_cores)
+    args = user.id, date, supply_id, mpan_cores
     threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

chellow/templates/e/dc_contract_edit.html

@@ -62,7 +62,11 @@
 	"hostname": "example.com",
 	"username": "username",
 	"password": "password",
-	"directories": ["downloads1", "downloads2"]}
+	"directories": ["downloads1", "downloads2"]
+	"mpan_map": {  \\ Optional
+	  "99 0993 2821 985": null,  \\ Ignore MPAN
+	},
+}
 			</pre>
 		</code>
 		<p>For the HTTPS protocol:</p>
@@ -81,6 +85,9 @@
       "name2": val2,
     }
   }
+	"mpan_map": {  \\ Optional
+	  "99 0993 2821 985": null,  \\ Ignore MPAN
+	},
 }
 
 				{% endraw %}

{chellow-1707396281.0.0.dist-info → chellow-1708358861.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: chellow
-Version: 1707396281.0.0
+Version: 1708358861.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1707396281.0.0.dist-info → chellow-1708358861.0.0.dist-info}/RECORD

@@ -24,10 +24,10 @@ chellow/e/dno_rate_parser.py,sha256=5MYEbcYngh7n6Szn2h9zLTwQKXH-WHKy6c80PYwNLLQ,
 chellow/e/duos.py,sha256=ISTcNqe9KNjVNM2Qs8IBoQxnmSXOt5W_G7tZxp4T78M,28870
 chellow/e/elexon.py,sha256=s8QdLUQMeylLhghxvvhLyOL3-0ifUnMfmo3vcbot_DY,5487
 chellow/e/energy_management.py,sha256=aXC2qlGt3FAODlNl_frWzVYAQrJLP8FFOiNX3m-QE_Y,12388
-chellow/e/hh_importer.py,sha256=G1CK3AQQuJaNXCxGEY3UU_CjqOzfVZT-PDR67b3zpA0,20362
-chellow/e/hh_parser_bg_csv.py,sha256=IOjM-QliKzvSXyhslmxz504XWnsGkVKEaCQ48SNeNeQ,2423
-chellow/e/hh_parser_df2.py,sha256=G4kt4N4mNjUa1Z7B1Ov6rAyZmWDyGEVFYscdghToMRo,3558
-chellow/e/hh_parser_simple_csv.py,sha256=lVRprIEjDpTQdeXm2xcLmRAyre2LWec8ueVwcCISZPo,2082
+chellow/e/hh_importer.py,sha256=nhmlWDNBm8U4salBud8ivdaDGF1lkY82NqQCrGuPYRI,20483
+chellow/e/hh_parser_bg_csv.py,sha256=W5SU2MSpa8BGA0VJw1JXF-IwbCNLFy8fe35yxLZ7gEw,2453
+chellow/e/hh_parser_df2.py,sha256=ynNNRMpLm9qs23tJjfm0vGSaAduXfPMw_7-WfQHXliQ,4209
+chellow/e/hh_parser_simple_csv.py,sha256=RN4QOLvTQeoPrpvXvQ9hkOBZnR5piybLfjCiSJdjpjs,2112
 chellow/e/laf_import.py,sha256=aqkcbjnvfBPszBLSNg6getP7iW1uWiTVHy6N5Z5x39U,5514
 chellow/e/lcc.py,sha256=q5iGbeYZdnozkIdbGuxXoJO-1KDO7AHqYOXEGBIaNHI,4963
 chellow/e/mdd_importer.py,sha256=9GN-at0DC3vOjd-0N2U856O3hInnjkMWWcRfxzQ5DjA,32042
@@ -76,12 +76,12 @@ chellow/reports/report_187.py,sha256=UvpaYHjyJFNV5puYq8_KxfzoBtVrwFgIGUOmC5oGA9A
 chellow/reports/report_215.py,sha256=_HZO_MhL51ttzmG6IrT8YVG6WmJMkRNWsXOW7AL7GWs,6766
 chellow/reports/report_219.py,sha256=o2eEg3mX9_raP2b4gjc2Gu4vqnMqcvvJBYQ1oQjxvpE,6637
 chellow/reports/report_231.py,sha256=E70sOYFS_Zc52LUfMY7SniQ1c6K3vPvDWvCvwaMUFrA,4210
-chellow/reports/report_233.py,sha256=KL9Qd3f-284w1Jj_ksUJ_OrE9Z_3RCz1zhRwZ9gdtLQ,4387
+chellow/reports/report_233.py,sha256=DLAmkoHFKH-N8OEIoX4jeCxAaCfJK0e7ssjFJJpPqzw,4334
 chellow/reports/report_241.py,sha256=se2V1tb5scvH9-RRuGG3U-IbqUYd-BZcNFwvTRjkbsw,5625
 chellow/reports/report_247.py,sha256=m2YgZY1YkAkSI2Qn3cNlFe2Gv8ToXfFs1zHiQd9RwbM,41556
 chellow/reports/report_29.py,sha256=KDFjgrLBv4WbG9efCdu_geMR7gT_QV9N97Wfdt7aDc4,2736
 chellow/reports/report_291.py,sha256=q04Y9EfMhQuEX4ObpmUs8vJTL76QwdG1A2-2HDVWhHQ,7587
-chellow/reports/report_33.py,sha256=LXLpsjfUTbCDuSKEE8o-LVpBLqRADpiGnyeilBgLGA0,16925
+chellow/reports/report_33.py,sha256=3WXPN0xV6PGMKY_Oc5bKR84JuiXirYlYdrghqIgYY_s,16675
 chellow/reports/report_387.py,sha256=KcClrEOK5IRvePp0Jgbzs1Gwf-_CqYmEtFFlXrr_Z3E,5670
 chellow/reports/report_41.py,sha256=QQeTshA1Og7N3wPaoZ8ynJzwsvZ1mgSFc7DDkVqIZoM,7447
 chellow/reports/report_429.py,sha256=0GCc0rQnOSG0fusw69yMMOCxnWApBd3P2sGWIg1py9M,12359
@@ -186,7 +186,7 @@ chellow/templates/e/dc_bill_edit.html,sha256=0GsN-ZIc4q-z_xs8igC2ZS6t4soo2SvB3qR
 chellow/templates/e/dc_bill_import.html,sha256=NHjMSoFizvFQIaPWuVE3nTCtMDTzJB0XmH8jXfV1tiA,2188
 chellow/templates/e/dc_bill_imports.html,sha256=lCaUR47r9KPr0VrQhEvVEaKexM5R_nmkxtzgpWZ0e9s,2135
 chellow/templates/e/dc_contract.html,sha256=_fq_s69kmbrGlScYngRYm4cPdRgh2roMvSfvG0J4xzA,2326
-chellow/templates/e/dc_contract_edit.html,sha256=BQPhcFqlBGaxBqinqxoeGZQPA6nmgfNAo9EcaOtycjM,2960
+chellow/templates/e/dc_contract_edit.html,sha256=wKK2pXN78VfSJ6gjfNRkikRFkYTiKtGL3UeVFIYSRwc,3115
 chellow/templates/e/dc_contract_hh_import.html,sha256=UODiBFNohb60MjH1w-9JW1JE0O9GR2uKPGw-lD7Da5g,848
 chellow/templates/e/dc_contract_hh_imports.html,sha256=eXFDGyzSgag4JRism81_p5yTzQOjCIXaVkQ8tl3dDcM,8172
 chellow/templates/e/dc_contracts.html,sha256=v8czpEcMzndngfWNXIKXib6Xrz9OwJjQAmNQNrQ6Y08,1705
@@ -363,6 +363,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1707396281.0.0.dist-info/METADATA,sha256=znQdcRupraUVzaIpiElWdAt0Zm8UtIuzJ1Vo7Ne4Mwk,12203
-chellow-1707396281.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
-chellow-1707396281.0.0.dist-info/RECORD,,
+chellow-1708358861.0.0.dist-info/METADATA,sha256=_bInYkXY1vjjvC7miNTD-mnFkrdyaxQ9nx3uOCNzYSE,12203
+chellow-1708358861.0.0.dist-info/WHEEL,sha256=TJPnKdtrSue7xZ_AVGkp9YXcvDrobsjBds1du3Nx6dc,87
+chellow-1708358861.0.0.dist-info/RECORD,,