chellow 1755614564.0.0__py3-none-any.whl → 1759155233.0.0__py3-none-any.whl

chellow/e/bill_parsers/csv.py

@@ -28,124 +28,162 @@ def parse_date(row, idx, is_finish=False):
         else:
             dt = to_utc(ct_datetime_parse(date_str, "%Y-%m-%d %H:%M"))
         return validate_hh_start(dt)
-    except ValueError as e:
+    except BaseException as e:
         raise BadRequest(
             f"Can't parse the date at column {idx}. The required spreadsheet format is "
             f"'YYYY-MM-DD HH:MM'. {e}"
         )
 
 
+def to_decimal(vals, dec_index, dec_name, is_money=False):
+    try:
+        dec_str = vals[dec_index]
+        dec_str = dec_str.replace(",", "")
+        dec = Decimal(dec_str)
+        if is_money:
+            dec += Decimal("0.00")
+            dec = round(dec, 2)
+        return dec
+    except InvalidOperation as e:
+        raise BadRequest(
+            f"The value '{dec_str}' can't be parsed as a decimal: {e}. It's in "
+            f"the '{dec_name}' column at position {dec_index} in {vals}."
+        )
+    except IndexError:
+        raise BadRequest(
+            f"The field '{dec_name}' can't be found. It's expected at "
+            f"position {dec_index} in the list of fields."
+        )
+    except ValueError:
+        raise BadRequest(
+            f"The {dec_name} field '{dec_str}' cannot be parsed as a number. The "
+            f"{dec_name} field is the {dec_index} field of {vals}."
+        )
+
+
+def _process_row(vals):
+    bill_type_code = vals[0]
+    account = vals[1]
+    mpan_core = parse_mpan_core(vals[2])
+    reference = vals[3]
+    issue_date = parse_date(vals, 4)
+    start_date = parse_date(vals, 5)
+    finish_date = parse_date(vals, 6, True)
+
+    kwh = to_decimal(vals, 7, "kwh")
+    net = to_decimal(vals, 8, "net", is_money=True)
+    vat = to_decimal(vals, 9, "vat", is_money=True)
+    gross = to_decimal(vals, 10, "gross", is_money=True)
+
+    if len(vals) > 11:
+        breakdown_str = vals[11].strip()
+        if len(breakdown_str) == 0:
+            breakdown = {}
+        else:
+            try:
+                breakdown = loads(breakdown_str)
+            except ZishLocationException as e:
+                raise BadRequest(str(e))
+    else:
+        raise BadRequest("There isn't a 'breakdown' field on the end.")
+
+    while vals[-1] == "" and len(vals) > 12:
+        del vals[-1]
+
+    reads = []
+    elements = []
+    i = 12
+    while i < len(vals):
+        typ = vals[i].strip().lower()
+        if typ == "read":
+            tpr_str = vals[i + 5].strip()
+            tpr_code = None if len(tpr_str) == 0 else tpr_str.zfill(5)
+            reads.append(
+                {
+                    "msn": vals[i + 1],
+                    "mpan": vals[i + 2],
+                    "coefficient": to_decimal(vals, i + 3, "coefficient"),
+                    "units": vals[i + 4],
+                    "tpr_code": tpr_code,
+                    "prev_date": parse_date(vals, i + 6),
+                    "prev_value": Decimal(vals[i + 7]),
+                    "prev_type_code": vals[i + 8],
+                    "pres_date": parse_date(vals, i + 9),
+                    "pres_value": Decimal(vals[i + 10]),
+                    "pres_type_code": vals[i + 11],
+                }
+            )
+            i += 12
+        elif typ == "element":
+            breakdown_str = vals[i + 5].strip()
+            if len(breakdown_str) == 0:
+                breakdown = {}
+            else:
+                try:
+                    breakdown = loads(breakdown_str)
+                except ZishLocationException as e:
+                    raise BadRequest(str(e))
+            elements.append(
+                {
+                    "name": vals[i + 1],
+                    "start_date": parse_date(vals, i + 2),
+                    "finish_date": parse_date(vals, i + 3),
+                    "net": to_decimal(vals, i + 4, "net", is_money=True),
+                    "breakdown": breakdown,
+                }
+            )
+            i += 6
+        else:
+            raise BadRequest("Record type {type} not recognized.")
+
+    if len(elements) == 0:
+        elements.append(
+            {
+                "name": "net",
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "net": net,
+                "breakdown": {},
+            }
+        )
+
+    return {
+        "bill_type_code": bill_type_code,
+        "account": account,
+        "mpan_core": mpan_core,
+        "reference": reference,
+        "issue_date": issue_date,
+        "start_date": start_date,
+        "finish_date": finish_date,
+        "kwh": kwh,
+        "net": net,
+        "vat": vat,
+        "gross": gross,
+        "breakdown": breakdown,
+        "reads": reads,
+        "elements": elements,
+    }
+
+
 class Parser:
     def __init__(self, f):
         self.reader = csv.reader(
-            StringIO(str(f.read(), "utf-8", errors="ignore")), skipinitialspace=True
+            StringIO(str(f.read(), "utf-8-sig", errors="ignore")), skipinitialspace=True
         )
         self.line_number = None
 
     def make_raw_bills(self):
-        raw_bills = []
-        next(iter(self.reader))  # skip title row
-        blank_set = set(("",))
         for self.line_number, self.vals in enumerate(self.reader, start=2):
             try:
-                # skip blank lines
-                if len(self.vals) == 0 or set(self.vals) == blank_set:
+                # skip blank lines and comment lines
+                if (
+                    len(self.vals) == 0
+                    or set(self.vals) == {""}
+                    or self.vals[0].strip().startswith("#")
+                ):
                     continue
-
-                bill_type_code = self.vals[0]
-                if bill_type_code.startswith("#"):
-                    continue  # skip comment lines
-                account = self.vals[1]
-                mpan_core = parse_mpan_core(self.vals[2])
-                reference = self.vals[3]
-                issue_date = parse_date(self.vals, 4)
-                start_date = parse_date(self.vals, 5)
-                finish_date = parse_date(self.vals, 6, True)
-
-                kwh = self.to_decimal(7, "kwh")
-                net = self.to_decimal(8, "net", True)
-                vat = self.to_decimal(9, "vat", True)
-                gross = self.to_decimal(10, "gross", True)
-
-                if len(self.vals) > 11:
-                    breakdown_str = self.vals[11].strip()
-                    if len(breakdown_str) == 0:
-                        breakdown = {}
-                    else:
-                        try:
-                            breakdown = loads(breakdown_str)
-                        except ZishLocationException as e:
-                            raise BadRequest(str(e))
-                else:
-                    raise BadRequest("There isn't a 'breakdown' field on the end.")
-
-                while self.vals[-1] == "" and len(self.vals) > 12:
-                    del self.vals[-1]
-
-                reads = []
-                for i in range(12, len(self.vals), 11):
-                    tpr_str = self.vals[i + 4].strip()
-                    tpr_code = None if len(tpr_str) == 0 else tpr_str.zfill(5)
-                    reads.append(
-                        {
-                            "msn": self.vals[i],
-                            "mpan": self.vals[i + 1],
-                            "coefficient": self.to_decimal(i + 2, "coefficient"),
-                            "units": self.vals[i + 3],
-                            "tpr_code": tpr_code,
-                            "prev_date": parse_date(self.vals, i + 5),
-                            "prev_value": Decimal(self.vals[i + 6]),
-                            "prev_type_code": self.vals[i + 7],
-                            "pres_date": parse_date(self.vals, i + 8),
-                            "pres_value": Decimal(self.vals[i + 9]),
-                            "pres_type_code": self.vals[i + 10],
-                        }
-                    )
-
-                raw_bill = {
-                    "bill_type_code": bill_type_code,
-                    "account": account,
-                    "mpan_core": mpan_core,
-                    "reference": reference,
-                    "issue_date": issue_date,
-                    "start_date": start_date,
-                    "finish_date": finish_date,
-                    "kwh": kwh,
-                    "net": net,
-                    "vat": vat,
-                    "gross": gross,
-                    "breakdown": breakdown,
-                    "reads": reads,
-                }
-                raw_bills.append(raw_bill)
+                yield _process_row(self.vals)
             except BadRequest as e:
                 raise BadRequest(
                     f"Problem at line {self.line_number} {self.vals}: {e.description}"
                 )
-
-        return raw_bills
-
-    def to_decimal(self, dec_index, dec_name, is_money=False):
-        try:
-            dec_str = self.vals[dec_index]
-            dec_str = dec_str.replace(",", "")
-            dec = Decimal(dec_str)
-            if is_money:
-                dec += Decimal("0.00")
-            return dec
-        except InvalidOperation as e:
-            raise BadRequest(
-                f"The value '{dec_str}' can't be parsed as a decimal: {e}. It's in "
-                f"the '{dec_name}' column at position {dec_index} in {self.vals}."
-            )
-        except IndexError:
-            raise BadRequest(
-                f"The field '{dec_name}' can't be found. It's expected at "
-                f"position {dec_index} in the list of fields."
-            )
-        except ValueError:
-            raise BadRequest(
-                f"The {dec_name} field '{dec_str}' cannot be parsed as a "
-                f"number. The {dec_name} field is the {dec_index} field of "
-                f"{self.vals}."
-            )

chellow/e/bill_parsers/drax_edi.py

@@ -24,131 +24,100 @@ read_type_map = {
 ELEMENT_MAP = {
     "AAH": {
         "AAHEDC": {
-            "139039": ("aahedc-gbp", "aahedc-rate", "aahedc-kwh"),
+            "139039": ("aahedc", "rate", "kwh"),
         },
     },
     "ADH": {
         "ADHOC": {
-            "020330": ("eii-gbp", None, None),
-            "064305": ("fit-gbp", None, None),
-            "590346": ("cfd-operational-gbp", None, None),
-            "637050": ("meter-rental-gbp", "meter-rental-rate", "meter-rental-days"),
+            "020330": ("eii", None, None),
+            "064305": ("fit", None, None),
+            "590346": ("cfd-operational", None, None),
+            "493988": ("reconciliation", None, None),
+            "637050": ("meter-rental", "rate", "days"),
         },
     },
     "BUS": {
         "BSUOS": {
-            "269100": ("bsuos-gbp", "bsuos-rate", "bsuos-kwh"),
+            "269100": ("bsuos", "rate", "kwh"),
         },
     },
     "CCL": {
         "CCL": {
-            "422733": ("ccl-gbp", "ccl-rate", "ccl-kwh"),
+            "422733": ("ccl", "rate", "kwh"),
         },
     },
     "CFD": {
         "CFD001": {
-            "273237": (
-                "cfd-operational-gbp",
-                "cfd-operational-rate",
-                "cfd-operational-kwh",
-            ),
-            "954379": ("cfd-interim-gbp", "cfd-interim-rate", "cfd-interim-kwh"),
-            "538249": (
-                "cm-settlement-levy-gbp",
-                "cm-settlement-levy-rate",
-                "cm-settlement-levy-kwh",
-            ),
-            "568307": ("capacity-gbp", "capacity-rate", "capacity-kwh"),
+            "273237": ("cfd-operational", "rate", "kwh"),
+            "954379": ("cfd-interim", "rate", "kwh"),
+            "538249": ("cm-settlement-levy", "rate", "kwh"),
+            "568307": ("capacity", "rate", "kwh"),
         },
     },
     "DCA": {
         "DCDA": {
-            "095469": ("meter-rental-gbp", "meter-rental-rate", "meter-rental-days"),
+            "095469": ("meter-rental", "rate", "days"),
         },
     },
     "DUS": {
         "DUS001": {
-            "794486": (
-                "duos-availability-gbp",
-                "duos-availability-rate",
-                "duos-availability-kva",
-            ),
-            "644819": ("duos-fixed-gbp", "duos-fixed-rate", "duos-fixed-days"),
-            "797790": (
-                "duos-reactive-gbp",
-                "duos-reactive-rate",
-                "duos-reactive-kvarh",
-            ),
-            "806318": ("duos-green-gbp", "duos-green-rate", "duos-green-kwh"),
-            "716514": ("duos-amber-gbp", "duos-amber-rate", "duos-amber-kwh"),
-            "769979": ("duos-red-gbp", "duos-red-rate", "duos-red-kwh"),
-            "709522": (
-                "duos-excess-availability-gbp",
-                "duos-excess-availability-rate",
-                "duos-excess-availability-kva",
-            ),
-            "209269": ("tnuos-gbp", "tnuos-rate", "tnuos-days"),
+            "794486": ("duos-availability", "rate", "kva"),
+            "644819": ("duos-fixed", "rate", "days"),
+            "797790": ("duos-reactive", "rate", "kvarh"),
+            "806318": ("duos-green", "rate", "kwh"),
+            "716514": ("duos-amber", "rate", "kwh"),
+            "769979": ("duos-red", "rate", "kwh"),
+            "709522": ("duos-excess-availability", "rate", "kva"),
+            "209269": ("tnuos", "rate", "days"),
         },
         "DUS002": {
-            "185913": ("duos-yellow-gbp", "duos-yellow-rate", "duos-yellow-kwh"),
-            "517270": ("duos-black-gbp", "duos-black-rate", "duos-black-kwh"),
-            "797790": (
-                "duos-reactive-gbp",
-                "duos-reactive-rate",
-                "duos-reactive-kvarh",
-            ),
-            "806318": ("duos-green-gbp", "duos-green-rate", "duos-green-kwh"),
-            "716514": ("duos-amber-gbp", "duos-amber-rate", "duos-amber-kwh"),
-            "709522": (
-                "duos-excess-availability-gbp",
-                "duos-excess-availability-rate",
-                "duos-excess-availability-kva",
-            ),
-            "769979": ("duos-red-gbp", "duos-red-rate", "duos-red-kwh"),
-            "644819": ("duos-fixed-gbp", "duos-fixed-rate", "duos-fixed-days"),
-            "794486": (
-                "duos-availability-gbp",
-                "duos-availability-rate",
-                "duos-availability-kva",
-            ),
-            "209269": ("tnuos-gbp", "tnuos-rate", "tnuos-days"),
-            "065950": ("eii-gbp", "eii-rate", "eii-kwh"),
+            "185913": ("duos-yellow", "rate", "kwh"),
+            "517270": ("duos-black", "rate", "kwh"),
+            "797790": ("duos-reactive", "rate", "kvarh"),
+            "806318": ("duos-green", "rate", "kwh"),
+            "716514": ("duos-amber", "rate", "kwh"),
+            "709522": ("duos-excess-availability", "rate", "kva"),
+            "769979": ("duos-red", "rate", "kwh"),
+            "644819": ("duos-fixed", "rate", "days"),
+            "794486": ("duos-availability", "rate", "kva"),
+            "209269": ("tnuos", "rate", "days"),
+            "065950": ("eii", "rate", "kwh"),
         },
         "DUSDIS": {
-            "122568": ("nrg-gsp-losses-gbp", "nrg-rate", "nrg-gsp-losses-kwh"),
+            "122568": ("nrg-gsp-losses", "rate", "kwh"),
         },
         "DUSTRN": {
-            "122568": ("nrg-nbp-losses-gbp", "nrg-rate", "nrg-nbp-losses-kwh"),
+            "122568": ("nrg-nbp-losses", "rate", "kwh"),
         },
     },
     "ELX": {
         "ELEXON": {
-            "489920": ("elexon-gbp", "elexon-rate", "elexon-nbp-kwh"),
+            "489920": ("elexon", "rate", "kwh"),
         },
     },
     "FIT": {
         "FIT_LV": {
-            "704107": ("fit-gbp", "fit-rate", "fit-kwh"),
+            "704107": ("fit", "rate", "kwh"),
         },
     },
     "NRG": {
         "HH0002": {
-            "033667": ("management-gbp", "management-rate", "management-kwh"),
-            "091890": ("shape-gbp", "shape-rate", "shape-kwh"),
-            "122568": ("nrg-msp-gbp", "nrg-rate", "nrg-msp-kwh"),
+            "033667": ("management", "rate", "kwh"),
+            "091890": ("shape", "rate", "kwh"),
+            "122568": ("nrg-msp", "rate", "kwh"),
         },
     },
     "REN": {
         "REN001": {
-            "229128": ("ro-gbp", "ro-rate", "ro-kwh"),
+            "229128": ("ro", "rate", "kwh"),
         },
         "REN002": {
-            "019090": ("rego-gbp", "rego-rate", "rego-kwh"),
+            "019090": ("rego", "rate", "kwh"),
         },
     },
     "TUS": {
         "TNUOS": {
-            "012069": ("triad-gbp", "triad-rate", "triad-kw"),
+            "012069": ("triad", "rate", "kw"),
         },
     },
 }
@@ -173,6 +142,7 @@ def _process_BCD(elements, headers):
     headers["issue_date"] = issue_date
     headers["bill_type_code"] = bill_type_code
     headers["reference"] = reference
+    headers["elements"] = []
 
 
 def _process_BTL(elements, headers):
@@ -194,6 +164,7 @@ def _process_BTL(elements, headers):
         "gross": Decimal("0.00") + to_decimal(tbtl) / Decimal("100"),
         "breakdown": headers["breakdown"],
         "reads": headers["reads"],
+        "elements": headers["elements"],
     }
 
 
@@ -311,13 +282,13 @@ def _process_CCD2(elements, headers):
 
 
 def _process_CCD3(elements, headers):
-    breakdown = headers["breakdown"]
+    elem_bd = {}
 
     supplier_code = elements["CCDE"][2]
     tariff_code = elements["TCOD"][0]
     mod_code = elements["TMOD"][0]
     try:
-        eln_gbp, eln_rate, eln_cons = ELEMENT_MAP[supplier_code][tariff_code][mod_code]
+        eln_name, eln_rate, eln_cons = ELEMENT_MAP[supplier_code][tariff_code][mod_code]
     except KeyError:
         raise BadRequest(
             f"Can't find the element key {supplier_code} -> {tariff_code} -> "
@@ -325,29 +296,35 @@ def _process_CCD3(elements, headers):
         )
 
     cons = elements["CONS"]
-    kwh = Decimal("0")
     if len(cons[0]) > 0:
         el_cons = to_decimal(cons) / Decimal("1000")
-        kwh = el_cons
-        breakdown[eln_cons] += kwh
+        elem_bd[eln_cons] = el_cons
+        if eln_name == "nrg-msp":
+            headers["kwh"] += el_cons
 
     bpri = elements["BPRI"]
     if len(bpri[0]) > 0:
         rate = to_decimal(bpri) / Decimal("100000")
-        if eln_rate in breakdown:
-            breakdown[eln_rate].add(rate)
-        else:
-            breakdown[eln_rate] = {rate}
+        elem_bd[eln_rate] = {rate}
 
+    net = Decimal("0.00")
     if "CTOT" in elements:
-        net = Decimal("0.00") + to_decimal(elements["CTOT"]) / Decimal("100")
-    else:
-        net = Decimal("0.00")
+        net += to_decimal(elements["CTOT"]) / Decimal("100")
 
-    breakdown[eln_gbp] += net
+    csdt = elements["CSDT"]
+    start_date = to_date(csdt[0])
+    cedt = elements["CEDT"]
+    finish_date = to_finish_date(cedt[0])
 
-    if eln_gbp == "nrg-msp-gbp":
-        headers["kwh"] += kwh
+    headers["elements"].append(
+        {
+            "name": eln_name,
+            "start_date": start_date,
+            "finish_date": finish_date,
+            "net": net,
+            "breakdown": elem_bd,
+        }
+    )
 
 
 def _process_CCD4(elements, headers):