chellow 1755614564.0.0__py3-none-any.whl → 1759155233.0.0__py3-none-any.whl

chellow/e/bill_parsers/settlement_dc_stark_xlsx.py

@@ -4,11 +4,227 @@ from decimal import Decimal, InvalidOperation
 from dateutil.relativedelta import relativedelta
 
 from openpyxl import load_workbook
+from openpyxl.utils import get_column_letter
 
 from werkzeug.exceptions import BadRequest
 
 from chellow.models import Session
-from chellow.utils import parse_mpan_core, to_utc
+from chellow.utils import parse_mpan_core, to_ct, to_utc
+
+COPS = ("2", "3", "5", "10")
+
+COLUMN_MAP = {
+    "quarterly unmetered charge": "unmetered charge",
+    "no. cop 5/10 meters": "no. cop 5 meters",
+    "annual cop 5/10 dc/da rate": "annual cop 5 dc/da rate",
+    "monthly cop 3 charge": "cop 3 charge",
+    "cop 5/10 charge": "cop 5 charge",
+    "monthly cop 5/10 charge": "cop 5 charge",
+    "total": "grand total",
+    "total ex vat": "grand total",
+    "total in vat": "grand total 2",
+    "total inc vat": "grand total 2",
+    "vat @ 20 %": "vat @ 20%",
+    "vat @20%": "vat @ 20%",
+}
+for cop in COPS:
+    COLUMN_MAP[f"quarterly cop {cop} charge"] = f"cop {cop} charge"
+
+
+def make_column_lookup(sheet):
+    column_lookup = {}
+    for cell in sheet[11]:
+        title = str(cell.value).strip().lower()
+        try:
+            title = COLUMN_MAP[title]
+        except KeyError:
+            pass
+        if title in column_lookup:
+            title += " 2"
+        column_lookup[title] = get_column_letter(cell.column)
+    return column_lookup
+
+
+def get_ct_date(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    val = cell.value
+    if not isinstance(val, Datetime):
+        raise BadRequest(f"Problem reading {val} as a timestamp at {cell.coordinate}.")
+    return to_ct(val)
+
+
+def get_start_date(sheet, col, row):
+    dt = get_ct_date(sheet, col, row)
+    return None if dt is None else to_utc(dt)
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_dec(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    if cell.value is None:
+        return None
+    else:
+        try:
+            return Decimal(str(cell.value))
+        except InvalidOperation as e:
+            raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_gbp(sheet, col, row):
+    gbp = get_dec(sheet, col, row)
+    if gbp is None:
+        return gbp
+    else:
+        return Decimal("0.00") + round(gbp, 2)
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+def _process_row(sess, sheet, row, issue_date, cl):
+    mpan_core = parse_mpan_core(str(get_int(sheet, "B", row)))
+    start_date_ct = get_ct_date(sheet, cl["bill from"], row)
+    start_date = to_utc(start_date_ct)
+    finish_date_ct = get_ct_date(sheet, cl["bill to"], row) + relativedelta(
+        hours=23, minutes=30
+    )
+    finish_date = to_utc(finish_date_ct)
+
+    elements = []
+    days = (finish_date_ct - start_date_ct).days + 1
+
+    el_titles = [("unmetered", "unmetered quant", "unmetered rate", "unmetered charge")]
+    for cop in ("2", "3", "5", "10"):
+        el_titles.append(
+            (
+                cop,
+                f"no. cop {cop} meters",
+                f"annual cop {cop} dc/da rate",
+                f"cop {cop} charge",
+            )
+        )
+
+    for cop, mpans_title, rate_title, net_title in el_titles:
+        if mpans_title in cl:
+            mpans = get_int(sheet, cl[mpans_title], row)
+            rate = get_dec(sheet, cl[rate_title], row)
+            net = get_gbp(sheet, cl[net_title], row)
+            if net not in (None, 0):
+                elements.append(
+                    {
+                        "name": "mpan",
+                        "start_date": start_date,
+                        "finish_date": finish_date,
+                        "net": net,
+                        "breakdown": {
+                            "rate": {rate},
+                            "days": days,
+                            "mpan-days": mpans * days,
+                            "cop": {cop},
+                        },
+                    }
+                )
+
+    for typ in ("adhoc", "regular"):
+        hand_visits_title = f"no. hand held visits ({typ})"
+        if hand_visits_title in cl:
+            hand_visits = get_dec(sheet, cl[hand_visits_title], row)
+            hand_rate = get_dec(sheet, cl[f"hand held visit ({typ}) rate"], row)
+            hand_gbp = get_gbp(sheet, cl[f"hand held visit ({typ}) charge"], row)
+            if hand_gbp != 0:
+                elements.append(
+                    {
+                        "name": "activity",
+                        "start_date": start_date,
+                        "finish_date": finish_date,
+                        "net": hand_gbp,
+                        "breakdown": {
+                            "rate": {hand_rate},
+                            "activity-name": {"hand_held_visit"},
+                            "visits": hand_visits,
+                        },
+                    }
+                )
+
+    annual_visits_count = get_int(sheet, cl["no. annual site visits"], row)
+    annual_visits_rate = get_dec(sheet, cl["annual site visit rate"], row)
+    annual_visits_gbp = get_gbp(sheet, cl["annual site visit charge"], row)
+    if annual_visits_gbp != 0:
+        elements.append(
+            {
+                "name": "activitiy",
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "net": annual_visits_gbp,
+                "breakdown": {
+                    "rate": {annual_visits_rate},
+                    "count": annual_visits_count,
+                    "activity-name": {"annual_site_visit"},
+                },
+            }
+        )
+
+    breakdown = {
+        "raw_lines": [],
+        "settlement-status": ["settlement"],
+    }
+    net = get_gbp(sheet, cl["grand total"], row)
+    sum_el = sum(el["net"] for el in elements)
+    if net != sum_el:
+        elements.append(
+            {
+                "name": "activitiy",
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "net": net - sum_el,
+                "breakdown": {
+                    "activity-name": {"discrepancy"},
+                },
+            }
+        )
+
+    vat = Decimal("0.00")
+    if "vat @ 20%" in cl:
+        vat += get_gbp(sheet, cl["vat @ 20%"], row)
+    gross = Decimal("0.00")
+    if "grand total 2" in cl:
+        gross += get_gbp(sheet, cl["grand total 2"], row)
+
+    return {
+        "bill_type_code": "N",
+        "kwh": Decimal(0),
+        "net": net,
+        "vat": vat,
+        "gross": gross,
+        "reads": [],
+        "breakdown": breakdown,
+        "account": mpan_core,
+        "issue_date": issue_date,
+        "start_date": start_date,
+        "finish_date": finish_date,
+        "mpan_core": mpan_core,
+        "reference": "_".join(
+            (
+                start_date_ct.strftime("%Y%m%d"),
+                finish_date_ct.strftime("%Y%m%d"),
+                to_ct(issue_date).strftime("%Y%m%d"),
+                mpan_core,
+            )
+        ),
+        "elements": elements,
+    }
 
 
 class Parser:
@@ -31,139 +247,26 @@ class Parser:
             self._title_line = line
         return line
 
-    def get_ct_date(self, col, row):
-        cell = self.get_cell(col, row)
-        val = cell.value
-        if not isinstance(val, Datetime):
-            raise BadRequest(
-                f"Problem reading {val} as a timestamp at {cell.coordinate}."
-            )
-        return val
-
-    def get_start_date(self, col, row):
-        dt = self.get_ct_date(col, row)
-        return None if dt is None else to_utc(dt)
-
-    def get_cell(self, col, row):
-        try:
-            coordinates = f"{col}{row}"
-            return self.sheet[coordinates]
-        except IndexError:
-            raise BadRequest(
-                f"Can't find the cell {coordinates} on sheet {self.sheet}."
-            )
-
-    def get_str(self, col, row):
-        return self.get_cell(col, row).value.strip()
-
-    def get_dec(self, col, row):
-        cell = self.get_cell(col, row)
-        try:
-            return Decimal(str(cell.value))
-        except InvalidOperation as e:
-            raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
-
-    def get_int(self, col, row):
-        return int(self.get_cell(col, row).value)
-
     def make_raw_bills(self):
-        row_index = None
         try:
             with Session() as sess:
                 bills = []
-                issue_date_str = self.get_str("A", 7)
-                issue_date = Datetime.strptime(issue_date_str[6:], "%d/%m/%Y %H:%M:%S")
+                issue_date_str = get_str(self.sheet, "A", 7)
+                issue_date = to_utc(
+                    to_ct(Datetime.strptime(issue_date_str[6:-3], "%d/%m/%Y %H:%M"))
+                )
+                column_lookup = make_column_lookup(self.sheet)
+
                 for row in range(12, len(self.sheet["A"]) + 1):
-                    val = self.get_cell("A", row).value
+                    val = get_cell(self.sheet, "A", row).value
                     if val is None or val == "":
-                        break
+                        continue
 
-                    self._set_last_line(row_index, val)
-                    mpan_core = parse_mpan_core(str(self.get_int("B", row)))
-                    start_date = self.get_start_date("D", row)
-                    finish_date = self.get_start_date("E", row) + relativedelta(
-                        hours=23, minutes=30
-                    )
-
-                    net = round(self.get_dec("AO", row), 2)
-
-                    cop_3_meters = self.get_int("G", row)
-                    cop_3_rate = self.get_dec("H", row)
-                    cop_3_gbp = self.get_dec("I", row)
-
-                    # Cop 5 meters
-                    self.get_int("J", row)
-                    cop_5_rate = self.get_dec("K", row)
-                    cop_5_gbp = self.get_dec("L", row)
-
-                    ad_hoc_visits = self.get_dec("P", row)
-                    ad_hoc_rate = self.get_dec("Q", row)
-                    ad_hoc_gbp = self.get_dec("R", row)
-                    activity_names = set()
-                    activity_gbp = Decimal("0")
-                    if ad_hoc_gbp != 0:
-                        activity_names.add("ad_hoc_visit")
-                        activity_gbp += ad_hoc_gbp
-
-                    annual_visits = self.get_int("S", row)
-                    annual_rate = self.get_dec("T", row)
-                    annual_gbp = self.get_dec("U", row)
-                    if annual_gbp != 0:
-                        activity_names.add("annual_visit")
-                        activity_gbp += annual_gbp
-
-                    if cop_3_meters > 0:
-                        cop = "3"
-                        mpan_rate = cop_3_rate
-                        mpan_gbp = cop_3_gbp
-                    else:
-                        cop = "5"
-                        mpan_rate = cop_5_rate
-                        mpan_gbp = cop_5_gbp
-
-                    breakdown = {
-                        "raw_lines": [],
-                        "cop": [cop],
-                        "settlement-status": ["settlement"],
-                        "mpan-rate": [mpan_rate],
-                        "mpan-gbp": mpan_gbp,
-                        "ad-hoc-visits": ad_hoc_visits,
-                        "ad-hoc-rate": [ad_hoc_rate],
-                        "ad-hoc-gbp-info": ad_hoc_gbp,
-                        "annual-visits-count": annual_visits,
-                        "annual-visits-rate": [annual_rate],
-                        "annual-visits-gbp-info": annual_gbp,
-                    }
-                    if len(activity_names) > 0:
-                        breakdown["activity-name"] = sorted(activity_names)
-
-                    if activity_gbp != 0:
-                        breakdown["activity-gbp"] = activity_gbp
-
-                    bills.append(
-                        {
-                            "bill_type_code": "N",
-                            "kwh": Decimal(0),
-                            "vat": Decimal("0.00"),
-                            "net": net,
-                            "gross": net,
-                            "reads": [],
-                            "breakdown": breakdown,
-                            "account": mpan_core,
-                            "issue_date": issue_date,
-                            "start_date": start_date,
-                            "finish_date": finish_date,
-                            "mpan_core": mpan_core,
-                            "reference": "_".join(
-                                (
-                                    start_date.strftime("%Y%m%d"),
-                                    finish_date.strftime("%Y%m%d"),
-                                    issue_date.strftime("%Y%m%d"),
-                                    mpan_core,
-                                )
-                            ),
-                        }
+                    self._set_last_line(row, val)
+                    bill = _process_row(
+                        sess, self.sheet, row, issue_date, column_lookup
                     )
+                    bills.append(bill)
                     sess.rollback()
 
         except BadRequest as e:

chellow/e/bill_parsers/sse_edi.py

@@ -3,7 +3,14 @@ from decimal import Decimal
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.edi_lib import parse_edi, to_ct_date, to_date, to_decimal, to_finish_date
+from chellow.edi_lib import (
+    parse_edi,
+    to_ct_date,
+    to_date,
+    to_decimal,
+    to_finish_date,
+    to_gbp,
+)
 from chellow.models import Era, Session
 from chellow.utils import HH, ct_datetime, parse_mpan_core, to_utc
 
@@ -196,6 +203,45 @@ def _process_BCD(elements, headers):
         headers["finish_date"] = to_finish_date(sumo[1])
 
 
+def _process_BTL(elements, headers):
+    uvlt = elements["UVLT"]
+    utva = elements["UTVA"]
+    tbtl = elements["TBTL"]
+
+    if headers["mpan_core"] is None:
+        sess = headers["sess"]
+        era = (
+            sess.query(Era)
+            .filter(Era.imp_supplier_account == headers["account"])
+            .first()
+        )
+        if era is not None:
+            headers["mpan_core"] = era.imp_mpan_core
+        sess.close()
+
+    if headers["is_ebatch"]:
+        for r in headers["reads"]:
+            if r["pres_type_code"] == "C":
+                r["pres_type_code"] = "E"
+
+    return {
+        "bill_type_code": headers["bill_type_code"],
+        "account": headers["account"],
+        "mpan_core": headers["mpan_core"],
+        "reference": headers["reference"],
+        "issue_date": headers["issue_date"],
+        "start_date": headers["start_date"],
+        "finish_date": headers["finish_date"],
+        "kwh": headers["kwh"],
+        "net": to_gbp(uvlt),
+        "vat": to_gbp(utva),
+        "gross": to_gbp(tbtl),
+        "breakdown": headers["breakdown"],
+        "reads": headers["reads"],
+        "elements": headers["elements"],
+    }
+
+
 def _process_MHD(elements, headers):
     message_type = elements["TYPE"][0]
     sess = headers["sess"]
@@ -204,7 +250,7 @@ def _process_MHD(elements, headers):
         headers["kwh"] = Decimal("0")
         headers["reads"] = []
         headers["breakdown"] = defaultdict(int, {"raw-lines": []})
-        headers["bill_elements"] = []
+        headers["elements"] = []
         headers["errors"] = []
         headers["sess"] = sess
         headers["mpan_core"] = None
@@ -215,9 +261,6 @@ def _process_MHD(elements, headers):
         headers["issue_date"] = None
         headers["start_date"] = None
         headers["finish_date"] = None
-        headers["net"] = Decimal("0.00")
-        headers["vat"] = Decimal("0.00")
-        headers["gross"] = Decimal("0.00")
     headers["message_type"] = message_type
 
 
@@ -323,6 +366,11 @@ def _process_CCD2(elements, headers):
 
     adjf = elements["ADJF"]
     cona = elements["CONA"]
+    nuct = elements["NUCT"]
+    csdt = elements["CSDT"]
+    cedt = elements["CEDT"]
+    ctot = elements["CTOT"]
+    cppu = elements["CPPU"]
 
     coefficient = Decimal(adjf[1]) / Decimal(100000)
     pres_reading_value = Decimal(prrd[0])
@@ -338,26 +386,28 @@ def _process_CCD2(elements, headers):
     if tpr == "kW":
         units = "kW"
         tpr = None
-        prefix = "md-"
+        el_name = "md"
     elif tpr == "kVA":
         units = "kVA"
         tpr = None
-        prefix = "md-"
+        el_name = "md"
     else:
         units = "kWh"
         headers["kwh"] += to_decimal(cona) / Decimal("1000")
-        prefix = tpr + "-"
+        el_name = tpr
 
-    nuct = elements["NUCT"]
-    breakdown = headers["breakdown"]
-    breakdown[prefix + "kwh"] += to_decimal(nuct) / Decimal("1000")
-    cppu = elements["CPPU"]
-    rate_key = prefix + "rate"
-    if rate_key not in breakdown:
-        breakdown[rate_key] = set()
-    breakdown[rate_key].add(to_decimal(cppu) / Decimal("100000"))
-    ctot = elements["CTOT"]
-    breakdown[prefix + "gbp"] += to_decimal(ctot) / Decimal("100")
+    headers["elements"].append(
+        {
+            "name": el_name,
+            "breakdown": {
+                cona[1].lower(): to_decimal(nuct) / Decimal("1000"),
+                "rate": {to_decimal(cppu) / Decimal("100000")},
+            },
+            "start_date": to_date(csdt[0]),
+            "finish_date": to_finish_date(cedt[0]),
+            "net": to_gbp(ctot),
+        }
+    )
 
     if mpan_core in WRONG_TPRS and pres_read_date == to_utc(
         ct_datetime(2020, 4, 1, 23, 30)
@@ -400,12 +450,9 @@ def _process_CCD3(elements, headers):
     tmod = elements["TMOD"]
     tmod0 = tmod[0]
     if tmod0 == "CCL":
-        prefix = kwh_prefix = "ccl-"
-    elif tmod0 in ["CQFITC", "CMFITC"]:
-        prefix = "fit-"
-        kwh_prefix = "fit-msp-"
-    elif tmod0 == "FITARR":
-        prefix = kwh_prefix = "fit-reconciliation-"
+        el_name = "ccl"
+    elif tmod0 in ("CQFITC", "CMFITC", "FITARR"):
+        el_name = "fit"
     else:
         tpr_code = tmod0
         if tpr_code not in tmod_map:
@@ -413,66 +460,51 @@ def _process_CCD3(elements, headers):
                 f"The TPR code {tpr_code} can't be found in the TPR "
                 f"list for mpan {headers['mpan']}."
             )
-        prefix = kwh_prefix = tmod_map[tpr_code] + "-"
+        el_name = tmod_map[tpr_code]
 
     nuct = elements["NUCT"]
-    breakdown = headers["breakdown"]
-    breakdown[kwh_prefix + "kwh"] += to_decimal(nuct) / Decimal("1000")
     cppu = elements["CPPU"]
-
-    rate_key = prefix + "rate"
-    if rate_key not in breakdown:
-        breakdown[rate_key] = set()
-    breakdown[rate_key].add(to_decimal(cppu) / Decimal("100000"))
-
     ctot = elements["CTOT"]
-    breakdown[prefix + "gbp"] += to_decimal(ctot) / Decimal("100")
+    csdt = elements["CSDT"]
+    start_date = to_date(csdt[0])
+    cedt = elements["CEDT"]
+    finish_date = to_finish_date(cedt[0])
+    headers["elements"].append(
+        {
+            "name": el_name,
+            "net": to_gbp(ctot),
+            "start_date": start_date,
+            "finish_date": finish_date,
+            "breakdown": {
+                "kwh": to_decimal(nuct) / Decimal("1000"),
+                "rate": {to_decimal(cppu) / Decimal("100000")},
+            },
+        }
+    )
 
 
 def _process_CCD4(elements, headers):
     ndrp = elements["NDRP"]
-    breakdown = headers["breakdown"]
-    if len(ndrp[0]) > 0:
-        breakdown["standing-days"] += to_decimal(ndrp)
     ctot = elements["CTOT"]
+    csdt = elements["CSDT"]
+    cedt = elements["CEDT"]
+
     if len(ctot[0]) > 0:
-        breakdown["standing-gbp"] += to_decimal(ctot) / Decimal("100")
+        breakdown = {}
+        el = {
+            "name": "standing",
+            "net": to_gbp(ctot),
+            "start_date": to_date(csdt[0]),
+            "finish_date": to_finish_date(cedt[0]),
+            "breakdown": breakdown,
+        }
+        if len(ndrp[0]) > 0:
+            breakdown["days"] = to_decimal(ndrp)
+        headers["elements"].append(el)
 
 
 def _process_MTR(elements, headers):
-    if headers["message_type"] == "UTLBIL":
-        if headers["mpan_core"] is None:
-            sess = headers["sess"]
-            era = (
-                sess.query(Era)
-                .filter(Era.imp_supplier_account == headers["account"])
-                .first()
-            )
-            if era is not None:
-                headers["mpan_core"] = era.imp_mpan_core
-            sess.close()
-
-        if headers["is_ebatch"]:
-            for r in headers["reads"]:
-                if r["pres_type_code"] == "C":
-                    r["pres_type_code"] = "E"
-
-        raw_bill = {
-            "bill_type_code": headers["bill_type_code"],
-            "account": headers["account"],
-            "mpan_core": headers["mpan_core"],
-            "reference": headers["reference"],
-            "issue_date": headers["issue_date"],
-            "start_date": headers["start_date"],
-            "finish_date": headers["finish_date"],
-            "kwh": headers["kwh"],
-            "net": headers["net"],
-            "vat": headers["vat"],
-            "gross": headers["gross"],
-            "breakdown": headers["breakdown"],
-            "reads": headers["reads"],
-        }
-        return raw_bill
+    pass
 
 
 def _process_MAN(elements, headers):
@@ -482,11 +514,11 @@ def _process_MAN(elements, headers):
 
 def _process_VAT(elements, headers):
     uvla = elements["UVLA"]
-    headers["net"] += to_decimal(uvla) / Decimal("100")
+    to_decimal(uvla) / Decimal("100")
     uvtt = elements["UVTT"]
-    headers["vat"] += to_decimal(uvtt) / Decimal("100")
+    to_decimal(uvtt) / Decimal("100")
     ucsi = elements["UCSI"]
-    headers["gross"] += to_decimal(ucsi) / Decimal("100")
+    to_decimal(ucsi) / Decimal("100")
 
 
 def _process_NOOP(elements, headers):
@@ -495,7 +527,7 @@ def _process_NOOP(elements, headers):
 
 CODE_FUNCS = {
     "BCD": _process_BCD,
-    "BTL": _process_NOOP,
+    "BTL": _process_BTL,
     "CCD1": _process_CCD1,
     "CCD2": _process_CCD2,
     "CCD3": _process_CCD3,