chellow 1757320031.0.0__py3-none-any.whl → 1759411815.0.0__py3-none-any.whl

chellow/e/bill_importer.py

@@ -5,6 +5,8 @@ import traceback
 from io import BytesIO
 from pkgutil import iter_modules
 
+from sqlalchemy import select
+
 from werkzeug.exceptions import BadRequest
 
 import chellow
@@ -13,12 +15,13 @@ from chellow.models import (
     Batch,
     BatchFile,
     BillType,
+    Contract,
     ReadType,
     Session,
     Supply,
     Tpr,
 )
-from chellow.utils import keydefaultdict, utc_datetime_now
+from chellow.utils import ct_datetime_now, keydefaultdict
 
 
 import_id = 0
@@ -31,24 +34,31 @@ def find_parser_names():
 
 
 class BillImport(threading.Thread):
-    def __init__(self, batch):
+    def __init__(self, batch_contract, now=None):
         threading.Thread.__init__(self)
         global import_id
         self.import_id = import_id
         import_id += 1
 
-        self.batch_id = batch.id
+        if isinstance(batch_contract, Batch):
+            self.contract_id = None
+            self.batch_id = batch_contract.id
+        elif isinstance(batch_contract, Contract):
+            self.contract_id = batch_contract.id
+            self.batch_id = None
+        else:
+            raise BadRequest("batch_contract must be a Batch or Contract.")
         self.successful_bills = []
         self.failed_bills = []
         self.log = collections.deque()
         self.bill_num = None
         self.parser = None
+        self.now = now
 
     def _log(self, msg):
         with import_lock:
-            self.log.appendleft(
-                f"{utc_datetime_now().strftime('%Y-%m-%d %H:%M:%S')} - {msg}"
-            )
+            ts = ct_datetime_now() if self.now is None else self.now
+            self.log.appendleft(f"{ts.strftime('%Y-%m-%d %H:%M:%S')} - {msg}")
 
     def status(self):
         if self.is_alive():
@@ -65,85 +75,117 @@ class BillImport(threading.Thread):
 
     def run(self):
         try:
+            batch_ids = []
             with Session() as sess:
-                batch = Batch.get_by_id(sess, self.batch_id)
-
-                bill_types = keydefaultdict(lambda k: BillType.get_by_code(sess, k))
+                batch_q = select(Batch).order_by(Batch.reference)
+                if self.batch_id is not None:
+                    batch = Batch.get_by_id(sess, self.batch_id)
+                    batch_q = batch_q.where(Batch.id == self.batch_id)
+                elif self.contract_id is not None:
+                    contract = Contract.get_by_id(sess, self.contract_id)
+                    batch_q = batch_q.where(Batch.contract == contract)
+                for batch in sess.scalars(batch_q):
+                    batch_ids.append(batch.id)
+
+            for batch_id in batch_ids:
+                with Session() as sess:
+                    batch = Batch.get_by_id(sess, batch_id)
+                    self._log(f"Importing bills from batch {batch.reference}.")
+                    bf_q = (
+                        select(BatchFile)
+                        .where(BatchFile.batch == batch)
+                        .order_by(BatchFile.upload_timestamp)
+                    )
+                    bill_types = keydefaultdict(lambda k: BillType.get_by_code(sess, k))
 
-                tprs = keydefaultdict(
-                    lambda k: None if k is None else Tpr.get_by_code(sess, k)
-                )
+                    tprs = keydefaultdict(
+                        lambda k: None if k is None else Tpr.get_by_code(sess, k)
+                    )
 
-                read_types = keydefaultdict(lambda k: ReadType.get_by_code(sess, k))
-
-                for bf in (
-                    sess.query(BatchFile)
-                    .filter(BatchFile.batch == batch)
-                    .order_by(BatchFile.upload_timestamp)
-                ):
-                    self.parser = _process_batch_file(sess, bf, self._log)
-                    for self.bill_num, raw_bill in enumerate(
-                        self.parser.make_raw_bills()
-                    ):
-                        if "error" in raw_bill:
-                            self.failed_bills.append(raw_bill)
-                        else:
-                            try:
-                                mpan_core = raw_bill["mpan_core"]
-                                supply = Supply.get_by_mpan_core(sess, mpan_core)
-                                with sess.begin_nested():
-                                    bill = batch.insert_bill(
-                                        sess,
-                                        raw_bill["account"],
-                                        raw_bill["reference"],
-                                        raw_bill["issue_date"],
-                                        raw_bill["start_date"],
-                                        raw_bill["finish_date"],
-                                        raw_bill["kwh"],
-                                        raw_bill["net"],
-                                        raw_bill["vat"],
-                                        raw_bill["gross"],
-                                        bill_types[raw_bill["bill_type_code"]],
-                                        raw_bill["breakdown"],
-                                        supply,
-                                    )
-                                    for raw_read in raw_bill["reads"]:
-                                        bill.insert_read(
+                    read_types = keydefaultdict(lambda k: ReadType.get_by_code(sess, k))
+
+                    for bf in sess.scalars(bf_q):
+                        self.parser = _process_batch_file(sess, bf, self._log)
+                        for self.bill_num, raw_bill in enumerate(
+                            self.parser.make_raw_bills()
+                        ):
+                            batch = bf.batch
+                            sum_elem = sum(el["net"] for el in raw_bill["elements"])
+                            raw_bill_net = raw_bill["net"]
+                            if sum_elem != raw_bill_net:
+                                raw_bill["error"] = (
+                                    f"The sum of the elements' net {sum_elem} doesn't "
+                                    f"equal the bill net {raw_bill_net}."
+                                )
+                            if "error" in raw_bill:
+                                self.failed_bills.append(raw_bill)
+                            else:
+                                try:
+                                    mpan_core = raw_bill["mpan_core"]
+                                    supply = Supply.get_by_mpan_core(sess, mpan_core)
+                                    with sess.begin_nested():
+                                        bill = batch.insert_bill(
                                             sess,
-                                            tprs[raw_read["tpr_code"]],
-                                            raw_read["coefficient"],
-                                            raw_read["units"],
-                                            raw_read["msn"],
-                                            raw_read["mpan"],
-                                            raw_read["prev_date"],
-                                            raw_read["prev_value"],
-                                            read_types[raw_read["prev_type_code"]],
-                                            raw_read["pres_date"],
-                                            raw_read["pres_value"],
-                                            read_types[raw_read["pres_type_code"]],
+                                            raw_bill["account"],
+                                            raw_bill["reference"],
+                                            raw_bill["issue_date"],
+                                            raw_bill["start_date"],
+                                            raw_bill["finish_date"],
+                                            raw_bill["kwh"],
+                                            raw_bill_net,
+                                            raw_bill["vat"],
+                                            raw_bill["gross"],
+                                            bill_types[raw_bill["bill_type_code"]],
+                                            raw_bill["breakdown"],
+                                            supply,
                                         )
-                                    self.successful_bills.append(raw_bill)
-                            except KeyError as e:
-                                err = raw_bill.get("error", "")
-                                raw_bill["error"] = err + " " + str(e)
-                                self.failed_bills.append(raw_bill)
-                            except BadRequest as e:
-                                raw_bill["error"] = str(e.description)
-                                self.failed_bills.append(raw_bill)
-
-                if len(self.failed_bills) == 0:
-                    sess.commit()
-                    self._log(
-                        "All the bills have been successfully loaded and attached "
-                        "to the batch."
-                    )
-                else:
-                    sess.rollback()
-                    self._log(
-                        f"The import has finished, but there were "
-                        f"{len(self.failed_bills)} "
-                        f"failures, and so the whole import has been rolled back."
-                    )
+                                        for raw_element in raw_bill["elements"]:
+                                            bill.insert_element(
+                                                sess,
+                                                raw_element["name"],
+                                                raw_element["start_date"],
+                                                raw_element["finish_date"],
+                                                raw_element["net"],
+                                                raw_element["breakdown"],
+                                            )
+                                        for raw_read in raw_bill["reads"]:
+                                            bill.insert_read(
+                                                sess,
+                                                tprs[raw_read["tpr_code"]],
+                                                raw_read["coefficient"],
+                                                raw_read["units"],
+                                                raw_read["msn"],
+                                                raw_read["mpan"],
+                                                raw_read["prev_date"],
+                                                raw_read["prev_value"],
+                                                read_types[raw_read["prev_type_code"]],
+                                                raw_read["pres_date"],
+                                                raw_read["pres_value"],
+                                                read_types[raw_read["pres_type_code"]],
+                                            )
+                                        self.successful_bills.append(raw_bill)
+                                except KeyError as e:
+                                    err = raw_bill.get("error", "")
+                                    raw_bill["error"] = err + " " + str(e)
+                                    self.failed_bills.append(raw_bill)
+                                except BadRequest as e:
+                                    raw_bill["error"] = str(e.description)
+                                    self.failed_bills.append(raw_bill)
+
+                    if len(self.failed_bills) == 0:
+                        sess.commit()
+                        self._log(
+                            "All the bills have been successfully loaded and attached "
+                            "to the batch."
+                        )
+                    else:
+                        sess.rollback()
+                        self._log(
+                            f"The import has finished, but there were "
+                            f"{len(self.failed_bills)} "
+                            f"failures, and so the whole import has been rolled back."
+                        )
+                        break
 
         except BadRequest as e:
             msg = f"Problem: {e.description}"
@@ -193,11 +235,25 @@ def start_bill_import(batch):
     return bi.import_id
 
 
+def start_bill_import_contract(contract):
+    with import_lock:
+        bi = BillImport(contract)
+        imports[bi.import_id] = bi
+        bi.start()
+
+    return bi.import_id
+
+
 def get_bill_import_ids(batch):
     with import_lock:
         return [k for k, v in imports.items() if v.batch_id == batch.id]
 
 
+def get_bill_import_ids_contract(contract):
+    with import_lock:
+        return [k for k, v in imports.items() if v.contract_id == contract.id]
+
+
 def get_bill_import(id):
     with import_lock:
         return imports[id]

chellow/e/bill_parsers/activity_mop_stark_xlsx.py

@@ -5,7 +5,99 @@ from openpyxl import load_workbook
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import parse_mpan_core, to_utc
+from chellow.utils import parse_mpan_core, to_ct, to_utc
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+def get_dec(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    try:
+        return Decimal(str(cell.value))
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_ct_date(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    val = cell.value
+    if isinstance(val, Datetime):
+        dt = val
+    elif isinstance(val, str):
+        dt = Datetime.strptime(val, "dd/mm/yyyy")
+    else:
+        raise BadRequest(
+            f"The value {val} at {cell.coordinate} is of type {type(val)}, but "
+            f"expected a timestamp or string."
+        )
+    return to_ct(dt)
+
+
+def _process_row(issue_date, sheet, row):
+    mpan_core = parse_mpan_core(str(get_int(sheet, "B", row)))
+
+    start_date = finish_date = to_utc(get_ct_date(sheet, "F", row))
+    activity_name_raw = get_str(sheet, "G", row)
+    activity_name = activity_name_raw.lower().replace(" ", "_")
+
+    net_dec = get_dec(sheet, "I", row)
+    net = round(net_dec, 2)
+
+    vat_dec = get_dec(sheet, "J", row)
+    vat = round(vat_dec, 2)
+
+    gross_dec = get_dec(sheet, "K", row)
+    gross = round(gross_dec, 2)
+
+    breakdown = {
+        "raw-lines": [],
+    }
+
+    return {
+        "bill_type_code": "N",
+        "kwh": Decimal(0),
+        "vat": vat,
+        "net": net,
+        "gross": gross,
+        "reads": [],
+        "breakdown": breakdown,
+        "account": mpan_core,
+        "issue_date": issue_date,
+        "start_date": start_date,
+        "finish_date": finish_date,
+        "mpan_core": mpan_core,
+        "reference": "_".join(
+            (
+                start_date.strftime("%Y%m%d"),
+                finish_date.strftime("%Y%m%d"),
+                issue_date.strftime("%Y%m%d"),
+                mpan_core,
+            )
+        ),
+        "elements": [
+            {
+                "name": "activity",
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "net": net,
+                "breakdown": {"name": {activity_name}},
+            }
+        ],
+    }
 
 
 class Parser:
@@ -17,42 +109,6 @@ class Parser:
         self._line_number = None
         self._title_line = None
 
-    def get_cell(self, col, row):
-        try:
-            coordinates = f"{col}{row}"
-            return self.sheet[coordinates]
-        except IndexError:
-            raise BadRequest(
-                f"Can't find the cell {coordinates} on sheet {self.sheet}."
-            )
-
-    def get_int(self, col, row):
-        return int(self.get_cell(col, row).value)
-
-    def get_dec(self, col, row):
-        cell = self.get_cell(col, row)
-        try:
-            return Decimal(str(cell.value))
-        except InvalidOperation as e:
-            raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
-
-    def get_str(self, col, row):
-        return self.get_cell(col, row).value.strip()
-
-    def get_ct_date(self, col, row):
-        cell = self.get_cell(col, row)
-        val = cell.value
-        if not isinstance(val, Datetime):
-            raise BadRequest(
-                f"The value {val} at {cell.coordinate} is of type {type(val)}, but "
-                f"expected a timestamp."
-            )
-        return val
-
-    def get_start_date(self, col, row):
-        dt = self.get_ct_date(col, row)
-        return None if dt is None else to_utc(dt)
-
     @property
     def line_number(self):
         return None if self._line_number is None else self._line_number + 1
@@ -67,59 +123,16 @@ class Parser:
     def make_raw_bills(self):
         try:
             bills = []
-            issue_date = self.get_start_date("C", 6)
-
+            issue_date = to_utc(get_ct_date(self.sheet, "C", 6))
             for row in range(12, len(self.sheet["A"]) + 1):
-                val = self.get_cell("B", row).value
+                val = get_cell(self.sheet, "B", row).value
                 if val is None or val == "":
                     break
-
                 self._set_last_line(row, val)
-                mpan_core = parse_mpan_core(str(self.get_int("B", row)))
-
-                start_date = finish_date = self.get_start_date("F", row)
-                activity_name_raw = self.get_str("G", row)
-                activity_name = activity_name_raw.lower().replace(" ", "_")
-
-                net_dec = self.get_dec("I", row)
-                net = round(net_dec, 2)
-
-                vat_dec = self.get_dec("J", row)
-                vat = round(vat_dec, 2)
-
-                gross_dec = self.get_dec("K", row)
-                gross = round(gross_dec, 2)
-
-                breakdown = {
-                    "raw-lines": [],
-                    "activity-name": [activity_name],
-                    "activity-gbp": net,
-                }
-
-                bills.append(
-                    {
-                        "bill_type_code": "N",
-                        "kwh": Decimal(0),
-                        "vat": vat,
-                        "net": net,
-                        "gross": gross,
-                        "reads": [],
-                        "breakdown": breakdown,
-                        "account": mpan_core,
-                        "issue_date": issue_date,
-                        "start_date": start_date,
-                        "finish_date": finish_date,
-                        "mpan_core": mpan_core,
-                        "reference": "_".join(
-                            (
-                                start_date.strftime("%Y%m%d"),
-                                finish_date.strftime("%Y%m%d"),
-                                issue_date.strftime("%Y%m%d"),
-                                mpan_core,
-                            )
-                        ),
-                    }
-                )
+
+                bill = _process_row(issue_date, self.sheet, row)
+                bills.append(bill)
+
         except BadRequest as e:
             raise BadRequest(f"Row number: {row} {e.description}")
 

chellow/e/bill_parsers/annual_mop_stark_xlsx.py

@@ -8,23 +8,22 @@ from openpyxl import load_workbook
 from werkzeug.exceptions import BadRequest
 
 
-from chellow.utils import parse_mpan_core, to_utc
+from chellow.utils import parse_mpan_core, to_ct, to_utc
 
 
 def get_ct_date(row, idx):
     cell = get_cell(row, idx)
     val = cell.value
-    if not isinstance(val, Datetime):
-        raise BadRequest(f"Problem reading {val} as a timestamp at {cell.coordinate}.")
-    return val
-
-
-def get_start_date(row, idx):
-    return to_utc(get_ct_date(row, idx))
-
-
-def get_finish_date(row, idx):
-    return to_utc(get_ct_date(row, idx) + relativedelta(hours=23, minutes=30))
+    if isinstance(val, Datetime):
+        dt = val
+    elif isinstance(val, str):
+        dt = Datetime.strptime(val, "dd/mm/yyyy")
+    else:
+        raise BadRequest(
+            f"The value {val} at {cell.coordinate} is of type {type(val)}, but "
+            f"expected a timestamp or string."
+        )
+    return to_ct(dt)
 
 
 def get_cell(row, idx):
@@ -52,6 +51,70 @@ def get_int(row, idx):
     return int(get_cell(row, idx).value)
 
 
+def _process_row(issue_date, row):
+
+    mpan_core = parse_mpan_core(str(get_int(row, 1)))
+    comm = get_str(row, 2)
+
+    settled_str = get_str(row, 3)
+    if settled_str == "Settled":
+        settlement_status = "settlement"
+    else:
+        settlement_status = "non_settlement"
+
+    start_date_ct = get_ct_date(row, 5)
+    finish_date_ct = get_ct_date(row, 6)
+    days = (finish_date_ct - start_date_ct).days + 1
+
+    start_date = to_utc(start_date_ct)
+    finish_date = to_utc(finish_date_ct + relativedelta(hours=23, minutes=30))
+
+    meter_rate = get_dec(row, 7)
+    net = round(get_dec(row, 8), 2)
+    vat = round(get_dec(row, 9), 2)
+    gross = round(get_dec(row, 10), 2)
+
+    breakdown = {
+        "raw-lines": [],
+    }
+    return {
+        "bill_type_code": "N",
+        "kwh": Decimal(0),
+        "net": net,
+        "vat": vat,
+        "gross": gross,
+        "reads": [],
+        "breakdown": breakdown,
+        "account": mpan_core,
+        "issue_date": issue_date,
+        "start_date": start_date,
+        "finish_date": finish_date,
+        "mpan_core": mpan_core,
+        "reference": "_".join(
+            (
+                start_date.strftime("%Y%m%d"),
+                finish_date.strftime("%Y%m%d"),
+                issue_date.strftime("%Y%m%d"),
+                mpan_core,
+            )
+        ),
+        "elements": [
+            {
+                "name": "meter",
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "breakdown": {
+                    "rate": {meter_rate},
+                    "comm": {comm},
+                    "settlement-status": {settlement_status},
+                    "days": days,
+                },
+                "net": net,
+            }
+        ],
+    }
+
+
 class Parser:
     def __init__(self, f):
         self.book = load_workbook(f, data_only=True)
@@ -76,61 +139,15 @@ class Parser:
         try:
             bills = []
             row = next(self.sheet.iter_rows(min_row=6, max_row=6, max_col=3))
-            issue_date = get_start_date(row, 2)
+            issue_date = to_utc(get_ct_date(row, 2))
             for row in self.sheet.iter_rows(min_row=12, max_col=11):
                 val = get_cell(row, 1).value
                 if val is None or val == "":
                     break
 
                 self._set_last_line(row[0].row, val)
-                mpan_core = parse_mpan_core(str(get_int(row, 1)))
-                comms = get_str(row, 2)
-
-                settled_str = get_str(row, 3)
-                if settled_str == "Settled":
-                    settlement_status = "settlement"
-                else:
-                    settlement_status = "non_settlement"
-
-                start_date = get_start_date(row, 5)
-                finish_date = get_finish_date(row, 6)
-
-                meter_rate = get_dec(row, 7)
-                net = round(get_dec(row, 8), 2)
-                vat = round(get_dec(row, 9), 2)
-                gross = round(get_dec(row, 10), 2)
-
-                breakdown = {
-                    "raw-lines": [],
-                    "comms": comms,
-                    "settlement-status": [settlement_status],
-                    "meter-rate": [meter_rate],
-                    "meter-gbp": net,
-                }
-                bills.append(
-                    {
-                        "bill_type_code": "N",
-                        "kwh": Decimal(0),
-                        "net": net,
-                        "vat": vat,
-                        "gross": gross,
-                        "reads": [],
-                        "breakdown": breakdown,
-                        "account": mpan_core,
-                        "issue_date": issue_date,
-                        "start_date": start_date,
-                        "finish_date": finish_date,
-                        "mpan_core": mpan_core,
-                        "reference": "_".join(
-                            (
-                                start_date.strftime("%Y%m%d"),
-                                finish_date.strftime("%Y%m%d"),
-                                issue_date.strftime("%Y%m%d"),
-                                mpan_core,
-                            )
-                        ),
-                    }
-                )
+                bill = _process_row(issue_date, row)
+                bills.append(bill)
         except BadRequest as e:
             raise BadRequest(f"Row number {row[0].row}: {e.description}")