chellow 1729081025.0.0__py3-none-any.whl → 1729755916.0.0__py3-none-any.whl

chellow/e/bill_parsers/engie_export_xlsx.py

@@ -0,0 +1,167 @@
+from datetime import datetime as Datetime
+from decimal import Decimal, InvalidOperation
+
+from dateutil.relativedelta import relativedelta
+
+from openpyxl import load_workbook
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import HH, parse_mpan_core, to_ct, to_utc
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_date_ct(sheet, col, row):
+    date_str = get_str(sheet, col, row)
+    return to_ct(Datetime.strptime(date_str, "%d/%m/%Y"))
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_dec(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    try:
+        return Decimal(str(cell.value))
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+ELEM_LOOKUP = {
+    "Admin Fee": {
+        "gbp": "admin-gbp",
+        "units": None,
+        "rate": None,
+    },
+    "VAT": None,
+    "GDUOS Charges": {
+        "gbp": "duos-gbp",
+        "units": None,
+        "rate": None,
+    },
+    "Power Generation": {
+        "gbp": "ssp-gbp",
+        "units": "ssp-kwh",
+        "rate": "ssp-rate",
+    },
+}
+
+
+def _make_raw_bills(book):
+    bills = {}
+    mpan_lookup = {}
+    sheet = book.worksheets[0]
+    for row in range(2, len(sheet["A"]) + 1):
+        val = get_cell(sheet, "A", row).value
+        if val is None or val == "":
+            break
+
+        description = get_str(sheet, "T", row)
+        desc_parts = [d.strip() for d in description.split("-")]
+        if len(desc_parts) > 1:
+            start_date_ct = to_ct(Datetime.strptime(desc_parts[1], "%b %y"))
+            finish_date_ct = start_date_ct + relativedelta(months=1) - HH
+        else:
+            start_date_ct = get_date_ct(sheet, "M", row)
+            finish_date_ct = get_date_ct(sheet, "N", row) + relativedelta(
+                hours=23, minutes=30
+            )
+
+        desc_elem = desc_parts[0]
+        start_date = to_utc(start_date_ct)
+        finish_date = to_utc(finish_date_ct)
+        reference = get_str(sheet, "G", row)
+        issue_date = to_utc(get_date_ct(sheet, "I", row))
+
+        bill_key = reference, start_date
+        try:
+            bill = bills[bill_key]
+        except KeyError:
+            bill = bills[bill_key] = {
+                "bill_type_code": "N",
+                "kwh": Decimal(0),
+                "vat": Decimal("0.00"),
+                "net": Decimal("0.00"),
+                "gross": Decimal("0.00"),
+                "reads": [],
+                "breakdown": {},
+                "issue_date": issue_date,
+                "start_date": start_date,
+                "finish_date": finish_date,
+                "reference": reference,
+            }
+
+        net = round(get_dec(sheet, "V", row), 2)
+        vat = round(get_dec(sheet, "W", row), 2)
+        gross = round(get_dec(sheet, "X", row), 2)
+        bill["net"] += net
+        bill["vat"] += vat
+        bill["gross"] += gross
+
+        mpan_core_str = get_str(sheet, "E", row)
+        if len(mpan_core_str) > 0:
+            mpan_lookup[reference] = parse_mpan_core(mpan_core_str)
+
+        mpan_core = mpan_lookup[reference]
+
+        bill["mpan_core"] = mpan_core
+        bill["account"] = mpan_core
+
+        titles = ELEM_LOOKUP[desc_elem]
+        if titles is None:
+            continue
+
+        breakdown = bill["breakdown"]
+        breakdown[titles["gbp"]] = net
+
+        units_title = titles["units"]
+        rate_title = titles["rate"]
+        units = get_dec(sheet, "P", row)
+        rate = get_dec(sheet, "U", row) / Decimal(1000)
+
+        if units_title is not None:
+            breakdown[units_title] = units
+        if rate_title is not None:
+            breakdown[rate_title] = [rate]
+    return bills.values()
+
+
+class Parser:
+    def __init__(self, f):
+        self.book = load_workbook(f, data_only=True)
+
+        self.last_line = None
+        self._line_number = None
+        self._title_line = None
+
+    @property
+    def line_number(self):
+        return None if self._line_number is None else self._line_number + 1
+
+    def _set_last_line(self, i, line):
+        self._line_number = i
+        self.last_line = line
+        if i == 0:
+            self._title_line = line
+        return line
+
+    def make_raw_bills(self):
+        row = bills = None
+        try:
+            bills = _make_raw_bills(self.book)
+        except BadRequest as e:
+            raise BadRequest(f"Row number: {row} {e.description}")
+
+        return bills

chellow/e/hh_importer.py

@@ -40,7 +40,15 @@ from chellow.utils import (
 processes = defaultdict(list)
 tasks = {}
 
-extensions = [".df2", ".simple.csv", ".bg.csv", ".vital.xlsx", ".edf.csv"]
+extensions = [
+    ".df2",
+    ".simple.csv",
+    ".bg.csv",
+    ".vital.xlsx",
+    ".edf.csv",
+    ".schneider.csv",
+    ".schneider.xlsx",
+]
 
 
 class HhDataImportProcess(threading.Thread):

chellow/e/hh_parser_schneider_csv.py

@@ -0,0 +1,75 @@
+import csv
+import itertools
+from codecs import iterdecode
+from datetime import datetime as Datetime
+from decimal import Decimal, InvalidOperation
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import parse_mpan_core, to_utc, validate_hh_start
+
+# "Time stamp","Value","Events","Comment","User"
+# "18/10/2024 09:30:00","88.0","","",""
+# "18/10/2024 09:00:00","89.1","","",""
+
+
+def create_parser(reader, mpan_map, messages):
+    return HhParserSchneiderCsv(reader, mpan_map, messages)
+
+
+def get_field(values, index, name):
+    if len(values) > index:
+        return values[index].strip()
+    else:
+        raise BadRequest(f"Can't find field {index}, {name}.")
+
+
+def parse_values(values):
+    start_date_str = get_field(values, 0, "Timestamp")
+    start_date = validate_hh_start(
+        to_utc(Datetime.strptime(start_date_str, "%d/%m/%Y %H:%M:%S"))
+    )
+
+    reading_str = get_field(values, 1, "Reading")
+    reading_str = reading_str.replace(",", "")
+    try:
+        reading = Decimal(reading_str)
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number {reading_str}. {e}")
+    return start_date, reading
+
+
+class HhParserSchneiderCsv:
+    def __init__(self, reader, mpan_map, messages):
+        s = iterdecode(reader, "utf-8")
+        self.shredder = zip(itertools.count(1), csv.reader(s))
+        next(self.shredder)  # skip the title line
+        self.line_number, self.values = next(self.shredder)
+        _, self.pres_reading = parse_values(self.values)
+        self.mpan_core = parse_mpan_core(next(iter(mpan_map.values())))
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        try:
+            self.line_number, self.values = next(self.shredder)
+            start_date, reading = parse_values(self.values)
+            datum = {
+                "mpan_core": self.mpan_core,
+                "channel_type": "ACTIVE",
+                "start_date": start_date,
+                "value": self.pres_reading - reading,
+                "status": "A",
+            }
+            self.pres_reading = reading
+            return datum
+        except BadRequest as e:
+            e.description = (
+                f"Problem at line number: {self.line_number}: {self.values}: "
+                f"{e.description}"
+            )
+            raise e
+
+    def close(self):
+        self.shredder.close()

chellow/e/hh_parser_schneider_xlsx.py

@@ -0,0 +1,118 @@
+from datetime import datetime as Datetime
+from decimal import Decimal, InvalidOperation
+
+from openpyxl import load_workbook
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import parse_mpan_core, to_ct, to_utc, validate_hh_start
+
+# "Time stamp","Value","Events","Comment","User"
+# "18/10/2024 09:30:00","88.0","","",""
+# "18/10/2024 09:00:00","89.1","","",""
+
+
+def create_parser(reader, mpan_map, messages):
+    return HhParserSchneiderXlsx(reader, mpan_map, messages)
+
+
+def get_cell(sheet, col, row):
+    try:
+        coordinates = f"{col}{row}"
+        return sheet[coordinates]
+    except IndexError:
+        raise BadRequest(f"Can't find the cell {coordinates} on sheet {sheet}.")
+
+
+def get_date(sheet, col, row):
+    cell = get_cell(sheet, col, row)
+    val = cell.value
+    if not isinstance(val, Datetime):
+        raise BadRequest(
+            f"Problem reading {val} (of type {type(val)}) as a timestamp at "
+            f"{cell.coordinate}."
+        )
+    return val
+
+
+def get_str(sheet, col, row):
+    return get_cell(sheet, col, row).value.strip()
+
+
+def get_dec(sheet, col, row):
+    return get_dec_from_cell(get_cell(sheet, col, row))
+
+
+def get_dec_from_cell(cell):
+    try:
+        return Decimal(str(cell.value))
+    except InvalidOperation as e:
+        raise BadRequest(f"Problem parsing the number at {cell.coordinate}. {e}")
+
+
+def get_int(sheet, col, row):
+    return int(get_cell(sheet, col, row).value)
+
+
+def find_hhs(sheet, set_line_number, mpan_core):
+    pres_reading = None
+    for row in range(2, len(sheet["A"]) + 1):
+        set_line_number(row)
+        try:
+            timestamp_cell = get_cell(sheet, "A", row)
+            timestamp_str = timestamp_cell.value
+            if timestamp_str is None:
+                continue
+
+            # 2024-10-02 00:00:00 +1H, DST
+            ts_str = timestamp_str.split(",")[0]
+            ts_naive = Datetime.strptime(ts_str[:19], "%Y-%m-%d %H:%M:%S")
+            # Sometimes has seconds
+            ts_naive = Datetime(
+                ts_naive.year,
+                ts_naive.month,
+                ts_naive.day,
+                ts_naive.hour,
+                ts_naive.minute,
+            )
+            is_dst = ts_str[21] == 1
+            if is_dst:
+                start_date = to_utc(to_ct(ts_naive))
+            else:
+                start_date = to_utc(ts_naive)
+            start_date = validate_hh_start(start_date)
+
+            reading = get_dec(sheet, "B", row)
+
+            if pres_reading is not None:
+                value = pres_reading - reading
+                if value >= 0:
+                    yield {
+                        "mpan_core": mpan_core,
+                        "channel_type": "ACTIVE",
+                        "start_date": start_date,
+                        "value": value,
+                        "status": "A",
+                    }
+            pres_reading = reading
+        except BadRequest as e:
+            e.description = f"Problem at line number: {row}: {e.description}"
+            raise e
+
+
+class HhParserSchneiderXlsx:
+    def __init__(self, input_stream, mpan_map, messages):
+        book = load_workbook(input_stream, data_only=True)
+        self.sheet = book.worksheets[0]
+        imp_name = get_str(self.sheet, "B", 1).strip()
+        self.mpan_core = parse_mpan_core(mpan_map[imp_name])
+        self.line_number = 0
+
+    def _set_line_number(self, line_number):
+        self.line_number = line_number
+
+    def __iter__(self):
+        return find_hhs(self.sheet, self._set_line_number, self.mpan_core)
+
+    def close(self):
+        self.shredder.close()

chellow/e/views.py

@@ -68,7 +68,6 @@ from chellow.models import (
     ReadType,
     RegisterRead,
     Report,
-    Scenario,
     Site,
     SiteEra,
     Snag,
@@ -80,7 +79,6 @@ from chellow.models import (
 )
 from chellow.utils import (
     HH,
-    c_months_c,
     c_months_u,
     csv_make_val,
     ct_datetime,
@@ -1220,7 +1218,7 @@ def dc_contract_edit_post(contract_id):
 
 
 @e.route("/dc_contracts/<int:contract_id>/hh_imports")
-def dc_contracts_hh_imports_get(contract_id):
+def dc_contract_hh_imports_get(contract_id):
     contract = Contract.get_dc_by_id(g.sess, contract_id)
     processes = chellow.e.hh_importer.get_hh_import_processes(contract.id)
     return render_template(
@@ -1232,7 +1230,7 @@ def dc_contracts_hh_imports_get(contract_id):
 
 
 @e.route("/dc_contracts/<int:contract_id>/hh_imports", methods=["POST"])
-def dc_contracts_hh_imports_post(contract_id):
+def dc_contract_hh_imports_post(contract_id):
     try:
         contract = Contract.get_dc_by_id(g.sess, contract_id)
 
@@ -3910,142 +3908,6 @@ def read_type_get(read_type_id):
     return render_template("read_type.html", read_type=read_type)
 
 
-@e.route("/scenarios")
-def scenarios_get():
-    scenarios = g.sess.query(Scenario).order_by(Scenario.name).all()
-    return render_template("scenarios.html", scenarios=scenarios)
-
-
-@e.route("/scenarios/add", methods=["POST"])
-def scenario_add_post():
-    try:
-        name = req_str("name")
-        properties = req_zish("properties")
-        scenario = Scenario.insert(g.sess, name, properties)
-        g.sess.commit()
-        return chellow_redirect(f"/scenarios/{scenario.id}", 303)
-    except BadRequest as e:
-        g.sess.rollback()
-        flash(e.description)
-        scenarios = g.sess.query(Scenario).order_by(Scenario.name)
-        return make_response(
-            render_template("scenario_add.html", scenarios=scenarios), 400
-        )
-
-
-@e.route("/scenarios/add")
-def scenario_add_get():
-    now = utc_datetime_now()
-    props = {
-        "scenario_start_month": now.month,
-        "scenario_start_year": now.year,
-        "scenario_duration": 1,
-    }
-    return render_template("scenario_add.html", initial_props=dumps(props))
-
-
-@e.route("/scenarios/<int:scenario_id>")
-def scenario_get(scenario_id):
-    start_date = None
-    finish_date = None
-    duration = 1
-
-    scenario = Scenario.get_by_id(g.sess, scenario_id)
-    props = scenario.props
-    site_codes = "\n".join(props.get("site_codes", []))
-    try:
-        duration = props["scenario_duration"]
-        _, finish_date_ct = list(
-            c_months_c(
-                start_year=props["scenario_start_year"],
-                start_month=props["scenario_start_month"],
-                months=duration,
-            )
-        )[-1]
-        finish_date = to_utc(finish_date_ct)
-    except KeyError:
-        pass
-
-    try:
-        start_date = to_utc(
-            ct_datetime(
-                props["scenario_start_year"],
-                props["scenario_start_month"],
-                props["scenario_start_day"],
-                props["scenario_start_hour"],
-                props["scenario_start_minute"],
-            )
-        )
-    except KeyError:
-        pass
-
-    try:
-        finish_date = to_utc(
-            ct_datetime(
-                props["scenario_finish_year"],
-                props["scenario_finish_month"],
-                props["scenario_finish_day"],
-                props["scenario_finish_hour"],
-                props["scenario_finish_minute"],
-            )
-        )
-    except KeyError:
-        pass
-    return render_template(
-        "scenario.html",
-        scenario=scenario,
-        scenario_start_date=start_date,
-        scenario_finish_date=finish_date,
-        scenario_duration=duration,
-        site_codes=site_codes,
-    )
-
-
-@e.route("/scenarios/<int:scenario_id>/edit")
-def scenario_edit_get(scenario_id):
-    scenario = Scenario.get_by_id(g.sess, scenario_id)
-    return render_template("scenario_edit.html", scenario=scenario)
-
-
-@e.route("/scenarios/<int:scenario_id>/edit", methods=["POST"])
-def scenario_edit_post(scenario_id):
-    try:
-        scenario = Scenario.get_by_id(g.sess, scenario_id)
-        name = req_str("name")
-        properties = req_zish("properties")
-        scenario.update(name, properties)
-        g.sess.commit()
-        return chellow_redirect(f"/scenarios/{scenario.id}", 303)
-    except BadRequest as e:
-        g.sess.rollback()
-        description = e.description
-        flash(description)
-        return make_response(
-            render_template(
-                "scenario_edit.html",
-                scenario=scenario,
-            ),
-            400,
-        )
-
-
-@e.route("/scenarios/<int:scenario_id>/edit", methods=["DELETE"])
-def scenario_edit_delete(scenario_id):
-    try:
-        scenario = Scenario.get_by_id(g.sess, scenario_id)
-        scenario.delete(g.sess)
-        g.sess.commit()
-        res = make_response()
-        res.headers["HX-Redirect"] = f"{chellow.utils.url_root}/e/scenarios"
-        return res
-    except BadRequest as e:
-        g.sess.rollback()
-        flash(e.description)
-        return make_response(
-            render_template("scenario_edit.html", scenario=scenario), 400
-        )
-
-
 @e.route("/sites/<int:site_id>/energy_management")
 def site_energy_management_get(site_id):
     site = Site.get_by_id(g.sess, site_id)