aemo-mdff-reader 2.0.2__py3-none-any.whl

aemo_mdff_reader/reader.py

@@ -0,0 +1,107 @@
+"""Backward-compatible NEMReader facade.
+
+The original :class:`NEMReader` class built an in-memory tree of records
+and walked it twice (once to build, once to flatten). The new
+implementation streams through :mod:`aemo_mdff_reader.parser` and only
+materialises results when the user asks for them — typically to a
+DataFrame or CSV.
+
+The legacy method names are preserved so existing code continues to
+work; new code should prefer :func:`aemo_mdff_reader.parser.parse` directly.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Iterator, List, Optional, Sequence
+
+from .parser import Columns, parse, to_columns, to_dataframe, write_csv
+from .types import IntervalReading
+
+# Kept for backward compatibility with the old INTERVAL_DATA_OUTPUT_HEADERS export.
+INTERVAL_DATA_OUTPUT_HEADERS = [
+    "NMI",
+    "MeterSerial",
+    "Register",
+    "Date",
+    "Interval",
+    "IntervalLength",
+    "UOM",
+    "IntervalValue",
+    "Quality",
+    "UpdateDatetime",
+]
+
+
+class NEMReader:
+    """Reads NEM12 / NEM13 files using the AEMO MDFF specification (v2.6).
+
+    See :mod:`aemo_mdff_reader.spec` for the spec version, effective date,
+    and the allowed-value enumerations referenced in the spec.
+    """
+
+    INTERVAL_DATA_OUTPUT_HEADERS = INTERVAL_DATA_OUTPUT_HEADERS
+
+    def __init__(self) -> None:
+        self._readings: Optional[List[IntervalReading]] = None
+        self._source_filename: Optional[str] = None
+
+    @property
+    def filename(self) -> Optional[str]:
+        return self._source_filename
+
+    @property
+    def readings(self) -> List[IntervalReading]:
+        if self._readings is None:
+            raise RuntimeError("Call read_from_file() or read_from_array() first.")
+        return self._readings
+
+    def read_from_array(self, array: Iterable[Sequence[str]]) -> None:
+        """Reads from an iterable of pre-split rows."""
+        self._readings = list(parse(array))
+
+    def read_from_file(self, filename: str) -> None:
+        """Read a NEM12 CSV file from disk and buffer every interval.
+
+        Memory note: this materialises the full reading list, so peak
+        usage scales with the file. For files larger than a few hundred
+        MiB use :func:`aemo_mdff_reader.parse` (streaming, O(1) memory) or
+        :func:`aemo_mdff_reader.iter_dataframes` (bounded, O(chunk_size))
+        instead.
+        """
+        self._source_filename = filename
+        self._readings = list(parse(filename))
+
+    def __iter__(self) -> Iterator[IntervalReading]:
+        """Iterate over the buffered readings.
+
+        Lets callers write ``for r in reader: ...`` after
+        :meth:`read_from_file` or :meth:`read_from_array` instead of
+        going through ``reader.readings``.
+        """
+        return iter(self.readings)
+
+    def __len__(self) -> int:
+        """Number of buffered readings (after ``read_from_file``)."""
+        return len(self.readings)
+
+    def to_dataframe(self) -> Any:
+        """Materialise readings into a pandas DataFrame.
+
+        Requires the ``pandas`` extra. When the buffered file path is
+        known, this re-parses via the columnar fast path for a 2–4×
+        speedup over iterating the in-memory readings list.
+        """
+        if self._source_filename is not None:
+            return to_dataframe(self._source_filename)
+        return to_dataframe(self.readings)
+
+    def to_csv(self, filename: str) -> int:
+        """Write readings as a flat CSV file. Returns the row count."""
+        return write_csv(self.readings, filename)
+
+    def to_columns(self) -> Columns:
+        """Return readings as a dict of column lists (no pandas required)."""
+        return to_columns(self.readings)
+
+
+__all__ = ["INTERVAL_DATA_OUTPUT_HEADERS", "NEMReader"]

aemo_mdff_reader/spec.py

@@ -0,0 +1,242 @@
+"""Constants from the AEMO MDFF specification (NEM12 / NEM13).
+
+This module mirrors the enumerations and allowed-value lists in the
+AEMO Meter Data File Format Specification — NEM12 & NEM13, v2.6,
+effective 29 September 2024. The spec is the source of truth; this
+module is a convenience surface for callers who want to validate
+parsed records against it.
+
+The parser itself does not enforce these enumerations strictly — it
+accepts any well-formed CSV — so downstream code is free to apply as
+much or as little validation as it needs.
+
+Reference (always points at the current AEMO publication):
+https://www.aemo.com.au/energy-systems/electricity/national-electricity-market-nem/market-operations/retail-and-metering/metering-procedures-guidelines-and-processes
+"""
+
+from __future__ import annotations
+
+from typing import Dict, FrozenSet
+
+SPEC_VERSION = "2.6"
+SPEC_EFFECTIVE_DATE = "2024-09-29"
+SPEC_DOCUMENT_TITLE = "Meter Data File Format Specification NEM12 & NEM13"
+SPEC_URL = (
+    "https://www.aemo.com.au/energy-systems/electricity/"
+    "national-electricity-market-nem/market-operations/retail-and-metering/"
+    "metering-procedures-guidelines-and-processes"
+)
+
+
+# -- Section 4.3, NMI data details (200) ------------------------------------
+
+#: IntervalLength values permitted by the spec, in minutes.
+#: The parser also accepts other divisors of 1440 (e.g. 1, 60) for tolerance
+#: with non-conforming files; pass through this constant if you want to
+#: reject anything outside the spec.
+ALLOWED_INTERVAL_LENGTHS: FrozenSet[int] = frozenset({5, 15, 30})
+
+
+# -- Section 5.3, Accumulation meter data (250) -----------------------------
+
+#: DirectionIndicator allowed values per the spec.
+DIRECTION_INDICATORS: Dict[str, str] = {
+    "I": "Import (energy flows from the connection point to the grid)",
+    "E": "Export (energy flows from the grid to the connection point)",
+}
+
+
+# -- Appendix A, Transaction code flags (500 / 550) -------------------------
+
+TRANSACTION_CODES: Dict[str, str] = {
+    "A": "Alteration",
+    "C": "Meter Reconfiguration",
+    "G": "Re-energisation",
+    "D": "De-energisation",
+    "E": "Estimate",
+    "N": "Normal Read",
+    "O": "Other (use when the original TransCode is unavailable, e.g. Historical Data)",
+    "S": "Special Read",
+    "R": "Removal of meter",
+}
+
+
+# -- Appendix B, allowed UOM values -----------------------------------------
+
+#: Units of measure allowed in the UOM field. Comparison is case-insensitive
+#: per the spec, so callers should normalise before lookup.
+UOM_VALUES: FrozenSet[str] = frozenset(
+    {
+        "MWh",
+        "kWh",
+        "Wh",
+        "MVArh",
+        "kVArh",
+        "VArh",
+        "MVAr",
+        "kVAr",
+        "VAr",
+        "MW",
+        "kW",
+        "W",
+        "MVAh",
+        "kVAh",
+        "VAh",
+        "MVA",
+        "kVA",
+        "VA",
+        "kV",
+        "V",
+        "kA",
+        "A",
+        "pf",
+    }
+)
+
+
+# -- Appendix C, Quality flags ----------------------------------------------
+
+QUALITY_FLAGS: Dict[str, str] = {
+    "A": "Actual Metering Data (no method flag required)",
+    "E": "Forward estimated data (method flag required, no reason code)",
+    "F": "Final substituted data (method flag and reason code required)",
+    "S": "Substituted data (method flag and reason code required)",
+    "V": (
+        "Variable data — only valid as the QualityMethod of a 300 record "
+        "to indicate that 400 rows carry per-interval flags. Not valid in "
+        "NEM13 files."
+    ),
+}
+
+
+# -- Appendix E, current reason codes ---------------------------------------
+
+REASON_CODES: Dict[int, str] = {
+    0: "Free text description",
+    1: "Meter/equipment changed",
+    2: "Extreme weather conditions",
+    3: "Quarantined premises",
+    5: "Blank screen",
+    6: "De-energised premises",
+    7: "Unable to locate meter",
+    8: "Vacant premises",
+    9: "Under investigation",
+    10: "Lock damaged unable to open",
+    11: "In wrong route",
+    12: "Locked premises",
+    13: "Locked gate",
+    14: "Locked meter box",
+    15: "Overgrown vegetation",
+    17: "Unsafe equipment/location",
+    18: "Read less than previous",
+    20: "Damaged equipment/panel",
+    21: "Main switch off",
+    22: "Meter/equipment seals missing",
+    23: "Reader error",
+    24: "Substituted/replaced data (data correction)",
+    25: "Unable to locate premises",
+    26: "Negative consumption (generation)",
+    27: "RoLR",
+    28: "CT/VT fault",
+    29: "Relay faulty/damaged",
+    31: "Not all meters read",
+    32: "Re-energised without readings",
+    33: "De-energised without readings",
+    34: "Meter not in handheld",
+    35: "Timeswitch faulty/reset required",
+    36: "Meter high/ladder required",
+    37: "Meter under churn",
+    38: "Unmarried lock",
+    39: "Reverse energy observed",
+    40: "Unrestrained livestock",
+    41: "Faulty Meter display/dials",
+    42: "Channel added/removed",
+    43: "Power outage",
+    44: "Meter testing",
+    45: "Readings failed to validate",
+    47: "Refused access",
+    48: "Dog on premises",
+    51: "Installation demolished",
+    52: "Access — blocked",
+    53: "Pests in meter box",
+    54: "Meter box damaged/faulty",
+    55: "Dials obscured",
+    60: "Illegal connection",
+    61: "Equipment tampered",
+    62: "NSRD window expired",
+    64: "Key required",
+    65: "Wrong key provided",
+    67: "Transfer",
+    68: "Zero consumption",
+    69: "Reading exceeds Substitute",
+    71: "Probe read error",
+    72: "Re-calculated based on Actual Metering Data",
+    73: "Low consumption",
+    74: "High consumption",
+    75: "Customer read",
+    76: "Communications fault",
+    77: "Estimation Forecast",
+    78: "Null Data",
+    79: "Power Outage Alarm",
+    80: "Short Interval Alarm",
+    81: "Long Interval Alarm",
+    87: "Reset occurred",
+    89: "Time reset occurred",
+    # Added in v2.6 (ICF_054 substitution type review, July 2023 REMP):
+    100: "Incorrect Meter Multiplier",
+    101: "Temporarily Connection Point unmetered",
+    102: "Customer By-Pass",
+    103: "Network By-Pass",
+    104: "Transposed Channel",
+    105: "Transposed Channel — UoM Correction",
+    106: "Transposed Channel — Reverse Polarity",
+    107: "Transposed Meter",
+    108: "Network by-pass extreme weather",
+    109: "Defined load method",
+}
+
+
+# -- Appendix F, obsolete reason codes (Historical Data only) ---------------
+
+OBSOLETE_REASON_CODES: Dict[int, str] = {
+    4: "Dangerous dog",
+    16: "Noxious weeds at premises preventing access",
+    19: "Consumer wanted",
+    30: "Meter stop switch on",
+    46: "Extreme weather/hot",
+    49: "Wet paint",
+    50: "Wrong tariff",
+    58: "Meter ok — supply failure",
+    70: "Probe reports tampering",
+    82: "CRC error",
+    83: "RAM checksum error",
+    84: "ROM checksum error",
+    85: "Data missing alarm",
+    86: "Clock error alarm",
+    88: "Watchdog timeout alarm",
+    90: "Test mode",
+    91: "Load control",
+    92: "Added interval (data correction)",
+    93: "Replaced interval (data correction)",
+    94: "Estimated interval (data correction)",
+    95: "Pulse overflow alarm",
+    96: "Data out of limits",
+    97: "Excluded data",
+    98: "Parity error",
+    99: "Energy type (register changed)",
+}
+
+
+__all__ = [
+    "ALLOWED_INTERVAL_LENGTHS",
+    "DIRECTION_INDICATORS",
+    "OBSOLETE_REASON_CODES",
+    "QUALITY_FLAGS",
+    "REASON_CODES",
+    "SPEC_DOCUMENT_TITLE",
+    "SPEC_EFFECTIVE_DATE",
+    "SPEC_URL",
+    "SPEC_VERSION",
+    "TRANSACTION_CODES",
+    "UOM_VALUES",
+]

aemo_mdff_reader/sql/__init__.py

@@ -0,0 +1,22 @@
+"""Optional SQL persistence layer.
+
+Importing this subpackage does NOT eagerly import :mod:`pymysql`.
+The dependency is only required when you actually instantiate
+:class:`Storer`. Install with ``pip install aemo-mdff-reader[mysql]``.
+"""
+
+from __future__ import annotations
+
+from .query import COLUMNS, QueryBuilder
+
+__all__ = ["COLUMNS", "DBCredentials", "QueryBuilder", "Storer"]
+
+
+def __getattr__(name):
+    # Lazy import so that ``import aemo_mdff_reader.sql`` does not require pymysql
+    # to be installed unless the caller actually uses the Storer.
+    if name in ("Storer", "DBCredentials"):
+        from .store import DBCredentials, Storer
+
+        return {"Storer": Storer, "DBCredentials": DBCredentials}[name]
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

aemo_mdff_reader/sql/query.py

@@ -0,0 +1,102 @@
+"""Contains classes and functions used for building queries in SQL"""
+# import statements
+
+IMPORT_TABLE = "imports"
+NMI_TABLE = "nmi"
+CHANNEL_TABLE = "channel"
+INTERVAL_TYPE_TABLE = "interval_type"
+INTERVAL_TABLE = "interval"
+INTERVAL_DATA_TABLE = "interval_data"
+INTERVAL_EVENT_TABLE = "interval_event"
+INTERVAL_B2B_TABLE = "interval_b2b"
+ACCUMULATION_DATA_TABLE = "accumulation_data"
+ACCUMULATION_B2B_TABLE = "accumulation_b2b"
+
+COLUMNS = {
+    IMPORT_TABLE: ["filename", "VersionHeader", "DateTime", "FromParticipant", "ToParticipant"],
+    NMI_TABLE: ["NMI", "NMIConfiguration"],
+    CHANNEL_TABLE: [
+        "NMI",
+        "RegisterID",
+        "NMISuffix",
+        "MDMDataStreamIdentifier",
+        "MeterSerialNumber",
+    ],
+    INTERVAL_TYPE_TABLE: [
+        "UOM",
+        "IntervalLength",
+        "NextScheduledReadDate",
+        "channelID",
+        "importID",
+    ],
+    INTERVAL_TABLE: [
+        "IntervalDate",
+        "QualityMethod",
+        "ReasonCode",
+        "ReasonDescription",
+        "UpdateDateTime",
+        "MSATSLoadDateTime",
+        "typeID",
+    ],
+    INTERVAL_DATA_TABLE: ["valueID", "value", "intervalID"],
+    INTERVAL_EVENT_TABLE: [
+        "StartInterval",
+        "EndInterval",
+        "QualityMethod",
+        "ReasonCode",
+        "ReasonDescription",
+        "intervalID",
+    ],
+    INTERVAL_B2B_TABLE: ["TransCode", "RetServiceOrder", "ReadDateTime", "IndexRead", "eventID"],
+    ACCUMULATION_DATA_TABLE: [],
+    ACCUMULATION_B2B_TABLE: [],
+}
+
+
+class QueryBuilder:
+    """
+    Builds a query from given information.
+    """
+
+    COLUMN_GEN = 'NULLIF("%s", "")'
+    NUM_WRAPPER = "CAST(%s AS %s)"
+    DECIMAL_TYPE_1 = "DECIMAL(18, 3)"
+    DECIMAL_TYPE_2 = "DECIMAL(22, 5)"
+    INT_TYPE = "INT"
+    INSERT_QUERY = """INSERT INTO %s
+                      (%s)
+                   """
+    VALUES_QUERY = """VALUES (%s)"""
+
+    @staticmethod
+    def insert_query(table, data):
+        """
+        Insert query builder
+        """
+        query = "INSERT IGNORE INTO `%s`\n" % table
+        query += "(" + ",".join(["`" + item + "`" for item in COLUMNS[table]]) + ")" + "\n"
+        query += "VALUES "
+        for instance in data:
+            list_value = [
+                (
+                    QueryBuilder.COLUMN_GEN % value
+                    if "@" not in value and "STR" not in value
+                    else value
+                )
+                for value in instance
+            ]
+            query += "("
+            for value in list_value:
+                query += value + ", "
+            query = query[:-2] + "),\n"
+        query = query[:-2] + ";"
+        return query
+
+    @staticmethod
+    def get_id_query():
+        """
+        Most recent ID retriever
+        """
+        query = "LAST_INSERT_ID();"
+
+        return query

aemo_mdff_reader/sql/store.py

@@ -0,0 +1,178 @@
+"""Storage for NEM12-13 within SQL Schema as specified."""
+
+# import statements
+
+import pymysql.cursors
+
+from .query import *
+
+# Load queries once session is imported
+QUERY_FOLDER = "queries"
+QUERIES = {}
+
+
+class Storer:
+    """
+    Stores Reader into the DB
+    """
+
+    def __init__(self, credentials):
+        if credentials is not None:
+            self.__connection = self.connect(**credentials)
+
+    @staticmethod
+    def connect(host, port, user, password, schema, directory):
+        """
+        Connects to a MySQL database with the
+        given credentials.
+
+        Args:
+            db_credentials (DBCredentials): The credentials
+                of the DB to connect to
+        Returns:
+            pymysql.connect. A connection to the DB
+        """
+        # assert isinstance(db_credentials, DBCredentials)
+        # connect to server
+        connection = pymysql.connect(
+            host=host,
+            user=user,
+            password=password,
+            db=schema,
+            charset="utf8mb4",
+            autocommit=True,
+            cursorclass=pymysql.cursors.Cursor,  # TODO: check if suitable cursor
+        )
+        return connection
+
+    def load_reader(self, reader):
+        """
+        loads reader
+        """
+        print(reader.filename)
+        with self.__connection.cursor() as cursor:
+            # first store header and filename
+            data = [
+                [
+                    reader.filename.replace("\\", "/"),
+                    reader.data.VersionHeader,
+                    f'STR_TO_DATE("{reader.data.DateTime}", "%Y%m%d%k%i%s")',
+                    reader.data.FromParticipant,
+                    reader.data.ToParticipant,
+                ]
+            ]
+            query = QueryBuilder.insert_query(IMPORT_TABLE, data)
+            resultset = cursor.execute(query)
+            if resultset == 0:
+                return None
+            cursor.execute("SELECT importID FROM imports WHERE filename LIKE %s", (data[0][0]))
+            importID = str(cursor.fetchone()[0])
+            print(importID)
+            channelID = None
+            typeID = None
+            intervalID = None
+            # now store nmi, channels and interval type
+            data_nmi = []
+            data_channel = []
+            data_interval_type = []
+            data_intervals = []
+            for subrecord in reader.data.subrecords:
+                if isinstance(subrecord, list):
+                    continue
+                if subrecord.RecordIndicator == "200":
+                    data_interval = []
+                    data_nmi.append([subrecord.NMI, subrecord.NMIConfiguration])
+                    data_channel.append(
+                        [
+                            subrecord.NMI,
+                            subrecord.RegisterID,
+                            subrecord.NMISuffix,
+                            subrecord.MDMDataStreamIdentifier,
+                            subrecord.MeterSerialNumber,
+                        ]
+                    )
+                    data_interval_type.append(
+                        [
+                            subrecord.UOM,
+                            subrecord.IntervalLength,
+                            f'STR_TO_DATE("{subrecord.NextScheduledReadDate}", "%Y%m%d%k%i%s")',
+                            channelID,
+                            importID,
+                        ]
+                    )
+                    # add each interval
+                    for subsubrecord in subrecord.subrecords:
+                        if subsubrecord.RecordIndicator == "300":
+                            interval_data = [
+                                (
+                                    key[0].replace("IntervalValue", ""),
+                                    subsubrecord.__getattr__(key[0]),
+                                    intervalID,
+                                )
+                                for key in subsubrecord.intervals
+                            ]
+                            data_interval.append(
+                                [
+                                    [
+                                        f'STR_TO_DATE("{subsubrecord.IntervalDate}", "%Y%m%d")',
+                                        subsubrecord.QualityMethod,
+                                        subsubrecord.ReasonCode,
+                                        subsubrecord.ReasonDescription,
+                                        f'STR_TO_DATE("{subsubrecord.UpdateDateTime}", "%Y%m%d%k%i%s")',
+                                        f'STR_TO_DATE("{subsubrecord.MSATSLoadDateTime}", "%Y%m%d%k%i%s")',
+                                        typeID,
+                                    ],
+                                    interval_data,
+                                ]
+                            )
+
+                    data_intervals.append(data_interval)
+            query = QueryBuilder.insert_query(NMI_TABLE, data_nmi)
+            resultset = cursor.execute(query)
+
+            for row in range(len(data_channel)):
+                query = QueryBuilder.insert_query(CHANNEL_TABLE, [data_channel[row]])
+                resultset = cursor.execute(query)
+                query = "SELECT channelID FROM `nem`.`channel` WHERE `NMI` = %s AND `NMISuffix` = %s AND `MeterSerialNumber` = %s"
+                cursor.execute(
+                    query, (data_channel[row][0], data_channel[row][2], data_channel[row][4])
+                )
+                channelID = str(cursor.fetchone()[0])
+                data_interval_type[row][-2] = channelID
+                cursor.execute(
+                    QueryBuilder.insert_query(INTERVAL_TYPE_TABLE, [data_interval_type[row]])
+                )
+                cursor.execute(
+                    """SELECT typeID FROM interval_type WHERE channelID = %s AND importID = %s""",
+                    (channelID, importID),
+                )
+                typeID = str(cursor.fetchone()[0])
+                for interval_row in data_intervals[row]:
+                    interval_row[0][-1] = typeID
+                    query = QueryBuilder.insert_query(INTERVAL_TABLE, [interval_row[0]])
+                    cursor.execute(query)
+                    cursor.execute(
+                        "SELECT intervalID FROM `interval` WHERE typeID = %s AND IntervalDate = %s"
+                        % (typeID, interval_row[0][0])
+                    )
+                    intervalID = str(cursor.fetchone()[0])
+                    interval_row[1] = [(val[0], val[1], intervalID) for val in interval_row[1]]
+                    query = QueryBuilder.insert_query(INTERVAL_DATA_TABLE, list(interval_row[1]))
+                    cursor.execute(query)
+
+    def close(self):
+        """
+        Closes the current connection
+        """
+        self.__connection.close()
+
+
+class DBCredentials:
+    """Credentials for SQL Database"""
+
+    def __init__(self, host, port, schema, username, password):
+        self.host = host
+        self.port = port
+        self.schema = schema
+        self.username = username
+        self.password = password