aemo-mdff-reader 2.0.2__py3-none-any.whl

aemo_mdff_reader/__init__.py

@@ -0,0 +1,84 @@
+"""aemo-mdff-reader — fast streaming reader for AEMO NEM12 / NEM13 files.
+
+Recommended entry points:
+
+>>> from aemo_mdff_reader import parse                  # NEM12 300 records
+>>> from aemo_mdff_reader import parse_accumulations    # NEM13 250 records
+>>> from aemo_mdff_reader import parse_all              # both, in file order
+
+Each iterator is lazy (O(1) memory). For pandas DataFrames, see
+:func:`to_dataframe` (NEM12) and :func:`to_accumulations_dataframe` (NEM13).
+
+Spec compatibility: AEMO MDFF v2.6 (effective 29 September 2024). The
+allowed-value tables — quality flags, transaction codes, reason codes,
+units of measure — are exposed as constants in :mod:`aemo_mdff_reader.spec`.
+"""
+
+from __future__ import annotations
+
+from . import aggregate, spec
+from .parser import (
+    NEM12ParseError,
+    iter_columns_chunks,
+    iter_dataframes,
+    nmi_checksum,
+    parse,
+    parse_accumulations,
+    parse_accumulations_to_columns,
+    parse_all,
+    parse_b2b,
+    parse_events,
+    parse_header,
+    parse_to_columns,
+    to_accumulations_dataframe,
+    to_columns,
+    to_dataframe,
+    validate_file,
+    validate_nmi,
+    write_accumulations_csv,
+    write_csv,
+)
+from .reader import INTERVAL_DATA_OUTPUT_HEADERS, NEMReader
+from .types import (
+    AccumulationReading,
+    B2BDetails,
+    Header,
+    IntervalEvent,
+    IntervalReading,
+    NMIDetails,
+)
+
+__version__ = "2.0.2"
+
+__all__ = [
+    "INTERVAL_DATA_OUTPUT_HEADERS",
+    "AccumulationReading",
+    "B2BDetails",
+    "Header",
+    "IntervalEvent",
+    "IntervalReading",
+    "NEM12ParseError",
+    "NEMReader",
+    "NMIDetails",
+    "__version__",
+    "aggregate",
+    "iter_columns_chunks",
+    "iter_dataframes",
+    "nmi_checksum",
+    "parse",
+    "parse_accumulations",
+    "parse_accumulations_to_columns",
+    "parse_all",
+    "parse_b2b",
+    "parse_events",
+    "parse_header",
+    "parse_to_columns",
+    "spec",
+    "to_accumulations_dataframe",
+    "to_columns",
+    "to_dataframe",
+    "validate_file",
+    "validate_nmi",
+    "write_accumulations_csv",
+    "write_csv",
+]

aemo_mdff_reader/aggregate.py

@@ -0,0 +1,168 @@
+"""Post-parse aggregation helpers.
+
+Lightweight composable iterators for the most common downstream
+operations on parsed NEM12 data: grouping by NMI/channel, and rolling
+intervals up to daily totals. Each function takes any iterable of
+:class:`IntervalReading` objects, so they compose with :func:`parse`,
+``parse_to_columns``, and the buffered :class:`NEMReader` facade.
+
+>>> from aemo_mdff_reader import parse
+>>> from aemo_mdff_reader.aggregate import daily_totals
+>>> for d in daily_totals(parse("metering.csv")):
+...     print(d.nmi, d.register_id, d.interval_date, d.total)
+"""
+
+from __future__ import annotations
+
+import itertools
+from datetime import datetime
+from typing import Iterable, Iterator, List, NamedTuple, Optional, Tuple, TypeVar
+
+from .types import IntervalReading
+
+_T = TypeVar("_T")
+
+
+def iter_chunks(items: Iterable[_T], size: int) -> Iterator[List[_T]]:
+    """Yield successive lists of ``size`` items from ``items``.
+
+    Useful for batching streaming reads when the downstream consumer
+    works in fixed-size lots — DB ``executemany`` / batch HTTP / etc. —
+    without materialising the whole stream in memory. Memory cost is
+    O(size), not O(file).
+
+    The final batch may be shorter than ``size`` if the input is not a
+    multiple of it. Empty input yields no batches.
+
+    >>> from aemo_mdff_reader import parse
+    >>> from aemo_mdff_reader.aggregate import iter_chunks
+    >>> for batch in iter_chunks(parse("metering.csv"), 5_000):
+    ...     db.executemany(INSERT_SQL, [r.to_dict() for r in batch])
+    """
+    if size <= 0:
+        raise ValueError(f"chunk size must be positive, got {size}")
+    it = iter(items)
+    while True:
+        chunk = list(itertools.islice(it, size))
+        if not chunk:
+            return
+        yield chunk
+
+
+class ChannelKey(NamedTuple):
+    """Composite key identifying a single metering channel."""
+
+    nmi: str
+    register_id: str
+    nmi_suffix: str
+
+
+class DailyTotal(NamedTuple):
+    """A single channel's total energy / quantity for a calendar day.
+
+    ``total`` is the sum of ``IntervalReading.value`` for every reading
+    that fell on ``interval_date``. ``interval_count`` is the number of
+    readings summed (useful for spotting partial days). ``unique_quality_flags``
+    is the set of distinct 1-character quality flags seen — a single
+    ``{"A"}`` means the day is fully Actual, anything else hints at
+    substituted or estimated data.
+    """
+
+    nmi: str
+    register_id: str
+    nmi_suffix: str
+    uom: str
+    interval_date: datetime
+    total: float
+    interval_count: int
+    unique_quality_flags: frozenset[str]
+
+
+def group_by_nmi(
+    readings: Iterable[IntervalReading],
+) -> Iterator[Tuple[ChannelKey, Iterator[IntervalReading]]]:
+    """Yield ``(ChannelKey, group)`` pairs, one per (NMI, register, suffix).
+
+    NEM12 files are emitted in 200-record blocks: every 300 row inside
+    a block shares the same NMI / register / suffix, and a new 200 row
+    starts a new block. We use :func:`itertools.groupby` over those
+    fields, which means each ``group`` iterator must be consumed before
+    the next pair is requested — exactly the same contract as
+    ``itertools.groupby``.
+
+    For a materialised list per group, wrap the inner iterator::
+
+        for key, group in group_by_nmi(parse(path)):
+            rows = list(group)
+            ...
+    """
+
+    def _key(r: IntervalReading) -> ChannelKey:
+        return ChannelKey(r.nmi, r.register_id, r.nmi_suffix)
+
+    yield from itertools.groupby(readings, key=_key)
+
+
+def daily_totals(readings: Iterable[IntervalReading]) -> Iterator[DailyTotal]:
+    """Yield one :class:`DailyTotal` per (channel, calendar day).
+
+    Within each 300 row, all readings share the same ``interval_date``
+    and the parser emits them contiguously, so the rollup is a streaming
+    O(1)-memory operation: we accumulate while ``(channel, date)`` stays
+    constant and emit when it changes.
+
+    Empty cells in the source are coerced to ``0.0`` upstream (see
+    :class:`IntervalReading`); they are summed as zero.
+    """
+    current_key: Optional[Tuple[ChannelKey, datetime]] = None
+    current_uom: str = ""
+    total = 0.0
+    count = 0
+    flags: set[str] = set()
+
+    for r in readings:
+        key = (ChannelKey(r.nmi, r.register_id, r.nmi_suffix), r.interval_date)
+        if current_key is None:
+            current_key = key
+            current_uom = r.uom
+        elif key != current_key:
+            yield DailyTotal(
+                nmi=current_key[0].nmi,
+                register_id=current_key[0].register_id,
+                nmi_suffix=current_key[0].nmi_suffix,
+                uom=current_uom,
+                interval_date=current_key[1],
+                total=total,
+                interval_count=count,
+                unique_quality_flags=frozenset(flags),
+            )
+            current_key = key
+            current_uom = r.uom
+            total = 0.0
+            count = 0
+            flags = set()
+        total += r.value
+        count += 1
+        if r.quality_method:
+            flags.add(r.quality_method[:1])
+
+    if current_key is not None:
+        yield DailyTotal(
+            nmi=current_key[0].nmi,
+            register_id=current_key[0].register_id,
+            nmi_suffix=current_key[0].nmi_suffix,
+            uom=current_uom,
+            interval_date=current_key[1],
+            total=total,
+            interval_count=count,
+            unique_quality_flags=frozenset(flags),
+        )
+
+
+__all__ = [
+    "ChannelKey",
+    "DailyTotal",
+    "daily_totals",
+    "group_by_nmi",
+    "iter_chunks",
+]

aemo_mdff_reader/cli.py

@@ -0,0 +1,231 @@
+"""Command-line interface for aemo-mdff-reader.
+
+Installed as the ``aemo-mdff-reader`` console script via the project's
+``[project.scripts]`` entry in ``pyproject.toml``.
+
+Usage::
+
+    aemo-mdff-reader INPUT [-o OUTPUT] [--records intervals|accumulations]
+                                   [--format csv|parquet]
+                                   [--nmi NMI [--nmi NMI ...]]
+                                   [--start YYYY-MM-DD] [--end YYYY-MM-DD]
+                                   [--validate]
+
+If ``OUTPUT`` is omitted, results are written to stdout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import date, datetime
+from typing import Iterable, Iterator, Optional, Sequence
+
+from . import __version__
+from .parser import (
+    parse,
+    parse_accumulations,
+    parse_accumulations_to_columns,
+    parse_to_columns,
+    validate_file,
+    write_accumulations_csv,
+    write_csv,
+)
+from .types import AccumulationReading, IntervalReading
+
+
+def _iso_date(value: str) -> date:
+    """Parse a YYYY-MM-DD argparse value, raising ArgumentTypeError on bad input."""
+    try:
+        return datetime.strptime(value, "%Y-%m-%d").date()
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError(f"expected YYYY-MM-DD date, got {value!r}") from exc
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="aemo-mdff-reader",
+        description=(
+            "Read an AEMO NEM12 / NEM13 file and emit a flat CSV. Streams in O(1) memory."
+        ),
+    )
+    p.add_argument("input", help="Path to the NEM12 / NEM13 input file (CSV).")
+    p.add_argument(
+        "-o",
+        "--output",
+        default="-",
+        help="Output path. Use '-' (the default) for stdout.",
+    )
+    p.add_argument(
+        "--records",
+        choices=("intervals", "accumulations"),
+        default="intervals",
+        help=(
+            "Which record type to emit. 'intervals' = NEM12 300 records "
+            "(default), 'accumulations' = NEM13 250 records."
+        ),
+    )
+    p.add_argument(
+        "--format",
+        choices=("csv", "parquet"),
+        default="csv",
+        help="Output format. 'parquet' requires pandas + pyarrow.",
+    )
+    p.add_argument(
+        "--nmi",
+        action="append",
+        default=None,
+        metavar="NMI",
+        help=(
+            "Restrict output to the given NMI. Repeat the flag to keep "
+            "more than one (e.g. --nmi A --nmi B). Default: all NMIs."
+        ),
+    )
+    p.add_argument(
+        "--start",
+        type=_iso_date,
+        default=None,
+        metavar="YYYY-MM-DD",
+        help="Drop rows whose interval / current-read date is before this date.",
+    )
+    p.add_argument(
+        "--end",
+        type=_iso_date,
+        default=None,
+        metavar="YYYY-MM-DD",
+        help="Drop rows whose interval / current-read date is after this date.",
+    )
+    p.add_argument(
+        "--validate",
+        action="store_true",
+        help=(
+            "Run structural validation against the AEMO MDFF spec and exit. "
+            "Returns 0 if the file is valid, 1 if issues were found."
+        ),
+    )
+    p.add_argument(
+        "--version",
+        action="version",
+        version=f"aemo-mdff-reader {__version__}",
+    )
+    return p
+
+
+def _filter_intervals(
+    readings: Iterable[IntervalReading],
+    *,
+    nmis: Optional[Sequence[str]],
+    start: Optional[date],
+    end: Optional[date],
+) -> Iterator[IntervalReading]:
+    nmi_set = set(nmis) if nmis else None
+    for r in readings:
+        if nmi_set is not None and r.nmi not in nmi_set:
+            continue
+        d = r.interval_date.date()
+        if start is not None and d < start:
+            continue
+        if end is not None and d > end:
+            continue
+        yield r
+
+
+def _filter_accumulations(
+    readings: Iterable[AccumulationReading],
+    *,
+    nmis: Optional[Sequence[str]],
+    start: Optional[date],
+    end: Optional[date],
+) -> Iterator[AccumulationReading]:
+    nmi_set = set(nmis) if nmis else None
+    for r in readings:
+        if nmi_set is not None and r.nmi not in nmi_set:
+            continue
+        if r.current_register_read_datetime is not None:
+            d = r.current_register_read_datetime.date()
+            if start is not None and d < start:
+                continue
+            if end is not None and d > end:
+                continue
+        yield r
+
+
+def _emit_parquet(args: argparse.Namespace) -> int:
+    try:
+        import pandas as pd
+    except ImportError:
+        print(
+            "parquet output requires pandas + pyarrow: pip install aemo-mdff-reader[parquet]",
+            file=sys.stderr,
+        )
+        return 2
+    if args.records == "accumulations":
+        cols = parse_accumulations_to_columns(args.input)
+        df = pd.DataFrame(cols)
+    else:
+        cols = parse_to_columns(args.input)
+        df = pd.DataFrame(cols)
+
+    # Apply --nmi / --start / --end filters in-DataFrame so the parquet
+    # output matches what the streaming CSV path would emit.
+    if args.nmi:
+        df = df[df["NMI"].isin(args.nmi)]
+    if args.records == "accumulations":
+        date_col = "CurrentRegisterReadDatetime"
+    else:
+        date_col = "IntervalDate"
+    if args.start is not None:
+        df = df[df[date_col] >= pd.Timestamp(args.start)]
+    if args.end is not None:
+        df = df[df[date_col] <= pd.Timestamp(args.end)]
+
+    df.to_parquet(args.output)
+    return 0
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    args = _build_parser().parse_args(argv)
+
+    if args.validate:
+        issues = validate_file(args.input)
+        if issues:
+            for msg in issues:
+                print(msg, file=sys.stderr)
+            return 1
+        print("OK", file=sys.stderr)
+        return 0
+
+    if args.format == "parquet":
+        return _emit_parquet(args)
+
+    out_target = sys.stdout if args.output == "-" else args.output
+
+    try:
+        if args.records == "accumulations":
+            acc_stream = _filter_accumulations(
+                parse_accumulations(args.input),
+                nmis=args.nmi,
+                start=args.start,
+                end=args.end,
+            )
+            write_accumulations_csv(acc_stream, out_target)
+        else:
+            int_stream = _filter_intervals(
+                parse(args.input),
+                nmis=args.nmi,
+                start=args.start,
+                end=args.end,
+            )
+            write_csv(int_stream, out_target)
+    except BrokenPipeError:
+        # Downstream consumer (e.g. ``head``) closed the pipe early —
+        # this is normal, not an error.
+        try:
+            sys.stdout.flush()
+        except BrokenPipeError:
+            pass
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())