telemetry-cli 1.0.0__py3-none-any.whl

telemetry_cli/pick/cli.py

@@ -0,0 +1,157 @@
+"""The pick command: extract and print fields from fixed-length binary records.
+
+    pick file.dat reclen [ fieldRequests | options | parameters ]
+    cat file.dat | pick reclen [ fieldRequests | options | parameters ]
+
+Run ``pick ?`` for help.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import BinaryIO, TextIO
+
+from .._cli import broken_pipe
+from .commandline import UsageError, parse, resolve, wants_help
+from .fields import FieldError, parse_fields
+from .formats import FormatError, render
+from .help import help_text
+
+
+class _Input:
+    """Record input with the Perl original's seek rules.
+
+    Files can seek anywhere; stdin can only skip forward (a request to go
+    back is ignored, as in the Perl).
+    """
+
+    def __init__(self, stream: BinaryIO, seekable: bool):
+        self.stream = stream
+        self.seekable = seekable
+        self.pos = 0
+
+    def seek(self, target: int) -> bool:
+        if not target:
+            return True
+        n = target - self.pos
+        if not n:
+            return True
+        if self.seekable:
+            if target < 0:
+                return False
+            self.pos = target
+            self.stream.seek(n, 1)
+            return True
+        if n < 1:
+            return True
+        self.pos = target
+        return len(self.stream.read(n)) > 0
+
+    def read(self, n: int) -> bytes:
+        return self.stream.read(n)
+
+
+def main(
+    argv: list[str] | None = None,
+    *,
+    stdin: BinaryIO | None = None,
+    stdout: BinaryIO | None = None,
+    stderr: TextIO | None = None,
+    host_order: str | None = None,
+) -> int:
+    """Run pick. Returns the exit status.
+
+    host_order ("big" or "little") overrides the machine's byte order, which
+    is used when the data's order isn't given; the tests use it to check
+    pick's big-endian behavior on any machine.
+    """
+    argv = sys.argv[1:] if argv is None else list(argv)
+    stdin = stdin if stdin is not None else sys.stdin.buffer
+    stdout = stdout if stdout is not None else sys.stdout.buffer
+    stderr = stderr if stderr is not None else sys.stderr
+    host = host_order or sys.byteorder
+    try:
+        return _run(argv, stdin, stdout, stderr, host)
+    except (UsageError, FieldError, FormatError) as e:
+        stdout.flush()
+        print(f"pick: {e}", file=stderr)
+        topics = getattr(e, "topics", "")
+        if isinstance(e, FormatError):
+            topics = "f"
+        if topics:
+            print(f"(see 'pick --help {topics}')", file=stderr)
+        return 1
+    except BrokenPipeError:
+        return broken_pipe(stdout)
+
+
+def _help_argv(argv: list[str]) -> list[str]:
+    """Turn ``--help [topic]`` / ``-h [topic]`` into pick's ``?topic`` form."""
+    if argv and argv[0] in ("--help", "-h"):
+        topic = argv[1] if len(argv) > 1 else ""
+        return ["??" if topic == "all" else "?" + topic.lstrip("?") if topic else "help"]
+    return argv
+
+
+def _run(argv, stdin, stdout, stderr, host) -> int:
+    cmd = parse(_help_argv(argv))
+    if wants_help(cmd.text):
+        stdout.write(
+            help_text(
+                cmd.text,
+                cmd.symbols if cmd.symbol_file else None,
+                cmd.symbol_file or "",
+            ).encode("latin-1")
+        )
+        return 0
+    plan = resolve(cmd)
+    opts = cmd.opts
+    order = host if cmd.order in (None, "native") else cmd.order
+    if opts["r"]:
+        order = "big" if order == "little" else "little"
+    fields = parse_fields(plan.fields_text, plan.size, binary=opts["u"])
+    if plan.size <= 0:  # the Perl printed empty records forever
+        raise UsageError("the record size must be at least 1 byte", "p")
+
+    if plan.file == "-":
+        if plan.from_stdin_default and not opts["q"]:
+            print("Reading from stdin", file=stderr)
+        source = _Input(stdin, seekable=False)
+        return _loop(source, plan, fields, opts, order, host, stdout)
+    try:
+        f = open(plan.file, "rb")  # noqa: SIM115
+    except OSError as e:
+        raise UsageError(f"can't read {plan.file}: {e.strerror}") from None
+    with f:
+        return _loop(_Input(f, seekable=True), plan, fields, opts, order, host, stdout)
+
+
+def _loop(source: _Input, plan, fields, opts, order, host, stdout) -> int:
+    if plan.head and not source.seek(plan.head):
+        raise UsageError("End of file while seeking past header")
+    newline = not (opts["u"] or opts["b"] or opts["s"])
+    record, lines = plan.start, 0
+    while True:
+        if not source.seek(record * plan.size + plan.head):
+            break
+        data = source.read(plan.size)
+        if len(data) != plan.size:
+            break
+        out = bytearray()
+        if opts["l"]:
+            out += f"{lines} ".encode()
+        if opts["n"]:
+            out += f"{record} ".encode()
+        if opts["m"]:
+            out += f"{source.pos} ".encode()
+        out += render(data, fields, order=order, host=host, fast_binary=opts["b"])
+        if newline:
+            out += b"\n"
+        stdout.write(out)
+        source.pos += plan.size
+        record += plan.skip + 1
+        lines += 1
+        if plan.stop >= 0 and record > plan.stop:
+            break
+    stdout.flush()
+    return 0

telemetry_cli/pick/commandline.py

@@ -0,0 +1,219 @@
+"""Reading a pick command line.
+
+pick doesn't parse its arguments one by one. Like the Perl original, it joins
+them into one string and removes pieces in this order: options (``-nlr``),
+numeric parameters (``start=5``, ``head=4z``), word parameters (``file=...``,
+``sym=...``), then expands symbols from a symbol file. What is left is an
+optional file name, the record size, and the field requests. This is why
+parameters can go anywhere on the command line.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass, field
+from importlib import resources
+from pathlib import Path
+
+from .._perl import truthy
+from .fields import DTYPES, NBYTES, parse_size
+
+OPTIONS = "mnblqrsu"
+NUM_PARAMS = "start|head|stop|skip|nrecs|every|length|size|header|rec"
+WORD_PARAMS = "file|filename|sym|symFile|symTable"
+MAX_SYMBOL_EXPANSIONS = 100_000
+
+
+class UsageError(ValueError):
+    """A problem with the command line. ``topics`` names help to suggest."""
+
+    def __init__(self, message: str, topics: str = ""):
+        super().__init__(message)
+        self.topics = topics
+
+
+@dataclass
+class Command:
+    opts: dict[str, bool]
+    params: dict[str, int | str | None]
+    text: str  # what's left after options, parameters and symbols are removed
+    symbols: dict[str, str] = field(default_factory=dict)
+    symbol_file: str | None = None
+    order: str | None = None  # order=big|little, if given
+
+
+def parse_options(s: str) -> tuple[str, dict[str, bool]]:
+    opts = dict.fromkeys(OPTIONS, False)
+    while m := re.search(r"\s([-]([a-zA-Z]+))", s):
+        tag, name = m.group(1), m.group(2)
+        for k in OPTIONS:
+            if k in name:
+                opts[k] = True
+                name = name.replace(k, "")
+        if name:
+            raise UsageError(f"Unrecognized option-->{name}<--", "o")
+        s = s.replace(tag, "", 1)
+    return s, opts
+
+
+def parse_params(s: str) -> tuple[str, dict]:
+    params: dict[str, int | str | None] = {}
+    num = re.compile(rf"(({NUM_PARAMS})\s*=\s*(\d+)([{DTYPES}]?))", re.IGNORECASE)
+    while m := num.search(s):
+        whole, key, value, unit = m.groups()
+        s = s[: m.start()] + s[m.end() :]
+        params[key] = int(value) * (NBYTES[unit] if unit else 1)
+    word = re.compile(rf"(({WORD_PARAMS})\s*=\s*(\S+))", re.IGNORECASE)
+    while m := word.search(s):
+        s = s[: m.start()] + s[m.end() :]
+        params[m.group(2)] = m.group(3)
+    return s, params
+
+
+def parse_order(s: str) -> tuple[str, str | None]:
+    """The order=big|little parameter (new in the Python version)."""
+    m = re.search(r"(?<!\S)order\s*=\s*(\S+)", s)
+    if not m:
+        return s, None
+    value = m.group(1).lower()
+    if value not in ("big", "little", "native"):
+        raise UsageError(f"order must be big, little or native, not {m.group(1)!r}", "p")
+    return s[: m.start()] + s[m.end() :], value
+
+
+def find_symbol_file(name: str) -> Path:
+    if os.path.exists(name):
+        return Path(name)
+    path = os.environ.get("PICKPATH", "") + "/" + name
+    if os.path.exists(path):
+        return Path(path)
+    bundled = resources.files("telemetry_cli.pick") / "symbols" / name
+    if "/" not in name and bundled.is_file():
+        return Path(str(bundled))
+    raise UsageError(f"Couldn't find symbol table-->{path}<--", "q")
+
+
+def read_symbols(path: Path) -> dict[str, str]:
+    symbols: dict[str, str] = {}
+    for line in path.read_text(encoding="latin-1").split("\n"):
+        line = re.sub(r";.*", "", line)
+        m = re.search(r"(\S+)\s*=\s*(.+)", line)
+        if m and truthy(m.group(1)) and truthy(m.group(2)):
+            symbols[m.group(1)] = m.group(2)
+    return symbols
+
+
+def expand_symbols(s: str, symbols: dict[str, str]) -> str:
+    """Replace symbol names with their definitions, repeatedly, as whole words.
+
+    Names match without regard to case, but the replacement is looked up by
+    the text as written, so a name typed in the wrong case expands to nothing
+    (as in the Perl).
+    """
+    if not symbols:
+        return s
+    try:
+        pattern = re.compile(r"\b(" + "|".join(symbols) + r")\b", re.IGNORECASE)
+    except re.error as e:
+        raise UsageError(f"can't use the symbol names in this table: {e}", "q") from None
+    for _ in range(MAX_SYMBOL_EXPANSIONS):
+        m = pattern.search(s)
+        if not m:
+            return s
+        s = s[: m.start()] + symbols.get(m.group(1), "") + s[m.end() :]
+    raise UsageError("symbol definitions refer to themselves without end", "q")
+
+
+def parse(argv: list[str]) -> Command:
+    s = " ".join(argv)
+    s, opts = parse_options(s)
+    s, params = parse_params(s)
+    s, order = parse_order(s)
+    if "=" in s:
+        raise UsageError(f"Unexpected '=' in command line after parsing parms:\n     {s}", "p")
+    cmd = Command(opts=opts, params=params, text=s, order=order)
+    for key in ("symTable", "symFile"):
+        if truthy(params.get(key)) and not truthy(params.get("sym")):
+            params["sym"] = params[key]
+            break
+    if truthy(params.get("sym")):
+        path = find_symbol_file(str(params["sym"]))
+        cmd.symbol_file = str(params["sym"]) if os.path.exists(str(params["sym"])) else str(path)
+        cmd.symbols = read_symbols(path)
+        cmd.text = expand_symbols(cmd.text, cmd.symbols)
+        for key in ("head", "size", "length"):
+            if key in cmd.symbols:
+                m = re.search(rf"(\d+)([{DTYPES}]?)", cmd.symbols[key])
+                params[key] = int(m.group(1)) * NBYTES.get(m.group(2) or "b", 1) if m else None
+    return cmd
+
+
+def wants_help(text: str) -> bool:
+    return "?" in text or not truthy(text) or "help" in text
+
+
+@dataclass
+class Plan:
+    """Where to read and which records, after the file name and size are found."""
+
+    file: str
+    size: int
+    head: int
+    start: int
+    stop: int
+    skip: int
+    fields_text: str
+    from_stdin_default: bool
+
+
+def resolve(cmd: Command) -> Plan:
+    params = cmd.params
+    s = cmd.text.lstrip()
+    args = re.split(r"\s", s) if s else []
+    while args and args[-1] == "":  # Perl's split drops trailing empty fields
+        args.pop()
+    file = params.get("file")
+    default_stdin = False
+    if not truthy(file):
+        if args and args[0] and os.path.exists(args[0]):
+            file = args.pop(0)
+            s = s.replace(file, "", 1)
+        else:
+            file = "-"
+            default_stdin = True
+    if "rec" in params:
+        params["start"] = params["stop"] = params["rec"]
+    head = params.get("head") or params.get("header") or 0
+    size = params.get("size") or params.get("length")
+    if not size:
+        first = args[0] if args else ""
+        size = parse_size(first)
+        if size is None:
+            raise UsageError(f"Can't find record size; looking at {first}")
+        s = s.replace(first, "", 1)
+    skip = params.get("skip")
+    if skip is None:
+        skip = params["every"] - 1 if "every" in params else 0
+        if skip < 0:  # the Perl reread the same record forever
+            raise UsageError("every must be at least 1", "p")
+    start, stop, nrecs = params.get("start"), params.get("stop"), params.get("nrecs")
+    if start is None or stop is None:
+        if start is not None:
+            stop = start + nrecs - 1 if nrecs is not None else -1
+        elif stop is not None:
+            start = stop - nrecs + 1 if nrecs is not None else 0
+        elif nrecs is not None:
+            start, stop = 0, nrecs - 1
+        else:
+            start, stop = 0, -1
+    return Plan(
+        file=str(file),
+        size=int(size),
+        head=int(head),
+        start=int(start),
+        stop=int(stop),
+        skip=int(skip),
+        fields_text=s,
+        from_stdin_default=default_stdin,
+    )

telemetry_cli/pick/fields.py

@@ -0,0 +1,166 @@
+"""pick's field-request language: data types, print requests, moves and groups.
+
+A print request is ``N type Offset :bitOffset:nBits formats`` (e.g. ``4d0+30``,
+``w5:1:7bdx``); a move is ``(+|-)N type`` (e.g. ``+5z``); and requests can be
+grouped with brackets and repeated (``20[ f +99f i -100f ]``). See
+``pick ?r``, ``pick ?m`` and ``pick ?g``.
+
+This follows the Perl original closely, including how groups are rewritten
+and how the current position may run past either end of the record.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .._perl import truthy
+
+DTYPES = "SAbcswuidfzZ"
+PFORMS = "udxnsgGoMPRIDbSAcU"
+
+SIZE_MATCH = re.compile(rf"^((\d+)([{DTYPES}])?)$")
+DATA_MATCH = re.compile(
+    rf"(^(\d+)?([{DTYPES}])((\d+)([+-]\d+)?)?(:(\d+)(?::(\d+))?)?([{PFORMS}]+)?$)"
+)
+MOVE_MATCH = re.compile(rf"((^[+-]\d+)([{DTYPES}])?$)")
+
+# bytes per item of each data type
+NBYTES = {"S": 1, "A": 1, "b": 1, "c": 1, "s": 2, "w": 2}
+NBYTES |= {"u": 4, "i": 4, "f": 4, "d": 8, "z": 8, "Z": 16}
+
+# default print formats
+DEFAULT_FORMAT = {"b": "x", "c": "c", "s": "d", "w": "u", "S": "S", "A": "A"}
+DEFAULT_FORMAT |= {"u": "u", "i": "d", "f": "g", "d": "g", "z": "RI", "Z": "RI"}
+
+# struct codes used to unpack each type (for complex types: one component)
+STRUCT = {"b": "B", "s": "h", "w": "H", "u": "I", "i": "i", "f": "f", "d": "d"}
+STRUCT |= {"z": "f", "Z": "d"}
+
+
+class FieldError(ValueError):
+    """A field request pick can't use."""
+
+
+@dataclass
+class Field:
+    """One print request, with its position resolved."""
+
+    src: str  # the request as written
+    pos: int  # byte position of the first item in the record (may be negative)
+    t: str  # data type letter
+    m: int  # number of items
+    formats: str  # print formats
+    nb: int  # bytes per item
+    first_bit: int  # bit offset (0 when none given)
+    nbits: int  # number of bits used
+    mask: int
+    bits: bool  # was a bit offset given?
+
+    @property
+    def out_bytes(self) -> int:
+        """Bytes of output per item for bit-limited formats (Perl's $nb)."""
+        return (7 + self.nbits) // 8
+
+
+def parse_size(token: str) -> int | None:
+    """A record size like ``256``, ``21d`` or ``4z``, in bytes; None if it isn't one."""
+    m = SIZE_MATCH.search(token)
+    if not m:
+        return None
+    return int(m.group(2)) * NBYTES[m.group(3) or "b"]
+
+
+class Parser:
+    """Parse field requests, tracking the current position as the Perl did."""
+
+    def __init__(self, record_size: int, *, binary: bool = False):
+        self.size = record_size
+        self.binary = binary  # -u: print everything as 'U'
+        self.pos = 0
+        self.fields: list[Field] = []
+
+    def field(self, f: str) -> None:
+        m = MOVE_MATCH.search(f)
+        if m:
+            t = m.group(3) or "b"
+            self.pos += int(m.group(2)) * NBYTES[t]
+            return
+        m = DATA_MATCH.search(f)
+        if not m:
+            raise FieldError(f"{f}:  Matched neither a move nor a data request")
+        _, mult, t, offset, n, byte_off, bitspec, first_bit, nbits, formats = m.groups()
+        mult = int(mult) if truthy(mult) else 1  # Perl: $m ||= 1, so "0i" is "i" ("00i" is not)
+        formats = formats or DEFAULT_FORMAT[t]
+        nb = NBYTES[t]
+        if offset:
+            o = int(byte_off) if byte_off else 0
+            cpos = nb * int(n) + o
+            self.pos = nb * (int(n) + mult) + o
+        else:
+            cpos = self.pos
+            self.pos += nb * mult
+        if self.pos > self.size:
+            raise FieldError("data requested beyond record length")
+        nbits = int(nbits) if nbits is not None else 8 * nb
+        # Perl built the mask one bit at a time in a 64-bit integer; 0 bits gave 1.
+        mask = (1 << min(max(nbits, 1), 64)) - 1
+        if self.binary:
+            formats = "U"
+        self.fields.append(
+            Field(
+                src=f,
+                pos=cpos,
+                t=t,
+                m=mult,
+                formats=formats,
+                nb=nb,
+                first_bit=int(first_bit) if first_bit else 0,
+                nbits=nbits,
+                mask=mask,
+                bits=bitspec is not None,
+            )
+        )
+
+    def token(self, t: str, groups: list[str]) -> None:
+        t = t.replace("\\", "")
+        m = re.search(r"(\d*)(Y)(\d+)", t)
+        if not m:
+            self.field(t)
+            return
+        repeat = int(m.group(1)) if truthy(m.group(1)) else 1
+        k = int(m.group(3))
+        body = groups[k] if k < len(groups) else ""
+        for _ in range(repeat):
+            for sub in body.split():
+                self.token(sub, groups)
+
+
+def extract_groups(text: str) -> tuple[str, list[str]]:
+    """Replace each innermost ``[ ... ]`` with ``Y<n>`` until none are left.
+
+    Returns the rewritten text and the group bodies. Like the Perl, the body is
+    used as a regular expression (with + and - escaped) to find the group again.
+    """
+    groups: list[str] = []
+    while m := re.search(r"\[([^\[\]]*)\]", text):
+        body = re.sub(r"([+-])", r"\\\1", m.group(1))
+        try:
+            pattern = re.compile(r"\[" + body + r"\]")
+        except re.error as e:
+            raise FieldError(f"can't parse group [{m.group(1)}]: {e}") from None
+        new = pattern.sub(f"Y{len(groups)} ", text, count=1)
+        if new == text:  # the Perl looped forever here
+            raise FieldError(f"can't parse group [{m.group(1)}]")
+        text = new
+        groups.append(body)
+    return text, groups
+
+
+def parse_fields(text: str, record_size: int, *, binary: bool = False) -> list[Field]:
+    """Parse the field-request part of a pick command line."""
+    text, groups = extract_groups(text)
+    p = Parser(record_size, binary=binary)
+    for tok in text.split():
+        p.token(tok, groups)
+    return p.fields