telemetry-cli 1.0.0__py3-none-any.whl

telemetry_cli/tgen/cli.py

@@ -0,0 +1,128 @@
+"""The tgen command: generate telemetry records from a command file.
+
+    tgen [ options ] command_file [ nframes [ firstframe ] ] > telemetry.dat
+    tgen --convert old_perl_command_file > new_command_file
+
+Run ``tgen -help`` for help.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import BinaryIO, TextIO
+
+from .. import __version__
+from .._cli import broken_pipe
+from .._getopt import OptionError, getoptions
+from .._perl import numify, truthy
+from . import cmdfile
+from .convert import convert
+from .runtime import Generator, TgenError
+
+SPECS = ["h|help|man", "r|random|random_fill", "f|fill=i", "n|notify=i", "q|quiet",
+         "v|version", "seed=i", "convert"]  # fmt: skip
+
+USAGE = """\
+Usage:
+    tgen [ options ] command_file [ nframes [ firstframe ] ] > telemetry.dat
+    tgen --convert perl_command_file > python_command_file
+    tgen -help
+"""
+
+HELP = (
+    USAGE
+    + """
+tgen reads a command file and writes records (frames) laid out as it says:
+nframes of them (default 1), numbering them from firstframe (default 0).
+
+Options:
+    -r, -random       fill unspecified bytes with random values
+    -f, -fill=n       fill unspecified bytes with the byte n (default 0)
+    -n, -notify=n     report progress on stderr every n frames (default 100)
+    -q, -quiet        no progress reports
+    -seed=n           seed the random numbers (rand and -r), for repeatable output
+    -convert          translate a Perl tgen command file to the Python form
+    -v, -version      show the version
+    -h, -help         show this help
+
+A command file holds item specifications, one per line:
+
+    name # offset # trigger # type # value
+
+Each field is Python code, evaluated for every frame. The value is packed
+with the type (a Perl pack template such as 'C', 'L', 'a10' or f"C{n}") and
+written at the byte offset, if the trigger is true. A negative offset means
+"evaluate the value but don't write it". Other lines are Python statements,
+run once at the start; lines starting with #, ! or ; are comments.
+
+In the code, I is the frame number, F the fill byte, and P.name.value an
+item's value. The full manual, with examples, is at
+    https://github.com/donalgrant/telemetry-cli/blob/main/docs/tgen.md
+"""
+)
+
+
+def main(
+    argv: list[str] | None = None,
+    *,
+    stdin: BinaryIO | None = None,
+    stdout: BinaryIO | None = None,
+    stderr: TextIO | None = None,
+) -> int:
+    argv = sys.argv[1:] if argv is None else list(argv)
+    stdout = stdout if stdout is not None else sys.stdout.buffer
+    stderr = stderr if stderr is not None else sys.stderr
+
+    def error(message: str, status: int = 1) -> int:
+        print(f"tgen: {message}", file=stderr)
+        return status
+
+    try:
+        o, args = getoptions(argv, SPECS)
+    except OptionError as e:
+        return error(f"{e} (see 'tgen -help')", 2)
+    if o.get("v"):
+        print(f"tgen (Python), from telemetry-cli {__version__}", file=stderr)
+        return 0
+    if o.get("h"):
+        stdout.write(HELP.encode())
+        return 0
+    if not args:
+        stderr.write(USAGE)
+        return 2
+    try:
+        with open(args[0], encoding="latin-1") as f:
+            text = f.read()
+    except OSError as e:
+        return error(f"Can't read {args[0]}:  {e.strerror}")
+    if o.get("convert"):
+        stdout.write(convert(text).encode("latin-1"))
+        return 0
+
+    def count(i, default):
+        if len(args) <= i:
+            return default
+        v = numify(args[i])
+        return int(v) if truthy(args[i]) and v else default  # Perl: shift || default
+
+    nframes, first = count(1, 1), count(2, 0)
+    fill = "random" if o.get("r") else o.get("f", 0)
+    try:
+        items, statements = cmdfile.parse(text)
+        gen = Generator(
+            items, statements, fill=fill, seed=o.get("seed"), stderr=stderr, source=args[0]
+        )
+        gen.run(
+            stdout,
+            nframes,
+            first,
+            notify=o.get("n") or 100,
+            quiet=bool(o.get("q")),
+            progress=lambda m: print(m, file=stderr),
+        )
+    except TgenError as e:
+        return error(str(e))
+    except BrokenPipeError:
+        return broken_pipe(stdout)
+    stdout.flush()
+    return 0

telemetry_cli/tgen/cmdfile.py

@@ -0,0 +1,68 @@
+"""Reading tgen command files.
+
+A command file has three kinds of lines:
+
+- comments: lines starting with ``#``, ``!`` or ``;`` in the first column;
+- item specifications, with at least four `` # `` separators (a ``#`` with
+  white space on both sides)::
+
+      name # offset # trigger # type # value
+
+- statements, run once before any record is generated.
+
+A line ending in a backslash continues on the next line.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+SEPARATOR = re.compile(r"\s+[#]\s+")
+
+
+@dataclass
+class Item:
+    name: str
+    offset: str
+    trigger: str
+    type: str
+    value: str
+    line: int  # where the specification starts in the file
+
+
+@dataclass
+class Statement:
+    code: str
+    line: int
+
+
+def logical_lines(text: str):
+    """(line number, text) of each logical line, with continuations joined."""
+    lines = text.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    i = 0
+    while i < len(lines):
+        start = i + 1
+        line = lines[i]
+        i += 1
+        while line.endswith("\\"):
+            line = line[:-1]
+            if i < len(lines):
+                line += lines[i]
+                i += 1
+        yield start, line
+
+
+def parse(text: str) -> tuple[list[Item], list[Statement]]:
+    items, statements = [], []
+    for n, line in logical_lines(text):
+        if line[:1] in ("#", "!", ";"):
+            continue
+        if len(SEPARATOR.findall(line)) >= 4:
+            fields = [f.strip() for f in SEPARATOR.split(line)]
+            items.append(Item(*fields[:5], line=n))  # further fields are ignored
+        elif line.strip():
+            statements.append(Statement(line.strip(), n))
+    return items, statements

telemetry_cli/tgen/convert.py

@@ -0,0 +1,278 @@
+"""Translate a Perl tgen command file into the Python form (``tgen --convert``).
+
+The translation is pattern-based. It handles what tgen command files usually
+contain: ``$::name`` variables, ``$P{item}{field}``, arithmetic, string
+concatenation, interpolated strings, ``map { ... } (a..b)``, ``++``/``--``,
+and statement modifiers (``x if y``). Anything else that looks like Perl is
+left as it is, with a ``# TODO(convert)`` comment line above it, so that
+running the result fails loudly rather than doing something different.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+
+from .cmdfile import SEPARATOR, logical_lines
+
+_PERLISH = re.compile(
+    r"\$|@|->|\bsub\b|=~|\bmy\b|\bprint\b(?!\()|\?|\blast\b|\bnext\b|\beq\b|\bne\b"
+)
+
+
+def convert(text: str) -> str:
+    out = []
+    for _, line in logical_lines(text):
+        if line[:1] in ("#", "!", ";") or not line.strip():
+            out.append(line)
+            continue
+        if len(SEPARATOR.findall(line)) >= 4:
+            parts = SEPARATOR.split(line)
+            seps = SEPARATOR.findall(line)
+            fields = [parts[0].strip()] + [expr(p.strip()) for p in parts[1:5]]
+            todo = [f for f in fields[1:] if _PERLISH.search(_strip_strings(f))]
+            rest = "".join(s + p for s, p in zip(seps[4:], parts[5:], strict=False))
+            new = fields[0] + "".join(s + f for s, f in zip(seps, fields[1:], strict=False)) + rest
+        else:
+            new = statement(line.strip())
+            todo = [new] if _PERLISH.search(_strip_strings(new)) else []
+        if todo:
+            out.append("# TODO(convert): this line still contains Perl")
+        out.append(new)
+    return "\n".join(out) + "\n"
+
+
+# --- pieces --------------------------------------------------------------------
+
+_STRING = re.compile(r"""("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')""")
+
+
+def _strip_strings(s: str) -> str:
+    return _STRING.sub('""', s)
+
+
+def _split_code(s: str):
+    """Alternating (code, string literal) pieces of s."""
+    pos = 0
+    for m in _STRING.finditer(s):
+        yield s[pos : m.start()], False
+        yield m.group(0), True
+        pos = m.end()
+    yield s[pos:], False
+
+
+def _variables(code: str) -> str:
+    code = re.sub(r"\$P\{\s*(\w+)\s*\}\s*\{\s*(\w+)\s*\}", r"P.\1.\2", code)
+    code = re.sub(r"[$@]::(\w+)", r"\1", code)
+    code = re.sub(r"\bmy\s+[$@](\w+)", r"\1", code)
+    code = re.sub(r"\$_\b", "_", code)
+    code = re.sub(r"[$@](\w+)", r"\1", code)
+    return code
+
+
+def _string(lit: str) -> str:
+    """A Perl string literal as Python: double-quoted strings interpolate."""
+    if lit.startswith("'"):
+        if "$" in lit:  # code in a string, for seek_file's offset and condition
+            return "'" + _code(lit[1:-1]) + "'"
+        return lit
+    body = lit[1:-1]
+    if not re.search(r"[$@]", body.replace("\\$", "").replace("\\@", "")):
+        return lit
+    body = body.replace("{", "{{").replace("}", "}}")
+    body = re.sub(r"\$(?:::)?(\w+)", r"{\1}", body)
+    return 'f"' + body + '"'
+
+
+def _ranges(code: str) -> str:
+    return re.sub(r"\(\s*([^()]+?|\([^()]*\))\s*\.\.\s*(\([^()]*\)|[^()]+?)\s*\)",
+                  r"range(\1, \2 + 1)", code)  # fmt: skip
+
+
+def _maps(code: str) -> str:
+    # [ map { EXPR } (LIST) ]  and  map { EXPR } (LIST)
+    code = re.sub(r"\[\s*map\s*\{(.*)\}\s*(range\(.*\)|\(.*\))\s*\]", r"[\1 for _ in \2]", code)
+    code = re.sub(r"\bmap\s*\{(.*)\}\s*(range\(.*\)|\(.*\))", r"[\1 for _ in \2]", code)
+    return code
+
+
+def _operators(code: str) -> str:
+    code = re.sub(r"&&", " and ", code)
+    code = re.sub(r"\|\|", " or ", code)
+    code = re.sub(r"!(?!=)", " not ", code)
+    code = re.sub(r"\beq\b", "==", code)
+    code = re.sub(r"\bne\b", "!=", code)
+    return code
+
+
+# Perl operators whose operands are integers, and the helpers that do the same
+_INTEGER_OPS = {ast.Mod: "pmod", ast.BitAnd: "pand", ast.BitOr: "por", ast.BitXor: "pxor",
+                ast.LShift: "pshl", ast.RShift: "pshr"}  # fmt: skip
+
+
+def _integer_ops(code: str) -> str:
+    """Rewrite %, &, |, ^, << and >> as calls to helpers with Perl's integer semantics.
+
+    Only those expressions change; the rest of the text is kept as written.
+    """
+    if not re.search(r"%|&|\||\^|<<|>>", _strip_strings(code)) or "\n" in code:
+        return code
+    try:
+        tree = ast.parse(code)
+    except SyntaxError:
+        return code  # still Perl; it will be marked for review
+    src = code.encode()
+
+    def needs(node) -> bool:
+        return any(isinstance(n, ast.BinOp) and type(n.op) in _INTEGER_OPS for n in ast.walk(node))
+
+    def span(node) -> tuple[int, int]:
+        return node.col_offset, node.end_col_offset
+
+    def render(node) -> bytes:
+        if isinstance(node, ast.BinOp) and type(node.op) in _INTEGER_OPS:
+            fn = _INTEGER_OPS[type(node.op)].encode()
+            return fn + b"(" + render(node.left) + b", " + render(node.right) + b")"
+        start, end = span(node)
+        text = src[start:end]
+        for child in sorted(
+            (c for c in ast.iter_child_nodes(node) if hasattr(c, "col_offset") and needs(c)),
+            key=lambda c: c.col_offset,
+            reverse=True,
+        ):
+            cs, ce = span(child)
+            text = text[: cs - start] + render(child) + text[ce - start :]
+        return text
+
+    out = src
+    for stmt in sorted(tree.body, key=lambda n: n.col_offset, reverse=True):
+        if needs(stmt):
+            s0, e0 = span(stmt)
+            out = out[:s0] + render(stmt) + out[e0:]
+    return out.decode()
+
+
+def _code(code: str) -> str:
+    return _operators(_maps(_ranges(_variables(code))))
+
+
+def _finish(code: str) -> str:
+    return _integer_ops(code)
+
+
+def _concat_pieces(s: str) -> list[str]:
+    """s split at Perl's string-concatenation operator (.), outside strings and brackets.
+
+    A dot next to a digit is part of a number, and ".." is a range.
+    """
+    pieces, depth, start, i, quote = [], 0, 0, 0, None
+    while i < len(s):
+        ch = s[i]
+        if quote:
+            if ch == "\\":
+                i += 1
+            elif ch == quote:
+                quote = None
+        elif ch in "'\"":
+            quote = ch
+        elif ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+        elif ch == "." and depth == 0:
+            before, after = s[i - 1 : i], s[i + 1 : i + 2]
+            if "." not in (before, after) and not (before.isdigit() or after.isdigit()):
+                pieces.append(s[start:i])
+                start = i + 1
+        i += 1
+    pieces.append(s[start:])
+    return pieces
+
+
+def _translate(s: str) -> str:
+    pieces = _concat_pieces(s)
+    if len(pieces) > 1:  # Perl's . joins strings, and numbers as text
+        return "cat(" + ", ".join(_translate(p.strip()) for p in pieces) + ")"
+    out = []
+    for piece, is_string in _split_code(s):
+        out.append(_string(piece) if is_string else _code(piece))
+    return "".join(out)
+
+
+def _increments(stmt: str) -> str:
+    """x++ / ++x / x-- / --x as assignments."""
+    m = re.fullmatch(r"\s*(?:(\+\+|--)\s*(\S+?)|(\S+?)\s*(\+\+|--))\s*", stmt)
+    if not m:
+        return stmt
+    op = m.group(1) or m.group(4)
+    target = m.group(2) or m.group(3)
+    return f"{target} = {target} {'+' if op == '++' else '-'} 1"
+
+
+def _statements(s: str) -> list[str]:
+    parts, depth, cur, quote = [], 0, "", None
+    for ch in s:
+        if quote:
+            cur += ch
+            if ch == quote:
+                quote = None
+            continue
+        if ch in "'\"":
+            quote = ch
+        elif ch in "([{":
+            depth += 1
+        elif ch in ")]}":
+            depth -= 1
+        elif ch == ";" and depth == 0:
+            parts.append(cur)
+            cur = ""
+            continue
+        cur += ch
+    parts.append(cur)
+    return [p.strip() for p in parts if p.strip()]
+
+
+def _modifier(stmt: str, *, as_expression: bool) -> str:
+    """Perl's `STATEMENT if CONDITION`."""
+    m = re.fullmatch(r"(.+?)\s+if\s+(.+)", stmt)
+    if not m or " else " in stmt:
+        return _increments(stmt)
+    body, cond = _increments(m.group(1)), m.group(2)
+    if not as_expression:
+        return f"if {cond}: {body}"
+    a = re.fullmatch(r"(\w+(?:\.\w+)*)\s*=\s*(.+)", body)
+    if a:
+        return f"{a.group(1)} = {a.group(2)} if {cond} else {a.group(1)}"
+    return f"({body}) if ({cond}) else None"
+
+
+def _print(stmt: str) -> str:
+    m = re.fullmatch(r"print\s+STDERR\s+(.+?)((?:\s+if\s+.+)?)", stmt)
+    if not m:
+        return stmt
+    return f'print({m.group(1)}, file=stderr, end=""){m.group(2)}'
+
+
+def expr(field: str) -> str:
+    """A Perl item field (an expression, or statements whose last value counts)."""
+    stmts = [_translate(_print(s)) for s in _statements(field)]
+    return _finish("; ".join(_modifier(s, as_expression=True) for s in stmts))
+
+
+def statement(line: str) -> str:
+    """A Perl statement line (run once)."""
+    code, comment = _split_comment(line)
+    stmts = [_translate(_print(s)) for s in _statements(code)]
+    new = _finish("; ".join(_modifier(s, as_expression=False) for s in stmts))
+    return new + (f"  {comment}" if comment else "")
+
+
+def _split_comment(line: str) -> tuple[str, str]:
+    """Code and a trailing # comment (outside strings)."""
+    pos = 0
+    for piece, is_string in _split_code(line):
+        if not is_string and "#" in piece:
+            i = pos + piece.index("#")
+            return line[:i].rstrip(), line[i:]
+        pos += len(piece)
+    return line, ""

telemetry_cli/tgen/examples/arb_data.tgen

@@ -0,0 +1,15 @@
+#
+# Arbitrarily-filled data file  --  generates 1024-byte records
+#
+
+# this following item specification gives only the value of the
+# final byte (byte 1023, with is the 1024th zero-offset byte);
+# the bytes at all earlier byte offsets are padded automatically.
+# $::F is a special variable in tgen which contains the value
+# of the fill byte.  The value is zero by default, but can
+# be specified on the command line as any other byte value using
+# the -f flag, or can be made random using the -r flag.
+
+data # 1023 # 1 # 'C' # F
+
+# end of command file

telemetry_cli/tgen/examples/calsweep.tgen

@@ -0,0 +1,32 @@
+#
+# Generate a binary data file
+#
+
+# some parameters to define a digital system
+
+pi=3.141592654  # the (approx) value of pi
+nf=10  # number of caltones to generate
+f0=5.0e6  # minimum caltone frequency
+f_max=35.0e6  # maximum caltone frequency
+df=(f_max-f0)/(nf-1)  # frequency step
+
+f = [ f0+_*df  for _ in range(1, nf + 1)]  # array of frequencies
+
+fs=90.0e6  # sampling rate
+a=2.0*pi/fs  # convenience coefficient
+
+ns=256  # number of samples to generate
+
+# below we have two item specifications:
+# 'calfreq' calculates which caltone frequency
+# we should use for the current pulse, and
+# data actually generates the sample values
+# to be written for the current pulse, using
+# the caltone frequency calculated previously.
+
+calfreq # -1 # 1 # X # pmod(I/32, nf)   # 32 pulses per caltone
+
+data # 0 # 1 # f"C{ns}" # [       128+int(128.0*cos(a*_*f[P.calfreq.value]))                                for _ in range(1, (ns-1) + 1)]
+
+# end of command file
+

telemetry_cli/tgen/examples/cond_fill.tgen

@@ -0,0 +1,17 @@
+#
+# Change fill behavior on every third pulse
+#
+
+zero_fill   # -1 # (pmod(I, 3))==0  # X # F=0
+rail_fill   # -1 # (pmod(I, 3))==1  # X # F=255
+rand_fill   # -1 # (pmod(I, 3))==2  # X # F="random"
+
+pulse_number  # 0  # 1  # 'C' # I
+last_byte     # 9  # 1  # 'C' # F
+    
+# Note that the -1 offset for the first three item specs 
+# means that the fields are only evaluated when the trigger field is true, 
+# but not written.  The pack-type is ignored in this case.
+
+# end of command file
+

telemetry_cli/tgen/examples/counter.tgen

@@ -0,0 +1,11 @@
+#
+# Generate different counters
+#
+
+every_pulse_byte       # 0 # 1 # 'C' # pmod(I, 256)
+every_other_pulse_byte # 1 # 1 # 'C' # pmod(I/2, 256)
+count_to_4             # 2 # 1 # 'C' # (pmod(I, 4)) + 1
+every_pulse_long       # 3 # 1 # 'L' # I
+
+# end of command file
+

telemetry_cli/tgen/examples/null_data.tgen

@@ -0,0 +1,11 @@
+#
+# Null data file  --  generates 1024-byte null-valued data items.
+#
+
+# this following item specification gives only the value of the
+# final byte (byte 1023, with is the 1024th zero-offset byte).
+# the bytes at all earlier byte offsets are null-padded automatically.
+
+data # 1023 # 1 # 'C' # 0
+
+# end of command file

telemetry_cli/tgen/examples/random_size.tgen

@@ -0,0 +1,20 @@
+#
+# random record lengths
+#
+
+# we make use of the fact that the byte-offset is actually
+# a Perl-evaluated statement for each pulse, so that it
+# can vary from one pulse to the next.  We'll write random
+# or fixed value data (using the -r or -f flag on the command line) 
+# but have a fixed "PN-Sequence" for the start of the records.
+
+pn   # 0                 # 1 # 'L' # 0x03915ed3
+
+# the following item gives a random byte offset between 4 
+# (so that we don't try to over-write the pn-sequence)
+# and 255 (maximum 256-byte record length).  The value
+# is the same as the fill byte value.
+
+data # int(rand()*252)+4 # 1 # 'C' # F
+
+# end of command file

telemetry_cli/tgen/examples/subcom.tgen

@@ -0,0 +1,14 @@
+#
+# Generate a "subcommutated header" -- use this command file
+#
+
+SUB_LENGTH=500  # size (bytes) of subcom data record
+SUB_BYTES=10  # number of subcom bytes per record
+FILE='subcom.tgen'  # use this file for example subcom data (run from its directory)
+
+counter   # 0 # 1 # 'L'   # I
+sub_index # 4 # 1 # 'S'   # pmod(I * SUB_BYTES, SUB_LENGTH)
+sub_data  # 6 # 1 # 'a10' # subcom_file(FILE,P.sub_index.value,10)
+
+# end of command file
+

telemetry_cli/tgen/examples/valueV_value.tgen

@@ -0,0 +1,20 @@
+#
+# Contrast the value and _value $P{}{} fields
+#
+    
+random_number # 0 # 1 # 'f' # rand(1.0)
+
+COUNT1=0  # this does a count on the rand(1.0) function
+COUNT2=0  # this one will be what we were probably wanting
+
+value_1       # 4 # 1 # 'S' # COUNT1 = COUNT1 + 1 if P.random_number._value>0.5 else COUNT1; COUNT1
+	      
+value_2       # 6 # 1 # 'S' # COUNT2 = COUNT2 + 1 if P.random_number.value>0.5 else COUNT2; COUNT2
+
+# The difference between the two is that _value does an 'eval' on the
+# content of the value field (in this case, the rand(1.0) function),
+# while value gives the result of the eval when the item was generated 
+# for this record.
+
+# end of command file
+