telemetry-cli 1.0.0__py3-none-any.whl

telemetry_cli/pick/symbols/moc

@@ -0,0 +1,42 @@
+; This is the symbol table for .moc files, which
+; are the radar motion compensation files for section 334.
+
+length=21d  ; length is 21 double-precision (8 byte) numbers
+
+; these are the components of the file
+
+xRec = d0G
+xrec = d0G
+time = d1G
+utc = d1G
+prf = d2 ; actually 1/prf...
+tprf = d2
+posS = d3
+posC = d4
+posH = d5
+pos = posS posC posH
+velS = d6
+velC = d7
+velH = d8
+vel = velS velC velH
+af1S = d9
+af1C = d10
+af1H = d11
+af1 = 3d9
+af2 = 3d12
+af3 = 3d15
+az = d18
+yaw = d18
+pitch = d19
+roll = d20
+orient = 3d18
+azD = d18D
+yawD = d18D
+pitchd = d19D
+rollD = d20D
+orientD = 3d18D
+all = 21d0
+af = af1 af2 af3
+angles = posS orientD
+velocities = posS vel
+tube = pos

telemetry_cli/pick/symbols/subcom

@@ -0,0 +1,83 @@
+;
+; pick symbol table for AIRSAR subcommutated data files
+; 
+length=4096
+;
+sub_len = w0+0u
+used_len = w0+2u
+;
+; data collection notes
+; 
+target  =  25A0+439
+invstg  =  25A0+464
+hp_opr  =  25A0+489
+crew    =  25A0+514
+mode    =  25A0+539
+status  =  25A0+564
+antenna =  25A0+589
+telev   =   8A0+638 ; target elevation in feet
+dser1   =  10A0+683 ; digital tape serial number HDDR #1
+dser2   =  10A0+683 ; digital tape serial number HDDR #2
+dser3   =  10A0+683 ; digital tape serial number HDDR #3
+notes   = 256A0+1131 ; operator's notes
+;
+; data in six-gun position
+; 
+six_month = b0+1412u ; month (1..12)
+six_day   = b0+1413u ; day   (1..31)
+six_year  = w0+1414u ; year  (1900..2999)
+six_hour  = b0+1416u ; hour  (0..23)
+six_min   = b0+1417u ; min   (0..59)
+six_sec   = b0+1418u ; sec   (0..60)
+six_nsec  = u0+1419u ; fractional seconds (0..999,999,999) [nano-seconds]
+;
+six_f_cnt = u0+1476u ; 32-bit frame count at GPS time tag
+;
+; True-Time data
+; 
+TTmonth = b0+2188u ; month (1..12)
+TTday   = b0+2189u ; day   (1..31)
+TTyear  = w0+2190u ; year  (1900..2999)
+TThour  = b0+2192u ; hour  (0..23)
+TTmin   = b0+2193u ; min   (0..59)
+TTsec   = b0+2194u ; sec   (0..60)
+TTnsec  = u0+2195u ; fractional seconds (0..999,999,999) [nano-seconds]
+;
+TTf_cnt = u0+2212u ; 32-bit frame count at True-Time time tag
+;
+; EO-17 message
+; 
+egi17TimeTag = w0+2496u ; 1553 time tag
+;
+; EO-18 message
+; 
+egi18GpsTimeHi = w0+2560u ; gps time tag word
+egi18GpsTime2  = w0+2562u ; gps time tag word
+egi18GpsTime3  = w0+2564u ; gps time tag word
+egi18GpsTimeLo = w0+2566u ; gps time tag word
+
+egi18GpsTimeHigh = u0+2560u ; gps time tag word
+egi18GpsTimeLow  = u0+2564u ; gps time tag word
+
+egi18UtcTimeHi = w0+2568u ; utc time tag word
+egi18UtcTime2  = w0+2570u ; utc time tag word
+egi18UtcTime3  = w0+2572u ; utc time tag word
+egi18UtcTimeLo = w0+2574u ; utc time tag word
+
+egi18UtcTimeHigh = u0+2568u ; utc time tag word
+egi18UtcTimeLow  = u0+2572u ; utc time tag word
+;
+; EO-19 message
+; 
+egi19TimeTag          = w0+2622u      ; Time tag 
+egi19PltfrmAzTimeTag  = w0+2654u      ; Platform azimuth time tag 
+egi19RollTimeTag      = w0+2656u      ; Roll time tag 
+egi19PitchTimeTag     = w0+2658u      ; Pitch time tag 
+;
+; end of record stuff
+; 
+motionSensorFlag = b0+3004d           ; Bit 0 set = IGI, bit 1 set = EGI 
+hdrVersion       = 6A0+3005           ; Date of this revision, eg. 980817 
+eosd             = 4A0+3011           ; Indicates end of used subheader 
+
+

telemetry_cli/pick/symbols/subcom_2002

@@ -0,0 +1,111 @@
+;
+; pick symbol table for AIRSAR subcommutated data files
+; 
+length=4096
+;
+sub_len = w0+0u
+used_len = w0+2u
+
+_ = -1 bs 
+
+;
+; data collection notes
+; 
+target  =  25A0+439
+invstg  =  25A0+464
+hp_opr  =  25A0+489
+crew    =  25A0+514
+mode    =  25A0+539
+status  =  25A0+564
+antenna =  25A0+589
+telev   =   8A0+638 ; target elevation in feet
+dser1   =  10A0+683 ; digital tape serial number HDDR #1
+dser2   =  10A0+683 ; digital tape serial number HDDR #2
+dser3   =  10A0+683 ; digital tape serial number HDDR #3
+notes   = 256A0+1131 ; operator's notes
+;
+; data in six-gun position
+; 
+six_month = b0+1412u ; month (1..12)
+six_day   = b0+1413u ; day   (1..31)
+six_year  = w0+1414u ; year  (1900..2999)
+six_hour  = b0+1416u ; hour  (0..23)
+six_min   = b0+1417u ; min   (0..59)
+six_sec   = b0+1418u ; sec   (0..60)
+six_nsec  = u0+1419u ; fractional seconds (0..999,999,999) [nano-seconds]
+;
+six_f_cnt = u0+1476u ; 32-bit frame count at GPS time tag
+;
+; True-Time data
+; 
+TTmonth = b0+2188u ; month (1..12)
+TTday   = b0+2189u ; day   (1..31)
+TTyear  = w0+2190u ; year  (1900..2999)
+TThour  = b0+2192u ; hour  (0..23)
+TTmin   = b0+2193u ; min   (0..59)
+TTsec   = b0+2194u ; sec   (0..60)
+TTnsec  = u0+2195u ; fractional seconds (0..999,999,999) [nano-seconds]
+;
+TTf_cnt = u0+2212u ; 32-bit frame count at True-Time time tag
+;
+; EO-17 message
+; 
+egi17TimeTag = w0+2496u ; 1553 time tag
+;
+; EO-18 message
+; 
+egi18GpsTimeHi = w0+2560u ; gps time tag word
+egi18GpsTime2  = w0+2562u ; gps time tag word
+egi18GpsTime3  = w0+2564u ; gps time tag word
+egi18GpsTimeLo = w0+2566u ; gps time tag word
+
+egi18GpsTimeHigh = u0+2560u ; gps time tag word
+egi18GpsTimeLow  = u0+2564u ; gps time tag word
+
+egi18UtcTimeHi = w0+2568u ; utc time tag word
+egi18UtcTime2  = w0+2570u ; utc time tag word
+egi18UtcTime3  = w0+2572u ; utc time tag word
+egi18UtcTimeLo = w0+2574u ; utc time tag word
+
+egi18UtcTimeHigh = u0+2568u ; utc time tag word
+egi18UtcTimeLow  = u0+2572u ; utc time tag word
+;
+; EO-19 message
+; 
+egi19TimeTag          = w0+2622u      ; Time tag 
+egi19PltfrmAzTimeTag  = w0+2654u      ; Platform azimuth time tag 
+egi19RollTimeTag      = w0+2656u      ; Roll time tag 
+egi19PitchTimeTag     = w0+2658u      ; Pitch time tag 
+;
+; end of record stuff
+; 
+motionSensorFlag = b0+3004d           ; Bit 0 set = IGI, bit 1 set = EGI 
+hdrVersion       = 6A0+3005           ; Date of this revision, eg. 980817 
+
+dcg_bw_l = 4A0+3011
+dcg_bw_c = 4A0+3015
+dcg_bw_p = 4A0+3019
+
+dcg_pd_l = 3A0+3023
+dcg_pd_c = 3A0+3026
+dcg_pd_p = 3A0+3029
+
+sampfreq_l = 4A0+3032
+sampfreq_c = 4A0+3036
+sampfreq_p = 4A0+3040
+
+dgps_time_J2000 = d0+3428G
+
+dgps_x = d0+3444G
+dgps_y = d0+3452G
+dgps_z = d0+3460G
+
+dgps_sigma_x = d0+3468
+dgps_sigma_y = d0+3476
+dgps_sigma_z = d0+3484
+
+cal_mode = b0+3492d
+
+eosd             = 4A0+3493           ; Indicates end of used subheader 
+
+

telemetry_cli/recl/__init__.py

@@ -0,0 +1 @@
+"""recl: estimate the record length of a binary file."""

telemetry_cli/recl/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m telemetry_cli.recl``."""
+
+import sys
+
+from .cli import main
+
+sys.exit(main())

telemetry_cli/recl/cli.py

@@ -0,0 +1,214 @@
+"""The recl command: estimate the record length of a binary file.
+
+    recl filename [ options ]
+    cat filename | recl - [ options ]
+
+Run ``recl -help`` for help.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import BinaryIO, TextIO
+
+import numpy as np
+
+from .._cli import broken_pipe
+from .._getopt import OptionError, getoptions
+from .._msg import Messenger
+from .._perl import perl_str
+from .corr import agreement, choose, compared
+
+SPECS = ["v|verbose", "h|help", "f|full", "q|quiet", "part|partial", "skip|header=i", "min=i",
+         "max=i", "minrecs=i", "maxbufs=i", "fact|factor|mult=i", "only=s", "limit=i",
+         "reduce=f"]  # fmt: skip
+
+HELP = """\
+recl - estimate the record length of a binary file
+
+SYNOPSIS
+    recl filename [ options ]
+    cat filename | recl - [ options ]
+
+DESCRIPTION
+    recl compares the file with itself shifted by each possible record
+    length R, and reports the R at which the most bytes are equal: records
+    of the same format tend to have the same bytes in the same places. It
+    prints
+
+        RESULT R      the estimated record length
+        CORR x        the percentage of bytes equal to the byte R bytes on
+                      (about 0.4 for unrelated random data)
+
+    and notes about the lengths it checked. Every multiple of the record
+    length agrees about as well as the length itself; recl reports the
+    smallest.
+
+    Unless -partial is given, recl assumes the file (after any header) holds
+    whole records, so it only checks lengths that divide its size.
+
+OPTIONS
+    Options can be abbreviated.
+
+    -skip=bytes (or -header)
+        Skip a header of this many bytes.
+    -min=bytes, -max=bytes
+        The shortest and longest record lengths to check. By default, 1 and
+        half the data (see -minrecs).
+    -minrecs=N
+        The file holds at least N records (default 2): the same as
+        -max=(size-skip)/N. -max takes precedence.
+    -fact=bytes (also -factor, -mult)
+        The record length is a multiple of this, e.g. 8 for doubles.
+    -only=list
+        Check only these lengths (comma-separated).
+    -partial
+        Don't assume whole records: check every length in range.
+    -maxbufs=N
+        Use only the first N buffers (of twice the longest length checked).
+    -limit=bytes
+        Use only the first this many bytes of data.
+    -reduce=factor
+        Use only the first 1/factor of the data (factor >= 1).
+    -full
+        Compare the data (or the part -limit or -reduce keep) as one piece,
+        rather than buffer by buffer.
+    -verbose
+        Show the score of every length in every buffer, and a sorted table.
+    -quiet
+        Show only the result.
+    -help
+        Show this help.
+
+The full manual is at
+    https://github.com/donalgrant/telemetry-cli/blob/main/docs/recl.md
+"""
+
+
+class ReclError(ValueError):
+    pass
+
+
+def candidate_lengths(databytes: int, lo: int, hi: int, fact: int, partial: bool) -> list[int]:
+    """Lengths in [lo, hi] that are multiples of fact (and divide databytes, unless partial)."""
+    return [
+        r for r in range(max(lo, 1), hi + 1) if r % fact == 0 and (partial or databytes % r == 0)
+    ]
+
+
+def main(
+    argv: list[str] | None = None,
+    *,
+    stdin: BinaryIO | None = None,
+    stdout: BinaryIO | None = None,
+    stderr: TextIO | None = None,
+) -> int:
+    argv = sys.argv[1:] if argv is None else list(argv)
+    stdin = stdin if stdin is not None else sys.stdin.buffer
+    stdout = stdout if stdout is not None else sys.stdout.buffer
+    stderr = stderr if stderr is not None else sys.stderr
+
+    def error(message: str, status: int = 1) -> int:
+        print(f"recl: {message}", file=stderr)
+        return status
+
+    try:
+        o, args = getoptions(argv, SPECS)
+    except OptionError as e:
+        return error(f"{e} (see 'recl -help')", 2)
+    if o.get("h"):
+        stdout.write(HELP.encode())
+        return 0
+    if not args:
+        return error("no input file given (use - for stdin); see 'recl -help'", 2)
+    try:
+        return _run(args[0], o, stdin, stdout)
+    except ReclError as e:
+        return error(str(e))
+    except OSError as e:
+        return error(f"Can't open {args[0]} for input: {e.strerror}")
+    except BrokenPipeError:
+        return broken_pipe(stdout)
+
+
+def _run(file: str, o: dict, stdin: BinaryIO, stdout: BinaryIO) -> int:
+    lines: list[str] = []
+
+    class _Out:
+        def write(self, s):
+            lines.append(s)
+
+    msg = Messenger(_Out())
+    if file == "-":
+        data = stdin.read()
+    else:
+        with open(file, "rb") as f:
+            data = f.read()
+
+    skip = o.get("skip", 0)
+    reduce = o.get("reduce", 1.0)
+    if reduce < 1:
+        raise ReclError("-reduce must be at least 1")
+    fact = o.get("fact", 1)
+    if fact < 1:
+        raise ReclError("-fact must be at least 1")
+    databytes = len(data) - skip
+    if databytes < 2:
+        raise ReclError(f"too little data: {max(databytes, 0)} bytes after the header")
+    hi = o["max"] if "max" in o else databytes // o.get("minrecs", 2)
+
+    if "only" in o:
+        try:
+            lengths = [int(x) for x in o["only"].split(",") if x.strip()]
+        except ValueError:
+            raise ReclError(
+                f"-only needs a comma-separated list of lengths: {o['only']!r}"
+            ) from None
+    else:
+        lengths = candidate_lengths(databytes, o.get("min", 1), hi, fact, bool(o.get("part")))
+    msg("Checking Record Lengths " + ", ".join(map(str, lengths)) + " bytes")
+    _flush(lines, stdout)
+    if not lengths:
+        raise ReclError("no record lengths to check; see the -min, -max and -fact options")
+    if min(lengths) < 1:
+        raise ReclError("record lengths must be at least 1 byte")
+
+    use = np.frombuffer(data, dtype=np.uint8)[skip:]
+    if "limit" in o:
+        use = use[: max(o["limit"], 0)]
+    use = use[: int(len(use) / reduce)]
+
+    bufsiz = 2 * max(lengths)
+    if o.get("f") or len(use) < bufsiz:
+        buffers = use[np.newaxis, :]  # one piece
+    else:
+        nbuf = len(use) // bufsiz
+        if "maxbufs" in o:
+            nbuf = min(nbuf, max(o["maxbufs"], 1))
+        buffers = use[: nbuf * bufsiz].reshape(nbuf, bufsiz)
+
+    scores = agreement(buffers, lengths)  # (buffers, lengths)
+    if o.get("v"):
+        running = np.cumsum(scores, axis=0)
+        for i in range(scores.shape[0]):
+            for j, r in enumerate(lengths):
+                msg(f"buf {i + 1} reclen {r} corr {perl_str(float(scores[i, j]))} "
+                    f"accum-corr {perl_str(float(running[i, j]))}")  # fmt: skip
+    mean = scores.mean(axis=0).tolist()
+    chosen = choose(lengths, mean, compared(buffers, lengths).tolist())
+    if o.get("v"):
+        msg("Table of Sorted Correlations")
+        for j in sorted(range(len(lengths)), key=lambda j: (-mean[j], lengths[j])):
+            msg(f"{100 * mean[j]:5.2f}% for reclen {lengths[j]}")
+    msg(lengths[chosen], "RESULT")
+    if not o.get("q"):
+        msg(perl_str(100.0 * mean[chosen]), "CORR")
+        msg(f"Based on {buffers.shape[0]} buffers, each of length {buffers.shape[1]}")
+    _flush(lines, stdout)
+    stdout.flush()
+    return 0
+
+
+def _flush(lines: list[str], stdout: BinaryIO) -> None:
+    stdout.write("".join(lines).encode())
+    lines.clear()

telemetry_cli/recl/corr.py

@@ -0,0 +1,83 @@
+"""Scoring candidate record lengths.
+
+For a record length R, the score of a buffer is the fraction of its bytes
+that equal the byte R bytes further on. For unrelated random data that is
+about 1/256; it is higher when records are R bytes long, since records of
+the same format tend to have the same bytes in the same places (sync words,
+flags, the high bytes of counters and slowly varying values). Scores are
+averaged over the buffers.
+
+(Comparing whole bytes works better than counting agreeing bits, which is
+what the Perl original set out to do: a counter's low bits agree more often
+at multiples of the record length than at the length itself, which pulls a
+bit count toward multiples, and bits agree often at small shifts in text.)
+
+A length R is scored only in buffers of at least 2R bytes (as in the Perl);
+elsewhere its score is 0.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def agreement(buffers: np.ndarray, lengths: list[int]) -> np.ndarray:
+    """Scores, shape (number of buffers, number of lengths)."""
+    nbuf, size = buffers.shape
+    out = np.zeros((nbuf, len(lengths)))
+    for j, r in enumerate(lengths):
+        if r < 1 or size < 2 * r:
+            continue
+        out[:, j] = (buffers[:, : size - r] == buffers[:, r:]).mean(axis=1)
+    return out
+
+
+def compared(buffers: np.ndarray, lengths: list[int]) -> np.ndarray:
+    """How many byte pairs were compared for each length, over all buffers."""
+    nbuf, size = buffers.shape
+    r = np.array(lengths)
+    return np.where(size >= 2 * r, 1.0 * (size - r) * nbuf, 0.0)
+
+
+def choose(
+    lengths: list[int],
+    scores: list[float],
+    counts: list[float] | None = None,
+    tolerance: float = 0.25,
+) -> int:
+    """The record length the scores point to.
+
+    Multiples of the record length score about as well as the length itself,
+    since the data repeat at those too; the longer ones only less precisely,
+    as fewer bytes overlap. So the answer is the smallest length that divides
+    the best-scoring one and scores nearly as well: within three standard
+    errors of it, or within ``tolerance`` of its height above a typical
+    score, whichever is more. It must also score at least half that height
+    above a typical score, so that a length with no structure of its own (1,
+    say) isn't chosen when the signal is weak.
+
+    A typical score is the lower quartile: with short records, most of the
+    candidates may be multiples of the record length, so the median may not
+    be typical. With fewer than five candidates, the lowest score stands in,
+    and the structure check is skipped.
+
+    ``counts`` are the numbers of byte pairs compared, for the standard errors.
+    """
+    best = max(range(len(lengths)), key=lambda j: (scores[j], -lengths[j]))
+    few = len(lengths) < 5
+    baseline = float(min(scores) if few else np.percentile(scores, 25))
+    height = scores[best] - baseline
+    p = min(max(baseline, 1 / 256), 1 - 1 / 256)
+
+    def stderr(j):  # of a fraction near the baseline
+        return float(np.sqrt(p * (1 - p) / counts[j])) if counts and counts[j] > 0 else 0.0
+
+    for j in sorted(range(len(lengths)), key=lambda j: lengths[j]):
+        r = lengths[j]
+        if not r or lengths[best] % r:
+            continue
+        margin = max(tolerance * height, 3 * np.hypot(stderr(j), stderr(best)))
+        # nearly as good as the best, and clearly better than a typical length
+        if scores[j] >= scores[best] - margin and (few or scores[j] - baseline >= height / 2):
+            return j
+    return best

telemetry_cli/recs/__init__.py

@@ -0,0 +1 @@
+"""recs: extract records from a stream of data by their start and end markers."""

telemetry_cli/recs/__main__.py

@@ -0,0 +1,7 @@
+"""Allow ``python -m telemetry_cli.recs``."""
+
+import sys
+
+from .cli import main
+
+sys.exit(main())