totalsync 0.1.0__py3-none-any.whl

packages/totalsync_2p/totalsync_2p/__init__.py

@@ -0,0 +1,5 @@
+"""TotalSync 2-photon data synchronization."""
+
+from .sync import synchronize
+
+__all__ = ['synchronize']

packages/totalsync_2p/totalsync_2p/cli.py

@@ -0,0 +1,76 @@
+"""Command-line interface for totalsync_2p."""
+
+import argparse
+import sys
+
+from .sync import synchronize
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Synchronize ScanImage tif recordings with TotalSync behavioral telemetry.'
+    )
+
+    input_group = parser.add_argument_group('input files')
+    input_group.add_argument(
+        '--tif-files',
+        nargs='+',
+        metavar='TIF',
+        help='One or more ScanImage tif files.',
+    )
+    input_group.add_argument(
+        '--b64-files',
+        nargs='+',
+        metavar='B64',
+        help='One or more TotalSync .b64 files, in the same order as --tif-files.',
+    )
+
+    parser.add_argument(
+        '--output-dir',
+        required=True,
+        metavar='DIR',
+        help='Directory where output files will be saved.',
+    )
+    parser.add_argument(
+        '--pin-sheet',
+        required=True,
+        metavar='JSON',
+        help='Path to the pin mapping JSON file.',
+    )
+
+    args = parser.parse_args()
+
+    if not args.tif_files or not args.b64_files:
+        parser.error('--tif-files and --b64-files are required.')
+
+    if len(args.tif_files) != len(args.b64_files):
+        parser.error(
+            f'Number of tif files ({len(args.tif_files)}) must match '
+            f'number of b64 files ({len(args.b64_files)}).'
+        )
+
+    errors = []
+    for i, (tif, b64) in enumerate(zip(args.tif_files, args.b64_files)):
+        print(f'\n[{i + 1}/{len(args.tif_files)}] Processing {tif} + {b64}')
+        try:
+            stats = synchronize(
+                tif_file=tif,
+                b64_file=b64,
+                output_dir=args.output_dir,
+                pin_sheet_file=args.pin_sheet,
+            )
+            barcode_info = 'barcode' if stats['has_barcode'] else 'no barcode (clock-based)'
+            print(f'  Done — session: {stats["session"]}, alignment: {barcode_info}')
+        except Exception as e:
+            print(f'  ERROR: {e}', file=sys.stderr)
+            errors.append((tif, b64, e))
+
+    if errors:
+        print(f'\n{len(errors)} file(s) failed:', file=sys.stderr)
+        for tif, b64, e in errors:
+            print(f'  {tif} / {b64}: {e}', file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

packages/totalsync_2p/totalsync_2p/sync.py

@@ -0,0 +1,299 @@
+"""Core synchronization logic for aligning tif frames to behavioral telemetry."""
+
+import pickle
+import re
+import warnings
+from pathlib import Path
+from typing import Dict, Union
+import numpy as np
+import pynapple as nap
+import tifffile
+from tqdm import tqdm
+
+from totalsync_utils import decode_single_file
+
+
+def extract_barcode_from_tif(tif_file: str) -> dict:
+    tag_structure = {
+        'image_description': 5,
+        'frame_timestamp': 3,
+        'auxTrigger0': 10,
+    }
+
+    with tifffile.TiffFile(tif_file) as tif:
+        n_pages = len(tif.pages)
+        ts = np.zeros(n_pages)
+        value = np.zeros(n_pages)
+        frame_n = np.zeros(n_pages)
+        for i, page in tqdm(enumerate(tif.pages), total=n_pages):
+            description = page.tags.values()[tag_structure['image_description']].value
+            timestamp = float(
+                description.split('\n')[tag_structure['frame_timestamp']].split('=')[-1]
+            )
+            aux_line = description.split('\n')[tag_structure['auxTrigger0']]
+            data = aux_line.split('=')[-1].strip(' [').strip(']').strip(' ]')
+            ts[i] = timestamp
+            frame_n[i] = i
+            if 0 < len(data) < 50:
+                try:
+                    value[i] = float(data)
+                except ValueError:
+                    pass
+
+    return {'ts': ts, 'value': value, 'frame_n': frame_n}
+
+
+def closest_match_indices_sorted(A, B, max_tolerance):
+    """Return one-to-one closest matches between sorted arrays A and B."""
+    A = np.asarray(A)
+    B = np.asarray(B)
+
+    i = 0
+    j = 0
+
+    match_A_indices = []
+    match_B_indices = []
+    match_A_values = []
+    match_B_values = []
+    match_abs_differences = []
+    match_differences = []
+
+    unmatched_A_indices = []
+    matched_B_indices = set()
+
+    import pandas as pd
+
+    while i < len(A):
+        a = A[i]
+
+        while j < len(B) and B[j] < a - max_tolerance:
+            j += 1
+
+        candidates = []
+
+        if j < len(B) and abs(B[j] - a) <= max_tolerance:
+            candidates.append(j)
+
+        if j > 0 and (j - 1) not in matched_B_indices:
+            if abs(B[j - 1] - a) <= max_tolerance:
+                candidates.append(j - 1)
+
+        candidates = [idx for idx in candidates if idx not in matched_B_indices]
+
+        if candidates:
+            best_j = min(candidates, key=lambda idx: abs(B[idx] - a))
+
+            match_A_indices.append(i)
+            match_B_indices.append(best_j)
+            match_A_values.append(A[i])
+            match_B_values.append(B[best_j])
+            match_abs_differences.append(abs(B[best_j] - A[i]))
+            match_differences.append(B[best_j] - A[i])
+            matched_B_indices.add(best_j)
+
+            if best_j == j:
+                j += 1
+        else:
+            unmatched_A_indices.append(i)
+
+        i += 1
+
+    unmatched_B_indices = np.array(
+        [idx for idx in range(len(B)) if idx not in matched_B_indices], dtype=int
+    )
+
+    matches = pd.DataFrame({
+        'A_index': np.array(match_A_indices, dtype=int),
+        'B_index': np.array(match_B_indices, dtype=int),
+        'A_value': np.array(match_A_values, dtype=A.dtype),
+        'B_value': np.array(match_B_values, dtype=B.dtype),
+        'abs_difference': np.array(match_abs_differences),
+        'difference': np.array(match_differences),
+    })
+
+    return matches, np.array(unmatched_A_indices, dtype=int), unmatched_B_indices
+
+
+def normalize_underscores(s: str) -> str:
+    return re.sub(r'_+', '_', s).rstrip('_')
+
+
+def fix_tsync_time(log_times: np.ndarray) -> np.ndarray:
+    skips = -np.where(
+        np.diff(log_times) < 0,
+        np.diff(log_times) - np.median(np.diff(log_times)),
+        0,
+    )
+    cs = np.cumsum(skips)
+    cs2 = np.hstack((0, cs))
+    return log_times + cs2
+
+def fill_gaps_in_tframes(t_frames: np.ndarray) -> np.ndarray:
+    median_interval = np.median(np.diff(t_frames))
+
+    t_frames_filled = t_frames.copy()
+    gap_locations = np.where(np.diff(t_frames_filled) > median_interval * 1.5)[0]
+
+    while len(gap_locations) > 0:
+        loc = gap_locations[0]
+        gap_size = np.diff(t_frames_filled)[loc]
+        t_frame_before = t_frames_filled[loc]
+        t_frame_after = t_frames_filled[loc + 1]
+        n_frames_to_fill = int(gap_size / median_interval)
+        frames_to_fill = np.linspace(t_frame_before, t_frame_after, n_frames_to_fill + 1)
+        t_frames_filled = np.insert(t_frames_filled, loc + 1, frames_to_fill[1:-1])
+        gap_locations = np.where(np.diff(t_frames_filled) > median_interval * 1.5)[0]
+
+    return t_frames_filled
+
+def synchronize(tif_file: str, b64_file: str, output_dir_str: str, pin_sheet_file: str, fill_gaps: bool=False, ignore_barcode: bool=False) -> Dict[str, Union[str, nap.Tsd, np.ndarray]]:
+    """Synchronize a tif imaging file with a b64 behavioral telemetry file.
+
+    Parameters
+    ----------
+    tif_file : str
+        Path to the ScanImage tif recording.
+    b64_file : str
+        Path to the TotalSync .b64 behavioral telemetry file.
+    output_dir_str: str
+        Directory where output files will be saved.
+    pin_sheet_file : str
+        Path to the pin mapping JSON file.
+    fill_gaps : bool, default=False
+        Whether to fill gaps in the timestamp sequence using linear interpolation
+
+    Returns
+    -------
+    dict
+        Statistics and synchronization results with keys:
+        - 'session': session name (stem of b64_file)
+        - 'has_barcode': whether barcode-based alignment was used
+        - 'max_ts_gap': maximum timestamp gap in the behavioral log
+        - 'gap_locations': timestamps where gaps occurred
+        - 'barcode_shift' (if has_barcode): time shift between barcode signals
+        - 'barcode_frame_matches' (if has_barcode): DataFrame of matched frames
+        - 'frames_time_idx': the resulting pynapple Tsd
+
+    Saved files (in output_dir)
+    ---------------------------
+    {session}_barcode_data.npz
+        Raw decoded telemetry arrays.
+    {session}_frames_time_idx.npz
+        Pynapple Tsd mapping scanner time -> tif frame index.
+    {session}_behavior_sync_stats.pkl
+        Dictionary of synchronization statistics.
+    behavior/{key_name}.npz
+        One pynapple Tsd (1-D channels) or TsdFrame (2-D channels) per decoded
+        telemetry key, time-indexed by the corrected TotalSync clock (µs).
+    """
+    output_dir = Path(output_dir_str)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    session = Path(b64_file).stem
+    stats = {'session': session}
+
+    # --- Decode behavioral telemetry ---
+    tsync_data = decode_single_file(b64_file, pin_json_path=pin_sheet_file)
+    np.savez(output_dir / f"{session}_barcode_data.npz", **tsync_data)
+
+    # --- Build frame-clock timestamps ---
+    frame_clock = tsync_data['Scanner Frame Clock (Input)'].astype(int)
+    log_times = tsync_data['startTS'].astype(int)
+
+    onsets = np.nonzero(np.diff(frame_clock) == 1)[0] + 1
+
+    stats['max_ts_gap'] = int(np.max(np.diff(log_times)))
+
+
+    tsync_time = fix_tsync_time(log_times)
+    stats['gap_locations'] = tsync_time[np.where(np.diff(log_times) > 30000)]
+    if len(stats['gap_locations']) > 0:
+        warnings.warn(f"There are gaps in the timestamps: {stats['gap_locations']}")
+    # --- Save behavioral telemetry as pynapple objects ---
+    behavior_dir = output_dir / "behavior"
+    behavior_dir.mkdir(parents=True, exist_ok=True)
+    for key in tsync_data:
+        key_name = normalize_underscores(
+            key.replace(" ", "_").replace("(", "_").replace(")", "_").replace("-", "_")
+        )
+        if tsync_data[key].ndim == 1:
+            tsd = nap.Tsd(t=tsync_time, d=tsync_data[key], time_units='us')
+            tsd.save(behavior_dir / f"{key_name}.npz")
+        elif tsync_data[key].ndim == 2:
+            tsd = nap.TsdFrame(t=tsync_time, d=tsync_data[key], time_units='us')
+            tsd.save(behavior_dir / f"{key_name}.npz")
+
+    t_frames = nap.Ts(tsync_time[onsets], time_units='us')
+
+    # --- Extract aux barcode from tif ---
+    aux_data = extract_barcode_from_tif(tif_file)
+
+    aux_high = np.nonzero(aux_data['value'])[0]
+    aux_barcode_ts = nap.Ts(aux_data['value'][aux_high], time_units='s')
+
+    has_barcode = 'Barcode (Scanner)' in tsync_data and len(aux_barcode_ts) > 0 and not ignore_barcode
+    stats['has_barcode'] = has_barcode
+
+    if has_barcode:
+        tsync_barcode = tsync_data['Barcode (Scanner)'].astype(int)
+        tsync_barcode_rising_edge = np.nonzero(np.diff(tsync_barcode) > 0)[0] + 1
+        tsync_barcode_ts = nap.Ts(tsync_time[tsync_barcode_rising_edge], time_units='us')
+        # shift_offset = (tsync_barcode_ts.t[0] - aux_barcode_ts.t[0])
+        shift_offset = t_frames.t[0]
+        aux_barcode_ts = nap.Ts(aux_barcode_ts.t + shift_offset, time_units='s')
+
+        barcode_group = nap.TsGroup({0: aux_barcode_ts, 1: tsync_barcode_ts})
+        crosscorrs = nap.compute_crosscorrelogram(
+            group=barcode_group, time_units='ms', windowsize=5000, binsize=1
+        )
+        shift = crosscorrs.idxmax().iloc[0] + shift_offset
+        stats['barcode_shift'] = float(shift)
+
+        matches, _, _ = closest_match_indices_sorted(
+            aux_data['ts'] + shift, t_frames.t, max_tolerance=0.025
+        )
+        stats['barcode_frame_matches'] = matches
+
+        frames_time_idx = nap.Tsd(
+            t=matches['B_value'].to_numpy(),
+            d=matches['A_index'].to_numpy(),
+            time_units='s',
+        )
+    else:
+        warnings.warn(
+            "No barcode detected in the tif file, frame/time alignment will be done "
+            "by assuming that the first Scanner frame clock pulse corresponds to the "
+            "first tif frame"
+        )
+        if len(stats['gap_locations']) > 0:
+            if fill_gaps:
+                t_frames = nap.Ts(fill_gaps_in_tframes(t_frames.t), time_units='s')
+                stats['orig_gap_locations'] = stats['gap_locations']
+                stats['orig_max_ts_gap'] = stats['max_ts_gap']
+                stats['gap_locations'] = np.empty(0)
+                stats['max_ts_gap'] = 0
+
+            else:
+                warnings.warn(
+                    "With no barcode, and gaps in the timestamps, alignment will be done "
+                    "up to the first gap"
+                )
+                # find the first gap location after the scanner was started
+                align_stop = stats['gap_locations'][np.where(stats['gap_locations'] > t_frames[0].t * 1e6)[0][0]]
+                ep = nap.IntervalSet(start=0, end=align_stop, time_units='us')
+                t_frames = t_frames.restrict(ep)
+
+        frames_time_idx = nap.Tsd(
+            t=t_frames.t,
+            d=np.arange(len(t_frames)),
+            time_units='s',
+        )
+
+    frames_time_idx.save(output_dir / f"{session}_frames_time_idx.npz")
+
+    stats['frames_time_idx'] = frames_time_idx
+
+    with open(output_dir / f"{session}_behavior_sync_stats.pkl", 'wb') as f:
+        pickle.dump({k: v for k, v in stats.items() if k != 'frames_time_idx'}, f)
+
+    return stats

packages/totalsync_utils/totalsync_utils/__init__.py

@@ -0,0 +1,5 @@
+"""TotalSync data decoding utilities."""
+
+from .decoder import decode_b64_files, decode_single_file, load_pin_mapping, apply_pin_mapping
+
+__all__ = ['decode_b64_files', 'decode_single_file', 'load_pin_mapping', 'apply_pin_mapping']

packages/totalsync_utils/totalsync_utils/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running as python -m totalsync_utils."""
+
+from .cli import main
+
+if __name__ == '__main__':
+    main()

packages/totalsync_utils/totalsync_utils/cli.py

@@ -0,0 +1,235 @@
+"""Command-line interface for TotalSync decoder."""
+
+import argparse
+import os
+import re
+import sys
+import warnings
+from pathlib import Path
+
+import numpy as np
+from scipy.io import savemat
+
+from .decoder import decode_b64_files
+
+
+def _fix_tsync_time(log_times: np.ndarray) -> np.ndarray:
+    skips = -np.where(
+        np.diff(log_times) < 0,
+        np.diff(log_times) - np.median(np.diff(log_times)),
+        0,
+    )
+    cs = np.cumsum(skips)
+    return log_times + np.hstack((0, cs))
+
+
+def _normalize_underscores(s: str) -> str:
+    return re.sub(r'_+', '_', s).rstrip('_')
+
+
+def save_as_pynapple(data_dict, output_dir, base_name):
+    """Save decoded b64 data as pynapple Tsd/TsdFrame files, time-indexed by corrected TotalSync clock."""
+    import pynapple as nap
+
+    log_times = data_dict['startTS'].astype(int)
+    tsync_time = _fix_tsync_time(log_times)
+
+    gap_locations = tsync_time[np.where(np.diff(log_times) > 30000)]
+    if len(gap_locations) > 0:
+        warnings.warn(f"{base_name}: gaps in timestamps at {gap_locations}")
+
+    behavior_dir = Path(output_dir) / base_name / "behavior"
+    behavior_dir.mkdir(parents=True, exist_ok=True)
+
+    for key, arr in data_dict.items():
+        if not isinstance(arr, np.ndarray):
+            continue
+        key_name = _normalize_underscores(
+            key.replace(" ", "_").replace("(", "_").replace(")", "_").replace("-", "_")
+        )
+        if arr.ndim == 1 and len(arr) == len(tsync_time):
+            nap.Tsd(t=tsync_time, d=arr, time_units='us').save(behavior_dir / f"{key_name}.npz")
+        elif arr.ndim == 2 and arr.shape[0] == len(tsync_time):
+            nap.TsdFrame(t=tsync_time, d=arr, time_units='us').save(behavior_dir / f"{key_name}.npz")
+
+    print(f"Saved: {behavior_dir}")
+
+
+def save_as_pynapple_concatenated(results_dict, output_dir):
+    """Concatenate multiple decoded b64 files into a single set of pynapple files.
+
+    Timestamps are shifted so each file's time axis starts at prev_end + 1 (in µs).
+    """
+    import pynapple as nap
+
+    all_tsync_times = []
+    offset = None
+    for name, data_dict in results_dict.items():
+        log_times = data_dict['startTS'].astype(int)
+        tsync_time = _fix_tsync_time(log_times)
+
+        gap_locations = tsync_time[np.where(np.diff(log_times) > 30000)]
+        if len(gap_locations) > 0:
+            warnings.warn(f"{name}: gaps in timestamps at {gap_locations}")
+
+        if offset is None:
+            shifted = tsync_time
+        else:
+            shifted = tsync_time - tsync_time[0] + offset + 1
+
+        all_tsync_times.append(shifted)
+        offset = int(shifted[-1])
+
+    tsync_time_cat = np.concatenate(all_tsync_times)
+
+    behavior_dir = Path(output_dir) / "behavior"
+    behavior_dir.mkdir(parents=True, exist_ok=True)
+
+    first_data = next(iter(results_dict.values()))
+    for key in first_data:
+        arrays = [d[key] for d in results_dict.values()
+                  if key in d and isinstance(d[key], np.ndarray)]
+        if len(arrays) != len(results_dict):
+            continue
+
+        arr = np.concatenate(arrays, axis=0)
+        key_name = _normalize_underscores(
+            key.replace(" ", "_").replace("(", "_").replace(")", "_").replace("-", "_")
+        )
+        if arr.ndim == 1 and len(arr) == len(tsync_time_cat):
+            nap.Tsd(t=tsync_time_cat, d=arr, time_units='us').save(behavior_dir / f"{key_name}.npz")
+        elif arr.ndim == 2 and arr.shape[0] == len(tsync_time_cat):
+            nap.TsdFrame(t=tsync_time_cat, d=arr, time_units='us').save(behavior_dir / f"{key_name}.npz")
+
+    print(f"Saved concatenated pynapple files to: {behavior_dir}")
+
+
+def save_as_npy(data_dict, output_dir, base_name):
+    """Save decoded data as .npy files."""
+    output_path = os.path.join(output_dir, f"{base_name}_decoded.npy")
+    np.save(output_path, data_dict)
+    print(f"Saved: {output_path}")
+
+
+def save_as_mat(data_dict, output_dir, base_name):
+    """Save decoded data as .mat file."""
+    output_path = os.path.join(output_dir, f"{base_name}_decoded.mat")
+    savemat(output_path, data_dict)
+    print(f"Saved: {output_path}")
+
+
+def main():
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description='Decode TotalSync .b64 files',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Decode files and print to console
+  totalsync-decode /path/to/data
+
+  # Save as .mat files
+  totalsync-decode /path/to/data --output /path/to/output --format mat
+
+  # Save as .npy files
+  totalsync-decode /path/to/data --output /path/to/output --format npy
+
+  # Use pin mapping to split channels by name
+  totalsync-decode /path/to/data --output /path/to/output --pin-json docs/pinSheet.json
+
+  # Save as pynapple Tsd/TsdFrame files (time-indexed by TotalSync clock)
+  totalsync-decode /path/to/data --output /path/to/output --format pynapple --pin-json docs/pinSheet.json
+
+  # Concatenate multiple b64 files into a single set of pynapple files
+  totalsync-decode /path/to/data --output /path/to/output --format pynapple --concatenate --pin-json docs/pinSheet.json
+        """
+    )
+
+    parser.add_argument(
+        'directory',
+        help='Directory containing .b64 files to decode'
+    )
+
+    parser.add_argument(
+        '-o', '--output',
+        help='Output directory (if not specified, data is only returned, not saved)',
+        default=None
+    )
+
+    parser.add_argument(
+        '-f', '--format',
+        choices=['npy', 'mat', 'pynapple'],
+        default='mat',
+        help='Output format for saved files (default: mat)'
+    )
+
+    parser.add_argument(
+        '-p', '--pin-json',
+        help='Path to pin mapping JSON file (e.g., pinSheet.json)',
+        default=None
+    )
+
+    parser.add_argument(
+        '--concatenate',
+        action='store_true',
+        help='Concatenate all b64 files into a single set of pynapple files (only with --format pynapple)'
+    )
+
+    parser.add_argument(
+        '-q', '--quiet',
+        action='store_true',
+        help='Suppress progress output'
+    )
+
+    args = parser.parse_args()
+
+    if args.concatenate and args.format != 'pynapple':
+        print("Error: --concatenate is only valid with --format pynapple", file=sys.stderr)
+        return 1
+
+    # Check input directory exists
+    if not os.path.isdir(args.directory):
+        print(f"Error: Directory not found: {args.directory}", file=sys.stderr)
+        return 1
+
+    # Create output directory if specified
+    if args.output:
+        os.makedirs(args.output, exist_ok=True)
+        if not os.path.isdir(args.output):
+            print(f"Error: Could not create output directory: {args.output}", file=sys.stderr)
+            return 1
+
+    # Check pin JSON file if provided
+    if args.pin_json and not os.path.isfile(args.pin_json):
+        print(f"Error: Pin JSON file not found: {args.pin_json}", file=sys.stderr)
+        return 1
+
+    # Decode files
+    try:
+        results = decode_b64_files(args.directory, verbose=not args.quiet, pin_json_path=args.pin_json)
+    except Exception as e:
+        print(f"Error during decoding: {e}", file=sys.stderr)
+        return 1
+
+    # Save results if output directory specified
+    if args.output:
+        if args.format == 'pynapple' and args.concatenate:
+            save_as_pynapple_concatenated(results, args.output)
+        else:
+            for name, data in results.items():
+                if args.format == 'npy':
+                    save_as_npy(data, args.output, name)
+                elif args.format == 'pynapple':
+                    save_as_pynapple(data, args.output, name)
+                else:  # mat
+                    save_as_mat(data, args.output, name)
+    else:
+        if not args.quiet:
+            print(f"\nDecoded {len(results)} file(s)")
+            print("No output directory specified - data not saved")
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())