filejack 1.0.0__tar.gz

filejack-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Staheos
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

filejack-1.0.0/PKG-INFO

@@ -0,0 +1,150 @@
+Metadata-Version: 2.4
+Name: filejack
+Version: 1.0.0
+Summary: File encoding/decoding to audio signals.
+Author: Staheos
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Staheos/FileJack
+Project-URL: Source, https://github.com/Staheos/FileJack
+Project-URL: Issues, https://github.com/Staheos/FileJack/issues
+Keywords: file,audio,jack,encoder,decoder
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.15.0
+Requires-Dist: scipy>=1.0.0
+Requires-Dist: reedsolo>=1.0.0
+Provides-Extra: test
+Requires-Dist: pytest>=8.0; extra == "test"
+Requires-Dist: pytest-cov; extra == "test"
+Requires-Dist: pytest-xdist[psutil]>=3.5; extra == "test"
+Dynamic: license-file
+
+# FileJack
+
+File encoding/decoding to audio signals.
+
+FileJack turns an arbitrary file into a WAV audio signal and back. The data is
+carried on a 12 kHz carrier using differential 8-PSK modulation, protected with
+Reed-Solomon error correction and a per-frame CRC. It is meant for sending files
+over an audio link (for example a 3.5 mm jack cable or an acoustic channel).
+
+## How it works
+
+The encoder splits the input into frames. Each frame carries a header
+(sequence number, total frame count, payload length), the payload and a CRC32,
+then gets Reed-Solomon encoded into a 255-byte codeword (223 data bytes + 32
+parity bytes, so up to 209 payload bytes per frame). Each codeword is mapped to
+symbols and modulated onto the carrier, preceded by a preamble and a syncword so
+the decoder can find frame boundaries.
+
+The decoder demodulates the recording, locks onto the preamble/syncword,
+recovers the symbols, runs Reed-Solomon correction and checks each frame's CRC.
+Valid frames are collected by sequence number and reassembled into the original
+file. Decoding runs across multiple threads over overlapping blocks, so partial
+or noisy recordings still recover whatever frames are intact.
+
+Signal parameters live in [filejack/values.py](filejack/values.py): 48 kHz
+sample rate, 4000 baud, 12 kHz carrier, 8-PSK.
+
+## Installation
+
+```
+pip install filejack
+```
+
+Or from source:
+
+```
+pip install .
+```
+
+Dependencies: numpy, scipy, reedsolo.
+
+## Command line usage
+
+Encoding writes a WAV file. By default it produces an anti-phase stereo signal
+(the right channel is the inverted left channel), which lets the decoder cancel
+common-mode noise by subtracting the channels.
+
+```
+filejack encode input.bin
+filejack encode input.bin output.wav
+filejack encode input.bin output.wav --mono
+```
+
+Decoding reads a WAV file and writes the reconstructed file. Stereo input is
+folded down automatically.
+
+```
+filejack decode output.wav
+filejack decode output.wav recovered.bin
+```
+
+Decode options:
+
+- `--fjf PATH` also saves the decoded frames into a `.fjf` file. This lets you
+  merge frames recovered from several recordings of the same transmission.
+- `--threads N` sets the number of decoding threads (default 12).
+
+The command is also available as `python -m filejack`.
+
+## Library usage
+
+```python
+import zlib
+from scipy.io import wavfile
+import numpy as np
+
+from filejack.encode_frames import encode_data
+from filejack.decode_data import decode_data
+from filejack.reconstruct_data import reconstruct_data
+from filejack.values import SAMPLE_RATE
+
+data = open("input.bin", "rb").read()
+
+# Encode to samples and write a WAV
+waveform = encode_data(data)
+wavfile.write("output.wav", SAMPLE_RATE, waveform)
+
+# Read a WAV and decode
+fs, rx = wavfile.read("output.wav")
+rx = rx.astype(np.float32)
+if rx.ndim == 2:
+	rx = (rx[:, 0] - rx[:, 1]) / 2
+
+frames_payload, total_expected, stats = decode_data(rx)
+recovered = reconstruct_data(frames_payload, total_expected)
+
+assert zlib.crc32(recovered) == zlib.crc32(data)
+```
+
+More runnable scripts are in [filejack/examples](filejack/examples), including
+`stereo_to_mono.py` for folding an anti-phase stereo recording down to mono.
+
+## .fjf files
+
+A `.fjf` (FileJack Frames) file stores decoded frames with their sequence
+numbers. Because each frame is independent, you can decode the same
+transmission several times, save each attempt to a `.fjf`, and merge them to
+fill in frames that were missed in individual passes. See
+[filejack/merge_frames.py](filejack/merge_frames.py) and the merge example.
+
+## Development
+
+```
+pip install .[test]
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).

filejack-1.0.0/README.md

@@ -0,0 +1,120 @@
+# FileJack
+
+File encoding/decoding to audio signals.
+
+FileJack turns an arbitrary file into a WAV audio signal and back. The data is
+carried on a 12 kHz carrier using differential 8-PSK modulation, protected with
+Reed-Solomon error correction and a per-frame CRC. It is meant for sending files
+over an audio link (for example a 3.5 mm jack cable or an acoustic channel).
+
+## How it works
+
+The encoder splits the input into frames. Each frame carries a header
+(sequence number, total frame count, payload length), the payload and a CRC32,
+then gets Reed-Solomon encoded into a 255-byte codeword (223 data bytes + 32
+parity bytes, so up to 209 payload bytes per frame). Each codeword is mapped to
+symbols and modulated onto the carrier, preceded by a preamble and a syncword so
+the decoder can find frame boundaries.
+
+The decoder demodulates the recording, locks onto the preamble/syncword,
+recovers the symbols, runs Reed-Solomon correction and checks each frame's CRC.
+Valid frames are collected by sequence number and reassembled into the original
+file. Decoding runs across multiple threads over overlapping blocks, so partial
+or noisy recordings still recover whatever frames are intact.
+
+Signal parameters live in [filejack/values.py](filejack/values.py): 48 kHz
+sample rate, 4000 baud, 12 kHz carrier, 8-PSK.
+
+## Installation
+
+```
+pip install filejack
+```
+
+Or from source:
+
+```
+pip install .
+```
+
+Dependencies: numpy, scipy, reedsolo.
+
+## Command line usage
+
+Encoding writes a WAV file. By default it produces an anti-phase stereo signal
+(the right channel is the inverted left channel), which lets the decoder cancel
+common-mode noise by subtracting the channels.
+
+```
+filejack encode input.bin
+filejack encode input.bin output.wav
+filejack encode input.bin output.wav --mono
+```
+
+Decoding reads a WAV file and writes the reconstructed file. Stereo input is
+folded down automatically.
+
+```
+filejack decode output.wav
+filejack decode output.wav recovered.bin
+```
+
+Decode options:
+
+- `--fjf PATH` also saves the decoded frames into a `.fjf` file. This lets you
+  merge frames recovered from several recordings of the same transmission.
+- `--threads N` sets the number of decoding threads (default 12).
+
+The command is also available as `python -m filejack`.
+
+## Library usage
+
+```python
+import zlib
+from scipy.io import wavfile
+import numpy as np
+
+from filejack.encode_frames import encode_data
+from filejack.decode_data import decode_data
+from filejack.reconstruct_data import reconstruct_data
+from filejack.values import SAMPLE_RATE
+
+data = open("input.bin", "rb").read()
+
+# Encode to samples and write a WAV
+waveform = encode_data(data)
+wavfile.write("output.wav", SAMPLE_RATE, waveform)
+
+# Read a WAV and decode
+fs, rx = wavfile.read("output.wav")
+rx = rx.astype(np.float32)
+if rx.ndim == 2:
+	rx = (rx[:, 0] - rx[:, 1]) / 2
+
+frames_payload, total_expected, stats = decode_data(rx)
+recovered = reconstruct_data(frames_payload, total_expected)
+
+assert zlib.crc32(recovered) == zlib.crc32(data)
+```
+
+More runnable scripts are in [filejack/examples](filejack/examples), including
+`stereo_to_mono.py` for folding an anti-phase stereo recording down to mono.
+
+## .fjf files
+
+A `.fjf` (FileJack Frames) file stores decoded frames with their sequence
+numbers. Because each frame is independent, you can decode the same
+transmission several times, save each attempt to a `.fjf`, and merge them to
+fill in frames that were missed in individual passes. See
+[filejack/merge_frames.py](filejack/merge_frames.py) and the merge example.
+
+## Development
+
+```
+pip install .[test]
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).

filejack-1.0.0/VERSION

@@ -0,0 +1 @@
+1.0.0

filejack-1.0.0/filejack/AsyncDecodeManager.py

@@ -0,0 +1,231 @@
+import queue
+import threading
+import time
+import zlib
+from collections import deque
+from concurrent.futures.thread import ThreadPoolExecutor
+from dataclasses import dataclass
+from fractions import Fraction
+from statistics import median
+
+from reedsolo import ReedSolomonError
+from scipy.signal import hilbert, butter, sosfiltfilt, resample_poly
+
+from scipy.io import wavfile
+
+from .conversions import *
+from .decode_frames import decode_frames
+from .values import *
+from .RSC import *
+import numpy as np
+# import sounddevice as sd
+
+
+@dataclass
+class Chunk:
+	stream_id: str
+	start_abs: int          # absolute sample index for this source stream
+	samples: np.ndarray     # float32
+
+class SourceBuffer:
+	def __init__(self, overlap: int):
+		self.total = 0
+		self.parts = deque()
+		self.overlap = overlap
+
+	def append(self, x: np.ndarray):
+		x = np.asarray(x, dtype=np.float32)
+
+		self.parts.append(x)
+		self.total += len(x)
+
+	def available_abs_end(self) -> int:
+		return self.total - 2 * self.overlap
+
+	def pop_slice(self, size) -> np.ndarray:
+		abs_start = 0
+		abs_end = size + self.overlap * 2
+
+		need = abs_end - abs_start
+
+		if need > self.total:
+			raise IndexError("slice not available yet")
+
+		out = np.empty(need, dtype=np.float32)
+		w = 0
+
+		while self.parts and w < need:
+			if len(self.parts[0]) <= need - w - 2 * self.overlap:
+				seg = self.parts.popleft()
+				out[w:w+len(seg)] = seg
+				w += len(seg)
+			else:
+				seg = self.parts[0][:need - w]
+				out[w:w+len(seg)] = seg
+				self.parts[0] = self.parts[0][need - w - 2 * self.overlap:]
+				w += len(seg)
+
+		self.total -= need - 2 * self.overlap
+		return out
+
+
+class AsyncDecodeManager:
+	def __init__(self, block_sec=3.0, overlap_sec=0.1):
+		self.block_samples_len = int(round(SAMPLE_RATE * block_sec))
+		self.overlap_samples_len = int(round(SAMPLE_RATE * overlap_sec))
+		self.buffer = np.array([], dtype=np.float32)
+		self.frames_payload = {}
+		self.total_expected = None
+		self.lock = threading.Lock()
+		self.rsc_error = 0
+		self.header_error = 0
+		self.crc_error = 0
+		self.duplicate_frames = 0
+
+		self.no_lock = 0
+		self.rsc_error = 0
+		self.header_error = 0
+		self.crc_error = 0
+
+		self.buffers: dict[str, SourceBuffer] = {}
+		self.next_block_start: dict[str, int] = {}
+
+		self.queue = queue.Queue(maxsize=2000)
+		self.executor = ThreadPoolExecutor(max_workers=32)
+		self.limit = threading.BoundedSemaphore(32)
+
+		self._stop = threading.Event()
+		self._thread = threading.Thread(target=self._run, daemon=True)
+
+		self.max_keep = 5 * self.block_samples_len + 3 * self.overlap_samples_len
+
+	def start(self):
+
+		self._stop = threading.Event()
+		self._thread.start()
+
+	def stop(self):
+		self._stop.set()
+		self._thread.join()
+		self.executor.shutdown(wait=True)
+
+	def push_chunk(self, chunk):
+		# If you have sources with different fs: resample here to target_fs (not shown)
+		self.queue.put(chunk)
+
+	def _ensure_stream(self, stream_id: str):
+		if stream_id not in self.buffers:
+			self.buffers[stream_id] = SourceBuffer(overlap=self.overlap_samples_len)
+			self.next_block_start[stream_id] = 0
+
+	def _run(self):
+		while not self._stop.is_set():
+			try:
+				chunk = self.queue.get(timeout=0.1)
+			except queue.Empty:
+				time.sleep(0.1)
+				continue
+
+			try:
+				self._ensure_stream(chunk.stream_id)
+				sb = self.buffers[chunk.stream_id]
+				sb.append(chunk.samples)
+				self._schedule_ready(chunk.stream_id)
+			finally:
+				self.queue.task_done()
+
+	def _schedule_ready(self, stream_id: str):
+		sb = self.buffers[stream_id]
+		next = self.next_block_start[stream_id]
+
+		while True:
+			start = max(0, next - self.overlap_samples_len)
+			end = start + self.block_samples_len + 2 * self.overlap_samples_len
+
+			# start = max(0, next)
+			# end = start + self.block_samples_len
+
+			if sb.available_abs_end() < self.block_samples_len:
+				break  # not enough samples yet
+
+			rx_block = sb.pop_slice(self.block_samples_len)
+
+			# submit decode job
+			# self.limit.acquire()
+			fut = self.executor.submit(
+				self._decode_one,
+				rx_block, stream_id, start, end
+			)
+			fut.add_done_callback(self._merge_result)
+
+			# advance to next valid block (no overlap in valid regions)
+			next += self.block_samples_len
+			self.next_block_start[stream_id] = next
+
+	def _decode_one(self, rx_block, stream_id: str, start, end):
+		frames_block, total_expected, ok_log, no_lock, rsc_error, header_error, crc_error = decode_frames(
+			rx_block, SAMPLE_RATE,
+			search_step=SAMPLES_PER_SYMBOL, quick=True
+		)
+		return (frames_block, total_expected, ok_log, no_lock, rsc_error, header_error, crc_error, stream_id, start, end)
+
+	def _merge_result(self, fut):
+		# self.limit.release()
+		payloads, total, ok_log, no_lock, rsc_error, header_error, crc_error, stream_id, start, end = fut.result()
+		with self.lock:
+			if total is not None and self.total_expected is None:
+				self.total_expected = total
+
+			for seq, payload in payloads.items():
+				self.no_lock += no_lock
+				self.rsc_error += rsc_error
+				self.header_error += header_error
+				self.crc_error += crc_error
+
+				if seq not in self.frames_payload:
+					self.frames_payload[seq] = payload
+				else:
+					self.duplicate_frames += 1
+
+			print(f"\rBlock(Sample): {end // self.block_samples_len}/{self.buffers[stream_id].available_abs_end() // self.block_samples_len}({end}/{self.buffers[stream_id].available_abs_end()})  Frames: {len(self.frames_payload)}(+{self.duplicate_frames})/{self.total_expected}  no_lock: {self.no_lock} rsc_error: {self.rsc_error} header_error: {self.header_error} crc_error: {self.crc_error}", end='')
+
+
+def feed_wav(manager: AsyncDecodeManager, stream_id: str, path: str, chunk_size=16000):
+	fs, rx = wavfile.read(path)
+	rx = rx.astype(np.float32)
+	pos = 0
+	while pos < len(rx):
+		x = rx[pos:pos+chunk_size]
+		manager.push_chunk(Chunk(stream_id=stream_id, start_abs=pos, samples=x))
+		pos += len(x)
+	manager.push_chunk(Chunk(stream_id, start_abs=pos, samples=np.zeros((manager.block_samples_len + manager.overlap_samples_len,), dtype=np.float32)))
+
+
+if __name__ == '__main__':
+	file = "bmp"
+
+	manager = AsyncDecodeManager()
+	manager.start()
+
+	# sample_pos = 0
+	# def callback(indata, frames, time_info, status):
+	# 	if status:
+	# 		print(status, flush=True)
+	#
+	# 	global sample_pos
+	# 	manager.push_chunk(Chunk(stream_id="mic", start_abs=sample_pos, samples=indata[:, 0].copy()))
+	# 	sample_pos += indata[:, 0].shape[0]
+	#
+	# stream = sd.InputStream(samplerate=SAMPLE_RATE, channels=1, dtype='float32', blocksize=4096, callback=callback)
+	#
+	# stream.start()
+	#
+	# while True:
+	# 	time.sleep(1)
+
+	feed_wav(manager, "wav", f"{file}.wav")
+	print("read wav")
+	# feed_wav(manager, "wav1", f"{file}1.wav")
+
+	manager.queue.join()
+	manager.stop()

filejack-1.0.0/filejack/RSC.py

@@ -0,0 +1,50 @@
+import struct
+from reedsolo import RSCodec
+
+from .values import *
+
+
+N = 255
+NSYM = 32
+K = N - NSYM
+RSC = RSCodec(NSYM)
+
+HDR_FMT = ">IIH"
+HDR_LEN = struct.calcsize(HDR_FMT)
+
+CRC_LEN = 4
+
+MAX_PAYLOAD = K - HDR_LEN - CRC_LEN
+
+
+def lfsr_bits(n: int, seed: int, taps=(7, 6)) -> list[int]:
+	"""
+	Simple LFSR bit generator.
+	taps are 1-indexed bit positions within the register size (max(taps)).
+	"""
+	m = max(taps)
+	mask = (1 << m) - 1
+	reg = seed & mask
+	if reg == 0:
+		reg = 1  # avoid stuck-at-zero
+
+	out = []
+	for _ in range(n):
+		# output bit (LSB)
+		out.append(reg & 1)
+
+		# feedback = XOR of tap bits
+		fb = 0
+		for t in taps:
+			fb ^= (reg >> (t - 1)) & 1
+
+		# shift right, insert feedback at MSB
+		reg = (reg >> 1) | (fb << (m - 1))
+		reg &= mask
+	return out
+
+def bits_to_steps(bits01: list[int]) -> list[int]:
+	return [PSK // 2 if b else 0 for b in bits01]
+
+PREAMBLE_STEPS = bits_to_steps(lfsr_bits(PREAMBLE_SYMS, seed=0x5D, taps=(7, 6)))
+SYNCWORD_STEPS = bits_to_steps(lfsr_bits(SYNCWORD_SYMS, seed=0x6B, taps=(7, 6)))

filejack-1.0.0/filejack/__init__.py

@@ -0,0 +1,6 @@
+from .conversions import bytes_to_symbols, symbols_to_bytes
+from .encode_frames import build_frames, encode_frames, encode_data
+from .decode_frames import decode_frames
+from .decode_data import decode_data
+from .merge_frames import merge_frames, save_fjf, load_fjf
+from .reconstruct_data import reconstruct_data

filejack-1.0.0/filejack/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+
+raise SystemExit(main())

filejack-1.0.0/filejack/cli.py

@@ -0,0 +1,96 @@
+import argparse
+import os
+import zlib
+
+import numpy as np
+from scipy.io import wavfile
+
+from .decode_data import decode_data
+from .encode_frames import encode_data
+from .merge_frames import save_fjf
+from .reconstruct_data import reconstruct_data
+from .values import *
+
+
+def encode_command(args) -> int:
+	output = args.output or args.input + ".wav"
+
+	data = open(args.input, 'rb').read()
+
+	waveform = encode_data(data)
+
+	if args.mono:
+		wavfile.write(output, SAMPLE_RATE, waveform)
+	else:
+		stereo = np.column_stack([waveform, -waveform])
+		wavfile.write(output, SAMPLE_RATE, stereo)
+
+	print(f"Encoded {len(data)} bytes into {output}")
+	print("Input CRC32:", zlib.crc32(data))
+	return 0
+
+
+def decode_command(args) -> int:
+	output = args.output
+	if output is None:
+		base, ext = os.path.splitext(args.input)
+		output = base + ".out"
+
+	fs_in, rx = wavfile.read(args.input)
+	rx = rx.astype(np.float32)
+
+	if fs_in != SAMPLE_RATE:
+		print(f"Expected sample rate {SAMPLE_RATE}, got {fs_in}")
+		return 1
+
+	if rx.ndim == 2:
+		# Anti-phase stereo: subtracting the channels doubles the signal and cancels common noise
+		rx = (rx[:, 0] - rx[:, 1]) / 2
+
+	frames_payload, total_expected, stats = decode_data(rx, threads_num=args.threads, progress=True)
+
+	if total_expected is None:
+		print("No frames decoded.")
+		return 1
+
+	if args.fjf:
+		save_fjf(total_expected, frames_payload, args.fjf)
+		print(f"Saved {len(frames_payload)} frames into {args.fjf}")
+
+	data = reconstruct_data(frames_payload, total_expected)
+
+	open(output, 'wb').write(data)
+
+	print("Total frames expected:", total_expected)
+	print("Frames received:", len(frames_payload))
+	print(f"Decoded {len(data)} bytes into {output}")
+	print("Output CRC32:", zlib.crc32(data))
+
+	if len(frames_payload) < total_expected:
+		return 1
+	return 0
+
+
+def main() -> int:
+	parser = argparse.ArgumentParser(prog="filejack", description="File encoding/decoding to audio signals.")
+	subparsers = parser.add_subparsers(dest="command", required=True)
+
+	encode_parser = subparsers.add_parser("encode", help="Encode a file into a WAV audio signal.")
+	encode_parser.add_argument("input", help="Path of the file to encode.")
+	encode_parser.add_argument("output", nargs='?', default=None, help="Path of the output WAV file. Defaults to <input>.wav")
+	encode_parser.add_argument("--mono", action="store_true", help="Write a mono WAV instead of anti-phase stereo.")
+	encode_parser.set_defaults(func=encode_command)
+
+	decode_parser = subparsers.add_parser("decode", help="Decode a WAV audio signal back into a file.")
+	decode_parser.add_argument("input", help="Path of the WAV file to decode.")
+	decode_parser.add_argument("output", nargs='?', default=None, help="Path of the output file. Defaults to <input without extension>.out")
+	decode_parser.add_argument("--fjf", default=None, help="Also save decoded frames into a .fjf file at this path.")
+	decode_parser.add_argument("--threads", type=int, default=12, help="Number of decoding threads. Defaults to 12.")
+	decode_parser.set_defaults(func=decode_command)
+
+	args = parser.parse_args()
+	return args.func(args)
+
+
+if __name__ == '__main__':
+	raise SystemExit(main())