syncoder 0.0.2a1__tar.gz → 0.0.4a1__tar.gz

{syncoder-0.0.2a1 → syncoder-0.0.4a1}/.gitignore

@@ -14,5 +14,6 @@ out/
 *.jpg
 *.bin
 .pypirc
+*.bat
 #any actual py files are in src.  anything in root is supposed to just for development.
 /*.py

{syncoder-0.0.2a1 → syncoder-0.0.4a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: syncoder
-Version: 0.0.2a1
+Version: 0.0.4a1
 Summary: A codec for DNA data storage
 Project-URL: Homepage, https://example.com/TBD
 Project-URL: Issues, https://example.com/TBD/issues

{syncoder-0.0.2a1 → syncoder-0.0.4a1}/src/syncoder/__init__.py

@@ -3,4 +3,4 @@ from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
 default_b32_alphabet = _default_b32_alphabet
 default_b32_alphabet_alt = _default_b32_alphabet_alt
 
-__version__ = "0.0.2a1"
+__version__ = "0.0.4a1"

{syncoder-0.0.2a1 → syncoder-0.0.4a1}/src/syncoder/codec.py

@@ -3,7 +3,7 @@ import struct
 import itertools
 import logging
 import typing
-from typing import Optional,Union, Callable, Tuple  #requires python 3.5 or later
+from typing import Optional,Union, Callable, Tuple, Literal, cast  #requires python 3.8 or later
 
 
 import numpy as np
@@ -62,15 +62,39 @@ def _baseN_to_int(digits: list[int], base: int) -> int:
         n = n*base + d
     return n
 
-def computeNStrands(nbytes:int, 
+def compute_num_strands(nbytes:int, 
                  inner_alphabet_size:int=32, 
                  inner_d:int=5, 
                  inner_n:Optional[int]=None, 
                  n_redundant_strands:int=5)->int:
-    
+    """
+    Computes the number of strands needed to encode a given number of bytes.
+
+    Args:
+        nbytes (int): The number of bytes to encode.
+        inner_alphabet_size (int, optional): Size of the inner alphabet (must be a prime power). Defaults to 32.
+        inner_d (int, optional): Inner codeword distance. Defaults to 5.
+        inner_n (Optional[int], optional): Number of symbols in inner block (n < inner_alphabet_size). Defaults to inner_alphabet_size-1.
+        n_redundant_strands (int, optional): Number of redundant strands. Defaults to 5.
+
+    Returns:
+        int: The minimum number of strands required to encode the given number of bytes.
+    """
     #TODO: do some algebra to solve for n_strands.
-    min_m=1
-    max_m=2**16-1
+    if inner_n is None:
+        inner_n = inner_alphabet_size - 1
+    
+    k = inner_n - (inner_d - 1)
+    bits_per_strand_max = np.ceil(np.log2(inner_alphabet_size))*k
+    bits_per_strand_min = np.floor(np.log2(inner_alphabet_size))*k-16 #16 bits for largest possible index.
+    #min_m=1
+    min_m = int( np.floor(nbytes*8 / bits_per_strand_max + n_redundant_strands) )
+    #max_m=2**16-1
+    max_m = int( np.ceil(nbytes*8 / bits_per_strand_min + n_redundant_strands) )
+    
+    _logger = logging.getLogger()
+    _t = _logger.getEffectiveLevel()
+    _logger.setLevel(logging.ERROR) #suppress debug messages
     while min_m<max_m:
         mid = (min_m+max_m)//2
         codec = BaseNBlockCodec(inner_alphabet_size=inner_alphabet_size,
@@ -82,14 +106,8 @@ def computeNStrands(nbytes:int,
             min_m = mid+1
         else:
             max_m = mid
+    _logger.setLevel(_t) #restore logger level
     return min_m
-    #index_bits = np.ceil(np.log2(n_strands)).astype(int)
-    #k = inner_n - (inner_d - 1)
-    #data_chunk_size = int(k*np.log2(inner_alphabet_size)-index_bits) // 8
-    #block_capacity_bytes = data_chunk_size*(n_strands - n_redundant_strands)
-
-
-
 
 class BaseNBlockCodec:
     def __init__(self, 
@@ -110,8 +128,6 @@ class BaseNBlockCodec:
             n_strands (int): Number of "strands" stored by this codec including redundant strands.
             max_strand_index (Optional[int]): Maximum guaranteed index. Default: n_strands.
         """
-        #TODO: handle more than 255 total strands.
-        #this was formally a parameter but probably better to keep it fixed or automatically calculated.
         if n_strands<=255:
             self.outer_alphabet_size_bytes = 1
         else:
@@ -119,6 +135,10 @@ class BaseNBlockCodec:
         if n_strands>(2**(8*self.outer_alphabet_size_bytes)-1):
             raise ValueError("Too many strands for outer code.")
         self.n_strands = n_strands
+        if max_strand_index is None:
+            max_strand_index = n_strands
+        if max_strand_index<n_strands:   
+            raise ValueError("max_strand_index must be greater than or equal to n_strands.")
 
         self.n_message_strands = n_strands - n_redundant_strands
 
@@ -140,13 +160,13 @@ class BaseNBlockCodec:
         #  sizes in bytes:
         #  |<---- self.data_chunk_size ----->|<- . ->|
         #                                        ^ = self.index_bits <-- in bits not bytes.
-        #  note: actual k remains alphabet_size-1, but the code is shorted (0 padded) to inner_n symbols. 
+        #  note: actual k remains alphabet_size-1, but the code is shortened (0 padded) to inner_n symbols. 
         k = inner_n - (inner_d - 1) 
         self.inner_k = k
         q = self.inner_coder.field.order
 
         #number of bytes needed to store the index
-        self.index_bits = np.ceil(np.log2(self.n_strands)).astype(int)
+        self.index_bits = np.ceil(np.log2(max_strand_index)).astype(int)
         logging.debug("bits per strand used for indexing: {}".format(self.index_bits))
 
         #message length in bytes (without index)
@@ -286,7 +306,7 @@ class BaseNBlockCodec:
         chunk_errors:list[int] = [-1] * self.n_strands
         for chunk in chunked_data:
             chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
-            chunk_index = chunk_int >> (self.data_chunk_size * 8) - index_start
+            chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
             _mask = (2 ** (self.data_chunk_size * 8)) - 1
             chunk_int = chunk_int & _mask
             chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
@@ -336,6 +356,63 @@ class BaseNBlockCodec:
                 logging.warning("Outer code errors found on strand {}, byte {}.".format(*e))
 
         return data_decoded, erasures, errors, chunk_errors
+    
+    def extract_index_dna(self, dna: str|bytes, words:list[str], alternate_words:list[str], error_check: bool = False, padding: Literal["left","right"]="right") -> int:
+        """
+        Extracts the index from the given DNA strand.
+
+        Args:
+            dna (str|bytes): The input DNA strand from which to extract the index.
+            error_check (bool, optional): If True, the inner_codec will be used to 
+                attempt to check for errors. Defaults to False.
+            padding Literal["left","right"], optional): Assume any missing bases are left or right of index.  Defaults to "right".
+                Note padded sequences are unlikely to error correct properly.
+            words (list[str]): The primary set of base words used, in order, for encoding.
+            alternate_words (list[str]): The alternate (aka synonymous) set of words used, in order, for encoding.
+
+        Returns:
+            int: The index. or -1 if error_check==True and unrecoverable error found.
+        """
+        word_len:int = len(words[0])
+        if isinstance(dna,bytes):
+            dna= dna.decode()
+        dna_s:str = cast(str,dna)
+        
+        if padding == "right": 
+            _p = self.inner_k*word_len - len(dna_s)
+            if _p>0: #if i have less than a full codeword of bases
+                dna_s = dna_s + "A"*_p  
+        elif padding == "left":
+            _p = self.inner_k*word_len - len(dna_s)
+            if _p>0: #if i have less than a full codeword of bases
+                dna_s = "A"*_p + dna_s 
+
+        data = dna_to_bN([dna_s],words,alternate_words)[0]
+         
+        return self.extract_index(data,error_check)
+
+    def extract_index(self, data: list[int], error_check: bool = False) -> int:
+        """
+        Extracts the index from the given DNA strand.
+
+        Args:
+            data list[int]: The input data from which to extract the index.
+                If a list[int] is given, it is assumed be be inner code symbols. 
+            error_check (bool, optional): If True, the inner_codec will be used to 
+                attempt to check for errors. Defaults to False.
+
+        Returns:
+            int: The index. or -1 if error_check==True and unrecoverable error found.
+        """
+        if error_check == False:
+            data = data[0:self.inner_k] #extract message
+        else:
+            data, n_corrected = self.inner_coder.decode(data,errors=True)[0].tolist()
+            if n_corrected < 0:
+                return -1
+        chunk_int = _baseN_to_int(data, self.inner_coder.field.order)
+        chunk_index = (chunk_int >> (self.data_chunk_size * 8))
+        return chunk_index
 
 
 #convert base N into DNA.

{syncoder-0.0.2a1 → syncoder-0.0.4a1}/tests/test_codec_pytest.py

@@ -4,7 +4,8 @@ import random
 import numpy as np
 from itertools import product
 
-import syncoder as codec
+import syncoder
+codec = syncoder
 
 #logging.basicConfig(level=logging.DEBUG,stream=sys.stdout)
 #numba_logger = logging.getLogger('numba')
@@ -91,7 +92,6 @@ def test_inserts():
   total_ibytes = len(insert_bytes)*n_inserts
   data_size = coder.block_capacity_bytes - total_ibytes 
 
-
   test_data = np.random.randint(0,255,data_size,dtype=np.uint8).tobytes()
 
   inserted_data = codec.insert_bytes(test_data,insert_bytes,coder.data_chunk_size,n_inserts)
@@ -107,3 +107,34 @@ def test_inserts():
     assert strand.decode().startswith(insert)
 
   assert codec.dna_to_bN(test_dna,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) == enc_d
+
+def test_compute_n_strands():
+  nbytes = 26600
+  ns = syncoder.compute_num_strands(nbytes,32,5,29,100)
+  data = np.random.randint(0,255,nbytes,dtype=np.uint8).tobytes()
+  coder = codec.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=29,n_strands=ns,n_redundant_strands=100)
+  enc_d = coder.encode(data)
+  assert len(enc_d) == ns
+
+def test_nonzero_index():
+  coder = syncoder.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=31,n_strands=10,n_redundant_strands=3,max_strand_index=3000)
+  data_size = coder.block_capacity_bytes
+
+  test_data = np.random.randint(0,255,data_size,dtype=np.uint8).tobytes()
+  enc_d = coder.encode(test_data,index_start=2500)
+  ded_d = coder.decode(enc_d,index_start=2500)
+  assert ded_d[0] == test_data
+
+  dna = codec.b32_to_DNA_optimize(enc_d,codec._default_b32_alphabet,codec._default_b32_alphabet_alt)  #type: ignore
+
+  index = coder.extract_index_dna(dna[0][0],
+                                  words = syncoder._default_b32_alphabet,
+                                  alternate_words= syncoder._default_b32_alphabet_alt,
+                                  error_check=False)
+  assert index == 2500
+  offset = 5
+  index = coder.extract_index_dna(dna[offset][0],
+                                  words = syncoder._default_b32_alphabet,
+                                  alternate_words= syncoder._default_b32_alphabet_alt,
+                                  error_check=False)
+  assert index == 2500+offset