syncoder 0.0.4a2__tar.gz → 0.0.5a0__tar.gz

{syncoder-0.0.4a2 → syncoder-0.0.5a0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: syncoder
-Version: 0.0.4a2
+Version: 0.0.5a0
 Summary: A codec for DNA data storage
 Project-URL: Homepage, https://example.com/TBD
 Project-URL: Issues, https://example.com/TBD/issues

syncoder-0.0.5a0/environments/conda_environment.yml

@@ -0,0 +1,10 @@
+name: codec-user
+channels:
+  - conda-forge 
+dependencies:
+  - python>=3.13
+  - numpy=2.2.6
+  - numba=0.61.*
+  - pip:
+    - galois>=0.4.6
+

syncoder-0.0.5a0/environments/conda_environment_dev.yml

@@ -0,0 +1,21 @@
+name: codec-dev
+channels:
+  - conda-forge 
+dependencies:
+  - python>=3.13
+  - pip
+  - "libblas=*=*mkl" 
+  - numpy=2.2.6 # version pinned for galois 0.4.6
+  - matplotlib=3.*
+  - numba=0.61.* # version pinned for galois 0.4.6
+  - pytest
+  - sphinx=7.*
+  - sphinx_rtd_theme
+  - biopython
+  - jupyter
+  - ipykernel
+  - twine
+  - pip:
+    - galois>=0.4.6
+    - build
+

syncoder-0.0.5a0/environments/conda_environment_test.yml

@@ -0,0 +1,15 @@
+name: codec-test
+channels:
+  - conda-forge 
+dependencies:
+  - python>=3.13
+  - pip
+  - numpy=2.2.6 # version pinned for galois 0.4.6
+  - numba=0.61.* # version pinned for galois 0.4.6
+  - pytest
+  - sphinx=7.*
+  - sphinx_rtd_theme
+  - twine
+  - pip:
+    - galois>=0.4.6
+    - build

{syncoder-0.0.4a2 → syncoder-0.0.5a0}/src/syncoder/__init__.py

@@ -3,4 +3,4 @@ from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
 default_b32_alphabet = _default_b32_alphabet
 default_b32_alphabet_alt = _default_b32_alphabet_alt
 
-__version__ = "0.0.4a2"
+__version__ = "0.0.5a0"

{syncoder-0.0.4a2 → syncoder-0.0.5a0}/src/syncoder/codec.py

@@ -124,7 +124,9 @@ class BaseNBlockCodec:
                  inner_n:Optional[int]=None, 
                  n_redundant_strands:int=5, 
                  n_strands:int=100,
-                 max_strand_index:Optional[int]=None): 
+                 max_strand_index:Optional[int]=None,
+                 index_type:Literal["binary", "inner symbol", "inner"]="binary",
+                 index_location:Literal["middle", "beginning", "end"]="middle"): 
         """  
         Initialize the Codec object.
 
@@ -135,6 +137,16 @@ class BaseNBlockCodec:
             n_redundant_strands (int): Number of redundant strands.
             n_strands (int): Number of "strands" stored by this codec including redundant strands.
             max_strand_index (Optional[int]): Maximum guaranteed index. Default: n_strands.
+            index_type (str): Type of index to use. Options: "binary", "inner symbol", "inner". Default: "binary".
+            index_location (str): Location of the index within the inner code. Options: "middle", "beginning", "end". Default: "middle".
+
+            index_type "binary" integrates the index into the data stream as a binary number.  
+                This may help packing efficiency.
+            index_type "inner symbol"/"inner" integrates the index inner code symbol symbols.
+                This may help packing efficiency and always simplifies index extraction.
+            index_location "middle" (default) places the index after the data symbols but before the redundancy symbols.
+            index_location "beginning" places the index before the data symbols.
+            index_location "end" places the index after the redundancy symbols. (not implemented)
         """
         if n_strands<=255:
             self.outer_alphabet_size_bytes = 1
@@ -147,6 +159,20 @@ class BaseNBlockCodec:
             max_strand_index = n_strands
         if max_strand_index<n_strands:   
             raise ValueError("max_strand_index must be greater than or equal to n_strands.")
+        
+        if index_type not in {"binary","inner symbol","inner"}:
+            raise ValueError("index_type must be 'binary' or 'inner symbol'.")
+        #"inner" and "inner symbol" are synomymous.
+        self.index_type = "inner" if index_type == "inner symbol" else index_type
+
+        if index_location in {"middle","beginning","end"}:
+            self.index_location = index_location
+            if index_location == "end":
+                raise NotImplementedError("index_location 'end' is not implemented.")
+            if index_location != "middle" and index_type == "binary":
+                raise ValueError("index_location 'beginning' or 'end' not supported for index_type 'binary'.")
+        else:
+            raise ValueError("index_location must be 'middle', 'beginning', or 'end'.")
 
         self.n_message_strands = n_strands - n_redundant_strands
 
@@ -178,25 +204,45 @@ class BaseNBlockCodec:
         logging.debug("bits per strand used for indexing: {}".format(self.index_bits))
 
         #message length in bytes (without index)
-        self.data_chunk_size = int(k*np.log2(q)-self.index_bits) // 8
+        if self.index_type == "binary":
+            self.data_chunk_size = int(k*np.log2(q)-self.index_bits) // 8
+        elif self.index_type == "inner":
+            self.index_symbols = len(_int_to_baseN(max_strand_index,q))
+            self.data_symbols = k-self.index_symbols
+            self.data_chunk_size = int(self.data_symbols*np.log2(q)) // 8
+            assert k == self.data_symbols + self.index_symbols
+            logging.debug("data chunk size in symbols: {}".format(self.data_symbols))
+            logging.debug(f"number of inner symbols: {k}; index symbols: {self.index_symbols}")
+
         logging.debug( "strand capacity in bytes: {}".format(self.data_chunk_size) )
+        #todo: only execute if logging level is debug or warning
+        self.__compute_waste()
 
-        _wasted_bits = k*np.log2(q) - (self.data_chunk_size*8+self.index_bits )
-        logging.debug( "{} bits wasted per strand".format( _wasted_bits ) )
-        logging.debug( "Maximum strand index (using wasted bits): {}".format(2**(int(self.index_bits+_wasted_bits))))
-        _wasted_symbols = _wasted_bits/np.log2(q)
-        if  _wasted_symbols>=1:
-            logging.warning( "{} symbols wasted per strand.  Consider decreasing inner_n.".format( _wasted_symbols) )
-        else:
-            logging.debug("{} symbols wasted per strand".format( _wasted_symbols))
- 
-         
         self.block_capacity_bytes = self.data_chunk_size*(self.n_strands - n_redundant_strands) 
         logging.debug( "codec message size: {} bytes".format(self.block_capacity_bytes) )
 
         #init the outer coder 
         self.outer_coder = reedsolo.RSCodec(n_redundant_strands, c_exp=8*self.outer_alphabet_size_bytes)
 
+    def __compute_waste(self):
+        k = self.inner_k
+        q = self.inner_coder.field.order
+        if self.index_type == "binary":
+            _wasted_bits = k*np.log2(q) - (self.data_chunk_size*8+self.index_bits )
+            logging.debug( "{} bits wasted per strand".format( _wasted_bits ) )
+            logging.debug( "Maximum strand index (using wasted bits): {}".format(2**(int(self.index_bits+_wasted_bits))))
+            _wasted_symbols = _wasted_bits/np.log2(q)
+        elif self.index_type == "inner":
+            data_chunk_symbols_used = self.data_chunk_size*8/np.log2(q)  
+            _wasted_symbols = self.data_symbols - data_chunk_symbols_used
+            logging.debug( f"Maximum strand index:{q**self.index_symbols}")
+        else:
+            raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
+        if  _wasted_symbols>=1:
+            logging.warning( "{} symbols wasted per strand.  Consider decreasing inner_n.".format( _wasted_symbols) )
+        else:
+            logging.debug("{} symbols wasted per strand".format( _wasted_symbols))
+
     def encode(self,data:bytes,index_start:int=0)->list[list[int]]:
         """ Encodes the given data using the outer and inner codes.
 
@@ -231,14 +277,18 @@ class BaseNBlockCodec:
         #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrr>|
         #  ...
         #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrr>|
-        #  3. inner code  (R= inner code redundancy applied to rows, * = inner code symbol)
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
+        #  3. inner code  (R= inner code redundancy applied to rows)
+        #                 (* = inner code symbol)
+        #                 (@= binary index concatenated to data bits)
+        #                 (a= binary index concatenated to outer code redundancy bits)
+        #     When index type is inner code, @ and a are full symbols.
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
         #  ...
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
-        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrrRRRRRRRR>|
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
+        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrraaaRRRRRRRR>|
         #  ...
-        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrrRRRRRRRR>|
+        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrraaaRRRRRRRR>|
 
         #1. Reshape data 
         logging.debug("applying outer code to {} bytes".format(len(data)))
@@ -273,11 +323,23 @@ class BaseNBlockCodec:
             d = chunk.tobytes()
             #use little endian to match x86/x64 byte order and avoid needing to pad byte string.
             chunk = int.from_bytes(d,'little') 
-            #add index to end (MS bits)
-            chunk += index<<(self.data_chunk_size*8)
+            if self.index_type == "binary":
+                #add index to end (MS bits)
+                chunk += index<<(self.data_chunk_size*8)
+
+                #but.. we still need to pad out the symbol stream.
+                chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)
+            elif self.index_type == "inner":
+                chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.data_symbols)
+                index_symbols = _int_to_baseN(index,self.inner_coder.field.order,length=self.index_symbols)
+                if self.index_location == "middle":
+                    chunk += index_symbols
+                elif self.index_location == "beginning":
+                    chunk = index_symbols + chunk
+                else:
+                    raise NotImplementedError("index_location 'end' is not implemented.")
+
             index += 1
-            #but.. we still need to pad out the symbol stream.
-            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)  
             chunks.append(chunk)
 
         #apply inner code
@@ -313,12 +375,23 @@ class BaseNBlockCodec:
         ordered_chunks:list[bytes|None] = [None] * self.n_strands
         chunk_errors:list[int] = [-1] * self.n_strands
         for chunk in chunked_data:
-            chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
-            chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
-            _mask = (2 ** (self.data_chunk_size * 8)) - 1
-            chunk_int = chunk_int & _mask
+            if self.index_type == "binary":
+                chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
+                chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
+                _mask = (2 ** (self.data_chunk_size * 8)) - 1
+                chunk_int = chunk_int & _mask
+            elif self.index_type == "inner":
+                if self.index_location == "middle":
+                    chunk_int = _baseN_to_int(chunk[0][:self.data_symbols].tolist(),self.inner_coder.field.order)
+                    chunk_index = _baseN_to_int(chunk[0][self.data_symbols:].tolist(),self.inner_coder.field.order) - index_start
+                elif self.index_location == "beginning":
+                    chunk_index = _baseN_to_int(chunk[0][:-self.data_symbols].tolist(),self.inner_coder.field.order) - index_start
+                    chunk_int = _baseN_to_int(chunk[0][-self.data_symbols:].tolist(),self.inner_coder.field.order)
+                else:
+                    raise NotImplementedError("index_location 'end' is not implemented.")
+            else:
+                raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
             chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
-            #print(chunk_index)
             try:
               ordered_chunks[chunk_index] = chunk_bytes
               chunk_errors[chunk_index] = chunk[1]
@@ -381,27 +454,26 @@ class BaseNBlockCodec:
         Returns:
             int: The index. or -1 if error_check==True and unrecoverable error found.
         """
-        word_len:int = len(words[0])
+        symbol_len:int = len(words[0])
         if isinstance(dna,bytes):
             dna= dna.decode()
         dna_s:str = cast(str,dna)
-        
-        if padding == "right": 
-            _p = self.inner_k*word_len - len(dna_s)
-            if _p>0: #if i have less than a full codeword of bases
-                dna_s = dna_s + "A"*_p  
-        elif padding == "left":
-            _p = self.inner_k*word_len - len(dna_s)
-            if _p>0: #if i have less than a full codeword of bases
-                dna_s = "A"*_p + dna_s 
 
+        num_missing = self.inner_k*symbol_len - len(dna_s) # in DNA bases
+        if num_missing>0: #if i have less than a full codeword of bases
+            if padding == "right": 
+                dna_s = dna_s + "A"*num_missing
+            elif padding == "left":
+                dna_s = "A"*num_missing + dna_s
+
+        #This could fail if padding makes a symbol not in words or alternate_words.
         data = dna_to_bN([dna_s],words,alternate_words)[0]
          
         return self.extract_index(data,error_check)
 
     def extract_index(self, data: list[int], error_check: bool = False) -> int:
         """
-        Extracts the index from the given DNA strand.
+        Extracts the index from the given inner code sequence.
 
         Args:
             data list[int]: The input data from which to extract the index.
@@ -415,11 +487,23 @@ class BaseNBlockCodec:
         if error_check == False:
             data = data[0:self.inner_k] #extract message
         else:
-            data, n_corrected = self.inner_coder.decode(data,errors=True)[0].tolist()
+            data_gf, n_corrected = self.inner_coder.decode(data,errors=True)
+            data = data_gf.tolist()
             if n_corrected < 0:
                 return -1
-        chunk_int = _baseN_to_int(data, self.inner_coder.field.order)
-        chunk_index = (chunk_int >> (self.data_chunk_size * 8))
+        if self.index_type == "binary":
+            chunk_int = _baseN_to_int(data, self.inner_coder.field.order)
+            chunk_index = (chunk_int >> (self.data_chunk_size * 8))
+        elif self.index_type == "inner":
+            if self.index_location == "middle":
+                chunk_index_baseN = data[self.data_symbols:] 
+            elif self.index_location == "beginning":
+                chunk_index_baseN = data[:-self.data_symbols]
+            else:
+                raise NotImplementedError("index_location 'end' is not implemented.")
+            chunk_index = _baseN_to_int(chunk_index_baseN, self.inner_coder.field.order)
+        else:
+            raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
         return chunk_index
 
 

{syncoder-0.0.4a2 → syncoder-0.0.5a0}/tests/test_codec_pytest.py

@@ -3,6 +3,7 @@
 import random
 import numpy as np
 from itertools import product
+from typing import Literal
 
 import syncoder
 codec = syncoder
@@ -12,11 +13,27 @@ codec = syncoder
 #numba_logger.setLevel(logging.WARNING)
 
 def test_encode_decode(): 
-  c = codec.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=30)
+  encode_decode(index_type="binary")
+  encode_decode(index_type="inner")
+  encode_decode(index_type="inner", index_loc="beginning")
+
+def encode_decode(index_type,index_loc:Literal["beginning","middle"]="middle"):
+  c = codec.BaseNBlockCodec(inner_alphabet_size=32,
+                            inner_d=5,
+                            inner_n=30,
+                            index_type=index_type,
+                            index_location=index_loc)
 
   in_text = (codec.lipsum + codec.lipsum)[0:c.block_capacity_bytes]
 
   coded = c.encode( in_text )
+  assert c.extract_index(coded[27]) == 27 #test extract index while we're at it
+  dnadata = syncoder.b32_to_DNA(coded,syncoder._default_b32_alphabet,syncoder._default_b32_alphabet_alt)
+  assert c.extract_index_dna(dnadata[27],
+                            words = syncoder._default_b32_alphabet,
+                            alternate_words= syncoder._default_b32_alphabet_alt,
+                            error_check=True) == 27
+
   random.shuffle(coded)
   #coded = coded[:-4] #erase a random strands
   corrupt_index = random.randint(0,len(coded[0])) 
@@ -76,6 +93,7 @@ def test_dna_to_bytes():
   dna_bytes, mask = codec.dna_to_bytes(DNA,alphabet,alphabet_alt)
   
   dna_baseN = _int_to_baseN(int.from_bytes(dna_bytes,"little"),len(alphabet))
+  #TODO: this can fail but only for some inputs randomly selected lookinto why.
   assert dna_baseN == answer32
 
   _t = np.array(dna_baseN)

syncoder-0.0.4a2/environments/conda_environment.yml

@@ -1,12 +0,0 @@
-name: codec-user
-channels:
-  - conda-forge 
-dependencies:
-  - python>=3.12
-  - numpy=2.*
-  - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
-  - pip:
-    - galois==0.4.*
-

syncoder-0.0.4a2/environments/conda_environment_dev.yml

@@ -1,20 +0,0 @@
-name: codec-dev
-channels:
-  - conda-forge 
-dependencies:
-  - python>=3.12
-  - numpy=2.*
-  - matplotlib=3.*
-  - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
-  - sphinx=7.*
-  - sphinx_rtd_theme
-  - biopython
-  - jupyter
-  - ipykernel
-  - pip:
-    - galois==0.4.*
-    - build
-    - twine
-

syncoder-0.0.4a2/environments/conda_environment_test.yml

@@ -1,14 +0,0 @@
-name: codec-test
-channels:
-  - conda-forge 
-dependencies:
-  - python>=3.12
-  - numpy=2.*
-  - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
-  - sphinx=7.*
-  - sphinx_rtd_theme
-  - pip:
-    - galois==0.4.*
-