syncoder 0.0.1a1__tar.gz → 0.0.3a1__tar.gz

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/.gitignore

@@ -13,5 +13,7 @@ out/
 *.zip
 *.jpg
 *.bin
+.pypirc
+*.bat
 #any actual py files are in src.  anything in root is supposed to just for development.
 /*.py

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: syncoder
-Version: 0.0.1a1
+Version: 0.0.3a1
 Summary: A codec for DNA data storage
 Project-URL: Homepage, https://example.com/TBD
 Project-URL: Issues, https://example.com/TBD/issues
@@ -8,6 +8,9 @@ Author-email: Christopher Takahashi <cnt@uw.edu>
 License-Expression: BSD-3-Clause
 License-File: LICENSE
 Requires-Python: >=3.9
+Requires-Dist: galois>=0.4
+Requires-Dist: numpy>=1.20
+Requires-Dist: reedsolo>=1.7
 Description-Content-Type: text/markdown
 
 # SyNCODER

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/environments/conda_environment.yml

@@ -2,7 +2,7 @@ name: codec-user
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - pytest
   - reedsolo=1.7.*

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/environments/conda_environment_dev.yml

@@ -2,7 +2,7 @@ name: codec-dev
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - matplotlib=3.*
   - pytest

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/environments/conda_environment_test.yml

@@ -2,7 +2,7 @@ name: codec-test
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - pytest
   - reedsolo=1.7.*

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/pyproject.toml

@@ -1,6 +1,11 @@
 [project]
 name = "syncoder"
-version = "0.0.1a1"
+dynamic = ["version"]
+dependencies = [
+    "numpy>=1.20",
+    "galois>=0.4",
+    "reedsolo>=1.7"
+]
 description = "A codec for DNA data storage"
 authors = [
     { name="Christopher Takahashi", email="cnt@uw.edu" }
@@ -24,4 +29,7 @@ build-backend = "hatchling.build"
 [tool.pytest.ini_options]
 testpaths = [
     "tests"
-]
+]
+
+[tool.hatch.version]
+path = "src/syncoder/__init__.py"

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/src/syncoder/__init__.py

@@ -1,4 +1,6 @@
 from .codec import *
 from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
 default_b32_alphabet = _default_b32_alphabet
-default_b32_alphabet_alt = _default_b32_alphabet_alt
+default_b32_alphabet_alt = _default_b32_alphabet_alt
+
+__version__ = "0.0.3a1"

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/src/syncoder/codec.py

@@ -62,15 +62,39 @@ def _baseN_to_int(digits: list[int], base: int) -> int:
         n = n*base + d
     return n
 
-def computeNStrands(nbytes:int, 
+def compute_num_strands(nbytes:int, 
                  inner_alphabet_size:int=32, 
                  inner_d:int=5, 
                  inner_n:Optional[int]=None, 
                  n_redundant_strands:int=5)->int:
-    
+    """
+    Computes the number of strands needed to encode a given number of bytes.
+
+    Args:
+        nbytes (int): The number of bytes to encode.
+        inner_alphabet_size (int, optional): Size of the inner alphabet (must be a prime power). Defaults to 32.
+        inner_d (int, optional): Inner codeword distance. Defaults to 5.
+        inner_n (Optional[int], optional): Number of symbols in inner block (n < inner_alphabet_size). Defaults to inner_alphabet_size-1.
+        n_redundant_strands (int, optional): Number of redundant strands. Defaults to 5.
+
+    Returns:
+        int: The minimum number of strands required to encode the given number of bytes.
+    """
     #TODO: do some algebra to solve for n_strands.
-    min_m=1
-    max_m=2**16-1
+    if inner_n is None:
+        inner_n = inner_alphabet_size - 1
+    
+    k = inner_n - (inner_d - 1)
+    bits_per_strand_max = np.ceil(np.log2(inner_alphabet_size))*k
+    bits_per_strand_min = np.floor(np.log2(inner_alphabet_size))*k-16 #16 bits for largest possible index.
+    #min_m=1
+    min_m = int( np.floor(nbytes*8 / bits_per_strand_max + n_redundant_strands) )
+    #max_m=2**16-1
+    max_m = int( np.ceil(nbytes*8 / bits_per_strand_min + n_redundant_strands) )
+    
+    _logger = logging.getLogger()
+    _t = _logger.getEffectiveLevel()
+    _logger.setLevel(logging.ERROR) #suppress debug messages
     while min_m<max_m:
         mid = (min_m+max_m)//2
         codec = BaseNBlockCodec(inner_alphabet_size=inner_alphabet_size,
@@ -82,14 +106,8 @@ def computeNStrands(nbytes:int,
             min_m = mid+1
         else:
             max_m = mid
+    _logger.setLevel(_t) #restore logger level
     return min_m
-    #index_bits = np.ceil(np.log2(n_strands)).astype(int)
-    #k = inner_n - (inner_d - 1)
-    #data_chunk_size = int(k*np.log2(inner_alphabet_size)-index_bits) // 8
-    #block_capacity_bytes = data_chunk_size*(n_strands - n_redundant_strands)
-
-
-
 
 class BaseNBlockCodec:
     def __init__(self, 
@@ -110,8 +128,6 @@ class BaseNBlockCodec:
             n_strands (int): Number of "strands" stored by this codec including redundant strands.
             max_strand_index (Optional[int]): Maximum guaranteed index. Default: n_strands.
         """
-        #TODO: handle more than 255 total strands.
-        #this was formally a parameter but probably better to keep it fixed or automatically calculated.
         if n_strands<=255:
             self.outer_alphabet_size_bytes = 1
         else:
@@ -119,6 +135,10 @@ class BaseNBlockCodec:
         if n_strands>(2**(8*self.outer_alphabet_size_bytes)-1):
             raise ValueError("Too many strands for outer code.")
         self.n_strands = n_strands
+        if max_strand_index is None:
+            max_strand_index = n_strands
+        if max_strand_index<n_strands:   
+            raise ValueError("max_strand_index must be greater than or equal to n_strands.")
 
         self.n_message_strands = n_strands - n_redundant_strands
 
@@ -142,10 +162,11 @@ class BaseNBlockCodec:
         #                                        ^ = self.index_bits <-- in bits not bytes.
         #  note: actual k remains alphabet_size-1, but the code is shorted (0 padded) to inner_n symbols. 
         k = inner_n - (inner_d - 1) 
+        self.inner_k = k
         q = self.inner_coder.field.order
 
         #number of bytes needed to store the index
-        self.index_bits = np.ceil(np.log2(self.n_strands)).astype(int)
+        self.index_bits = np.ceil(np.log2(max_strand_index)).astype(int)
         logging.debug("bits per strand used for indexing: {}".format(self.index_bits))
 
         #message length in bytes (without index)
@@ -248,7 +269,7 @@ class BaseNBlockCodec:
             chunk += index<<(self.data_chunk_size*8)
             index += 1
             #but.. we still need to pad out the symbol stream.
-            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_coder.k)
+            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)  
             chunks.append(chunk)
 
         #apply inner code
@@ -285,7 +306,7 @@ class BaseNBlockCodec:
         chunk_errors:list[int] = [-1] * self.n_strands
         for chunk in chunked_data:
             chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
-            chunk_index = chunk_int >> (self.data_chunk_size * 8) - index_start
+            chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
             _mask = (2 ** (self.data_chunk_size * 8)) - 1
             chunk_int = chunk_int & _mask
             chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
@@ -386,7 +407,7 @@ def b32_to_DNA_optimize(file_data:list[ArrayLike],words:list[str], alternate_wor
     mask: list of ints. 0 for word, 1 for alternate word, -1 for don't care.  None for no mask.
     nmasked: number of masked words.  -1 for all.
     penalty_fn: function that takes a DNA sequence and returns a score.  penalty_fn(DNA_seq:str)->int
-  """
+  """      
   if nmasked<0: #mask all
     return [b32_to_DNA_optimize_single(x,words,alternate_words,mask,penalty_fn=penalty_fn) for x in file_data]
   else:
@@ -398,7 +419,6 @@ def b32_to_DNA_optimize(file_data:list[ArrayLike],words:list[str], alternate_wor
 def b32_to_DNA_optimize_single(strand_data:ArrayLike, words:list[str], alternate_words:list[str], mask:Union[ArrayLike,None]=None ,penalty_fn=None)->Tuple[bytes,int]:
   """
     TODO: rename to bN_to_DNA_optimize_single
-    TODO: add mask support.
     strand_data: 1d list of integral types
     mask: list of ints. 0 for word, 1 for alternate word, -1 for don't care.  None for no mask.
   """

{syncoder-0.0.1a1 → syncoder-0.0.3a1}/tests/test_codec_pytest.py

@@ -4,7 +4,8 @@ import random
 import numpy as np
 from itertools import product
 
-import syncoder as codec
+import syncoder
+codec = syncoder
 
 #logging.basicConfig(level=logging.DEBUG,stream=sys.stdout)
 #numba_logger = logging.getLogger('numba')
@@ -23,12 +24,14 @@ def test_encode_decode():
 
   out_text = c.decode( coded )[0]
   assert in_text == out_text
+  #check inner_n is causing the correct number of bases
+  assert len(coded[0]) == 30 
 
 def test_coded_to_bases():
   #TODO: test optimize too
 
   test_encoded_data = np.random.randint(0,32,(10,31),dtype=np.uint8)
-  DNA_with_scores = codec.b32_to_DNA_optimize(test_encoded_data,codec._default_b32_alphabet,codec._default_b32_alphabet_alt)
+  DNA_with_scores = codec.b32_to_DNA_optimize(test_encoded_data,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) # type: ignore
   DNA = [x[0] for x in DNA_with_scores]
   test_decoded_data  = codec.dna_to_b32(DNA,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) 
   print(test_encoded_data.flatten())
@@ -89,7 +92,6 @@ def test_inserts():
   total_ibytes = len(insert_bytes)*n_inserts
   data_size = coder.block_capacity_bytes - total_ibytes 
 
-
   test_data = np.random.randint(0,255,data_size,dtype=np.uint8).tobytes()
 
   inserted_data = codec.insert_bytes(test_data,insert_bytes,coder.data_chunk_size,n_inserts)
@@ -99,9 +101,26 @@ def test_inserts():
   data_decodec,__oe,__oer,__ie = coder.decode(enc_d)
   assert data_decodec == inserted_data
 
-  test_dna_out = codec.b32_to_DNA_optimize(enc_d,codec._default_b32_alphabet,codec._default_b32_alphabet_alt,mask=imask,nmasked=n_inserts)
+  test_dna_out = codec.b32_to_DNA_optimize(enc_d,codec._default_b32_alphabet,codec._default_b32_alphabet_alt,mask=imask,nmasked=n_inserts) # type: ignore
   test_dna = [x[0] for x in test_dna_out]
   for strand in test_dna[:n_inserts]:
     assert strand.decode().startswith(insert)
 
   assert codec.dna_to_bN(test_dna,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) == enc_d
+
+def test_compute_n_strands():
+  nbytes = 26600
+  ns = syncoder.compute_num_strands(nbytes,32,5,29,100)
+  data = np.random.randint(0,255,nbytes,dtype=np.uint8).tobytes()
+  coder = codec.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=29,n_strands=ns,n_redundant_strands=100)
+  enc_d = coder.encode(data)
+  assert len(enc_d) == ns
+
+def test_nonzero_index():
+  coder = syncoder.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=31,n_strands=10,n_redundant_strands=3,max_strand_index=3000)
+  data_size = coder.block_capacity_bytes
+
+  test_data = np.random.randint(0,255,data_size,dtype=np.uint8).tobytes()
+  enc_d = coder.encode(test_data,index_start=2500)
+  ded_d = coder.decode(enc_d,index_start=2500)
+  assert ded_d[0] == test_data