syncoder 0.0.0__tar.gz → 0.0.2a1__tar.gz

syncoder-0.0.2a1/.gitignore

@@ -0,0 +1,18 @@
+#some temp and build directories
+__pycache__
+dist/*
+Docs/
+data/
+out/
+#ingnore data files that may be used or generated during dev.
+*.ipynb
+*.dna
+*.pdf
+*.txt
+*.json
+*.zip
+*.jpg
+*.bin
+.pypirc
+#any actual py files are in src.  anything in root is supposed to just for development.
+/*.py

syncoder-0.0.2a1/PKG-INFO

@@ -0,0 +1,42 @@
+Metadata-Version: 2.4
+Name: syncoder
+Version: 0.0.2a1
+Summary: A codec for DNA data storage
+Project-URL: Homepage, https://example.com/TBD
+Project-URL: Issues, https://example.com/TBD/issues
+Author-email: Christopher Takahashi <cnt@uw.edu>
+License-Expression: BSD-3-Clause
+License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: galois>=0.4
+Requires-Dist: numpy>=1.20
+Requires-Dist: reedsolo>=1.7
+Description-Content-Type: text/markdown
+
+# SyNCODER
+
+## Project Overview
+A DNA codec. (Fill in.)
+
+## Features
+Fill in here.
+
+## Installation
+`codec.py` Can be included in a project and used as a local module or installed and used from `pip` :
+```
+pip install --pre syncoder
+```
+Note the requirement for the `--pre` flag as there is no official post release yet.
+
+## Usage
+To be completed.  For now see `tests/test_codec_pytest.py` for examples
+
+## Testing
+It is sufficient to run 
+```
+pytest
+```
+from the root directory.  Currently coverage is only for features and parameters in active use.  More tests to come.
+
+## License
+BSD-3 clause.  see `LICENSE` file in this directory.

syncoder-0.0.2a1/README.md

@@ -0,0 +1,27 @@
+# SyNCODER
+
+## Project Overview
+A DNA codec. (Fill in.)
+
+## Features
+Fill in here.
+
+## Installation
+`codec.py` Can be included in a project and used as a local module or installed and used from `pip` :
+```
+pip install --pre syncoder
+```
+Note the requirement for the `--pre` flag as there is no official post release yet.
+
+## Usage
+To be completed.  For now see `tests/test_codec_pytest.py` for examples
+
+## Testing
+It is sufficient to run 
+```
+pytest
+```
+from the root directory.  Currently coverage is only for features and parameters in active use.  More tests to come.
+
+## License
+BSD-3 clause.  see `LICENSE` file in this directory.

{syncoder-0.0.0 → syncoder-0.0.2a1/environments}/conda_environment.yml

@@ -2,7 +2,7 @@ name: codec-user
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - pytest
   - reedsolo=1.7.*

{syncoder-0.0.0 → syncoder-0.0.2a1/environments}/conda_environment_dev.yml

@@ -2,7 +2,7 @@ name: codec-dev
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - matplotlib=3.*
   - pytest
@@ -13,7 +13,8 @@ dependencies:
   - biopython
   - jupyter
   - ipykernel
-  - build
   - pip:
     - galois==0.4.*
+    - build
+    - twine
 

{syncoder-0.0.0 → syncoder-0.0.2a1/environments}/conda_environment_test.yml

@@ -2,7 +2,7 @@ name: codec-test
 channels:
   - conda-forge 
 dependencies:
-  - python=3.12
+  - python>=3.12
   - numpy=2.*
   - pytest
   - reedsolo=1.7.*

{syncoder-0.0.0 → syncoder-0.0.2a1}/pyproject.toml

@@ -1,6 +1,11 @@
 [project]
 name = "syncoder"
-version = "0.0.0"
+dynamic = ["version"]
+dependencies = [
+    "numpy>=1.20",
+    "galois>=0.4",
+    "reedsolo>=1.7"
+]
 description = "A codec for DNA data storage"
 authors = [
     { name="Christopher Takahashi", email="cnt@uw.edu" }
@@ -19,4 +24,12 @@ packages = ["syncoder"]
 
 [build-system]
 requires = ["hatchling >= 1.26"]
-build-backend = "hatchling.build"
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests"
+]
+
+[tool.hatch.version]
+path = "src/syncoder/__init__.py"

syncoder-0.0.2a1/src/syncoder/__init__.py

@@ -0,0 +1,6 @@
+from .codec import *
+from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
+default_b32_alphabet = _default_b32_alphabet
+default_b32_alphabet_alt = _default_b32_alphabet_alt
+
+__version__ = "0.0.2a1"

{syncoder-0.0.0 → syncoder-0.0.2a1}/src/syncoder/codec.py

@@ -2,7 +2,8 @@ import os
 import struct
 import itertools
 import logging
-from typing import Optional,Union, Callable  #requires python 3.5 or later
+import typing
+from typing import Optional,Union, Callable, Tuple  #requires python 3.5 or later
 
 
 import numpy as np
@@ -18,7 +19,7 @@ import reedsolo
 
 
 lipsum = b"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum." #cspell:disable-line
-  
+
 
 def _int_to_baseN(n:int,base:int,length:int=-1)->list[int]:
     """Converts an integer to a base N number in little endian order.
@@ -141,6 +142,7 @@ class BaseNBlockCodec:
         #                                        ^ = self.index_bits <-- in bits not bytes.
         #  note: actual k remains alphabet_size-1, but the code is shorted (0 padded) to inner_n symbols. 
         k = inner_n - (inner_d - 1) 
+        self.inner_k = k
         q = self.inner_coder.field.order
 
         #number of bytes needed to store the index
@@ -247,7 +249,7 @@ class BaseNBlockCodec:
             chunk += index<<(self.data_chunk_size*8)
             index += 1
             #but.. we still need to pad out the symbol stream.
-            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_coder.k)
+            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)  
             chunks.append(chunk)
 
         #apply inner code
@@ -255,7 +257,7 @@ class BaseNBlockCodec:
 
         return ic_chunks
 
-    def decode(self, data: list[list[int]], index_start: int = 0) -> bytes:
+    def decode(self, data: list[list[int]], index_start: int = 0) -> Tuple[bytes,ArrayLike,ArrayLike,ArrayLike]:
         """
         Decodes the given data.
 
@@ -274,18 +276,20 @@ class BaseNBlockCodec:
         """
         # inner decode
         ## TODO: handle erasures in inner code.
-        chunked_data = [self.inner_coder.decode(d,errors=True) for d in data]
+
+        # I'm sure it'll be a (FieldArray,int) ----------------V
+        chunked_data:list[Tuple[galois.FieldArray,int]] = [self.inner_coder.decode(d,errors=True) for d in data] #type:ignore
         
 
         # handle outer decode
-        ordered_chunks = [None] * self.n_strands
-        chunk_errors = [-1] * self.n_strands
+        ordered_chunks:list[bytes|None] = [None] * self.n_strands
+        chunk_errors:list[int] = [-1] * self.n_strands
         for chunk in chunked_data:
             chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
             chunk_index = chunk_int >> (self.data_chunk_size * 8) - index_start
             _mask = (2 ** (self.data_chunk_size * 8)) - 1
             chunk_int = chunk_int & _mask
-            chunk_bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
+            chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
             #print(chunk_index)
             try:
               ordered_chunks[chunk_index] = chunk_bytes
@@ -299,6 +303,8 @@ class BaseNBlockCodec:
             if c is None:
                 erasures.append(i)  # remember where erasure is
                 ordered_chunks[i] = b'\x00' * self.data_chunk_size  # fill in erasures with zeros
+        #we're all bytes now, but the checker can't tell that.
+        ordered_chunks_bytes:list[bytes] = typing.cast(list[bytes],ordered_chunks)
 
         if self.outer_coder.c_exp==8:
             outer_dtype = np.uint8
@@ -306,7 +312,7 @@ class BaseNBlockCodec:
             outer_dtype = np.uint16
         else:
             raise ValueError("Unsupported outer code byte size.")
-        data_np_encoded = np.array([np.frombuffer(c, dtype=outer_dtype) for c in ordered_chunks]).transpose()
+        data_np_encoded = np.array([np.frombuffer(c, dtype=outer_dtype) for c in ordered_chunks_bytes]).transpose()
         data_decoded_results = [self.outer_coder.decode(dc, erase_pos=erasures) for dc in data_np_encoded]
         data_decoded_chunks = np.array([d[0] for d in data_decoded_results],dtype=outer_dtype).transpose()
         data_decoded = data_decoded_chunks.flatten().tobytes()
@@ -336,7 +342,7 @@ class BaseNBlockCodec:
 _default_b32_alphabet = ['GCT', 'ACT', 'AGT', 'CTC', 'TGC', 'GAG', 'GCA', 'ATG', 'AGA', 'AGC', 'CAC', 'CAT', 'CAG', 'CTG', 'TCT', 'GAC', 'GTC', 'GTG', 'ACA', 'ACG', 'ATC', 'CTA', 'CGA', 'CGT', 'TAC', 'TAG', 'TCA', 'TCG', 'TGA', 'TGT', 'GAT', 'GTA']
 _default_b32_alphabet_alt = ['ATA', 'AAC', 'CCT', 'TAT', 'CCG', 'TGG', 'CGC', 'CGG', 'GTT', 'GGT', 'GCG', 'TCC', 'CCA', 'GGA', 'ACC', 'AGG', 'TAA', 'GCC', 'AAA', 'CCC', 'GGG', 'GGC', 'TTA', 'TTT', 'GAA', 'AAG', 'CTT', 'TTG', 'TTC', 'AAT', 'CAA', 'ATT']
 
-def find_avoid(seq:str, avoids:list)->tuple[int]:
+def find_avoid(seq:str, avoids:list)->tuple[int,int]:
     """find index and length of avoided sequence """
     for a in avoids:
         loc = seq.find(a)
@@ -374,39 +380,52 @@ def longest_binder(a:str,b:str):
 
 
 
-def b32_to_DNA_optimize(file_data:list[list[int]],words:list[str], alternate_words:list[str], mask:Union[list[int],None]=None, penalty_fn:Union[Callable[[str],int],None]=None)->list[str]:
+def b32_to_DNA_optimize(file_data:list[ArrayLike],words:list[str], alternate_words:list[str], mask:Union[ArrayLike,None]=None, nmasked=-1, penalty_fn:Union[Callable[[str],int],None]=None)->list[Tuple[bytes,int]]:
   """ 
     TODO: rename to bN_to_DNA_optimize, and add test.
+    file_data: 2d list of integral types (symbols)
     mask: list of ints. 0 for word, 1 for alternate word, -1 for don't care.  None for no mask.
+    nmasked: number of masked words.  -1 for all.
     penalty_fn: function that takes a DNA sequence and returns a score.  penalty_fn(DNA_seq:str)->int
-  """
-  return [b32_to_DNA_optimize_single(x,words,alternate_words,penalty_fn=penalty_fn) for x in file_data]
+  """      
+  if nmasked<0: #mask all
+    return [b32_to_DNA_optimize_single(x,words,alternate_words,mask,penalty_fn=penalty_fn) for x in file_data]
+  else:
+      masked_part = [b32_to_DNA_optimize_single(x,words,alternate_words,mask,penalty_fn=penalty_fn) for x in file_data[0:nmasked]]
+      unmasked_part = [b32_to_DNA_optimize_single(x,words,alternate_words,mask=None,penalty_fn=penalty_fn) for x in file_data[nmasked:]]
+      return masked_part + unmasked_part
 
 
-def b32_to_DNA_optimize_single(strand_data:list[int],words:list[str], alternate_words:list[str], mask:Union[list[int],None]=None ,penalty_fn=None)->str:
+def b32_to_DNA_optimize_single(strand_data:ArrayLike, words:list[str], alternate_words:list[str], mask:Union[ArrayLike,None]=None ,penalty_fn=None)->Tuple[bytes,int]:
   """
     TODO: rename to bN_to_DNA_optimize_single
-    TODO: add mask support.
+    strand_data: 1d list of integral types
     mask: list of ints. 0 for word, 1 for alternate word, -1 for don't care.  None for no mask.
   """
+  strand_data = np.array(strand_data)
+  
   if mask is not None:
-    mutable_ind = np.where(np.array(mask)==-1)
+    mask = np.array(mask)
+    _m = np.full(len(strand_data),-1,dtype=np.int8)
+    _m[:len(mask)] = np.array(mask.copy()) #Probably don't need this copy.
+    mask = _m
+    mutable_ind = np.where(mask==-1)
+    mask[mutable_ind] = 0
     #raise NotImplementedError("mask not implemented yet")
   else:
+    mask = np.zeros(len(strand_data),dtype=np.uint8)
     mutable_ind = np.arange(len(strand_data))
-    mask = mask.copy()
-    mask[mutable_ind] = 0
-  
-  strand_data = np.array(strand_data)
-  words = np.array(list(zip(words,alternate_words)),dtype="S")
+    
+  words_lookup = np.array(list(zip(words,alternate_words)),dtype="S")
   picks = np.zeros(len(strand_data),dtype=np.uint8)
   picks[0:len(mask)] = mask
-  dna_seq = b"".join(words[strand_data,picks])
+  dna_seq = b"".join(words_lookup[strand_data,picks])
   if penalty_fn is None:
      return dna_seq,0
   score = penalty_fn(dna_seq)
 
   #greedy search, stop when you can't find a better solution one step away.
+  #this may end on a local minimum.
   icount = 0
   while True:
     iteration_best_picks = picks.copy()
@@ -415,7 +434,7 @@ def b32_to_DNA_optimize_single(strand_data:list[int],words:list[str], alternate_
     for i in mutable_ind:
       new_picks = picks.copy()
       new_picks[i] = not new_picks[i]
-      new_dna_seq = b"".join(words[strand_data,new_picks])
+      new_dna_seq = b"".join(words_lookup[strand_data,new_picks])
       new_score = penalty_fn(new_dna_seq)
       if new_score < iteration_best_score:
         iteration_best_picks = new_picks
@@ -457,11 +476,11 @@ def b32_to_DNA(file_data:list[list[int]],words:list[str], alternate_words:list[s
         dna.append(_s)
     return dna
 
-def dna_to_bN(dna: Union[list[str],list[bytes]] ,words:list[str], alternate_words:list[str])->list[int]:
-    """ takes a list of strings and converts them to a list of base N ints"""
+def dna_to_bN(dna: Union[list[str],list[bytes]] ,words:list[str], alternate_words:list[str])->list[list[int]]:
+    """ takes a list of strings and converts them to a list of lists of base N ints"""
 
-    if isinstance(dna[0],bytes):
-        dna = [x.decode() for x in dna]
+    if isinstance(dna[0],bytes): #assume the rest are bytes too.
+        dna = [x.decode() for x in typing.cast(list[bytes],dna)]
 
     wordlen = len(words[0])
     #build the reverse lookup table
@@ -471,16 +490,16 @@ def dna_to_bN(dna: Union[list[str],list[bytes]] ,words:list[str], alternate_word
         bNlut[alternate_words[ind]] = ind
 
 
-    bNdatalist = []
+    bNdatalist:list[list[int]] = []
     for d in dna:
-        bNdata = []
+        bNdata:list[int] = []
         for ind in range(0,len(d),wordlen):
             w = d[ind:ind+wordlen] 
             bNdata.append(bNlut[w])
         bNdatalist.append(bNdata)
     return bNdatalist
 
-def dna_to_b32(dna: Union[list[str],list[bytes]] ,words:list[str], alternate_words:list[str])->list[int]:
+def dna_to_b32(dna: Union[list[str],list[bytes]] ,words:list[str], alternate_words:list[str])->list[list[int]]:
     assert len(words) == 32
     return dna_to_bN(dna,words,alternate_words)
 
@@ -602,15 +621,21 @@ def remove_bytes(data:bytes, insert:int|bytes, chunk_len:int, num_inserts:int) -
     Returns:
         bytes: The modified byte array with the inserted bytes removed.
     """
+    have_insert_seq = False
     if isinstance(insert,bytes):
         insert_len = len(insert)
+        have_insert_seq = True
+    elif isinstance(insert,int):
+        if insert <= 0:
+            raise ValueError("insert must be a positive integer")   
+        insert_len:int = insert
     else:
-        insert = None
+        raise ValueError("insert must be a bytes or int")
     
     data = bytearray(data)
     for i in reversed(range(num_inserts)):
         ins_index = i * chunk_len
-        if insert is not None:
+        if have_insert_seq:
             if data[ins_index:ins_index + insert_len] != insert:
                 raise ValueError("insert bytes do not match bytes to be removed")
         del data[ins_index:ins_index + insert_len]

{syncoder-0.0.0 → syncoder-0.0.2a1}/tests/test_codec_pytest.py

@@ -4,7 +4,7 @@ import random
 import numpy as np
 from itertools import product
 
-import codec
+import syncoder as codec
 
 #logging.basicConfig(level=logging.DEBUG,stream=sys.stdout)
 #numba_logger = logging.getLogger('numba')
@@ -23,12 +23,14 @@ def test_encode_decode():
 
   out_text = c.decode( coded )[0]
   assert in_text == out_text
+  #check inner_n is causing the correct number of bases
+  assert len(coded[0]) == 30 
 
 def test_coded_to_bases():
   #TODO: test optimize too
 
-  test_encoded_data = np.random.randint(0,32,(10,31))
-  DNA_with_scores = codec.b32_to_DNA_optimize(test_encoded_data,codec._default_b32_alphabet,codec._default_b32_alphabet_alt)
+  test_encoded_data = np.random.randint(0,32,(10,31),dtype=np.uint8)
+  DNA_with_scores = codec.b32_to_DNA_optimize(test_encoded_data,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) # type: ignore
   DNA = [x[0] for x in DNA_with_scores]
   test_decoded_data  = codec.dna_to_b32(DNA,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) 
   print(test_encoded_data.flatten())
@@ -54,6 +56,8 @@ def test_dna_to_bN():
   assert np.array_equal(data, decode_data)
 
 def test_dna_to_bytes():
+  _int_to_baseN = codec.codec._int_to_baseN
+
   #TODO: adapt to workwith not b32 too.
   #answer = [64, 79, 231, 126, 228, 124, 3]
   alphabet =codec._default_b32_alphabet
@@ -70,7 +74,7 @@ def test_dna_to_bytes():
 
   dna_bytes, mask = codec.dna_to_bytes(DNA,alphabet,alphabet_alt)
   
-  dna_baseN = codec._int_to_baseN(int.from_bytes(dna_bytes,"little"),len(alphabet))
+  dna_baseN = _int_to_baseN(int.from_bytes(dna_bytes,"little"),len(alphabet))
   assert dna_baseN == answer32
 
   _t = np.array(dna_baseN)
@@ -78,3 +82,28 @@ def test_dna_to_bytes():
   generated_dna = "".join(full_alphabet[dna_baseN_with_alt])
  
   assert generated_dna == DNA
+
+def test_inserts():
+  coder = codec.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=29,n_strands=2000,n_redundant_strands=100)
+  n_inserts = 200
+  insert = "GCTCTTCCCACCACCATTGCCTTCTTCTTG"
+  insert_bytes, imask = codec.dna_to_bytes(insert,codec._default_b32_alphabet,codec._default_b32_alphabet_alt)
+  total_ibytes = len(insert_bytes)*n_inserts
+  data_size = coder.block_capacity_bytes - total_ibytes 
+
+
+  test_data = np.random.randint(0,255,data_size,dtype=np.uint8).tobytes()
+
+  inserted_data = codec.insert_bytes(test_data,insert_bytes,coder.data_chunk_size,n_inserts)
+  assert len(inserted_data)  == coder.block_capacity_bytes
+
+  enc_d = coder.encode(inserted_data)
+  data_decodec,__oe,__oer,__ie = coder.decode(enc_d)
+  assert data_decodec == inserted_data
+
+  test_dna_out = codec.b32_to_DNA_optimize(enc_d,codec._default_b32_alphabet,codec._default_b32_alphabet_alt,mask=imask,nmasked=n_inserts) # type: ignore
+  test_dna = [x[0] for x in test_dna_out]
+  for strand in test_dna[:n_inserts]:
+    assert strand.decode().startswith(insert)
+
+  assert codec.dna_to_bN(test_dna,codec._default_b32_alphabet,codec._default_b32_alphabet_alt) == enc_d

syncoder-0.0.0/.gitignore

@@ -1,11 +0,0 @@
-__pycache__/*
-Docs/
-*.ipynb
-*.dna
-*.pdf
-*.txt
-*.json
-*.zip
-*.jpg
-*.bin
-debugging.py

syncoder-0.0.0/PKG-INFO

@@ -1,13 +0,0 @@
-Metadata-Version: 2.4
-Name: syncoder
-Version: 0.0.0
-Summary: A codec for DNA data storage
-Project-URL: Homepage, https://example.com/TBD
-Project-URL: Issues, https://example.com/TBD/issues
-Author-email: Christopher Takahashi <cnt@uw.edu>
-License-Expression: BSD-3-Clause
-License-File: LICENSE
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-
-Readme coming soon.

syncoder-0.0.0/README.md

@@ -1 +0,0 @@
-Readme coming soon.

syncoder-0.0.0/foo.py

@@ -1,4 +0,0 @@
-
-from src import *
-
-print(nextfn(5))

syncoder-0.0.0/src/syncoder/__init__.py

@@ -1 +0,0 @@
-__all__ = ["bar.nextfn"]