syncoder 0.0.4a2__tar.gz → 0.0.6a0__tar.gz

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/PKG-INFO

@@ -1,16 +1,15 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: syncoder
-Version: 0.0.4a2
+Version: 0.0.6a0
 Summary: A codec for DNA data storage
 Project-URL: Homepage, https://example.com/TBD
 Project-URL: Issues, https://example.com/TBD/issues
 Author-email: Christopher Takahashi <cnt@uw.edu>
 License-Expression: BSD-3-Clause
 License-File: LICENSE
-Requires-Python: >=3.9
-Requires-Dist: galois>=0.4
-Requires-Dist: numpy>=1.20
-Requires-Dist: reedsolo>=1.7
+Requires-Python: >=3.11
+Requires-Dist: galois>=0.4.8
+Requires-Dist: numpy>=2
 Description-Content-Type: text/markdown
 
 # SyNCODER
@@ -20,6 +19,8 @@ A DNA codec. (Fill in.)
 
 ## Features
 Fill in here.
+### DDSA Sector 0/1 generator
+(Status: incomplete) Generates [sector 0](https://www.snia.org/standards/technology-standards-software/standards-portfolio/dna-data-storage-sector-zero) and [sector 1](https://www.snia.org/standards/technology-standards-software/standards-portfolio/dna-data-storage-sector-one) compliant with the DNA Data Storange Alliance (DDSA) [DNA Archive Rosetta Stone specifiation](https://dnastoragealliance.org/publications/).  Note: The maximum sector 1 size is currenty 325950 bytes (limited by reserved index size).
 
 ## Installation
 `codec.py` Can be included in a project and used as a local module or installed and used from `pip` :

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/README.md

@@ -1,27 +1,29 @@
-# SyNCODER
-
-## Project Overview
-A DNA codec. (Fill in.)
-
-## Features
-Fill in here.
-
-## Installation
-`codec.py` Can be included in a project and used as a local module or installed and used from `pip` :
-```
-pip install --pre syncoder
-```
-Note the requirement for the `--pre` flag as there is no official post release yet.
-
-## Usage
-To be completed.  For now see `tests/test_codec_pytest.py` for examples
-
-## Testing
-It is sufficient to run 
-```
-pytest
-```
-from the root directory.  Currently coverage is only for features and parameters in active use.  More tests to come.
-
-## License
-BSD-3 clause.  see `LICENSE` file in this directory.
+# SyNCODER
+
+## Project Overview
+A DNA codec. (Fill in.)
+
+## Features
+Fill in here.
+### DDSA Sector 0/1 generator
+(Status: incomplete) Generates [sector 0](https://www.snia.org/standards/technology-standards-software/standards-portfolio/dna-data-storage-sector-zero) and [sector 1](https://www.snia.org/standards/technology-standards-software/standards-portfolio/dna-data-storage-sector-one) compliant with the DNA Data Storange Alliance (DDSA) [DNA Archive Rosetta Stone specifiation](https://dnastoragealliance.org/publications/).  Note: The maximum sector 1 size is currenty 325950 bytes (limited by reserved index size).
+
+## Installation
+`codec.py` Can be included in a project and used as a local module or installed and used from `pip` :
+```
+pip install --pre syncoder
+```
+Note the requirement for the `--pre` flag as there is no official post release yet.
+
+## Usage
+To be completed.  For now see `tests/test_codec_pytest.py` for examples
+
+## Testing
+It is sufficient to run 
+```
+pytest
+```
+from the root directory.  Currently coverage is only for features and parameters in active use.  More tests to come.
+
+## License
+BSD-3 clause.  see `LICENSE` file in this directory.

syncoder-0.0.6a0/environments/conda_environment.yml

@@ -0,0 +1,10 @@
+name: codec-user
+channels:
+  - conda-forge 
+dependencies:
+  - python>=3.13
+  - numpy=2.*
+  - numba=0.62.*
+  - pip:
+    - galois>=0.4.8
+

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/environments/conda_environment_dev.yml

@@ -2,19 +2,20 @@ name: codec-dev
 channels:
   - conda-forge 
 dependencies:
-  - python>=3.12
-  - numpy=2.*
+  - python>=3.13
+  - pip
+  - "libblas=*=*mkl" 
+  - numpy 
   - matplotlib=3.*
+  - numba=0.62.* # version pinned for galois 0.4.8
   - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
   - sphinx=7.*
   - sphinx_rtd_theme
   - biopython
   - jupyter
   - ipykernel
+  - twine
   - pip:
-    - galois==0.4.*
+    - galois>=0.4.8
     - build
-    - twine
 

syncoder-0.0.6a0/environments/conda_environment_pipeline.yml

@@ -0,0 +1,14 @@
+name: codec-pipeline
+channels:
+  - bioconda
+  - conda-forge 
+dependencies:
+  - bioconda::pandaseq
+  - bioconda::cutadapt
+  - bioconda::vsearch
+  - python>=3.13
+  - numpy=2.*
+  - numba=0.62.*
+  - pip:
+    - galois>=0.4.8
+

syncoder-0.0.6a0/environments/conda_environment_test.yml

@@ -0,0 +1,15 @@
+name: codec-test
+channels:
+  - conda-forge 
+dependencies:
+  - python>=3.13
+  - pip
+  - numpy=2.2.6 
+  - numba=0.62.* # version pinned for galois 0.4.8
+  - pytest
+  - sphinx=7.*
+  - sphinx_rtd_theme
+  - twine
+  - pip:
+    - galois>=0.4.8
+    - build

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/pyproject.toml

@@ -1,35 +1,34 @@
-[project]
-name = "syncoder"
-dynamic = ["version"]
-dependencies = [
-    "numpy>=1.20",
-    "galois>=0.4",
-    "reedsolo>=1.7"
-]
-description = "A codec for DNA data storage"
-authors = [
-    { name="Christopher Takahashi", email="cnt@uw.edu" }
-]
-readme = "README.md"
-requires-python = ">=3.9"
-license = "BSD-3-Clause"
-license-files = ["LICENSE"]
-
-[project.urls]
-Homepage = "https://example.com/TBD"
-Issues = "https://example.com/TBD/issues"
-
-[tool.setuptools]
-packages = ["syncoder"]
-
-[build-system]
-requires = ["hatchling >= 1.26"]
-build-backend = "hatchling.build"
-
-[tool.pytest.ini_options]
-testpaths = [
-    "tests"
-]
-
-[tool.hatch.version]
+[project]
+name = "syncoder"
+dynamic = ["version"]
+dependencies = [
+    "numpy>=2",
+    "galois>=0.4.8" 
+]
+description = "A codec for DNA data storage"
+authors = [
+    { name="Christopher Takahashi", email="cnt@uw.edu" }
+]
+readme = "README.md"
+requires-python = ">=3.11"
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+
+[project.urls]
+Homepage = "https://example.com/TBD"
+Issues = "https://example.com/TBD/issues"
+
+[tool.setuptools]
+packages = ["syncoder"]
+
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests"
+]
+
+[tool.hatch.version]
 path = "src/syncoder/__init__.py"

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/src/syncoder/__init__.py

@@ -1,6 +1,6 @@
-from .codec import *
-from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
-default_b32_alphabet = _default_b32_alphabet
-default_b32_alphabet_alt = _default_b32_alphabet_alt
-
-__version__ = "0.0.4a2"
+from .codec import *
+from .codec import _default_b32_alphabet, _default_b32_alphabet_alt
+default_b32_alphabet = _default_b32_alphabet
+default_b32_alphabet_alt = _default_b32_alphabet_alt
+
+__version__ = "0.0.6a0"

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/src/syncoder/codec.py

@@ -1,8 +1,7 @@
-import os
-import struct
-import itertools
 import logging
 import typing
+import json
+from . import sector01
 from typing import Optional,Union, Callable, Tuple, Literal, cast  #requires python 3.8 or later
 
 
@@ -11,17 +10,7 @@ from numpy.typing import ArrayLike
 
 import galois  
 
-#import a version of reedsolo that supports multi instance
-import importlib.metadata
-import packaging.version
-rsvers = importlib.metadata.version('reedsolo')
-if packaging.version.Version(rsvers) <= packaging.version.Version("2.1.3b1"):
-    from . import reedsoloMulti as reedsolo
-else:
-    import reedsolo 
-
 #### ROADMAP ####
-# - integrate the baseN to DNA function here.  (generalize from b32fn)
 # - Strand prefix sub-encoder.
 # - Sample objective functions (at least minimize homopolymers and avoid annealing.)
 
@@ -124,7 +113,9 @@ class BaseNBlockCodec:
                  inner_n:Optional[int]=None, 
                  n_redundant_strands:int=5, 
                  n_strands:int=100,
-                 max_strand_index:Optional[int]=None): 
+                 max_strand_index:Optional[int]=None,
+                 index_type:Literal["binary", "inner symbol", "inner"]="binary",
+                 index_location:Literal["middle", "beginning", "end"]="middle"): 
         """  
         Initialize the Codec object.
 
@@ -135,6 +126,15 @@ class BaseNBlockCodec:
             n_redundant_strands (int): Number of redundant strands.
             n_strands (int): Number of "strands" stored by this codec including redundant strands.
             max_strand_index (Optional[int]): Maximum guaranteed index. Default: n_strands.
+            index_type (str): Type of index to use. Options: "binary", "inner symbol", "inner". Default: "binary".
+                index_type "binary" integrates the index into the data stream as a binary number.  
+                    This may help packing efficiency.
+                index_type "inner symbol"/"inner" integrates the index inner code symbol symbols.
+                    This may help packing efficiency and always simplifies index extraction.
+            index_location (str): Location of the index within the inner code. Options: "middle", "beginning", "end". Default: "middle".
+                index_location "middle" (default) places the index after the data symbols but before the redundancy symbols.
+                index_location "beginning" places the index before the data symbols.
+                index_location "end" places the index after the redundancy symbols. (not implemented)
         """
         if n_strands<=255:
             self.outer_alphabet_size_bytes = 1
@@ -147,6 +147,21 @@ class BaseNBlockCodec:
             max_strand_index = n_strands
         if max_strand_index<n_strands:   
             raise ValueError("max_strand_index must be greater than or equal to n_strands.")
+        self.max_strand_index = max_strand_index
+        
+        if index_type not in {"binary","inner symbol","inner"}:
+            raise ValueError("index_type must be 'binary' or 'inner symbol'.")
+        #"inner" and "inner symbol" are synomymous.
+        self.index_type = "inner" if index_type == "inner symbol" else index_type
+
+        if index_location in {"middle","beginning","end"}:
+            self.index_location = index_location
+            if index_location == "end":
+                raise NotImplementedError("index_location 'end' is not implemented.")
+            if index_location != "middle" and index_type == "binary":
+                raise ValueError("index_location 'beginning' or 'end' not supported for index_type 'binary'.")
+        else:
+            raise ValueError("index_location must be 'middle', 'beginning', or 'end'.")
 
         self.n_message_strands = n_strands - n_redundant_strands
 
@@ -157,6 +172,7 @@ class BaseNBlockCodec:
             #      make sure extra sequence numbers are not added to inner blocks.  and or maybe add a 
             #      parameter for the number of inner blocks per strand.
             assert inner_n<inner_alphabet_size
+        self.inner_n = inner_n
         
         self.inner_coder = galois.ReedSolomon(n=inner_alphabet_size-1,d=inner_d,field=galois.GF(inner_alphabet_size))
         #  in units of symbols (there are q=inner_alphabet_size possible symbols)
@@ -173,29 +189,127 @@ class BaseNBlockCodec:
         self.inner_k = k
         q = self.inner_coder.field.order
 
-        #number of bytes needed to store the index
-        self.index_bits = np.ceil(np.log2(max_strand_index)).astype(int)
-        logging.debug("bits per strand used for indexing: {}".format(self.index_bits))
 
         #message length in bytes (without index)
-        self.data_chunk_size = int(k*np.log2(q)-self.index_bits) // 8
+        if self.index_type == "binary":
+            #number of bytes needed to store the index
+            self.index_bits = np.ceil(np.log2(max_strand_index)).astype(int)
+            logging.debug("bits per strand used for indexing: {}".format(self.index_bits))
+            self.data_chunk_size = int(k*np.log2(q)-self.index_bits) // 8
+        elif self.index_type == "inner":
+            self.index_symbols = len(_int_to_baseN(max_strand_index,q))
+            self.data_symbols = k-self.index_symbols
+            self.data_chunk_size = int(self.data_symbols*np.log2(q)) // 8
+            assert k == self.data_symbols + self.index_symbols
+            logging.debug("data chunk size in symbols: {}".format(self.data_symbols))
+            logging.debug(f"number of inner symbols: {k}; index symbols: {self.index_symbols}")
+
         logging.debug( "strand capacity in bytes: {}".format(self.data_chunk_size) )
+        #todo: only execute if logging level is debug or warning
+        self.__compute_waste()
+
+        self.block_capacity_bytes = self.data_chunk_size*(self.n_strands - n_redundant_strands) 
+        logging.debug( "codec message size: {} bytes".format(self.block_capacity_bytes) )
 
-        _wasted_bits = k*np.log2(q) - (self.data_chunk_size*8+self.index_bits )
-        logging.debug( "{} bits wasted per strand".format( _wasted_bits ) )
-        logging.debug( "Maximum strand index (using wasted bits): {}".format(2**(int(self.index_bits+_wasted_bits))))
-        _wasted_symbols = _wasted_bits/np.log2(q)
+        #init the outer coder 
+        if self.outer_alphabet_size_bytes==2 and self.data_chunk_size%2!=0:
+            raise ValueError("data_chunk_size must be even for 16 bit outer code.")
+        #c=0 to match reedsolo library's default for code compatibility with prior versions of this codec.
+        #   This may be changed in a future major version.
+        self.outer_coder_fast = galois.ReedSolomon(n=2**(self.outer_alphabet_size_bytes*8)-1,
+                                                   d=n_redundant_strands+1,
+                                                   c=0,
+                                                   field=galois.GF(2**(self.outer_alphabet_size_bytes*8)))
+
+    def __compute_waste(self):
+        k = self.inner_k
+        q = self.inner_coder.field.order
+        if self.index_type == "binary":
+            _wasted_bits = k*np.log2(q) - (self.data_chunk_size*8+self.index_bits )
+            logging.debug( "{} bits wasted per strand".format( _wasted_bits ) )
+            logging.debug( "Maximum strand index (using wasted bits): {}".format(2**(int(self.index_bits+_wasted_bits))))
+            _wasted_symbols = _wasted_bits/np.log2(q)
+        elif self.index_type == "inner":
+            data_chunk_symbols_used = self.data_chunk_size*8/np.log2(q)  
+            _wasted_symbols = self.data_symbols - data_chunk_symbols_used
+            logging.debug( f"Maximum strand index:{q**self.index_symbols}")
+        else:
+            raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
         if  _wasted_symbols>=1:
             logging.warning( "{} symbols wasted per strand.  Consider decreasing inner_n.".format( _wasted_symbols) )
         else:
             logging.debug("{} symbols wasted per strand".format( _wasted_symbols))
- 
-         
-        self.block_capacity_bytes = self.data_chunk_size*(self.n_strands - n_redundant_strands) 
-        logging.debug( "codec message size: {} bytes".format(self.block_capacity_bytes) )
 
-        #init the outer coder 
-        self.outer_coder = reedsolo.RSCodec(n_redundant_strands, c_exp=8*self.outer_alphabet_size_bytes)
+    def generate_sector1(self, alphabets,vendor_id = "UNKNOWN", primer_len=40, file_meta_data=None):
+        #change to lazy imports when python 3.15 is common.
+        #these don't take particularly long to import but i don't want to bloat things more.
+        import sys
+        import uuid
+        import json
+        from collections import OrderedDict
+
+        alen = len(alphabets[0])
+        abases = len(alphabets[0][0])
+
+        for a in alphabets:
+            if len(a) != alen:
+                raise ValueError("All alphabets must be of the same length")
+            for s in a:
+                if len(s) != abases:
+                    raise ValueError("All elements of alphabets must be the same number of bases")
+        
+        files_nstrands = 0
+        if file_meta_data is not None:
+            for block_name, block_meta in file_meta_data.items():
+                strand_count_code_block = block_meta["blen"]//self.data_chunk_size+self.outer_coder_fast.d-1
+                strand_count = strand_count_code_block*len(block_meta["istart"])
+                files_nstrands+=strand_count
+        
+        #encode as one string to save space.
+        alphabets = [" ".join(x) for x in alphabets]
+
+        sector1 = OrderedDict(sector01.get_sector_1_obj())
+        #reserve octets 4-5 for the number of strands in sector 1.
+        u = bytearray(uuid.uuid4().bytes)
+        u[6] = 8<<4
+        u[8] = (u[8] & 0b00111111) | 0b10000000
+        u[4] = 0
+        u[5] = 0        
+        sector1["id"] = str(uuid.UUID(bytes=bytes(u) ))
+        sector1["codec"]["ver"] = sys.modules['.'.join(__name__.split('.')[:-1])].__version__
+        codec_params = sector1["codec"]["params"]
+        codec_params["alphs"] = alphabets
+        codec_params["innerd"] = self.inner_coder.d
+        codec_params["innern"] = self.inner_n
+        codec_params["nstrands"] = self.n_strands
+        codec_params["neccstrand"] = self.outer_coder_fast.d-1
+        codec_params["imax"] = self.max_strand_index
+        codec_params["itype"] = self.index_type
+        codec_params["ilocation"] = self.index_location
+        sector1["vendorid"] = vendor_id
+        seq_params = sector1["seq"]
+        osize = self.inner_n*abases+primer_len
+        seq_params["osizemin"] = osize
+        seq_params["osizemedian"] = osize
+        seq_params["osizemax"] = osize
+        seq_params["ocount"] = files_nstrands
+        #these two don't make sense
+        seq_params["orepmin"] = 1
+        seq_params["orepmax"] = 100
+
+        if file_meta_data is not None:
+            sector1["addl"] = {"fmeta": file_meta_data}
+
+        sector1.move_to_end("id",last=False) #guarentee this is in the first code block.
+        #add this in when file meta is present.
+        #s1_len = len(json.dumps(sector1,separators=(',', ':')).encode('utf-8'))
+        #s1_strands = s1_len//self.data_chunk_size + (1 if s1_len%self.data_chunk_size>0 else 0)
+        #s1_redundant_strands = 50*(s1_len//self.block_capacity_bytes + (1 if s1_len%self.block_capacity_bytes>0 else 0))
+        #s1_strands += s1_redundant_strands
+
+        #sector1["id"] = uid+f":{s1_strands:05d}"
+
+        return sector1
 
     def encode(self,data:bytes,index_start:int=0)->list[list[int]]:
         """ Encodes the given data using the outer and inner codes.
@@ -231,23 +345,27 @@ class BaseNBlockCodec:
         #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrr>|
         #  ...
         #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrr>|
-        #  3. inner code  (R= inner code redundancy applied to rows, * = inner code symbol)
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
+        #  3. inner code  (R= inner code redundancy applied to rows)
+        #                 (* = inner code symbol)
+        #                 (@= binary index concatenated to data bits)
+        #                 (a= binary index concatenated to outer code redundancy bits)
+        #     When index type is inner code, @ and a are full symbols.
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
         #  ...
-        #  |<***************** self.data_chunk_size *************RRRRRRRR>|
-        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrrRRRRRRRR>|
+        #  |<***************** self.data_chunk_size *************@@@RRRRRRRR>|
+        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrraaaRRRRRRRR>|
         #  ...
-        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrrRRRRRRRR>|
+        #  |<rrrrrrrrrrrrrrrrr self.data_chunk_size rrrrrrrrrrrrraaaRRRRRRRR>|
 
         #1. Reshape data 
         logging.debug("applying outer code to {} bytes".format(len(data)))
         # reshape data so columns can be coded independently.
-        if self.outer_coder.c_exp==8:
+        if self.outer_alphabet_size_bytes==1:
             data_np = np.frombuffer(data,dtype=np.uint8)
             assert len(data_np)%self.data_chunk_size == 0
             data_np = data_np.reshape(-1,self.data_chunk_size)
-        elif self.outer_coder.c_exp==16:
+        elif self.outer_alphabet_size_bytes==2:
             data_np = np.frombuffer(data,dtype=np.uint16)
             if self.data_chunk_size%2!=0:
                 raise ValueError("data_chunk_size must be even for 16 bit outer code.")
@@ -259,7 +377,8 @@ class BaseNBlockCodec:
             raise ValueError("Unsupported outer code byte size.")
 
         #2. apply the outer code (encode columns of data)
-        data_enc_cols = np.array([self.outer_coder.encode(dc) for dc in data_np.transpose()],dtype=data_np.dtype)
+        data_enc_cols = np.array([self.outer_coder_fast.encode(dc) for dc in data_np.transpose()],dtype=data_np.dtype)
+
         data_enc_chunks = data_enc_cols.transpose()
         
         logging.debug("outer coded strands: {}".format(data_enc_chunks.shape[0]))
@@ -273,11 +392,23 @@ class BaseNBlockCodec:
             d = chunk.tobytes()
             #use little endian to match x86/x64 byte order and avoid needing to pad byte string.
             chunk = int.from_bytes(d,'little') 
-            #add index to end (MS bits)
-            chunk += index<<(self.data_chunk_size*8)
+            if self.index_type == "binary":
+                #add index to end (MS bits)
+                chunk += index<<(self.data_chunk_size*8)
+
+                #but.. we still need to pad out the symbol stream.
+                chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)
+            elif self.index_type == "inner":
+                chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.data_symbols)
+                index_symbols = _int_to_baseN(index,self.inner_coder.field.order,length=self.index_symbols)
+                if self.index_location == "middle":
+                    chunk += index_symbols
+                elif self.index_location == "beginning":
+                    chunk = index_symbols + chunk
+                else:
+                    raise NotImplementedError("index_location 'end' is not implemented.")
+
             index += 1
-            #but.. we still need to pad out the symbol stream.
-            chunk = _int_to_baseN(chunk,self.inner_coder.field.order,length=self.inner_k)  
             chunks.append(chunk)
 
         #apply inner code
@@ -285,16 +416,17 @@ class BaseNBlockCodec:
 
         return ic_chunks
 
-    def decode(self, data: list[list[int]], index_start: int = 0) -> Tuple[bytes,ArrayLike,ArrayLike,ArrayLike]:
+    def decode(self, data: list[list[int]], index_start: int = 0, n_strands: int|None = None) -> Tuple[bytes|None,ArrayLike,ArrayLike,ArrayLike]:
         """
         Decodes the given data.
 
         Args:
             data (list[list[int]]): The encoded data to be decoded.
             index_start (int, optional): The starting index for decoding. Defaults to 0.
+            n_strands (int, optional): The number of strands expected. Defaults to n_strands set in constructor.
 
         Returns:
-            bytes: The decoded data.
+            bytes|None: The decoded data. Or None if decoding failed.
             outer erasures: list of erasures in outer code.
             outer errors: list of errors in outer code.
             inner errors: list of number of errors in inner code by strand index. -1 -> strand not present.
@@ -302,28 +434,49 @@ class BaseNBlockCodec:
         Raises:
             None
         """
+        if n_strands is None:
+            n_strands = self.n_strands
+
         # inner decode
         ## TODO: handle erasures in inner code.
 
         # I'm sure it'll be a (FieldArray,int) ----------------V
-        chunked_data:list[Tuple[galois.FieldArray,int]] = [self.inner_coder.decode(d,errors=True) for d in data] #type:ignore
+        chunked_data:list[Tuple[galois.FieldArray,int]] = [self.inner_coder.decode(d,errors=True) for d in data if d != []] #type:ignore
         
 
         # handle outer decode
-        ordered_chunks:list[bytes|None] = [None] * self.n_strands
-        chunk_errors:list[int] = [-1] * self.n_strands
+        ordered_chunks:list[bytes|None] = [None] * n_strands
+        chunk_errors:list[int] = [-1] * n_strands
         for chunk in chunked_data:
-            chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
-            chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
-            _mask = (2 ** (self.data_chunk_size * 8)) - 1
-            chunk_int = chunk_int & _mask
-            chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
-            #print(chunk_index)
+            #TODO: decide if its better to throw these out or keep them.
+            #if chunk[1]<0: #can't decode and the address may be bad so discard.
+            #    continue
+            if self.index_type == "binary":
+                chunk_int = _baseN_to_int(chunk[0].tolist(), self.inner_coder.field.order)
+                chunk_index = (chunk_int >> (self.data_chunk_size * 8)) - index_start
+                _mask = (2 ** (self.data_chunk_size * 8)) - 1
+                chunk_int = chunk_int & _mask
+            elif self.index_type == "inner":
+                if self.index_location == "middle":
+                    chunk_int = _baseN_to_int(chunk[0][:self.data_symbols].tolist(),self.inner_coder.field.order)
+                    chunk_index = _baseN_to_int(chunk[0][self.data_symbols:].tolist(),self.inner_coder.field.order) - index_start
+                elif self.index_location == "beginning":
+                    chunk_index = _baseN_to_int(chunk[0][:-self.data_symbols].tolist(),self.inner_coder.field.order) - index_start
+                    chunk_int = _baseN_to_int(chunk[0][-self.data_symbols:].tolist(),self.inner_coder.field.order)
+                else:
+                    raise NotImplementedError("index_location 'end' is not implemented.")
+            else:
+                raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
+            #this can be to big sometimes.  TODO handle more gracefully
             try:
-              ordered_chunks[chunk_index] = chunk_bytes
-              chunk_errors[chunk_index] = chunk[1]
-            except IndexError:
-              logging.info("strand index out of bounds:" + str(chunk_index))
+                chunk_bytes:bytes = chunk_int.to_bytes(self.data_chunk_size, 'little')
+                try:
+                    ordered_chunks[chunk_index] = chunk_bytes
+                    chunk_errors[chunk_index] = chunk[1]
+                except IndexError:
+                    logging.info("strand index out of bounds:" + str(chunk_index))
+            except OverflowError:
+                logging.info("chunk overflow for strand index:" + str(chunk_index)+ "intval: "+ str(chunk_int))
 
         # identify erasures
         erasures = []
@@ -334,20 +487,32 @@ class BaseNBlockCodec:
         #we're all bytes now, but the checker can't tell that.
         ordered_chunks_bytes:list[bytes] = typing.cast(list[bytes],ordered_chunks)
 
-        if self.outer_coder.c_exp==8:
+        # report erasures (lost strands) in outer code
+        logging.info("erasures (lost) chunks: {}".format(erasures))
+
+        if self.outer_alphabet_size_bytes==1:
             outer_dtype = np.uint8
-        elif self.outer_coder.c_exp==16:
+        elif self.outer_alphabet_size_bytes==2:
             outer_dtype = np.uint16
         else:
             raise ValueError("Unsupported outer code byte size.")
-        data_np_encoded = np.array([np.frombuffer(c, dtype=outer_dtype) for c in ordered_chunks_bytes]).transpose()
-        data_decoded_results = [self.outer_coder.decode(dc, erase_pos=erasures) for dc in data_np_encoded]
-        data_decoded_chunks = np.array([d[0] for d in data_decoded_results],dtype=outer_dtype).transpose()
-        data_decoded = data_decoded_chunks.flatten().tobytes()
+        data_np_encoded = np.array([np.frombuffer(c, dtype=outer_dtype) for c in ordered_chunks_bytes]).transpose() 
+
+        has_erasuers = len(erasures)>0
+        erasures_bool = np.zeros(data_np_encoded.shape[1],dtype=bool)
+        if has_erasuers:
+            erasures_bool[erasures] = True
+        data_decoded_results = [self.outer_coder_fast.decode(dc,errors=True,erasures=erasures_bool) for dc in data_np_encoded]
+        data_errata_col_pos = [] #galois doesn't report error positions
+        data_num_errors = [int(x[1]) for x in data_decoded_results]
+        if -1 in data_num_errors:
+            logging.error("Outer code decode failed on at least one column.")
+            return None, erasures, data_num_errors, chunk_errors
+            #raise ValueError(f"Fast mode outer code decode failed on at least one column.  numerrors:{data_num_errors}")
+        logging.info("Found {} outer code column errors.".format(data_num_errors))
+        data_decoded_chunks = np.array([np.array(d[0]) for d in data_decoded_results],dtype=outer_dtype).transpose()
 
-        # report erasures (lost strands) in outer code
-        logging.info("erasures (lost) chunks: {}".format(erasures))
-        data_errata_col_pos = [e[2] for e in data_decoded_results]
+        data_decoded = data_decoded_chunks.flatten().tobytes()
 
         errors = []
         for i, pos in enumerate(data_errata_col_pos):
@@ -361,53 +526,74 @@ class BaseNBlockCodec:
             if e[0] in erasures:
                 pass
             else:
-                logging.warning("Outer code errors found on strand {}, byte {}.".format(*e))
+                logging.info("Outer code errors found on strand {}, byte {}.".format(*e))
+
+        return data_decoded, erasures, data_num_errors, chunk_errors # pyright: ignore[reportPossiblyUnboundVariable] 
 
-        return data_decoded, erasures, errors, chunk_errors
     
-    def extract_index_dna(self, dna: str|bytes, words:list[str], alternate_words:list[str], error_check: bool = False, padding: Literal["left","right"]="right") -> int:
+    def extract_index_dna(self, dna: str|bytes, words:list[str], alternate_words:list[str], error_check: bool|Literal["erasures"] = False, padding: Literal["left","right"]="right") -> int:
         """
         Extracts the index from the given DNA strand.
 
         Args:
             dna (str|bytes): The input DNA strand from which to extract the index.
-            error_check (bool, optional): If True, the inner_codec will be used to 
-                attempt to check for errors. Defaults to False.
-            padding Literal["left","right"], optional): Assume any missing bases are left or right of index.  Defaults to "right".
-                Note padded sequences are unlikely to error correct properly.
+            error_check (bool|Literal["erasures"], optional): If True, the inner_codec will be used to 
+                attempt to check for errors.  if "erasures", the inner codec will check only if it encounters an N.
+                Defaults to False.
+            padding Literal["left","right"], optional): Assume any missing bases are left or right of index.  
+                Defaults to "right".
+                Note: padded sequences are unlikely to error correct properly.
             words (list[str]): The primary set of base words used, in order, for encoding.
             alternate_words (list[str]): The alternate (aka synonymous) set of words used, in order, for encoding.
 
         Returns:
             int: The index. or -1 if error_check==True and unrecoverable error found.
         """
-        word_len:int = len(words[0])
+        symbol_len:int = len(words[0])
         if isinstance(dna,bytes):
             dna= dna.decode()
         dna_s:str = cast(str,dna)
-        
-        if padding == "right": 
-            _p = self.inner_k*word_len - len(dna_s)
-            if _p>0: #if i have less than a full codeword of bases
-                dna_s = dna_s + "A"*_p  
-        elif padding == "left":
-            _p = self.inner_k*word_len - len(dna_s)
-            if _p>0: #if i have less than a full codeword of bases
-                dna_s = "A"*_p + dna_s 
 
+        num_missing = self.inner_k*symbol_len - len(dna_s) # in DNA bases
+        if num_missing>0: #if i have less than a full codeword of bases
+            if padding == "right": 
+                dna_s = dna_s + "A"*num_missing
+            elif padding == "left":
+                dna_s = "A"*num_missing + dna_s
+
+
+        if error_check == "erasures":
+            erasures = np.zeros(len(dna_s)//symbol_len,dtype=bool)
+            _temp = []
+            for i,c in enumerate(dna_s):
+                if c.upper() not in "ACGT":
+                    erasures[i//symbol_len] = True
+                    _temp.append("A")
+                else:
+                    _temp.append(c)
+            has_erasures = bool(np.any(erasures))
+            if has_erasures: #don't bother if nothing has changed.
+                dna_s = "".join(_temp) 
+        else:
+            has_erasures = False
+            erasures = None
+
+        ec = (has_erasures and error_check=="erasures") or (error_check==True)
+        #This could fail if padding makes a symbol not in words or alternate_words.
         data = dna_to_bN([dna_s],words,alternate_words)[0]
          
-        return self.extract_index(data,error_check)
+        return self.extract_index(data,ec,erasures)
 
-    def extract_index(self, data: list[int], error_check: bool = False) -> int:
+    def extract_index(self, data: list[int], error_check: bool = False, erasures: np.ndarray|None = None) -> int:
         """
-        Extracts the index from the given DNA strand.
+        Extracts the index from the given inner code sequence.
 
         Args:
             data list[int]: The input data from which to extract the index.
                 If a list[int] is given, it is assumed be be inner code symbols. 
             error_check (bool, optional): If True, the inner_codec will be used to 
                 attempt to check for errors. Defaults to False.
+            erasures (np.ndarray|None, optional): A boolean array indicating which positions are erased. Defaults to None.
 
         Returns:
             int: The index. or -1 if error_check==True and unrecoverable error found.
@@ -415,11 +601,23 @@ class BaseNBlockCodec:
         if error_check == False:
             data = data[0:self.inner_k] #extract message
         else:
-            data, n_corrected = self.inner_coder.decode(data,errors=True)[0].tolist()
+            data_gf, n_corrected = self.inner_coder.decode(data,errors=True,erasures=erasures) #type:ignore
+            data = data_gf.tolist()
             if n_corrected < 0:
                 return -1
-        chunk_int = _baseN_to_int(data, self.inner_coder.field.order)
-        chunk_index = (chunk_int >> (self.data_chunk_size * 8))
+        if self.index_type == "binary":
+            chunk_int = _baseN_to_int(data, self.inner_coder.field.order)
+            chunk_index = (chunk_int >> (self.data_chunk_size * 8))
+        elif self.index_type == "inner":
+            if self.index_location == "middle":
+                chunk_index_baseN = data[self.data_symbols:] 
+            elif self.index_location == "beginning":
+                chunk_index_baseN = data[:-self.data_symbols]
+            else:
+                raise NotImplementedError("index_location 'end' is not implemented.")
+            chunk_index = _baseN_to_int(chunk_index_baseN, self.inner_coder.field.order)
+        else:
+            raise ValueError("Unknown index type: {}".format(self.index_type)) #should be impossible to get here.
         return chunk_index
 
 
@@ -465,7 +663,7 @@ def longest_binder(a:str,b:str):
 
 
 
-def b32_to_DNA_optimize(file_data:list[ArrayLike],words:list[str], alternate_words:list[str], mask:Union[ArrayLike,None]=None, nmasked=-1, penalty_fn:Union[Callable[[str],int],None]=None)->list[Tuple[bytes,int]]:
+def b32_to_DNA_optimize(file_data:list[ArrayLike],words:list[str], alternate_words:list[str], mask:Union[ArrayLike,None]=None, nmasked:int =-1, penalty_fn:Union[Callable[[str],int],None]=None)->list[Tuple[bytes,int]]:
   """ 
     TODO: rename to bN_to_DNA_optimize, and add test.
     file_data: 2d list of integral types (symbols)
@@ -726,3 +924,79 @@ def remove_bytes(data:bytes, insert:int|bytes, chunk_len:int, num_inserts:int) -
         del data[ins_index:ins_index + insert_len]
     return bytes(data)
 
+
+########################
+# Sector 0 and 1 funcs #
+########################
+
+
+def sector_1_codec():
+    """Returns a BaseNBlockCodec object with parameters suitable for encoding sector 1 of the SyNCODER format."""
+    return BaseNBlockCodec(inner_alphabet_size=32,
+                             inner_d=5,
+                             inner_n=70//3, 
+                             n_strands=255,
+                             n_redundant_strands=50,
+                             max_strand_index=2**(5*3)-1,#32,767 strands max.
+                             index_type="inner")
+
+def generate_sector1(codec:BaseNBlockCodec,alphabets:list[list[str]],vendor_id:str, file_meta_data:dict, primer_len:int=40, length:int=-1) -> list[bytes]:
+    """Generates the sector 1 object for the SyNCODER format.
+
+    Args:
+        alphabets (list of list of str): A list of alphabets, where each alphabet is a list of strings representing the symbols in that alphabet.
+        vendor_id (str): A DNA string representing the vendor ID. 
+        file_meta_data (dict): A dictionary containing metadata about the files being encoded.
+        primer_len (int, optional): The length of the primers to be used in sector 1. Defaults to 40.
+        length (int, optional): The number of strands in the sector 1 object. If -1, the number of strands will be determined automatically. Defaults to -1.
+
+    Returns:
+        list[bytes]: A list of bytes representing the encoded sector 1 object.
+    """
+
+    assert len(vendor_id) == 35 #required by spec.
+    s1codec = sector_1_codec()
+    sector1 = codec.generate_sector1(alphabets,vendor_id,primer_len,file_meta_data)
+
+    if length>=0:
+        nstarnds_ashex = hex(length)[2:].rjust(4,'0')
+        id_parts = sector1["id"].split("-")
+        id_parts[1] = nstarnds_ashex
+        sector1["id"] = "-".join(id_parts)
+
+    s1json = json.dumps(sector1,separators=(',', ':')).encode('utf-8')
+
+    padding = s1codec.data_chunk_size - (len(s1json) % s1codec.data_chunk_size)
+    if padding != s1codec.data_chunk_size:
+        # Distribute padding bytes by inserting at most one space after commas
+        new_chars = bytearray(b" ") #put one space at the start.
+        rem = padding-1
+        for c in s1json:
+            new_chars.append(c)
+            if c == ',' and rem > 0:
+                new_chars.append(ord(' ')) # add a space after the comma
+                rem -= 1
+        if rem > 0: #append any remaining spaces at the end (probably wont happen)
+            new_chars.extend(b" " * rem)
+        s1json_padded = bytes(new_chars)
+        assert json.loads(s1json_padded) == sector1, "Padded JSON does not match original data"
+        s1json = s1json_padded
+
+    encoded_s1 = []
+    cbstart = 0
+    istart = 0
+    while cbstart < len(s1json): 
+        encoded_s1_block= s1codec.encode(s1json[cbstart:(cbstart+s1codec.block_capacity_bytes)],index_start=istart) 
+        istart+=len(encoded_s1_block)
+        cbstart+=s1codec.block_capacity_bytes
+        encoded_s1 += encoded_s1_block
+    
+    #reoncode sector 1 with updated id.
+    if length<0:
+        #update sector1 id with number of strands in s1.
+        return generate_sector1(codec,alphabets,vendor_id,file_meta_data,primer_len,length=len(encoded_s1))
+
+    #convert to base32 pad with a random base to meet the 70 base requirement for sector 1.
+    s1DNA = [x[0]+bytes(np.random.choice([b"A",b"C",b"T",b"G"])) for x in b32_to_DNA_optimize(encoded_s1,_default_b32_alphabet,_default_b32_alphabet_alt)]
+    s1DNA_with_primers = [sector01.s1fp_ext.encode("utf-8") + x + sector01.s1rp_rc_ext.encode("utf-8") for x in s1DNA]
+    return s1DNA_with_primers

syncoder-0.0.6a0/src/syncoder/sector01.py

@@ -0,0 +1,57 @@
+import json
+
+s1fp = "ACAACGCTCAAATCAGGG"
+s1rp = "GTGACGGTAGGTGAAATC"
+s1fp_ext = "GCACAACGCTCAAATCAGGG"
+s1rp_ext = "CGGTGACGGTAGGTGAAATC"
+s1rp_rc = "GATTTCACCTACCGTCAC"
+s1rp_rc_ext = "GATTTCACCTACCGTCACCG"
+JSON_template_mandatory = \
+"""
+{
+  "id": null,
+  "seq": {
+    "osizemin": null,
+    "osizemedian": null,
+    "osizemax": null,
+    "orepmin": null,
+    "orepmax": null,
+    "ocount": null,
+    "adapter": false,
+    "natural": true,
+    "coverageinfo": "30x"
+  },
+  "codec": {
+    "id": "ATGTATAGCGCTACGAGCTGCAGTCATCTGATCTC",
+    "vendor": "UW MISL",
+    "name": "SyNCODER",
+    "ver": null,
+    "uri": "http://misl.bio",
+    "params": {}
+  },
+  "vendorid": null
+}
+"""
+
+JSON_template_optional = \
+"""
+{
+  "desc": null,
+  "hash": {
+    "alg": "SHA256",
+    "val": null,
+    "addl": {}
+  },
+  "ts": null,
+  "vendor": {
+    "name": "UW MISL",
+    "uri": "http://misl.bio",
+    "contact": null
+  },
+  "opt": {}
+}
+"""
+
+def get_sector_1_obj():
+  return json.loads(JSON_template_mandatory)
+

{syncoder-0.0.4a2 → syncoder-0.0.6a0}/tests/test_codec_pytest.py

@@ -1,8 +1,7 @@
-#write a test for codec.py
-
 import random
 import numpy as np
 from itertools import product
+from typing import Literal
 
 import syncoder
 codec = syncoder
@@ -12,11 +11,27 @@ codec = syncoder
 #numba_logger.setLevel(logging.WARNING)
 
 def test_encode_decode(): 
-  c = codec.BaseNBlockCodec(inner_alphabet_size=32,inner_d=5,inner_n=30)
+  encode_decode(index_type="binary")
+  encode_decode(index_type="inner")
+  encode_decode(index_type="inner", index_loc="beginning")
+
+def encode_decode(index_type,index_loc:Literal["beginning","middle"]="middle"):
+  c = codec.BaseNBlockCodec(inner_alphabet_size=32,
+                            inner_d=5,
+                            inner_n=30,
+                            index_type=index_type,
+                            index_location=index_loc)
 
   in_text = (codec.lipsum + codec.lipsum)[0:c.block_capacity_bytes]
 
   coded = c.encode( in_text )
+  assert c.extract_index(coded[27]) == 27 #test extract index while we're at it
+  dnadata = syncoder.b32_to_DNA(coded,syncoder._default_b32_alphabet,syncoder._default_b32_alphabet_alt)
+  assert c.extract_index_dna(dnadata[27],
+                            words = syncoder._default_b32_alphabet,
+                            alternate_words= syncoder._default_b32_alphabet_alt,
+                            error_check=True) == 27
+
   random.shuffle(coded)
   #coded = coded[:-4] #erase a random strands
   corrupt_index = random.randint(0,len(coded[0])) 
@@ -27,6 +42,32 @@ def test_encode_decode():
   #check inner_n is causing the correct number of bases
   assert len(coded[0]) == 30 
 
+def test_encode_decode_fast_erase():
+  index_type = "inner"
+  index_loc:Literal["beginning","middle"]="middle"
+  c = codec.BaseNBlockCodec(inner_alphabet_size=32,
+                            inner_d=5,
+                            inner_n=30,
+                            index_type=index_type,
+                            index_location=index_loc,
+                            n_strands = 2001,
+                            n_redundant_strands=100
+                            )
+
+  in_text = np.random.bytes( c.block_capacity_bytes ) 
+  coded = c.encode( in_text ) 
+  dnadata = syncoder.b32_to_DNA(coded,syncoder._default_b32_alphabet,syncoder._default_b32_alphabet_alt)
+ 
+  random.shuffle(coded)
+  coded = coded[:-90] #erase a random strands
+  #corrupt_index = random.randint(0,len(coded[0])) 
+  #coded[0][corrupt_index] = 6 
+
+  out_text = c.decode( coded)[0]
+  assert in_text == out_text
+  #check inner_n is causing the correct number of bases
+  assert len(coded[0]) == 30 
+
 def test_coded_to_bases():
   #TODO: test optimize too
 
@@ -76,6 +117,7 @@ def test_dna_to_bytes():
   dna_bytes, mask = codec.dna_to_bytes(DNA,alphabet,alphabet_alt)
   
   dna_baseN = _int_to_baseN(int.from_bytes(dna_bytes,"little"),len(alphabet))
+  #TODO: this can fail but only for some inputs randomly selected lookinto why.
   assert dna_baseN == answer32
 
   _t = np.array(dna_baseN)

syncoder-0.0.4a2/environments/conda_environment.yml

@@ -1,12 +0,0 @@
-name: codec-user
-channels:
-  - conda-forge 
-dependencies:
-  - python>=3.12
-  - numpy=2.*
-  - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
-  - pip:
-    - galois==0.4.*
-

syncoder-0.0.4a2/environments/conda_environment_test.yml

@@ -1,14 +0,0 @@
-name: codec-test
-channels:
-  - conda-forge 
-dependencies:
-  - python>=3.12
-  - numpy=2.*
-  - pytest
-  - reedsolo=1.7.*
-  - pip=24.*
-  - sphinx=7.*
-  - sphinx_rtd_theme
-  - pip:
-    - galois==0.4.*
-