labfreed 0.0.4__py3-none-any.whl → 0.2.0b0__py3-none-any.whl

labfreed/trex/__init__.py

@@ -0,0 +1,16 @@
+from .trex import TREX
+from .value_segments import NumericSegment, DateSegment, BoolSegment, AlphanumericSegment, TextSegment, ErrorSegment
+from .table_segment import TableSegment, ColumnHeader, TableRow
+
+__all__ = [
+    "TREX",
+    "NumericSegment",
+    "DateSegment",
+    "BoolSegment",
+    "AlphanumericSegment",
+    "TextSegment",
+    "ErrorSegment",
+    "TableSegment",
+    "ColumnHeader",
+    "TableRow"
+]

labfreed/trex/python_convenience/__init__.py

@@ -0,0 +1,3 @@
+from .pyTREX import pyTREX  # noqa: F401
+from .data_table import DataTable  # noqa: F401
+from .quantity import Quantity  # noqa: F401

labfreed/trex/python_convenience/data_table.py

@@ -0,0 +1,45 @@
+
+
+from datetime import date, datetime, time
+from pydantic import BaseModel, Field, PrivateAttr
+
+from labfreed.utilities.base36 import base36
+from labfreed.trex.python_convenience.quantity import Quantity
+
+
+class DataTable(BaseModel):
+    _row_template:list[str, Quantity | datetime | time | date | bool | str | base36] =  PrivateAttr(default_factory=list)
+    col_names: list[str] = Field(default_factory=list)
+    data:list[str, Quantity | datetime | time | date | bool | str | base36] = Field(default_factory=list)
+    
+    @property
+    def row_template(self):
+        return self._row_template
+       
+    def append(self, row:list):
+        if not isinstance(row, list):
+            raise ValueError('row must be a list of values')
+        if not self._row_template:
+            self._row_template = row.copy()
+        if not len(row) == len(self._row_template):
+            raise ValueError('row is not of same length as the row template.')
+        if not self.col_names:
+            self.col_names = [f"Col{i}" for i in range(len(self._row_template))]
+        
+        # make sure int and float have a unit, if the row_tempalet has one
+        for i, e in enumerate(row):
+            if isinstance(e, float|int) and isinstance(self._row_template[i], Quantity):
+                unit = self._row_template[i].unit
+                row[i] = Quantity(value=e, unit=unit)     
+        self.data.append(row)
+        
+        
+    def extend(self, iterable):
+        for item in iterable:
+            if not len(item) == len(self._row_template):
+                raise ValueError('row is not of same length as the row template.')
+            self.data.append(item) 
+        
+        
+
+    

labfreed/trex/python_convenience/pyTREX.py

@@ -0,0 +1,242 @@
+
+from datetime import date, datetime, time
+import logging
+import re
+
+from pydantic import RootModel
+from labfreed.well_known_keys.unece.unece_units import unece_unit
+from labfreed.trex.python_convenience.data_table import DataTable
+from labfreed.utilities.base36 import from_base36, base36, to_base36
+
+from labfreed.trex.python_convenience.quantity import Quantity, unece_unit_code_from_quantity
+from labfreed.trex.table_segment import ColumnHeader, TableSegment
+from labfreed.trex.trex import TREX
+from labfreed.trex.trex_base_models import AlphanumericValue, BinaryValue, BoolValue, DateValue, ErrorValue, NumericValue, TextValue
+from labfreed.trex.value_segments import BoolSegment, TextSegment, NumericSegment, AlphanumericSegment, DateSegment, ValueSegment
+
+
+class pyTREX(RootModel[dict[str, Quantity | datetime | time | date | bool | str | base36 | DataTable]]):
+    ''' A wrapper around dict, which knows how to convert to and from TREX. 
+        It restricts the types allowed as values. Keys must be str.
+    '''
+    model_config = {'arbitrary_types_allowed':True} # needed to allow Quantity and DataTable w/o implementing the pydantic schema
+    '''@private'''
+
+    
+    @classmethod
+    def from_trex(cls, trex:TREX):
+        '''Creates a pyTREX from a TREX'''
+        return {seg.key: _trex_segment_to_python_type(seg) for seg in trex.segments}
+             
+             
+    def to_trex(self):
+        '''Creates a TREX'''
+        segments = list()
+        for k, v in self.root.items():
+            if isinstance(v, bool):
+                value = _bool_value_from_python_type(v)
+                segments.append(BoolSegment(key=k, value=value.value))
+            elif isinstance(v, Quantity):
+                unece_code = unece_unit_code_from_quantity(v)
+                value = _numeric_value_from_python_type(v.value)
+                segments.append(NumericSegment(key=k, value=value.value, type=unece_code))
+            elif isinstance(v, (int, float)):
+                value = _numeric_value_from_python_type(v)
+                segments.append(NumericSegment(key=k, value=value.value, type='C63'))  # unitless
+            elif isinstance(v, (datetime, time, date)):
+                value = _date_value_from_python_type(v)
+                segments.append(DateSegment(key=k, value=value.value))
+            elif isinstance(v, str):
+                if re.fullmatch(r'[A-Z0-9\-\.]*', v):
+                    value = _alphanumeric_value_from_python_type(v)
+                    segments.append(AlphanumericSegment(key=k, value=value.value))
+                else:
+                    v = to_base36(v)
+                    value = _text_value_from_python_type(v)
+                    segments.append(TextSegment(key=k, value=value.value))
+            elif isinstance(v, base36):
+                value = _text_value_from_python_type(v)
+                segments.append(TextSegment(key=k, value=value.value))
+                
+            elif isinstance(v, DataTable):
+                v:DataTable = v
+                headers = list()
+                for nm, rt in zip(v.col_names, v.row_template):
+                    if isinstance(rt, bool): # must come first otherwise int matches the bool
+                        t = 'T.B'
+                    elif isinstance(rt, Quantity):
+                        unece_code = unece_unit_code_from_quantity(rt)
+                        t = unece_code
+                    elif isinstance(rt, (datetime, time, date)):
+                        t = 'T.D'     
+                    elif isinstance(rt, str):
+                        if re.fullmatch(r'[A-Z0-9\-\.]*', rt):
+                            t = 'T.A'
+                        else:
+                            v = to_base36(rt)
+                            t = 'T.X'
+                    elif isinstance(rt, base36):
+                        t = 'T.X'
+                    
+                    headers.append(ColumnHeader(key=nm, type=t))
+                data = []
+                for row in v.data:
+                    r = []
+                    for e in row:
+                        if isinstance(e, bool): # must come first otherwise int matches the bool
+                            r.append(_bool_value_from_python_type(e))
+                        elif isinstance(e, Quantity):
+                            r.append(_numeric_value_from_python_type(e.value))
+                        elif isinstance(e, (int, float)):
+                            r.append(_numeric_value_from_python_type(e))
+                        elif isinstance(e, (datetime, time, date)):
+                            r.append(_date_value_from_python_type(e))
+                        elif isinstance(e, str):
+                            if re.fullmatch(r'[A-Z0-9\-\.]*', e):
+                                r.append(_alphanumeric_value_from_python_type(e))
+                            else:
+                                e = to_base36(e)
+                                r.append(_text_value_from_python_type(e))
+                        elif isinstance(e, base36):
+                            r.append(_text_value_from_python_type(e))
+                    data.append(r)
+                segments.append(TableSegment(key=k, column_headers=headers, data=data))
+        return TREX(segments=segments)
+    
+    # make the usual dict methods available, for convenience
+    def __getitem__(self, key): return self.root[key]
+    def __setitem__(self, key, value): self.root[key] = value
+    def update(self, *args, **kwargs):
+        return self.root.update(*args, **kwargs)
+    def keys(self): return self.root.keys()
+    def values(self): return self.root.values()
+    def items(self): return self.root.items()
+    def __contains__(self, key): return key in self.root
+    def __iter__(self): return iter(self.root)
+    def __len__(self): return len(self.root)
+    
+  
+  
+# Helper functions to convert python types to TREX types    
+
+def _numeric_value_from_python_type(v:int|float):
+    return NumericValue(value = str(v))
+
+
+def _date_value_from_python_type(v:date|time|datetime):    
+    sd = ""
+    st = ""
+    if isinstance(v, date) or isinstance(v, datetime):
+        sd = v.strftime('%Y%m%d')
+    if isinstance(v, time) or isinstance(v, datetime):
+        if v.microsecond:
+            st = v.strftime("T%H%M%S.") + f"{v.microsecond // 1000:03d}"
+        elif v.second:
+            st = v.strftime("T%H%M%S")
+        else:
+            st = v.strftime("T%H%M")
+                        
+    return DateValue(value = sd + st)
+    
+    
+def _bool_value_from_python_type(v:bool):
+    return BoolValue(value = 'T' if v else 'F')
+
+
+def _alphanumeric_value_from_python_type(v:str):
+    return AlphanumericValue(value = v)
+
+
+def _text_value_from_python_type(v:base36|str):
+    if isinstance(v, str):
+        logging.info('Got str for text value > converting to base36')
+        out =  to_base36(v).root
+    else:
+        out =  v.root
+    return TextValue(value = out)
+        
+        
+def _binary_value_from_python_type(v:base36|str):
+    if isinstance(v, str):
+        out = v
+    else:
+        out = v.root
+    return BinaryValue(value = out)
+    
+
+def _error_value_from_python_type(v:str):
+    return ErrorValue(value = v)
+    
+
+
+# Helper functions to convert from TREX types to python types
+def _trex_segment_to_python_type(v):
+    '''Converts a TREX segment to a python value. Note the segment key must be handles outside.'''
+    if isinstance(v, NumericSegment):
+        num_val = _trex_value_to_python_type(v)
+        u = unece_unit(v.type)
+        unit = u.get('symbol')
+        return Quantity(value=num_val, unit=unit)
+    
+    # value segments are derived from their respective value type
+    elif isinstance(v, ValueSegment):
+        return _trex_value_to_python_type(v)
+
+    elif isinstance(v, TableSegment):
+        table = DataTable(col_names=[ch.key for ch in v.column_headers])
+        for row in v.data:
+            r = []
+            for e, h in zip(row, v.column_headers):
+                if isinstance(e, NumericValue):
+                    u = unece_unit(h.type)
+                    unit = u.get('symbol')
+                    r.append(Quantity(value=e.value, unit=unit))
+                else:
+                    r.append(_trex_value_to_python_type(e))
+            table.append(r)
+        return table
+        
+
+
+def _trex_value_to_python_type(v):
+    '''Converts a TREX value to the corresponding python type'''
+    if isinstance(v, NumericValue):
+        if '.' not in v.value and 'E' not in v.value: 
+            return int(v)
+        else:
+            return float(v.value)  
+        
+    elif isinstance(v,DateValue):
+        d = v._date_time_dict
+        if d.get('year') and d.get('hour'): # input is only a time
+            return datetime(**d)
+        elif d.get('year'):
+            return date(**d)
+        else:
+            return time(**d)
+            
+    elif isinstance(v, BoolValue):
+        if v.value == 'T':
+            return True
+        elif v.value == 'F':
+            return False
+        else:
+            Exception(f'{v} is not valid boolean. That really should not have been possible -- Contact the maintainers of the library')
+                
+    elif isinstance(v, AlphanumericValue):
+        return v.value
+        
+    elif isinstance(v, TextValue):
+        decoded = from_base36(v.value)
+        return decoded
+        
+    elif isinstance(v, BinaryValue):
+        decoded = bytes(from_base36(v.value))
+        return decoded
+        
+    elif isinstance(v, ErrorValue):
+        return v.value
+        
+    else:
+        raise (TypeError(f'Invalid type {type(v)} of segment'))
+ 

labfreed/trex/python_convenience/quantity.py

@@ -0,0 +1,46 @@
+from pydantic import BaseModel, model_validator
+from labfreed.well_known_keys.unece.unece_units import unece_units
+
+
+class Quantity(BaseModel):
+    ''' Represents a quantity'''
+    value: float|int
+    unit: str
+    significant_digits: int|None = None
+    
+    @model_validator(mode='after')
+    def significat_digits_for_int(self):
+        if isinstance(self.value, int):
+            self.significant_digits = 0
+        return self
+        
+    @property
+    def float(self) -> float:
+        ''' for clarity returns the value'''
+        return self.value
+    
+    def __str__(self):
+        unit_symbol = self.unit
+        if self.unit == "dimensionless" or not self.unit:
+            unit_symbol = ""
+        if self.significant_digits is not None:
+            val = f"{self.value:.{self.significant_digits}f}"
+        else:
+            val = str(self.value)
+        return f"{val} {unit_symbol}"
+      
+    def __repr__(self):
+        return f'Quantity: {self.__repr__()}'
+    
+    
+    
+    
+def unece_unit_code_from_quantity(q:Quantity):
+        by_name =   [ u['commonCode'] for u in unece_units() if u.get('name','') == q.unit] 
+        by_symbol = [ u['commonCode'] for u in unece_units() if u.get('symbol','') == q.unit]
+        code = list(set(by_name) | set(by_symbol))
+        if len(code) != 1:
+            raise ValueError(f'No UNECE unit code found for Quantity {str(q)}' ) 
+        return code[0]
+    
+    

labfreed/trex/table_segment.py

@@ -0,0 +1,227 @@
+       
+    
+from collections import Counter
+import logging
+import re
+
+from pydantic import RootModel, model_validator
+from labfreed.trex.trex_base_models import Value
+from labfreed.well_known_keys.unece.unece_units import unece_unit_codes
+from labfreed.labfreed_infrastructure import LabFREED_BaseModel, ValidationMsgLevel, _quote_texts
+from labfreed.trex.trex_base_models import AlphanumericValue, BinaryValue, BoolValue, DateValue, ErrorValue, NumericValue, TREX_Segment, TextValue, str_to_value_type
+
+
+class ColumnHeader(LabFREED_BaseModel):
+    '''Header of a table Column'''
+    key:str
+    type:str
+               
+    @model_validator(mode='after')     
+    def _validate_key(self):
+        if not_allowed_chars := set(re.sub(r'[A-Z0-9\.-]', '', self.key)):
+            self._add_validation_message(
+                source=f"TREX table column {self.key}",
+                level= ValidationMsgLevel.ERROR,
+                msg=f"Column header key contains invalid characters: {_quote_texts(not_allowed_chars)}",
+                highlight_pattern = f'{self.key}$',
+                highlight_sub=not_allowed_chars
+            )
+        return self
+    
+    @model_validator(mode='after')
+    def _validate_type(self):
+        valid_types = unece_unit_codes() + ['T.D', 'T.B', 'T.A', 'T.T', 'T.X', 'E']
+        if self.type not in valid_types:
+            self._add_validation_message(
+                    source=f"TREX table column {self.key}",
+                    level= ValidationMsgLevel.ERROR,
+                    msg=f"Type '{self.type}' is invalid. Must be 'T.D', 'T.B', 'T.A', 'T.T', 'T.X', 'E' or a UNECE unit",
+                    highlight_pattern = self.type
+            )
+        return self
+
+class TableRow(RootModel[list[Value]]):
+    """
+    Represents a row in a table.
+
+    This class is a Pydantic RootModel that wraps a `list[ValueMixin]`.
+    Each element in the list corresponds to a cell in the row.
+
+    All common list operations (indexing, iteration, append, pop, etc.) are supported.
+    Internally, it wraps a list in the `.root` attribute.
+    """
+    def serialize(self):
+        return ':'.join([e.serialize() for e in self.root])
+    
+    def __len__(self):
+        return len(self.root)
+    
+    def __iter__(self):
+        return iter(self.root)
+    
+    def __repr__(self):
+        return f"TableRow({self.root!r})  # wraps list[{Value.__name__}]"
+    
+  
+class TableSegment(TREX_Segment):
+    '''TREX Segment which represents tabular data'''
+    key:str   
+    column_headers: list[ColumnHeader]
+    data: list[TableRow]
+    
+    @property
+    def column_names(self):
+        return [h.key for h in self.column_headers]
+    
+    @property
+    def column_types(self):
+        return [h.type for h in self.column_headers]
+    
+    @model_validator(mode='after')
+    def _validate_sizes(self):
+        sizes = [len(self.column_headers)]
+        sizes.extend( [ len(row) for row in self.data ] ) 
+        most_common_len, count = Counter(sizes).most_common(1)[0]        
+        
+        if len(self.column_headers) != most_common_len:
+            self._add_validation_message(
+                    source=f"Table {self.key}",
+                    level= ValidationMsgLevel.ERROR,
+                    msg=f"Size mismatch: Table header contains {self.column_names} keys, while most rows have {most_common_len}",
+                    highlight_pattern = self.key
+            )
+            expected_row_len = most_common_len
+        else:
+            expected_row_len = len(self.column_headers)
+            
+
+        for i, row in enumerate(self.data):
+            if len(row) != expected_row_len:
+                self._add_validation_message(
+                    source=f"Table {self.key}",
+                    level= ValidationMsgLevel.ERROR,
+                    msg=f"Size mismatch: Table row {i} contains {len(row)} elements. Expected size is {expected_row_len}",
+                    highlight_pattern = row.serialize()
+                )
+        return self
+    
+    @model_validator(mode='after')
+    def _validate_data_types(self):
+        expected_types = self.column_types
+        i = 0
+        for row in self.data:
+            for e, t_expected, nm in zip(row, expected_types, self.column_names):
+                try:
+                    match t_expected:
+                        case 'T.D':
+                            assert isinstance(e, DateValue)
+                        case 'T.B':
+                            assert isinstance(e, BoolValue)
+                        case 'T.A':
+                            assert isinstance(e, AlphanumericValue)              
+                        case 'T.T':
+                            assert isinstance(e, TextValue)
+                        case 'T.X':
+                            assert isinstance(e, BinaryValue)
+                        case 'E':
+                            assert isinstance(e, ErrorValue)
+                        case _:
+                            assert isinstance(e, NumericValue)
+                except AssertionError:
+                    self._add_validation_message(
+                        source=f"Table {self.key}",
+                        level= ValidationMsgLevel.ERROR,
+                        msg=f"Type mismatch: Table row {i}, column {nm} is of wrong type. According to the header it should be {t_expected}",
+                        highlight_pattern = row.serialize(),
+                        highlight_sub=[c for c in e.value]
+                    )
+                    
+                if msg := e.errors():
+                    for m in msg:
+                        self._add_validation_message(
+                            source=f"Table {self.key}",
+                            level= ValidationMsgLevel.ERROR,
+                            msg=m.msg,
+                            highlight_pattern = row.serialize(),
+                            highlight_sub=[c for c in e.value]
+                        )
+                i += 1
+                
+            
+    def _get_col_index(self, col:str|int):
+        if isinstance(col, str):
+            col_index = self.column_names.index(col)
+        elif isinstance(col, int):
+            col_index = col
+        else:
+            raise TypeError(f"Column must be specified as string or int: {col.__name__}")
+        return col_index
+    
+    
+    
+    def serialize(self):
+        header = ':'.join([f'{h.key}${h.type}' for h in self.column_headers])        
+        data = '::'.join([r.serialize() for r in self.data])
+        s = f'{self.key}$${header}::{data}'
+        return s
+    
+
+    def n_rows(self) -> int:
+        return len(self.data)
+    
+    def n_cols(self) -> int:
+        return len(self.column_headers)
+      
+    def row_data(self, row:int) -> list:
+        out = self.data[row]
+        return out
+    
+    
+    def column_data(self, col:str|int) -> list:
+        col_index = self._get_col_index(col)
+        out =  [row[col_index] for row in self.data]
+        return out
+     
+     
+    def cell_data(self, row:int, col:str|int):
+        try:
+            col_index = self._get_col_index(col)
+            value = self.data[row][col_index]
+        except ValueError:
+            logging.warning(f"row {row}, column {col} not found")
+            return None 
+        return value
+   
+
+
+
+ 
+
+def _deserialize_table_segment_from_trex_segment_str(trex_segment_str) -> TableSegment:
+    # re_table_pattern = re.compile(f"(?P<tablename>[\w\.-]*?)\$\$(?P<header>[\w\.,\$:]*?)::(?P<body>.*)")
+    # re_col_head_pattern = re.compile(f"(?P<name>[\w\.-]*?)\$(?P<unit>[\w\.]*)")
+    re_table_pattern = re.compile(r"(?P<tablename>.+?)\$\$(?P<header>.+?)::(?P<body>.+)")
+    
+    matches = re_table_pattern.match(trex_segment_str) 
+    if not matches:
+        return None
+    name, header, body = matches.groups()
+    
+    column_headers_str = header.split(':')
+    
+    headers = []
+    for colum_header in column_headers_str:
+         ch = colum_header.split('$')
+         col_key = ch[0]
+         col_type = ch[1] if len(ch) > 1 else ''
+         headers.append(ColumnHeader(key=col_key, type=col_type))
+    
+    data = [row.split(':') for row in body.split('::') ]
+    col_types = [h.type for h in headers]
+    # convert to correct value types
+    data_with_types = [[str_to_value_type(c,t) for c, t in zip(r, col_types)] for r in data]
+    data = [ TableRow(r) for r in data_with_types]
+             
+    out = TableSegment(column_headers=headers, data=data_with_types, key=name)
+    return out
+        

labfreed/trex/trex.py

@@ -0,0 +1,69 @@
+from collections import Counter
+from typing import Self
+from pydantic import Field, field_validator
+
+from labfreed.labfreed_infrastructure import LabFREED_BaseModel
+from labfreed.trex.table_segment import _deserialize_table_segment_from_trex_segment_str
+from labfreed.trex.trex_base_models import TREX_Segment
+from labfreed.trex.value_segments import _deserialize_value_segment_from_trex_segment_str
+
+
+class TREX(LabFREED_BaseModel):
+    '''Represents a T-REX extension'''
+    segments: list[TREX_Segment] = Field(default_factory=list)
+       
+    @classmethod
+    def deserialize(cls, data) -> Self:
+        segment_strings = data.split('+')
+        segments = list()
+        for s in segment_strings:
+            # there are only two valid options. The segment is a scalar or a table. 
+            # Constructors do the parsing anyways and raise exceptions if invalid data
+            # try both options and then let it fail
+            segment = _deserialize_table_segment_from_trex_segment_str(s)
+            if not segment:
+                segment = _deserialize_value_segment_from_trex_segment_str(s)
+            if not segment:
+                raise ValueError('TREX contains neither valid value segment nor table')
+                
+            segments.append(segment)
+        trex = TREX(segments=segments)
+        return trex
+        
+    
+    def serialize(self):
+        seg_strings = list()
+        for s in self.segments:
+            seg_strings.append(s.serialize())
+        s_out = '+'.join(seg_strings)
+        return s_out
+    
+       
+    def get_segment(self, segment_key:str) -> TREX_Segment:
+        '''Get a segment by key'''
+        s = [s for s in self.segments if s.key == segment_key]
+        if s:
+            return s[0]
+        else:
+            return None
+        
+    
+    def __str__(self):
+        s = self.serialize().replace('+', '\n+').replace('::', '::\n ')
+        return s
+            
+        
+        
+    @field_validator('segments')
+    @classmethod
+    def _validate_segments(cls, segments):
+        segment_keys = [s.key for s in segments]
+        duplicates = [item for item, count in Counter(segment_keys).items() if count > 1]
+        if duplicates:
+            raise ValueError(f"Duplicate segment keys: {','.join(duplicates)}")
+        return segments
+      
+    
+        
+
+