sacc 1.0.2__py3-none-any.whl → 2.0__py3-none-any.whl

sacc/__init__.py

@@ -3,4 +3,7 @@ from .windows import Window, BandpowerWindow, TopHatWindow, LogTopHatWindow  # n
 from .data_types import standard_types, parse_data_type_name, build_data_type_name  # noqa
 from .tracers import BaseTracer  # noqa
 from .covariance import BaseCovariance  # noqa
-__version__ = '1.0.2' #noqa
+from .tracer_uncertainty import NZLinearUncertainty, NZShiftUncertainty, NZShiftStretchUncertainty  # noqa
+from .io import BaseIO  # noqa
+from . import io  # noqa
+__version__ = '2.0' #noqa

sacc/covariance.py

@@ -1,12 +1,12 @@
-from astropy.io import fits
 from astropy.table import Table
 import scipy.linalg
 import numpy as np
+import warnings
 
 from .utils import invert_spd_matrix
+from .io import BaseIO, ONE_OBJECT_PER_TABLE, ONE_OBJECT_MULTIPLE_TABLES
 
-
-class BaseCovariance:
+class BaseCovariance(BaseIO):
     """
     The abstract base class for covariances in different forms.
     These are not currently designed to be modified after creation.
@@ -26,7 +26,7 @@ class BaseCovariance:
     cov_type: string
         The type of the covariance (class variable)
     """
-    _covariance_classes = {}
+    _sub_classes = {}
 
     def __init__(self):
         """Abstract superclass constructor.
@@ -37,13 +37,34 @@ class BaseCovariance:
         self._dense = None
         self._dense_inverse = None
 
-    # This method gets called whenever a subclass is
-    # defined.  The keyword argument in the class definition
-    # (e.g. cov_type='full' below is passed to this class method)
-    @classmethod
-    def __init_subclass__(cls, cov_type):
-        cls._covariance_classes[cov_type] = cls
-        cls.cov_type = cov_type
+        # At the moment we only allow one covariance object per table,
+        # so this is only used for consistency when saving objects.
+        self.name = "cov"
+
+    def __eq__(self, other):
+        """
+        Test for equality
+
+        Parameters
+        ----------
+        other: object
+            The other object to test for equality
+
+        Returns
+        -------
+        equal: bool
+            True if the objects are equal
+        """
+        if not isinstance(other, self.__class__):
+            return False
+        # We do not test the inverse; we rely on the fact that
+        # if the dense matrices are equal, then the inverses will be equal.
+        # We are also relying on each subclass to have an instance variable
+        # 'size'.
+        return self.name == other.name and \
+            self.size == other.size and \
+            ((self._dense is None and other._dense is None) or \
+            np.allclose(self._dense, other._dense))
 
     @classmethod
     def from_hdu(cls, hdu):
@@ -64,8 +85,11 @@ class BaseCovariance:
         instance: BaseCovariance
             A covariance instance
         """
+        warnings.warn("You are using an older SACC legacy SACC file format with old covariance data."
+                      " Consider updating it by loading it and saving it again with the latest SACC version.",
+                      DeprecationWarning)
         subclass_name = hdu.header['saccclss']
-        subclass = cls._covariance_classes[subclass_name]
+        subclass = cls._sub_classes[subclass_name]
         return subclass.from_hdu(hdu)
 
     @classmethod
@@ -80,32 +104,29 @@ class BaseCovariance:
         Parameters
         ----------
         cov: list[array] or array
-            If a list, the total length of all the arrays in it
-            should equal n.  If an array, it should be either 1D of
-            length n or 2D of shape (n x n).
+            If a list, it should be a list of array-like objects each of which
+            can be coerced into a 2d array. A BlockDiagonalCovariance will be
+            returned.
 
-        n: int
-            length of the data vector to which this covariance applies
+            If an array, it should be either 1D or 2d and square. Either a
+            DiagonalCovariance or a FullCovariance will be returned.
         """
         if isinstance(cov, list):
-            s = 0
             for block in cov:
                 block = np.atleast_2d(block)
                 if (block.ndim != 2) or (block.shape[0] != block.shape[1]):
                     raise ValueError("Covariance block has wrong size "
                                      f"or shape {block.shape}")
-                s += block.shape[0]
             return BlockDiagonalCovariance(cov)
-        else:
-            cov = np.array(cov).squeeze()
-            if cov.ndim == 0:
-                return DiagonalCovariance(np.atleast_1d(cov))
-            if cov.ndim == 1:
-                return DiagonalCovariance(cov)
-            if (cov.ndim != 2) or (cov.shape[0] != cov.shape[1]):
-                raise ValueError("Covariance is not a 2D square matrix "
-                                 f"- shape: {cov.shape}")
-            return FullCovariance(cov)
+        cov = np.array(cov).squeeze()
+        if cov.ndim == 0:
+            return DiagonalCovariance(np.atleast_1d(cov))
+        if cov.ndim == 1:
+            return DiagonalCovariance(cov)
+        if (cov.ndim != 2) or (cov.shape[0] != cov.shape[1]):
+            raise ValueError("Covariance is not a 2D square matrix "
+                             f"- shape: {cov.shape}")
+        return FullCovariance(cov)
 
     @property
     def dense(self):
@@ -140,7 +161,7 @@ class BaseCovariance:
         return self._dense_inverse
 
 
-class FullCovariance(BaseCovariance, cov_type='full'):
+class FullCovariance(BaseCovariance, type_name='full'):
     """
     A covariance subclass representing a full matrix with correlations
     anywhere.  Represented as an n x n matrix.
@@ -153,15 +174,40 @@ class FullCovariance(BaseCovariance, cov_type='full'):
     covmat: 2D array
         The matrix itself, of shape (size x size)
     """
+
+    storage_type = ONE_OBJECT_PER_TABLE
+
     def __init__(self, covmat):
         self.covmat = np.atleast_2d(covmat)
         self.size = self.covmat.shape[0]
         super().__init__()
 
-    def to_hdu(self):
+    def __eq__(self, other):
+        return super().__eq__(other) and \
+            np.allclose(self.covmat, other.covmat)
+
+    @classmethod
+    def from_hdu(cls, hdu):
+        """
+
+        Load a covariance object from the data in the HDU
+        LEGACY METHOD: new sacc files will use from_table
+
+        Parameters
+        ----------
+        hdu: astropy.fits.ImageHDU instance
+
+        Returns
+        -------
+        cov: FullCovariance
+            Loaded covariance object
+        """
+        C = hdu.data
+        return cls(C)
+
+    def to_table(self):
         """
-        Make an astropy FITS HDU object with this covariance in it.
-        This is represented as an image.
+        Make an astropy table object with this covariance in it.
 
         Parameters
         ----------
@@ -169,32 +215,33 @@ class FullCovariance(BaseCovariance, cov_type='full'):
 
         Returns
         -------
-        hdu: astropy.fits.ImageHDU instance
-            HDU that can be used to reconstruct the object.
+        table: astropy.table.Table instance
+            Table that can be used to reconstruct the object.
         """
-        hdu = fits.ImageHDU(self.covmat)
-        hdu.header['EXTNAME'] = 'covariance'
-        hdu.header['SACCTYPE'] = 'cov'
-        hdu.header['SACCCLSS'] = self.cov_type
-        hdu.header['SIZE'] = self.size
-        return hdu
+        col_names = [f'col_{i}' for i in range(self.size)]
+        cols = [self.covmat[i] for i in range(self.size)]
+        table = Table(data=cols, names=col_names)
+        table.meta['SIZE'] = self.size
+        return table
 
     @classmethod
-    def from_hdu(cls, hdu):
+    def from_table(cls, table):
         """
-        Load a covariance object from the data in the HDU
+        Load a covariance object from the data in the table
 
         Parameters
         ----------
-        hdu: astropy.fits.ImageHDU instance
+        table: astropy.table.Table instance
 
         Returns
         -------
         cov: FullCovariance
             Loaded covariance object
         """
-        C = hdu.data
-        return cls(C)
+        size = table.meta['SIZE']
+        covmat = np.array([table[f'col_{i}'] for i in range(size)])
+        return cls(covmat)
+
 
     def keeping_indices(self, indices):
         """
@@ -240,7 +287,7 @@ class FullCovariance(BaseCovariance, cov_type='full'):
         return self.covmat.copy()
 
 
-class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
+class BlockDiagonalCovariance(BaseCovariance, type_name='block'):
     """A covariance subclass representing block diagonal covariances
 
     Block diagonal covariances have sub-blocks that are full dense matrices,
@@ -258,6 +305,9 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
     size: int
         overall total size of the matrix
     """
+
+    storage_type = ONE_OBJECT_MULTIPLE_TABLES
+
     def __init__(self, blocks):
         """Create a BlockDiagonalCovariance object from a list of blocks
 
@@ -271,34 +321,17 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
         self.size = sum(self.block_sizes)
         super().__init__()
 
-    def to_hdu(self):
-        """Write a FITS HDU from the data, ready to be saved.
-
-        The data in the HDU is stored as a single 1 x size image,
-        and the header contains the information needed to reconstruct it.
-
-        Parameters
-        ----------
-        None
-
-        Returns
-        -------
-        hdu: astropy.fits.ImageHDU object
-            HDU containing data and metadata
-        """
-        hdu = fits.ImageHDU(np.concatenate([b.flatten() for b in self.blocks]))
-        hdu.name = 'covariance'
-        hdu.header['sacctype'] = 'cov'
-        hdu.header['saccclss'] = self.cov_type
-        hdu.header['size'] = self.size
-        hdu.header['blocks'] = len(self.blocks)
-        for i, s in enumerate(self.block_sizes):
-            hdu.header[f'size_{i}'] = s
-        return hdu
+    def __eq__(self, other):
+        return super().__eq__(other) and \
+            self.block_sizes == other.block_sizes and \
+            all(np.allclose(b1, b2)
+                for b1, b2
+                in zip(self.blocks, other.blocks))
 
     @classmethod
     def from_hdu(cls, hdu):
         """Read a covariance object from a loaded FITS HDU.
+        LEGACY METHOD: new sacc files will use from_tables
 
         Parameters
         ----------
@@ -320,6 +353,59 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
             blocks.append(B)
         return cls(blocks)
 
+    @classmethod
+    def from_tables(cls, tables):
+        """
+        Load a covariance object from the data in the tables
+
+        Parameters
+        ----------
+        tables: list
+            list of astropy.table.Table instances in block order
+
+        Returns
+        -------
+        cov: BlockDiagonalCovariance
+            Loaded covariance object
+        """
+
+        blocks = []
+        # Get the block count from the first table
+        nblock = list(tables.values())[0].meta['SACCBCNT']
+        for i in range(nblock):
+            table = tables[f'block_{i}']
+            block_size = table.meta['SACCBSZE']
+            cols = [table[f'block_col_{i}'] for i in range(block_size)]
+            blocks.append(np.array(cols))
+        return cls(blocks)
+
+    def to_tables(self):
+        """
+        Make an astropy table object with this covariance in it.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        table: astropy.table.Table instance
+            Table that can be used to reconstruct the object.
+        """
+        tables = {}
+        nblock = len(self.blocks)
+        for j, block in enumerate(self.blocks):
+            b = len(block)
+            col_names = [f'block_col_{i}' for i in range(b)]
+            cols = [block[i] for i in range(b)]
+            table = Table(data=cols, names=col_names)
+            table.meta['SIZE'] = self.size
+            table.meta['SACCBIDX'] = j
+            table.meta['SACCBCNT'] = nblock
+            table.meta['SACCBSZE'] = b
+            tables[f'block_{j}'] = table
+        return tables
+
     def get_block(self, indices):
         """Read a (not necessarily contiguous) sublock of the matrix
 
@@ -378,7 +464,7 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
             blocks = [self.blocks[i][m][:, m] for i, m in
                       enumerate(block_masks)]
             return self.__class__(blocks)
-        elif (np.diff(indices) > 0).all():
+        if (np.diff(indices) > 0).all():
             s = 0
             sub_blocks = []
             for block, sz in zip(self.blocks, self.block_sizes):
@@ -387,10 +473,9 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
                 sub_blocks.append(block[m][:, m])
                 s += sz
             return self.__class__(sub_blocks)
-        else:
-            C = scipy.linalg.block_diag(*self.blocks)
-            C = C[indices][:, indices]
-            return FullCovariance(C)
+        C = scipy.linalg.block_diag(*self.blocks)
+        C = C[indices][:, indices]
+        return FullCovariance(C)
 
     def _get_dense_inverse(self):
         # Invert all the blocks individually and then
@@ -405,7 +490,7 @@ class BlockDiagonalCovariance(BaseCovariance, cov_type='block'):
         return scipy.linalg.block_diag(*self.blocks)
 
 
-class DiagonalCovariance(BaseCovariance, cov_type='diagonal'):
+class DiagonalCovariance(BaseCovariance, type_name='diagonal'):
     """A covariance subclass representing covariances that are
     purely diagonal.
 
@@ -417,6 +502,9 @@ class DiagonalCovariance(BaseCovariance, cov_type='diagonal'):
     diag: array
         The diagonal terms in the covariance (i.e. the variances)
     """
+
+    storage_type = ONE_OBJECT_PER_TABLE
+
     def __init__(self, variances):
         """
         Create a DiagonalCovariance object from the variances
@@ -431,26 +519,10 @@ class DiagonalCovariance(BaseCovariance, cov_type='diagonal'):
         self.size = len(self.diag)
         super().__init__()
 
-    def to_hdu(self):
-        """
-        Make an astropy FITS HDU object with this covariance in it.
-        In this can a binary table HDU is created.
+    def __eq__(self, other):
+        return super().__eq__(other) and \
+            np.allclose(self.diag, other.diag)
 
-        Parameters
-        ----------
-        None
-
-        Returns
-        -------
-        hdu: astropy.fits.BinTableHDU instance
-            HDU that can be used to reconstruct the object.
-        """
-        table = Table(names=['variance'], data=[self.diag])
-        hdu = fits.table_to_hdu(table)
-        hdu.name = 'covariance'
-        hdu.header['sacctype'] = 'cov'
-        hdu.header['saccclss'] = self.cov_type
-        return hdu
 
     def keeping_indices(self, indices):
         """
@@ -472,6 +544,40 @@ class DiagonalCovariance(BaseCovariance, cov_type='diagonal'):
         D = self.diag[indices]
         return self.__class__(D)
 
+    @classmethod
+    def from_table(cls, table):
+        """
+        Load a covariance object from the data in the table
+
+        Parameters
+        ----------
+        table: astropy.table.Table instance
+
+        Returns
+        -------
+        cov: DiagonalCovariance
+            Loaded covariance object
+        """
+        D = table['variance']
+        return cls(D)
+
+    def to_table(self):
+        """
+        Make an astropy table object with this covariance in it.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        table: astropy.table.Table instance
+            Table that can be used to reconstruct the object.
+        """
+        table = Table(data=[self.diag], names=['variance'])
+        table.meta['SIZE'] = self.size
+        return table
+
     @classmethod
     def from_hdu(cls, hdu):
         """

sacc/data_types.py

@@ -1,8 +1,9 @@
 from collections import namedtuple
 from astropy.table import Table
 
-from .utils import (Namespace, hide_null_values,
+from .utils import (Namespace, hide_null_values, numpy_to_vanilla,
                     null_values, camel_case_split_and_lowercase)
+from .io import BaseIO, MULTIPLE_OBJECTS_PER_TABLE
 
 # The format for a data type name looks like this:
 # {sources}_{properties}_{statistic_type}[_{statistic_subtype}]
@@ -220,8 +221,7 @@ def build_data_type_name(sources, properties, statistic, subtype=None):
                                                 for s in properties[1:]])
     if subtype:
         return f"{sources}_{properties}_{statistic}_{subtype}"
-    else:
-        return f"{sources}_{properties}_{statistic}"
+    return f"{sources}_{properties}_{statistic}"
 
 
 # This makes a namespace object, so you can do:
@@ -232,7 +232,7 @@ def build_data_type_name(sources, properties, statistic, subtype=None):
 standard_types = Namespace(*required_tags.keys())
 
 
-class DataPoint:
+class DataPoint(BaseIO, type_name="DataPoint"):
     """A class for a single data point (one scalar value).
 
     Data points have a type, zero or more tracers, a value,
@@ -258,6 +258,8 @@ class DataPoint:
         Dictionary of further data point metadata, such as binning
         info, angles, etc.
     """
+    storage_type = MULTIPLE_OBJECTS_PER_TABLE
+    _sub_classes = {}
     def __init__(self, data_type, tracers, value,
                  ignore_missing_tags=False, **tags):
         """Create a new data point.
@@ -297,12 +299,38 @@ class DataPoint:
                                      f"{data_type} "
                                      "(ignore_missing_tags=False)")
 
+
     def __repr__(self):
-        t = ", ".join(f'{k}={v}' for (k, v) in self.tags.items())
+        t = ", ".join(f'{k}={v}' for (k, v) in self.tags.items() if k != 'sacc_ordering')
         st = f"DataPoint(data_type='{self.data_type}', "
         st += f"tracers={self.tracers}, value={self.value}, {t})"
         return st
 
+    def __eq__(self, other):
+        """
+        Check equality with another DataPoint.
+
+        This method compares the current DataPoint instance with another
+        to determine if they are equivalent. Two DataPoints are considered
+        equal if they have the same data_type, tracers, value, and tags.
+
+        Parameters
+        ----------
+        other: DataPoint
+            The other DataPoint instance to compare against.
+
+        Returns
+        -------
+        bool
+            True if the DataPoints are equal, False otherwise.
+        """
+        if not isinstance(other, DataPoint):
+            return False
+        return (self.data_type == other.data_type and
+                self.tracers == other.tracers and
+                self.value == other.value and
+                self.tags == other.tags)
+
     def get_tag(self, tag, default=None):
         """
         Get the value of the the named tag, or None if not found.
@@ -356,10 +384,37 @@ class DataPoint:
         tracers = [f'tracer_{i}' for i in range(ntracer)]
         return tracers, tags
 
+    @classmethod
+    def to_tables(cls, data, lookups=None):
+        data_by_type = {}
+        for i, d in enumerate(data):
+            d.tags['sacc_ordering'] = i
+            # Get the data type name
+            data_type = d.data_type
+            if data_type not in data_by_type:
+                data_by_type[data_type] = []
+            # Add the data point to the table for this type
+            data_by_type[data_type].append(d)
+
+        tables = {}
+        for data_type, data_points in data_by_type.items():
+            # Convert the data points to a table
+            table = cls.to_table(data_points, lookups)
+            # Add metadata to the table
+            table.meta['SACCTYPE'] = 'data'
+            table.meta['SACCNAME'] = data_type
+            tables[data_type] = table
+
+        # Now remove the temporary ordering tag
+        for d in data:
+            del d.tags['sacc_ordering']
+
+        return tables
+
     @classmethod
     def to_table(cls, data, lookups=None):
         """
-        Convert a list of data points to a single homogenous table
+        Convert a list of data points to a single homogenous table.
 
         Since data points can have varying tags, this method uses
         null values to represent non-present tags.
@@ -424,11 +479,11 @@ class DataPoint:
         data = []
         for row in table:
             # Get basic data elements
-            tracers = tuple([row[f'tracer_{i}'] for i in range(nt)])
-            value = row['value']
+            tracers = tuple([numpy_to_vanilla(row[f'tracer_{i}']) for i in range(nt)])
+            value = numpy_to_vanilla(row['value'])
 
             # Deal with tags.  First just pull out all remaining columns
-            tags = {name: row[name] for name in tag_names}
+            tags = {numpy_to_vanilla(name): row[name] for name in tag_names}
             for k, v in list(tags.items()):
                 # Deal with any tags that we should replace.
                 # This is mainly used for Window instances.