roman-snpit-snappl 0.16.0__tar.gz → 0.17.0__tar.gz

{roman_snpit_snappl-0.16.0/roman_snpit_snappl.egg-info → roman_snpit_snappl-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roman_snpit_snappl
-Version: 0.16.0
+Version: 0.17.0
 Summary: General, database, and photometry utilities for the Roman SNPIT
 Author: Roman Supernova Project Infrastructure Team
 Maintainer-email: Roman SN PIT <raknop@lbl.gov>
@@ -22,7 +22,7 @@ Requires-Dist: pycryptodome<4.0.0,>=3.23.0
 Requires-Dist: python-dateutil<3.0.0,>=2.9.0.post0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: pyyaml<7.0.0,>=6.0.3
-Requires-Dist: rkwebutil<3.0.0,>=2.2.3
+Requires-Dist: rkwebutil<3.0.0,>=2.4.0
 Requires-Dist: roman-datamodels<0.28.0,>=0.27.0
 Requires-Dist: scipy<2.0.0,>=1.16.2
 Requires-Dist: simplejson<4.0.0,>=3.20.2

roman_snpit_snappl-0.17.0/changes/83.snappl.rst

@@ -0,0 +1 @@
+Added extremely basic static source sim to snappl.

roman_snpit_snappl-0.17.0/changes/85.feature.rst

@@ -0,0 +1,2 @@
+Add database connection to l2image, go back to using collection in
+DiaObject, some other small changes.

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/docker/webserver/Dockerfile

@@ -58,6 +58,33 @@ RUN source /venv/bin/activate && \
        "pytest>=8.4.2,<9.0.0" \
        "remote-pdb==2.1.0"
 
+# TEMPORARY
+# This installs the preqreqs of snappl.  Really, we should
+#   just let that be installed when we do pip install .
+#   below.  However, during development, when editing
+#   the package a lot, it's faster to build the docker
+#   image if there's a previous step that already has the prereqs.
+#   In the mean time, KEEP THIS SYNCED WITH pyproject.toml
+RUN source /venv/bin/activate && \
+    pip --no-cache install \
+        "astropy>=7.1.0,<8.0.0" \
+        "fitsio>=1.2.5,<2.0.0" \
+        "galsim>=2.7.2,<3.0.0" \
+        "gwcs>=0.26.0,<0.27.0" \
+        "h5py>=3.14.0,<4.0.0" \
+        "numpy>=2.2.6,<3.0.0" \
+        "pandas>=2.3.3,<3.0.0" \
+        "photutils>=2.3.0,<3.0.0" \
+        "pycryptodome>=3.23.0,<4.0.0" \
+        "python-dateutil>=2.9.0.post0,<3.0.0" \
+        "pytz>=2025.2" \
+        "pyyaml>=6.0.3,<7.0.0" \
+        "rkwebutil>=2.3.0,<3.0.0" \
+        "roman-datamodels>=0.27.0,<0.28.0" \
+        "scipy>=1.16.2,<2.0.0" \
+        "simplejson>=3.20.2,<4.0.0"
+# END TEMPORARY
+
 RUN mkdir -p /usr/src/snappl_install
 WORKDIR /usr/src/snappl_install
 COPY . /usr/src/snappl_install
@@ -112,6 +139,7 @@ ENV PYTHONPATH=/roman-snpit-db
 RUN mkdir /sessions
 
 COPY docker/webserver/config-test.yaml /roman-snpit-db/
+COPY docker/webserver/roman-snpit-server.py /roman-snpit-db/
 
 ENTRYPOINT [ "tail", "-f", "/etc/issue" ]
 

roman_snpit_snappl-0.17.0/docker/webserver/config-test.yaml

@@ -0,0 +1,27 @@
+# This is the config file used for the tests.
+# You should not use it for real.  Ideally, you
+# should have a config stored as a secret somewhere
+# which you mount (say to /secrets/conifg.yaml),
+# and then set the env var SNPIT_CONFIG to point
+# to that config.
+
+system:
+  webserver:
+    flask_secret_key: o6950bi77gvbc2jie49lklotwll69ayv
+    sessionstore: /sessions
+    emailfrom: nobody@nowhere.org
+    smtpserver: mailhog
+    smtpport: 1025
+    smtpusessl: False
+    smtpusername: NULL
+    smtppassword: NULL
+  
+  db:
+    postgres_host: postgres
+    postgres_port: 5432
+    postgres_database: roman_snpit
+    postgres_username: postgres
+    postgres_password: fragile
+    # These next two are for debugging purposes and should always be false in production
+    echoqueries: true
+    alwaysexplain: false

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
   "python-dateutil>=2.9.0.post0,<3.0.0",
   "pytz>=2025.2",
   "pyyaml>=6.0.3,<7.0.0",
-  "rkwebutil>=2.2.3,<3.0.0",
+  "rkwebutil>=2.4.0,<3.0.0",
   "roman-datamodels>=0.27.0,<0.28.0",
   "scipy>=1.16.2,<2.0.0",
   "simplejson>=3.20.2,<4.0.0"

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0/roman_snpit_snappl.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: roman_snpit_snappl
-Version: 0.16.0
+Version: 0.17.0
 Summary: General, database, and photometry utilities for the Roman SNPIT
 Author: Roman Supernova Project Infrastructure Team
 Maintainer-email: Roman SN PIT <raknop@lbl.gov>
@@ -22,7 +22,7 @@ Requires-Dist: pycryptodome<4.0.0,>=3.23.0
 Requires-Dist: python-dateutil<3.0.0,>=2.9.0.post0
 Requires-Dist: pytz>=2025.2
 Requires-Dist: pyyaml<7.0.0,>=6.0.3
-Requires-Dist: rkwebutil<3.0.0,>=2.2.3
+Requires-Dist: rkwebutil<3.0.0,>=2.4.0
 Requires-Dist: roman-datamodels<0.28.0,>=0.27.0
 Requires-Dist: scipy<2.0.0,>=1.16.2
 Requires-Dist: simplejson<4.0.0,>=3.20.2

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/roman_snpit_snappl.egg-info/SOURCES.txt

@@ -60,7 +60,9 @@ changes/79.snappl.rst
 changes/8.snappl.rst
 changes/81.snappl.rst
 changes/82.snappl.rst
+changes/83.snappl.rst
 changes/84.feature.rst
+changes/85.feature.rst
 changes/9.snappl.rst
 docker/postgres/Dockerfile
 docker/postgres/postgresql.conf
@@ -109,6 +111,7 @@ snappl/utils.py
 snappl/wcs.py
 snappl/_dev/__init__.py
 snappl/_dev/scm_version.py
+snappl/admin/load_ou2024_l2images.py
 snappl/admin/load_snana_ou2024_diaobject.py
 snappl/data/README.rst
 snappl/db/baseview.py

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/roman_snpit_snappl.egg-info/requires.txt

@@ -10,7 +10,7 @@ pycryptodome<4.0.0,>=3.23.0
 python-dateutil<3.0.0,>=2.9.0.post0
 pytz>=2025.2
 pyyaml<7.0.0,>=6.0.3
-rkwebutil<3.0.0,>=2.2.3
+rkwebutil<3.0.0,>=2.4.0
 roman-datamodels<0.28.0,>=0.27.0
 scipy<2.0.0,>=1.16.2
 simplejson<4.0.0,>=3.20.2

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/snappl/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.16.0'
-__version_tuple__ = version_tuple = (0, 16, 0)
+__version__ = version = '0.17.0'
+__version_tuple__ = version_tuple = (0, 17, 0)
 
-__commit_id__ = commit_id = 'g9ae526c23'
+__commit_id__ = commit_id = 'gde6e5cc0b'

roman_snpit_snappl-0.17.0/snappl/admin/load_ou2024_l2images.py

@@ -0,0 +1,141 @@
+import uuid
+import pathlib
+import multiprocessing
+import functools
+
+import psycopg
+
+from snappl.image import OpenUniverse2024FITSImage
+from snappl.logger import SNLogger
+from snappl.utils import asUUID
+from snappl.provenance import Provenance
+import snappl.db.db
+
+
+# python multiprocesing irritates me; it seems you can't
+#   send a class method as the function
+def parse_fits_file( relpath, base_path=None, provid=None ):
+    base_path = pathlib.Path( base_path )
+    provid = asUUID( provid )
+    # import random
+    # import remote_pdb;
+    # remote_pdb.RemotePdb( '127.0.0.1', random.randint( 4000, 5000 ) ).set_trace()
+    image = OpenUniverse2024FITSImage( path=base_path / relpath )
+    header = image.get_fits_header()
+    wcs = image.get_wcs()
+
+    width = int( header['NAXIS2'] )
+    height = int( header['NAXIS1'] )
+    ra, dec = wcs.pixel_to_world( width / 2., height / 2. )
+    ra_corner_00, dec_corner_00 = wcs.pixel_to_world( 0., 0. )
+    ra_corner_10, dec_corner_10 = wcs.pixel_to_world( width-1, 0. )
+    ra_corner_01, dec_corner_01 = wcs.pixel_to_world( 0., height-1 )
+    ra_corner_11, dec_corner_11 = wcs.pixel_to_world( width-1, height-1 )
+    exptime = float( header['EXPTIME'] )
+
+    params = { 'id': uuid.uuid4(),
+               'provenance_id': provid,
+               'pointing': image.pointing,
+               'sca': image.sca,
+               'filter': image.band,
+               'ra': ra,
+               'dec': dec,
+               'ra_corner_00': ra_corner_00,
+               'ra_corner_01': ra_corner_01,
+               'ra_corner_10': ra_corner_10,
+               'ra_corner_11': ra_corner_11,
+               'dec_corner_00': dec_corner_00,
+               'dec_corner_01': dec_corner_01,
+               'dec_corner_10': dec_corner_10,
+               'dec_corner_11': dec_corner_11,
+               'filepath': str( relpath ),
+               'width': width,
+               'height': height,
+               'format': 1,
+               'mjd_start': image.mjd,
+               'exptime': exptime,
+               'properties': psycopg.types.json.Jsonb( {} )
+              }
+    return params
+
+
+class OU2024_L2image_loader:
+    def __init__( self, provid, base_path ):
+        self.provid = provid.id if isinstance( provid, Provenance ) else provid
+        self.base_path = pathlib.Path( base_path )
+        self.dbcon = None
+
+
+    def collect_ou2024_l2image_paths( self, relpath ):
+        subdirs = []
+        imagefiles = []
+
+        for fullpath in ( self.base_path / relpath ).iterdir():
+            fullpath = fullpath.resolve()
+            if fullpath.is_dir():
+                subdirs.append( fullpath.relative_to( self.base_path ) )
+            elif ( fullpath.name[-5:] == '.fits' ) or ( fullpath.name[-8:] == '.fits.gz' ):
+                imagefiles.append( fullpath.relative_to( self.base_path ) )
+
+        for subdir in subdirs:
+            imagefiles.extend( self.collect_ou2024_l2image_paths(subdir) )
+
+        return imagefiles
+
+
+    def save_to_db( self ):
+        if len( self.copydata ) > 0:
+            SNLogger.info( f"Loading {len(self.copydata)} images to database..." )
+            snappl.db.db.L2Image.bulk_insert_or_upsert( self.copydata, dbcon=self.dbcon )
+            self.totloaded += len( self.copydata )
+            self.copydata = []
+
+    def append_to_copydata( self, relpath ):
+        self.copydata.append( relpath )
+        if len(self.copydata) % self.loadevery == 0:
+            self.save_to_db()
+
+    def omg( self, e ):
+        self.errors.append( e )
+
+    def __call__( self, dbcon=None, loadevery=1000, nprocs=1 ):
+        toload = self.collect_ou2024_l2image_paths( '.' )
+        self.totloaded = 0
+        self.copydata = []
+        self.loadevery = loadevery
+        self.errors = []
+
+        SNLogger.info( f"Loading {len(toload)} files in {nprocs} processes...." )
+        do_parse_fits_file = functools.partial( parse_fits_file,
+                                                base_path=self.base_path,
+                                                provid=self.provid )
+
+        with snappl.db.db.DBCon( dbcon ) as self.dbcon:
+            if nprocs > 1:
+                with multiprocessing.Pool( nprocs ) as pool:
+                    for path in toload:
+                        pool.apply_async( do_parse_fits_file,
+                                          args=[ str(path) ],
+                                          callback=self.append_to_copydata,
+                                          error_callback=self.omg
+                                         )
+                    pool.close()
+                    pool.join()
+                if len( self.errors ) > 0:
+                    nl = "\n"
+                    SNLogger.error( f"Got errors loading FITS files:\n{nl.join(str(e) for e in self.errors)}" )
+                    raise RuntimeError( "Massive failure." )
+
+            elif nprocs == 1:
+                for path in toload:
+                    self.append_to_copydata( do_parse_fits_file( path ) )
+
+            else:
+                raise ValueError( "Dude, nprocs needs to be positive, not {nprocs}" )
+
+            # Get any residual ones that didn't pass the "send to db" threshold
+            self.save_to_db()
+
+            SNLogger.info( f"Loaded {self.totloaded} of {len(toload)} images to database." )
+
+        self.dbcon = None

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/snappl/config.py

@@ -16,6 +16,11 @@ class NoValue:
     pass
 
 
+class NotFoundValue:
+    """Used internally by Config, ignore."""
+    pass
+
+
 class Config:
     """Interface for yaml config file.
 
@@ -349,7 +354,7 @@ class Config:
 
 
     @staticmethod
-    def get( configfile=None, setdefault=None, static=True ):
+    def get( configfile=None, setdefault=None, static=True, clone=None ):
         """Returns a Config object.
 
         Parameters
@@ -396,6 +401,15 @@ class Config:
             modify the config.  Call Config.get(static=False) to get a
             modifiable version of the default config.
 
+        clone : Config, default None
+            If given, return a clone of this Config object.  The
+            returned object, if all is working properly, is a deep copy,
+            so it should be safe to mangle it.  It's not possibe to set
+            a cloned config object as default, nor is the cloned object
+            ever set as a singleton, so setdefault and static are both
+            ignored and treated as False when making a clone.
+
+
         Returns
         -------
             Config object
@@ -418,6 +432,12 @@ class Config:
         default", then an exception will be raised.
 
         """
+
+        if clone is not None:
+            if configfile is not None:
+                raise ValueError( "Only specify one of clone or configfile." )
+            return Config( clone=clone, _ok_to_call=True )
+
         if configfile is None:
             if Config._default is not None:
                 configfile = Config._default
@@ -540,6 +560,7 @@ class Config:
             SNLogger.exception( f'Exception trying to load config from {configfile}' )
             raise e
 
+
     def value( self, field, default=NoValue(), struct=None ):
         """Get a value from the config structure.
 
@@ -596,49 +617,8 @@ class Config:
 
         """
 
-        if struct is None:
-            struct = self._data
-        fields, isleaf, curfield, ifield = self._fieldsep( field )
-
-        if isinstance( struct, list ):
-            if ifield is None:
-                raise ValueError( f'Failed to parse {curfield} as an integer index' )
-            if ifield >= len(struct):
-                raise ValueError( f'{ifield} >= {len(struct)}, the length of the list' )
-            if isleaf:
-                return_value = struct[ifield]
-            else:
-                try:
-                    return_value = self.value( ".".join(fields[1:]), default, struct[ifield] )
-                except Exception as e:
-                    if isinstance(default, NoValue):
-                        raise ValueError( f'Error getting list element {ifield}' ) from e
-                    else:
-                        return_value = default
-        elif isinstance( struct, dict ):
-            if curfield not in struct:
-                if isinstance(default, NoValue):
-                    raise ValueError( f'Field {curfield} doesn\'t exist' )
-                else:
-                    return_value = default
-            elif isleaf:
-                return_value = struct[curfield]
-            else:
-                try:
-                    return_value = self.value( ".".join(fields[1:]), default, struct[curfield] )
-                except Exception as e:
-                    if isinstance(default, NoValue):
-                        raise ValueError( f'Error getting field {field} from {curfield}' ) from e
-                    else:
-                        return_value = default
-        else:
-            if not isleaf:
-                raise ValueError( f'Tried to get field {field} of scalar {curfield}!' )
-            return_value = struct
-
-        if isinstance(return_value, (dict, list)):
-            return_value = copy.deepcopy( return_value )
-        return return_value
+        _, _, value = self._parent_key_and_value( field, parent=None, struct=struct, default=default )
+        return value
 
 
     def set_value( self, field, value, structpass=None, appendlists=False ):
@@ -736,6 +716,97 @@ class Config:
                     structpass.struct = [ structchuck.struct ]
 
 
+    def _parent_key_and_value( self, field, parent=None, struct=None, fullfield=None, default=NoValue ):
+        fullfield = fullfield if fullfield is not None else field
+        if struct is None:
+            struct = self._data
+        fields, isleaf, curfield, ifield = self._fieldsep( field )
+
+        if isinstance( struct, list ):
+            if ifield is None:
+                raise ValueError( f'Failed to parse {curfield} of {fullfield} as an integer index' )
+            elif ifield < 0:
+                if isinstance( default, NoValue ):
+                    raise ValueError( f'Array index {ifield} is negative for {fullfield}' )
+                else:
+                    return_parent = struct
+                    return_key = ifield
+                    return_value = default
+            elif ifield >= len(struct):
+                if isinstance( default, NoValue ):
+                    raise ValueError( f'{ifield} >= {len(struct)}, the length of the list for {fullfield}' )
+                else:
+                    return_parent = struct
+                    return_key = ifield
+                    return_value = default
+            elif isleaf:
+                return_parent = struct
+                return_key = ifield
+                return_value = struct[ifield]
+            else:
+                return_parent, return_key, return_value = self._parent_key_and_value( ".".join(fields[1:]),
+                                                                                      parent=struct,
+                                                                                      struct=struct[ifield],
+                                                                                      fullfield=fullfield,
+                                                                                      default=default )
+
+        elif isinstance( struct, dict ):
+            if curfield not in struct:
+                if isinstance( default, NoValue ):
+                    raise ValueError( f"Can't find field {fullfield}" )
+                else:
+                    return_parent = struct
+                    return_key = curfield
+                    return_value = default
+            elif isleaf:
+                return_parent = struct
+                return_key = curfield
+                return_value = struct[curfield]
+            else:
+                return_parent, return_key, return_value = self._parent_key_and_value( ".".join(fields[1:]),
+                                                                                      parent=struct,
+                                                                                      struct=struct[curfield],
+                                                                                      fullfield=fullfield,
+                                                                                      default=default )
+
+        else:
+            if not isleaf:
+                raise ValueError( f'Tried to get field {field} of scalar {curfield} in {fullfield}!' )
+            return_parent = parent
+            return_key = curfield
+            return_value = struct
+
+        if isinstance(return_value, (dict, list)):
+            return_value = copy.deepcopy( return_value )
+
+        return return_parent, return_key, return_value
+
+
+    def delete_field( self, field, missing_ok=False ):
+        """Remove a field from the config.
+
+        Use this this with great care.
+
+        """
+        if self._static:
+            raise RuntimeError( "Not permitted to modify static Config object." )
+
+        parent, key, value = self._parent_key_and_value( field, default=NotFoundValue() )
+        if isinstance( value, NotFoundValue ):
+            if missing_ok:
+                return
+            else:
+                raise ValueError( f"Can't find config field {field} to delete it." )
+
+        if isinstance( parent, list ):
+            parent.pop( key )
+        elif isinstance( parent, dict ):
+            del parent[key]
+        else:
+            # ... I think this shouldn't ever actually happen
+            raise RuntimeError( "Rob, figure out how to cope with this." )
+
+
     @classmethod
     def _fieldsep( cls, field ):
         """Parses a period-separated config specifier string.  Internal use only.
@@ -764,6 +835,33 @@ class Config:
         return fields, isleaf, curfield, ifield
 
 
+    @classmethod
+    def _is_parent_field( cls, parent, child ):
+        parent = parent.split( "." )
+        child = child.split( "." )
+        if len(parent) >= len(child):
+            return False
+        return child[ 0 : len(parent) ] == parent
+
+
+    @classmethod
+    def _allkeys( cls, struct, base="" ):
+        keys = []
+
+        if isinstance( struct, dict ):
+            for key, val in struct.items():
+                keys.append( f"{base}{key}" )
+                keys.extend( cls._allkeys( val, base=f'{base}{key}.' ) )
+        elif isinstance( struct, list ):
+            for i, val in enumerate( struct ):
+                keys.append( f"{base}{i}" )
+                keys.extend( cls._allkeys( val, base=f'{base}{i}.' ) )
+        else:
+            return []
+
+        return keys
+
+
     def _pathify( self, fname ):
         fname = pathlib.Path( fname )
         if fname.is_absolute():
@@ -888,5 +986,67 @@ class Config:
                 _dict[key] = getattr( args, arg )
 
 
+    def dump_to_dict_for_params( self, omitkeys=['system'], keepkeys=None ):
+        """Dump the config to a dictionary suitable for use in a Provenance params field.
+
+        Specify one of omitkeys or keepkeys.
+
+        Parameters
+        ----------
+          omitkeys: None, or list of str
+            This is a list of keys to delete from the config before
+            exporting it.  (The internal state of the config will not be
+            affected, only what is exported.)  Be careful not to list a
+            subkey of a key that's already earlier in the list, or
+            you'll get errors.
+
+            By default, the top-level key "system" is deleted, as per
+            the Roman SNPIT standard that this holds all of the (but
+            only the) system-specific config needed to run at a
+            particular place.  (system should not include anything that
+            changes the behavior of the code.)
+
+            However, this default is a bit profligate, as it will keep
+            all of the config options for all codes, not just the code
+            you're running right now.  Use with thought.
+
+          keepkeys: None, or list of str
+            This is a list of keeps to keep in the export.  Currently,
+            only top-level keys are supported.
+
+        Returns
+        -------
+          dict
+            This is a deep copy of the internal dictionary, so ideally
+            you should be able to do anything you want to it without
+            screwing up the internal config state.
+
+        """
+
+        if ( omitkeys is not None ) and ( keepkeys is not None ):
+            raise ValueError( "Only specify one of omitkeys or keepkeys." )
+
+        cfg = Config.get( clone=self )
+
+        if omitkeys is not None:
+            for kw in omitkeys:
+                cfg.delete_field( kw )
+
+        elif keepkeys is not None:
+            allkeys = Config._allkeys( cfg._data )
+            allkeys.reverse()
+            for kw in allkeys:
+                if not ( ( kw in keepkeys )
+                         or any( [ Config._is_parent_field( i, kw ) for i in keepkeys ] )
+                         or any( [ Config._is_parent_field( kw, i ) for i in keepkeys ] )
+                        ):
+                    cfg.delete_field( kw )
+
+        else:
+            raise ValueError( "Must specify either omitkeys or keepkeys." )
+
+        return cfg._data
+
+
 if __name__ == "__main__":
     Config.init()

{roman_snpit_snappl-0.16.0 → roman_snpit_snappl-0.17.0}/snappl/db/baseview.py

@@ -71,7 +71,7 @@ class BaseView( flask.views.View ):
             #   writes out NaN which is not standard JSON and which
             #   the javascript JSON parser chokes on.  Sigh.
             if isinstance( retval, dict ) or isinstance( retval, list ):
-                SNLogger.warning( f"Dumping to json: {retval}" )
+                # SNLogger.warning( f"Dumping to json: {retval}" )
                 return ( simplejson.dumps( retval, ignore_nan=True, cls=SNPITJsonEncoder ),
                          200, { 'Content-Type': 'application/json' } )
             elif isinstance( retval, str ):