vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/nwp/algo/ifsroot.py

@@ -0,0 +1,311 @@
+"""
+Abstract base class for any AlgoComponent leveraging the Arpege/IFS code.
+"""
+
+from bronx.fancies import loggers
+import footprints
+
+from vortex.algo.components import Parallel, ParallelIoServerMixin, AlgoComponentError
+from vortex.syntax.stdattrs import model
+from vortex.tools import grib
+
+from . import ifsnaming  # @UnusedImport
+from ..syntax.stdattrs import algo_member
+from ..tools import satrad, drhook
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class IFSParallel(Parallel, ParallelIoServerMixin,
+                  satrad.SatRadDecoMixin, drhook.DrHookDecoMixin, grib.EcGribDecoMixin):
+    """Abstract IFSModel parallel algo components."""
+
+    _abstract = True
+    _footprint = [
+        model, algo_member,
+        dict(
+            info = 'Abstract AlgoComponent for anything based on Arpege/IFS.',
+            attr = dict(
+                kind = dict(
+                    info            = 'The kind of processing we want the Arpege/IFS binary to perform.',
+                    default         = 'ifsrun',
+                    doc_zorder      = 90,
+                ),
+                model = dict(
+                    values  = ['arpege', 'arp', 'arp_court', 'aladin', 'ald',
+                               'arome', 'aro', 'aearp', 'pearp', 'ifs', 'alaro', 'harmoniearome']
+                ),
+                ioname = dict(
+                    default = 'nwpioserv',
+                ),
+                binarysingle = dict(
+                    default = 'basicnwp',
+                ),
+                conf = dict(
+                    info = 'The configuration number given to Arpege/IFS.',
+                    type            = int,
+                    optional        = True,
+                    default         = 1,
+                    doc_visibility  = footprints.doc.visibility.ADVANCED,
+                ),
+                timescheme = dict(
+                    info = 'The timescheme that will be used by Arpege/IFS model.',
+                    optional        = True,
+                    default         = 'sli',
+                    values          = ['eul', 'eulerian', 'sli', 'semilag'],
+                    remap           = dict(
+                        eulerian = 'eul',
+                        semilag  = 'sli'
+                    ),
+                    doc_visibility  = footprints.doc.visibility.ADVANCED,
+                ),
+                timestep = dict(
+                    info     = 'The timestep of the Arpege/IFS model.',
+                    type     = float,
+                    optional = True,
+                    default  = 600.,
+                ),
+                fcterm = dict(
+                    info     = 'The forecast term of the Arpege/IFS model.',
+                    type     = int,
+                    optional = True,
+                    default  = 0,
+                ),
+                fcunit = dict(
+                    info     = 'The unit used in the *fcterm* attribute.',
+                    optional = True,
+                    default  = 'h',
+                    values   = ['h', 'hour', 't', 'step', 'timestep'],
+                    remap = dict(
+                        hour = 'h',
+                        step = 't',
+                        timestep = 't',
+                    )
+                ),
+                xpname = dict(
+                    info = 'The default labelling of files used in Arpege/IFS model.',
+                    optional        = True,
+                    default         = 'XPVT',
+                    doc_visibility  = footprints.doc.visibility.ADVANCED,
+                ),
+            )
+        )
+    ]
+
+    def fstag(self):
+        """Extend default tag with ``kind`` value."""
+        return super().fstag() + '.' + self.kind
+
+    def valid_executable(self, rh):
+        """Be sure that the specifed executable is ifsmodel compatible."""
+        valid = super().valid_executable(rh)
+        try:
+            return valid and bool(rh.resource.realkind == 'ifsmodel')
+        except (ValueError, TypeError):
+            return False
+
+    def spawn_hook(self):
+        """Usually a good habit to dump the fort.4 namelist."""
+        super().spawn_hook()
+        if self.system.path.exists('fort.4'):
+            self.system.subtitle('{:s} : dump namelist <fort.4>'.format(self.realkind))
+            self.system.cat('fort.4', output=False)
+
+    def spawn_command_options(self):
+        """Dictionary provided for command line factory."""
+        return dict(
+            name=(self.xpname + 'xxxx')[:4].upper(),
+            conf=self.conf,
+            timescheme=self.timescheme,
+            timestep=self.timestep,
+            fcterm=self.fcterm,
+            fcunit=self.fcunit,
+        )
+
+    def naming_convention(self, kind, rh, actualfmt=None, **kwargs):
+        """Create an appropriate :class:`IFSNamingConvention`.
+
+        :param str kind: The :class:`IFSNamingConvention` object kind.
+        :param rh: The binary's ResourceHandler.
+        :param actualfmt: The format of the target file.
+        :param dict kwargs: Any argument you may see fit.
+        """
+        nc_args = dict(model=self.model,
+                       conf=self.conf,
+                       xpname=self.xpname)
+        nc_args.update(kwargs)
+        nc = footprints.proxy.ifsnamingconv(kind=kind,
+                                            actualfmt=actualfmt,
+                                            cycle=rh.resource.cycle,
+                                            **nc_args)
+        if nc is None:
+            raise AlgoComponentError("No IFSNamingConvention was found.")
+        return nc
+
+    def do_climfile_fixer(self, rh, convkind, actualfmt=None, geo=None, **kwargs):
+        """Is it necessary to fix the climatology file ? (i.e link in the appropriate file).
+
+        :param rh: The binary's ResourceHandler.
+        :param str convkind: The :class:`IFSNamingConvention` object kind.
+        :param actualfmt: The format of the climatology file.
+        :param geo: The geometry of the desired geometry file.
+        :param dict kwargs: Any argument you may see fit (used to create and call
+                            the IFSNamingConvention object.
+        """
+        nc = self.naming_convention(kind=convkind, rh=rh, actualfmt=actualfmt, **kwargs)
+        nc_args = dict()
+        if geo:
+            nc_args['area'] = geo.area
+        nc_args.update(kwargs)
+        return not self.system.path.exists(nc(** nc_args))
+
+    def climfile_fixer(self, rh, convkind,
+                       month, geo=None, notgeo=None, actualfmt=None,
+                       inputrole=None, inputkind=None, **kwargs):
+        """Fix the climatology files (by choosing the appropriate month, geometry, ...)
+
+        :param rh: The binary's ResourceHandler.
+        :param str convkind: The :class:`IFSNamingConvention` object kind.
+        :param ~bronx.stdtypes.date.Month month: The climatlogy file month
+        :param geo: The climatlogy file geometry
+        :param notgeo: Exclude these geometries during the climatology file lookup
+        :param actualfmt: The format of the climatology file.
+        :param inputrole: The section's role in which Climatology files are looked for.
+        :param inputkind: The section's realkind in which Climatology files are looked for/
+        :param dict kwargs: Any argument you may see fit (used to create and call
+                            the IFSNamingConvention object).
+        """
+        if geo is not None and notgeo is not None:
+            raise ValueError('*geo* and *notgeo* cannot be provided together.')
+
+        def check_month(actualrh):
+            return bool(hasattr(actualrh.resource, 'month') and
+                        actualrh.resource.month == month)
+
+        def check_month_and_geo(actualrh):
+            return (check_month(actualrh) and
+                    actualrh.resource.geometry.tag == geo.tag)
+
+        def check_month_and_notgeo(actualrh):
+            return (check_month(actualrh) and
+                    actualrh.resource.geometry.tag != notgeo.tag)
+
+        if geo:
+            checker = check_month_and_geo
+        elif notgeo:
+            checker = check_month_and_notgeo
+        else:
+            checker = check_month
+
+        nc = self.naming_convention(kind=convkind, rh=rh, actualfmt=actualfmt, **kwargs)
+        nc_args = dict()
+        if geo:
+            nc_args['area'] = geo.area
+        nc_args.update(kwargs)
+        target_name = nc(** nc_args)
+
+        self.system.remove(target_name)
+
+        logger.info("Linking in the %s file (%s) for month %s.", convkind, target_name, month)
+        rc = self.setlink(initrole=inputrole, initkind=inputkind, inittest=checker,
+                          initname=target_name)
+        return target_name if rc else None
+
+    def all_localclim_fixer(self, rh, month, convkind='targetclim', actualfmt=None,
+                            inputrole=('LocalClim', 'TargetClim', 'BDAPClim'),
+                            inputkind='clim_bdap', **kwargs):
+        """Fix all the local/BDAP climatology files (by choosing the appropriate month)
+
+        :param rh: The binary's ResourceHandler.
+        :param ~bronx.stdtypes.date.Month month: The climatology file month
+        :param str convkind: The :class:`IFSNamingConvention` object kind.
+        :param actualfmt: The format of the climatology file.
+        :param inputrole: The section's role in which Climatology files are looked for.
+        :param inputkind: The section's realkind in which Climatology files are looked for/
+        :param dict kwargs: Any argument you may see fit (used to create and call
+                            the IFSNamingConvention object.
+        :return: The list of linked files
+        """
+
+        def check_month(actualrh):
+            return bool(hasattr(actualrh.resource, 'month') and
+                        actualrh.resource.month == month)
+
+        nc = self.naming_convention(kind=convkind, rh=rh, actualfmt=actualfmt, **kwargs)
+        dealtwith = list()
+
+        for tclimrh in [x.rh for x in self.context.sequence.effective_inputs(
+                role=inputrole, kind=inputkind,
+        ) if x.rh.resource.month == month]:
+            thisclim = tclimrh.container.localpath()
+            thisname = nc(area=tclimrh.resource.geometry.area)
+            if thisclim != thisname:
+                logger.info("Linking in the %s to %s for month %s.", thisclim, thisname, month)
+                self.system.symlink(thisclim, thisname)
+                dealtwith.append(thisname)
+
+        return dealtwith
+
+    def find_namelists(self, opts=None):
+        """Find any namelists candidates in actual context inputs."""
+        return [x.rh
+                for x in self.context.sequence.effective_inputs(kind=('namelist', 'namelistfp'))]
+
+    def _set_nam_macro(self, namcontents, namlocal, macro, value):
+        """Set a namelist macro and log it!"""
+        namcontents.setmacro(macro, value)
+        logger.info('Setup macro %s=%s in %s', macro, str(value), namlocal)
+
+    def prepare_namelist_delta(self, rh, namcontents, namlocal):
+        """Apply a namelist delta depending on the cycle of the binary."""
+        # TODO: The mapping between the dict that contains the settings
+        # (i.e elf.spawn_command_options()) and actual namelist keys should
+        # be done by an extra class ... and it could be generalized to mpi
+        # setup by the way !
+        nam_updated = False
+        # For cy41 onward, replace some namelist macros with the command line
+        # arguments
+        if rh.resource.cycle >= 'cy41':
+            if 'NAMARG' in namcontents:
+                opts_arg = self.spawn_command_options()
+                self._set_nam_macro(namcontents, namlocal, 'CEXP', opts_arg['name'])
+                self._set_nam_macro(namcontents, namlocal, 'TIMESTEP', opts_arg['timestep'])
+                fcstop = '{:s}{:d}'.format(opts_arg['fcunit'], opts_arg['fcterm'])
+                self._set_nam_macro(namcontents, namlocal, 'FCSTOP', fcstop)
+                nam_updated = True
+            else:
+                logger.info('No NAMARG block in %s', namlocal)
+
+        if self.member is not None:
+            for macro_name in ('MEMBER', 'PERTURB'):
+                self._set_nam_macro(namcontents, namlocal, macro_name, self.member)
+            nam_updated = True
+        return nam_updated
+
+    def prepare_namelists(self, rh, opts=None):
+        """Update each of the namelists."""
+        namcandidates = self.find_namelists(opts)
+        self.system.subtitle('Namelist candidates')
+        for nam in namcandidates:
+            nam.quickview()
+        for namrh in namcandidates:
+            namc = namrh.contents
+            if self.prepare_namelist_delta(rh, namc, namrh.container.actualpath()):
+                if namc.dumps_needs_update:
+                    logger.info('Rewritting the %s namelists file.', namrh.container.actualpath())
+                    namc.rewrite(namrh.container)
+
+    def prepare(self, rh, opts):
+        """Set some variables according to target definition."""
+        super().prepare(rh, opts)
+        # Namelist fixes
+        self.prepare_namelists(rh, opts)
+
+    def execute_single(self, rh, opts):
+        """Standard IFS-Like execution parallel execution."""
+        if rh.resource.cycle < 'cy46':
+            self.system.ls(output='dirlst')
+        super().execute_single(rh, opts)

vortex/nwp/algo/monitoring.py

@@ -0,0 +1,202 @@
+"""
+AlgoComponents dedicated to computations related to observations monitoring.
+"""
+
+from bronx.fancies import loggers
+
+from vortex.algo.components import Parallel
+from vortex.syntax.stdattrs import a_date, a_model, a_cutoff
+from ..tools import odb, drhook
+
+#: Automatic export of Monitoring class
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+class OdbMonitoring(Parallel, odb.OdbComponentDecoMixin, drhook.DrHookDecoMixin):
+    """Compute monitoring statistics."""
+
+    _footprint = dict(
+        attr = dict(
+            kind = dict(
+                values = ['monitoring'],
+            ),
+            npool = dict(
+                default = 1,
+                optional = True,
+            ),
+            obs = dict(
+                values = ['all', 'used'],
+            ),
+            date = a_date,
+            model= a_model,
+            cutoff = a_cutoff,
+            start = dict(
+                type  = bool,
+                default = False,
+                optional = True,
+            ),
+            cumul = dict(
+                type = bool,
+                default = True,
+                optional = True,
+            ),
+            extend = dict(
+                type = bool,
+                default = False,
+                optional = True,
+            ),
+            stage = dict(
+                values = ['can', 'surf', 'surface', 'atm', 'atmospheric'],
+                remap = dict(can='surf', surface='surf', atmospheric='atm'),
+                info = 'The processing stage of the ODB base.',
+            ),
+        )
+    )
+
+    def _fix_nam_macro(self, rh, macro, value):
+        """Set a given namelist macro and issue a log message."""
+        rh.contents.setmacro(macro, value)
+        logger.info('Setup %s macro to %s in %s', macro, value, rh.container.actualpath())
+
+    def prepare(self, rh, opts):
+        """Update some variables in the namelist and check the presence of the accumulated statistics file."""
+
+        sh = self.system
+
+        # Looking for input observations
+
+        # Virtual upper-air observations database
+        obsatm_virt = [
+            x for x in self.lookupodb(fatal=False)
+            if (x.rh.resource.stage.startswith('matchup') or
+                x.rh.resource.stage.startswith('screening')) and x.rh.resource.part == 'virtual'
+        ]
+
+        # Single upper-air observations database
+        obsatm_single = [
+            x for x in self.lookupodb(fatal=False)
+            if x.rh.resource.stage.startswith('matchup') or x.rh.resource.stage.startswith('screening')
+        ]
+        if len(obsatm_single) > 1:
+            obsatm_single = []
+
+        # Surface observations database
+        obssurf = [
+            x for x in self.lookupodb(fatal=False)
+            if x.rh.resource.stage.startswith('canari') and (x.rh.resource.part == 'surf' or
+                                                             x.rh.resource.part == 'ground')
+        ]
+
+        # One database at a time
+        if not (obsatm_virt or obsatm_single) and self.stage == 'atm':
+            raise ValueError('Could not find any ODB matchup or screening ECMA database')
+        if not obssurf and self.stage == 'surf':
+            raise ValueError('Could not find any ODB surface ECMA database')
+
+        # Set actual ODB paths
+        if obsatm_virt:
+            ecma = obsatm_virt.pop(0)
+        elif obsatm_single:
+            ecma = obsatm_single.pop(0)
+        else:
+            ecma = obssurf.pop(0)
+        ecma_path = sh.path.abspath(ecma.rh.container.localpath())
+        self.odb.fix_db_path(ecma.rh.resource.layout, ecma_path)
+        self.env.IOASSIGN = sh.path.join(ecma_path, 'IOASSIGN')
+        logger.info('Setting ODB env %s = %s.', 'IOASSIGN', sh.path.join(ecma_path, 'IOASSIGN'))
+
+        # Let ancestors handling most of the env setting
+        super().prepare(rh, opts)
+
+        # Force to start a new accumulated statistics file if first day and first hour of the month
+        mnt_start = self.start
+
+        if not mnt_start and int(self.date.day) == 1 and int(self.date.hh) == 0 and not self.extend:
+            logger.info('First day and first hour of the month : force start attribute to True.')
+            mnt_start = True
+
+        mnt_cumul = self.cumul
+        if self.cutoff == 'production':
+            mnt_cumul = False
+            logger.info('No output accumulated statistics file will be produced because '
+                        'cutoff = production : force cumul to False')
+
+        # Monitoring namelist
+        namrh = self.context.sequence.effective_inputs(
+            role='Namelist',
+            kind='namelist',)
+        if len(namrh) != 1:
+            logger.critical('There must be exactly one namelist for monitoring. Stop.')
+            raise ValueError('There must be exactly one namelist for monitoring. Stop.')
+        namrh = namrh[0].rh
+
+        # Cumulated statistics file
+        cumulrh = self.context.sequence.effective_inputs(
+            role='Cumulated monitoring statistics',
+            kind='accumulated_stats',
+        )
+
+        if len(cumulrh) > 1:
+            logger.critical('There must be at most one accumulated statistics file.Stop.')
+            raise ValueError('There must be one accumulated statistics file or none.Stop.')
+        else:
+            if len(cumulrh) == 0:
+                if not mnt_start:
+                    if mnt_cumul:
+                        logger.critical('There must be one input accumulated statistics file. Stop.')
+                        raise ValueError('There must be one input accumulated statistics file. Stop.')
+                    else:
+                        logger.info('No input accumulated statistics file is necessary.')
+                        logger.info('No output accumulated statistics file will be produced.')
+                else:
+                    if mnt_cumul:
+                        logger.info('No input accumulated statistics file. It will be created by the binary.')
+                    else:
+                        logger.info('No output accumulated statistics file will be produced.')
+            else:
+                cumulrh = cumulrh[0].rh
+                if not mnt_cumul:
+                    logger.info('No input accumulated statistics file is necessary(start=False).')
+                    cumulrh.container.clear()
+                else:
+                    if mnt_start:
+                        logger.info('No input accumulated statistics file is necessary (start=True)')
+                        cumulrh.container.clear()
+
+        self._fix_nam_macro(namrh, 'JOUR', int(self.date.ymd))
+        self._fix_nam_macro(namrh, 'RES', int(self.date.hh))
+
+        self._fix_nam_macro(namrh, 'LLADMON', mnt_cumul)
+        self._fix_nam_macro(namrh, 'LLADAJ', mnt_cumul and not mnt_start)
+
+        self._fix_nam_macro(namrh, 'LLFLAG', self.obs != 'all')
+
+        self._fix_nam_macro(namrh, 'LLARO', self.model == 'arome')
+        self._fix_nam_macro(namrh, 'LLVRP', self.model == 'varpack')
+        self._fix_nam_macro(namrh, 'LLCAN', self.stage == 'surf')
+
+        if namrh.contents.dumps_needs_update:
+            namrh.contents.rewrite(namrh.container)
+        namrh.container.cat()
+
+    def postfix(self, rh, opts):
+        """Remove all empty files and find out if any special resources have been produced."""
+
+        sh = self.system
+        self.system.dir(output=False, fatal=False)
+        allfiles = sh.ls()
+        for f in allfiles:
+            if self.system.path.getsize(f) == 0:
+                logger.info('Remove %s because size of %s is zero.', f, f)
+                sh.remove(f)
+
+        obspoint_out = sh.ls('point.*')
+        if obspoint_out:
+            dest = 'obslocationpack'
+            logger.info('Creating an OBSLOCATION pack: %s', dest)
+            sh.mkdir(dest)
+            for fname in obspoint_out:
+                sh.mv(fname, dest)
+        self.system.dir(output=False, fatal=False)