vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/nwp/data/modelstates.py

@@ -0,0 +1,334 @@
+"""
+Resources to handle any NWP model state variable.
+"""
+
+import re
+
+from bronx.fancies import loggers
+
+from vortex.data.flow import GeoFlowResource
+from vortex.syntax.stdattrs import term_deco, term
+from vortex.syntax.stddeco import namebuilding_insert
+from bronx.stdtypes.date import Time
+
+from ..tools.igastuff import archive_suffix
+from vortex.data.geometries import CurvlinearGeometry
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+@namebuilding_insert('src', lambda s: [s.filling, s.model])
+class AbstractAnalysis(GeoFlowResource):
+    """Analysis resource.
+
+    It can be an atmospheric, surface or full analysis (full = atmospheric + surface).
+    """
+    _abstract = True
+    _footprint = dict(
+        info = 'Analysis',
+        attr = dict(
+            kind = dict(
+                values   = ['analysis', 'analyse', 'atm_analysis']
+            ),
+            nativefmt = dict(
+                values   = ['fa', 'grib', 'lfi', 'netcdf', 'txt', 'unknown'],
+                default  = 'fa',
+            ),
+            filtering = dict(
+                info         = "The filtering that was applied during the generating process.",
+                optional    = True,
+                values      = ['dfi'],
+                doc_zorder  = -5,
+            ),
+            filling = dict(
+                info     = "The content/coverage of the analysis.",
+                optional = True,
+                default  = 'full',
+                values   = ['surface', 'surf', 'atmospheric', 'atm', 'full', 'soil'],
+                remap    = dict(
+                    surface     = 'surf',
+                    atmospheric = 'atm',
+                ),
+            )
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'analysis'
+
+
+class Analysis3D(AbstractAnalysis):
+    """3D Analysis resource (i.e. the resource has no term attribute).
+
+    The data is assumed to be valid for **date** (i.e. term = 0).
+    """
+
+    _footprint = [
+        term,
+        dict(
+            attr = dict(
+                term = dict(
+                    values = [Time(0), ],
+                    optional = True,
+                    default = Time(0)
+                )
+            )
+        )
+    ]
+
+    def archive_basename(self):
+        """OP ARCHIVE specific naming convention."""
+        ananame = 'analyse'
+        if 'surf' in self.filling:
+            if re.match('aladin|arome', self.model):
+                ananame = 'analyse_surf'
+            elif self.model == 'surfex':
+                ananame = 'analyse'
+            elif self.model in ('hycom', 'mfwam'):
+                ananame = '(prefix:modelkey)(termfix:modelkey)(suffix:modelkey)'
+            else:
+                ananame = 'analyse_surface1'
+
+        if self.filtering is not None:
+            if 'aladin' in self.model:
+                ananame = 'ANALYSE_DFI'
+
+        if self.model == 'surfex':
+            ananame += '.sfx'
+
+        return ananame
+
+    def olive_basename(self):
+        """OLIVE specific naming convention."""
+        olivename_map = {'atm': 'TRAJ' + self.model[:4].upper() + '+0000',
+                         'surf': 'surfanalyse',
+                         'full': 'analyse'}
+        if self.model != 'arpege':
+            olivename_map['surf'] = 'analyse'
+            if self.model == 'surfex':
+                olivename_map = {k: x + '.sfx' for k, x in olivename_map.items()}
+        return olivename_map[self.filling]
+
+    def iga_pathinfo(self):
+        """Standard path information for IGA inline cache."""
+        if self.model == 'arome':
+            if self.filling == 'surf':
+                directory = 'fic_day'
+            else:
+                directory = 'workdir/analyse'
+        elif self.model == 'arpege':
+            if self.filling == 'surf':
+                directory = 'workdir/analyse'
+            else:
+                directory = 'autres'
+        elif self.model in ('hycom', 'mfwam'):
+            if self.filling == 'surf':
+                directory = 'guess'
+        elif self.model == 'surfex':
+            directory = 'fic_day'
+        else:
+            if self.filling == 'surf':
+                directory = 'autres'
+            else:
+                directory = 'workdir/analyse'
+        return dict(
+            fmt=directory,
+            model=self.model,
+            nativefmt=self.nativefmt,
+        )
+
+
+class Analysis4D(AbstractAnalysis):
+    """4D Analysis resource (i.e. the resource has a term attribute)."""
+
+    _footprint = [
+        term_deco,
+        dict(
+            attr = dict(
+                term = dict(
+                    outcast = [Time(0), ]
+                )
+            )
+        )
+    ]
+
+
+class InitialCondition(AbstractAnalysis):
+    """
+    Class for initial condition resources : anything from which a model run can be performed.
+    """
+    _footprint = dict(
+        info = 'Initial condition',
+        attr = dict(
+            kind = dict(
+                values   = ['initial_condition', 'ic', 'starting_point'],
+                remap    = dict(autoremap = 'first'),
+            ),
+        )
+    )
+
+    @property
+    def term(self):
+        """Fake term for duck typing."""
+        return Time(0)
+
+    @property
+    def realkind(self):
+        return 'ic'
+
+    def olive_basename(self):
+        """OLIVE specific naming convention."""
+        logger.warning("The member number is only known by the provider, so the generic historic name is returned.")
+        return 'ICMSH' + self.model[:4].upper() + '+' + self.term.fmthour
+
+    def archive_basename(self):
+        """OP ARCHIVE specific naming convention."""
+        return 'ICFC_(memberfix:member)'
+
+
+class Historic(GeoFlowResource):
+    """
+    Class for historical state of a model (e.g. from a forecast).
+    """
+    _footprint = [
+        term_deco,
+        dict(
+            info = 'Historic forecast file',
+            attr = dict(
+                kind = dict(
+                    values = ['historic', 'modelstate'],
+                    remap = dict(
+                        modelstate = 'historic'
+                    )
+                ),
+                subset = dict(
+                    # Dummy argument but avoid priority related messages with footprints
+                    info = 'With Historical files, leave subset empty...',
+                    optional = True,
+                    values = [None, ],
+                ),
+                nativefmt = dict(
+                    values = ['fa', 'grib', 'lfi', 'netcdf', 'unknown', 'nc'],
+                    remap = dict(nc='netcdf'),
+                    default = 'fa',
+                ),
+            )
+        )
+    ]
+
+    @property
+    def realkind(self):
+        return 'historic'
+
+    def archive_basename(self):
+        """OP ARCHIVE specific naming convention."""
+        if self.model in ('mfwam', 'hycom'):
+            prefix = '(prefix:modelkey)'
+            midfix = ''
+        else:
+            prefix = '(icmshfix:modelkey)'
+            midfix = '(histfix:igakey)'
+        termfix = '(termfix:modelkey)'
+        suffix = '(suffix:modelkey)'
+
+        if self.geometry.lam and re.match('testms1|testmp1|testmp2', self.geometry.area):
+            suffix = '.r' + archive_suffix(self.model, self.cutoff, self.date)
+
+        if self.model == 'mocage':
+            prefix = 'HM'
+            midfix = self.geometry.area
+            if self.nativefmt == 'netcdf':
+                suffix = '.nc'
+
+        return prefix + midfix + termfix + suffix
+
+    def olive_basename(self):
+        """OLIVE specific naming convention."""
+        if self.model == 'mesonh':
+            return '.'.join(
+                (self.model.upper(), self.geometry.area[:4].upper() + '+' + self.term.fmthour, self.nativefmt)
+            )
+        else:
+            return 'ICMSH' + self.model[:4].upper() + '+' + self.term.fmthour
+
+    def _geo2basename_info(self, add_stretching=True):
+        """Return an array describing the geometry for the Vortex's name builder."""
+        if isinstance(self.geometry, CurvlinearGeometry) and self.model == 'hycom':
+            # return the old naming convention for surges restart files
+            lgeo = [self.geometry.area, self.geometry.rnice]
+            return lgeo
+        else:
+            return super()._geo2basename_info(add_stretching=add_stretching)
+
+
+@namebuilding_insert('filtername', lambda s: s.subset)
+class HistoricSubset(GeoFlowResource):
+    """
+    Class for a subset of the historical state of a model (e.g. from a forecast).
+    """
+    _footprint = [
+        term_deco,
+        dict(
+            info = 'Subset of an historic forecast file',
+            attr = dict(
+                kind = dict(
+                    values = ['historic', 'modelstate'],
+                    remap = dict(
+                        modelstate = 'historic'
+                    )
+                ),
+                subset = dict(
+                    info = "The subset of fields contained in this data.",
+                ),
+                nativefmt = dict(
+                    values = ['fa', 'grib', 'lfi', 'netcdf', 'unknown', 'nc'],
+                    remap = dict(nc='netcdf'),
+                    default = 'fa',
+                ),
+            )
+        )
+    ]
+
+    @property
+    def realkind(self):
+        return 'historic'
+
+
+class BiasDFI(GeoFlowResource):
+    """
+    Class for some kind of DFI bias (please add proper documentation).
+    """
+    _footprint = [
+        term_deco,
+        dict(
+            info = 'DFI bias file',
+            attr = dict(
+                kind = dict(
+                    values = ['biasdfi', 'dfibias'],
+                    remap = dict(
+                        dfibias = 'biasdfi'
+                    )
+                ),
+                nativefmt = dict(
+                    values = ['fa'],
+                    default = 'fa',
+                ),
+            )
+        )
+    ]
+
+    @property
+    def realkind(self):
+        return 'biasdfi'
+
+    def archive_basename(self):
+        """OP ARCHIVE specific naming convention."""
+        return 'BIASDFI+{:04d}'.format(self.term.hour)
+
+    def olive_basename(self):
+        """OLIVE specific naming convention."""
+        return 'BIASDFI{:s}+{:04d}'.format(self.model[:4].upper(), self.term.hour)

vortex/nwp/data/monitoring.py

@@ -0,0 +1,220 @@
+"""
+Various Resources to handle data produce by the obserations monitoring.
+"""
+
+from bronx.fancies import loggers
+
+from vortex.data.flow import FlowResource
+from vortex.syntax.stddeco import namebuilding_append, namebuilding_insert
+from .consts import GenvModelResource
+
+#: Automatic export of Observations class
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+@namebuilding_insert('src', lambda s: [s.stage, s.obs])
+class Monitoring(FlowResource):
+    """Abstract monitoring resource."""
+
+    _abstract = True
+    _footprint = dict(
+        info = 'Observations monitoring file',
+        attr = dict(
+            kind = dict(
+                values = ['monitoring', ],
+            ),
+            nativefmt = dict(
+                values =['ascii', 'binary', 'txt', 'bin'],
+                remap = dict(ascii='txt', binary='bin')
+            ),
+            stage = dict(
+                values = ['can', 'surf', 'surface', 'atm', 'atmospheric'],
+                remap = dict(can='surf', surface='surf', atmospheric='atm'),
+                info = 'The processing stage of the ODB base.',
+            ),
+            obs = dict(
+                values = ['all', 'used'],
+                info = 'The processing part of the ODB base.',
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'monitoring'
+
+
+class MntObsThreshold(GenvModelResource):
+    """Observations threshold file.
+
+    A GenvKey can be given.
+    """
+
+    _footprint = dict(
+        info='Observations threshold',
+        attr=dict(
+            kind=dict(
+                values=['obs_threshold']
+            ),
+            gvar=dict(
+                default='monitoring_seuils_obs'
+            ),
+            source=dict(
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'obs_threshold'
+
+    def gget_urlquery(self):
+        """GGET specific query : ``extract``."""
+        return 'extract=' + self.source
+
+
+@namebuilding_insert('period', lambda s: s.periodicity)
+class MntCumulStat(Monitoring):
+    """Accumulated statistics file."""
+
+    _footprint = dict(
+        info='Monthly accumulated statistics',
+        attr=dict(
+            kind=dict(
+                values=['accumulated_stats']
+            ),
+            nativefmt=dict(
+                values=['binary', 'bin'],
+                default='bin',
+                optional = True
+            ),
+            periodicity = dict(
+                values = ['monthly',
+                          'weekly_on_mondays',
+                          'weekly_on_sundays'],
+                default = 'monthly',
+                optional =True
+            )
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'accumulated_stats'
+
+
+@namebuilding_append('src', lambda s: s.monitor)
+class MntStat(Monitoring):
+    """Monitoring statistics file."""
+
+    _footprint = dict(
+        info='Monitoring statistics',
+        attr=dict(
+            kind=dict(
+                values=['monitoring_stats']
+            ),
+            nativefmt=dict(
+                values=['ascii', 'txt'],
+                default='txt',
+                optional=True
+            ),
+            monitor=dict(
+                values=['bias', 'analysis'],
+                remap=dict(cy='analysis', deb='bias'),
+            )
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'monitoring_stats'
+
+
+class MntGrossErrors(Monitoring):
+    """Gross errors file."""
+
+    _footprint = dict(
+        info='Gross errors',
+        attr=dict(
+            kind=dict(
+                values=['gross_errors']
+            ),
+            nativefmt = dict(
+                values=['ascii', 'txt'],
+                default='txt',
+                optional=True
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'gross_errors'
+
+
+class MntNbMessages(Monitoring):
+    """Number of messages for each observations type"""
+
+    _footprint = dict(
+        info='Obs messages',
+        attr=dict(
+            kind=dict(
+                values=['nbmessages']
+            ),
+            nativefmt = dict(
+                values=['ascii', 'txt'],
+                default='txt',
+                optional=True
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'nbmessages'
+
+
+class MntMissingObs(Monitoring):
+    """Missing observations."""
+
+    _footprint = dict(
+        info='Missing observations',
+        attr=dict(
+            kind=dict(
+                values=['missing_obs']
+            ),
+            nativefmt = dict(
+                values=['ascii', 'txt'],
+                default='txt',
+                optional=True
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'missing_obs'
+
+
+class MntObsLocation(Monitoring):
+    """Observations location."""
+
+    _footprint = dict(
+        info='Observations location',
+        attr=dict(
+            kind=dict(
+                values=['obslocation']
+            ),
+            nativefmt = dict(
+                values=['obslocationpack'],
+                default='obslocationpack',
+                optional=True
+            ),
+        )
+    )
+
+    @property
+    def realkind(self):
+        return 'obslocation'