vortex-nwp 2.0.0b1__py3-none-any.whl

vortex/nwp/algo/fpserver.py

@@ -0,0 +1,927 @@
+"""
+AlgoComponents for the next generation of Fullpos runs (based on the 903
+configuration).
+"""
+
+import collections
+import functools
+import math
+import re
+from itertools import filterfalse
+import time
+
+from bronx.compat.functools import cached_property
+from bronx.datagrip.namelist import NamelistBlock
+from bronx.stdtypes.date import Time, Date
+from bronx.fancies import loggers
+
+import footprints
+
+from vortex.algo.components import AlgoComponentError
+import vortex.layout.monitor as _lmonitor
+
+from .ifsroot import IFSParallel
+from ..syntax.stdattrs import outputid_deco
+
+#: No automatic export
+__all__ = []
+
+logger = loggers.getLogger(__name__)
+
+
+fullpos_server_flypoll_pickle = '.fullpos_server_flypoll'
+
+
+class FullPosServerFlyPollPersistantState:
+    """Persistent storage object for Fullpos's polling method."""
+
+    def __init__(self):
+        self.cursor = collections.defaultdict(functools.partial(Time, -9999))
+        self.found = collections.defaultdict(list)
+
+
+def fullpos_server_flypoll(sh, outputprefix, termfile, directories=('.', ), **kwargs):  # @UnusedVariable
+    """Check sub-**directories** to determine wether new output files are available or not."""
+    new = list()
+    for directory in directories:
+        with sh.cdcontext(directory, create=True):
+            if sh.path.exists(fullpos_server_flypoll_pickle):
+                fpoll_st = sh.pickle_load(fullpos_server_flypoll_pickle)
+            else:
+                fpoll_st = FullPosServerFlyPollPersistantState()
+            try:
+                if sh.path.exists(termfile):
+                    with open(termfile) as wfh:
+                        rawcursor = wfh.readline().rstrip('\n')
+                    try:
+                        cursor = Time(rawcursor)
+                    except TypeError:
+                        logger.warning('Unable to convert "%s" to a Time object', rawcursor)
+                        return new
+                    pre = re.compile(r'^{:s}\w*\+(\d+(?::\d\d)?)(?:\.\w+)?$'.format(outputprefix))
+                    candidates = [pre.match(f) for f in sh.listdir()]
+                    lnew = list()
+                    for candidate in filterfalse(lambda c: c is None, candidates):
+                        if candidate.group(0).endswith('.d'):
+                            continue
+                        ctime = Time(candidate.group(1))
+                        if ctime > fpoll_st.cursor[outputprefix] and ctime <= cursor:
+                            lnew.append(candidate.group(0))
+                    fpoll_st.cursor[outputprefix] = cursor
+                    fpoll_st.found[outputprefix].extend(lnew)
+                    new.extend([sh.path.normpath(sh.path.join(directory, anew))
+                                for anew in lnew])
+            finally:
+                sh.pickle_dump(fpoll_st, fullpos_server_flypoll_pickle)
+    return new
+
+
+class FullposServerDiscoveredInputs:
+    """Holds all kind of information on input files."""
+
+    def __init__(self):
+        self.inidata = dict()
+        self.tododata = list()
+        self.guessdata = list()
+        self.termscount = collections.defaultdict(int)
+        self.anyexpected = False
+        self.inputsminlen = 0
+        self.firstprefix = None
+
+    def actual_suffixlen(self, minlen=None):
+        """Find out the required suffixlen."""
+        if minlen is None:
+            minlen = self.inputsminlen
+        return max(minlen,
+                   int(math.floor(math.log10(len(self.tododata)))))
+
+
+class FullPosServer(IFSParallel):
+    """Fullpos Server for geometry transforms & post-processing in IFS-like Models.
+
+    Input/Output files are labelled as follows:
+
+        * Let ``INPUTFILE_0`` denote an input file name (the user can choose
+          whichever name she/he wants provided that the associated input's
+          section has the "ModelState" role).
+        * For FA files: the corresponding output file will be
+          ``NPUTFILE_0.domain.out`` where ``domain`` is the domain name chosen
+          by the user in the namelist. If a Surfex output file is also created
+          (it depends on the namelist) it will be named ``NPUTFILE_0.domain.sfx.out``
+        * For GRIB files: the corresponding output file will be
+          ``NPUTFILE_0.domain.grib.out``.
+        * Some or all output files may be pre-positionned (data generated by the
+          c903 will be appended on them). The corresponding section's role needs
+          to be "OutputGuess". The local filename of the section, needs to match
+          the expected output filename. In our exemple it would be
+          ``NPUTFILE_0.domain.out``.
+
+    :note: To use this algocomponent, the c903's server needs to be activated
+           in the namelist (NFPSERVER != 0).
+
+    :note: With the current IFS/Arpege code, in order for the output's polling
+           to work properly,  an ``ECHFP`` whiteness file have to be incremented
+           by the server, in each of the output directories.
+
+    :note: Climatology files are not managed (only few sanity checks are
+           performed). The user needs to name the input climatology file
+           consistently with the c903' namelist. For role="InitialClim" sections,
+           the filename should be something like ``Const.Clim.m[month]``. For
+           role="TargetClim" sections the filename should be something like
+           ``const.clim.[domain].m[month]`` where ``[domain]`` corresponds to
+           the name of the output domain (as specified in the namelist file)
+           and ``[month]`` corresponds to the month being dealt with (on 2 digits).
+
+    Interesting features:
+
+        * Input files can be expected (for on the fly processing)
+        * Input files are dealt with in arbitrary order depending on their
+          availability (useful for ensemble processing).
+        * Output files can be promised
+
+    """
+
+    _INITIALCONDITION_ROLE = re.compile(r'InitialCondition((?:\w+)?)')
+    _INPUTDATA_ROLE_STR = 'ModelState'
+    _INPUTDATA_ROLE = re.compile(r'ModelState((?:\w+)?)')
+    _OUTPUTGUESS_ROLE = 'OutputGuess'
+
+    _MODELSIDE_INPUTPREFIX0 = 'ICM'
+    _MODELSIDE_INPUTPREFIX1 = 'SH'
+    _MODELSIDE_OUTPUTPREFIX = 'PF'
+    _MODELSIDE_OUTPUTPREFIX_GRIB = 'GRIBPF'
+    _MODELSIDE_TERMFILE = './ECHFP'
+    _MODELSIDE_OUT_SUFFIXLEN_MIN = 4
+    _MODELSIDE_IND_SUFFIXLEN_MIN = 4
+    _MODELSIDE_INE_SUFFIXLEN_MIN = dict(grib=6)
+
+    _SERVERSYNC_RAISEONEXIT = False
+    _SERVERSYNC_RUNONSTARTUP = False
+    _SERVERSYNC_STOPONEXIT = False
+
+    _footprint = [
+        outputid_deco,
+        dict(
+            attr = dict(
+                kind = dict(
+                    values   = ['fpserver', ],
+                ),
+                outdirectories = dict(
+                    info     = "The list of possible output directories.",
+                    type     = footprints.stdtypes.FPList,
+                    default  = footprints.stdtypes.FPList(['.', ]),
+                    optional = True,
+                ),
+                append_domain = dict(
+                    info     = ("If defined, the output file for domain append_domain " +
+                                "will be made a copy of the input file (prior to the " +
+                                "server run"),
+                    optional = True,
+                ),
+                basedate=dict(
+                    info     = "The run date of the coupling generating process",
+                    type     = Date,
+                    optional = True
+                ),
+                xpname = dict(
+                    default  = 'FPOS'
+                ),
+                conf = dict(
+                    default  = 903,
+                ),
+                timestep=dict(
+                    default  = 1.,
+                ),
+                timeout = dict(
+                    type     = int,
+                    optional = True,
+                    default  = 300,
+                ),
+                refreshtime = dict(
+                    info     = "How frequently are the expected input files looked for ? (seconds)",
+                    type     = int,
+                    optional = True,
+                    default  = 20,
+                ),
+                server_run = dict(
+                    # This is a rw attribute: it will be managed internally
+                    values   = [True, False]
+                ),
+                serversync_method = dict(
+                    default  = 'simple_socket',
+                ),
+                serversync_medium = dict(
+                    default  = 'nextfile_wait',
+                ),
+                maxpollingthreads = dict(
+                    type     = int,
+                    optional = True,
+                    default  = 8,
+                ),
+                flypoll = dict(
+                    default  = 'internal',
+                ),
+                defaultformat = dict(
+                    info = "Format for the legacy output files.",
+                    default = 'fa',
+                    optional = True
+                )
+            )
+        )
+    ]
+
+    @property
+    def realkind(self):
+        return 'fullpos'
+
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self._flyput_mapping_d = dict()
+
+    def flyput_outputmapping(self, item):
+        """Map an output file to its final name."""
+        sh = self.system
+        for out_re, data in self._flyput_mapping_d.items():
+            m_re = out_re.match(sh.path.basename(item))
+            if m_re:
+                return (sh.path.join(sh.path.dirname(item),
+                                     data[0].format(m_re.group('fpdom'),
+                                                    m_re.group('suffix'))),
+                        data[1])
+
+    @cached_property
+    def inputs(self):
+        """Retrieve the lists in input sections/ResourceHandlers."""
+        discovered = FullposServerDiscoveredInputs()
+
+        # Initial conditions
+        inisec = self.context.sequence.effective_inputs(role=self._INITIALCONDITION_ROLE)
+        if inisec:
+            for s in inisec:
+                iprefix = (self._INITIALCONDITION_ROLE.match(s.alternate
+                                                             if s.role is None else
+                                                             s.role).group(1) or
+                           self._MODELSIDE_INPUTPREFIX1)
+                fprefix = self._MODELSIDE_INPUTPREFIX0 + iprefix
+                if fprefix in discovered.inidata:
+                    raise AlgoComponentError('Only one Initial Condition is allowed.')
+                else:
+                    discovered.inidata[fprefix] = s
+
+        # Model states
+        todosec0 = self.context.sequence.effective_inputs(role=self._INPUTDATA_ROLE)
+        todosec1 = collections.defaultdict(list)
+        discovered.anyexpected = any([isec.rh.is_expected() for isec in todosec0])
+        hasterms = all([hasattr(isec.rh.resource, 'term') for isec in todosec0])
+        # Sort things up (if possible)
+        if hasterms:
+            logger.info('Sorting input data based on the actual term.')
+            todosec0 = sorted(todosec0, key=lambda s: self._actual_term(s.rh))
+        if todosec0:
+            for iseq, s in enumerate(todosec0):
+                rprefix = (self._INPUTDATA_ROLE.match(s.alternate
+                                                      if s.role is None else
+                                                      s.role).group(1) or
+                           self._MODELSIDE_INPUTPREFIX1)
+                todosec1[rprefix].append(s)
+                if iseq == 0:
+                    # Find the "default" prefix and suffix len based on the first section
+                    discovered.firstprefix = rprefix
+                    discovered.inputsminlen = self._MODELSIDE_INE_SUFFIXLEN_MIN.get(
+                        s.rh.container.actualfmt, self._MODELSIDE_IND_SUFFIXLEN_MIN
+                    )
+            iprefixes = sorted(todosec1.keys())
+            if len(iprefixes) == 1:
+                for s in todosec0:
+                    discovered.tododata.append({self._MODELSIDE_INPUTPREFIX0 + iprefixes[0]: s})
+            else:
+                if len({len(secs) for secs in todosec1.values()}) > 1:
+                    raise AlgoComponentError('Inconsistent number of input data.')
+                for sections in zip(* [iter(todosec1[i]) for i in iprefixes]):
+                    discovered.tododata.append({self._MODELSIDE_INPUTPREFIX0 + k: v
+                                                for k, v in zip(iprefixes, sections)})
+
+        # Detect the number of terms based on the firstprefix
+        if hasterms:
+            for sections in discovered.tododata:
+                act_term = self._actual_term(sections[self._MODELSIDE_INPUTPREFIX0 +
+                                                      discovered.firstprefix].rh)
+                discovered.termscount[act_term] += 1
+
+        # Look for guesses of output files
+        guesses_sec0 = collections.defaultdict(list)
+        guess_entry = collections.namedtuple('guess_entry', ('sdir', 'prefix', 'domain', 'suffix', 'sec'))
+        for sec in self.context.sequence.effective_inputs(role=self._OUTPUTGUESS_ROLE):
+            s_lpath = sec.rh.container.localpath()
+            s_match = self._o_algo_re.match(self.system.path.basename(s_lpath))
+            if s_match:
+                guesses_sec0[s_match.group('base')].append(guess_entry(
+                    self.system.path.dirname(s_lpath),
+                    self._o_auto_prefix('grib' if s_match.group('grib') else self.defaultformat),
+                    s_match.group('fpdom'),
+                    s_match.group('suffix'),
+                    sec
+                ))
+                discovered.anyexpected = discovered.anyexpected or sec.rh.is_expected()
+            else:
+                logger.warning('Improper name for the following output guess < %s >. Ignoring it.', s_lpath)
+        # Pair them with input file (based on their name)
+        for iinput in discovered.tododata:
+            isec = iinput[self._MODELSIDE_INPUTPREFIX0 + discovered.firstprefix]
+            discovered.guessdata.append(
+                guesses_sec0.pop(self.system.path.basename(isec.rh.container.localpath()), ())
+            )
+        if guesses_sec0:
+            logger.warning('Some input data were left unsed: < %s >', guesses_sec0)
+            logger.info('discovered guessdata are: < %s >', discovered.guessdata)
+
+        return discovered
+
+    @cached_property
+    def object_namelists(self):
+        """The list of object's namelists."""
+        namrhs = [isec.rh
+                  for isec in self.context.sequence.effective_inputs(role='ObjectNamelist')
+                  if isec.rh.resource.realkind == 'namelist_fpobject']
+        # Update the object's content
+        for namrh in namrhs:
+            namsave = False
+            if namrh.resource.fp_cmodel is not None:
+                self._set_nam_macro(namrh.contents, namrh.container.localpath(),
+                                    'FP_CMODEL', namrh.resource.fp_cmodel)
+                namsave = True
+            if namrh.resource.fp_lextern is not None:
+                self._set_nam_macro(namrh.contents, namrh.container.localpath(),
+                                    'FP_LEXTERN', namrh.resource.fp_lextern)
+                namsave = True
+            if namrh.resource.fp_terms is not None:
+                if not self.inputs.termscount:
+                    raise AlgoComponentError('In this use case, all input data must have a term attribute')
+                active_terms = {Time(t) for t in namrh.resource.fp_terms}
+                # Generate the list of NFPOSTS
+                global_i = 0
+                nfposts = list()
+                for term, n_term in sorted(self.inputs.termscount.items()):
+                    if term in active_terms:
+                        nfposts.extend(range(global_i, global_i + n_term))
+                    global_i += n_term
+                # Get the NAMFPC block
+                try:
+                    nfpc = namrh.contents['NAMFPC']
+                except KeyError:
+                    raise AlgoComponentError('NAMFPC should be defined in {:s}'
+                                             .format(namrh.container.localpath()))
+                # Sanity check
+                for k in nfpc.keys():
+                    if k.startswith('NFPOSTS'):
+                        raise AlgoComponentError('&NAMFPC NFPOSTS*(*) / entries should not be defined in {:s}'
+                                                 .format(namrh.container.localpath()))
+                # Write NFPOSTS to NAMFPC
+                nfpc['NFPOSTS(0)'] = - len(nfposts)
+                for i, v in enumerate(nfposts):
+                    nfpc['NFPOSTS({:d})'.format(i + 1)] = - v
+                logger.info("The NAMFPC namelist in %s was updated.",
+                            namrh.container.localpath())
+                logger.debug("The updated NAMFPC namelist in %s is:\n%s",
+                             namrh.container.localpath(), nfpc)
+                namsave = True
+            if namsave:
+                namrh.save()
+        return namrhs
+
+    @cached_property
+    def xxtmapping(self):
+        """A handy dictionary about selection namelists."""
+        namxxrh = collections.defaultdict(dict)
+        for isec in self.context.sequence.effective_inputs(role='FullPosSelection',
+                                                           kind='namselect'):
+            dpath = self.system.path.dirname(isec.rh.container.localpath())
+            namxxrh[dpath][isec.rh.resource.term] = isec.rh
+        if namxxrh and not self.inputs.termscount:
+            raise AlgoComponentError('In this use case, all input data must have a term attribute')
+        return namxxrh
+
+    @cached_property
+    def _i_fmt(self):
+        """The input files format (as expected by the c903)."""
+        return ('{:s}' + '{:s}+'.format(self.xpname) +
+                '{:0' + str(self.inputs.actual_suffixlen()) + 'd}')
+
+    @cached_property
+    def _o_raw_fmt(self):
+        """The output files format (as imposed by the c903)."""
+        return ('{:s}' + '{:s}'.format(self.xpname) + '{:s}+' +
+                '{:0' + str(self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN)) + 'd}{:s}')
+
+    @cached_property
+    def _o_re_fmt(self):
+        """The output files regex (as imposed by the c903)."""
+        return ('^{:s}' + '{:s}'.format(self.xpname) + r'(?P<fpdom>\w+)\+' +
+                '{:0' + str(self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN)) +
+                r'd}(?P<suffix>(?:\.sfx)?)$')
+
+    @cached_property
+    def _o_init_re_fmt(self):
+        """The output files regex (as imposed by the c903)."""
+        return ('^{:s}' + '{:s}'.format(self.xpname) +
+                r'(?P<fpdom>\w+){:s}(?P<suffix>(?:\.sfx)?)$')
+
+    @cached_property
+    def _o_algo_re(self):
+        """The regex for any output (as imposed by our AlgoComponent)."""
+        return re.compile(r'(?P<base>.+)\.(?P<fpdom>\w+)(?P<suffix>(?:\.sfx)?)(?P<grib>(?:\.grib)?)\.out$')
+
+    @cached_property
+    def _o_suffix(self):
+        """The FAs output suffix (as imposed by our AlgoComponent)."""
+        return '.{:s}{:s}.out'
+
+    @cached_property
+    def _o_grb_suffix(self):
+        """The GRIBs output suffix (as imposed by our AlgoComponent)."""
+        return '.{:s}{:s}.grib.out'
+
+    def _o_auto_prefix(self, fmt):
+        """Return the appropriate output files prefix (as imposed by the c903)."""
+        return dict(grib=self._MODELSIDE_OUTPUTPREFIX_GRIB).get(
+            fmt, self._MODELSIDE_OUTPUTPREFIX
+        )
+
+    def _actual_term(self, rhandler):
+        """Compute the actual Resource Handler term."""
+        rterm = rhandler.resource.term
+        if self.basedate is not None:
+            rterm += rhandler.resource.date - self.basedate
+        return rterm
+
+    def _add_output_mapping(self, outputs_mapping, i, out_re, out_fname):
+        """Add mappings for output file."""
+        # FA/default file
+        re_default = out_re.format(self._MODELSIDE_OUTPUTPREFIX, i)
+        what_default = (out_fname + self._o_suffix, self.defaultformat)
+        outputs_mapping[re.compile(re_default)] = what_default
+        # GRIB files
+        re_grib = out_re.format(self._MODELSIDE_OUTPUTPREFIX_GRIB, i)
+        what_grib = (out_fname + self._o_grb_suffix, 'grib')
+        outputs_mapping[re.compile(re_grib)] = what_grib
+        logger.info('Output %s mapped as %s. Output %s mapped as %s.',
+                    re_default, what_default[0], re_grib, what_grib[0])
+
+    def _link_input(self, iprefix, irh, i, inputs_mapping, outputs_mapping):
+        """Link an input file and update the mappings dictionaries."""
+        sourcepath = irh.container.localpath()
+        inputs_mapping[sourcepath] = self._i_fmt.format(iprefix, i)
+        self.system.cp(sourcepath, inputs_mapping[sourcepath], intent='in', fmt=irh.container.actualfmt)
+        logger.info('%s copied as %s.', sourcepath, inputs_mapping[sourcepath])
+        if iprefix == self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix:
+            self._add_output_mapping(outputs_mapping, i, self._o_re_fmt,
+                                     self.system.path.basename(sourcepath))
+            if self.append_domain:
+                outputpath = self._o_raw_fmt.format(
+                    self._o_auto_prefix(irh.container.actualfmt), self.append_domain, i, ''
+                )
+
+                if self.outdirectories:
+                    todo = [self.system.path.join(d, outputpath) for d in self.outdirectories]
+                else:
+                    todo = [outputpath, ]
+                for a_outputpath in todo:
+                    self.system.cp(sourcepath, a_outputpath, intent='inout', fmt=irh.container.actualfmt)
+                    logger.info('output file prepared: %s copied (rw) to %s.', sourcepath, a_outputpath)
+
+    def _move_output_guess(self, iguess, i):
+        """Move the output file guesses to their final location."""
+        sourcepath = iguess.sec.rh.container.localpath()
+        destpath = self.system.path.join(
+            iguess.sdir,
+            self._o_raw_fmt.format(iguess.prefix, iguess.domain, i, iguess.suffix)
+        )
+        self.system.mv(sourcepath, destpath, fmt=iguess.sec.rh.container.actualfmt)
+        logger.info('output guess %s was moved to %s.', sourcepath, destpath)
+
+    def _link_xxt(self, todorh, i):
+        """If necessary, link in the appropriate xxtNNNNNNMM file."""
+        for sdir, tdict in self.xxtmapping.items():
+            xxtrh = tdict.get(self._actual_term(todorh), None)
+            if xxtrh is not None:
+                xxtsource = self.system.path.relpath(xxtrh.container.abspath,
+                                                     sdir)
+                # The file is expected to follow the xxtDDDDHHMM syntax where DDDD
+                # is the number of days
+                days_hours = (i // 24) * 100 + i % 24
+                xxttarget = 'xxt{:06d}00'.format(days_hours)
+                xxttarget = self.system.path.join(sdir, xxttarget)
+                self.system.symlink(xxtsource, xxttarget)
+                logger.info('XXT %s linked in as %s.', xxtsource, xxttarget)
+
+    def _init_poll_and_move(self, outputs_mapping):
+        """Deal with the PF*INIT file."""
+        sh = self.system
+        candidates = self.system.glob('{:s}{:s}*INIT'.format(self._MODELSIDE_OUTPUTPREFIX, self.xpname))
+        outputnames = list()
+        for thisdata in candidates:
+            mappeddata = None
+            for out_re, data in outputs_mapping.items():
+                m_re = out_re.match(thisdata)
+                if m_re:
+                    mappeddata = (sh.path.join(sh.path.dirname(thisdata),
+                                               data[0].format(m_re.group('fpdom'),
+                                                              m_re.group('suffix'))),
+                                  data[1])
+                    break
+            if mappeddata is None:
+                raise AlgoComponentError('The mapping failed for {:s}.'.format(thisdata))
+            # Already dealt with ?
+            if not self.system.path.exists(mappeddata[0]):
+                logger.info('Linking <%s> to <%s> (fmt=%s).', thisdata, mappeddata[0], mappeddata[1])
+                outputnames.append(mappeddata[0])
+                self.system.cp(thisdata, mappeddata[0], intent='in', fmt=mappeddata[1])
+        return outputnames
+
+    def _poll_and_move(self, outputs_mapping):
+        """Call **io_poll** and rename available output files."""
+        sh = self.system
+        data = self.manual_flypolling()
+        outputnames = list()
+        for thisdata in data:
+            mappeddata = None
+            for out_re, data in outputs_mapping.items():
+                m_re = out_re.match(sh.path.basename(thisdata))
+                if m_re:
+                    mappeddata = (sh.path.join(sh.path.dirname(thisdata),
+                                               data[0].format(m_re.group('fpdom'),
+                                                              m_re.group('suffix'))),
+                                  data[1])
+                    break
+            if mappeddata is None:
+                raise AlgoComponentError('The mapping failed for {:s}.'.format(thisdata))
+            logger.info('Linking <%s> to <%s> (fmt=%s).', thisdata, mappeddata[0], mappeddata[1])
+            outputnames.append(mappeddata[0])
+            self.system.cp(thisdata, mappeddata[0], intent='in', fmt=mappeddata[1])
+        return outputnames
+
+    def _deal_with_promises(self, outputs_mapping, pollingcb):
+        if self.promises:
+            seen = pollingcb(outputs_mapping)
+            for afile in seen:
+                candidates = [x for x in self.promises
+                              if x.rh.container.abspath == self.system.path.abspath(afile)]
+                if candidates:
+                    logger.info('The output data is promised <%s>', afile)
+                    bingo = candidates.pop()
+                    bingo.put(incache=True)
+
+    def prepare(self, rh, opts):
+        """Various sanity checks + namelist tweaking."""
+        super().prepare(rh, opts)
+
+        if self.object_namelists:
+            self.system.subtitle('Object Namelists customisation')
+        for o_nam in self.object_namelists:
+            # a/c cy44: &NAMFPIOS NFPDIGITS=__SUFFIXLEN__, /
+            self._set_nam_macro(o_nam.contents, o_nam.container.localpath(), 'SUFFIXLEN',
+                                self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN))
+            if o_nam.contents.dumps_needs_update:
+                logger.info('Rewritting the %s namelists file.', o_nam.container.actualpath())
+                o_nam.save()
+
+        self.system.subtitle('Dealing with various input files')
+
+        # Sanity check over climfiles and geometries
+        input_geo = {sec.rh.resource.geometry
+                     for sdict in self.inputs.tododata for sec in sdict.values()}
+        if len(input_geo) == 0:
+            raise AlgoComponentError('No input data are provided, ...')
+        elif len(input_geo) > 1:
+            raise AlgoComponentError('Multiple geometries are not allowed for input data.')
+        else:
+            input_geo = input_geo.pop()
+
+        input_climgeo = {x.rh.resource.geometry
+                         for x in self.context.sequence.effective_inputs(role=('InputClim',
+                                                                               'InitialClim'))}
+        if len(input_climgeo) == 0:
+            logger.info('No input clim provided. Going on without it...')
+        elif len(input_climgeo) > 1:
+            raise AlgoComponentError('Multiple geometries are not allowed for input climatology.')
+        else:
+            if input_climgeo.pop() != input_geo:
+                raise AlgoComponentError('The input data and input climatology geometries does not match.')
+
+        # Initial Condition geometry sanity check
+        if self.inputs.inidata and any([sec.rh.resource.geometry != input_geo
+                                        for sec in self.inputs.inidata.values()]):
+            raise AlgoComponentError('The Initial Condition geometry differs from other input data.')
+
+        # Sanity check on target climatology files
+        target_climgeos = {x.rh.resource.geometry
+                           for x in self.context.sequence.effective_inputs(role='TargetClim')}
+        if len(target_climgeos) == 0:
+            logger.info('No target clim are provided. Going on without it...')
+
+        # Sanity check on selection namelists
+        if self.xxtmapping:
+            for tdict in self.xxtmapping.values():
+                if ({self._actual_term(sec.rh)
+                     for sdict in self.inputs.tododata
+                     for sec in sdict.values()} < set(tdict.keys())):
+                    raise AlgoComponentError("The list of terms between input data and selection namelists differs")
+        else:
+            logger.info("No selection namelists detected. That's fine")
+
+        # Link in the initial condition file (if necessary)
+        for iprefix, isec in self.inputs.inidata.items():
+            i_init = '{:s}{:s}INIT'.format(iprefix, self.xpname)
+            if isec.rh.container.basename != i_init:
+                self.system.cp(isec.rh.container.localpath(), i_init,
+                               intent='in', fmt=isec.rh.container.actualfmt)
+                logger.info('Initial condition file %s copied as %s.',
+                            isec.rh.container.localpath(), i_init)
+
+    def find_namelists(self, opts=None):
+        """Find any namelists candidates in actual context inputs."""
+        return [x.rh
+                for x in self.context.sequence.effective_inputs(role='Namelist',
+                                                                kind='namelist')]
+
+    def prepare_namelist_delta(self, rh, namcontents, namlocal):
+        super().prepare_namelist_delta(rh, namcontents, namlocal)
+        # With cy43: &NAMCT0 CSCRIPT_PPSERVER=__SERVERSYNC_SCRIPT__, /
+        if self.inputs.anyexpected:
+            self._set_nam_macro(namcontents, namlocal, 'SERVERSYNC_SCRIPT',
+                                self.system.path.join('.', self.serversync_medium))
+        else:
+            # Do not harass the filesystem...
+            self._set_nam_macro(namcontents, namlocal, 'SERVERSYNC_SCRIPT', ' ')
+        # With cy43: &NAMCT0 CFPNCF=__IOPOLL_WHITNESSFILE__, /
+        self._set_nam_macro(namcontents, namlocal, 'IOPOLL_WHITNESSFILE', self._MODELSIDE_TERMFILE)
+        # With cy43: No matching namelist key
+        # a/c cy44: &NAMFPIOS NFPDIGITS=__SUFFIXLEN__, /
+        self._set_nam_macro(namcontents, namlocal, 'SUFFIXLEN',
+                            self.inputs.actual_suffixlen(self._MODELSIDE_OUT_SUFFIXLEN_MIN))
+        # No matching namelist yet
+        self._set_nam_macro(namcontents, namlocal, 'INPUT_SUFFIXLEN',
+                            self.inputs.actual_suffixlen())
+        # With cy43: &NAMCT0 NFRPOS=__INPUTDATALEN__, /
+        self._set_nam_macro(namcontents, namlocal, 'INPUTDATALEN', - len(self.inputs.tododata))
+        # Auto generate the list of namelists for the various objects
+        if self.object_namelists:
+            if 'NAMFPOBJ' not in namcontents or len(namcontents['NAMFPOBJ']) == 0:
+                nb_o = NamelistBlock('NAMFPOBJ')
+                nb_o['NFPOBJ'] = len(self.object_namelists)
+                for i_nam, nam in enumerate(self.object_namelists):
+                    if nam.resource.fp_conf:
+                        nb_o['NFPCONF({:d})'.format(i_nam + 1)] = nam.resource.fp_conf
+                    nb_o['CNAMELIST({:d})'.format(i_nam + 1)] = nam.container.localpath()
+                namcontents['NAMFPOBJ'] = nb_o
+                logger.info('The following namelist block has been added to "%s":\n%s',
+                            namlocal, nb_o.dumps())
+            else:
+                logger.warning('The NAMFPOBJ namelist in "%s" is not empty. Leaving it as it is',
+                               namlocal)
+        # Just in case FP_CMODEL is defined in the main namelist
+        if self.outputid is not None and any(['FP_CMODEL' in nam_b.macros()
+                                              for nam_b in namcontents.values()]):
+            self._set_nam_macro(namcontents, namlocal, 'FP_CMODEL', self.outputid)
+        return True
+
+    def spawn_pre_dirlisting(self):
+        """Print a directory listing just before run."""
+        super().spawn_pre_dirlisting()
+        for sdir in self.outdirectories:
+            self.system.subtitle('{:s} : {:s} sub-directory listing (pre-execution)'
+                                 .format(self.realkind, sdir))
+            self.system.dir(sdir, output=False, fatal=False)
+
+    def spawn_hook(self):
+        """Usually a good habit to dump the fort.4 namelist."""
+        super().spawn_hook()
+        for o_nam in self.object_namelists:
+            self.system.subtitle('{:s} : dump namelist <{:s}>'
+                                 .format(self.realkind, o_nam.container.localpath()))
+            self.system.cat(o_nam.container.localpath(), output=False)
+
+    def execute(self, rh, opts):
+        """Server still or Normal execution depending on the input sequence."""
+        sh = self.system
+
+        # Input and Output mapping
+        inputs_mapping = dict()
+        outputs_mapping = dict()
+
+        # Initial condition file ?
+        if self.inputs.inidata:
+            for iprefix, isec in self.inputs.inidata.items():
+                # The initial condition resource may be expected
+                self.grab(isec)
+                # Fix potential links and output mappings
+                sourcepath = isec.rh.container.basename
+                if iprefix == self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix:
+                    self._add_output_mapping(outputs_mapping, 'INIT',
+                                             self._o_init_re_fmt, sourcepath)
+                i_init = '{:s}{:s}INIT'.format(iprefix, self.xpname)
+                if isec.rh.container.basename != i_init:
+                    self.system.cp(sourcepath, i_init,
+                                   intent='in', fmt=isec.rh.container.actualfmt)
+                    logger.info('Initial condition file %s copied as %s.',
+                                isec.rh.container.localpath(), i_init)
+        else:
+            if self.inputs.tododata:
+                # Just in case the INIT file is transformed
+                fakesource = self._MODELSIDE_INPUTPREFIX0 + self.inputs.firstprefix + self.xpname + 'INIT'
+                self._add_output_mapping(outputs_mapping, 'INIT', self._o_init_re_fmt, fakesource)
+
+        # Initialise the flying stuff
+        self.flyput = False  # Do not use flyput every time...
+        flyprefixes = set()
+        for s in self.promises:
+            lpath = s.rh.container.localpath()
+            if lpath.endswith('.grib.out'):
+                flyprefixes.add(self._MODELSIDE_OUTPUTPREFIX_GRIB)
+            elif lpath.endswith('.out'):
+                flyprefixes.add(self._MODELSIDE_OUTPUTPREFIX)
+        self.io_poll_args = tuple(flyprefixes)
+        self.io_poll_kwargs = dict(directories=tuple(set(self.outdirectories)))
+        for directory in set(self.outdirectories):
+            sh.mkdir(directory)  # Create possible output directories
+        if self.flypoll == 'internal':
+            self.io_poll_method = functools.partial(fullpos_server_flypoll, sh)
+            self.io_poll_kwargs['termfile'] = sh.path.basename(self._MODELSIDE_TERMFILE)
+        self.flymapping = True
+        self._flyput_mapping_d = outputs_mapping
+
+        # Deal with XXT files
+        if self.xxtmapping:
+            for i, istuff in enumerate(self.inputs.tododata):
+                self._link_xxt(istuff[self._MODELSIDE_INPUTPREFIX0 +
+                                      self.inputs.firstprefix].rh, i)
+
+        if self.inputs.anyexpected:
+            # Some server sync here...
+            self.server_run = True
+            self.system.subtitle('Starting computation with server_run=T')
+
+            # Process the data in chronological order ?
+            ordered_processing = (self.xxtmapping or
+                                  any([o_rh.resource.fp_terms is not None
+                                       for o_rh in self.object_namelists]))
+            if ordered_processing:
+                logger.info('Input data will be processed chronologicaly.')
+
+            # IO poll settings
+            self.io_poll_kwargs['nthreads'] = self.maxpollingthreads
+
+            # Is there already an Initial Condition file ?
+            # If so, start the binary...
+            if self.inputs.inidata:
+                super().execute(rh, opts)
+                # Did the server stopped ?
+                if not self.server_alive():
+                    logger.error("Server initialisation failed.")
+                    return
+                self._deal_with_promises(outputs_mapping, self._init_poll_and_move)
+
+            # Setup the InputMonitor
+            all_entries = set()
+            metagang = _lmonitor.MetaGang()
+            cur_term = None
+            cur_term_gangs = set()
+            prev_term_gangs = set()
+            for istuff, iguesses in zip(self.inputs.tododata, self.inputs.guessdata):
+                iinputs = {_lmonitor.InputMonitorEntry(s) for s in istuff.values()}
+                iinputs |= {_lmonitor.InputMonitorEntry(g.sec) for g in iguesses}
+                iterm = self._actual_term(istuff[self._MODELSIDE_INPUTPREFIX0 +
+                                                 self.inputs.firstprefix].rh)
+                all_entries.update(iinputs)
+                bgang = _lmonitor.BasicGang()
+                bgang.add_member(* iinputs)
+                igang = _lmonitor.MetaGang()
+                igang.info = (istuff, iguesses, iterm)
+                igang.add_member(bgang)
+                # If needed, wait for the previous terms to complete
+                if ordered_processing:
+                    if cur_term is not None and cur_term != iterm:
+                        # Detect term's change
+                        prev_term_gangs = cur_term_gangs
+                        cur_term_gangs = set()
+                    if prev_term_gangs:
+                        # Wait for the gangs of the previous terms
+                        igang.add_member(* prev_term_gangs)
+                    # Save things up for the next time
+                    cur_term_gangs.add(igang)
+                    cur_term = iterm
+                metagang.add_member(igang)
+            bm = _lmonitor.ManualInputMonitor(self.context, all_entries,
+                                              caching_freq=self.refreshtime,)
+
+            # Start the InputMonitor
+            tmout = False
+            current_i = 0
+            server_stopped = False
+            with bm:
+                while not bm.all_done or len(bm.available) > 0:
+
+                    # Fetch available inputs and sort them
+                    ibatch = list()
+                    while metagang.has_collectable():
+                        thegang = metagang.pop_collectable()
+                        ibatch.append(thegang.info)
+                    ibatch.sort(key=lambda item: item[2])  # Sort according to the term
+
+                    # Deal with the various available inputs
+                    for (istuff, iguesses, iterm) in ibatch:
+                        sh.highlight("The Fullpos Server is triggered (step={:d})..."
+                                     .format(current_i))
+
+                        # Link for the init file (if needed)
+                        if current_i == 0 and not self.inputs.inidata:
+                            for iprefix, isec in istuff.items():
+                                i_init = '{:s}{:s}INIT'.format(iprefix, self.xpname)
+                                if not sh.path.exists(i_init):
+                                    sh.cp(isec.rh.container.localpath(), i_init,
+                                          intent='in', fmt=isec.rh.container.actualfmt)
+                                    logger.info('%s copied as %s. For initialisation purposes only.',
+                                                isec.rh.container.localpath(), i_init)
+                            super().execute(rh, opts)
+                            # Did the server stopped ?
+                            if not self.server_alive():
+                                logger.error("Server initialisation failed.")
+                                return
+                            self._deal_with_promises(outputs_mapping, self._init_poll_and_move)
+
+                        # Link input files
+                        for iprefix, isec in istuff.items():
+                            self._link_input(iprefix, isec.rh, current_i,
+                                             inputs_mapping, outputs_mapping)
+                        for iguess in iguesses:
+                            self._move_output_guess(iguess, current_i)
+
+                        # Let's go...
+                        super().execute(rh, opts)
+                        self._deal_with_promises(outputs_mapping, self._poll_and_move)
+                        current_i += 1
+
+                        # Did the server stopped ?
+                        if not self.server_alive():
+                            server_stopped = True
+                            if not bm.all_done:
+                                logger.error("The server stopped but everything wasn't processed...")
+                            break
+
+                    if server_stopped:
+                        break
+
+                    if not (bm.all_done or metagang.has_collectable()):
+                        # Timeout ?
+                        tmout = bm.is_timedout(self.timeout)
+                        if tmout:
+                            break
+                        # Wait a little bit :-)
+                        time.sleep(1)
+                        bm.health_check(interval=30)
+
+            for failed_file in [e.section.rh.container.localpath()
+                                for e in bm.failed.values()]:
+                logger.error("We were unable to fetch the following file: %s", failed_file)
+                if self.fatal:
+                    self.delayed_exception_add(IOError("Unable to fetch {:s}".format(failed_file)),
+                                               traceback=False)
+
+            if tmout:
+                raise OSError("The waiting loop timed out")
+
+        else:
+            # Direct Run !
+            self.server_run = False
+            self.system.subtitle('Starting computation with server_run=F')
+
+            # Link for the inifile (if needed)
+            if not self.inputs.inidata:
+                for iprefix, isec in self.inputs.tododata[0].items():
+                    i_init = '{:s}{:s}INIT'.format(iprefix, self.xpname)
+                    if not sh.path.exists(i_init):
+                        sh.cp(isec.rh.container.localpath(), i_init,
+                              intent='in', fmt=isec.rh.container.actualfmt)
+                        logger.info('%s copied as %s. For initialisation purposes only.',
+                                    isec.rh.container.localpath(), i_init)
+
+            # Create all links well in advance
+            for i, (iinputs, iguesses) in enumerate(zip(self.inputs.tododata,
+                                                        self.inputs.guessdata)):
+                for iprefix, isec in iinputs.items():
+                    self._link_input(iprefix, isec.rh, i, inputs_mapping, outputs_mapping)
+                for iguess in iguesses:
+                    self._move_output_guess(iguess, i)
+
+            # On the fly ?
+            if self.promises:
+                self.flyput = True
+
+            # Let's roll !
+            super().execute(rh, opts)
+
+        # Map all outputs to destination (using io_poll)
+        self.io_poll_args = tuple([self._MODELSIDE_OUTPUTPREFIX,
+                                   self._MODELSIDE_OUTPUTPREFIX_GRIB, ])
+        self._init_poll_and_move(outputs_mapping)
+        self._poll_and_move(outputs_mapping)