vortex-nwp 2.1.2__tar.gz → 2.2.0__tar.gz

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.1.2
+Version: 2.2.0
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "vortex-nwp"
-version = "2.1.2"
+version = "2.2.0"
 description = "A Python library to write Numerical Weather Prediction pipelines components"
 readme = "README.md"
 requires-python = ">=3.7"

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/__init__.py

@@ -53,10 +53,10 @@ from .toolbox import algo as task
 
 from . import nwp as nwp  # footprints import
 
-__version__ = "2.1.2"
+__version__ = "2.2.0"
 __prompt__ = "Vortex v-" + __version__ + ":"
 
-__nextversion__ = "2.1.3"
+__nextversion__ = "2.2.1"
 __tocinfoline__ = "VORTEX core package"
 
 __all__ = [

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/data/abstractstores.py

@@ -14,7 +14,12 @@ from bronx.system import hash as hashutils
 import footprints
 
 from vortex import sessions
-from vortex.config import from_config, ConfigurationError, is_defined
+from vortex.config import (
+    from_config,
+    ConfigurationError,
+    is_defined,
+    get_from_config_w_default,
+)
 from vortex.syntax.stdattrs import (
     hashalgo,
     hashalgo_avail_list,
@@ -438,6 +443,7 @@ class MultiStore(footprints.FootprintBase):
             attr=dict(
                 scheme=dict(alias=("protocol",)),
                 netloc=dict(type=Namespace, alias=("domain", "namespace")),
+                username=dict(type=str),
                 refillstore=dict(
                     type=bool,
                     optional=True,
@@ -792,6 +798,9 @@ class ArchiveStore(Store):
                 netloc=dict(
                     values=["open.archive.fr"],
                 ),
+                username=dict(
+                    type=str,
+                ),
                 storehash=dict(
                     values=hashalgo_avail_list,
                 ),
@@ -843,6 +852,13 @@ class ArchiveStore(Store):
     def underlying_archive_kind(self):
         return "std"
 
+    @property
+    def archive_entry(self):
+        pattern = get_from_config_w_default(
+            section="storage", key="rootdir", default="~/vortex"
+        )
+        return pattern.replace("%usr%", self.username)
+
     @property
     def actual_storage(self):
         """This archive network name (potentially read form the configuration file)."""
@@ -894,6 +910,7 @@ class ArchiveStore(Store):
                 storage=self.actual_storage,
                 tube=self.actual_storetube,
                 readonly=self.readonly,
+                entry=self.archive_entry,
             )
             self._archives_object_stack.add(self._archive)
         return self._archive
@@ -909,25 +926,12 @@ class ArchiveStore(Store):
 
     archive = property(_get_archive, _set_archive, _del_archive)
 
-    def _inarchiveformatpath(self, remote):
-        # Remove extra slashes
-        formatted = remote["path"].lstrip(self.system.path.sep)
-        # Store head ?
-        if self.storehead:
-            formatted = self.system.path.join(self.storehead, formatted)
-        # Store root (if specified)
-        pathroot = remote.get("root", self.storeroot)
-        if pathroot is not None:
-            formatted = self.system.path.join(pathroot, formatted)
-        return formatted
-
     def inarchivecheck(self, remote, options):
         """Use the archive object to check if **remote** exists."""
         # Try to delete the md5 file but ignore errors...
         if self._hash_check_or_delete(self.inarchivecheck, remote, options):
             return self.archive.check(
-                self._inarchiveformatpath(remote),
-                username=remote.get("username", None),
+                remote["path"],
                 fmt=options.get("fmt", "foo"),
                 compressionpipeline=self._actual_cpipeline,
             )
@@ -937,8 +941,7 @@ class ArchiveStore(Store):
     def inarchivelocate(self, remote, options):
         """Use the archive object to obtain **remote** physical location."""
         return self.archive.fullpath(
-            self._inarchiveformatpath(remote),
-            username=remote.get("username", None),
+            remote["path"],
             fmt=options.get("fmt", "foo"),
             compressionpipeline=self._actual_cpipeline,
         )
@@ -946,15 +949,13 @@ class ArchiveStore(Store):
     def inarchivelist(self, remote, options):
         """Use the archive object to list available files."""
         return self.archive.list(
-            self._inarchiveformatpath(remote),
-            username=remote.get("username", None),
+            remote["path"],
         )
 
     def inarchiveprestageinfo(self, remote, options):
         """Returns the prestaging informations"""
         return self.archive.prestageinfo(
-            self._inarchiveformatpath(remote),
-            username=remote.get("username", None),
+            remote["path"],
             fmt=options.get("fmt", "foo"),
             compressionpipeline=self._actual_cpipeline,
         )
@@ -965,16 +966,15 @@ class ArchiveStore(Store):
             "inarchiveget on %s://%s/%s (to: %s)",
             self.scheme,
             self.netloc,
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
         )
         rc = self.archive.retrieve(
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
             intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
             fmt=options.get("fmt", "foo"),
             info=options.get("rhandler", None),
-            username=remote["username"],
             compressionpipeline=self._actual_cpipeline,
         )
         return rc and self._hash_get_check(
@@ -987,16 +987,15 @@ class ArchiveStore(Store):
             "inarchiveearlyget on %s://%s/%s (to: %s)",
             self.scheme,
             self.netloc,
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
         )
         rc = self.archive.earlyretrieve(
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
             intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
             fmt=options.get("fmt", "foo"),
             info=options.get("rhandler", None),
-            username=remote["username"],
             compressionpipeline=self._actual_cpipeline,
         )
         return rc
@@ -1007,17 +1006,16 @@ class ArchiveStore(Store):
             "inarchivefinaliseget on %s://%s/%s (to: %s)",
             self.scheme,
             self.netloc,
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
         )
         rc = self.archive.finaliseretrieve(
             result_id,
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
             intent=options.get("intent", ARCHIVE_GET_INTENT_DEFAULT),
             fmt=options.get("fmt", "foo"),
             info=options.get("rhandler", None),
-            username=remote["username"],
             compressionpipeline=self._actual_cpipeline,
         )
         return rc and self._hash_get_check(
@@ -1030,16 +1028,15 @@ class ArchiveStore(Store):
             "inarchiveput to %s://%s/%s (from: %s)",
             self.scheme,
             self.netloc,
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
         )
         rc = self.archive.insert(
-            self._inarchiveformatpath(remote),
+            remote["path"],
             local,
             intent=ARCHIVE_PUT_INTENT,
             fmt=options.get("fmt", "foo"),
             info=options.get("rhandler"),
-            username=remote["username"],
             compressionpipeline=self._actual_cpipeline,
             enforcesync=options.get("enforcesync", False),
             usejeeves=options.get("delayed", None),
@@ -1051,15 +1048,14 @@ class ArchiveStore(Store):
             "inarchivedelete on %s://%s/%s",
             self.scheme,
             self.netloc,
-            self._inarchiveformatpath(remote),
+            remote["path"],
         )
         # Try to delete the md5 file but ignore errors...
         self._hash_check_or_delete(self.inarchivedelete, remote, options)
         return self.archive.delete(
-            self._inarchiveformatpath(remote),
+            remote["path"],
             fmt=options.get("fmt", "foo"),
             info=options.get("rhandler", None),
-            username=remote["username"],
             compressionpipeline=self._actual_cpipeline,
         )
 
@@ -1081,6 +1077,9 @@ class CacheStore(Store):
             netloc=dict(
                 values=["open.cache.fr"],
             ),
+            username=dict(
+                type=str,
+            ),
             storehash=dict(
                 values=hashalgo_avail_list,
             ),
@@ -1130,7 +1129,7 @@ class CacheStore(Store):
     def _get_cache(self):
         if not self._cache:
             self._cache = footprints.proxy.caches.default(
-                entry=self.location,
+                entry=self.cache_entry,
                 rtouch=self.rtouch,
                 rtouchskip=self.rtouchskip,
                 readonly=self.readonly,
@@ -1255,6 +1254,7 @@ class PromiseStore(footprints.FootprintBase):
         attr=dict(
             scheme=dict(alias=("protocol",)),
             netloc=dict(type=Namespace, alias=("domain", "namespace")),
+            username=dict(type=str),
             storetrack=dict(
                 type=bool,
                 default=True,
@@ -1282,12 +1282,14 @@ class PromiseStore(footprints.FootprintBase):
             scheme=self.proxyscheme,
             netloc=self.prstorename,
             storetrack=self.storetrack,
+            username=self.username,
         )
         if self.promise is None:
             logger.critical(
-                "Could not find store scheme <%s> netloc <%s>",
+                "Could not find store scheme <%s> netloc <%s> username <%s>",
                 self.proxyscheme,
                 self.prstorename,
+                self.username,
             )
             raise ValueError("Could not get a Promise Store")
 
@@ -1296,12 +1298,14 @@ class PromiseStore(footprints.FootprintBase):
             scheme=self.proxyscheme,
             netloc=self.netloc,
             storetrack=self.storetrack,
+            username=self.username,
         )
         if self.other is None:
             logger.critical(
-                "Could not find store scheme <%s> netloc <%s>",
+                "Could not find store scheme <%s> netloc <%s> username <%s>",
                 self.proxyscheme,
                 self.netloc,
+                self.username,
             )
             raise ValueError("Could not get an Other Store")
 

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/data/geometries.py

@@ -759,17 +759,47 @@ hgeometry_deco = footprints.DecorativeFootprint(
 # Load default geometries when the module is first imported
 
 
-def load(inifile="@geometries.ini", refresh=False, verbose=True):
+def _get_user_config_dir():
+    """Get the user's vortex configuration directory.
+
+    This function is separated out to make it easier to mock in tests.
+    """
+    from pathlib import Path
+    from vortex.tools.env import Environment
+
+    env = Environment(active=False)
+    return Path(env.HOME) / ".vortexrc"
+
+
+def load(
+    inifile="@geometries.ini",
+    refresh=False,
+    verbose=True,
+    _user_config_dir=None,
+):
     """Load a set of pre-defined geometries from a configuration file.
 
     The class that will be instantiated depends on the "kind" keyword..
+
+    :param _user_config_dir: Override user config directory (for testing)
     """
     iniconf = configparser.ConfigParser()
+
+    # Load from vortex distribution
     with importlib.resources.open_text(
         "vortex.data",
         "geometries.ini",
     ) as fh:
         iniconf.read_file(fh)
+
+    # Load from user's config directory if it exists
+    if _user_config_dir is None:
+        _user_config_dir = _get_user_config_dir()
+    user_geometries = _user_config_dir / "geometries.ini"
+    if user_geometries.exists():
+        with open(user_geometries, encoding="utf-8") as fh:
+            iniconf.read_file(fh)
+
     for item in iniconf.sections():
         gdesc = dict(iniconf.items(item))
         gkind = gdesc.get("kind")

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/data/handlers.py

@@ -485,6 +485,7 @@ class Handler:
             return footprints.proxy.store(
                 scheme=self._uridata.pop("scheme"),
                 netloc=self._uridata.pop("netloc"),
+                username=self._uridata.pop("username"),
                 **stopts,
             )
         else:

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/data/providers.py

@@ -14,6 +14,7 @@ from bronx.fancies import loggers
 import footprints
 from footprints import proxy as fpx
 
+import vortex
 from vortex import config
 from vortex.syntax.stdattrs import (
     xpid,
@@ -56,6 +57,7 @@ class Provider(footprints.FootprintBase):
                 info="The username that will be used whenever necessary.",
                 optional=True,
                 default=None,
+                access="rwx",
                 alias=("user", "logname"),
             ),
         ),
@@ -66,6 +68,9 @@ class Provider(footprints.FootprintBase):
         logger.debug("Abstract provider init %s", self.__class__)
         super().__init__(*args, **kw)
 
+        if not self.username:
+            self.username = vortex.ticket().glove.user
+
     def _str_more(self):
         """Additional information to print representation."""
         try:
@@ -279,7 +284,7 @@ class Vortex(Provider):
 
     _DEFAULT_NAME_BUILDER = names.VortexNameBuilder()
     _CUSTOM_NAME_BUILDERS = dict()
-    _SPECIAL_EXPS = ("oper", "dble", "test", "mirr")
+    _SPECIAL_EXPS = ("OPER", "DBLE", "TEST", "MIRR")
 
     _footprint = [
         block,
@@ -366,8 +371,8 @@ class Vortex(Provider):
             self._namebuilder = self._CUSTOM_NAME_BUILDERS[self.namebuild]
         else:
             self._namebuilder = self._DEFAULT_NAME_BUILDER
-        if self.experiment in (n.upper() for n in self._SPECIAL_EXPS):
-            self.experiment = self.experiment.lower()
+        if self.experiment in (n.lower() for n in self._SPECIAL_EXPS):
+            self.experiment = self.experiment.upper()
 
         # Ensure compatibility with deprecated namespace attribute
         # Under the hood the namespace attribute is still used to

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/data/stores.py

@@ -741,19 +741,6 @@ class VortexStdBaseArchiveStore(_VortexBaseArchiveStore):
     def remap_read(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         remote = copy.copy(remote)
-        try:
-            remote["root"] = config.from_config(
-                section="storage",
-                key="rootdir",
-            )
-        except config.ConfigurationError as e:
-            msg = (
-                "Trying to write to archive but location is not configured. "
-                'Make sure key "rootdir" is defined in storage section of '
-                "the configuration.\n"
-                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
-            )
-            raise config.ConfigurationError(msg) from e
         return remote
 
     remap_write = remap_read
@@ -801,22 +788,13 @@ class VortexOpBaseArchiveStore(_VortexBaseArchiveStore):
         ),
     )
 
+    @property
+    def archive_entry(self):
+        return config.from_config(section="storage", key="op_rootdir")
+
     def remap_read(self, remote, options):
         """Reformulates the remote path to compatible vortex namespace."""
         remote = copy.copy(remote)
-        try:
-            remote["root"] = config.from_config(
-                section="storage",
-                key="op_rootdir",
-            )
-        except config.ConfigurationError as e:
-            msg = (
-                "Trying to write to operational data archive but location"
-                ' is not configured. Make sure key "op_rootdir" is defined in '
-                "the storage section of the configuration.\n"
-                "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/configuration.html#storage"
-            )
-            raise config.ConfigurationError(msg) from e
         xpath = remote["path"].split("/")
         if len(xpath) >= 5 and re.match(r"^\d{8}T\d{2,4}", xpath[4]):
             # If a date is detected
@@ -915,7 +893,11 @@ class VortexArchiveStore(MultiStore):
 
     def alternates_fpextras(self):
         """Deal with some ArchiveStores' specific attributes."""
-        return dict(storehead=self.storehead, storesync=self.storesync)
+        return dict(
+            username=self.username,
+            storehead=self.storehead,
+            storesync=self.storesync,
+        )
 
 
 class _VortexCacheBaseStore(CacheStore, _VortexStackedStorageMixin):
@@ -993,9 +975,20 @@ class VortexCacheMtStore(_VortexCacheBaseStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
-        self.location = get_cache_location()
+    @property
+    def cache_entry(self):
+        try:
+            cacheloc = config.from_config(
+                section="data-tree",
+                key="rootdir",
+            )
+        except config.ConfigurationError:
+            cacheloc = os.path.join(os.environ["HOME"], ".vortex.d")
+
+        if self.username != self.system.glove.user:
+            return os.path.join(cacheloc, self.username)
+
+        return cacheloc
 
 
 class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
@@ -1016,19 +1009,17 @@ class VortexCacheOp2ResearchStore(_VortexCacheBaseStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
+    @property
+    def cache_entry(self):
         if not config.is_defined(section="data-tree", key="op_rootdir"):
-            raise config.ConfigurationError(
+            msg = (
                 "Using special experiment but corresponding cache location "
                 'is not configured. Bet sure to set "op_rootdir" in configuration. '
                 "See https://vortex-nwp.readthedocs.io/en/latest/user-guide/oper-dble-data-trees"
             )
-        cachepath = config.from_config(
-            section="data-tree",
-            key="op_rootdir",
-        )
-        self.location = os.path.join(cachepath, "vortex")
+            raise config.ConfigurationError(msg)
+
+        return config.from_config(section="data-tree", key="op_rootdir")
 
 
 class _AbstractVortexCacheMultiStore(MultiStore):
@@ -1103,6 +1094,9 @@ class VortexCacheStore(_AbstractVortexCacheMultiStore):
             f"{self.netloc.firstname}.stacked-cache-mt.fr",
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
     """The go to store for data cached by VORTEX operational experiments.
@@ -1145,6 +1139,9 @@ class VortexVsopCacheStore(_AbstractVortexCacheMultiStore):
             ]
         return todo
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class _AbstractVortexStackMultiStore(MultiStore):
     """Any Cache based Vortex multi store."""
@@ -1267,6 +1264,9 @@ class VortexStoreLegacy(MultiStore):
             for d in (".cache.fr", ".archive-legacy.fr")
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class VortexStore(MultiStore):
     """Combined cache and archive VORTEX stores.
@@ -1315,6 +1315,9 @@ class VortexStore(MultiStore):
             )
         ]
 
+    def alternates_fpextras(self):
+        return dict(username=self.username)
+
 
 class PromiseCacheStore(VortexCacheMtStore):
     """Some kind of vortex cache for EXPECTED resources."""
@@ -1332,9 +1335,9 @@ class PromiseCacheStore(VortexCacheMtStore):
         ),
     )
 
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
-        self.location = os.path.join(get_cache_location(), "promise")
+    @property
+    def cache_promise(self):
+        return os.path.join(super().cache_entry, "promise")
 
     @staticmethod
     def _add_default_options(options):

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/sessions.py

@@ -85,8 +85,9 @@ def exit():
     """Ask all inactive sessions to close, then close the active one."""
     tags = keys()
     xtag = Ticket.tag_focus()
-    tags.remove(xtag)
-    tags.append(xtag)
+    if xtag in tags:
+        tags.remove(xtag)
+        tags.append(xtag)
     ok = True
     for s in [get(tag=x) for x in tags]:
         ok = s.exit() and ok

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex/tools/storage.py

@@ -548,6 +548,11 @@ class AbstractArchive(Storage):
             tube=dict(
                 info="How to communicate with the archive ?",
             ),
+            entry=dict(
+                optional=False,
+                type=str,
+                info="The absolute path to the archive space",
+            ),
         ),
     )
 
@@ -560,23 +565,20 @@ class AbstractArchive(Storage):
     def realkind(self):
         return "archive"
 
-    def _formatted_path(self, rawpath, **kwargs):
-        root = kwargs.get("root", None)
-        if root is not None:
-            rawpath = self.sh.path.join(root, rawpath.lstrip("/"))
+    def _formatted_path(self, subpath, **kwargs):
+        path = self.sh.path.join(self.entry, subpath.lstrip("/"))
+
         # Deal with compression
         compressionpipeline = kwargs.get("compressionpipeline", None)
         if compressionpipeline is not None:
-            rawpath += compressionpipeline.suffix
-        return self.sh.anyft_remote_rewrite(
-            rawpath, fmt=kwargs.get("fmt", "foo")
-        )
+            path += compressionpipeline.suffix
+        return self.sh.anyft_remote_rewrite(path, fmt=kwargs.get("fmt", "foo"))
 
     def _actual_proxy_method(self, pmethod):
         """Create a proxy method based on the **pmethod** actual method."""
 
-        def actual_proxy(item, *kargs, **kwargs):
-            path = self._formatted_path(item, **kwargs)
+        def actual_proxy(subpath, *kargs, **kwargs):
+            path = self._formatted_path(subpath, **kwargs)
             if path is None:
                 raise ValueError("The archive's path is void.")
             return pmethod(path, *kargs, **kwargs)

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex_nwp.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vortex-nwp
-Version: 2.1.2
+Version: 2.2.0
 Summary: A Python library to write Numerical Weather Prediction pipelines components
 Author-email: The Vortex Team <vortex.support@meteo.fr>
 License: CECILL-C

{vortex_nwp-2.1.2 → vortex_nwp-2.2.0}/src/vortex_nwp.egg-info/SOURCES.txt

@@ -159,6 +159,7 @@ tests/test_ecmwf_interface.py
 tests/test_env.py
 tests/test_epygram.py
 tests/test_gco.py
+tests/test_geometries.py
 tests/test_ifstools.py
 tests/test_import.py
 tests/test_iosponge.py
@@ -182,4 +183,5 @@ tests/test_targets.py
 tests/test_templates.py
 tests/test_toolsodb.py
 tests/test_uget.py
+tests/test_user_remote_dtree.py
 tests/test_vortexnames.py