girder 3.2.3.dev16__tar.gz → 3.2.3.dev18__tar.gz

{girder-3.2.3.dev16 → girder-3.2.3.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: girder
-Version: 3.2.3.dev16
+Version: 3.2.3.dev18
 Summary: Web-based data management platform
 Home-page: https://girder.readthedocs.org
 Author: Kitware, Inc.

{girder-3.2.3.dev16 → girder-3.2.3.dev18}/girder/cli/mount.py

@@ -1,3 +1,5 @@
+#!/usrbin/env python3
+
 import errno
 import functools
 import hashlib
@@ -27,6 +29,10 @@ from girder.utility.model_importer import ModelImporter
 from girder.utility.server import configureServer
 
 
+def _hashkey_first(funcname, *args, **kwargs):
+    return (funcname, args[1])
+
+
 class ServerFuse(fuse.Operations):
     """
     This class handles FUSE operations that are non-default.  It exposes the
@@ -65,6 +71,15 @@ class ServerFuse(fuse.Operations):
         options = options or {}
         self.cache = cachetools.TTLCache(maxsize=10000, ttl=int(options.pop('stat_cache_ttl', 1)))
         self.diskcache = None
+        self._mount_stats = {
+            'open': 0,
+            'read': 0,
+            'release': 0,
+            'dir': 0,
+            'bytesread': 0,
+            'pathlookups': 0,
+            'directpathchecks': 0,
+        }
         self._configure_disk_cache(options)
 
     def _configure_disk_cache(self, cacheopts):
@@ -108,7 +123,10 @@ class ServerFuse(fuse.Operations):
             chunk = int(options.pop('chunk', 128 * 1024))
             self.diskcache = {
                 'chunk': chunk,
-                'cache': diskcache.Cache(**options)
+                'cache': diskcache.Cache(**options),
+                'hits': 0,
+                'miss': 0,
+                'bytesread': 0,
             }
 
     def __call__(self, op, path, *args, **kwargs):
@@ -138,7 +156,7 @@ class ServerFuse(fuse.Operations):
                 logger.debug('<- %s (length %d) %r', op, len(ret), ret[:16])
 
     @cachetools.cachedmethod(lambda self: self.cache, key=functools.partial(
-        cachetools.keys.hashkey, '_get_path'))
+        _hashkey_first, '_get_path'))
     def _get_path(self, path):
         """
         Given a fuse path, return the associated resource.
@@ -146,6 +164,7 @@ class ServerFuse(fuse.Operations):
         :param path: path within the fuse.
         :returns: a Girder resource dictionary.
         """
+        self._mount_stats['pathlookups'] += 1
         # If asked about a file in top level directory or the top directory,
         # return that it doesn't exist.  Other methods should handle '',
         # '/user', and 'collection' before calling this method.
@@ -241,6 +260,20 @@ class ServerFuse(fuse.Operations):
                 entries.append(self._name(file, 'file'))
         return entries
 
+    def _mount_stats_repr(self):
+        import json
+
+        curstats = self._mount_stats.copy()
+        curstats['openfiles'] = {}
+        for fh in self.openFiles:
+            try:
+                curstats['openfiles'][fh] = self.openFiles[fh]['path']
+            except Exception:
+                pass
+        if self.diskcache:
+            curstats['diskcache'] = {k: v for k, v in self.diskcache.items() if k != 'cache'}
+        return (json.dumps(curstats, indent=2) + '\n').encode()
+
     # We don't handle extended attributes or ioctl.
     getxattr = None
     listxattr = None
@@ -279,7 +312,7 @@ class ServerFuse(fuse.Operations):
         return 0
 
     @cachetools.cachedmethod(lambda self: self.cache, key=functools.partial(
-        cachetools.keys.hashkey, 'getattr'))
+        _hashkey_first, 'getattr'))
     def getattr(self, path, fh=None):
         """
         Get the attributes dictionary of a path.
@@ -293,6 +326,13 @@ class ServerFuse(fuse.Operations):
             attr = self._defaultStat.copy()
             attr['st_mode'] = 0o500 | stat.S_IFDIR
             attr['st_size'] = 0
+        elif path.rstrip('/') == '/stats':
+            attr = self._defaultStat.copy()
+            attr['st_ino'] = -1
+            attr['st_nlink'] = 1
+            attr['st_ctime'] = attr['st_mtime'] = int(time.time() * 1e9)
+            attr['st_mode'] = 0o400 | stat.S_IFREG
+            attr['st_size'] = len(self._mount_stats_repr())
         else:
             resource = self._get_path(path)
             attr = self._stat(resource['document'], resource['model'])
@@ -313,10 +353,14 @@ class ServerFuse(fuse.Operations):
         :param fh: an open file handle.
         :returns: a block of up to <size> bytes.
         """
+        if path.rstrip('/') == '/stats':
+            data = self._mount_stats_repr()
+            return data[offset:offset + size]
         with self.openFilesLock:
             if fh not in self.openFiles:
                 raise fuse.FuseOSError(errno.EBADF)
             info = self.openFiles[fh]
+        self._mount_stats['read'] += 1
         if self.diskcache and info.get('allowcache'):
             result = b''
             for idx in range(
@@ -327,6 +371,8 @@ class ServerFuse(fuse.Operations):
                 key = '%s-%d-%d' % (info['hash'], idxoffset, idxlen)
                 try:
                     data = self.diskcache['cache'].get(key, None, read=True)
+                    if data is not None:
+                        self.diskcache['hits'] += 1
                 except Exception:
                     logger.exception('diskcache threw an exception in get')
                     data = None
@@ -337,6 +383,8 @@ class ServerFuse(fuse.Operations):
                         handle = info['handle']
                         handle.seek(idxoffset)
                         data = handle.read(idxlen)
+                        self.diskcache['miss'] += 1
+                        self.diskcache['bytesread'] += len(data)
                     try:
                         self.diskcache['cache'][key] = data
                     except Exception:
@@ -347,6 +395,7 @@ class ServerFuse(fuse.Operations):
                 else:
                     data.seek(max(0, offset - idxoffset))
                     result += data.read(size - len(result))
+            self._mount_stats['bytesread'] += len(result)
             return result
         with info['lock']:
             if 'handle' not in info:
@@ -356,6 +405,7 @@ class ServerFuse(fuse.Operations):
                     open(info['directpath'], 'rb'))
             handle = info['handle']
             handle.seek(offset)
+            self._mount_stats['bytesread'] += size
             return handle.read(size)
 
     def readdir(self, path, fh):
@@ -367,10 +417,11 @@ class ServerFuse(fuse.Operations):
             specified.
         :returns: a list of names.  This always includes . and ..
         """
+        self._mount_stats['dir'] += 1
         path = path.rstrip('/')
         result = ['.', '..']
         if path == '':
-            result.extend(['collection', 'user'])
+            result.extend(['collection', 'user', 'stats'])
         elif path in ('/user', '/collection'):
             model = path[1:]
             docList = ModelImporter.model(model).find({}, sort=None)
@@ -381,6 +432,16 @@ class ServerFuse(fuse.Operations):
             result.extend(self._list(resource['document'], resource['model']))
         return result
 
+    @cachetools.cachedmethod(lambda self: self.cache, key=functools.partial(
+        _hashkey_first, '_get_direct_path'))
+    def _get_direct_path(self, path, doc):
+        self._mount_stats['directpathchecks'] += 1
+        try:
+            directpath = File().getAssetstoreAdapter(doc).getLocalFilePath(doc)
+        except FilePathException:
+            directpath = None
+        return directpath
+
     def open(self, path, flags):
         """
         Open a path and return a descriptor.
@@ -390,6 +451,10 @@ class ServerFuse(fuse.Operations):
             read only.
         :returns: a file descriptor.
         """
+        if path.rstrip('/') == '/stats':
+            fh = self.nextFH
+            self.nextFH += 1
+            return fh
         resource = self._get_path(path)
         if resource['model'] != 'file':
             return super().open(path, flags)
@@ -409,18 +474,15 @@ class ServerFuse(fuse.Operations):
             'allowcache': True,
             'lock': threading.Lock(),
         }
-        try:
-            directpath = File().getAssetstoreAdapter(
-                resource['document']).getLocalFilePath(resource['document'])
-            if directpath:
-                info['allowcache'] = False
-                info['directpath'] = directpath
-        except FilePathException:
-            pass
+        directpath = self._get_direct_path(path, resource['document'])
+        if directpath:
+            info['allowcache'] = False
+            info['directpath'] = directpath
         with self.openFilesLock:
             fh = self.nextFH
             self.nextFH += 1
             self.openFiles[fh] = info
+            self._mount_stats['open'] += 1
         return fh
 
     def release(self, path, fh):
@@ -438,6 +500,7 @@ class ServerFuse(fuse.Operations):
                         self.openFiles[fh]['handle'].close()
                         del self.openFiles[fh]['handle']
                     del self.openFiles[fh]
+                    self._mount_stats['release'] += 1
             else:
                 return super().release(path, fh)
         return 0

{girder-3.2.3.dev16 → girder-3.2.3.dev18}/girder/utility/abstract_assetstore_adapter.py

@@ -65,6 +65,8 @@ class FileHandle:
             raise GirderException('Read exceeds maximum allowed size.')
         data = io.BytesIO()
         length = 0
+        if self._stream is None:
+            self._stream = self._adapter.downloadFile(self._file, offset=self._pos, headers=False)()
         for chunk in itertools.chain(self._prev, self._stream):
             chunkLen = len(chunk)
 
@@ -103,7 +105,7 @@ class FileHandle:
 
         if self._pos != oldPos:
             self._prev = []
-            self._stream = self._adapter.downloadFile(self._file, offset=self._pos, headers=False)()
+            self._stream = None
 
     def close(self):
         pass

{girder-3.2.3.dev16 → girder-3.2.3.dev18}/girder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: girder
-Version: 3.2.3.dev16
+Version: 3.2.3.dev18
 Summary: Web-based data management platform
 Home-page: https://girder.readthedocs.org
 Author: Kitware, Inc.