sshfs-offline 0.0.1__py3-none-any.whl

sshfs-offline/cli.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python
+
+import errno
+from logging import getLogger
+import os
+from pathlib import Path
+
+import getpass
+import sys
+
+import metrics
+import sftp
+from sftp import fixPath
+
+from fuse import FUSE, FuseOSError, Operations
+
+import data
+import metadata
+import log
+
+class Main(Operations):
+    '''
+    A simple SFTP filesystem. Requires paramiko: http://www.lag.net/paramiko/
+
+    You need to be able to login o remote host without entering a password.
+    '''
+
+    HOME_DIR = str(Path.home())
+    CACHE_TIMEOUT = 5 * 60
+                        
+    def __init__(self, args): 
+        self.debug = args.debug       
+        host = args.host
+        user = args.user
+        remotedir = args.remotedir
+        port = args.port
+               
+        self.log = getLogger(log.MAIN)
+        
+        metrics.counts = metrics.Metrics()
+        sftp.manager = sftp.SFTPManager(host, user, remotedir, port) 
+        metadata.cache = metadata.Metadata(host, remotedir, args.cachetimeout)
+        data.cache = data.Data(host, remotedir)
+
+        sftp.manager.sftp() # verify connection to host
+
+    def init(self, path):
+        metrics.counts.incr('init')
+        metrics.counts.start()
+        log.Log().setupConfig(self.debug)
+        sftp.manager.startKeepalive()        
+         
+    def chmod(self, path, mode): 
+        try: 
+            self.log.debug('-> chmod: %s %s', path, mode)   
+            metrics.counts.incr('chmod')     
+            metadata.cache.deleteMetadata(path)
+            sftp.manager.sftp().chmod(fixPath(path), mode)
+            self.log.debug('<- chmod: %s', path) 
+        except Exception as e:
+            self.log.error('<- chmod: %s %s', path, mode) 
+            metrics.counts.incr('chmod_except') 
+            raise e
+
+    def chown(self, path, uid, gid):
+        try:
+            self.log.debug('-> chown: %s %s %s', path, uid, gid) 
+            metrics.counts.incr('chown') 
+            metadata.cache.deleteMetadata(path)
+            sftp.manager.sftp().chown(fixPath(path), uid, gid)  
+            self.log.debug('<- chown: %s', path)  
+        except Exception as e:
+            self.log.error('<- chown: %s %s %s', path, uid, gid) 
+            metrics.counts.incr('chown_except') 
+            raise e
+        
+    def create(self, path, mode):  
+        try:
+            self.log.debug('-> create: %s %s', path, mode)  
+            metrics.counts.incr('create')     
+            metadata.cache.deleteMetadata(path)
+            metadata.cache.deleteParentMetadata(path)
+            f = sftp.manager.sftp().open(fixPath(path), 'w')
+            f.chmod(mode)
+            f.close()
+            self.log.debug('<- create: %s', path)             
+            return 0
+        except Exception as e:
+            self.log.error('<- create: %s %s', path, mode) 
+            metrics.counts.incr('create_except')  
+            raise e       
+
+    def destroy(self, path):  
+        try:
+            self.log.debug('-> destroy: %s', path)  
+            metrics.counts.incr('destroy')     
+            sftp.manager.sftp().close()        
+            self.log.debug('<- destroy: %s', path) 
+        except Exception as e:
+            self.log.error('<- destroy: %s', path)  
+            metrics.counts.incr('destroy_except') 
+            raise e
+        finally:
+            metrics.counts.stop()
+            sftp.manager.stop()
+
+    def getattr(self, path, fh=None):
+        try:
+            self.log.debug('-> getattr: %s', path)
+            metrics.counts.incr('getattr')
+            d = metadata.cache.getattr(path)
+            if d != None:
+                if d == {}:
+                    raise FuseOSError(errno.ENOENT)
+                else:
+                    self.log.debug('<- getattr: %s', path)
+                    return d # cache hit
+            
+            try:
+                st = sftp.manager.sftp().lstat(fixPath(path))            
+            except IOError as e: 
+                metadata.cache.getattr_save(path, {}) # negative cache entry          
+                raise FuseOSError(errno.ENOENT)
+
+            d = dict((key, getattr(st, key)) for key in (
+                'st_atime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid'))
+            metadata.cache.getattr_save(path, d)
+            self.log.debug('<- getattr: %s %s', path, d)
+            return d
+        except Exception as e:
+            if not isinstance(e,  OSError) and OSError(e).errno != errno.ENOENT:                
+                self.log.error('<- getattr: %s %s', path, e)
+                metrics.counts.incr('getattr_except') 
+            raise e        
+        
+    def statfs(self, path): 
+        try:
+            self.log.debug('-> statfs: %s', path) 
+            metrics.counts.incr('statfs')      
+            stv = data.cache.statvfs(path)        
+            dic = dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
+                'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
+                'f_frsize', 'f_namemax'))
+            self.log.debug('<- statfs: %s %s', path, dic)  
+            return dic
+        except Exception as e:
+            self.log.error('<- statfs: %s', path) 
+            metrics.counts.incr('statfs_except')       
+            raise e
+
+    def mkdir(self, path, mode): 
+        try: 
+            self.log.debug('-> mkdir: %s %s', path, mode) 
+            metrics.counts.incr('mkdir')      
+            metadata.cache.deleteMetadata(path)
+            metadata.cache.deleteParentMetadata(path)
+            sftp.manager.sftp().mkdir(fixPath(path), mode)
+            self.log.debug('<- mkdir: %s', path)
+        except Exception as e:
+            self.log.error('<- mkdir: %s %s', path, mode) 
+            metrics.counts.incr('mkdir_except')  
+            raise e
+
+    def read(self, path, size, offset, fh):  
+        try:
+            self.log.debug('-> read: %s size=%d offset=%d', path, size, offset)
+            metrics.counts.incr('read')
+
+            buf = data.cache.read(path, size, offset, fh)
+
+            self.log.debug('<- read: %s %d', path, len(buf))
+            return buf
+        except Exception as e:
+            self.log.error('<- read: %s %s %d', path, size, offset) 
+            metrics.counts.incr('read_except') 
+            raise e
+        
+    def readdir(self, path, fh):
+        try:
+            self.log.debug('-> readdir: %s', path)
+            metrics.counts.incr('readdir')
+            s = metadata.cache.readdir(path)
+            if s != None:
+                self.log.debug('<- readdir: %s %d', path, len(s))
+                return s
+            s = ['.', '..'] + [name
+                                for name in sftp.manager.sftp().listdir(fixPath(path))]
+            metadata.cache.readdir_save(path, s)        
+            s = metadata.cache.readdir(path)
+            self.log.debug('<- readdir: %s %d', path, len(s))
+            return s
+        except Exception as e:
+            self.log.error('<- readdir: %s', path)
+            metrics.counts.incr('readdir_except') 
+            raise e
+
+    def readlink(self, path):
+        try:
+            self.log.debug('-> readlink: %s', path)
+            metrics.counts.incr('readlink')
+            link = metadata.cache.readlink(path)
+            if link == None:        
+                link = sftp.manager.sftp().readlink(fixPath(path))
+                metadata.cache.readlink_save(path, link)
+            
+            self.log.debug('<- readlink: %s %s', path, link)
+            return link
+        except Exception as e:
+            self.log.error('<- readlink: %s', path)
+            metrics.counts.incr('readlink_except') 
+            raise e
+
+    def rename(self, old, new):
+        try:
+            self.log.debug('-> rename: %s %s', old, new) 
+            metrics.counts.incr('rename')       
+            metadata.cache.deleteMetadata(old)
+            sftp.manager.sftp().rename(fixPath(old), fixPath(new))
+            self.log.debug('<- rename: %s %s', old, new)
+        except Exception as e:
+            self.log.error('<- rename: %s %s', old, new) 
+            metrics.counts.incr('rename_except')   
+            raise e
+
+    def rmdir(self, path):  
+        try:
+            self.log.debug('-> rmdir: %s', path)   
+            metrics.counts.incr('rmdir')  
+            metadata.cache.deleteMetadata(path)
+            metadata.cache.deleteParentMetadata(path)
+            sftp.manager.sftp().rmdir(fixPath(path))
+            self.log.debug('<- rmdir: %s', path)    
+        except Exception as e:
+            self.log.error('<- rmdir: %s', path)  
+            metrics.counts.incr('rmdir_except') 
+            raise e
+
+    def symlink(self, target, source): 
+        try:
+            self.log.debug('-> symlink: %s %s', target, source)   
+            metrics.counts.incr('symlink')        
+            sftp.manager.sftp().symlink(fixPath(source), fixPath(target))
+            self.log.debug('<- symlink: %s %s', target, source)     
+        except Exception as e:
+            self.log.error('<- symlink: %s %s', target, source) 
+            metrics.counts.incr('symlink_except')   
+            raise e
+
+    def truncate(self, path, length, fh=None):  
+        try:
+            self.log.debug('-> truncate: %s %d', path, length)  
+            metrics.counts.incr('truncate')         
+            metadata.cache.deleteMetadata(path)
+            data.cache.deleteStaleFile(path)
+            sftp.manager.sftp().truncate(fixPath(path), length)
+            self.log.debug('<- truncate: %s', path)   
+        except Exception as e:
+            self.log.error('<- truncate: %s %d', path, length)  
+            metrics.counts.incr('truncate_except') 
+            raise e  
+
+    def unlink(self, path):  
+        try: 
+            self.log.debug('-> unlink: %s', path)    
+            metrics.counts.incr('unlink')     
+            metadata.cache.deleteMetadata(path)
+            metadata.cache.deleteParentMetadata(path)
+            data.cache.deleteStaleFile(path)
+            sftp.manager.sftp().unlink(fixPath(path))
+            self.log.debug('<- unlink: %s', path)    
+        except Exception as e:
+            self.log.error('<- unlink: %s', path)   
+            metrics.counts.incr('unlink_except') 
+            raise e
+
+    def utimens(self, path, times=None):
+        try:
+            self.log.debug('-> utimens: %s', path) 
+            metrics.counts.incr('utimens')   
+            metadata.cache.deleteMetadata(path)
+            data.cache.deleteStaleFile(path)
+            sftp.manager.sftp().utime(fixPath(path), times)
+            self.log.debug('<- utimens: %s', path)   
+        except Exception as e:
+            self.log.error('<- utimens: %s', path) 
+            metrics.counts.incr('utimens_except') 
+            raise e 
+
+    def write(self, path, buf, offset, fh): 
+        try:       
+            self.log.debug('-> write: %s %d', path, offset)
+            metrics.counts.incr('write')
+            metadata.cache.deleteMetadata(path)  
+            data.cache.removeStaleBlocks(path)
+            #self.log.debug('write: write to remote file %s %d', path, offset)
+            with sftp.manager.sftp().open(fixPath(path), 'r+') as file:
+                file.seek(offset, 0)
+                file.write(buf)
+                file.close()
+            self.log.debug('<- write: %s %d', path, len(buf))
+            return len(buf)
+        except Exception as e:
+            self.log.error('<- write: %s %d', path, offset)
+            metrics.counts.incr('write_except') 
+            raise e           
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()  
+    parser.description = 'To unmount use: fusermount -u mountpoint'
+    parser.add_argument('host', help='remote host name')
+    parser.add_argument('mountpoint', help='local mount point (eg, ~/mnt)')
+    parser.add_argument('-p', '--port', help='port number (default=22)', default=22)
+    parser.add_argument('-u', '--user', help='user on remote host', default=getpass.getuser())
+    parser.add_argument('-d', '--remotedir', help='directory on remote host (eg, ~/)', default=Main.HOME_DIR)
+    parser.add_argument('--debug', help='run in debug mode', action='store_true')
+    parser.add_argument('--cachetimeout', type=int, help='duration in seconds to keep metadata cached (default is 5 minutes)', default=Main.CACHE_TIMEOUT)
+
+    args = parser.parse_args()   
+
+    log.Log().setupConfig(debug=args.debug)            
+    
+    main = Main(args)    
+
+    #print(args.host, args.login)
+    #exit()
+    #breakpoint()
+    try:
+        fuse = FUSE(
+            main,
+            args.mountpoint,
+            foreground=args.debug,
+            nothreads=False,
+            allow_other=True,
+        )
+    except Exception as e:
+        pass
+
+if __name__ == '__main__':
+    main()

sshfs-offline/data.py

@@ -0,0 +1,166 @@
+
+import math
+from pathlib import Path
+import os
+import log
+
+from logging import getLogger
+import queue
+import threading
+import time
+
+import metrics
+import sftp
+from sftp import fixPath
+
+from errno import ENOENT
+
+import metadata
+
+from fuse import FuseOSError
+
+class Data:
+    '''
+    On demand data file cache.   The files are cached in 64k chunks (blocks).  Only the blocks of the file that is read by
+    the user are cached.  Subsequent reads for the same data block are very fast.
+    '''
+    DATA_DIR = os.path.join(Path.home(), '.sshfs-offline', 'data') 
+    BLOCK_SIZE = sftp.BLOCK_SIZE  
+ 
+    def __init__(self, host: str, basedir: str):
+        self.log = getLogger(log.DATA)
+            
+        # make data cache directory ~/.sshfs-offline/data
+        self.dataDir = os.path.join(Data.DATA_DIR, host, os.path.splitroot(basedir)[-1])
+        if not os.path.exists(self.dataDir):
+            os.makedirs(self.dataDir) 
+
+        self.fileReaderQueue = queue.Queue()
+
+        threading.Thread(target=self.fileReaderThread).start()        
+     
+    def _dataPath(self, path: str) -> str:
+        #p = path.replace('/','%').replace('\\', '%')
+        return os.path.join(self.dataDir, path[1:]) 
+      
+    def statvfs(self, path: str):
+        self.log.debug('statvfs: %s', path)              
+        dataPath = self._dataPath(path)
+       
+        if os.path.exists(dataPath):
+            return os.statvfs(dataPath) 
+                    
+    def deleteStaleFile(self, path, mtime: float=None ): 
+        self.log.debug('deleteStaleFile: %s', path)    
+        if not sftp.manager.isConnected():
+            return
+        
+        dataPath = self._dataPath(path)        
+     
+        if os.path.exists(dataPath) and os.path.isfile(path):                               
+            if (mtime == None or os.lstat(dataPath).st_ctime < mtime):
+                self.log.debug('deleteStaleFile: deleting %s', path) 
+                metrics.counts.incr('deleteStaleFile')
+                os.unlink(dataPath)  
+                metadata.cache.deleteMetadata(path, [metadata.Metadata.BLOCKMAP])             
+
+    def read(self, path, size, offset, fh):  
+        #self.log.debug('read: %s input: size=%d offset=%d fd=%d', path, size, offset, fh)
+
+        buf = bytearray()
+
+        dataPath = self._dataPath(path)
+        d = os.path.dirname(dataPath)
+        if not os.path.exists(d):
+            os.makedirs(d)
+        if not os.path.exists(dataPath):
+            with open(dataPath, 'wb') as file:
+                fileSize = 0
+                st = metadata.cache.getattr(path)
+                if st != None:
+                    fileSize = st['st_size']
+                else:
+                    fileSize = sftp.manager.sftp().lstat(fixPath(path)).st_size
+                file.truncate(fileSize)       
+
+        blockMap = metadata.cache.blockmap(path)
+        blockNumSlice = range(math.floor(offset / Data.BLOCK_SIZE) , min(math.ceil((offset + size) / Data.BLOCK_SIZE), len(blockMap))) 
+              
+        try:
+            if not 1 in blockMap[blockNumSlice[0]:blockNumSlice[-1]+1]:            
+                with sftp.manager.sftp().open(fixPath(path), 'rb') as file:
+                    file.seek(blockNumSlice[0]*Data.BLOCK_SIZE)
+                    tempBuf = file.read(len(blockNumSlice)*Data.BLOCK_SIZE)
+
+                with open(dataPath, 'rb+') as file:
+                    file.seek(blockNumSlice[0]*Data.BLOCK_SIZE)
+                    file.write(tempBuf)
+
+                for blockNum in blockNumSlice:                    
+                    blockMap[blockNum] = 1
+                metadata.cache.blockmap_save(path, blockMap)
+
+                blockOffset = offset%Data.BLOCK_SIZE
+                buf = tempBuf[blockOffset:min(len(tempBuf), blockOffset+size)]
+
+                # More unread blocks?
+                if 0 in blockMap:
+                    self.fileReaderQueue.put(path)
+            else:                     
+                for blockNum in blockNumSlice:
+                    if blockMap[blockNum] == 0:
+                        #self.log.debug('read: %s get block %d from remote', path, blockNum)
+                        with sftp.manager.sftp().open(fixPath(path), 'rb') as file:                                       
+                            file.seek(blockNum*Data.BLOCK_SIZE)
+                            block = file.read(Data.BLOCK_SIZE) 
+                                                
+                        with open(dataPath, 'rb+') as file:
+                            file.seek(blockNum*Data.BLOCK_SIZE)
+                            file.write(block) 
+
+                        blockMap[blockNum] = 1
+                        metadata.cache.blockmap_save(path, blockMap)
+
+                        if len(buf) == 0:
+                            blockOffset = offset%Data.BLOCK_SIZE                            
+                            buf = block[blockOffset : min(Data.BLOCK_SIZE, blockOffset+size)]
+                        else:
+                            buf += block[0 : min(Data.BLOCK_SIZE, size-len(buf))]
+                    else:
+                        with open(dataPath, 'rb') as file:
+                            if len(buf) == 0:
+                                file.seek(offset)                        
+                                buf = file.read(min(size, Data.BLOCK_SIZE-offset%Data.BLOCK_SIZE))                    
+                            else:
+                                file.seek(blockNum*Data.BLOCK_SIZE)    
+                                buf += file.read(min(Data.BLOCK_SIZE, size-len(buf)))                           
+        except Exception as e:            
+            self.log.error('read: %s size=%d offset=%d', path, size, offset)
+            self.log.error('read: %s blockMap=%s', path, blockMap)
+            self.log.error('read: %s blockMap=%s', path, blockNumSlice)
+            raise e
+       
+        #self.log.debug('read: %s %d', path,= len(buf))
+        return bytes(buf)  
+
+    def fileReaderThread(self):
+        while True:
+            path = self.fileReaderQueue.get()
+            self.log.debug('-> fileReaderThread %s', path)
+            blockMap = metadata.cache.blockmap(path)  
+            unreadBlockFound = False          
+            for i in range(0, len(blockMap)):
+                if blockMap[i] == 0: 
+                    size = Data.BLOCK_SIZE
+                    offset = i * Data.BLOCK_SIZE
+                    self.log.debug('<- fileReaderThread %s size=%s offset=%s', path, size, offset)
+                    self.read(path, size, offset, 0) 
+                    unreadBlockFound = True                    
+                    break
+
+            if unreadBlockFound:
+                metrics.counts.incr('fileReaderThread')
+            else:
+                self.log.debug('<- fileReaderThread %s all blocks read', path)
+
+cache: Data = None

sshfs-offline/log.py

@@ -0,0 +1,49 @@
+    
+import logging
+import os
+from pathlib import Path
+
+MAIN        = 'main    '
+SFTP        = 'sftp    '
+METADATA    = 'metadata'
+DATA        = 'data    '
+METRICS     = 'metrics'
+
+FUSE        = 'fuse'
+PARAMIKO    = 'paramiko'
+
+class Log:
+    def __init__(self):
+        self.logDir = os.path.join(Path.home(), '.sshfs-offline')
+        if not os.path.exists(self.logDir):
+            os.makedirs(self.logDir)
+        self.formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s %(message)s')
+
+    def setupConfig(self, debug: bool):    
+
+        ## debug logging
+        if debug:            
+            logging.getLogger(FUSE).setLevel(logging.WARNING)    
+            logging.getLogger(PARAMIKO).setLevel(logging.WARNING)            
+            logging.basicConfig(
+                format='%(asctime)s:%(levelname)s:%(name)s %(message)s',
+                datefmt='%H:%M:%S',
+                level=logging.DEBUG                                
+            ) 
+                
+        # error logging 
+        for name in [MAIN, SFTP, METADATA, DATA, FUSE, PARAMIKO]:
+            logger = logging.getLogger(name)
+            errorHandler = logging.FileHandler(os.path.join(self.logDir, 'error.log'), mode='w')
+            errorHandler.setFormatter(self.formatter) 
+            errorHandler.setLevel(logging.ERROR)
+            logger.addHandler(errorHandler)
+            if not debug:
+                logger.setLevel(logging.ERROR)   
+
+        # metrics logging
+        metricsHandler = logging.FileHandler(os.path.join(self.logDir, 'metrics.log'), mode='w')
+        metricsHandler.setFormatter(self.formatter)             
+        logger = logging.getLogger(METRICS)
+        logger.addHandler(metricsHandler)
+        logger.setLevel(logging.DEBUG)        

sshfs-offline/metadata.py

@@ -0,0 +1,166 @@
+
+import math
+from pathlib import Path
+import os
+import log
+
+import json
+from logging import getLogger
+
+import shutil
+import time
+
+from errno import ENOENT
+
+import data
+import metrics
+import sftp
+
+from fuse import FuseOSError
+
+class Metadata:
+    '''
+    Metadata cache for getattr, readdir and read link operations.
+    '''
+    METADATA_DIR = os.path.join(Path.home(), '.sshfs-offline', 'metadata')
+    GETATTR = 'getattr'
+    READDIR = 'readdir'
+    READLINK = 'readlink'
+    BLOCKMAP = 'blockmap'
+    
+    def __init__(self, host: str, basedir: str, cachetimeout: float):
+        self.log = getLogger(log.METADATA)
+
+        self.cachetimeout = cachetimeout
+        
+        self.metadataDir = os.path.join(Metadata.METADATA_DIR, host, os.path.splitroot(basedir)[-1])
+        if not os.path.exists(self.metadataDir):
+            os.makedirs(self.metadataDir)
+
+    def deleteMetadata(self, path, files=[GETATTR, READDIR, READLINK]):
+        if not sftp.manager.isConnected():
+            return
+        
+        mdPath = self._metadataPath(path)
+        if os.path.exists(mdPath):
+            for file in files:
+                filePath = os.path.join(mdPath, file)
+                if os.path.exists(filePath):
+                    metrics.counts.incr('deleteMetadata')
+                    os.unlink(filePath)
+        
+    def deleteParentMetadata(self, path):
+        if not sftp.manager.isConnected():
+            return
+        
+        p = os.path.split(path)[0]
+        self.deleteMetadata(p)
+        
+    # 'st_atime', 'st_gid', 'st_mode', 'st_mtime', 'st_size', 'st_uid'    
+    def getattr(self, path)-> dict:
+        return self._readCache(path, Metadata.GETATTR)
+        
+    def getattr_save(self, path, dic: dict):        
+        if dic == {}:
+            data.cache.deleteStaleFile(path)            
+            self._storeCache(path, Metadata.GETATTR, dic)
+        elif dic != None:           
+            data.cache.deleteStaleFile(path, dic['st_mtime']) 
+            self._storeCache(path, Metadata.GETATTR, dic)
+       
+    def readdir(self, path)-> list[str]:       
+        return self._readCache(path, Metadata.READDIR)
+        
+    def readdir_save(self, path, s: list[str]=None):
+        self._storeCache(path, Metadata.READDIR, s)        
+                
+    def readlink(self, path:str) -> str | None:        
+        return self._readCache(path, Metadata.READLINK)
+           
+    def readlink_save(self, path:str, link: str=None):        
+        self._storeCache(path, Metadata.READLINK, link)        
+    
+    def blockmap(self, path:str) -> bytearray | None:                   
+        bm = self._readCache(path, Metadata.BLOCKMAP)
+        metrics.counts.incr('blockmap')
+        # not sure how the blockMap can be zero length?
+        if bm != None and len(bm) == 0:
+            bm = None
+            self.log.warning('blockmap: %s zero length blockMap?', path)
+
+        if bm == None:
+            fileSize = 0
+            st = self.getattr(path)
+            if st != None:
+                fileSize = st['st_size']               
+            else:
+                fileSize = sftp.manager.sftp().lstat(sftp.fixPath(path)).st_size
+                
+            bm = bytearray(math.ceil(fileSize/data.cache.BLOCK_SIZE))            
+        
+        return bm    
+    
+    def blockmap_save(self, path:str, blockMap: bytearray):
+        metrics.counts.incr('blockmap_save')
+        self._storeCache(path, Metadata.BLOCKMAP, blockMap)        
+    
+    # 
+    # Private methods:
+    #
+
+    def _metadataPath(self, path: str, operation: str=None) -> str:
+        p = path.replace('/','%').replace('\\', '%')
+        d = os.path.join(self.metadataDir, p)
+        if not os.path.exists(d):
+            os.mkdir(d)
+        if operation == None:
+            return d
+        else:
+            return os.path.join(d, operation)
+                   
+    def _storeCache(self, path, operation, d: dict | list[str] | str | bytearray):  
+        self.log.debug('_storeCace.%s: %s', operation, path)     
+        if not sftp.manager.isConnected():
+            return
+         
+        p = self._metadataPath(path, operation)
+        if operation == Metadata.BLOCKMAP:            
+            with open(p, "wb") as file:
+                file.write(bytes(d))
+        else:
+            with open(p, "w") as file:
+                if d == ENOENT:
+                    json.dump(d, None)
+                else:
+                    json.dump(d, file, indent=4)
+
+    def _readCache(self, path, operation) -> dict | list[str] | str | bytearray:       
+        metadataPath = self._metadataPath(path, operation)        
+        if os.path.exists(metadataPath):
+            if operation == Metadata.BLOCKMAP:
+                with open(metadataPath, 'rb') as file:
+                    buf = file.read()
+                    self.log.debug('_readCache.%s: %s %s', operation, path, str(buf))   
+                    metrics.counts.incr('blockmap_hit')
+                    return bytearray(buf)
+            else:
+                if time.time() > os.lstat(metadataPath).st_ctime + self.cachetimeout and sftp.manager.isConnected():            
+                    self.log.debug('_readCache.%s: expired %s', operation, path)
+                    os.unlink(metadataPath)
+                    metrics.counts.incr(operation+'_expired')    
+                    return None
+                else:                
+                    with open(metadataPath, 'r') as file:                    
+                        d = json.load(file)
+                        logMd = ''  
+                        if operation != Metadata.READDIR:
+                            logMd = d
+                        self.log.debug('_readCache.%s: %s %s', operation, path, logMd)  
+                        metrics.counts.incr(operation+'_hit')                             
+                        return d
+                    
+        self.log.debug('readCache.%s: not found %s', operation, path)
+        return None
+          
+
+cache: Metadata = None

sshfs-offline/metrics.py

@@ -0,0 +1,64 @@
+
+from logging import getLogger
+import copy
+import logging
+import threading
+import time
+import log
+
+
+class Metrics:
+    def __init__(self):
+       self.log = getLogger(log.METRICS)   
+       self.counts: dict[str,int] = dict()
+       self.prevCounts: dict[str, int] = dict()
+       self.stopped = False
+
+    def start(self):
+        threading.Thread(target=self.captureLoop).start()
+        
+    def incr(self, name: str): 
+        if name in self.counts:
+            self.counts[name] += 1
+        else:
+            self.counts[name] = 1                   
+
+    def _logCounts(self):
+        lines: list[str] = []
+        diff = 0
+        keys = list(self.counts.keys())
+        keys.sort()
+        for key in keys:
+            if key in self.prevCounts: 
+                diff = self.counts[key] - self.prevCounts[key]
+            else:
+                diff = self.counts[key]
+            if diff > 0:
+                lines.append('\n   {}: {}'.format(key.ljust(16), diff))
+        
+        self.prevCounts = copy.deepcopy(self.counts)
+        
+        if len(lines) > 0:
+            self.log.info(''.join(lines))
+
+    def captureLoop(self):
+        try:            
+            while True:                
+                time.sleep(10)
+                if self.stopped:
+                    break
+                self._logCounts()                
+        except Exception as e:
+            self.log.error('Exception: %s', e)
+
+    def stop(self):
+        self.log.info('metrics_stop')
+        self.stopped = True
+
+counts: Metrics
+
+
+            
+
+
+       

sshfs-offline/sftp.py

@@ -0,0 +1,188 @@
+import errno
+from logging import getLogger
+import logging
+import os
+from pathlib import Path
+import time
+from typing import Iterator
+import paramiko
+import threading
+
+import getpass
+import socket
+
+from fuse import FuseOSError
+
+import metrics
+import log
+
+BLOCK_SIZE = 131072
+WINDOW_SIZE = 1073741824 
+
+def fixPath(path):
+    return os.path.splitroot(path)[-1]
+
+class Connection:
+    def __init__(self, sshClient: paramiko.SSHClient, sftpClient: paramiko.SFTPClient):
+        self.sshClient: paramiko.SSHClient  = sshClient
+        self.sftpClient: paramiko.SFTPClient = sftpClient
+        self.offline = False
+
+class SftpOffline:
+    def close(self) -> None:
+        pass
+    def listdir(self, path: str = ".") -> list[str]:
+        raise FuseOSError(errno.ENETDOWN)
+    def listdir_attr(self, path: str = ".") -> list[paramiko.SFTPAttributes]:
+        raise FuseOSError(errno.ENETDOWN)
+    def listdir_iter(self, path: bytes | str = ".", read_aheads: int = 50) -> Iterator[paramiko.SFTPAttributes]:
+        raise FuseOSError(errno.ENETDOWN)
+    def open(self, filename: bytes | str, mode: str = "r", bufsize: int = -1) -> paramiko.SFTPFile:
+        raise FuseOSError(errno.ENETDOWN)
+    file = open
+    def remove(self, path: bytes | str) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    unlink = remove
+    def rename(self, oldpath: bytes | str, newpath: bytes | str) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def posix_rename(self, oldpath: bytes | str, newpath: bytes | str) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def mkdir(self, path: bytes | str, mode: int = 511) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def rmdir(self, path: bytes | str) -> None:
+        raise FuseOSError(errno.ENETDOWN)        
+    def stat(self, path: bytes | str) -> paramiko.SFTPAttributes:
+        raise FuseOSError(errno.ENETDOWN)
+    def lstat(self, path: bytes | str) -> paramiko.SFTPAttributes:
+        raise FuseOSError(errno.ENETDOWN)
+    def symlink(self, source: bytes | str, dest: bytes | str) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def chmod(self, path: bytes | str, mode: int) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def chown(self, path: bytes | str, uid: int, gid: int) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def utime(self, path: bytes | str, times: tuple[float, float] | None) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def truncate(self, path: bytes | str, size: int) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def readlink(self, path: bytes | str) -> str | None:
+        raise FuseOSError(errno.ENETDOWN)
+    def normalize(self, path: bytes | str) -> str:
+        raise FuseOSError(errno.ENETDOWN)
+    def chdir(self, path: None | bytes | str = None) -> None:
+        raise FuseOSError(errno.ENETDOWN)
+    def getcwd(self) -> str | None:
+        raise FuseOSError(errno.ENETDOWN)         
+
+class SFTPManager:
+    def __init__(self, host, user, remotedir, port):
+        self.log = getLogger(log.SFTP)
+        self.host = host
+        self.user = user 
+        self.password = None      
+        self.remotedir = remotedir
+        self.port = port 
+        self.local = threading.local()
+        self.connections: dict[str, Connection] = dict() 
+        self.offline = False
+        self.keepaliveStarted = False   
+        self.keepaliveStopped = False  
+
+    def isConnected(self):
+        return not isinstance(self.sftp(), SftpOffline) and not self.offline
+
+    def sftp(self) -> paramiko.SFTPClient | SftpOffline:                    
+        threadId = threading.get_native_id()
+        if self.offline:
+            if threadId in self.connections:
+                self.sftpClose()
+            return SftpOffline()
+        
+        if (threadId not in self.connections or 
+            not (self.connections[threadId].sshClient.get_transport().is_active() and
+                 self.connections[threadId].sshClient.get_transport().is_alive())):  
+            if threadId in self.connections:
+                self.sftpClose()      
+            sshClient = paramiko.SSHClient()
+            sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            sshClient.load_system_host_keys()            
+            try:
+                sshClient.connect(self.host, port=self.port, username=self.user, password=self.password)
+               #sshClient.get_transport().set_keepalive(60)                
+            except socket.gaierror:
+                self.log.debug('sftp: Cannot connect to host '+self.host)
+                print('Cannot connect to host ' + self.host + '.   Only cached data will be available.')
+                metrics.counts.incr('sftp_connect_err') 
+                return SftpOffline()
+            except OSError as e:
+                self.log.debug('sftp: %s', e)
+                print('{}.   Only cached data will be available.'.format(e))
+                metrics.counts.incr('sftp_network_err') 
+                return SftpOffline()
+            except paramiko.ssh_exception.AuthenticationException:
+                self.password = getpass.getpass("Enter password: ")
+                try:
+                    sshClient.connect(self.host, port=self.port, username=self.user, password=self.password)
+                except paramiko.ssh_exception.AuthenticationException:
+                    self.log.debug("sftp: Authentication failed")
+                    print('Invalid user or password')
+                    metrics.counts.incr('sftp_auth_err') 
+                    exit(1)
+            
+            metrics.counts.incr('sftp_connected') 
+            sshClient.get_transport().default_window_size = WINDOW_SIZE
+            self.connections[threadId] = Connection(sshClient, sshClient.open_sftp())
+            self.connections[threadId].sftpClient.SFTP_FILE_OBJECT_BLOCK_SIZE = BLOCK_SIZE
+            try:
+                self.connections[threadId].sftpClient.chdir(self.remotedir)
+                metrics.counts.incr('sftp_chdir') 
+            except IOError:
+                self.log.debug('--remotedir '+self.remotedir+' not found on host '+self.host)
+                print('--remotedir '+self.remotedir+' not found on host '+self.host)
+                metrics.counts.incr('sftp_chdir_err') 
+                exit(1)
+                        
+                           
+        return self.connections[threadId].sftpClient    
+    
+    def sftpClose(self):
+        metrics.counts.incr('sftp_close')
+        threadId = threading.get_native_id()
+        val = self.connections.pop(threadId)
+        val.sftpClient.close()
+        val.sshClient.close() 
+
+    def startKeepalive(self):
+        if self.keepaliveStarted == False:
+            metrics.counts.incr('sftp_start')
+            self.keepaliveStarted = True
+            threading.Thread(target=self.keepaliveThread).start()
+
+    def keepaliveThread(self):        
+        metrics.counts.incr('sftp_keepaliveThread')
+        while True:
+            if self.keepaliveStopped:
+                metrics.counts.incr('sftp_stopped')
+                break
+            time.sleep(10)
+            sshClient = paramiko.SSHClient()
+            sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            sshClient.load_system_host_keys()            
+            try:
+                sshClient.connect(self.host, port=self.port, username=self.user, password=self.password)                              
+            except Exception as e:
+                if self.offline == False:
+                    self.offline = True
+                    metrics.counts.incr('sftp_offline')
+            else:                
+                if self.offline:
+                    metrics.counts.incr('sftp_online')
+                    self.offline = False                    
+                sshClient.close()
+
+    def stop(self):
+        self.log.info('sftp_stop')
+        metrics.counts.incr('sftp_stop')
+        self.keepaliveStopped = True
+
+manager: SFTPManager = None

sshfs_offline-0.0.1.dist-info/METADATA

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: sshfs-offline
+Version: 0.0.1
+Summary: SSH File System with offline access to cached files.
+Author-email: Dave Christenson <davechri58@gmail.com>
+License: MIT
+Keywords: offline,cache,sshfs,fuse-filesystem,fuse
+Classifier: Development Status :: 4 - Beta
+Classifier: Topic :: System :: Filesystems
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+sshfs-offline
+=============
+
+SSH File System with offline access to cached files.
+
+Features:
+
+  - Based on FUSE (Filesystem in Userspace framework for Linux)
+
+  - Metadata and Data are cached locally to improve performance.
+
+  - Offline access to cached data when the remote host is not reachable
+
+  - Read/Write file system
+
+Install Script:
+===============
+
+```sh
+$ pip install sshfs-offline
+
+```
+
+How to mount a filesystem
+=========================
+
+Usage:
+
+    ```sh
+    usage: sshfs-offline [-h] [-p PORT] [-u USER] [-d REMOTEDIR] [--debug] [--cachetimeout CACHETIMEOUT] host mountpoint
+
+    To unmount use: fusermount -u mountpoint
+
+    positional arguments:
+      host                  remote host name
+      mountpoint            local mount point (eg, ~/mnt)
+
+    options:
+      -h, --help            show this help message and exit
+      -p PORT, --port PORT  port number (default=22)
+      -u USER, --user USER  user on remote host
+      -d REMOTEDIR, --remotedir REMOTEDIR
+                            directory on remote host (eg, ~/)
+      --debug               run in debug mode
+      --cachetimeout CACHETIMEOUT
+                            duration in seconds to keep metadata cached (default is 5 minutes)
+
+    ```
+
+Example:
+
+    ```sh
+    sshfs-offline localhost ~/mnt
+    ```
+
+Note, that it's recommended to run it as user, not as root.  For this
+to work the mountpoint must be owned by the user.  If the username is
+different on the host you are connecting to, then use the --user option.
+
+If you need to enter a password sshfs-offline will ask for it. 
+You can also specify a remote directory using --remotedir.  The default
+is your home directory.
+
+The cache timeout defaults to 5 minutes, and can be set with the -cachetimeout option.
+
+To unmount the filesystem:
+
+    fusermount -u mountpoint
+
+Cache Implementation
+====================
+
+The data and metadata are cached in the **.sshfs-offline** directory.  In this example, the **test/myfile.txt** file has two 132K blocks.  The data is cached in the **data** sub-directory, and the metadata is cached in the **metadata** sub-directory.
+
+```sh
+➜  .sshfs-offline
+├── data
+│   └── localhost   # host name
+│       └── home
+│           └── dave
+│               └── test
+│                   └── myfile.txt 
+└── metadata
+    └── localhost   # host name
+        └── home
+            └── user                
+                ├── %test        # test direcotry
+                │   ├── getattr  # lstat status for directory
+                │   └── readdir  # directory entries
+                └── %test%myfile.txt  # test/myfile.txt file
+                    ├── blockmap      # track blocks that are cached
+                    └── getattr       # lstat status for file 
+```
+
+Debugging
+=========
+
+* Metrics are logged to the **~/.sshfs-offline/metrics.log** file.
+* In production  (--debug=False), the log level is set to **warning**, and logs are writtend to the **~/.sshfs-offline/error.log** file.
+* If the --debug option is specified, the log level is set to **debug**, the process is run in the foreground, and logs are written to stdout.
+
+Using the tail command to follow the metrics:
+```sh
+$ tail -f ~/.sshfs-offline
+2025-09-23 07:26:19,237:INFO:metrics 
+   getattr         : 181
+   getattr_hit     : 181
+   init            : 1
+   readdir         : 26
+   readdir_hit     : 26
+   readlink        : 104
+   readlink_hit    : 104
+   sftp_chdir      : 1
+   sftp_connected  : 1
+   sftp_healthThread: 1
+   sftp_start      : 1
+```
+
+Development
+===========
+
+Create Virtual Environment:
+
+```sh
+$ python3 -m venv my-venv-name
+$ source ~/my-venv-name/bin/activate
+$ pip install -r requirements.txt
+```
+
+Mount filesystem:
+
+```sh
+$ ./sshfs-offline/cli.py localhost ~/mnt
+
+```
+

sshfs_offline-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+sshfs-offline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sshfs-offline/cli.py,sha256=-CHus9EJbC0D4UoPrqSnff8PmMiM7kqQZCgyp37yKZ8,12755
+sshfs-offline/data.py,sha256=PiOheuWASrKLVqwBI2BmDl7FzFkU0x5-yCiWxZ9AFQI,6863
+sshfs-offline/log.py,sha256=l32kcFTl8Tpu6r7WbOoIHRCzjqRglKB1LGWxntalr8Q,1768
+sshfs-offline/metadata.py,sha256=cIkUDBLw4VLHLg7ydKzovv86HTIClgGGJEtafqCAGLo,6009
+sshfs-offline/metrics.py,sha256=KfYR9lKYvuN1EBttY_0AKfHSpJ-RJQKDivn4II_eugc,1553
+sshfs-offline/sftp.py,sha256=-kSFhRm8w-PCaUmMNiE67pKEA4KwLWfkffgnucpgsIM,7967
+sshfs_offline-0.0.1.dist-info/licenses/LICENSE,sha256=w1JMi3ef157H69w-vQjh4eV-7ZohqZBOuLZ2DA8-whg,1074
+sshfs_offline-0.0.1.dist-info/METADATA,sha256=3EKwDJJ0HPAlXDAdHlCpgxC0ZbNGag-DIwnXfkQ7paU,4865
+sshfs_offline-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sshfs_offline-0.0.1.dist-info/entry_points.txt,sha256=jnvW0_UWvhp8GejltsEbnxSRxWEfKD9VFNDFOmdSfNY,57
+sshfs_offline-0.0.1.dist-info/top_level.txt,sha256=bPyPvvBdF2qxSh7IdS8nGelR9d9tMix5NYhHMZaqmpQ,14
+sshfs_offline-0.0.1.dist-info/RECORD,,

sshfs_offline-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

sshfs_offline-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sshfs-offline = sshfs_offline.cli:main

sshfs_offline-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 David Christenson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sshfs_offline-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+sshfs-offline