pyLodStorage 0.0.45__tar.gz → 0.0.47__tar.gz

{pylodstorage-0.0.45/pylodstorage.egg-info → pylodstorage-0.0.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.45
+Version: 0.0.47
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.45 → pylodstorage-0.0.47}/lodstorage/entity.py

@@ -75,12 +75,15 @@ class EntityManager(YamlAbleMixin, JsonPickleMixin):
             if config.cacheFile is not None:
                 return config.cacheFile
         ''' get the path to the file for my cached data '''  
-        if mode is StoreMode.JSON:  
-            cachepath="%s/%s-%s.%s" % (cachedir,self.name,"events",'json')
+        if mode is StoreMode.JSON or mode is StoreMode.JSONABLE:  
+            cachepath="%s/%s-%s.%s" % (cachedir,self.name,self.entityPluralName,'json')
         elif mode is StoreMode.SPARQL:
             cachepath="%s %s" % ('SPARQL',config.endpoint)    
         elif mode is StoreMode.SQL:
-            cachepath="%s/%s.db" % (cachedir,config.tableName)
+            tableName=config.tableName
+            if tableName is None:
+                tableName=self.name
+            cachepath="%s/%s.db" % (cachedir,tableName)
         else:
             cachepath="undefined cachepath for %s" % (mode)
         return cachepath     
@@ -88,7 +91,7 @@ class EntityManager(YamlAbleMixin, JsonPickleMixin):
     def removeCacheFile(self):
         '''  remove my cache file '''
         mode=self.config.mode
-        if mode is StoreMode.JSON:
+        if mode is StoreMode.JSON or mode is StoreMode.JSONABLE:
             cacheFile=self.getCacheFile(mode=mode)
             if os.path.isfile(cacheFile):
                 os.remove(cacheFile)
@@ -119,8 +122,8 @@ class EntityManager(YamlAbleMixin, JsonPickleMixin):
         result=False
         config=self.config
         mode=self.config.mode
-        if mode is StoreMode.JSON:
-            result=os.path.isfile(self.getCacheFile(config=self.config,mode=StoreMode.JSON))
+        if mode is StoreMode.JSON or mode is StoreMode.JSONABLE:
+            result=os.path.isfile(self.getCacheFile(config=self.config,mode=mode))
         elif mode is StoreMode.SPARQL:
             # @FIXME - make abstract
             query=config.prefix+"""
@@ -164,7 +167,7 @@ GROUP by ?source
             self.cacheFile=self.store(listOfDicts)
         else:
             # fromStore also sets self.cacheFile
-            listOfDicts=self.fromStore()
+            self.fromStore()
         return listOfDicts      
         
     def fromStore(self,cacheFile=None):
@@ -184,6 +187,9 @@ GROUP by ?source
         mode=self.config.mode
         if mode is StoreMode.JSON:   
             JSONem=JsonPickleMixin.readJson(cacheFile)    
+        elif mode is StoreMode.JSONABLE:
+            # TODO: implement
+            pass
         elif mode is StoreMode.SPARQL:
             # @FIXME make abstract
             eventQuery="""
@@ -213,12 +219,13 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             raise Exception("unsupported store mode %s" % self.mode)
           
         if JSONem is not None:
+            # TODO FIXME
             return JSONem
         else:
             self.showProgress("read %d %s from %s in %5.1f s" % (len(listOfDicts),self.entityPluralName,self.name,time.time()-startTime))     
             return listOfDicts
         
-    def store(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,sampleRecordCount=1):
+    def store(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,fixNone=True,sampleRecordCount=1):
         ''' 
         store my entities 
         
@@ -231,10 +238,10 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
         '''
         config=self.config
         mode=config.mode
-        if mode is StoreMode.JSON:    
+        if mode is StoreMode.JSON or mode is StoreMode.JSONABLE:    
             if cacheFile is None:
                 cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.JSON)
-            self.showProgress ("storing %d events for %s to cache %s" % (len(self.events),self.name,cacheFile))
+            self.showProgress (f"storing {len(listOfDicts)} {self.entityPluralName} for {self.name} to cache {cacheFile}")
             self.writeJson(cacheFile)
         elif mode is StoreMode.SPARQL:
             startTime=time.time()
@@ -252,7 +259,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             sqldb=self.getSQLDB(cacheFile)
             self.showProgress ("storing %d %s for %s to %s:%s" % (len(listOfDicts),self.entityPluralName,self.name,config.mode,cacheFile)) 
             entityInfo=sqldb.createTable(listOfDicts, config.tableName, "eventId",withDrop=True,sampleRecordCount=sampleRecordCount)   
-            self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany)
+            self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany,fixNone=fixNone)
             self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
         else:
             raise Exception("unsupported store mode %s" % self.mode)  

{pylodstorage-0.0.45 → pylodstorage-0.0.47}/lodstorage/storageconfig.py

@@ -10,7 +10,7 @@ class StoreMode(Enum):
     '''
     possible supported storage modes
     '''
-    JSON = 1
+    JSON = 1      # JSON Pickle
     JSONABLE = 2
     SQL = 3
     SPARQL = 4
@@ -20,17 +20,6 @@ class StorageConfig(object):
     '''
     a storage configuration
     '''
-    
-    def ensureDirectoryExists(self,file_path:str):
-        '''
-        check that the given path exists and otherwise create it
-        
-        Args
-            file_path(str): the path to check
-        '''
-        directory = os.path.dirname(file_path)
-        if not os.path.exists(directory):
-            os.makedirs(directory)
            
     def getCachePath(self,ensureExists=True)->str:
         '''
@@ -41,8 +30,10 @@ class StorageConfig(object):
         '''
         home = str(Path.home())
         cachedir=f"{home}/.{self.cacheDirName}"
+        
         if ensureExists:
-            self.ensureDirectoryExists(cachedir)
+            if not os.path.exists(cachedir):
+                os.makedirs(cachedir)
         return cachedir
 
     def __init__(self, mode=StoreMode.SQL,cacheDirName="lodstorage",cacheFile=None,withShowProgress=True,profile=True,debug=False,errorDebug=True):

{pylodstorage-0.0.45 → pylodstorage-0.0.47/pylodstorage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.45
+Version: 0.0.47
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.45 → pylodstorage-0.0.47}/setup.py

@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='pylodstorage',
-    version='0.0.45',
+    version='0.0.47',
 
     packages=['lodstorage',],
     author='Wolfgang Fahl',