pyLodStorage 0.0.78__tar.gz → 0.0.80__tar.gz

{pylodstorage-0.0.78/pylodstorage.egg-info → pylodstorage-0.0.80}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.78
+Version: 0.0.80
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.78 → pylodstorage-0.0.80}/lodstorage/entity.py

@@ -5,6 +5,7 @@ Created on 2020-08-19
 '''
 from lodstorage.yamlablemixin import YamlAbleMixin
 from lodstorage.jsonpicklemixin import JsonPickleMixin
+from lodstorage.jsonable import JSONAble
 from lodstorage.storageconfig import StorageConfig, StoreMode
 from lodstorage.sparql import SPARQL
 from lodstorage.sql import SQLDB
@@ -102,7 +103,7 @@ class EntityManager(YamlAbleMixin, JsonPickleMixin,JSONAbleList):
             cacheFile=self.getCacheFile(mode=mode)
             if os.path.isfile(cacheFile):
                 os.remove(cacheFile)
-    
+                    
     def getSQLDB(self,cacheFile):
         '''
         get the SQL database for the given cacheFile
@@ -114,6 +115,23 @@ class EntityManager(YamlAbleMixin, JsonPickleMixin,JSONAbleList):
         sqldb=self.sqldb=SQLDB(cacheFile,debug=config.debug,errorDebug=config.errorDebug)
         return sqldb
     
+    def initSQLDB(self,sqldb,listOfDicts=None,withCreate:bool=True,withDrop:bool=True,sampleRecordCount=-1):
+        '''
+        initialize my sql DB  
+        
+        Args:
+            listOfDicts(list): the list of dicts to analyze for type information
+            withDrop(boolean): true if the existing Table should be dropped
+            withCreate(boolean): true if the create Table command should be executed - false if only the entityInfo should be returned
+            sampleRecordCount(int): the number of records to analyze for type information
+        Return:
+            EntityInfo: the entity information such as CREATE Table command
+        '''
+        if listOfDicts is None:
+            listOfDicts=JSONAble.getJsonTypeSamplesForClass(self.clazz)
+        entityInfo=sqldb.createTable(listOfDicts, self.tableName, primaryKey=self.primaryKey,withCreate=withCreate,withDrop=withDrop,sampleRecordCount=sampleRecordCount) 
+        return entityInfo
+    
     def setNone(self,record,fields):
         '''
         make sure the given fields in the given record are set to none
@@ -167,14 +185,14 @@ GROUP by ?source
             raise Exception("unsupported mode %s" % self.mode)            
         return result     
     
-    def fromCache(self,force:bool=False,getListOfDicts=None,withDrop=True,sampleRecordCount=-1):
+    def fromCache(self,force:bool=False,getListOfDicts=None,append=False,sampleRecordCount=-1):
         '''
         get my entries from the cache or from the callback provided
         
         Args:
             force(bool): force ignoring the cache
             getListOfDicts(callable): a function to call for getting the data 
-            withDrop(bool): True if the table should be (re) created
+            append(bool): True if records should be appended
             sampleRecordCount(int): the number of records to analyze for type information
             
         Returns:
@@ -191,7 +209,7 @@ GROUP by ?source
             listOfDicts=getListOfDicts()
             duration=time.time()-startTime
             self.showProgress(f"got {len(listOfDicts)} {self.entityPluralName} in {duration:5.1f} s")   
-            self.cacheFile=self.storeLoD(listOfDicts,withDrop=withDrop,sampleRecordCount=sampleRecordCount)
+            self.cacheFile=self.storeLoD(listOfDicts,append=append,sampleRecordCount=sampleRecordCount)
             self.setListFromLoD(listOfDicts)
         else:
             # fromStore also sets self.cacheFile
@@ -270,14 +288,14 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             lod.append(entity.__dict__)
         return lod
     
-    def store(self,limit=10000000,batchSize=250,withDrop=True,fixNone=True,sampleRecordCount=-1)->str:
+    def store(self,limit=10000000,batchSize=250,append=False,fixNone=True,sampleRecordCount=-1)->str:
         '''
         store my list of dicts
         
         Args:
             limit(int): maximum number of records to store per batch
             batchSize(int): size of batch for storing
-             withDrop(bool): True if the table should be (re) created
+            append(bool): True if records should be appended
             fixNone(bool): if True make sure the dicts are filled with None references for each record
             sampleRecordCount(int): the number of records to analyze for type information
         
@@ -285,9 +303,9 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             str: The cachefile being used
         '''
         lod=self.getLoD()
-        return self.storeLoD(lod,limit=limit,batchSize=batchSize,withDrop=withDrop,fixNone=fixNone,sampleRecordCount=sampleRecordCount)
+        return self.storeLoD(lod,limit=limit,batchSize=batchSize,append=append,fixNone=fixNone,sampleRecordCount=sampleRecordCount)
         
-    def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,withDrop=True,fixNone=True,sampleRecordCount=1)->str:
+    def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,append=False,fixNone=True,sampleRecordCount=1)->str:
         ''' 
         store my entities 
         
@@ -296,7 +314,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             limit(int): maximum number of records to store
             batchSize(int): size of batch for storing
             cacheFile(string): the name of the storage e.g path to JSON or sqlite3 file
-            withDrop(bool): True if the table should be (re) created
+            append(bool): True if records should be appended
             fixNone(bool): if True make sure the dicts are filled with None references for each record
             sampleRecordCount(int): the number of records to analyze for type information
             
@@ -331,7 +349,13 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
                 cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
             sqldb=self.getSQLDB(cacheFile)
             self.showProgress ("storing %d %s for %s to %s:%s" % (len(listOfDicts),self.entityPluralName,self.name,config.mode,cacheFile)) 
-            entityInfo=sqldb.createTable(listOfDicts, self.tableName, primaryKey=self.primaryKey,withDrop=withDrop,sampleRecordCount=sampleRecordCount)   
+            if append:
+                withDrop=False
+                withCreate=False
+            else:
+                withDrop=True
+                withCreate=True
+            entityInfo=self.initSQLDB(sqldb,listOfDicts,withCreate=withCreate,withDrop=withDrop,sampleRecordCount=sampleRecordCount)
             self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany,fixNone=fixNone)
             self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
         else:

{pylodstorage-0.0.78 → pylodstorage-0.0.80}/lodstorage/jsonable.py

@@ -122,10 +122,21 @@ class JSONAble(object):
             cls=self.__class__
             if isinstance(self, JSONAbleList) and not hasattr(cls, 'getSamples'):
                 cls=self.clazz
-            if hasattr(cls, 'getSamples'):
-                getSamples=getattr(cls,'getSamples');
-                if callable(getSamples):
-                    return getSamples()
+            return JSONAble.getJsonTypeSamplesForClass(cls)
+        return None
+    
+    @staticmethod
+    def getJsonTypeSamplesForClass(cls):
+        '''
+        return the type samples for the given class
+        
+        Return:
+            list: a list of dict that specify the types by example
+        '''
+        if hasattr(cls, 'getSamples'):
+            getSamples=getattr(cls,'getSamples');
+            if callable(getSamples):
+                return getSamples()
         return None
     
     @staticmethod

{pylodstorage-0.0.78 → pylodstorage-0.0.80/pylodstorage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.78
+Version: 0.0.80
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.78 → pylodstorage-0.0.80}/setup.py

@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='pylodstorage',
-    version='0.0.78',
+    version='0.0.80',
 
     packages=['lodstorage',],
     author='Wolfgang Fahl',