pyLodStorage 0.0.76__tar.gz → 0.0.78__tar.gz

{pylodstorage-0.0.76/pylodstorage.egg-info → pylodstorage-0.0.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.76
+Version: 0.0.78
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.76 → pylodstorage-0.0.78}/lodstorage/entity.py

@@ -10,6 +10,7 @@ from lodstorage.sparql import SPARQL
 from lodstorage.sql import SQLDB
 from lodstorage.lod import LOD
 from lodstorage.jsonable import JSONAbleList
+import sys
 import os
 import time
 
@@ -148,26 +149,33 @@ GROUP by ?source
         elif mode is StoreMode.SQL:
             cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.SQL)
             if os.path.isfile(cacheFile):
-                sqlQuery="SELECT COUNT(*) AS count FROM %s" % self.tableName
+                sqlQuery=f"SELECT COUNT(*) AS count FROM {self.tableName}" 
                 try:
                     sqlDB=self.getSQLDB(cacheFile)
-                    countResult=sqlDB.query(sqlQuery)
-                    count=countResult[0]['count']
-                    result=count>100
+                    countResults=sqlDB.query(sqlQuery)
+                    countResult=countResults[0]
+                    count=countResult['count']
+                    result=count>=0
                 except Exception as ex:
+                    msg=str(ex)
+                    if self.debug:
+                        print(msg,file=sys.stderr)
+                        sys.stderr.flush()
                     # e.g. sqlite3.OperationalError: no such table: Event_crossref
                     pass      
         else:
             raise Exception("unsupported mode %s" % self.mode)            
         return result     
     
-    def fromCache(self,force:bool=False,getListOfDicts=None,sampleRecordCount=-1):
+    def fromCache(self,force:bool=False,getListOfDicts=None,withDrop=True,sampleRecordCount=-1):
         '''
-        get my entries from the cache or from the callback provide
+        get my entries from the cache or from the callback provided
         
         Args:
             force(bool): force ignoring the cache
             getListOfDicts(callable): a function to call for getting the data 
+            withDrop(bool): True if the table should be (re) created
+            sampleRecordCount(int): the number of records to analyze for type information
             
         Returns:
             the list of Dicts and as a side effect setting self.cacheFile
@@ -176,11 +184,14 @@ GROUP by ?source
             startTime=time.time()
             self.showProgress(f"getting {self.entityPluralName} for {self.name} ...")
             if getListOfDicts is None:
-                getListOfDicts=self.getListOfDicts
+                if hasattr(self, "getListOfDicts"):
+                    getListOfDicts=self.getListOfDicts
+                else:
+                    raise  Exception("from Cache failed and no secondary cache via getListOfDicts specified")
             listOfDicts=getListOfDicts()
             duration=time.time()-startTime
             self.showProgress(f"got {len(listOfDicts)} {self.entityPluralName} in {duration:5.1f} s")   
-            self.cacheFile=self.storeLoD(listOfDicts,sampleRecordCount=sampleRecordCount)
+            self.cacheFile=self.storeLoD(listOfDicts,withDrop=withDrop,sampleRecordCount=sampleRecordCount)
             self.setListFromLoD(listOfDicts)
         else:
             # fromStore also sets self.cacheFile
@@ -259,13 +270,14 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             lod.append(entity.__dict__)
         return lod
     
-    def store(self,limit=10000000,batchSize=250,fixNone=True,sampleRecordCount=-1)->str:
+    def store(self,limit=10000000,batchSize=250,withDrop=True,fixNone=True,sampleRecordCount=-1)->str:
         '''
         store my list of dicts
         
         Args:
-            limit(int): maximum number of records to store
+            limit(int): maximum number of records to store per batch
             batchSize(int): size of batch for storing
+             withDrop(bool): True if the table should be (re) created
             fixNone(bool): if True make sure the dicts are filled with None references for each record
             sampleRecordCount(int): the number of records to analyze for type information
         
@@ -273,9 +285,9 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             str: The cachefile being used
         '''
         lod=self.getLoD()
-        return self.storeLoD(lod,limit=limit,batchSize=batchSize,fixNone=fixNone,sampleRecordCount=sampleRecordCount)
+        return self.storeLoD(lod,limit=limit,batchSize=batchSize,withDrop=withDrop,fixNone=fixNone,sampleRecordCount=sampleRecordCount)
         
-    def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,fixNone=True,sampleRecordCount=1)->str:
+    def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,withDrop=True,fixNone=True,sampleRecordCount=1)->str:
         ''' 
         store my entities 
         
@@ -284,6 +296,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
             limit(int): maximum number of records to store
             batchSize(int): size of batch for storing
             cacheFile(string): the name of the storage e.g path to JSON or sqlite3 file
+            withDrop(bool): True if the table should be (re) created
             fixNone(bool): if True make sure the dicts are filled with None references for each record
             sampleRecordCount(int): the number of records to analyze for type information
             
@@ -318,7 +331,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
                 cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
             sqldb=self.getSQLDB(cacheFile)
             self.showProgress ("storing %d %s for %s to %s:%s" % (len(listOfDicts),self.entityPluralName,self.name,config.mode,cacheFile)) 
-            entityInfo=sqldb.createTable(listOfDicts, self.tableName, primaryKey=self.primaryKey,withDrop=True,sampleRecordCount=sampleRecordCount)   
+            entityInfo=sqldb.createTable(listOfDicts, self.tableName, primaryKey=self.primaryKey,withDrop=withDrop,sampleRecordCount=sampleRecordCount)   
             self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany,fixNone=fixNone)
             self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
         else:

{pylodstorage-0.0.76 → pylodstorage-0.0.78}/lodstorage/sql.py

@@ -56,7 +56,7 @@ class SQLDB(object):
         '''
         self.c.execute(ddlCmd)
         
-    def createTable(self,listOfRecords,entityName,primaryKey=None,withDrop=False,sampleRecordCount=1,failIfTooFew=True):
+    def createTable(self,listOfRecords,entityName:str,primaryKey:str=None,withCreate:bool=True,withDrop:bool=False,sampleRecordCount=1,failIfTooFew=True):
         '''
         derive  Data Definition Language CREATE TABLE command from list of Records by examining first recorda
         as defining sample record and execute DDL command
@@ -68,6 +68,7 @@ class SQLDB(object):
            entityName(string): the entity / table name to use
            primaryKey(string): the key/column to use as a  primary key
            withDrop(boolean): true if the existing Table should be dropped
+           withCreate(boolean): true if the create Table command should be executed - false if only the entityInfo should be returned
            sampleRecords(int): number of sampleRecords expected and to be inspected
            failIfTooFew(boolean): raise an Exception if to few sampleRecords else warn only
         Returns:
@@ -87,10 +88,11 @@ class SQLDB(object):
         entityInfo=EntityInfo(sampleRecords,entityName,primaryKey,debug=self.debug)
         if withDrop:
             self.c.execute(entityInfo.dropTableCmd) 
-        try:
-            self.c.execute(entityInfo.createTableCmd)
-        except sqlite3.OperationalError as oe:
-            raise Exception(f"createTable failed with error {oe} for {entityInfo.createTableCmd}")
+        if withCreate:
+            try:
+                self.c.execute(entityInfo.createTableCmd)
+            except sqlite3.OperationalError as oe:
+                raise Exception(f"createTable failed with error {oe} for {entityInfo.createTableCmd}")
         return entityInfo
     
     def getDebugInfo(self,record,index,executeMany):

{pylodstorage-0.0.76 → pylodstorage-0.0.78/pylodstorage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.0.76
+Version: 0.0.78
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.0.76 → pylodstorage-0.0.78}/setup.py

@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name='pylodstorage',
-    version='0.0.76',
+    version='0.0.78',
 
     packages=['lodstorage',],
     author='Wolfgang Fahl',