pyLodStorage 0.0.1__py3-none-any.whl

pylodstorage-0.0.1.dist-info/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pylodstorage-0.0.1.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: pylodstorage
+Version: 0.0.1
+Summary: python List of Dict (Table) Storage library
+Home-page: https://github.com/WolfgangFahl/pyLodStorage
+Author: Wolfgang Fahl
+Author-email: wf@bitplan.com
+Maintainer: Wolfgang Fahl
+License: Apache License
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/markdown
+
+# pyLoDStorage
+python List of Dict (Table) Storage library
+
+[![pypi](https://img.shields.io/pypi/pyversions/pyLoDStorage)](https://pypi.org/project/pyLoDStorage/)
+[![Github Actions Build](https://github.com/WolfgangFahl/pyLoDStorage/workflows/Build/badge.svg?branch=master)](https://github.com/WolfgangFahl/pyLoDStorage/actions?query=workflow%3ABuild+branch%3Amaster)
+[![GitHub issues](https://img.shields.io/github/issues/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues/?q=is%3Aissue+is%3Aclosed)
+[![License](https://img.shields.io/github/license/WolfgangFahl/pyLoDStorage.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+
+What it is
+==========
+pyLoDStorage allows to store table like data (List of Dicts) via Sqlite3
+
+Installation
+============
+```bash
+pip install pylodstorage
+```
+
+Usage
+=====
+see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tests)
+

pylodstorage-0.0.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+storage/sample.py,sha256=V4-OtXUpmfgPEnuhIE9OeY4kKMcQCmN1ZSIwltbq8Vk,2609
+storage/sql.py,sha256=YCqMubFPh7p0qsxKD-psvMwHCBLYKi-dxXR7wKWMuxQ,16226
+storage/uml.py,sha256=jxV2y3Iaho-Sk4UWSWDXK9IglZcIiu8IvZ-4CoNGG9A,5082
+pylodstorage-0.0.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pylodstorage-0.0.1.dist-info/METADATA,sha256=i-BqQrlbLx1aCbOIPIUPkk402gHPNqjgoPzoHQYwFr0,1526
+pylodstorage-0.0.1.dist-info/WHEEL,sha256=EVRjI69F5qVjm_YgqcTXPnTAv3BfSUr0WVAHuSP3Xoo,92
+pylodstorage-0.0.1.dist-info/top_level.txt,sha256=yphqphSuwRzNyEfTnt0xnNv02quslUAR1yj7mKZ_KxI,8
+pylodstorage-0.0.1.dist-info/RECORD,,

pylodstorage-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.35.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pylodstorage-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+storage

storage/sample.py

@@ -0,0 +1,71 @@
+'''
+Created on 2020-08-24
+
+@author: wf
+'''
+import urllib.request
+import json
+from datetime import date, datetime
+
+class Sample(object):
+    '''
+    Sample dataset generator
+    '''
+
+    def __init__(self):
+        '''
+        Constructor
+        '''
+        
+    @staticmethod         
+    def getSample(size):         
+        listOfDicts=[]
+        for index in range(size):
+            listOfDicts.append({'pkey': "index%d" %index, 'cindex': index})
+        return listOfDicts    
+    
+    @staticmethod
+    def getCountries():
+        countryJsonUrl="https://gist.githubusercontent.com/erdem/8c7d26765831d0f9a8c62f02782ae00d/raw/248037cd701af0a4957cce340dabb0fd04e38f4c/countries.json"
+        with urllib.request.urlopen(countryJsonUrl) as url:
+            countryList=json.loads(url.read().decode())
+        return countryList
+    
+    @staticmethod
+    def getCities():
+        '''
+        get a list of cities
+        '''
+        cityJsonUrl="https://raw.githubusercontent.com/lutangar/cities.json/master/cities.json"
+        with urllib.request.urlopen(cityJsonUrl) as url:
+            cityList=json.loads(url.read().decode())
+        for city in cityList:
+            city['cityId']="%s-%s" % (city['country'],city['name'])    
+        return cityList
+    
+    @staticmethod
+    def dob(isoDateString):
+        ''' get the date of birth from the given iso date state'''
+        #if sys.version_info >= (3, 7):
+        #    dt=datetime.fromisoformat(isoDateString)
+        #else:
+        dt=datetime.strptime(isoDateString,"%Y-%m-%d")  
+        return dt.date()   
+    
+    @staticmethod
+    def getRoyals():
+        listOfDicts=[
+            {'name': 'Elizabeth Alexandra Mary Windsor', 'born': Sample.dob('1926-04-21'), 'numberInLine': 0, 'wikidataurl': 'https://www.wikidata.org/wiki/Q9682' },
+            {'name': 'Charles, Prince of Wales',         'born': Sample.dob('1948-11-14'), 'numberInLine': 1, 'wikidataurl': 'https://www.wikidata.org/wiki/Q43274' },
+            {'name': 'George of Cambridge',              'born': Sample.dob('2013-07-22'), 'numberInLine': 3, 'wikidataurl': 'https://www.wikidata.org/wiki/Q1359041'},
+            {'name': 'Harry Duke of Sussex',             'born': Sample.dob('1984-09-15'), 'numberInLine': 6, 'wikidataurl': 'https://www.wikidata.org/wiki/Q152316'}
+        ]
+        today=date.today()
+        for person in listOfDicts:
+            born=person['born']
+            age=(today - born).days / 365.2425
+            person['age']=age
+            person['ofAge']=age>=18
+            person['lastmodified']=datetime.now()
+        return listOfDicts
+        

storage/sql.py

@@ -0,0 +1,441 @@
+'''
+Created on 2020-08-24
+
+@author: wf
+'''
+# python standard library
+import sqlite3
+import datetime
+import io
+import time
+import sys
+import re
+
+
+class SQLDB(object):
+    '''
+    Structured Query Language Database wrapper
+    
+    :ivar dbname(string): name of the database
+    :ivar debug(boolean): True if debug info should be provided
+    :ivar errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
+    '''
+    RAM=":memory:"
+
+    def __init__(self,dbname=':memory:',connection=None,debug=False, errorDebug=False):
+        '''
+        Construct me for the given dbname and debug
+        
+        Args:
+        
+           dbname(string): name of the database - default is a RAM based database
+           connection(Connection): an optional connectio to be reused 
+           debug(boolean): if True switch on debug
+           errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
+        '''
+        self.dbname=dbname
+        self.debug=debug
+        self.errorDebug=errorDebug
+        if connection is None:
+            self.c=sqlite3.connect(dbname,detect_types=sqlite3.PARSE_DECLTYPES)
+        else:
+            self.c=connection
+        
+    def close(self):
+        ''' close my connection '''
+        self.c.close()    
+        
+    def createTable(self,listOfRecords,entityName,primaryKey=None,withDrop=False,sampleRecordCount=1):
+        '''
+        derive  Data Definition Language CREATE TABLE command from list of Records by examining first recorda
+        as defining sample record and execute DDL command
+        
+        auto detect column types see e.g. https://stackoverflow.com/a/57072280/1497139
+        
+        Args:
+           listOfRecords(list): a list of Dicts
+           entityName(string): the entity / table name to use
+           primaryKey(string): the key/column to use as a  primary key
+           withDrop(boolean): true if the existing Table should be dropped
+           sampleRecords(int): number of sampleRecords expected and to be inspected
+           
+        Returns:
+           EntityInfo: meta data information for the created table
+        '''
+        if len(listOfRecords)<sampleRecordCount:
+            raise Exception("Need %d sample records to createTable" % (sampleRecordCount))
+        sampleRecords=listOfRecords[:sampleRecordCount]
+        entityInfo=EntityInfo(sampleRecords,entityName,primaryKey,debug=self.debug)
+        if withDrop:
+            self.c.execute(entityInfo.dropTableCmd) 
+        self.c.execute(entityInfo.createTableCmd)
+        return entityInfo
+    
+    def getDebugInfo(self,record,index,executeMany):
+        '''
+        get the debug info for the given record at the given index depending on the state of executeMany
+        
+        Args:
+            record(dict): the record to show
+            index(int): the index of the record
+            executeMany(boolean): if True the record may be valid else not
+        '''
+        debugInfo=""
+        if not executeMany:
+            if self.errorDebug:
+                debugInfo="\nrecord  #%d=%s" % (index,repr(record))
+        return debugInfo
+        
+       
+    def store(self,listOfRecords,entityInfo,executeMany=False):
+        '''
+        store the given list of records based on the given entityInfo
+        
+        Args:
+          
+           listOfRecords(list): the list of Dicts to be stored
+           entityInfo(EntityInfo): the meta data to be used for storing
+        '''
+        insertCmd=entityInfo.insertCmd
+        record=None
+        index=0
+        try:
+            if executeMany:
+                self.c.executemany(insertCmd,listOfRecords)
+            else:
+                for record in listOfRecords:
+                    index+=1
+                    self.c.execute(insertCmd,record)
+            self.c.commit()
+        except sqlite3.ProgrammingError as pe:
+            msg=pe.args[0]
+            if "You did not supply a value for binding" in msg:
+                columnIndex=int(re.findall(r'\d+',msg)[0])
+                columnName=list(entityInfo.typeMap.keys())[columnIndex-1]
+                debugInfo=self.getDebugInfo(record, index, executeMany)
+                raise Exception("%s\nfailed: no value supplied for column '%s'%s" % (insertCmd,columnName,debugInfo))
+            else:
+                raise pe
+        except sqlite3.InterfaceError as ie:
+            msg=ie.args[0]
+            if "Error binding parameter" in msg:
+                columnName=re.findall(r':[_a-zA-Z]\w*',msg)[0]
+                debugInfo=self.getDebugInfo(record, index, executeMany)
+                raise Exception("%s\nfailed: error binding column '%s'%s" % (insertCmd,columnName,debugInfo))
+            else:
+                raise ie
+        
+    def query(self,sqlQuery):
+        '''
+        run the given sqlQuery and return a list of Dicts
+        
+        Args:
+            
+            sqlQuery(string): the SQL query to be executed
+                
+        Returns:
+            list: a list of Dicts
+        '''
+        if self.debug:
+            print(sqlQuery)
+        # https://stackoverflow.com/a/13735506/1497139
+        cur=self.c.cursor()
+        query = cur.execute(sqlQuery)
+        colname = [ d[0] for d in query.description ]
+        resultList=[]
+        for row in query:
+            record=dict(zip(colname, row))
+            resultList.append(record)
+        cur.close()
+        return resultList
+        
+    def queryAll(self,entityInfo,fixDates=True):
+        '''
+        query all records for the given entityName/tableName
+        
+        Args:
+           entityName(string): name of the entity/table to qury
+           fixDates(boolean): True if date entries should be returned as such and not as strings
+        '''  
+        sqlQuery='SELECT * FROM %s' % entityInfo.name
+        resultList=self.query(sqlQuery)
+        if fixDates:
+            entityInfo.fixDates(resultList)
+        return resultList
+    
+    def getTableList(self):
+        '''
+        get the schema information from this database
+        '''
+        tableQuery="SELECT name FROM sqlite_master WHERE type='table'"
+        tableList=self.query(tableQuery)
+        for table in tableList:
+            tableName=table['name']
+            columnQuery="PRAGMA table_info('%s')" % tableName
+            columns=self.query(columnQuery)
+            table['columns']=columns
+        return tableList
+              
+    def progress(self,status, remaining, total):
+        '''
+        show progress
+        '''
+        print('Backup %s at %5.0f%%' % ("... " if status==0 else "done",(total-remaining)/total*100)) 
+    
+    def backup(self,backupDB,action="Backup",profile=False,showProgress=200,doClose=True):
+        '''
+        create backup of this SQLDB to the given backup db
+        
+        see https://stackoverflow.com/a/59042442/1497139
+        
+        Args:
+            backupDB(string): the path to the backupdb or SQLDB.RAM for in memory
+            action(string): the action to display
+            profile(boolean): True if timing information shall be shown
+            showProgress(int): show progress at each showProgress page (0=show no progress)
+        '''
+        if sys.version_info <= (3, 6):
+            raise Exception("backup via stdlibrary not available in python <=3.6 use copyToDB instead")
+        startTime=time.time()
+        bck=sqlite3.connect(backupDB)
+        if showProgress>0:
+            progress=self.progress
+        else:
+            progress=None
+        with bck:
+            self.c.backup(bck,pages=showProgress,progress=progress)        
+        elapsed=time.time()-startTime
+        if profile:
+            print("%s to %s took %5.1f s" % (action,backupDB,elapsed))
+        if doClose:
+            bck.close()
+            return None
+        else:
+            return bck
+        
+    def showDump(self,dump,limit=10):
+        '''
+        show the given dump up to the given limit
+        
+        Args:
+            dump(string): the SQL dump to show
+            limit(int): the maximum number of lines to display
+        '''
+        s=io.StringIO(dump)
+        index=0
+        for line in s:
+            if index <= limit:
+                print(line)
+                index+=1    
+            else:
+                break        
+        
+    def executeDump(self,connection,dump,title,maxErrors=100,errorDisplayLimit=12,profile=True):
+        '''
+        execute the given dump for the given connection
+        
+        Args:
+            connection(Connection): the sqlite3 connection to use
+            dump(string): the SQL commands for the dump
+            title(string): the title of the dump
+            maxErrors(int): maximum number of errors to be tolerated before stopping and doing a rollback
+            profile(boolean): True if profiling information should be shown
+        Returns:
+            a list of errors
+        '''
+        if self.debug:
+            self.showDump(dump)
+        startTime=time.time()    
+        if profile:
+            print("dump of %s has size %4.1f MB" % (title,len(dump)/1024/1024))
+        errors=[]
+        index=0
+        # fixes https://github.com/WolfgangFahl/ProceedingsTitleParser/issues/37
+        for line in dump.split(";\n"):
+            try:
+                connection.execute(line)
+            except  sqlite3.OperationalError as soe:
+                msg="SQL error %s in line %d:\n\t%s" % (soe,index,line)
+                errors.append(msg)
+                if len(errors)<=errorDisplayLimit:
+                    print(msg)    
+                if len(errors)>=maxErrors:
+                    connection.execute("ROLLBACK;")
+                    break
+                
+            index=index+1
+        if profile:    
+            print("finished executing dump %s with %d lines and %d errors in %5.1f s" % (title,index,len(errors),time.time()-startTime))    
+        return errors
+    
+    def copyTo(self,copyDB,profile=True):
+        '''
+        copy my content to another database
+        
+        Args:
+            
+           copyDB(Connection): the target database
+           profile(boolean): if True show profile information
+        '''
+        startTime=time.time()
+        dump="\n".join(self.c.iterdump())
+        #cursor.executescript(dump)
+        if profile:
+            print("finished getting dump of %s in %5.1f s" % (self.dbname,time.time()-startTime))
+        dumpErrors=self.executeDump(copyDB.c,dump,self.dbname,profile=profile)
+        return dumpErrors
+       
+        
+    @staticmethod
+    def restore(backupDB,restoreDB,profile=False,showProgress=200,debug=False):
+        '''
+        restore the restoreDB from the given backup DB
+        
+        Args:
+            backupDB(string): path to the backupDB e.g. backup.db
+            restoreDB(string): path to the restoreDB or in Memory SQLDB.RAM
+            profile(boolean): True if timing information should be shown
+            showProgress(int): show progress at each showProgress page (0=show no progress)
+        '''
+        backupSQLDB=SQLDB(backupDB)
+        connection=backupSQLDB.backup(restoreDB,action="Restore",profile=profile,showProgress=showProgress,doClose=False)
+        restoreSQLDB=SQLDB(restoreDB,connection=connection,debug=debug)
+        return restoreSQLDB
+    
+        
+class EntityInfo(object):
+    """
+    holds entity meta Info 
+    
+    :ivar name(string): entity name = table name
+    
+    :ivar primaryKey(string): the name of the primary key column
+    
+    :ivar typeMap(dict): maps column names to python types
+    
+    :ivar debug(boolean): True if debug information should be shown
+    
+    """
+        
+    def __init__(self,sampleRecords,name,primaryKey=None,debug=False):
+        '''
+        construct me from the given name and primary key
+        
+        Args:
+           name(string): the name of the entity
+           primaryKey(string): the name of the primary key column
+           debug(boolean): True if debug information should be shown
+        '''
+        self.sampleRecords=sampleRecords
+        self.name=name
+        self.primaryKey=primaryKey
+        self.debug=debug
+        self.typeMap={}
+        self.sqlTypeMap={}
+        self.createTableCmd=self.getCreateTableCmd(sampleRecords)
+        self.dropTableCmd="DROP TABLE IF EXISTS %s" % self.name
+        self.insertCmd=self.getInsertCmd()
+        
+    def getCreateTableCmd(self,sampleRecords):
+        '''
+        get the CREATE TABLE DDL command for the given sample records
+        
+        Args:
+            sampleRecords(list): a list of Dicts of sample Records    
+            
+        Returns:
+            string: CREATE TABLE DDL command for this entity info 
+            
+        Example:   
+      
+        .. code-block:: sql
+            
+            CREATE TABLE Person(name TEXT PRIMARY KEY,born DATE,numberInLine INTEGER,wikidataurl TEXT,age FLOAT,ofAge BOOLEAN)
+    
+        '''
+        ddlCmd="CREATE TABLE %s(" %self.name
+        delim=""
+        for sampleRecord in sampleRecords:
+            for key,value in sampleRecord.items():
+                sqlType=None
+                valueType=None
+                if value is None:
+                    if len(sampleRecords)==1:
+                        print("Warning sampleRecord column %s is None - using TEXT as type" % key)
+                        valueType=str
+                else:
+                    valueType=type(value)
+                if valueType == str:
+                    sqlType="TEXT"
+                elif valueType == int:
+                    sqlType="INTEGER"
+                elif valueType == float:
+                    sqlType="FLOAT"
+                elif valueType == bool:
+                    sqlType="BOOLEAN"      
+                elif valueType == datetime.date:
+                    sqlType="DATE"    
+                elif valueType== datetime.datetime:
+                    sqlType="TIMESTAMP"
+                else:
+                    if valueType is not None:
+                        msg="warning: unsupported type %s for column %s " % (str(valueType),key)
+                        print(msg)
+                if sqlType is not None and valueType is not None:
+                    self.addType(key,valueType,sqlType)
+        for key,sqlType in self.sqlTypeMap.items():        
+            ddlCmd+="%s%s %s%s" % (delim,key,sqlType," PRIMARY KEY" if key==self.primaryKey else "")
+            delim=","
+        ddlCmd+=")"  
+        if self.debug:
+            print (ddlCmd)    
+        return ddlCmd
+        
+    def getInsertCmd(self):
+        '''
+        get the INSERT command for this entityInfo
+        
+        Returns:
+            the INSERT INTO SQL command for his entityInfo e.g.
+                 
+        Example:   
+      
+        .. code-block:: sql
+
+            INSERT INTO Person (name,born,numberInLine,wikidataurl,age,ofAge) values (?,?,?,?,?,?).
+
+        '''
+        columns =','.join(self.typeMap.keys())
+        placeholders=':'+',:'.join(self.typeMap.keys())
+        insertCmd="INSERT INTO %s (%s) values (%s)" % (self.name, columns,placeholders)
+        if self.debug:
+            print(insertCmd)
+        return insertCmd
+        
+    def addType(self,column,valueType,sqlType):
+        '''
+        add the python type for the given column to the typeMap
+        
+        Args:
+           column(string): the name of the column
+           
+           valueType(type): the python type of the column
+        '''
+        if not column in self.typeMap:
+            self.typeMap[column]=valueType     
+            self.sqlTypeMap[column]=sqlType          
+        
+    def fixDates(self,resultList):
+        '''
+        fix date entries in the given resultList by parsing the date content e.g.
+        converting '1926-04-21' back to datetime.date(1926, 4, 21)
+        
+        Args:
+            resultList(list): the list of records to be fixed
+        '''
+        for record in resultList:
+            for key,valueType in self.typeMap.items():
+                if valueType==datetime.date:
+                    dt=datetime.datetime.strptime(record[key],"%Y-%m-%d")  
+                    dateValue=dt.date()  
+                    record[key]=dateValue

storage/uml.py

@@ -0,0 +1,187 @@
+'''
+Created on 2020-09-04
+
+@author: wf
+'''
+from collections import Counter
+
+class UML(object):
+    '''
+    UML diagrams via plantuml
+    
+    '''
+
+    skinparams="""
+' BITPlan Corporate identity skin params
+' Copyright (c) 2015 BITPlan GmbH
+' see http://wiki.bitplan.com/PlantUmlSkinParams#BITPlanCI
+' skinparams generated by com.bitplan.restmodelmanager
+skinparam note {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam component {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam package {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam usecase {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam activity {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam classAttribute {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam interface {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam class {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam object {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+hide Circle
+' end of skinparams '
+"""
+
+    def __init__(self, debug=False):
+        '''
+        Constructor
+        Args:
+            debug(boolean): True if debug information should be shown
+        '''
+        self.debug=debug
+        
+    def generalizeColumn(self,tableList,colName):
+        ''' 
+        remove the column with the given name from all tables in the tablelist and
+        return it
+        
+        Args:
+            tableList(list): a list of Tables
+            colName(string): the name of the column to generalize
+        
+        Returns:
+            string: the column having been generalized and removed
+        '''
+        gCol=None
+        for table in tableList:
+            for col in table['columns']:
+                if col['name']==colName:
+                    gCol=col
+                    table['columns'].remove(col)
+        return gCol                
+        
+    def getGeneral(self,tableList,name):
+        '''
+        derive a general table from the given table list
+        Args:
+            tableList(list): a list of tables
+            name(string): name of the general table
+            
+        Returns:
+            at table dict for the generalized table
+        '''
+        general={'name':name,'columns':[]}
+        colCount=Counter()
+        for table in tableList:
+            for col in table['columns']:
+                columnId="%s.%s" % (col['name'],col['type'])
+                if self.debug:
+                    print (columnId) 
+                colCount[columnId]+=1
+        for columnId,count in colCount.items():
+            if count==len(tableList): 
+                colName=columnId.split('.')[0]
+                generalCol=self.generalizeColumn(tableList, colName)
+                general['columns'].append(generalCol)    
+        return general
+            
+    def tableListToPlantUml(self,tableList, packageName=None,generalizeTo=None,withSkin=True):
+        '''
+        convert tableList to PlantUml notation
+        
+        Args:
+            tableList(list): the tableList list of Dicts from getTableList() to convert
+            packageName(string): optional packageName to be added
+            generalizeTo(string): optional name of a general table to be derived
+            withSkin(boolean): if True add default BITPlan skin parameters 
+            
+        Returns:
+            string: the Plantuml notation for the entities in columns of the given tablelist
+        '''
+        uml=""
+        indent=""
+        inherit=""
+        if packageName is not None:
+            uml+="package %s {\n" % packageName
+            indent="  "
+        if generalizeTo is not None:
+            generalTable=self.getGeneral(tableList,generalizeTo)
+            for table in tableList:
+                inherit+="%s%s <|-- %s\n" % (indent,generalizeTo,table['name'])
+            tableList.insert(0,generalTable)
+        for table in tableList:
+            colUml=""
+            for col in table['columns']:
+                mandatory="*" if col['notnull']==1 else ""
+                pk="<<PK>>" if col['pk']==1 else ""
+                colUml+="%s  %s%s : %s %s\n" % (indent,mandatory,col['name'],col['type'],pk)
+            uml+="%sentity %s {\n%s%s}\n" % (indent,table['name'],colUml,indent)
+        uml+=inherit    
+        if packageName is not None:
+            uml+="}\n"
+        if withSkin:
+            uml+=UML.skinparams    
+        return uml
+    
+    
+