pyLodStorage 0.0.1__tar.gz

pylodstorage-0.0.1/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: pylodstorage
+Version: 0.0.1
+Summary: python List of Dict (Table) Storage library
+Home-page: https://github.com/WolfgangFahl/pyLodStorage
+Author: Wolfgang Fahl
+Author-email: wf@bitplan.com
+Maintainer: Wolfgang Fahl
+License: Apache License
+Description: # pyLoDStorage
+        python List of Dict (Table) Storage library
+        
+        [![pypi](https://img.shields.io/pypi/pyversions/pyLoDStorage)](https://pypi.org/project/pyLoDStorage/)
+        [![Github Actions Build](https://github.com/WolfgangFahl/pyLoDStorage/workflows/Build/badge.svg?branch=master)](https://github.com/WolfgangFahl/pyLoDStorage/actions?query=workflow%3ABuild+branch%3Amaster)
+        [![GitHub issues](https://img.shields.io/github/issues/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues)
+        [![GitHub closed issues](https://img.shields.io/github/issues-closed/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues/?q=is%3Aissue+is%3Aclosed)
+        [![License](https://img.shields.io/github/license/WolfgangFahl/pyLoDStorage.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+        
+        What it is
+        ==========
+        pyLoDStorage allows to store table like data (List of Dicts) via Sqlite3
+        
+        Installation
+        ============
+        ```bash
+        pip install pylodstorage
+        ```
+        
+        Usage
+        =====
+        see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tests)
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/markdown

pylodstorage-0.0.1/README.md

@@ -0,0 +1,22 @@
+# pyLoDStorage
+python List of Dict (Table) Storage library
+
+[![pypi](https://img.shields.io/pypi/pyversions/pyLoDStorage)](https://pypi.org/project/pyLoDStorage/)
+[![Github Actions Build](https://github.com/WolfgangFahl/pyLoDStorage/workflows/Build/badge.svg?branch=master)](https://github.com/WolfgangFahl/pyLoDStorage/actions?query=workflow%3ABuild+branch%3Amaster)
+[![GitHub issues](https://img.shields.io/github/issues/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues)
+[![GitHub closed issues](https://img.shields.io/github/issues-closed/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues/?q=is%3Aissue+is%3Aclosed)
+[![License](https://img.shields.io/github/license/WolfgangFahl/pyLoDStorage.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+
+What it is
+==========
+pyLoDStorage allows to store table like data (List of Dicts) via Sqlite3
+
+Installation
+============
+```bash
+pip install pylodstorage
+```
+
+Usage
+=====
+see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tests)

pylodstorage-0.0.1/pylodstorage.egg-info/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: pylodstorage
+Version: 0.0.1
+Summary: python List of Dict (Table) Storage library
+Home-page: https://github.com/WolfgangFahl/pyLodStorage
+Author: Wolfgang Fahl
+Author-email: wf@bitplan.com
+Maintainer: Wolfgang Fahl
+License: Apache License
+Description: # pyLoDStorage
+        python List of Dict (Table) Storage library
+        
+        [![pypi](https://img.shields.io/pypi/pyversions/pyLoDStorage)](https://pypi.org/project/pyLoDStorage/)
+        [![Github Actions Build](https://github.com/WolfgangFahl/pyLoDStorage/workflows/Build/badge.svg?branch=master)](https://github.com/WolfgangFahl/pyLoDStorage/actions?query=workflow%3ABuild+branch%3Amaster)
+        [![GitHub issues](https://img.shields.io/github/issues/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues)
+        [![GitHub closed issues](https://img.shields.io/github/issues-closed/WolfgangFahl/pyLoDStorage.svg)](https://github.com/WolfgangFahl/pyLoDStorage/issues/?q=is%3Aissue+is%3Aclosed)
+        [![License](https://img.shields.io/github/license/WolfgangFahl/pyLoDStorage.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+        
+        What it is
+        ==========
+        pyLoDStorage allows to store table like data (List of Dicts) via Sqlite3
+        
+        Installation
+        ============
+        ```bash
+        pip install pylodstorage
+        ```
+        
+        Usage
+        =====
+        see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tests)
+Platform: UNKNOWN
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Description-Content-Type: text/markdown

pylodstorage-0.0.1/pylodstorage.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+setup.py
+pylodstorage.egg-info/PKG-INFO
+pylodstorage.egg-info/SOURCES.txt
+pylodstorage.egg-info/dependency_links.txt
+pylodstorage.egg-info/top_level.txt
+storage/__init__.py
+storage/sample.py
+storage/sql.py
+storage/uml.py

pylodstorage-0.0.1/pylodstorage.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pylodstorage-0.0.1/pylodstorage.egg-info/top_level.txt

@@ -0,0 +1 @@
+storage

pylodstorage-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pylodstorage-0.0.1/setup.py

@@ -0,0 +1,29 @@
+# ! important
+# see https://stackoverflow.com/a/27868004/1497139
+from setuptools import setup
+
+# read the contents of your README file
+from os import path
+this_directory = path.abspath(path.dirname(__file__))
+with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+setup(
+    name='pylodstorage',
+    version='0.0.1',
+
+    packages=['storage',],
+    author='Wolfgang Fahl',
+    author_email='wf@bitplan.com',
+    maintainer='Wolfgang Fahl',
+    url='https://github.com/WolfgangFahl/pyLodStorage',
+    license='Apache License',
+    description='python List of Dict (Table) Storage library',
+    classifiers=[
+            'Programming Language :: Python',
+            'Programming Language :: Python :: 3.7',
+            'Programming Language :: Python :: 3.8'
+    ],
+    long_description=long_description,
+    long_description_content_type='text/markdown'
+)

pylodstorage-0.0.1/storage/sample.py

@@ -0,0 +1,71 @@
+'''
+Created on 2020-08-24
+
+@author: wf
+'''
+import urllib.request
+import json
+from datetime import date, datetime
+
+class Sample(object):
+    '''
+    Sample dataset generator
+    '''
+
+    def __init__(self):
+        '''
+        Constructor
+        '''
+        
+    @staticmethod         
+    def getSample(size):         
+        listOfDicts=[]
+        for index in range(size):
+            listOfDicts.append({'pkey': "index%d" %index, 'cindex': index})
+        return listOfDicts    
+    
+    @staticmethod
+    def getCountries():
+        countryJsonUrl="https://gist.githubusercontent.com/erdem/8c7d26765831d0f9a8c62f02782ae00d/raw/248037cd701af0a4957cce340dabb0fd04e38f4c/countries.json"
+        with urllib.request.urlopen(countryJsonUrl) as url:
+            countryList=json.loads(url.read().decode())
+        return countryList
+    
+    @staticmethod
+    def getCities():
+        '''
+        get a list of cities
+        '''
+        cityJsonUrl="https://raw.githubusercontent.com/lutangar/cities.json/master/cities.json"
+        with urllib.request.urlopen(cityJsonUrl) as url:
+            cityList=json.loads(url.read().decode())
+        for city in cityList:
+            city['cityId']="%s-%s" % (city['country'],city['name'])    
+        return cityList
+    
+    @staticmethod
+    def dob(isoDateString):
+        ''' get the date of birth from the given iso date state'''
+        #if sys.version_info >= (3, 7):
+        #    dt=datetime.fromisoformat(isoDateString)
+        #else:
+        dt=datetime.strptime(isoDateString,"%Y-%m-%d")  
+        return dt.date()   
+    
+    @staticmethod
+    def getRoyals():
+        listOfDicts=[
+            {'name': 'Elizabeth Alexandra Mary Windsor', 'born': Sample.dob('1926-04-21'), 'numberInLine': 0, 'wikidataurl': 'https://www.wikidata.org/wiki/Q9682' },
+            {'name': 'Charles, Prince of Wales',         'born': Sample.dob('1948-11-14'), 'numberInLine': 1, 'wikidataurl': 'https://www.wikidata.org/wiki/Q43274' },
+            {'name': 'George of Cambridge',              'born': Sample.dob('2013-07-22'), 'numberInLine': 3, 'wikidataurl': 'https://www.wikidata.org/wiki/Q1359041'},
+            {'name': 'Harry Duke of Sussex',             'born': Sample.dob('1984-09-15'), 'numberInLine': 6, 'wikidataurl': 'https://www.wikidata.org/wiki/Q152316'}
+        ]
+        today=date.today()
+        for person in listOfDicts:
+            born=person['born']
+            age=(today - born).days / 365.2425
+            person['age']=age
+            person['ofAge']=age>=18
+            person['lastmodified']=datetime.now()
+        return listOfDicts
+        

pylodstorage-0.0.1/storage/sql.py

@@ -0,0 +1,441 @@
+'''
+Created on 2020-08-24
+
+@author: wf
+'''
+# python standard library
+import sqlite3
+import datetime
+import io
+import time
+import sys
+import re
+
+
+class SQLDB(object):
+    '''
+    Structured Query Language Database wrapper
+    
+    :ivar dbname(string): name of the database
+    :ivar debug(boolean): True if debug info should be provided
+    :ivar errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
+    '''
+    RAM=":memory:"
+
+    def __init__(self,dbname=':memory:',connection=None,debug=False, errorDebug=False):
+        '''
+        Construct me for the given dbname and debug
+        
+        Args:
+        
+           dbname(string): name of the database - default is a RAM based database
+           connection(Connection): an optional connectio to be reused 
+           debug(boolean): if True switch on debug
+           errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
+        '''
+        self.dbname=dbname
+        self.debug=debug
+        self.errorDebug=errorDebug
+        if connection is None:
+            self.c=sqlite3.connect(dbname,detect_types=sqlite3.PARSE_DECLTYPES)
+        else:
+            self.c=connection
+        
+    def close(self):
+        ''' close my connection '''
+        self.c.close()    
+        
+    def createTable(self,listOfRecords,entityName,primaryKey=None,withDrop=False,sampleRecordCount=1):
+        '''
+        derive  Data Definition Language CREATE TABLE command from list of Records by examining first recorda
+        as defining sample record and execute DDL command
+        
+        auto detect column types see e.g. https://stackoverflow.com/a/57072280/1497139
+        
+        Args:
+           listOfRecords(list): a list of Dicts
+           entityName(string): the entity / table name to use
+           primaryKey(string): the key/column to use as a  primary key
+           withDrop(boolean): true if the existing Table should be dropped
+           sampleRecords(int): number of sampleRecords expected and to be inspected
+           
+        Returns:
+           EntityInfo: meta data information for the created table
+        '''
+        if len(listOfRecords)<sampleRecordCount:
+            raise Exception("Need %d sample records to createTable" % (sampleRecordCount))
+        sampleRecords=listOfRecords[:sampleRecordCount]
+        entityInfo=EntityInfo(sampleRecords,entityName,primaryKey,debug=self.debug)
+        if withDrop:
+            self.c.execute(entityInfo.dropTableCmd) 
+        self.c.execute(entityInfo.createTableCmd)
+        return entityInfo
+    
+    def getDebugInfo(self,record,index,executeMany):
+        '''
+        get the debug info for the given record at the given index depending on the state of executeMany
+        
+        Args:
+            record(dict): the record to show
+            index(int): the index of the record
+            executeMany(boolean): if True the record may be valid else not
+        '''
+        debugInfo=""
+        if not executeMany:
+            if self.errorDebug:
+                debugInfo="\nrecord  #%d=%s" % (index,repr(record))
+        return debugInfo
+        
+       
+    def store(self,listOfRecords,entityInfo,executeMany=False):
+        '''
+        store the given list of records based on the given entityInfo
+        
+        Args:
+          
+           listOfRecords(list): the list of Dicts to be stored
+           entityInfo(EntityInfo): the meta data to be used for storing
+        '''
+        insertCmd=entityInfo.insertCmd
+        record=None
+        index=0
+        try:
+            if executeMany:
+                self.c.executemany(insertCmd,listOfRecords)
+            else:
+                for record in listOfRecords:
+                    index+=1
+                    self.c.execute(insertCmd,record)
+            self.c.commit()
+        except sqlite3.ProgrammingError as pe:
+            msg=pe.args[0]
+            if "You did not supply a value for binding" in msg:
+                columnIndex=int(re.findall(r'\d+',msg)[0])
+                columnName=list(entityInfo.typeMap.keys())[columnIndex-1]
+                debugInfo=self.getDebugInfo(record, index, executeMany)
+                raise Exception("%s\nfailed: no value supplied for column '%s'%s" % (insertCmd,columnName,debugInfo))
+            else:
+                raise pe
+        except sqlite3.InterfaceError as ie:
+            msg=ie.args[0]
+            if "Error binding parameter" in msg:
+                columnName=re.findall(r':[_a-zA-Z]\w*',msg)[0]
+                debugInfo=self.getDebugInfo(record, index, executeMany)
+                raise Exception("%s\nfailed: error binding column '%s'%s" % (insertCmd,columnName,debugInfo))
+            else:
+                raise ie
+        
+    def query(self,sqlQuery):
+        '''
+        run the given sqlQuery and return a list of Dicts
+        
+        Args:
+            
+            sqlQuery(string): the SQL query to be executed
+                
+        Returns:
+            list: a list of Dicts
+        '''
+        if self.debug:
+            print(sqlQuery)
+        # https://stackoverflow.com/a/13735506/1497139
+        cur=self.c.cursor()
+        query = cur.execute(sqlQuery)
+        colname = [ d[0] for d in query.description ]
+        resultList=[]
+        for row in query:
+            record=dict(zip(colname, row))
+            resultList.append(record)
+        cur.close()
+        return resultList
+        
+    def queryAll(self,entityInfo,fixDates=True):
+        '''
+        query all records for the given entityName/tableName
+        
+        Args:
+           entityName(string): name of the entity/table to qury
+           fixDates(boolean): True if date entries should be returned as such and not as strings
+        '''  
+        sqlQuery='SELECT * FROM %s' % entityInfo.name
+        resultList=self.query(sqlQuery)
+        if fixDates:
+            entityInfo.fixDates(resultList)
+        return resultList
+    
+    def getTableList(self):
+        '''
+        get the schema information from this database
+        '''
+        tableQuery="SELECT name FROM sqlite_master WHERE type='table'"
+        tableList=self.query(tableQuery)
+        for table in tableList:
+            tableName=table['name']
+            columnQuery="PRAGMA table_info('%s')" % tableName
+            columns=self.query(columnQuery)
+            table['columns']=columns
+        return tableList
+              
+    def progress(self,status, remaining, total):
+        '''
+        show progress
+        '''
+        print('Backup %s at %5.0f%%' % ("... " if status==0 else "done",(total-remaining)/total*100)) 
+    
+    def backup(self,backupDB,action="Backup",profile=False,showProgress=200,doClose=True):
+        '''
+        create backup of this SQLDB to the given backup db
+        
+        see https://stackoverflow.com/a/59042442/1497139
+        
+        Args:
+            backupDB(string): the path to the backupdb or SQLDB.RAM for in memory
+            action(string): the action to display
+            profile(boolean): True if timing information shall be shown
+            showProgress(int): show progress at each showProgress page (0=show no progress)
+        '''
+        if sys.version_info <= (3, 6):
+            raise Exception("backup via stdlibrary not available in python <=3.6 use copyToDB instead")
+        startTime=time.time()
+        bck=sqlite3.connect(backupDB)
+        if showProgress>0:
+            progress=self.progress
+        else:
+            progress=None
+        with bck:
+            self.c.backup(bck,pages=showProgress,progress=progress)        
+        elapsed=time.time()-startTime
+        if profile:
+            print("%s to %s took %5.1f s" % (action,backupDB,elapsed))
+        if doClose:
+            bck.close()
+            return None
+        else:
+            return bck
+        
+    def showDump(self,dump,limit=10):
+        '''
+        show the given dump up to the given limit
+        
+        Args:
+            dump(string): the SQL dump to show
+            limit(int): the maximum number of lines to display
+        '''
+        s=io.StringIO(dump)
+        index=0
+        for line in s:
+            if index <= limit:
+                print(line)
+                index+=1    
+            else:
+                break        
+        
+    def executeDump(self,connection,dump,title,maxErrors=100,errorDisplayLimit=12,profile=True):
+        '''
+        execute the given dump for the given connection
+        
+        Args:
+            connection(Connection): the sqlite3 connection to use
+            dump(string): the SQL commands for the dump
+            title(string): the title of the dump
+            maxErrors(int): maximum number of errors to be tolerated before stopping and doing a rollback
+            profile(boolean): True if profiling information should be shown
+        Returns:
+            a list of errors
+        '''
+        if self.debug:
+            self.showDump(dump)
+        startTime=time.time()    
+        if profile:
+            print("dump of %s has size %4.1f MB" % (title,len(dump)/1024/1024))
+        errors=[]
+        index=0
+        # fixes https://github.com/WolfgangFahl/ProceedingsTitleParser/issues/37
+        for line in dump.split(";\n"):
+            try:
+                connection.execute(line)
+            except  sqlite3.OperationalError as soe:
+                msg="SQL error %s in line %d:\n\t%s" % (soe,index,line)
+                errors.append(msg)
+                if len(errors)<=errorDisplayLimit:
+                    print(msg)    
+                if len(errors)>=maxErrors:
+                    connection.execute("ROLLBACK;")
+                    break
+                
+            index=index+1
+        if profile:    
+            print("finished executing dump %s with %d lines and %d errors in %5.1f s" % (title,index,len(errors),time.time()-startTime))    
+        return errors
+    
+    def copyTo(self,copyDB,profile=True):
+        '''
+        copy my content to another database
+        
+        Args:
+            
+           copyDB(Connection): the target database
+           profile(boolean): if True show profile information
+        '''
+        startTime=time.time()
+        dump="\n".join(self.c.iterdump())
+        #cursor.executescript(dump)
+        if profile:
+            print("finished getting dump of %s in %5.1f s" % (self.dbname,time.time()-startTime))
+        dumpErrors=self.executeDump(copyDB.c,dump,self.dbname,profile=profile)
+        return dumpErrors
+       
+        
+    @staticmethod
+    def restore(backupDB,restoreDB,profile=False,showProgress=200,debug=False):
+        '''
+        restore the restoreDB from the given backup DB
+        
+        Args:
+            backupDB(string): path to the backupDB e.g. backup.db
+            restoreDB(string): path to the restoreDB or in Memory SQLDB.RAM
+            profile(boolean): True if timing information should be shown
+            showProgress(int): show progress at each showProgress page (0=show no progress)
+        '''
+        backupSQLDB=SQLDB(backupDB)
+        connection=backupSQLDB.backup(restoreDB,action="Restore",profile=profile,showProgress=showProgress,doClose=False)
+        restoreSQLDB=SQLDB(restoreDB,connection=connection,debug=debug)
+        return restoreSQLDB
+    
+        
+class EntityInfo(object):
+    """
+    holds entity meta Info 
+    
+    :ivar name(string): entity name = table name
+    
+    :ivar primaryKey(string): the name of the primary key column
+    
+    :ivar typeMap(dict): maps column names to python types
+    
+    :ivar debug(boolean): True if debug information should be shown
+    
+    """
+        
+    def __init__(self,sampleRecords,name,primaryKey=None,debug=False):
+        '''
+        construct me from the given name and primary key
+        
+        Args:
+           name(string): the name of the entity
+           primaryKey(string): the name of the primary key column
+           debug(boolean): True if debug information should be shown
+        '''
+        self.sampleRecords=sampleRecords
+        self.name=name
+        self.primaryKey=primaryKey
+        self.debug=debug
+        self.typeMap={}
+        self.sqlTypeMap={}
+        self.createTableCmd=self.getCreateTableCmd(sampleRecords)
+        self.dropTableCmd="DROP TABLE IF EXISTS %s" % self.name
+        self.insertCmd=self.getInsertCmd()
+        
+    def getCreateTableCmd(self,sampleRecords):
+        '''
+        get the CREATE TABLE DDL command for the given sample records
+        
+        Args:
+            sampleRecords(list): a list of Dicts of sample Records    
+            
+        Returns:
+            string: CREATE TABLE DDL command for this entity info 
+            
+        Example:   
+      
+        .. code-block:: sql
+            
+            CREATE TABLE Person(name TEXT PRIMARY KEY,born DATE,numberInLine INTEGER,wikidataurl TEXT,age FLOAT,ofAge BOOLEAN)
+    
+        '''
+        ddlCmd="CREATE TABLE %s(" %self.name
+        delim=""
+        for sampleRecord in sampleRecords:
+            for key,value in sampleRecord.items():
+                sqlType=None
+                valueType=None
+                if value is None:
+                    if len(sampleRecords)==1:
+                        print("Warning sampleRecord column %s is None - using TEXT as type" % key)
+                        valueType=str
+                else:
+                    valueType=type(value)
+                if valueType == str:
+                    sqlType="TEXT"
+                elif valueType == int:
+                    sqlType="INTEGER"
+                elif valueType == float:
+                    sqlType="FLOAT"
+                elif valueType == bool:
+                    sqlType="BOOLEAN"      
+                elif valueType == datetime.date:
+                    sqlType="DATE"    
+                elif valueType== datetime.datetime:
+                    sqlType="TIMESTAMP"
+                else:
+                    if valueType is not None:
+                        msg="warning: unsupported type %s for column %s " % (str(valueType),key)
+                        print(msg)
+                if sqlType is not None and valueType is not None:
+                    self.addType(key,valueType,sqlType)
+        for key,sqlType in self.sqlTypeMap.items():        
+            ddlCmd+="%s%s %s%s" % (delim,key,sqlType," PRIMARY KEY" if key==self.primaryKey else "")
+            delim=","
+        ddlCmd+=")"  
+        if self.debug:
+            print (ddlCmd)    
+        return ddlCmd
+        
+    def getInsertCmd(self):
+        '''
+        get the INSERT command for this entityInfo
+        
+        Returns:
+            the INSERT INTO SQL command for his entityInfo e.g.
+                 
+        Example:   
+      
+        .. code-block:: sql
+
+            INSERT INTO Person (name,born,numberInLine,wikidataurl,age,ofAge) values (?,?,?,?,?,?).
+
+        '''
+        columns =','.join(self.typeMap.keys())
+        placeholders=':'+',:'.join(self.typeMap.keys())
+        insertCmd="INSERT INTO %s (%s) values (%s)" % (self.name, columns,placeholders)
+        if self.debug:
+            print(insertCmd)
+        return insertCmd
+        
+    def addType(self,column,valueType,sqlType):
+        '''
+        add the python type for the given column to the typeMap
+        
+        Args:
+           column(string): the name of the column
+           
+           valueType(type): the python type of the column
+        '''
+        if not column in self.typeMap:
+            self.typeMap[column]=valueType     
+            self.sqlTypeMap[column]=sqlType          
+        
+    def fixDates(self,resultList):
+        '''
+        fix date entries in the given resultList by parsing the date content e.g.
+        converting '1926-04-21' back to datetime.date(1926, 4, 21)
+        
+        Args:
+            resultList(list): the list of records to be fixed
+        '''
+        for record in resultList:
+            for key,valueType in self.typeMap.items():
+                if valueType==datetime.date:
+                    dt=datetime.datetime.strptime(record[key],"%Y-%m-%d")  
+                    dateValue=dt.date()  
+                    record[key]=dateValue

pylodstorage-0.0.1/storage/uml.py

@@ -0,0 +1,187 @@
+'''
+Created on 2020-09-04
+
+@author: wf
+'''
+from collections import Counter
+
+class UML(object):
+    '''
+    UML diagrams via plantuml
+    
+    '''
+
+    skinparams="""
+' BITPlan Corporate identity skin params
+' Copyright (c) 2015 BITPlan GmbH
+' see http://wiki.bitplan.com/PlantUmlSkinParams#BITPlanCI
+' skinparams generated by com.bitplan.restmodelmanager
+skinparam note {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam component {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam package {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam usecase {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam activity {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam classAttribute {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam interface {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam class {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+skinparam object {
+  BackGroundColor #FFFFFF
+  FontSize 12
+  ArrowColor #FF8000
+  BorderColor #FF8000
+  FontColor black
+  FontName Technical
+}
+hide Circle
+' end of skinparams '
+"""
+
+    def __init__(self, debug=False):
+        '''
+        Constructor
+        Args:
+            debug(boolean): True if debug information should be shown
+        '''
+        self.debug=debug
+        
+    def generalizeColumn(self,tableList,colName):
+        ''' 
+        remove the column with the given name from all tables in the tablelist and
+        return it
+        
+        Args:
+            tableList(list): a list of Tables
+            colName(string): the name of the column to generalize
+        
+        Returns:
+            string: the column having been generalized and removed
+        '''
+        gCol=None
+        for table in tableList:
+            for col in table['columns']:
+                if col['name']==colName:
+                    gCol=col
+                    table['columns'].remove(col)
+        return gCol                
+        
+    def getGeneral(self,tableList,name):
+        '''
+        derive a general table from the given table list
+        Args:
+            tableList(list): a list of tables
+            name(string): name of the general table
+            
+        Returns:
+            at table dict for the generalized table
+        '''
+        general={'name':name,'columns':[]}
+        colCount=Counter()
+        for table in tableList:
+            for col in table['columns']:
+                columnId="%s.%s" % (col['name'],col['type'])
+                if self.debug:
+                    print (columnId) 
+                colCount[columnId]+=1
+        for columnId,count in colCount.items():
+            if count==len(tableList): 
+                colName=columnId.split('.')[0]
+                generalCol=self.generalizeColumn(tableList, colName)
+                general['columns'].append(generalCol)    
+        return general
+            
+    def tableListToPlantUml(self,tableList, packageName=None,generalizeTo=None,withSkin=True):
+        '''
+        convert tableList to PlantUml notation
+        
+        Args:
+            tableList(list): the tableList list of Dicts from getTableList() to convert
+            packageName(string): optional packageName to be added
+            generalizeTo(string): optional name of a general table to be derived
+            withSkin(boolean): if True add default BITPlan skin parameters 
+            
+        Returns:
+            string: the Plantuml notation for the entities in columns of the given tablelist
+        '''
+        uml=""
+        indent=""
+        inherit=""
+        if packageName is not None:
+            uml+="package %s {\n" % packageName
+            indent="  "
+        if generalizeTo is not None:
+            generalTable=self.getGeneral(tableList,generalizeTo)
+            for table in tableList:
+                inherit+="%s%s <|-- %s\n" % (indent,generalizeTo,table['name'])
+            tableList.insert(0,generalTable)
+        for table in tableList:
+            colUml=""
+            for col in table['columns']:
+                mandatory="*" if col['notnull']==1 else ""
+                pk="<<PK>>" if col['pk']==1 else ""
+                colUml+="%s  %s%s : %s %s\n" % (indent,mandatory,col['name'],col['type'],pk)
+            uml+="%sentity %s {\n%s%s}\n" % (indent,table['name'],colUml,indent)
+        uml+=inherit    
+        if packageName is not None:
+            uml+="}\n"
+        if withSkin:
+            uml+=UML.skinparams    
+        return uml
+    
+    
+