pyLodStorage 0.0.30__tar.gz → 0.0.32__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/PKG-INFO +1 -1
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/jsonable.py +50 -1
- pylodstorage-0.0.32/lodstorage/lod.py +92 -0
- pylodstorage-0.0.32/lodstorage/plot.py +79 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sample.py +8 -6
- pylodstorage-0.0.32/lodstorage/tabulateCounter.py +33 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/PKG-INFO +1 -1
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/SOURCES.txt +2 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/setup.py +1 -1
- pylodstorage-0.0.30/lodstorage/lod.py +0 -52
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/LICENSE +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/README.md +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/__init__.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/mwTable.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/query.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/schema.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sparql.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sql.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/uml.py +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/dependency_links.txt +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/requires.txt +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/top_level.txt +0 -0
- {pylodstorage-0.0.30 → pylodstorage-0.0.32}/setup.cfg +0 -0
|
@@ -298,7 +298,7 @@ class JSONAbleList(JSONAble):
|
|
|
298
298
|
Container class
|
|
299
299
|
'''
|
|
300
300
|
|
|
301
|
-
def __init__(self,listName:str=None,clazz=None,tableName:str=None):
|
|
301
|
+
def __init__(self,listName:str=None,clazz=None,tableName:str=None,initList:bool=True):
|
|
302
302
|
'''
|
|
303
303
|
Constructor
|
|
304
304
|
|
|
@@ -306,6 +306,7 @@ class JSONAbleList(JSONAble):
|
|
|
306
306
|
listName(str): the name of the list attribute to be used for storing the List
|
|
307
307
|
clazz(class): a class to be used for Object relational mapping (if any)
|
|
308
308
|
tableName(str): the name of the "table" to be used
|
|
309
|
+
initList(bool): True if the list should be initialized
|
|
309
310
|
'''
|
|
310
311
|
self.clazz=clazz
|
|
311
312
|
if listName is None:
|
|
@@ -318,6 +319,54 @@ class JSONAbleList(JSONAble):
|
|
|
318
319
|
self.tableName=listName
|
|
319
320
|
else:
|
|
320
321
|
self.tableName=tableName
|
|
322
|
+
if initList:
|
|
323
|
+
self.__dict__[self.listName]=[]
|
|
324
|
+
|
|
325
|
+
def getList(self):
|
|
326
|
+
'''
|
|
327
|
+
get my list
|
|
328
|
+
'''
|
|
329
|
+
return self.__dict__[self.listName]
|
|
330
|
+
|
|
331
|
+
def getLookup(self,attrName:str,withDuplicates:bool=False):
|
|
332
|
+
'''
|
|
333
|
+
create a lookup dictionary by the given attribute name
|
|
334
|
+
|
|
335
|
+
Args:
|
|
336
|
+
attrName(str): the attribute to lookup
|
|
337
|
+
withDuplicates(bool): whether to retain single values or lists
|
|
338
|
+
|
|
339
|
+
Return:
|
|
340
|
+
a dictionary for lookup
|
|
341
|
+
'''
|
|
342
|
+
lookup={}
|
|
343
|
+
duplicates=[]
|
|
344
|
+
for entity in self.getList():
|
|
345
|
+
value=None
|
|
346
|
+
if isinstance(entity,dict):
|
|
347
|
+
if attrName in entity:
|
|
348
|
+
value=entity[attrName]
|
|
349
|
+
else:
|
|
350
|
+
if hasattr(entity, attrName):
|
|
351
|
+
value=getattr(entity,attrName)
|
|
352
|
+
if value is not None:
|
|
353
|
+
if value in lookup:
|
|
354
|
+
if withDuplicates:
|
|
355
|
+
lookupResult=lookup[value]
|
|
356
|
+
lookupResult.append(entity)
|
|
357
|
+
else:
|
|
358
|
+
duplicates.append(entity)
|
|
359
|
+
else:
|
|
360
|
+
if withDuplicates:
|
|
361
|
+
lookupResult=[entity]
|
|
362
|
+
else:
|
|
363
|
+
lookupResult=entity
|
|
364
|
+
lookup[value]=lookupResult
|
|
365
|
+
if withDuplicates:
|
|
366
|
+
return lookup
|
|
367
|
+
else:
|
|
368
|
+
return lookup,duplicates
|
|
369
|
+
|
|
321
370
|
|
|
322
371
|
def getJsonData(self):
|
|
323
372
|
'''
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2021-01-31
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
from lodstorage.jsonable import JSONAbleList
|
|
7
|
+
|
|
8
|
+
class LOD(object):
|
|
9
|
+
'''
|
|
10
|
+
list of Dict aka Table
|
|
11
|
+
'''
|
|
12
|
+
|
|
13
|
+
def __init__(self,name):
|
|
14
|
+
'''
|
|
15
|
+
Constructor
|
|
16
|
+
'''
|
|
17
|
+
self.name=name
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
def getFields(listOfDicts,sampleCount:int=None):
|
|
22
|
+
if sampleCount is None:
|
|
23
|
+
sampleCount=len(listOfDicts)
|
|
24
|
+
fields=[]
|
|
25
|
+
for row in listOfDicts:
|
|
26
|
+
for key in row.keys():
|
|
27
|
+
if not key in fields:
|
|
28
|
+
fields.append(key)
|
|
29
|
+
return fields
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
def setNone4List(listOfDicts,fields):
|
|
33
|
+
'''
|
|
34
|
+
set the given fields to None for the records in the given listOfDicts
|
|
35
|
+
if they are not set
|
|
36
|
+
Args:
|
|
37
|
+
listOfDicts(list): the list of records to work on
|
|
38
|
+
fields(list): the list of fields to set to None
|
|
39
|
+
'''
|
|
40
|
+
for record in listOfDicts:
|
|
41
|
+
LOD.setNone(record, fields)
|
|
42
|
+
|
|
43
|
+
@staticmethod
|
|
44
|
+
def setNone(record,fields):
|
|
45
|
+
'''
|
|
46
|
+
make sure the given fields in the given record are set to none
|
|
47
|
+
Args:
|
|
48
|
+
record(dict): the record to work on
|
|
49
|
+
fields(list): the list of fields to set to None
|
|
50
|
+
'''
|
|
51
|
+
for field in fields:
|
|
52
|
+
if not field in record:
|
|
53
|
+
record[field]=None
|
|
54
|
+
|
|
55
|
+
'''
|
|
56
|
+
https://stackoverflow.com/questions/33542997/python-intersection-of-2-lists-of-dictionaries/33543164
|
|
57
|
+
'''
|
|
58
|
+
@staticmethod
|
|
59
|
+
def sortKey(d,key=None):
|
|
60
|
+
''' get the sort key for the given dict d with the given key
|
|
61
|
+
'''
|
|
62
|
+
if key is None:
|
|
63
|
+
# https://stackoverflow.com/a/60765557/1497139
|
|
64
|
+
return hash(tuple(d.items()))
|
|
65
|
+
else:
|
|
66
|
+
return d[key]
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def intersect(listOfDict1,listOfDict2,key=None):
|
|
70
|
+
'''
|
|
71
|
+
get the intersection of the two lists of Dicts by the given key
|
|
72
|
+
'''
|
|
73
|
+
i1=iter(sorted(listOfDict1, key=lambda k: LOD.sortKey(k, key)))
|
|
74
|
+
i2=iter(sorted(listOfDict2, key=lambda k: LOD.sortKey(k, key)))
|
|
75
|
+
c1=next(i1)
|
|
76
|
+
c2=next(i2)
|
|
77
|
+
lr=[]
|
|
78
|
+
while True:
|
|
79
|
+
try:
|
|
80
|
+
val1=LOD.sortKey(c1,key)
|
|
81
|
+
val2=LOD.sortKey(c2,key)
|
|
82
|
+
if val1<val2:
|
|
83
|
+
c1=next(i1)
|
|
84
|
+
elif val1>val2:
|
|
85
|
+
c2=next(i2)
|
|
86
|
+
else:
|
|
87
|
+
lr.append(c1)
|
|
88
|
+
c1=next(i1)
|
|
89
|
+
c2=next(i2)
|
|
90
|
+
except StopIteration:
|
|
91
|
+
break
|
|
92
|
+
return lr
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2020-07-05
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
import matplotlib.pyplot as plt
|
|
7
|
+
from collections import Counter
|
|
8
|
+
import numpy as np
|
|
9
|
+
import os
|
|
10
|
+
|
|
11
|
+
class Plot(object):
|
|
12
|
+
'''
|
|
13
|
+
create Plot based on counters
|
|
14
|
+
see https://stackoverflow.com/questions/19198920/using-counter-in-python-to-build-histogram
|
|
15
|
+
'''
|
|
16
|
+
def __init__(self, valueList,title,xlabel=None,ylabel=None,gformat='.png',fontsize=12,plotdir=None,debug=False):
|
|
17
|
+
'''
|
|
18
|
+
Constructor
|
|
19
|
+
'''
|
|
20
|
+
self.counter=Counter(valueList)
|
|
21
|
+
self.valueList=valueList
|
|
22
|
+
self.title=title
|
|
23
|
+
self.xlabel=xlabel
|
|
24
|
+
self.ylabel=ylabel
|
|
25
|
+
self.fontsize=fontsize
|
|
26
|
+
self.gformat=gformat
|
|
27
|
+
self.debug=debug
|
|
28
|
+
path=os.path.dirname(__file__)
|
|
29
|
+
if plotdir is not None:
|
|
30
|
+
self.plotdir=plotdir
|
|
31
|
+
else:
|
|
32
|
+
self.plotdir=path+"/../plots/"
|
|
33
|
+
os.makedirs(self.plotdir,exist_ok=True)
|
|
34
|
+
|
|
35
|
+
def titleMe(self):
|
|
36
|
+
''' set my title and labels '''
|
|
37
|
+
plt.title(self.title, fontsize=self.fontsize)
|
|
38
|
+
if self.xlabel is not None:
|
|
39
|
+
plt.xlabel(self.xlabel)
|
|
40
|
+
if self.ylabel is not None:
|
|
41
|
+
plt.ylabel(self.ylabel)
|
|
42
|
+
|
|
43
|
+
def showMe(self,mode='show',close=True):
|
|
44
|
+
''' show me in the given mode '''
|
|
45
|
+
if mode=="show":
|
|
46
|
+
plt.show()
|
|
47
|
+
else:
|
|
48
|
+
plt.savefig(self.plotdir+self.title+self.gformat)
|
|
49
|
+
if close:
|
|
50
|
+
plt.close()
|
|
51
|
+
|
|
52
|
+
def barchart(self,mode='show'):
|
|
53
|
+
''' barchart based histogram for the given counter '''
|
|
54
|
+
labels, values = zip(*self.counter.items())
|
|
55
|
+
indexes = np.arange(len(labels))
|
|
56
|
+
width = 1
|
|
57
|
+
self.titleMe()
|
|
58
|
+
plt.bar(indexes, values, width)
|
|
59
|
+
plt.xticks(indexes + width * 0.5, labels)
|
|
60
|
+
plt.yticks(np.arange(1,max(values)+1,step=1))
|
|
61
|
+
self.showMe(mode)
|
|
62
|
+
|
|
63
|
+
def showDebug(self):
|
|
64
|
+
print(" value list: ",self.valueList)
|
|
65
|
+
print("counter items: ",self.counter.items())
|
|
66
|
+
print("counter values: ",self.counter.values())
|
|
67
|
+
print("counter keys: ",self.counter.keys())
|
|
68
|
+
|
|
69
|
+
def hist(self,mode="show"):
|
|
70
|
+
''' create histogram for the given counter '''
|
|
71
|
+
if self.debug:
|
|
72
|
+
self.showDebug()
|
|
73
|
+
self.titleMe()
|
|
74
|
+
# see https://stackoverflow.com/a/2162045/1497139
|
|
75
|
+
plt.hist(self.valueList,bins=len(self.counter.keys()))
|
|
76
|
+
self.showMe(mode)
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
|
|
@@ -12,6 +12,7 @@ class Sample(object):
|
|
|
12
12
|
'''
|
|
13
13
|
Sample dataset generator
|
|
14
14
|
'''
|
|
15
|
+
cityList=None
|
|
15
16
|
|
|
16
17
|
def __init__(self):
|
|
17
18
|
'''
|
|
@@ -37,12 +38,13 @@ class Sample(object):
|
|
|
37
38
|
'''
|
|
38
39
|
get a list of cities
|
|
39
40
|
'''
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
city
|
|
45
|
-
|
|
41
|
+
if Sample.cityList is None:
|
|
42
|
+
cityJsonUrl="https://raw.githubusercontent.com/lutangar/cities.json/master/cities.json"
|
|
43
|
+
with urllib.request.urlopen(cityJsonUrl) as url:
|
|
44
|
+
Sample.cityList=json.loads(url.read().decode())
|
|
45
|
+
for city in Sample.cityList:
|
|
46
|
+
city['cityId']="%s-%s" % (city['country'],city['name'])
|
|
47
|
+
return Sample.cityList
|
|
46
48
|
|
|
47
49
|
@staticmethod
|
|
48
50
|
def dob(isoDateString):
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2021-06-13
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
from tabulate import tabulate
|
|
7
|
+
|
|
8
|
+
class TabulateCounter(object):
|
|
9
|
+
'''
|
|
10
|
+
helper for tabulating Counters
|
|
11
|
+
'''
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def __init__(self, counter):
|
|
15
|
+
'''
|
|
16
|
+
Constructor
|
|
17
|
+
'''
|
|
18
|
+
self.counter=counter
|
|
19
|
+
|
|
20
|
+
def mostCommonTable(self,headers=["#","key","count","%"],tablefmt='pretty',limit=50):
|
|
21
|
+
'''
|
|
22
|
+
get the most common Table
|
|
23
|
+
'''
|
|
24
|
+
bins=len(self.counter.keys())
|
|
25
|
+
limit=min(bins,limit)
|
|
26
|
+
total=sum(self.counter.values())
|
|
27
|
+
binTable=[("total",bins,total)]
|
|
28
|
+
for i,bintuple in enumerate(self.counter.most_common(limit)):
|
|
29
|
+
key,count=bintuple
|
|
30
|
+
binTable.append((i+1,key,count,count/total*100.0))
|
|
31
|
+
|
|
32
|
+
table=tabulate(binTable,headers=headers,tablefmt=tablefmt,floatfmt=".2f")
|
|
33
|
+
return table
|
|
@@ -5,11 +5,13 @@ lodstorage/__init__.py
|
|
|
5
5
|
lodstorage/jsonable.py
|
|
6
6
|
lodstorage/lod.py
|
|
7
7
|
lodstorage/mwTable.py
|
|
8
|
+
lodstorage/plot.py
|
|
8
9
|
lodstorage/query.py
|
|
9
10
|
lodstorage/sample.py
|
|
10
11
|
lodstorage/schema.py
|
|
11
12
|
lodstorage/sparql.py
|
|
12
13
|
lodstorage/sql.py
|
|
14
|
+
lodstorage/tabulateCounter.py
|
|
13
15
|
lodstorage/uml.py
|
|
14
16
|
pylodstorage.egg-info/PKG-INFO
|
|
15
17
|
pylodstorage.egg-info/SOURCES.txt
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
'''
|
|
2
|
-
Created on 2021-01-31
|
|
3
|
-
|
|
4
|
-
@author: wf
|
|
5
|
-
'''
|
|
6
|
-
|
|
7
|
-
class LOD(object):
|
|
8
|
-
'''
|
|
9
|
-
list of Dict aka Table
|
|
10
|
-
'''
|
|
11
|
-
|
|
12
|
-
def __init__(self,name):
|
|
13
|
-
'''
|
|
14
|
-
Constructor
|
|
15
|
-
'''
|
|
16
|
-
self.name=name
|
|
17
|
-
pass
|
|
18
|
-
|
|
19
|
-
@staticmethod
|
|
20
|
-
def getFields(listOfDicts,sampleCount:int=None):
|
|
21
|
-
if sampleCount is None:
|
|
22
|
-
sampleCount=len(listOfDicts)
|
|
23
|
-
fields=[]
|
|
24
|
-
for row in listOfDicts:
|
|
25
|
-
for key in row.keys():
|
|
26
|
-
if not key in fields:
|
|
27
|
-
fields.append(key)
|
|
28
|
-
return fields
|
|
29
|
-
|
|
30
|
-
@staticmethod
|
|
31
|
-
def setNone4List(listOfDicts,fields):
|
|
32
|
-
'''
|
|
33
|
-
set the given fields to None for the records in the given listOfDicts
|
|
34
|
-
if they are not set
|
|
35
|
-
Args:
|
|
36
|
-
listOfDicts(list): the list of records to work on
|
|
37
|
-
fields(list): the list of fields to set to None
|
|
38
|
-
'''
|
|
39
|
-
for record in listOfDicts:
|
|
40
|
-
LOD.setNone(record, fields)
|
|
41
|
-
|
|
42
|
-
@staticmethod
|
|
43
|
-
def setNone(record,fields):
|
|
44
|
-
'''
|
|
45
|
-
make sure the given fields in the given record are set to none
|
|
46
|
-
Args:
|
|
47
|
-
record(dict): the record to work on
|
|
48
|
-
fields(list): the list of fields to set to None
|
|
49
|
-
'''
|
|
50
|
-
for field in fields:
|
|
51
|
-
if not field in record:
|
|
52
|
-
record[field]=None
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|