pyLodStorage 0.0.30__tar.gz → 0.0.32__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/PKG-INFO +1 -1
  2. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/jsonable.py +50 -1
  3. pylodstorage-0.0.32/lodstorage/lod.py +92 -0
  4. pylodstorage-0.0.32/lodstorage/plot.py +79 -0
  5. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sample.py +8 -6
  6. pylodstorage-0.0.32/lodstorage/tabulateCounter.py +33 -0
  7. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/PKG-INFO +1 -1
  8. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/SOURCES.txt +2 -0
  9. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/setup.py +1 -1
  10. pylodstorage-0.0.30/lodstorage/lod.py +0 -52
  11. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/LICENSE +0 -0
  12. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/README.md +0 -0
  13. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/__init__.py +0 -0
  14. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/mwTable.py +0 -0
  15. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/query.py +0 -0
  16. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/schema.py +0 -0
  17. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sparql.py +0 -0
  18. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/sql.py +0 -0
  19. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/lodstorage/uml.py +0 -0
  20. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/dependency_links.txt +0 -0
  21. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/requires.txt +0 -0
  22. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/pylodstorage.egg-info/top_level.txt +0 -0
  23. {pylodstorage-0.0.30 → pylodstorage-0.0.32}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pylodstorage
3
- Version: 0.0.30
3
+ Version: 0.0.32
4
4
  Summary: python List of Dict (Table) Storage library
5
5
  Home-page: https://github.com/WolfgangFahl/pyLodStorage
6
6
  Author: Wolfgang Fahl
@@ -298,7 +298,7 @@ class JSONAbleList(JSONAble):
298
298
  Container class
299
299
  '''
300
300
 
301
- def __init__(self,listName:str=None,clazz=None,tableName:str=None):
301
+ def __init__(self,listName:str=None,clazz=None,tableName:str=None,initList:bool=True):
302
302
  '''
303
303
  Constructor
304
304
 
@@ -306,6 +306,7 @@ class JSONAbleList(JSONAble):
306
306
  listName(str): the name of the list attribute to be used for storing the List
307
307
  clazz(class): a class to be used for Object relational mapping (if any)
308
308
  tableName(str): the name of the "table" to be used
309
+ initList(bool): True if the list should be initialized
309
310
  '''
310
311
  self.clazz=clazz
311
312
  if listName is None:
@@ -318,6 +319,54 @@ class JSONAbleList(JSONAble):
318
319
  self.tableName=listName
319
320
  else:
320
321
  self.tableName=tableName
322
+ if initList:
323
+ self.__dict__[self.listName]=[]
324
+
325
+ def getList(self):
326
+ '''
327
+ get my list
328
+ '''
329
+ return self.__dict__[self.listName]
330
+
331
+ def getLookup(self,attrName:str,withDuplicates:bool=False):
332
+ '''
333
+ create a lookup dictionary by the given attribute name
334
+
335
+ Args:
336
+ attrName(str): the attribute to lookup
337
+ withDuplicates(bool): whether to retain single values or lists
338
+
339
+ Return:
340
+ a dictionary for lookup
341
+ '''
342
+ lookup={}
343
+ duplicates=[]
344
+ for entity in self.getList():
345
+ value=None
346
+ if isinstance(entity,dict):
347
+ if attrName in entity:
348
+ value=entity[attrName]
349
+ else:
350
+ if hasattr(entity, attrName):
351
+ value=getattr(entity,attrName)
352
+ if value is not None:
353
+ if value in lookup:
354
+ if withDuplicates:
355
+ lookupResult=lookup[value]
356
+ lookupResult.append(entity)
357
+ else:
358
+ duplicates.append(entity)
359
+ else:
360
+ if withDuplicates:
361
+ lookupResult=[entity]
362
+ else:
363
+ lookupResult=entity
364
+ lookup[value]=lookupResult
365
+ if withDuplicates:
366
+ return lookup
367
+ else:
368
+ return lookup,duplicates
369
+
321
370
 
322
371
  def getJsonData(self):
323
372
  '''
@@ -0,0 +1,92 @@
1
+ '''
2
+ Created on 2021-01-31
3
+
4
+ @author: wf
5
+ '''
6
+ from lodstorage.jsonable import JSONAbleList
7
+
8
+ class LOD(object):
9
+ '''
10
+ list of Dict aka Table
11
+ '''
12
+
13
+ def __init__(self,name):
14
+ '''
15
+ Constructor
16
+ '''
17
+ self.name=name
18
+ pass
19
+
20
+ @staticmethod
21
+ def getFields(listOfDicts,sampleCount:int=None):
22
+ if sampleCount is None:
23
+ sampleCount=len(listOfDicts)
24
+ fields=[]
25
+ for row in listOfDicts:
26
+ for key in row.keys():
27
+ if not key in fields:
28
+ fields.append(key)
29
+ return fields
30
+
31
+ @staticmethod
32
+ def setNone4List(listOfDicts,fields):
33
+ '''
34
+ set the given fields to None for the records in the given listOfDicts
35
+ if they are not set
36
+ Args:
37
+ listOfDicts(list): the list of records to work on
38
+ fields(list): the list of fields to set to None
39
+ '''
40
+ for record in listOfDicts:
41
+ LOD.setNone(record, fields)
42
+
43
+ @staticmethod
44
+ def setNone(record,fields):
45
+ '''
46
+ make sure the given fields in the given record are set to none
47
+ Args:
48
+ record(dict): the record to work on
49
+ fields(list): the list of fields to set to None
50
+ '''
51
+ for field in fields:
52
+ if not field in record:
53
+ record[field]=None
54
+
55
+ '''
56
+ https://stackoverflow.com/questions/33542997/python-intersection-of-2-lists-of-dictionaries/33543164
57
+ '''
58
+ @staticmethod
59
+ def sortKey(d,key=None):
60
+ ''' get the sort key for the given dict d with the given key
61
+ '''
62
+ if key is None:
63
+ # https://stackoverflow.com/a/60765557/1497139
64
+ return hash(tuple(d.items()))
65
+ else:
66
+ return d[key]
67
+
68
+ @staticmethod
69
+ def intersect(listOfDict1,listOfDict2,key=None):
70
+ '''
71
+ get the intersection of the two lists of Dicts by the given key
72
+ '''
73
+ i1=iter(sorted(listOfDict1, key=lambda k: LOD.sortKey(k, key)))
74
+ i2=iter(sorted(listOfDict2, key=lambda k: LOD.sortKey(k, key)))
75
+ c1=next(i1)
76
+ c2=next(i2)
77
+ lr=[]
78
+ while True:
79
+ try:
80
+ val1=LOD.sortKey(c1,key)
81
+ val2=LOD.sortKey(c2,key)
82
+ if val1<val2:
83
+ c1=next(i1)
84
+ elif val1>val2:
85
+ c2=next(i2)
86
+ else:
87
+ lr.append(c1)
88
+ c1=next(i1)
89
+ c2=next(i2)
90
+ except StopIteration:
91
+ break
92
+ return lr
@@ -0,0 +1,79 @@
1
+ '''
2
+ Created on 2020-07-05
3
+
4
+ @author: wf
5
+ '''
6
+ import matplotlib.pyplot as plt
7
+ from collections import Counter
8
+ import numpy as np
9
+ import os
10
+
11
+ class Plot(object):
12
+ '''
13
+ create Plot based on counters
14
+ see https://stackoverflow.com/questions/19198920/using-counter-in-python-to-build-histogram
15
+ '''
16
+ def __init__(self, valueList,title,xlabel=None,ylabel=None,gformat='.png',fontsize=12,plotdir=None,debug=False):
17
+ '''
18
+ Constructor
19
+ '''
20
+ self.counter=Counter(valueList)
21
+ self.valueList=valueList
22
+ self.title=title
23
+ self.xlabel=xlabel
24
+ self.ylabel=ylabel
25
+ self.fontsize=fontsize
26
+ self.gformat=gformat
27
+ self.debug=debug
28
+ path=os.path.dirname(__file__)
29
+ if plotdir is not None:
30
+ self.plotdir=plotdir
31
+ else:
32
+ self.plotdir=path+"/../plots/"
33
+ os.makedirs(self.plotdir,exist_ok=True)
34
+
35
+ def titleMe(self):
36
+ ''' set my title and labels '''
37
+ plt.title(self.title, fontsize=self.fontsize)
38
+ if self.xlabel is not None:
39
+ plt.xlabel(self.xlabel)
40
+ if self.ylabel is not None:
41
+ plt.ylabel(self.ylabel)
42
+
43
+ def showMe(self,mode='show',close=True):
44
+ ''' show me in the given mode '''
45
+ if mode=="show":
46
+ plt.show()
47
+ else:
48
+ plt.savefig(self.plotdir+self.title+self.gformat)
49
+ if close:
50
+ plt.close()
51
+
52
+ def barchart(self,mode='show'):
53
+ ''' barchart based histogram for the given counter '''
54
+ labels, values = zip(*self.counter.items())
55
+ indexes = np.arange(len(labels))
56
+ width = 1
57
+ self.titleMe()
58
+ plt.bar(indexes, values, width)
59
+ plt.xticks(indexes + width * 0.5, labels)
60
+ plt.yticks(np.arange(1,max(values)+1,step=1))
61
+ self.showMe(mode)
62
+
63
+ def showDebug(self):
64
+ print(" value list: ",self.valueList)
65
+ print("counter items: ",self.counter.items())
66
+ print("counter values: ",self.counter.values())
67
+ print("counter keys: ",self.counter.keys())
68
+
69
+ def hist(self,mode="show"):
70
+ ''' create histogram for the given counter '''
71
+ if self.debug:
72
+ self.showDebug()
73
+ self.titleMe()
74
+ # see https://stackoverflow.com/a/2162045/1497139
75
+ plt.hist(self.valueList,bins=len(self.counter.keys()))
76
+ self.showMe(mode)
77
+ pass
78
+
79
+
@@ -12,6 +12,7 @@ class Sample(object):
12
12
  '''
13
13
  Sample dataset generator
14
14
  '''
15
+ cityList=None
15
16
 
16
17
  def __init__(self):
17
18
  '''
@@ -37,12 +38,13 @@ class Sample(object):
37
38
  '''
38
39
  get a list of cities
39
40
  '''
40
- cityJsonUrl="https://raw.githubusercontent.com/lutangar/cities.json/master/cities.json"
41
- with urllib.request.urlopen(cityJsonUrl) as url:
42
- cityList=json.loads(url.read().decode())
43
- for city in cityList:
44
- city['cityId']="%s-%s" % (city['country'],city['name'])
45
- return cityList
41
+ if Sample.cityList is None:
42
+ cityJsonUrl="https://raw.githubusercontent.com/lutangar/cities.json/master/cities.json"
43
+ with urllib.request.urlopen(cityJsonUrl) as url:
44
+ Sample.cityList=json.loads(url.read().decode())
45
+ for city in Sample.cityList:
46
+ city['cityId']="%s-%s" % (city['country'],city['name'])
47
+ return Sample.cityList
46
48
 
47
49
  @staticmethod
48
50
  def dob(isoDateString):
@@ -0,0 +1,33 @@
1
+ '''
2
+ Created on 2021-06-13
3
+
4
+ @author: wf
5
+ '''
6
+ from tabulate import tabulate
7
+
8
+ class TabulateCounter(object):
9
+ '''
10
+ helper for tabulating Counters
11
+ '''
12
+
13
+
14
+ def __init__(self, counter):
15
+ '''
16
+ Constructor
17
+ '''
18
+ self.counter=counter
19
+
20
+ def mostCommonTable(self,headers=["#","key","count","%"],tablefmt='pretty',limit=50):
21
+ '''
22
+ get the most common Table
23
+ '''
24
+ bins=len(self.counter.keys())
25
+ limit=min(bins,limit)
26
+ total=sum(self.counter.values())
27
+ binTable=[("total",bins,total)]
28
+ for i,bintuple in enumerate(self.counter.most_common(limit)):
29
+ key,count=bintuple
30
+ binTable.append((i+1,key,count,count/total*100.0))
31
+
32
+ table=tabulate(binTable,headers=headers,tablefmt=tablefmt,floatfmt=".2f")
33
+ return table
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pylodstorage
3
- Version: 0.0.30
3
+ Version: 0.0.32
4
4
  Summary: python List of Dict (Table) Storage library
5
5
  Home-page: https://github.com/WolfgangFahl/pyLodStorage
6
6
  Author: Wolfgang Fahl
@@ -5,11 +5,13 @@ lodstorage/__init__.py
5
5
  lodstorage/jsonable.py
6
6
  lodstorage/lod.py
7
7
  lodstorage/mwTable.py
8
+ lodstorage/plot.py
8
9
  lodstorage/query.py
9
10
  lodstorage/sample.py
10
11
  lodstorage/schema.py
11
12
  lodstorage/sparql.py
12
13
  lodstorage/sql.py
14
+ lodstorage/tabulateCounter.py
13
15
  lodstorage/uml.py
14
16
  pylodstorage.egg-info/PKG-INFO
15
17
  pylodstorage.egg-info/SOURCES.txt
@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
11
11
 
12
12
  setup(
13
13
  name='pylodstorage',
14
- version='0.0.30',
14
+ version='0.0.32',
15
15
 
16
16
  packages=['lodstorage',],
17
17
  author='Wolfgang Fahl',
@@ -1,52 +0,0 @@
1
- '''
2
- Created on 2021-01-31
3
-
4
- @author: wf
5
- '''
6
-
7
- class LOD(object):
8
- '''
9
- list of Dict aka Table
10
- '''
11
-
12
- def __init__(self,name):
13
- '''
14
- Constructor
15
- '''
16
- self.name=name
17
- pass
18
-
19
- @staticmethod
20
- def getFields(listOfDicts,sampleCount:int=None):
21
- if sampleCount is None:
22
- sampleCount=len(listOfDicts)
23
- fields=[]
24
- for row in listOfDicts:
25
- for key in row.keys():
26
- if not key in fields:
27
- fields.append(key)
28
- return fields
29
-
30
- @staticmethod
31
- def setNone4List(listOfDicts,fields):
32
- '''
33
- set the given fields to None for the records in the given listOfDicts
34
- if they are not set
35
- Args:
36
- listOfDicts(list): the list of records to work on
37
- fields(list): the list of fields to set to None
38
- '''
39
- for record in listOfDicts:
40
- LOD.setNone(record, fields)
41
-
42
- @staticmethod
43
- def setNone(record,fields):
44
- '''
45
- make sure the given fields in the given record are set to none
46
- Args:
47
- record(dict): the record to work on
48
- fields(list): the list of fields to set to None
49
- '''
50
- for field in fields:
51
- if not field in record:
52
- record[field]=None
File without changes
File without changes
File without changes