pyLodStorage 0.0.32__tar.gz → 0.0.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/PKG-INFO +2 -1
- pylodstorage-0.0.34/lodstorage/csv.py +126 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/jsonable.py +36 -5
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/lod.py +2 -1
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/pylodstorage.egg-info/PKG-INFO +2 -1
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/pylodstorage.egg-info/SOURCES.txt +1 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/setup.py +3 -2
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/LICENSE +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/README.md +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/__init__.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/mwTable.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/plot.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/query.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/sample.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/schema.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/sparql.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/sql.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/tabulateCounter.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/lodstorage/uml.py +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/pylodstorage.egg-info/dependency_links.txt +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/pylodstorage.egg-info/requires.txt +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/pylodstorage.egg-info/top_level.txt +0 -0
- {pylodstorage-0.0.32 → pylodstorage-0.0.34}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pylodstorage
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.34
|
|
4
4
|
Summary: python List of Dict (Table) Storage library
|
|
5
5
|
Home-page: https://github.com/WolfgangFahl/pyLodStorage
|
|
6
6
|
Author: Wolfgang Fahl
|
|
@@ -65,4 +65,5 @@ Classifier: Programming Language :: Python
|
|
|
65
65
|
Classifier: Programming Language :: Python :: 3.6
|
|
66
66
|
Classifier: Programming Language :: Python :: 3.7
|
|
67
67
|
Classifier: Programming Language :: Python :: 3.8
|
|
68
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
68
69
|
Description-Content-Type: text/markdown
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import io
|
|
3
|
+
|
|
4
|
+
from lodstorage.lod import LOD
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CSV(LOD):
|
|
8
|
+
'''
|
|
9
|
+
helper for converting data in csv format to list of dicts (LoD) and vice versa
|
|
10
|
+
'''
|
|
11
|
+
|
|
12
|
+
@staticmethod
|
|
13
|
+
def restoreFromCSVFile(filePath:str, headerNames:list=None, withPostfix:bool=False):
|
|
14
|
+
'''
|
|
15
|
+
restore LOD from given csv file
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
filePath(str): file name
|
|
19
|
+
headerNames(list): Names of the headers that should be used. If None it is assumed that the header is given.
|
|
20
|
+
withPostfix(bool): If False the file type is appended to given filePath. Otherwise file type MUST be given with filePath.
|
|
21
|
+
|
|
22
|
+
Returns:
|
|
23
|
+
list of dicts (LoD) containing the content of the given csv file
|
|
24
|
+
'''
|
|
25
|
+
if not withPostfix:
|
|
26
|
+
filePath += ".csv"
|
|
27
|
+
csvStr = CSV.readFile(filePath)
|
|
28
|
+
lod = CSV.fromCSV(csvStr, headerNames)
|
|
29
|
+
return lod
|
|
30
|
+
|
|
31
|
+
@staticmethod
|
|
32
|
+
def fromCSV(csvString:str, headerNames:list=None):
|
|
33
|
+
'''
|
|
34
|
+
convert given csv string to list of dicts (LOD)
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
csvStr(str): csv string that should be converted to LOD
|
|
38
|
+
headerNames(list): Names of the headers that should be used. If None it is assumed that the header is given.
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
list of dicts (LoD) containing the content of the given csv string
|
|
42
|
+
'''
|
|
43
|
+
reader = csv.DictReader(io.StringIO(csvString), headerNames)
|
|
44
|
+
lod=list(reader)
|
|
45
|
+
CSV.fixTypes(lod)
|
|
46
|
+
return lod
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def storeToCSVFile(lod:list, filePath:str, withPostfix:bool=False):
|
|
50
|
+
'''
|
|
51
|
+
converts the given lod to CSV file.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
lod(list): lod that should be converted to csv file
|
|
55
|
+
filePath(str): file name the csv should be stored to
|
|
56
|
+
withPostfix(bool): If False the file type is appended to given filePath. Otherwise file type MUST be given with filePath.
|
|
57
|
+
Returns:
|
|
58
|
+
csv string of the given lod
|
|
59
|
+
'''
|
|
60
|
+
if not withPostfix:
|
|
61
|
+
filePath += ".csv"
|
|
62
|
+
csvStr=CSV.toCSV(lod)
|
|
63
|
+
CSV.writeFile(csvStr, filePath)
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def toCSV(lod:list):
|
|
67
|
+
'''
|
|
68
|
+
converts the given lod to CSV string.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
lod(list): lod that should be converted to csv string
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
csv string of the given lod
|
|
75
|
+
'''
|
|
76
|
+
fields = LOD.getFields(lod)
|
|
77
|
+
keys=fields
|
|
78
|
+
csvString = ""
|
|
79
|
+
csvStream = io.StringIO(csvString)
|
|
80
|
+
dict_writer = csv.DictWriter(csvStream, keys)
|
|
81
|
+
dict_writer.writeheader()
|
|
82
|
+
dict_writer.writerows(lod)
|
|
83
|
+
csvStream.seek(0)
|
|
84
|
+
csvString = csvStream.read()
|
|
85
|
+
return csvString
|
|
86
|
+
|
|
87
|
+
@staticmethod
|
|
88
|
+
def readFile(filename: str) -> str:
|
|
89
|
+
"""
|
|
90
|
+
Reads the given filename and returns it as string
|
|
91
|
+
Args:
|
|
92
|
+
filename: Name of the file that should be returned as string
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
Content of the file as string
|
|
96
|
+
"""
|
|
97
|
+
content = ""
|
|
98
|
+
with open(filename, 'r') as file:
|
|
99
|
+
content = file.read()
|
|
100
|
+
return content
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def writeFile(content:str, filename: str) -> str:
|
|
104
|
+
"""
|
|
105
|
+
Write the given str to the given filename
|
|
106
|
+
Args:
|
|
107
|
+
content(str): string that should be written into the file
|
|
108
|
+
filename: Name of the file the given str should be written to
|
|
109
|
+
Returns:
|
|
110
|
+
Nothing
|
|
111
|
+
"""
|
|
112
|
+
with open(filename, 'w') as file:
|
|
113
|
+
file.write(content)
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def fixTypes(lod:list):
|
|
117
|
+
"""
|
|
118
|
+
fixes the types of the given LoD.
|
|
119
|
+
|
|
120
|
+
"""
|
|
121
|
+
for record in lod:
|
|
122
|
+
for key, value in record.items():
|
|
123
|
+
# fix empty csv value: "cell1,,cell3" converts the second value to empty string instead of None
|
|
124
|
+
if value == '':
|
|
125
|
+
record[key] = None
|
|
126
|
+
|
|
@@ -9,6 +9,9 @@ import datetime
|
|
|
9
9
|
import sys
|
|
10
10
|
import re
|
|
11
11
|
|
|
12
|
+
from lodstorage.lod import LOD
|
|
13
|
+
|
|
14
|
+
|
|
12
15
|
class JSONAbleSettings():
|
|
13
16
|
'''
|
|
14
17
|
settings for JSONAble - put in a separate class so they would not be
|
|
@@ -117,6 +120,8 @@ class JSONAble(object):
|
|
|
117
120
|
'''
|
|
118
121
|
if hasattr(self, '__class__'):
|
|
119
122
|
cls=self.__class__
|
|
123
|
+
if isinstance(self, JSONAbleList) and not hasattr(cls, 'getSamples'):
|
|
124
|
+
cls=self.clazz
|
|
120
125
|
if hasattr(cls, 'getSamples'):
|
|
121
126
|
getSamples=getattr(cls,'getSamples');
|
|
122
127
|
if callable(getSamples):
|
|
@@ -151,14 +156,15 @@ class JSONAble(object):
|
|
|
151
156
|
with open(jsonFilePath,"w") as jsonFile:
|
|
152
157
|
jsonFile.write(jsonStr)
|
|
153
158
|
|
|
154
|
-
def storeToJsonFile(self,storeFilePrefix):
|
|
159
|
+
def storeToJsonFile(self,storeFilePrefix, limitToSampleFields:bool=False):
|
|
155
160
|
'''
|
|
156
161
|
store me with the given storeFilePrefix
|
|
157
162
|
|
|
158
163
|
Args:
|
|
159
164
|
storeFilePrefix(string): the prefix for the JSON file name
|
|
165
|
+
limitToSampleFields(bool): If True the returned JSON is limited to the attributes/fields that are present in the samples. Otherwise all attributes of the object will be included. Default is False.
|
|
160
166
|
'''
|
|
161
|
-
JSONAble.storeJsonToFile(self.toJSON(), "%s.json" % storeFilePrefix)
|
|
167
|
+
JSONAble.storeJsonToFile(self.toJSON(limitToSampleFields), "%s.json" % storeFilePrefix)
|
|
162
168
|
|
|
163
169
|
def restoreFromJsonFile(self,storeFilePrefix):
|
|
164
170
|
'''
|
|
@@ -224,12 +230,35 @@ class JSONAble(object):
|
|
|
224
230
|
else:
|
|
225
231
|
return ""
|
|
226
232
|
|
|
227
|
-
def toJSON(self):
|
|
233
|
+
def toJSON(self, limitToSampleFields:bool=False):
|
|
228
234
|
'''
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
limitToSampleFields(bool): If True the returned JSON is limited to the attributes/fields that are present in the samples. Otherwise all attributes of the object will be included. Default is False.
|
|
238
|
+
|
|
229
239
|
Returns:
|
|
230
240
|
a recursive JSON dump of the dicts of my objects
|
|
231
241
|
'''
|
|
232
|
-
|
|
242
|
+
data={}
|
|
243
|
+
if limitToSampleFields:
|
|
244
|
+
samples=self.getJsonTypeSamples()
|
|
245
|
+
sampleFields = LOD.getFields(samples)
|
|
246
|
+
if isinstance(self, JSONAbleList):
|
|
247
|
+
limitedRecords=[]
|
|
248
|
+
for record in self.__dict__[self.listName]:
|
|
249
|
+
limitedRecord={}
|
|
250
|
+
for key, value in record.__dict__.items():
|
|
251
|
+
if key in sampleFields:
|
|
252
|
+
limitedRecord[key] = value
|
|
253
|
+
limitedRecords.append(limitedRecord)
|
|
254
|
+
data[self.listName]=limitedRecords
|
|
255
|
+
else:
|
|
256
|
+
for key,value in self.__dict__.items():
|
|
257
|
+
if key in sampleFields:
|
|
258
|
+
data[key]=value
|
|
259
|
+
else:
|
|
260
|
+
data=self.__dict__
|
|
261
|
+
jsonStr=json.dumps(data, default=lambda v: self.toJsonAbleValue(v),
|
|
233
262
|
sort_keys=True, indent=JSONAbleSettings.indent)
|
|
234
263
|
return jsonStr
|
|
235
264
|
|
|
@@ -292,6 +321,7 @@ class JSONAble(object):
|
|
|
292
321
|
jsonStr = JSONAble.singleQuoteToDoubleQuote(jsonStr)
|
|
293
322
|
return jsonStr
|
|
294
323
|
return jsonDict
|
|
324
|
+
|
|
295
325
|
|
|
296
326
|
class JSONAbleList(JSONAble):
|
|
297
327
|
'''
|
|
@@ -508,7 +538,8 @@ class Types(JSONAble):
|
|
|
508
538
|
valueType=None
|
|
509
539
|
if value is None:
|
|
510
540
|
if warnOnNone:
|
|
511
|
-
|
|
541
|
+
if self.debug:
|
|
542
|
+
print("Warning sampleRecord field %s is None - using string as type" % key)
|
|
512
543
|
valueType=str
|
|
513
544
|
else:
|
|
514
545
|
valueType=type(value)
|
|
@@ -3,7 +3,6 @@ Created on 2021-01-31
|
|
|
3
3
|
|
|
4
4
|
@author: wf
|
|
5
5
|
'''
|
|
6
|
-
from lodstorage.jsonable import JSONAbleList
|
|
7
6
|
|
|
8
7
|
class LOD(object):
|
|
9
8
|
'''
|
|
@@ -20,6 +19,8 @@ class LOD(object):
|
|
|
20
19
|
@staticmethod
|
|
21
20
|
def getFields(listOfDicts,sampleCount:int=None):
|
|
22
21
|
if sampleCount is None:
|
|
22
|
+
if listOfDicts is None:
|
|
23
|
+
return None
|
|
23
24
|
sampleCount=len(listOfDicts)
|
|
24
25
|
fields=[]
|
|
25
26
|
for row in listOfDicts:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pylodstorage
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.34
|
|
4
4
|
Summary: python List of Dict (Table) Storage library
|
|
5
5
|
Home-page: https://github.com/WolfgangFahl/pyLodStorage
|
|
6
6
|
Author: Wolfgang Fahl
|
|
@@ -65,4 +65,5 @@ Classifier: Programming Language :: Python
|
|
|
65
65
|
Classifier: Programming Language :: Python :: 3.6
|
|
66
66
|
Classifier: Programming Language :: Python :: 3.7
|
|
67
67
|
Classifier: Programming Language :: Python :: 3.8
|
|
68
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
68
69
|
Description-Content-Type: text/markdown
|
|
@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
|
|
|
11
11
|
|
|
12
12
|
setup(
|
|
13
13
|
name='pylodstorage',
|
|
14
|
-
version='0.0.
|
|
14
|
+
version='0.0.34',
|
|
15
15
|
|
|
16
16
|
packages=['lodstorage',],
|
|
17
17
|
author='Wolfgang Fahl',
|
|
@@ -36,7 +36,8 @@ setup(
|
|
|
36
36
|
'Programming Language :: Python',
|
|
37
37
|
'Programming Language :: Python :: 3.6',
|
|
38
38
|
'Programming Language :: Python :: 3.7',
|
|
39
|
-
'Programming Language :: Python :: 3.8'
|
|
39
|
+
'Programming Language :: Python :: 3.8',
|
|
40
|
+
'Programming Language :: Python :: 3.9'
|
|
40
41
|
],
|
|
41
42
|
long_description=long_description,
|
|
42
43
|
long_description_content_type='text/markdown'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|