pyLodStorage 0.0.24__tar.gz → 0.0.26__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/PKG-INFO +1 -1
- pylodstorage-0.0.26/lodstorage/lod.py +52 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/sparql.py +15 -3
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/sql.py +5 -26
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/pylodstorage.egg-info/PKG-INFO +1 -1
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/pylodstorage.egg-info/SOURCES.txt +1 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/setup.py +1 -1
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/README.md +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/__init__.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/jsonable.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/mwTable.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/query.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/sample.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/schema.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/lodstorage/uml.py +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/pylodstorage.egg-info/dependency_links.txt +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/pylodstorage.egg-info/requires.txt +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/pylodstorage.egg-info/top_level.txt +0 -0
- {pylodstorage-0.0.24 → pylodstorage-0.0.26}/setup.cfg +0 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2021-01-31
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
|
|
7
|
+
class LOD(object):
|
|
8
|
+
'''
|
|
9
|
+
list of Dict aka Table
|
|
10
|
+
'''
|
|
11
|
+
|
|
12
|
+
def __init__(self,name):
|
|
13
|
+
'''
|
|
14
|
+
Constructor
|
|
15
|
+
'''
|
|
16
|
+
self.name=name
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
@staticmethod
|
|
20
|
+
def getFields(listOfDicts,sampleCount:int=None):
|
|
21
|
+
if sampleCount is None:
|
|
22
|
+
sampleCount=len(listOfDicts)
|
|
23
|
+
fields=[]
|
|
24
|
+
for row in listOfDicts:
|
|
25
|
+
for key in row.keys():
|
|
26
|
+
if not key in fields:
|
|
27
|
+
fields.append(key)
|
|
28
|
+
return fields
|
|
29
|
+
|
|
30
|
+
@staticmethod
|
|
31
|
+
def setNone4List(listOfDicts,fields):
|
|
32
|
+
'''
|
|
33
|
+
set the given fields to None for the records in the given listOfDicts
|
|
34
|
+
if they are not set
|
|
35
|
+
Args:
|
|
36
|
+
listOfDicts(list): the list of records to work on
|
|
37
|
+
fields(list): the list of fields to set to None
|
|
38
|
+
'''
|
|
39
|
+
for record in listOfDicts:
|
|
40
|
+
LOD.setNone(record, fields)
|
|
41
|
+
|
|
42
|
+
@staticmethod
|
|
43
|
+
def setNone(record,fields):
|
|
44
|
+
'''
|
|
45
|
+
make sure the given fields in the given record are set to none
|
|
46
|
+
Args:
|
|
47
|
+
record(dict): the record to work on
|
|
48
|
+
fields(list): the list of fields to set to None
|
|
49
|
+
'''
|
|
50
|
+
for field in fields:
|
|
51
|
+
if not field in record:
|
|
52
|
+
record[field]=None
|
|
@@ -5,6 +5,7 @@ Created on 2020-08-14
|
|
|
5
5
|
'''
|
|
6
6
|
from SPARQLWrapper import SPARQLWrapper2
|
|
7
7
|
from SPARQLWrapper.Wrapper import POSTDIRECTLY, POST
|
|
8
|
+
from lodstorage.lod import LOD
|
|
8
9
|
import datetime
|
|
9
10
|
import time
|
|
10
11
|
from sys import stderr
|
|
@@ -269,18 +270,20 @@ class SPARQL(object):
|
|
|
269
270
|
jsonResult=queryResult.convert()
|
|
270
271
|
return self.getResults(jsonResult)
|
|
271
272
|
|
|
272
|
-
def queryAsListOfDicts(self,queryString):
|
|
273
|
+
def queryAsListOfDicts(self,queryString,fixNone:bool=False,sampleCount:int=None):
|
|
273
274
|
'''
|
|
274
275
|
get a list of dicts for the given query (to allow round-trip results for insertListOfDicts)
|
|
275
276
|
|
|
276
277
|
Args:
|
|
277
278
|
queryString(string): the SPARQL query to execute
|
|
279
|
+
fixNone(bool): if True add None values for empty columns in Dict
|
|
280
|
+
sampleCount(int): the number of samples to check
|
|
278
281
|
|
|
279
282
|
Returns:
|
|
280
283
|
list: a list ofDicts
|
|
281
284
|
'''
|
|
282
285
|
records=self.query(queryString)
|
|
283
|
-
listOfDicts=self.asListOfDicts(records)
|
|
286
|
+
listOfDicts=self.asListOfDicts(records,fixNone=fixNone,sampleCount=sampleCount)
|
|
284
287
|
return listOfDicts
|
|
285
288
|
|
|
286
289
|
@staticmethod
|
|
@@ -303,17 +306,22 @@ class SPARQL(object):
|
|
|
303
306
|
print(str(ve))
|
|
304
307
|
return dt
|
|
305
308
|
|
|
306
|
-
def asListOfDicts(self,records):
|
|
309
|
+
def asListOfDicts(self,records,fixNone:bool=False,sampleCount:int=None):
|
|
307
310
|
'''
|
|
308
311
|
convert SPARQL result back to python native
|
|
309
312
|
|
|
310
313
|
Args:
|
|
311
314
|
record(list): the list of bindings
|
|
315
|
+
fixNone(bool): if True add None values for empty columns in Dict
|
|
316
|
+
sampleCount(int): the number of samples to check
|
|
312
317
|
|
|
313
318
|
Returns:
|
|
314
319
|
list: a list of Dicts
|
|
315
320
|
'''
|
|
316
321
|
resultList=[]
|
|
322
|
+
fields=None
|
|
323
|
+
if fixNone:
|
|
324
|
+
fields=LOD.getFields(records, sampleCount)
|
|
317
325
|
for record in records:
|
|
318
326
|
resultDict={}
|
|
319
327
|
for keyValue in record.items():
|
|
@@ -338,6 +346,10 @@ class SPARQL(object):
|
|
|
338
346
|
else:
|
|
339
347
|
resultValue=value.value
|
|
340
348
|
resultDict[key]=resultValue
|
|
349
|
+
if fixNone:
|
|
350
|
+
for field in fields:
|
|
351
|
+
if not field in resultDict:
|
|
352
|
+
resultDict[field]=None
|
|
341
353
|
resultList.append(resultDict)
|
|
342
354
|
return resultList
|
|
343
355
|
|
|
@@ -10,6 +10,7 @@ import io
|
|
|
10
10
|
import time
|
|
11
11
|
import sys
|
|
12
12
|
import re
|
|
13
|
+
from lodstorage.lod import LOD
|
|
13
14
|
|
|
14
15
|
class SQLDB(object):
|
|
15
16
|
'''
|
|
@@ -43,31 +44,7 @@ class SQLDB(object):
|
|
|
43
44
|
|
|
44
45
|
def close(self):
|
|
45
46
|
''' close my connection '''
|
|
46
|
-
self.c.close()
|
|
47
|
-
|
|
48
|
-
@staticmethod
|
|
49
|
-
def setNone4List(listOfDicts,fields):
|
|
50
|
-
'''
|
|
51
|
-
set the given fields to None for the records in the given listOfDicts
|
|
52
|
-
if they are not set
|
|
53
|
-
Args:
|
|
54
|
-
listOfDicts(list): the list of records to work on
|
|
55
|
-
fields(list): the list of fields to set to None
|
|
56
|
-
'''
|
|
57
|
-
for record in listOfDicts:
|
|
58
|
-
SQLDB.setNone(record, fields)
|
|
59
|
-
|
|
60
|
-
@staticmethod
|
|
61
|
-
def setNone(record,fields):
|
|
62
|
-
'''
|
|
63
|
-
make sure the given fields in the given record are set to none
|
|
64
|
-
Args:
|
|
65
|
-
record(dict): the record to work on
|
|
66
|
-
fields(list): the list of fields to set to None
|
|
67
|
-
'''
|
|
68
|
-
for field in fields:
|
|
69
|
-
if not field in record:
|
|
70
|
-
record[field]=None
|
|
47
|
+
self.c.close()
|
|
71
48
|
|
|
72
49
|
def execute(self,ddlCmd):
|
|
73
50
|
'''
|
|
@@ -139,6 +116,8 @@ class SQLDB(object):
|
|
|
139
116
|
|
|
140
117
|
listOfRecords(list): the list of Dicts to be stored
|
|
141
118
|
entityInfo(EntityInfo): the meta data to be used for storing
|
|
119
|
+
executeMany(bool): if True the insert command is done with many/all records at once
|
|
120
|
+
fixNone(bool): if True make sure empty columns in the listOfDict are filled with "None" values
|
|
142
121
|
'''
|
|
143
122
|
insertCmd=entityInfo.insertCmd
|
|
144
123
|
record=None
|
|
@@ -146,7 +125,7 @@ class SQLDB(object):
|
|
|
146
125
|
try:
|
|
147
126
|
if executeMany:
|
|
148
127
|
if fixNone:
|
|
149
|
-
|
|
128
|
+
LOD.setNone4List(listOfRecords, entityInfo.typeMap.keys())
|
|
150
129
|
self.c.executemany(insertCmd,listOfRecords)
|
|
151
130
|
else:
|
|
152
131
|
for record in listOfRecords:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|