pyLodStorage 0.0.41__tar.gz → 0.0.43__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pylodstorage-0.0.41/pylodstorage.egg-info → pylodstorage-0.0.43}/PKG-INFO +1 -1
- pylodstorage-0.0.43/lodstorage/entity.py +255 -0
- pylodstorage-0.0.43/lodstorage/jsonpicklemixin.py +60 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/lod.py +26 -0
- pylodstorage-0.0.43/lodstorage/storageconfig.py +89 -0
- pylodstorage-0.0.43/lodstorage/yamlablemixin.py +40 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43/pylodstorage.egg-info}/PKG-INFO +1 -1
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/SOURCES.txt +4 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/setup.py +1 -1
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/LICENSE +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/README.md +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/__init__.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/csv.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/jsonable.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/mwTable.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/plot.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/query.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sample.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/schema.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sparql.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sql.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/tabulateCounter.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/uml.py +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/dependency_links.txt +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/requires.txt +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/top_level.txt +0 -0
- {pylodstorage-0.0.41 → pylodstorage-0.0.43}/setup.cfg +0 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2020-08-19
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
from lodstorage.yamlablemixin import YamlAbleMixin
|
|
7
|
+
from lodstorage.jsonpicklemixin import JsonPickleMixin
|
|
8
|
+
from lodstorage.storageconfig import StorageConfig, StoreMode
|
|
9
|
+
from lodstorage.sparql import SPARQL
|
|
10
|
+
from lodstorage.sql import SQLDB
|
|
11
|
+
from lodstorage.lod import LOD
|
|
12
|
+
import os
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
class EntityManager(YamlAbleMixin, JsonPickleMixin):
|
|
16
|
+
'''
|
|
17
|
+
generic entity manager
|
|
18
|
+
'''
|
|
19
|
+
|
|
20
|
+
def __init__(self,name,entityName,entityPluralName,config=None,debug=False):
|
|
21
|
+
'''
|
|
22
|
+
Constructor
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
name(string): name of this eventManager
|
|
26
|
+
entityName(string): entityType to be managed e.g. Country
|
|
27
|
+
entityPluralName(string): plural of the the entityType e.g. Countries
|
|
28
|
+
config(StorageConfig): the configuration to be used if None a default configuration will be used
|
|
29
|
+
debug(boolean): override debug setting when default of config is used via config=None
|
|
30
|
+
'''
|
|
31
|
+
self.name=name
|
|
32
|
+
self.entityName=entityName
|
|
33
|
+
self.entityPluralName=entityPluralName
|
|
34
|
+
if config is None:
|
|
35
|
+
config=StorageConfig.getDefault()
|
|
36
|
+
if config.tableName is None:
|
|
37
|
+
config.tableName=entityName
|
|
38
|
+
if debug:
|
|
39
|
+
config.debug=debug
|
|
40
|
+
self.config=config
|
|
41
|
+
cacheFile=self.getCacheFile(config=config,mode=config.mode)
|
|
42
|
+
self.showProgress ("Creating %smanager(%s) for %s using cache %s" % (self.entityName,config.mode,self.name,cacheFile))
|
|
43
|
+
if config.mode is StoreMode.SPARQL:
|
|
44
|
+
if config.endpoint is None:
|
|
45
|
+
raise Exception("no endpoint set for mode sparql")
|
|
46
|
+
self.endpoint=config.endpoint
|
|
47
|
+
self.sparql=SPARQL(config.endpoint,debug=config.debug,profile=config.profile)
|
|
48
|
+
elif config.mode is StoreMode.SQL:
|
|
49
|
+
self.executeMany=False # may be True when issues are fixed
|
|
50
|
+
|
|
51
|
+
def storeMode(self):
|
|
52
|
+
'''
|
|
53
|
+
return my store mode
|
|
54
|
+
'''
|
|
55
|
+
return self.config.mode
|
|
56
|
+
|
|
57
|
+
def showProgress(self,msg):
|
|
58
|
+
''' display a progress message
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
msg(string): the message to display
|
|
62
|
+
'''
|
|
63
|
+
if self.config.withShowProgress:
|
|
64
|
+
print (msg,flush=True)
|
|
65
|
+
|
|
66
|
+
def getCacheFile(self,config=None,mode=StoreMode.SQL):
|
|
67
|
+
'''
|
|
68
|
+
get the cache file for this event manager
|
|
69
|
+
Args:
|
|
70
|
+
config(StorageConfig): if None get the cache for my mode
|
|
71
|
+
mode(StoreMode): the storeMode to use
|
|
72
|
+
'''
|
|
73
|
+
cachedir=EntityManager.getCachePath()
|
|
74
|
+
if config is not None and config.cacheFile is not None:
|
|
75
|
+
return config.cacheFile
|
|
76
|
+
''' get the path to the file for my cached data '''
|
|
77
|
+
if mode is StoreMode.JSON:
|
|
78
|
+
cachepath="%s/%s-%s.%s" % (cachedir,self.name,"events",'json')
|
|
79
|
+
elif mode is StoreMode.SPARQL:
|
|
80
|
+
cachepath="%s %s" % ('SPARQL',config.endpoint)
|
|
81
|
+
elif mode is StoreMode.SQL:
|
|
82
|
+
cachepath="%s/%s.db" % (cachedir,config.tableName)
|
|
83
|
+
else:
|
|
84
|
+
cachepath="undefined cachepath for %s" % (mode)
|
|
85
|
+
return cachepath
|
|
86
|
+
|
|
87
|
+
def getSQLDB(self,cacheFile):
|
|
88
|
+
'''
|
|
89
|
+
get the SQL database for the given cacheFile
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
cacheFile(string): the file to get the SQL db from
|
|
93
|
+
'''
|
|
94
|
+
config=self.config
|
|
95
|
+
sqldb=self.sqldb=SQLDB(cacheFile,debug=config.debug,errorDebug=config.errorDebug)
|
|
96
|
+
return sqldb
|
|
97
|
+
|
|
98
|
+
def setNone(self,record,fields):
|
|
99
|
+
'''
|
|
100
|
+
make sure the given fields in the given record are set to none
|
|
101
|
+
Args:
|
|
102
|
+
record(dict): the record to work on
|
|
103
|
+
fields(list): the list of fields to set to None
|
|
104
|
+
'''
|
|
105
|
+
LOD.setNone(record,fields)
|
|
106
|
+
|
|
107
|
+
def isCached(self):
|
|
108
|
+
''' check whether there is a file containing cached
|
|
109
|
+
data for me '''
|
|
110
|
+
result=False
|
|
111
|
+
config=self.config
|
|
112
|
+
mode=self.config.mode
|
|
113
|
+
if mode is StoreMode.JSON:
|
|
114
|
+
result=os.path.isfile(self.getCacheFile(config=self.config,mode=StoreMode.JSON))
|
|
115
|
+
elif mode is StoreMode.SPARQL:
|
|
116
|
+
# @FIXME - make abstract
|
|
117
|
+
query=config.prefix+"""
|
|
118
|
+
SELECT ?source (COUNT(?source) AS ?sourcecount)
|
|
119
|
+
WHERE {
|
|
120
|
+
?event cr:Event_source ?source.
|
|
121
|
+
}
|
|
122
|
+
GROUP by ?source
|
|
123
|
+
"""
|
|
124
|
+
sourceCountList=self.sparql.queryAsListOfDicts(query)
|
|
125
|
+
for sourceCount in sourceCountList:
|
|
126
|
+
source=sourceCount['source'];
|
|
127
|
+
recordCount=sourceCount['sourcecount']
|
|
128
|
+
if source==self.name and recordCount>100:
|
|
129
|
+
result=True
|
|
130
|
+
elif mode is StoreMode.SQL:
|
|
131
|
+
cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.SQL)
|
|
132
|
+
if os.path.isfile(cacheFile):
|
|
133
|
+
sqlQuery="SELECT COUNT(*) AS count FROM %s" % config.tableName
|
|
134
|
+
try:
|
|
135
|
+
sqlDB=self.getSQLDB(cacheFile)
|
|
136
|
+
countResult=sqlDB.query(sqlQuery)
|
|
137
|
+
count=countResult[0]['count']
|
|
138
|
+
result=count>100
|
|
139
|
+
except Exception as ex:
|
|
140
|
+
# e.g. sqlite3.OperationalError: no such table: Event_crossref
|
|
141
|
+
pass
|
|
142
|
+
else:
|
|
143
|
+
raise Exception("unsupported mode %s" % self.mode)
|
|
144
|
+
return result
|
|
145
|
+
|
|
146
|
+
def fromCache(self):
|
|
147
|
+
'''
|
|
148
|
+
get my entries from the cache
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
the list of Dicts and as a side effect setting self.cacheFile
|
|
152
|
+
'''
|
|
153
|
+
if not self.isCached():
|
|
154
|
+
listOfDicts=self.getListOfDicts()
|
|
155
|
+
self.cacheFile=self.store(listOfDicts)
|
|
156
|
+
else:
|
|
157
|
+
# fromStore also sets self.cacheFile
|
|
158
|
+
listOfDicts=self.fromStore()
|
|
159
|
+
return listOfDicts
|
|
160
|
+
|
|
161
|
+
def fromStore(self,cacheFile=None):
|
|
162
|
+
'''
|
|
163
|
+
restore me from the store
|
|
164
|
+
Args:
|
|
165
|
+
cacheFile(String): the cacheFile to use if None use the preconfigured Cachefile
|
|
166
|
+
Returns:
|
|
167
|
+
list: list of dicts or JSON entitymanager
|
|
168
|
+
'''
|
|
169
|
+
startTime=time.time()
|
|
170
|
+
if cacheFile is None:
|
|
171
|
+
cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
|
|
172
|
+
self.cacheFile=cacheFile
|
|
173
|
+
self.showProgress("reading %s for %s from cache %s" % (self.entityPluralName,self.name,cacheFile))
|
|
174
|
+
JSONem=None
|
|
175
|
+
mode=self.config.mode
|
|
176
|
+
if mode is StoreMode.JSON:
|
|
177
|
+
JSONem=JsonPickleMixin.readJson(cacheFile)
|
|
178
|
+
elif mode is StoreMode.SPARQL:
|
|
179
|
+
# @FIXME make abstract
|
|
180
|
+
eventQuery="""
|
|
181
|
+
PREFIX cr: <http://cr.bitplan.com/>
|
|
182
|
+
SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate ?url ?source WHERE {
|
|
183
|
+
OPTIONAL { ?event cr:Event_eventId ?eventId. }
|
|
184
|
+
OPTIONAL { ?event cr:Event_acronym ?acronym. }
|
|
185
|
+
OPTIONAL { ?event cr:Event_series ?series. }
|
|
186
|
+
OPTIONAL { ?event cr:Event_title ?title. }
|
|
187
|
+
OPTIONAL { ?event cr:Event_year ?year. }
|
|
188
|
+
OPTIONAL { ?event cr:Event_country ?country. }
|
|
189
|
+
OPTIONAL { ?event cr:Event_city ?city. }
|
|
190
|
+
OPTIONAL { ?event cr:Event_startDate ?startDate. }
|
|
191
|
+
OPTIONAL { ?event cr:Event_endDate ?endDate. }
|
|
192
|
+
OPTIONAL { ?event cr:Event_url ?url. }
|
|
193
|
+
?event cr:Event_source ?source FILTER(?source='%s').
|
|
194
|
+
}
|
|
195
|
+
""" % self.name
|
|
196
|
+
listOfDicts=self.sparql.queryAsListOfDicts(eventQuery)
|
|
197
|
+
elif mode is StoreMode.SQL:
|
|
198
|
+
sqlQuery="SELECT * FROM %s" % self.config.tableName
|
|
199
|
+
sqlDB=self.getSQLDB(cacheFile)
|
|
200
|
+
listOfDicts=sqlDB.query(sqlQuery)
|
|
201
|
+
sqlDB.close()
|
|
202
|
+
pass
|
|
203
|
+
else:
|
|
204
|
+
raise Exception("unsupported store mode %s" % self.mode)
|
|
205
|
+
|
|
206
|
+
if JSONem is not None:
|
|
207
|
+
return JSONem
|
|
208
|
+
else:
|
|
209
|
+
self.showProgress("read %d %s from %s in %5.1f s" % (len(listOfDicts),self.entityPluralName,self.name,time.time()-startTime))
|
|
210
|
+
return listOfDicts
|
|
211
|
+
|
|
212
|
+
def store(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,sampleRecordCount=1):
|
|
213
|
+
'''
|
|
214
|
+
store my entities
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
listOfDicts(list): the list of dicts to store
|
|
218
|
+
limit(int): maximumn number of records to store
|
|
219
|
+
batchSize(int): size of batch for storing
|
|
220
|
+
cacheFile(string): the name of the storage e.g path to JSON or sqlite3 file
|
|
221
|
+
sampleRecordCount(int): the number of records to analyze for type information
|
|
222
|
+
'''
|
|
223
|
+
config=self.config
|
|
224
|
+
mode=config.mode
|
|
225
|
+
if mode is StoreMode.JSON:
|
|
226
|
+
if cacheFile is None:
|
|
227
|
+
cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.JSON)
|
|
228
|
+
self.showProgress ("storing %d events for %s to cache %s" % (len(self.events),self.name,cacheFile))
|
|
229
|
+
self.writeJson(cacheFile)
|
|
230
|
+
elif mode is StoreMode.DGRAPH:
|
|
231
|
+
startTime=time.time()
|
|
232
|
+
self.showProgress ("storing %d %s for %s to %s" % (len(self.events),self.entityPluralName,self.name,self.mode))
|
|
233
|
+
self.dgraph.addData(listOfDicts,limit=limit,batchSize=batchSize)
|
|
234
|
+
self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
|
|
235
|
+
elif mode is StoreMode.SPARQL:
|
|
236
|
+
startTime=time.time()
|
|
237
|
+
# @ FIXME make abstract
|
|
238
|
+
self.showProgress ("storing %d events for %s to %s" % (len(self.events),self.name,self.mode))
|
|
239
|
+
entityType="cr:Event"
|
|
240
|
+
prefixes="PREFIX cr: <http://cr.bitplan.com/>"
|
|
241
|
+
primaryKey="eventId"
|
|
242
|
+
self.sparql.insertListOfDicts(listOfDicts, entityType, primaryKey, prefixes,limit=limit,batchSize=batchSize)
|
|
243
|
+
self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
|
|
244
|
+
elif mode is StoreMode.SQL:
|
|
245
|
+
startTime=time.time()
|
|
246
|
+
if cacheFile is None:
|
|
247
|
+
cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
|
|
248
|
+
sqldb=self.getSQLDB(cacheFile)
|
|
249
|
+
self.showProgress ("storing %d %s for %s to %s:%s" % (len(listOfDicts),self.entityPluralName,self.name,config.mode,cacheFile))
|
|
250
|
+
entityInfo=sqldb.createTable(listOfDicts, config.tableName, "eventId",withDrop=True,sampleRecordCount=sampleRecordCount)
|
|
251
|
+
self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany)
|
|
252
|
+
self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
|
|
253
|
+
else:
|
|
254
|
+
raise Exception("unsupported store mode %s" % self.mode)
|
|
255
|
+
return cacheFile
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# Json persistence
|
|
2
|
+
import jsonpickle
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class JsonPickleMixin(object):
|
|
7
|
+
'''
|
|
8
|
+
allow reading and writing derived objects from a jsonpickle file
|
|
9
|
+
'''
|
|
10
|
+
debug = False
|
|
11
|
+
|
|
12
|
+
# read me from a yaml file
|
|
13
|
+
@staticmethod
|
|
14
|
+
def readJson(name, postfix=".json"):
|
|
15
|
+
jsonFileName = name
|
|
16
|
+
if not name.endswith(postfix):
|
|
17
|
+
jsonFileName = name + postfix
|
|
18
|
+
# is there a jsonFile for the given name
|
|
19
|
+
if os.path.isfile(jsonFileName):
|
|
20
|
+
if JsonPickleMixin.debug:
|
|
21
|
+
print("reading %s" % (jsonFileName))
|
|
22
|
+
with open(jsonFileName) as jsonFile:
|
|
23
|
+
json = jsonFile.read()
|
|
24
|
+
result = jsonpickle.decode(json)
|
|
25
|
+
if (JsonPickleMixin.debug):
|
|
26
|
+
print (json)
|
|
27
|
+
print (result)
|
|
28
|
+
return result
|
|
29
|
+
else:
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
def asJson(self)->str:
|
|
33
|
+
'''
|
|
34
|
+
convert me to JSON
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
str: a JSON String with my JSON representation
|
|
38
|
+
'''
|
|
39
|
+
json = jsonpickle.encode(self)
|
|
40
|
+
return json
|
|
41
|
+
|
|
42
|
+
def writeJson(self, name:str, postfix:str=".json"):
|
|
43
|
+
'''
|
|
44
|
+
write me to the json file with the given name (optionally without postfix)
|
|
45
|
+
|
|
46
|
+
name(str): name of the file (optionally without ".json" postfix)
|
|
47
|
+
postfix(str): default file extension
|
|
48
|
+
'''
|
|
49
|
+
if not name.endswith(postfix):
|
|
50
|
+
jsonFileName = name + postfix
|
|
51
|
+
else:
|
|
52
|
+
jsonFileName = name
|
|
53
|
+
json = self.asJson()
|
|
54
|
+
if JsonPickleMixin.debug:
|
|
55
|
+
print("writing %s" % (jsonFileName))
|
|
56
|
+
print(json)
|
|
57
|
+
print(self)
|
|
58
|
+
jsonFile = open(jsonFileName, "w")
|
|
59
|
+
jsonFile.write(json)
|
|
60
|
+
jsonFile.close()
|
|
@@ -152,3 +152,29 @@ class LOD(object):
|
|
|
152
152
|
return lookup
|
|
153
153
|
else:
|
|
154
154
|
return lookup,duplicates
|
|
155
|
+
|
|
156
|
+
@classmethod
|
|
157
|
+
def handleListTypes(cls,lod,doFilter=False,separator=","):
|
|
158
|
+
'''
|
|
159
|
+
handle list types in the given list of dicts
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
cls: this class
|
|
163
|
+
lod(list): a list of dicts
|
|
164
|
+
doFilter(bool): True if records containing lists value items should be filtered
|
|
165
|
+
separator(str): the separator to use when converting lists
|
|
166
|
+
'''
|
|
167
|
+
# see https://stackoverflow.com/a/1207485/1497139
|
|
168
|
+
for i in range(len(lod) - 1, -1, -1):
|
|
169
|
+
record=lod[i]
|
|
170
|
+
if isinstance(record,dict):
|
|
171
|
+
for key in record:
|
|
172
|
+
value=record[key]
|
|
173
|
+
if isinstance(value,list):
|
|
174
|
+
if doFilter:
|
|
175
|
+
del lod[i]
|
|
176
|
+
continue
|
|
177
|
+
else:
|
|
178
|
+
newValue=separator.join(value)
|
|
179
|
+
record[key]=newValue
|
|
180
|
+
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
'''
|
|
2
|
+
Created on 2020-08-29
|
|
3
|
+
|
|
4
|
+
@author: wf
|
|
5
|
+
'''
|
|
6
|
+
from enum import Enum
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
class StoreMode(Enum):
|
|
11
|
+
'''
|
|
12
|
+
possible supported storage modes
|
|
13
|
+
'''
|
|
14
|
+
JSON = 1
|
|
15
|
+
JSONABLE = 2
|
|
16
|
+
SQL = 3
|
|
17
|
+
SPARQL = 4
|
|
18
|
+
YAML = 5
|
|
19
|
+
|
|
20
|
+
class StorageConfig(object):
|
|
21
|
+
'''
|
|
22
|
+
a storage configuration
|
|
23
|
+
'''
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def getCachePath(cls,name:str)->str:
|
|
27
|
+
'''
|
|
28
|
+
get the path to the default cache
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
name(str): the name of the cache to use
|
|
32
|
+
'''
|
|
33
|
+
home = str(Path.home())
|
|
34
|
+
cachedir=f"{home}/{name}"
|
|
35
|
+
return cachedir
|
|
36
|
+
|
|
37
|
+
def __init__(self, mode=StoreMode.SQL,cacheFile=None,withShowProgress=True,profile=True,debug=False,errorDebug=True):
|
|
38
|
+
'''
|
|
39
|
+
Constructor
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
mode(StoreMode): the storage mode e.g. sql
|
|
43
|
+
cacheFile(string): the common cacheFile to use (if any)
|
|
44
|
+
withShowProgress(boolean): True if progress should be shown
|
|
45
|
+
profile(boolean): True if timing / profiling information should be shown
|
|
46
|
+
debug(boolean): True if debugging information should be shown
|
|
47
|
+
errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
|
|
48
|
+
'''
|
|
49
|
+
self.mode=mode
|
|
50
|
+
self.cacheFile=cacheFile
|
|
51
|
+
self.profile=profile
|
|
52
|
+
self.withShowProgress=withShowProgress
|
|
53
|
+
self.debug=debug
|
|
54
|
+
self.errorDebug=errorDebug
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def getDefault(debug=False):
|
|
58
|
+
return StorageConfig.getSQL(debug)
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def getSQL(debug=False):
|
|
62
|
+
config=StorageConfig(mode=StoreMode.SQL,debug=debug)
|
|
63
|
+
config.tableName=None
|
|
64
|
+
return config
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def getJSON(debug=False):
|
|
68
|
+
config=StorageConfig(mode=StoreMode.JSON,debug=debug)
|
|
69
|
+
return config
|
|
70
|
+
|
|
71
|
+
@staticmethod
|
|
72
|
+
def getJsonAble(debug=False):
|
|
73
|
+
config=StorageConfig(mode=StoreMode.JSONABLE,debug=debug)
|
|
74
|
+
return config
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def getSPARQL(prefix,endpoint,host,debug=False):
|
|
78
|
+
config=StorageConfig(mode=StoreMode.SPARQL,debug=debug)
|
|
79
|
+
config.prefix=prefix
|
|
80
|
+
config.host=host
|
|
81
|
+
config.endpoint=endpoint
|
|
82
|
+
return config
|
|
83
|
+
|
|
84
|
+
@staticmethod
|
|
85
|
+
def getYaml(debug=False):
|
|
86
|
+
config=StorageConfig(mode=StoreMode.YAML,debug=debug)
|
|
87
|
+
return config
|
|
88
|
+
|
|
89
|
+
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
# part of https://github.com/WolfgangFahl/ProceedingsTitleParser
|
|
3
|
+
|
|
4
|
+
# Yaml persistence
|
|
5
|
+
import yaml
|
|
6
|
+
import io
|
|
7
|
+
import os
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class YamlAbleMixin(object):
|
|
11
|
+
""" allow reading and writing derived objects from a yaml file"""
|
|
12
|
+
debug = False
|
|
13
|
+
|
|
14
|
+
# read me from a yaml file
|
|
15
|
+
@staticmethod
|
|
16
|
+
def readYaml(name):
|
|
17
|
+
yamlFile = name
|
|
18
|
+
if not yamlFile.endswith(".yaml"):
|
|
19
|
+
yamlFile = yamlFile + ".yaml"
|
|
20
|
+
# is there a yamlFile for the given name
|
|
21
|
+
if os.path.isfile(yamlFile):
|
|
22
|
+
with io.open(yamlFile, 'r') as stream:
|
|
23
|
+
if YamlAbleMixin.debug:
|
|
24
|
+
print("reading %s" % (yamlFile))
|
|
25
|
+
result = yaml.load(stream, Loader=yaml.Loader)
|
|
26
|
+
if (YamlAbleMixin.debug):
|
|
27
|
+
print (result)
|
|
28
|
+
return result
|
|
29
|
+
else:
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
# write me to my yaml file
|
|
33
|
+
def writeYaml(self, name):
|
|
34
|
+
yamlFile = name
|
|
35
|
+
if not yamlFile.endswith(".yaml"):
|
|
36
|
+
yamlFile = yamlFile + ".yaml"
|
|
37
|
+
with io.open(yamlFile, 'w', encoding='utf-8') as stream:
|
|
38
|
+
yaml.dump(self, stream)
|
|
39
|
+
if YamlAbleMixin.debug:
|
|
40
|
+
print (yaml.dump(self))
|
|
@@ -3,7 +3,9 @@ README.md
|
|
|
3
3
|
setup.py
|
|
4
4
|
lodstorage/__init__.py
|
|
5
5
|
lodstorage/csv.py
|
|
6
|
+
lodstorage/entity.py
|
|
6
7
|
lodstorage/jsonable.py
|
|
8
|
+
lodstorage/jsonpicklemixin.py
|
|
7
9
|
lodstorage/lod.py
|
|
8
10
|
lodstorage/mwTable.py
|
|
9
11
|
lodstorage/plot.py
|
|
@@ -12,8 +14,10 @@ lodstorage/sample.py
|
|
|
12
14
|
lodstorage/schema.py
|
|
13
15
|
lodstorage/sparql.py
|
|
14
16
|
lodstorage/sql.py
|
|
17
|
+
lodstorage/storageconfig.py
|
|
15
18
|
lodstorage/tabulateCounter.py
|
|
16
19
|
lodstorage/uml.py
|
|
20
|
+
lodstorage/yamlablemixin.py
|
|
17
21
|
pylodstorage.egg-info/PKG-INFO
|
|
18
22
|
pylodstorage.egg-info/SOURCES.txt
|
|
19
23
|
pylodstorage.egg-info/dependency_links.txt
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|