pyLodStorage 0.0.41__tar.gz → 0.0.43__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {pylodstorage-0.0.41/pylodstorage.egg-info → pylodstorage-0.0.43}/PKG-INFO +1 -1
  2. pylodstorage-0.0.43/lodstorage/entity.py +255 -0
  3. pylodstorage-0.0.43/lodstorage/jsonpicklemixin.py +60 -0
  4. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/lod.py +26 -0
  5. pylodstorage-0.0.43/lodstorage/storageconfig.py +89 -0
  6. pylodstorage-0.0.43/lodstorage/yamlablemixin.py +40 -0
  7. {pylodstorage-0.0.41 → pylodstorage-0.0.43/pylodstorage.egg-info}/PKG-INFO +1 -1
  8. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/SOURCES.txt +4 -0
  9. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/setup.py +1 -1
  10. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/LICENSE +0 -0
  11. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/README.md +0 -0
  12. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/__init__.py +0 -0
  13. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/csv.py +0 -0
  14. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/jsonable.py +0 -0
  15. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/mwTable.py +0 -0
  16. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/plot.py +0 -0
  17. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/query.py +0 -0
  18. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sample.py +0 -0
  19. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/schema.py +0 -0
  20. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sparql.py +0 -0
  21. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/sql.py +0 -0
  22. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/tabulateCounter.py +0 -0
  23. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/lodstorage/uml.py +0 -0
  24. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/dependency_links.txt +0 -0
  25. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/requires.txt +0 -0
  26. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/pylodstorage.egg-info/top_level.txt +0 -0
  27. {pylodstorage-0.0.41 → pylodstorage-0.0.43}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pylodstorage
3
- Version: 0.0.41
3
+ Version: 0.0.43
4
4
  Summary: python List of Dict (Table) Storage library
5
5
  Home-page: https://github.com/WolfgangFahl/pyLodStorage
6
6
  Author: Wolfgang Fahl
@@ -0,0 +1,255 @@
1
+ '''
2
+ Created on 2020-08-19
3
+
4
+ @author: wf
5
+ '''
6
+ from lodstorage.yamlablemixin import YamlAbleMixin
7
+ from lodstorage.jsonpicklemixin import JsonPickleMixin
8
+ from lodstorage.storageconfig import StorageConfig, StoreMode
9
+ from lodstorage.sparql import SPARQL
10
+ from lodstorage.sql import SQLDB
11
+ from lodstorage.lod import LOD
12
+ import os
13
+ import time
14
+
15
+ class EntityManager(YamlAbleMixin, JsonPickleMixin):
16
+ '''
17
+ generic entity manager
18
+ '''
19
+
20
+ def __init__(self,name,entityName,entityPluralName,config=None,debug=False):
21
+ '''
22
+ Constructor
23
+
24
+ Args:
25
+ name(string): name of this eventManager
26
+ entityName(string): entityType to be managed e.g. Country
27
+ entityPluralName(string): plural of the the entityType e.g. Countries
28
+ config(StorageConfig): the configuration to be used if None a default configuration will be used
29
+ debug(boolean): override debug setting when default of config is used via config=None
30
+ '''
31
+ self.name=name
32
+ self.entityName=entityName
33
+ self.entityPluralName=entityPluralName
34
+ if config is None:
35
+ config=StorageConfig.getDefault()
36
+ if config.tableName is None:
37
+ config.tableName=entityName
38
+ if debug:
39
+ config.debug=debug
40
+ self.config=config
41
+ cacheFile=self.getCacheFile(config=config,mode=config.mode)
42
+ self.showProgress ("Creating %smanager(%s) for %s using cache %s" % (self.entityName,config.mode,self.name,cacheFile))
43
+ if config.mode is StoreMode.SPARQL:
44
+ if config.endpoint is None:
45
+ raise Exception("no endpoint set for mode sparql")
46
+ self.endpoint=config.endpoint
47
+ self.sparql=SPARQL(config.endpoint,debug=config.debug,profile=config.profile)
48
+ elif config.mode is StoreMode.SQL:
49
+ self.executeMany=False # may be True when issues are fixed
50
+
51
+ def storeMode(self):
52
+ '''
53
+ return my store mode
54
+ '''
55
+ return self.config.mode
56
+
57
+ def showProgress(self,msg):
58
+ ''' display a progress message
59
+
60
+ Args:
61
+ msg(string): the message to display
62
+ '''
63
+ if self.config.withShowProgress:
64
+ print (msg,flush=True)
65
+
66
+ def getCacheFile(self,config=None,mode=StoreMode.SQL):
67
+ '''
68
+ get the cache file for this event manager
69
+ Args:
70
+ config(StorageConfig): if None get the cache for my mode
71
+ mode(StoreMode): the storeMode to use
72
+ '''
73
+ cachedir=EntityManager.getCachePath()
74
+ if config is not None and config.cacheFile is not None:
75
+ return config.cacheFile
76
+ ''' get the path to the file for my cached data '''
77
+ if mode is StoreMode.JSON:
78
+ cachepath="%s/%s-%s.%s" % (cachedir,self.name,"events",'json')
79
+ elif mode is StoreMode.SPARQL:
80
+ cachepath="%s %s" % ('SPARQL',config.endpoint)
81
+ elif mode is StoreMode.SQL:
82
+ cachepath="%s/%s.db" % (cachedir,config.tableName)
83
+ else:
84
+ cachepath="undefined cachepath for %s" % (mode)
85
+ return cachepath
86
+
87
+ def getSQLDB(self,cacheFile):
88
+ '''
89
+ get the SQL database for the given cacheFile
90
+
91
+ Args:
92
+ cacheFile(string): the file to get the SQL db from
93
+ '''
94
+ config=self.config
95
+ sqldb=self.sqldb=SQLDB(cacheFile,debug=config.debug,errorDebug=config.errorDebug)
96
+ return sqldb
97
+
98
+ def setNone(self,record,fields):
99
+ '''
100
+ make sure the given fields in the given record are set to none
101
+ Args:
102
+ record(dict): the record to work on
103
+ fields(list): the list of fields to set to None
104
+ '''
105
+ LOD.setNone(record,fields)
106
+
107
+ def isCached(self):
108
+ ''' check whether there is a file containing cached
109
+ data for me '''
110
+ result=False
111
+ config=self.config
112
+ mode=self.config.mode
113
+ if mode is StoreMode.JSON:
114
+ result=os.path.isfile(self.getCacheFile(config=self.config,mode=StoreMode.JSON))
115
+ elif mode is StoreMode.SPARQL:
116
+ # @FIXME - make abstract
117
+ query=config.prefix+"""
118
+ SELECT ?source (COUNT(?source) AS ?sourcecount)
119
+ WHERE {
120
+ ?event cr:Event_source ?source.
121
+ }
122
+ GROUP by ?source
123
+ """
124
+ sourceCountList=self.sparql.queryAsListOfDicts(query)
125
+ for sourceCount in sourceCountList:
126
+ source=sourceCount['source'];
127
+ recordCount=sourceCount['sourcecount']
128
+ if source==self.name and recordCount>100:
129
+ result=True
130
+ elif mode is StoreMode.SQL:
131
+ cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.SQL)
132
+ if os.path.isfile(cacheFile):
133
+ sqlQuery="SELECT COUNT(*) AS count FROM %s" % config.tableName
134
+ try:
135
+ sqlDB=self.getSQLDB(cacheFile)
136
+ countResult=sqlDB.query(sqlQuery)
137
+ count=countResult[0]['count']
138
+ result=count>100
139
+ except Exception as ex:
140
+ # e.g. sqlite3.OperationalError: no such table: Event_crossref
141
+ pass
142
+ else:
143
+ raise Exception("unsupported mode %s" % self.mode)
144
+ return result
145
+
146
+ def fromCache(self):
147
+ '''
148
+ get my entries from the cache
149
+
150
+ Returns:
151
+ the list of Dicts and as a side effect setting self.cacheFile
152
+ '''
153
+ if not self.isCached():
154
+ listOfDicts=self.getListOfDicts()
155
+ self.cacheFile=self.store(listOfDicts)
156
+ else:
157
+ # fromStore also sets self.cacheFile
158
+ listOfDicts=self.fromStore()
159
+ return listOfDicts
160
+
161
+ def fromStore(self,cacheFile=None):
162
+ '''
163
+ restore me from the store
164
+ Args:
165
+ cacheFile(String): the cacheFile to use if None use the preconfigured Cachefile
166
+ Returns:
167
+ list: list of dicts or JSON entitymanager
168
+ '''
169
+ startTime=time.time()
170
+ if cacheFile is None:
171
+ cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
172
+ self.cacheFile=cacheFile
173
+ self.showProgress("reading %s for %s from cache %s" % (self.entityPluralName,self.name,cacheFile))
174
+ JSONem=None
175
+ mode=self.config.mode
176
+ if mode is StoreMode.JSON:
177
+ JSONem=JsonPickleMixin.readJson(cacheFile)
178
+ elif mode is StoreMode.SPARQL:
179
+ # @FIXME make abstract
180
+ eventQuery="""
181
+ PREFIX cr: <http://cr.bitplan.com/>
182
+ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate ?url ?source WHERE {
183
+ OPTIONAL { ?event cr:Event_eventId ?eventId. }
184
+ OPTIONAL { ?event cr:Event_acronym ?acronym. }
185
+ OPTIONAL { ?event cr:Event_series ?series. }
186
+ OPTIONAL { ?event cr:Event_title ?title. }
187
+ OPTIONAL { ?event cr:Event_year ?year. }
188
+ OPTIONAL { ?event cr:Event_country ?country. }
189
+ OPTIONAL { ?event cr:Event_city ?city. }
190
+ OPTIONAL { ?event cr:Event_startDate ?startDate. }
191
+ OPTIONAL { ?event cr:Event_endDate ?endDate. }
192
+ OPTIONAL { ?event cr:Event_url ?url. }
193
+ ?event cr:Event_source ?source FILTER(?source='%s').
194
+ }
195
+ """ % self.name
196
+ listOfDicts=self.sparql.queryAsListOfDicts(eventQuery)
197
+ elif mode is StoreMode.SQL:
198
+ sqlQuery="SELECT * FROM %s" % self.config.tableName
199
+ sqlDB=self.getSQLDB(cacheFile)
200
+ listOfDicts=sqlDB.query(sqlQuery)
201
+ sqlDB.close()
202
+ pass
203
+ else:
204
+ raise Exception("unsupported store mode %s" % self.mode)
205
+
206
+ if JSONem is not None:
207
+ return JSONem
208
+ else:
209
+ self.showProgress("read %d %s from %s in %5.1f s" % (len(listOfDicts),self.entityPluralName,self.name,time.time()-startTime))
210
+ return listOfDicts
211
+
212
+ def store(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,sampleRecordCount=1):
213
+ '''
214
+ store my entities
215
+
216
+ Args:
217
+ listOfDicts(list): the list of dicts to store
218
+ limit(int): maximumn number of records to store
219
+ batchSize(int): size of batch for storing
220
+ cacheFile(string): the name of the storage e.g path to JSON or sqlite3 file
221
+ sampleRecordCount(int): the number of records to analyze for type information
222
+ '''
223
+ config=self.config
224
+ mode=config.mode
225
+ if mode is StoreMode.JSON:
226
+ if cacheFile is None:
227
+ cacheFile=self.getCacheFile(config=self.config,mode=StoreMode.JSON)
228
+ self.showProgress ("storing %d events for %s to cache %s" % (len(self.events),self.name,cacheFile))
229
+ self.writeJson(cacheFile)
230
+ elif mode is StoreMode.DGRAPH:
231
+ startTime=time.time()
232
+ self.showProgress ("storing %d %s for %s to %s" % (len(self.events),self.entityPluralName,self.name,self.mode))
233
+ self.dgraph.addData(listOfDicts,limit=limit,batchSize=batchSize)
234
+ self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
235
+ elif mode is StoreMode.SPARQL:
236
+ startTime=time.time()
237
+ # @ FIXME make abstract
238
+ self.showProgress ("storing %d events for %s to %s" % (len(self.events),self.name,self.mode))
239
+ entityType="cr:Event"
240
+ prefixes="PREFIX cr: <http://cr.bitplan.com/>"
241
+ primaryKey="eventId"
242
+ self.sparql.insertListOfDicts(listOfDicts, entityType, primaryKey, prefixes,limit=limit,batchSize=batchSize)
243
+ self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
244
+ elif mode is StoreMode.SQL:
245
+ startTime=time.time()
246
+ if cacheFile is None:
247
+ cacheFile=self.getCacheFile(config=self.config,mode=self.config.mode)
248
+ sqldb=self.getSQLDB(cacheFile)
249
+ self.showProgress ("storing %d %s for %s to %s:%s" % (len(listOfDicts),self.entityPluralName,self.name,config.mode,cacheFile))
250
+ entityInfo=sqldb.createTable(listOfDicts, config.tableName, "eventId",withDrop=True,sampleRecordCount=sampleRecordCount)
251
+ self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany)
252
+ self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
253
+ else:
254
+ raise Exception("unsupported store mode %s" % self.mode)
255
+ return cacheFile
@@ -0,0 +1,60 @@
1
+ # Json persistence
2
+ import jsonpickle
3
+ import os
4
+
5
+
6
+ class JsonPickleMixin(object):
7
+ '''
8
+ allow reading and writing derived objects from a jsonpickle file
9
+ '''
10
+ debug = False
11
+
12
+ # read me from a yaml file
13
+ @staticmethod
14
+ def readJson(name, postfix=".json"):
15
+ jsonFileName = name
16
+ if not name.endswith(postfix):
17
+ jsonFileName = name + postfix
18
+ # is there a jsonFile for the given name
19
+ if os.path.isfile(jsonFileName):
20
+ if JsonPickleMixin.debug:
21
+ print("reading %s" % (jsonFileName))
22
+ with open(jsonFileName) as jsonFile:
23
+ json = jsonFile.read()
24
+ result = jsonpickle.decode(json)
25
+ if (JsonPickleMixin.debug):
26
+ print (json)
27
+ print (result)
28
+ return result
29
+ else:
30
+ return None
31
+
32
+ def asJson(self)->str:
33
+ '''
34
+ convert me to JSON
35
+
36
+ Returns:
37
+ str: a JSON String with my JSON representation
38
+ '''
39
+ json = jsonpickle.encode(self)
40
+ return json
41
+
42
+ def writeJson(self, name:str, postfix:str=".json"):
43
+ '''
44
+ write me to the json file with the given name (optionally without postfix)
45
+
46
+ name(str): name of the file (optionally without ".json" postfix)
47
+ postfix(str): default file extension
48
+ '''
49
+ if not name.endswith(postfix):
50
+ jsonFileName = name + postfix
51
+ else:
52
+ jsonFileName = name
53
+ json = self.asJson()
54
+ if JsonPickleMixin.debug:
55
+ print("writing %s" % (jsonFileName))
56
+ print(json)
57
+ print(self)
58
+ jsonFile = open(jsonFileName, "w")
59
+ jsonFile.write(json)
60
+ jsonFile.close()
@@ -152,3 +152,29 @@ class LOD(object):
152
152
  return lookup
153
153
  else:
154
154
  return lookup,duplicates
155
+
156
+ @classmethod
157
+ def handleListTypes(cls,lod,doFilter=False,separator=","):
158
+ '''
159
+ handle list types in the given list of dicts
160
+
161
+ Args:
162
+ cls: this class
163
+ lod(list): a list of dicts
164
+ doFilter(bool): True if records containing lists value items should be filtered
165
+ separator(str): the separator to use when converting lists
166
+ '''
167
+ # see https://stackoverflow.com/a/1207485/1497139
168
+ for i in range(len(lod) - 1, -1, -1):
169
+ record=lod[i]
170
+ if isinstance(record,dict):
171
+ for key in record:
172
+ value=record[key]
173
+ if isinstance(value,list):
174
+ if doFilter:
175
+ del lod[i]
176
+ continue
177
+ else:
178
+ newValue=separator.join(value)
179
+ record[key]=newValue
180
+
@@ -0,0 +1,89 @@
1
+ '''
2
+ Created on 2020-08-29
3
+
4
+ @author: wf
5
+ '''
6
+ from enum import Enum
7
+ import os
8
+ from pathlib import Path
9
+
10
+ class StoreMode(Enum):
11
+ '''
12
+ possible supported storage modes
13
+ '''
14
+ JSON = 1
15
+ JSONABLE = 2
16
+ SQL = 3
17
+ SPARQL = 4
18
+ YAML = 5
19
+
20
+ class StorageConfig(object):
21
+ '''
22
+ a storage configuration
23
+ '''
24
+
25
+ @classmethod
26
+ def getCachePath(cls,name:str)->str:
27
+ '''
28
+ get the path to the default cache
29
+
30
+ Args:
31
+ name(str): the name of the cache to use
32
+ '''
33
+ home = str(Path.home())
34
+ cachedir=f"{home}/{name}"
35
+ return cachedir
36
+
37
+ def __init__(self, mode=StoreMode.SQL,cacheFile=None,withShowProgress=True,profile=True,debug=False,errorDebug=True):
38
+ '''
39
+ Constructor
40
+
41
+ Args:
42
+ mode(StoreMode): the storage mode e.g. sql
43
+ cacheFile(string): the common cacheFile to use (if any)
44
+ withShowProgress(boolean): True if progress should be shown
45
+ profile(boolean): True if timing / profiling information should be shown
46
+ debug(boolean): True if debugging information should be shown
47
+ errorDebug(boolean): True if debug info should be provided on errors (should not be used for production since it might reveal data)
48
+ '''
49
+ self.mode=mode
50
+ self.cacheFile=cacheFile
51
+ self.profile=profile
52
+ self.withShowProgress=withShowProgress
53
+ self.debug=debug
54
+ self.errorDebug=errorDebug
55
+
56
+ @staticmethod
57
+ def getDefault(debug=False):
58
+ return StorageConfig.getSQL(debug)
59
+
60
+ @staticmethod
61
+ def getSQL(debug=False):
62
+ config=StorageConfig(mode=StoreMode.SQL,debug=debug)
63
+ config.tableName=None
64
+ return config
65
+
66
+ @staticmethod
67
+ def getJSON(debug=False):
68
+ config=StorageConfig(mode=StoreMode.JSON,debug=debug)
69
+ return config
70
+
71
+ @staticmethod
72
+ def getJsonAble(debug=False):
73
+ config=StorageConfig(mode=StoreMode.JSONABLE,debug=debug)
74
+ return config
75
+
76
+ @staticmethod
77
+ def getSPARQL(prefix,endpoint,host,debug=False):
78
+ config=StorageConfig(mode=StoreMode.SPARQL,debug=debug)
79
+ config.prefix=prefix
80
+ config.host=host
81
+ config.endpoint=endpoint
82
+ return config
83
+
84
+ @staticmethod
85
+ def getYaml(debug=False):
86
+ config=StorageConfig(mode=StoreMode.YAML,debug=debug)
87
+ return config
88
+
89
+
@@ -0,0 +1,40 @@
1
+ #!/usr/bin/python3
2
+ # part of https://github.com/WolfgangFahl/ProceedingsTitleParser
3
+
4
+ # Yaml persistence
5
+ import yaml
6
+ import io
7
+ import os
8
+
9
+
10
+ class YamlAbleMixin(object):
11
+ """ allow reading and writing derived objects from a yaml file"""
12
+ debug = False
13
+
14
+ # read me from a yaml file
15
+ @staticmethod
16
+ def readYaml(name):
17
+ yamlFile = name
18
+ if not yamlFile.endswith(".yaml"):
19
+ yamlFile = yamlFile + ".yaml"
20
+ # is there a yamlFile for the given name
21
+ if os.path.isfile(yamlFile):
22
+ with io.open(yamlFile, 'r') as stream:
23
+ if YamlAbleMixin.debug:
24
+ print("reading %s" % (yamlFile))
25
+ result = yaml.load(stream, Loader=yaml.Loader)
26
+ if (YamlAbleMixin.debug):
27
+ print (result)
28
+ return result
29
+ else:
30
+ return None
31
+
32
+ # write me to my yaml file
33
+ def writeYaml(self, name):
34
+ yamlFile = name
35
+ if not yamlFile.endswith(".yaml"):
36
+ yamlFile = yamlFile + ".yaml"
37
+ with io.open(yamlFile, 'w', encoding='utf-8') as stream:
38
+ yaml.dump(self, stream)
39
+ if YamlAbleMixin.debug:
40
+ print (yaml.dump(self))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pylodstorage
3
- Version: 0.0.41
3
+ Version: 0.0.43
4
4
  Summary: python List of Dict (Table) Storage library
5
5
  Home-page: https://github.com/WolfgangFahl/pyLodStorage
6
6
  Author: Wolfgang Fahl
@@ -3,7 +3,9 @@ README.md
3
3
  setup.py
4
4
  lodstorage/__init__.py
5
5
  lodstorage/csv.py
6
+ lodstorage/entity.py
6
7
  lodstorage/jsonable.py
8
+ lodstorage/jsonpicklemixin.py
7
9
  lodstorage/lod.py
8
10
  lodstorage/mwTable.py
9
11
  lodstorage/plot.py
@@ -12,8 +14,10 @@ lodstorage/sample.py
12
14
  lodstorage/schema.py
13
15
  lodstorage/sparql.py
14
16
  lodstorage/sql.py
17
+ lodstorage/storageconfig.py
15
18
  lodstorage/tabulateCounter.py
16
19
  lodstorage/uml.py
20
+ lodstorage/yamlablemixin.py
17
21
  pylodstorage.egg-info/PKG-INFO
18
22
  pylodstorage.egg-info/SOURCES.txt
19
23
  pylodstorage.egg-info/dependency_links.txt
@@ -11,7 +11,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
11
11
 
12
12
  setup(
13
13
  name='pylodstorage',
14
- version='0.0.41',
14
+ version='0.0.43',
15
15
 
16
16
  packages=['lodstorage',],
17
17
  author='Wolfgang Fahl',
File without changes
File without changes
File without changes