pyLodStorage 0.4.8__tar.gz → 0.4.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pylodstorage-0.4.8/pylodstorage.egg-info → pylodstorage-0.4.10}/PKG-INFO +4 -1
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/README.md +2 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/entity.py +7 -6
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/query.py +12 -2
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/querymain.py +26 -20
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/sparql.py +22 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/trulytabular.py +20 -7
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/version.py +2 -2
- {pylodstorage-0.4.8 → pylodstorage-0.4.10/pylodstorage.egg-info}/PKG-INFO +4 -1
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/pylodstorage.egg-info/SOURCES.txt +1 -0
- pylodstorage-0.4.10/sampledata/dblp.yaml +13 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/endpoints.yaml +8 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/wikidata.yaml +14 -1
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/setup.py +2 -1
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/LICENSE +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/__init__.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/csv.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/jsonable.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/jsonpicklemixin.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/lod.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/mwTable.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/plot.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/sample.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/schema.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/sql.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/storageconfig.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/tabulateCounter.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/uml.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/xml.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/lodstorage/yamlablemixin.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/pylodstorage.egg-info/dependency_links.txt +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/pylodstorage.egg-info/entry_points.txt +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/pylodstorage.egg-info/requires.txt +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/pylodstorage.egg-info/top_level.txt +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/__init__.py +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/formats.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/got.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/queries.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/scholia.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/trulytabular.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/sampledata/wf.yaml +0 -0
- {pylodstorage-0.4.8 → pylodstorage-0.4.10}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pylodstorage
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.10
|
|
4
4
|
Summary: python List of Dict (Table) Storage library
|
|
5
5
|
Home-page: https://github.com/WolfgangFahl/pyLodStorage
|
|
6
6
|
Author: Wolfgang Fahl
|
|
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.7
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.8
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.9
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
19
|
Description-Content-Type: text/markdown
|
|
19
20
|
License-File: LICENSE
|
|
20
21
|
|
|
@@ -68,3 +69,5 @@ see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tree/master/tests)
|
|
|
68
69
|
### Authors
|
|
69
70
|
* [Wolfgang Fahl](http://www.bitplan.com/Wolfgang_Fahl)
|
|
70
71
|
* [Tim Holzheim](https://www.semantic-mediawiki.org/wiki/Tim_Holzheim)
|
|
72
|
+
|
|
73
|
+
|
|
@@ -290,7 +290,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
|
|
|
290
290
|
lod.append(entity.__dict__)
|
|
291
291
|
return lod
|
|
292
292
|
|
|
293
|
-
def store(self,limit=10000000,batchSize=250,append=False,fixNone=True,sampleRecordCount=-1)->str:
|
|
293
|
+
def store(self,limit=10000000,batchSize=250,append=False,fixNone=True,sampleRecordCount=-1,replace:bool=False)->str:
|
|
294
294
|
'''
|
|
295
295
|
store my list of dicts
|
|
296
296
|
|
|
@@ -300,14 +300,15 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
|
|
|
300
300
|
append(bool): True if records should be appended
|
|
301
301
|
fixNone(bool): if True make sure the dicts are filled with None references for each record
|
|
302
302
|
sampleRecordCount(int): the number of records to analyze for type information
|
|
303
|
+
replace(bool): if True allow replace for insert
|
|
303
304
|
|
|
304
305
|
Return:
|
|
305
306
|
str: The cachefile being used
|
|
306
307
|
'''
|
|
307
308
|
lod=self.getLoD()
|
|
308
|
-
return self.storeLoD(lod,limit=limit,batchSize=batchSize,append=append,fixNone=fixNone,sampleRecordCount=sampleRecordCount)
|
|
309
|
+
return self.storeLoD(lod,limit=limit,batchSize=batchSize,append=append,fixNone=fixNone,sampleRecordCount=sampleRecordCount,replace=replace)
|
|
309
310
|
|
|
310
|
-
def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,append=False,fixNone=True,sampleRecordCount=1)->str:
|
|
311
|
+
def storeLoD(self,listOfDicts,limit=10000000,batchSize=250,cacheFile=None,append=False,fixNone=True,sampleRecordCount=1,replace:bool=False)->str:
|
|
311
312
|
'''
|
|
312
313
|
store my entities
|
|
313
314
|
|
|
@@ -319,7 +320,7 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
|
|
|
319
320
|
append(bool): True if records should be appended
|
|
320
321
|
fixNone(bool): if True make sure the dicts are filled with None references for each record
|
|
321
322
|
sampleRecordCount(int): the number of records to analyze for type information
|
|
322
|
-
|
|
323
|
+
replace(bool): if True allow replace for insert
|
|
323
324
|
Return:
|
|
324
325
|
str: The cachefile being used
|
|
325
326
|
'''
|
|
@@ -358,8 +359,8 @@ SELECT ?eventId ?acronym ?series ?title ?year ?country ?city ?startDate ?endDate
|
|
|
358
359
|
withDrop=True
|
|
359
360
|
withCreate=True
|
|
360
361
|
entityInfo=self.initSQLDB(sqldb,listOfDicts,withCreate=withCreate,withDrop=withDrop,sampleRecordCount=sampleRecordCount)
|
|
361
|
-
self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany,fixNone=fixNone)
|
|
362
|
+
self.sqldb.store(listOfDicts, entityInfo,executeMany=self.executeMany,fixNone=fixNone,replace=replace)
|
|
362
363
|
self.showProgress ("store for %s done after %5.1f secs" % (self.name,time.time()-startTime))
|
|
363
364
|
else:
|
|
364
|
-
raise Exception("unsupported store mode
|
|
365
|
+
raise Exception(f"unsupported store mode {self.mode}")
|
|
365
366
|
return cacheFile
|
|
@@ -244,7 +244,7 @@ class QueryResultDocumentation():
|
|
|
244
244
|
class Query(object):
|
|
245
245
|
''' a Query e.g. for SPAQRL '''
|
|
246
246
|
|
|
247
|
-
def __init__(self,name:str,query:str,lang='sparql',endpoint:str=None,database:str="blazegraph",title:str=None,description:str=None,prefixes=None,tryItUrl:str=None,formats:list=None,debug=False):
|
|
247
|
+
def __init__(self,name:str,query:str,lang='sparql',endpoint:str=None,database:str="blazegraph",title:str=None,description:str=None,limit:int=None,prefixes=None,tryItUrl:str=None,formats:list=None,debug=False):
|
|
248
248
|
'''
|
|
249
249
|
constructor
|
|
250
250
|
Args:
|
|
@@ -255,6 +255,7 @@ class Query(object):
|
|
|
255
255
|
database(string): the type of database e.g. "blazegraph"
|
|
256
256
|
title(string): the header/title of the query
|
|
257
257
|
description(string): the description of the query
|
|
258
|
+
limit(int): the limit of the query default: None
|
|
258
259
|
prefixes(list): list of prefixes to be resolved
|
|
259
260
|
tryItUrl(str): the url of a "tryit" webpage
|
|
260
261
|
formats(list): key,value pairs of ValueFormatters to be applied
|
|
@@ -269,6 +270,7 @@ class Query(object):
|
|
|
269
270
|
|
|
270
271
|
self.title=title=name if title is None else title
|
|
271
272
|
self.description="" if description is None else description
|
|
273
|
+
self.limit=limit
|
|
272
274
|
self.prefixes=prefixes
|
|
273
275
|
self.debug=debug
|
|
274
276
|
self.formats=formats
|
|
@@ -596,6 +598,15 @@ class Endpoint(JSONAble):
|
|
|
596
598
|
"database": "blazegraph",
|
|
597
599
|
"method": "POST",
|
|
598
600
|
"prefixes": "PREFIX bd: <http://www.bigdata.com/rdf#>\nPREFIX cc: <http://creativecommons.org/ns#>"
|
|
601
|
+
},
|
|
602
|
+
{
|
|
603
|
+
"name": "dbis-jena",
|
|
604
|
+
"lang": "sparql",
|
|
605
|
+
"endpoint": "https://confident.dbis.rwth-aachen.de/jena/",
|
|
606
|
+
"website": "https://confident.dbis.rwth-aachen.de",
|
|
607
|
+
"auth": "BASIC",
|
|
608
|
+
"user": "secret",
|
|
609
|
+
"password": "#not public - example not usable for access#",
|
|
599
610
|
}
|
|
600
611
|
]
|
|
601
612
|
return samples
|
|
@@ -612,7 +623,6 @@ class Endpoint(JSONAble):
|
|
|
612
623
|
'''
|
|
613
624
|
self.method="POST"
|
|
614
625
|
self.lang="SPARQL"
|
|
615
|
-
|
|
616
626
|
|
|
617
627
|
def __str__(self):
|
|
618
628
|
'''
|
|
@@ -12,6 +12,7 @@ __updated__ = Version.updated
|
|
|
12
12
|
DEBUG = 0
|
|
13
13
|
|
|
14
14
|
import json
|
|
15
|
+
import re
|
|
15
16
|
import requests
|
|
16
17
|
import sys
|
|
17
18
|
import traceback
|
|
@@ -21,12 +22,11 @@ from argparse import ArgumentParser
|
|
|
21
22
|
from argparse import RawDescriptionHelpFormatter
|
|
22
23
|
|
|
23
24
|
from lodstorage.csv import CSV
|
|
24
|
-
from lodstorage.query import Query,QueryManager, EndpointManager, Format, ValueFormatter
|
|
25
|
+
from lodstorage.query import Query,QueryManager, Endpoint, EndpointManager, Format, ValueFormatter
|
|
25
26
|
from lodstorage.sparql import SPARQL
|
|
26
27
|
from lodstorage.sql import SQLDB
|
|
27
28
|
from lodstorage.xml import Lod2Xml
|
|
28
29
|
|
|
29
|
-
|
|
30
30
|
class QueryMain:
|
|
31
31
|
'''
|
|
32
32
|
Commandline handler
|
|
@@ -45,7 +45,6 @@ class QueryMain:
|
|
|
45
45
|
qm=QueryManager(lang=args.language,debug=debug,queriesPath=args.queriesPath)
|
|
46
46
|
query=None
|
|
47
47
|
queryCode=args.query
|
|
48
|
-
endpointConf=None
|
|
49
48
|
formats=None
|
|
50
49
|
# preload ValueFormatter
|
|
51
50
|
ValueFormatter.getFormats(args.formatsPath)
|
|
@@ -64,6 +63,8 @@ class QueryMain:
|
|
|
64
63
|
if args.queryName not in qm.queriesByName:
|
|
65
64
|
raise Exception(f"named query {args.queryName} not available")
|
|
66
65
|
query=qm.queriesByName[args.queryName]
|
|
66
|
+
if query.limit is None and args.limit is not None:
|
|
67
|
+
query.limit=args.limit
|
|
67
68
|
formats=query.formats
|
|
68
69
|
queryCode=query.query
|
|
69
70
|
if debug or args.showQuery:
|
|
@@ -79,24 +80,27 @@ class QueryMain:
|
|
|
79
80
|
if queryCode:
|
|
80
81
|
if debug or args.showQuery:
|
|
81
82
|
print(f"{args.language}:\n{queryCode}")
|
|
83
|
+
endpointConf=Endpoint()
|
|
84
|
+
endpointConf.method='POST'
|
|
82
85
|
if args.endpointName:
|
|
83
86
|
endpointConf=endpoints.get(args.endpointName)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
87
|
+
query.tryItUrl=endpointConf.website
|
|
88
|
+
query.database=endpointConf.database
|
|
89
|
+
else:
|
|
90
|
+
endpointConf.endpoint=query.endpoint
|
|
91
|
+
if args.method:
|
|
92
|
+
endpointConf.method=args.method
|
|
93
|
+
if query.limit:
|
|
94
|
+
if "limit" in queryCode or "LIMIT" in queryCode:
|
|
95
|
+
queryCode=re.sub(r"(limit|LIMIT)\s+(\d+)", f"LIMIT {query.limit}", queryCode)
|
|
91
96
|
else:
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
sparql=SPARQL(endPointUrl,method=method)
|
|
97
|
+
queryCode+=f"\nLIMIT {query.limit}"
|
|
98
|
+
if args.language=="sparql":
|
|
99
|
+
sparql=SPARQL.fromEndpointConf(endpointConf)
|
|
96
100
|
if args.prefixes and endpointConf is not None:
|
|
97
101
|
queryCode = f"{endpointConf.prefixes}\n{queryCode}"
|
|
98
102
|
if args.raw:
|
|
99
|
-
qres = cls.rawQuery(
|
|
103
|
+
qres = cls.rawQuery(endpointConf, query=query.query, resultFormat=args.format, mimeType=args.mimeType)
|
|
100
104
|
print(qres)
|
|
101
105
|
return
|
|
102
106
|
if "wikidata" in args.endpointName and formats is None:
|
|
@@ -126,12 +130,12 @@ class QueryMain:
|
|
|
126
130
|
raise Exception(f"format {args.format} not supported yet")
|
|
127
131
|
|
|
128
132
|
@staticmethod
|
|
129
|
-
def rawQuery(
|
|
133
|
+
def rawQuery(endpointConf, query, resultFormat, mimeType):
|
|
130
134
|
"""
|
|
131
135
|
returns raw result of the endpoint
|
|
132
136
|
|
|
133
137
|
Args:
|
|
134
|
-
|
|
138
|
+
endpointConf: EndPoint
|
|
135
139
|
query(str): query
|
|
136
140
|
resultFormat(str): format of the result
|
|
137
141
|
mimeType(str): mimeType
|
|
@@ -150,8 +154,9 @@ class QueryMain:
|
|
|
150
154
|
}
|
|
151
155
|
else:
|
|
152
156
|
headers={}
|
|
153
|
-
|
|
154
|
-
|
|
157
|
+
endpoint=endpointConf.endpoint
|
|
158
|
+
method=endpointConf.method
|
|
159
|
+
response = requests.request(method, endpoint, headers=headers, data=payload, params=params)
|
|
155
160
|
return response.text
|
|
156
161
|
|
|
157
162
|
def mainSQL(argv=None):
|
|
@@ -184,7 +189,7 @@ def main(argv=None,lang=None): # IGNORE:C0111
|
|
|
184
189
|
program_license = '''%s
|
|
185
190
|
|
|
186
191
|
Created by %s on %s.
|
|
187
|
-
Copyright 2020-
|
|
192
|
+
Copyright 2020-2023 Wolfgang Fahl. All rights reserved.
|
|
188
193
|
|
|
189
194
|
Licensed under the Apache License 2.0
|
|
190
195
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
@@ -205,6 +210,7 @@ USAGE
|
|
|
205
210
|
parser.add_argument('--method',help="method to be used for SPARQL queries")
|
|
206
211
|
parser.add_argument('-f','--format', type=Format, choices=list(Format))
|
|
207
212
|
parser.add_argument('-li','--list',action="store_true",help="show the list of available queries")
|
|
213
|
+
parser.add_argument('--limit',type=int,default=None,help="set limit parameter of query")
|
|
208
214
|
parser.add_argument('-le','--listEndpoints',action="store_true",help="show the list of available endpoints")
|
|
209
215
|
parser.add_argument("-m", "--mimeType",help="MIME-type to use for the raw query")
|
|
210
216
|
parser.add_argument("-p", "--prefixes",action="store_true",help="add predefined prefixes for endpoint")
|
|
@@ -47,6 +47,24 @@ class SPARQL(object):
|
|
|
47
47
|
self.sparql=SPARQLWrapper2(url)
|
|
48
48
|
self.method=method
|
|
49
49
|
self.sparql.agent=agent
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def fromEndpointConf(cls,endpointConf)->"SPARQL":
|
|
53
|
+
"""
|
|
54
|
+
create a SPARQL endpoint from the given EndpointConfiguration
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
endpointConf(Endpoint): the endpoint configuration to be used
|
|
58
|
+
"""
|
|
59
|
+
sparql=SPARQL(url=endpointConf.endpoint,method=endpointConf.method)
|
|
60
|
+
if hasattr(endpointConf, "auth"):
|
|
61
|
+
authMethod=None
|
|
62
|
+
if endpointConf.auth=="BASIC":
|
|
63
|
+
authMethod=BASIC
|
|
64
|
+
elif endpointConf.auth=="DIGEST":
|
|
65
|
+
authMethod=DIGEST
|
|
66
|
+
sparql.addAuthentication(endpointConf.user,endpointConf.passwd,method=authMethod)
|
|
67
|
+
return sparql
|
|
50
68
|
|
|
51
69
|
def addAuthentication(self, username:str, password:str, method: Union[BASIC, DIGEST] = BASIC):
|
|
52
70
|
"""
|
|
@@ -91,6 +109,10 @@ class SPARQL(object):
|
|
|
91
109
|
def getValues(self,sparqlQuery:str,attrList:list):
|
|
92
110
|
'''
|
|
93
111
|
get Values for the given sparlQuery and attribute list
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
sparqlQuery(str): the query which did not return any values
|
|
115
|
+
attrList(list): the list of attributes
|
|
94
116
|
'''
|
|
95
117
|
if self.debug:
|
|
96
118
|
print(sparqlQuery)
|
|
@@ -153,7 +153,7 @@ class WikidataItem:
|
|
|
153
153
|
'''
|
|
154
154
|
a wikidata Item
|
|
155
155
|
'''
|
|
156
|
-
def __init__(self,qid:str,lang:str="en",sparql:SPARQL=None):
|
|
156
|
+
def __init__(self,qid:str,lang:str="en",sparql:SPARQL=None,debug:bool=False):
|
|
157
157
|
'''
|
|
158
158
|
construct me with the given item id, language and optional SPARQL access
|
|
159
159
|
|
|
@@ -161,6 +161,7 @@ class WikidataItem:
|
|
|
161
161
|
qid(str): the item Id
|
|
162
162
|
lang(str): the language to use
|
|
163
163
|
sparql(SPARQL): the sparql access to use
|
|
164
|
+
debug(bool): if True switch on debugging
|
|
164
165
|
'''
|
|
165
166
|
self.qid=qid
|
|
166
167
|
# numeric qid
|
|
@@ -169,7 +170,7 @@ class WikidataItem:
|
|
|
169
170
|
self.lang=lang
|
|
170
171
|
self.sparql=sparql
|
|
171
172
|
if sparql is not None:
|
|
172
|
-
self.qlabel,self.description=WikidataItem.getLabelAndDescription(sparql, self.qid, self.lang)
|
|
173
|
+
self.qlabel,self.description=WikidataItem.getLabelAndDescription(sparql, self.qid, self.lang,debug=debug)
|
|
173
174
|
self.varname=Variable.validVarName(self.qlabel)
|
|
174
175
|
self.itemVarname=f"{self.varname}Item"
|
|
175
176
|
self.labelVarname=f"{self.varname}"
|
|
@@ -243,13 +244,14 @@ class WikidataItem:
|
|
|
243
244
|
return sparql
|
|
244
245
|
|
|
245
246
|
@classmethod
|
|
246
|
-
def getLabelAndDescription(cls,sparql:SPARQL, itemId:str,lang:str="en"):
|
|
247
|
+
def getLabelAndDescription(cls,sparql:SPARQL, itemId:str,lang:str="en",debug:bool=False):
|
|
247
248
|
'''
|
|
248
249
|
get the label for the given item and language
|
|
249
250
|
|
|
250
251
|
Args:
|
|
251
252
|
itemId(str): the wikidata Q/P id
|
|
252
253
|
lang(str): the language of the label
|
|
254
|
+
debug(bool): if True output debug information
|
|
253
255
|
|
|
254
256
|
Returns:
|
|
255
257
|
(str,str): the label and description as a tuple
|
|
@@ -267,10 +269,20 @@ WHERE
|
|
|
267
269
|
?item schema:description ?itemDescription.
|
|
268
270
|
FILTER(LANG(?itemDescription) = "{lang}")
|
|
269
271
|
}}"""
|
|
270
|
-
|
|
272
|
+
try:
|
|
273
|
+
if debug:
|
|
274
|
+
msg=f"getLabelAndDescription for wikidata Item {itemId} with query:\n{query}"
|
|
275
|
+
print(msg)
|
|
276
|
+
labelAndDescription=sparql.getValues(query, ["itemLabel","itemDescription"])
|
|
277
|
+
except Exception as ex:
|
|
278
|
+
msg=f"getLabelAndDescription failed for wikidata Item {itemId}:{str(ex)}"
|
|
279
|
+
if debug:
|
|
280
|
+
print(msg)
|
|
281
|
+
raise Exception(msg)
|
|
282
|
+
return labelAndDescription
|
|
271
283
|
|
|
272
284
|
@classmethod
|
|
273
|
-
def getItemsByLabel(cls,sparql:SPARQL,itemLabel:str,lang:str="en")->list:
|
|
285
|
+
def getItemsByLabel(cls,sparql:SPARQL,itemLabel:str,lang:str="en",debug:bool=False)->list:
|
|
274
286
|
'''
|
|
275
287
|
get a Wikidata items by the given label
|
|
276
288
|
|
|
@@ -278,6 +290,7 @@ WHERE
|
|
|
278
290
|
sparql(SPARQL): the SPARQL endpoint to use
|
|
279
291
|
itemLabel(str): the label of the items
|
|
280
292
|
lang(str): the language of the label
|
|
293
|
+
debug(bool): if True show debugging information
|
|
281
294
|
|
|
282
295
|
Returns:
|
|
283
296
|
a list of potential items
|
|
@@ -307,7 +320,7 @@ WHERE {{
|
|
|
307
320
|
for record in qLod:
|
|
308
321
|
url=record["item"]
|
|
309
322
|
qid=re.sub(r"http://www.wikidata.org/entity/(.*)",r"\1",url)
|
|
310
|
-
item=WikidataItem(qid)
|
|
323
|
+
item=WikidataItem(qid,debug=debug)
|
|
311
324
|
item.url=url
|
|
312
325
|
item.qlabel=record["itemLabel"]
|
|
313
326
|
item.varname=Variable.validVarName(item.qlabel)
|
|
@@ -345,7 +358,7 @@ class TrulyTabular(object):
|
|
|
345
358
|
self.subclassPredicate=subclassPredicate
|
|
346
359
|
self.where=f"\n {where}" if where is not None else ""
|
|
347
360
|
self.lang=lang
|
|
348
|
-
self.item=WikidataItem(itemQid,sparql=self.sparql,lang=lang)
|
|
361
|
+
self.item=WikidataItem(itemQid,sparql=self.sparql,lang=lang,debug=self.debug)
|
|
349
362
|
self.queryManager=TrulyTabular.getQueryManager(debug=self.debug)
|
|
350
363
|
self.properties=WikidataProperty.getPropertiesByIds(self.sparql,propertyIds,lang)
|
|
351
364
|
self.properties.update(WikidataProperty.getPropertiesByLabels(self.sparql, propertyLabels, lang))
|
|
@@ -9,7 +9,7 @@ class Version(object):
|
|
|
9
9
|
Version handling for pyLoDStorage
|
|
10
10
|
'''
|
|
11
11
|
name="pylodstorage"
|
|
12
|
-
version='0.4.
|
|
12
|
+
version='0.4.10'
|
|
13
13
|
date = '2020-09-10'
|
|
14
|
-
updated = '
|
|
14
|
+
updated = '2023-03-21'
|
|
15
15
|
description='python List of Dict (Table) Storage library'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pylodstorage
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.10
|
|
4
4
|
Summary: python List of Dict (Table) Storage library
|
|
5
5
|
Home-page: https://github.com/WolfgangFahl/pyLodStorage
|
|
6
6
|
Author: Wolfgang Fahl
|
|
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.7
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.8
|
|
16
16
|
Classifier: Programming Language :: Python :: 3.9
|
|
17
17
|
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
19
|
Description-Content-Type: text/markdown
|
|
19
20
|
License-File: LICENSE
|
|
20
21
|
|
|
@@ -68,3 +69,5 @@ see [test cases](https://github.com/WolfgangFahl/pyLoDStorage/tree/master/tests)
|
|
|
68
69
|
### Authors
|
|
69
70
|
* [Wolfgang Fahl](http://www.bitplan.com/Wolfgang_Fahl)
|
|
70
71
|
* [Tim Holzheim](https://www.semantic-mediawiki.org/wiki/Tim_Holzheim)
|
|
72
|
+
|
|
73
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Sample Queries for dblp
|
|
3
|
+
#
|
|
4
|
+
# WF 2023-01-22
|
|
5
|
+
'PredicatesByFrequency':
|
|
6
|
+
title: All available dblp predicates by frequency
|
|
7
|
+
description: All available dblp predicates by frequency with percentage
|
|
8
|
+
sparql: |
|
|
9
|
+
SELECT ?p ?count ?percent WHERE {
|
|
10
|
+
{ SELECT ?p (COUNT(?p) AS ?count) WHERE { ?s ?p ?o } GROUP BY ?p }
|
|
11
|
+
BIND(100 * ?count / SUM(?count) AS ?percent)
|
|
12
|
+
}
|
|
13
|
+
ORDER BY DESC(?count)
|
|
@@ -76,6 +76,14 @@
|
|
|
76
76
|
PREFIX wdv: <http://www.wikidata.org/value/>
|
|
77
77
|
PREFIX wikibase: <http://wikiba.se/ontology#>
|
|
78
78
|
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
|
79
|
+
'qlever-dblp':
|
|
80
|
+
lang: sparql
|
|
81
|
+
method: POST
|
|
82
|
+
database: qlever
|
|
83
|
+
endpoint: https://qlever.cs.uni-freiburg.de/api/dblp
|
|
84
|
+
website: https://qlever.cs.uni-freiburg.de/dblp
|
|
85
|
+
prefixes: |
|
|
86
|
+
|
|
79
87
|
'wikidata-openlinksw':
|
|
80
88
|
lang: sparql
|
|
81
89
|
endpoint: https://wikidata.demo.openlinksw.com/sparql
|
|
@@ -126,6 +126,19 @@
|
|
|
126
126
|
?item rdfs:label ?itemLabel.
|
|
127
127
|
filter (lang(?itemLabel) = "en").
|
|
128
128
|
}
|
|
129
|
+
'UniversityLogos':
|
|
130
|
+
sparql: |
|
|
131
|
+
# University Logos
|
|
132
|
+
# WF 2023-03-11
|
|
133
|
+
SELECT ?item ?itemLabel ?logo
|
|
134
|
+
WHERE
|
|
135
|
+
{
|
|
136
|
+
?item wdt:P31 wd:Q3918.
|
|
137
|
+
?item rdfs:label ?itemLabel.
|
|
138
|
+
FILTER(LANG(?itemLabel)="en")
|
|
139
|
+
?item wdt:P154 ?logo
|
|
140
|
+
}
|
|
141
|
+
ORDER BY ?itemLabel
|
|
129
142
|
'MachadoDeAssis':
|
|
130
143
|
# https://stackoverflow.com/questions/71809883/sparql-query-dates-json-results-in-the-same-format-that-in-web-portal
|
|
131
144
|
sparql: |
|
|
@@ -134,7 +147,7 @@
|
|
|
134
147
|
PREFIX wdt: <http://www.wikidata.org/prop/direct/>
|
|
135
148
|
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
|
136
149
|
SELECT ?author ?authorLabel ?birthDate
|
|
137
|
-
WHERE
|
|
150
|
+
WHERE
|
|
138
151
|
{
|
|
139
152
|
VALUES ?authorLabel {
|
|
140
153
|
"Machado de Assis"@pt
|
|
@@ -39,7 +39,8 @@ setup(
|
|
|
39
39
|
'Programming Language :: Python :: 3.7',
|
|
40
40
|
'Programming Language :: Python :: 3.8',
|
|
41
41
|
'Programming Language :: Python :: 3.9',
|
|
42
|
-
'Programming Language :: Python :: 3.10'
|
|
42
|
+
'Programming Language :: Python :: 3.10',
|
|
43
|
+
'Programming Language :: Python :: 3.11'
|
|
43
44
|
],
|
|
44
45
|
entry_points={
|
|
45
46
|
'console_scripts': [
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|