pyLodStorage 0.3.4__tar.gz → 0.3.6__tar.gz

{pylodstorage-0.3.4/pylodstorage.egg-info → pylodstorage-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.3.4
+Version: 0.3.6
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.3.4 → pylodstorage-0.3.6}/lodstorage/query.py

@@ -580,12 +580,19 @@ class Endpoint(JSONAble):
                 "lang": "sparql",
                 "endpoint": "https://query.wikidata.org/sparql",
                 "website": "https://query.wikidata.org/",
+                "database": "blazegraph",
                 "method": "POST",
                 "prefixes": "PREFIX bd: <http://www.bigdata.com/rdf#>\nPREFIX cc: <http://creativecommons.org/ns#>"
             }
         ]
         return samples
     
+    @classmethod
+    def getDefault(cls):
+        endpointConf=Endpoint()
+        endpointConf.fromDict(Endpoint.getSamples()[0])
+        return endpointConf
+    
     def __init__(self):
         '''
         constructor for setting defaults

{pylodstorage-0.3.4 → pylodstorage-0.3.6}/lodstorage/trulytabular.py

@@ -5,7 +5,7 @@ Created on 2022-04-14
 '''
 import datetime
 from lodstorage.sparql import SPARQL
-from lodstorage.query import Query,QueryManager,YamlPath
+from lodstorage.query import Query,QueryManager,YamlPath,Endpoint
 from lodstorage.version import Version
 from pathlib import Path
 import os
@@ -74,23 +74,20 @@ class WikidataProperty():
         '''
         # the result dict
         wdProperties={}
-        valuesClause=""
-        for propertyLabel in propertyLabels:
-            valuesClause+=f'   "{propertyLabel}"@{lang}\n'
-        query="""
-# get the property for the given labels
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-PREFIX wikibase: <http://wikiba.se/ontology#>
-SELECT ?property ?propertyLabel WHERE {
-  VALUES ?propertyLabel {
-%s
-  }
-  ?property rdf:type wikibase:Property;
-  rdfs:label ?propertyLabel.
-  FILTER((LANG(?propertyLabel)) = "%s")
-}""" % (valuesClause,lang)
-        cls.addPropertiesForQuery(wdProperties,sparql,query)
+        if len(propertyLabels)>0:
+            valuesClause=""
+            for propertyLabel in propertyLabels:
+                valuesClause+=f'   "{propertyLabel}"@{lang}\n'
+            query=f"""# get the properties for the given labels
+{WikidataItem.getPrefixes()}
+SELECT ?property ?propertyLabel WHERE {{
+  VALUES ?propertyLabel {{
+{valuesClause}
+  }}
+  ?property rdf:type wikibase:Property;rdfs:label ?propertyLabel.
+  FILTER(LANG(?propertyLabel) = "{lang}")
+}}"""
+            cls.addPropertiesForQuery(wdProperties,sparql,query)
         return wdProperties
     
     @classmethod
@@ -105,23 +102,21 @@ SELECT ?property ?propertyLabel WHERE {
         '''
         # the result dict
         wdProperties={}
-        valuesClause=""
-        for propertyId in propertyIds:
-            valuesClause+=f'   wd:{propertyId}\n'
-        query="""
+        if len(propertyIds)>0:
+            valuesClause=""
+            for propertyId in propertyIds:
+                valuesClause+=f'   wd:{propertyId}\n'
+            query=f"""
 # get the property for the given property Ids
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-PREFIX wikibase: <http://wikiba.se/ontology#>
-SELECT ?property ?propertyLabel WHERE {
-  VALUES ?property {
-%s
-  }
-  ?property rdf:type wikibase:Property;
-  rdfs:label ?propertyLabel.
-  FILTER((LANG(?propertyLabel)) = "%s")
-}""" % (valuesClause,lang)
-        cls.addPropertiesForQuery(wdProperties,sparql,query)
+{WikidataItem.getPrefixes()}
+SELECT ?property ?propertyLabel WHERE {{
+  VALUES ?property {{
+{valuesClause}
+  }}
+  ?property rdf:type wikibase:Property;rdfs:label ?propertyLabel.
+  FILTER(LANG(?propertyLabel) = "{lang}")
+}}""" 
+            cls.addPropertiesForQuery(wdProperties,sparql,query)
         return wdProperties
         
     @classmethod    
@@ -160,6 +155,8 @@ class WikidataItem:
             sparql(SPARQL): the sparql access to use
         '''
         self.qid=qid
+        # numeric qid
+        self.qnumber=int(qid[1:])
         self.url=f"https://www.wikidata.org/wiki/{self.qid}"
         self.lang=lang
         self.sparql=sparql
@@ -195,13 +192,16 @@ class WikidataItem:
     
     @classmethod
     def getPrefixes(cls):
-        prefixes="""PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        # see also https://www.wikidata.org/wiki/EntitySchema:E49
+        prefixes="""PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX schema: <http://schema.org/>
 PREFIX wd: <http://www.wikidata.org/entity/>
 PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+PREFIX wikibase: <http://wikiba.se/ontology#>
 PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>"""
         return prefixes
-    
+
     @classmethod
     def getLabelAndDescription(cls,sparql:SPARQL, itemId:str,lang:str="en"):
         '''
@@ -223,9 +223,9 @@ WHERE
     wd:{itemId}
   }}
   ?item rdfs:label ?itemLabel.
-  FILTER (lang(?itemLabel) = "{lang}").
+  FILTER (LANG(?itemLabel) = "{lang}").
   ?item schema:description ?itemDescription.
-  FILTER((LANG(?itemDescription)) = "{lang}")
+  FILTER(LANG(?itemDescription) = "{lang}")
 }}""" 
         return sparql.getValues(query, ["itemLabel","itemDescription"])
         
@@ -247,17 +247,21 @@ WHERE
 # e.g. we'll find human=Q5 as the oldest type for the label "human" first
 # and then the newer ones such as "race in Warcraft"
 {cls.getPrefixes()}
-SELECT ?itemId ?item ?itemLabel ?itemDescription
+SELECT 
+  #?itemId 
+  ?item 
+  ?itemLabel 
+  ?itemDescription
 WHERE {{ 
   VALUES ?itemLabel {{
     {valuesClause}
   }}
-  BIND (xsd:integer(SUBSTR(STR(?item),33)) AS ?itemId)
+  #BIND (xsd:integer(SUBSTR(STR(?item),33)) AS ?itemId)
   ?item rdfs:label ?itemLabel. 
-  ?item schema:description ?itemDescription
+  ?item schema:description ?itemDescription.
   FILTER(LANG(?itemDescription)="{lang}")
-}} ORDER BY ?itemId""" 
-
+}} 
+#ORDER BY ?itemId""" 
         qLod=sparql.queryAsListOfDicts(query)
         items=[]
         for record in qLod:
@@ -269,7 +273,8 @@ WHERE {{
             item.varname=Variable.validVarName(item.qlabel)
             item.description=record["itemDescription"]
             items.append(item)
-        return items
+        sortedItems=sorted(items,key=lambda item: item.qnumber)
+        return sortedItems
         
 class TrulyTabular(object):
     '''
@@ -277,9 +282,8 @@ class TrulyTabular(object):
     
     checks "how tabular" a query based on a list of properties of an itemclass is
     '''
-    endpoint="https://query.wikidata.org/sparql"
 
-    def __init__(self, itemQid, propertyLabels:list=[],propertyIds:list=[],where:str=None,endpoint=None,method="POST",lang="en",debug=False):
+    def __init__(self, itemQid, propertyLabels:list=[],propertyIds:list=[],where:str=None,endpointConf=None,lang="en",debug=False):
         '''
         Constructor
         
@@ -292,10 +296,10 @@ class TrulyTabular(object):
         '''
         self.itemQid=itemQid
         self.debug=debug
-        if endpoint is None:
-            endpoint=TrulyTabular.endpoint
-        self.endpoint=endpoint
-        self.sparql=SPARQL(endpoint,method=method)
+        if endpointConf is None:
+            endpointConf=Endpoint.getDefault()
+        self.endpointConf=endpointConf
+        self.sparql=SPARQL(endpointConf.endpoint,method=self.endpointConf.method)
         self.sparql.debug=self.debug
         self.where=f"\n  {where}" if where is not None else ""
         self.lang=lang
@@ -329,6 +333,8 @@ WHERE
 }}"""
         try:
             count=self.sparql.getValue(query, "count")
+            # workaround https://github.com/ad-freiburg/qlever/issues/717
+            count=int(count)
         except Exception as ex:
             self.error=ex
             count=None
@@ -475,32 +481,41 @@ WHERE {{
             whereClause(str): an extra WhereClause to use
         '''
         if whereClause is None:
-            whereClause=f"?item wdt:P31 wd:{self.itemQid};";
-        else:
-            whereClause+=";"     
+            whereClause=f"?item wdt:P31 wd:{self.itemQid}";
+            if self.endpointConf.database!="qlever":
+                whereClause+=";?p ?id"
+        whereClause+="."  
         itemText=self.getItemText()
-        if self.endpoint=="https://query.wikidata.org/sparql":
-            optimizerHint='      hint:Query hint:optimizer "None".\n'
-        else:
-            optimizerHint=''
         sparqlQuery=f"""# get the most frequently used properties for
 # {itemText}
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX schema: <http://schema.org/>
-PREFIX wd: <http://www.wikidata.org/entity/>
-PREFIX wdt: <http://www.wikidata.org/prop/direct/>
-PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>
+{WikidataItem.getPrefixes()}
 SELECT ?prop ?propLabel ?count WHERE {{
-  {{
-    SELECT ?prop (COUNT(DISTINCT ?item) AS ?count) WHERE {{
-{optimizerHint}        
-      {whereClause}
-      ?p ?id.
+  {{"""
+        if self.endpointConf.database=="qlever":
+            sparqlQuery+=f"""
+    SELECT ?p (COUNT(DISTINCT ?item) AS ?count) WHERE {{"""
+        else:
+            sparqlQuery+=f"""
+    SELECT ?prop (COUNT(DISTINCT ?item) AS ?count) WHERE {{"""
+        if self.endpointConf.database=="blazegraph":
+            sparqlQuery+=f"""
+      hint:Query hint:optimizer "None"."""
+        sparqlQuery+=f"""
+      {whereClause}"""
+        if self.endpointConf.database=="qlever":
+            sparqlQuery+=f"""  
+      ?item ql:has-predicate ?p 
+    }} GROUP BY ?p
+  }}
+  ?prop wikibase:directClaim ?p."""
+        else:
+            sparqlQuery+=f"""
       ?prop wikibase:directClaim ?p.
     }}
     GROUP BY ?prop ?propLabel
-  }}
-  ?prop rdfs:label ?propLabel
+  }}"""
+        sparqlQuery+=f"""
+  ?prop rdfs:label ?propLabel.
   FILTER(LANG(?propLabel) = "{self.lang}")      
 }}
 ORDER BY DESC (?count)
@@ -525,16 +540,17 @@ ORDER BY DESC (?count)
         sparql=f"""
 # Count all {self.asText(long=True)} items
 # with the given {propertyLabel}({propertyId}) https://www.wikidata.org/wiki/Property:{propertyId} 
+{WikidataItem.getPrefixes()}
 SELECT ?item ?itemLabel (COUNT (?value) AS ?count)
 WHERE
 {{
   # instance of {self.item.qlabel}
   ?item wdt:P31 wd:{self.itemQid}.{self.where}
   ?item rdfs:label ?itemLabel.
-  filter (lang(?itemLabel) = "en").
+  FILTER (LANG(?itemLabel) = "{self.lang}").
   # {propertyLabel}
   ?item {wdProperty.getPredicate()} ?value.
-}} GROUP by ?item ?itemLabel
+}} GROUP BY ?item ?itemLabel
 """
         if asFrequency:
             freqDesc="frequencies"

{pylodstorage-0.3.4 → pylodstorage-0.3.6}/lodstorage/version.py

@@ -9,7 +9,7 @@ class Version(object):
     Version handling for pyLoDStorage
     '''
     name="pylodstorage"
-    version='0.3.4'
+    version='0.3.6'
     date = '2020-09-10'
-    updated = '2022-07-30'
+    updated = '2022-07-31'
     description='python List of Dict (Table) Storage library'

{pylodstorage-0.3.4 → pylodstorage-0.3.6/pylodstorage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.3.4
+Version: 0.3.6
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.3.4 → pylodstorage-0.3.6}/sampledata/endpoints.yaml

@@ -4,6 +4,7 @@
   lang: sparql
   endpoint: https://query.wikidata.org/sparql
   website: https://query.wikidata.org/
+  database: blazegraph
   method: POST
   prefixes: |
     PREFIX bd: <http://www.bigdata.com/rdf#>
@@ -37,10 +38,10 @@
     PREFIX wdv: <http://www.wikidata.org/value/>
     PREFIX wikibase: <http://wikiba.se/ontology#>
     PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
-
 'qlever-wikidata':
   lang: sparql
   method: GET
+  database: qlever
   endpoint: https://qlever.cs.uni-freiburg.de/api/wikidata
   website: https://qlever.cs.uni-freiburg.de/wikidata
   prefixes: |
@@ -75,10 +76,10 @@
     PREFIX wdv: <http://www.wikidata.org/value/>
     PREFIX wikibase: <http://wikiba.se/ontology#>
     PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
-
 'qlever-wikidata-proxy':
   lang: sparql
   method: GET
+  database: qlever
   endpoint: https://qlever.cs.uni-freiburg.de/api/wikidata-proxy
   website: https://qlever.cs.uni-freiburg.de/wikidata
   prefixes: |
@@ -117,6 +118,7 @@
     lang: sparql
     endpoint: https://wikidata.demo.openlinksw.com/sparql
     website: https://wikidata.demo.openlinksw.com/sparql
+    database: virtuoso
     prefixes: |
       PREFIX bd: <http://www.bigdata.com/rdf#>
       PREFIX cc: <http://creativecommons.org/ns#>