pyLodStorage 0.2.14__tar.gz → 0.3.1__tar.gz

{pylodstorage-0.2.14/pylodstorage.egg-info → pylodstorage-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.2.14
+Version: 0.3.1
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.2.14 → pylodstorage-0.3.1}/lodstorage/trulytabular.py

@@ -6,11 +6,13 @@ Created on 2022-04-14
 import datetime
 from lodstorage.sparql import SPARQL
 from lodstorage.query import Query,QueryManager,YamlPath
+from lodstorage.version import Version
 from pathlib import Path
 import os
 import textwrap
 import re
 
+
 class Variable:
     '''
     Variable e.g. name handling
@@ -192,7 +194,16 @@ class WikidataItem:
         return text
     
     @classmethod
-    def getLabelAndDescription(self,sparql:SPARQL, itemId:str,lang:str="en"):
+    def getPrefixes(cls):
+        prefixes="""PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX schema: <http://schema.org/>
+PREFIX wd: <http://www.wikidata.org/entity/>
+PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>"""
+        return prefixes
+    
+    @classmethod
+    def getLabelAndDescription(cls,sparql:SPARQL, itemId:str,lang:str="en"):
         '''
         get  the label for the given item and language
         
@@ -203,23 +214,19 @@ class WikidataItem:
         Returns:
             (str,str): the label and description as a tuple
         '''
-        query="""
-# get the label for the given item
-PREFIX wd: <http://www.wikidata.org/entity/>
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
-PREFIX schema: <http://schema.org/>
+        query=f"""# get the label for the given item
+{cls.getPrefixes()}        
 SELECT ?itemLabel ?itemDescription
 WHERE
-{
-  VALUES ?item {
-    wd:%s
-  }
+{{
+  VALUES ?item {{
+    wd:{itemId}
+  }}
   ?item rdfs:label ?itemLabel.
-  filter (lang(?itemLabel) = "%s").
+  FILTER (lang(?itemLabel) = "{lang}").
   ?item schema:description ?itemDescription.
-  FILTER((LANG(?itemDescription)) = "%s")
-}""" % (itemId,lang,lang)
+  FILTER((LANG(?itemDescription)) = "{lang}")
+}}""" 
         return sparql.getValues(query, ["itemLabel","itemDescription"])
         
     @classmethod
@@ -236,23 +243,20 @@ WHERE
             a list of potential items
         '''
         valuesClause=f'   "{itemLabel}"@{lang}\n'
-        query="""# get the items that have the given label in the given language
+        query=f"""# get the items that have the given label in the given language
 # e.g. we'll find human=Q5 as the oldest type for the label "human" first
 # and then the newer ones such as "race in Warcraft"
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-PREFIX schema: <http://schema.org/>
-PREFIX wd: <http://www.wikidata.org/entity/>
-PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>
+{cls.getPrefixes()}
 SELECT ?itemId ?item ?itemLabel ?itemDescription
-WHERE { 
-  VALUES ?itemLabel {
-%s
-  }
+WHERE {{ 
+  VALUES ?itemLabel {{
+    {valuesClause}
+  }}
   BIND (xsd:integer(SUBSTR(STR(?item),33)) AS ?itemId)
   ?item rdfs:label ?itemLabel. 
   ?item schema:description ?itemDescription
-  FILTER(LANG(?itemDescription)="%s")
-} ORDER BY ?itemId""" % (valuesClause,lang)
+  FILTER(LANG(?itemDescription)="{lang}")
+}} ORDER BY ?itemId""" 
 
         qLod=sparql.queryAsListOfDicts(query)
         items=[]
@@ -316,8 +320,7 @@ class TrulyTabular(object):
         itemText=self.getItemText()
         query=f"""# Count all items with the given type
 # {itemText}
-PREFIX wd: <http://www.wikidata.org/entity/>
-PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+{WikidataItem.getPrefixes()}
 SELECT (COUNT (DISTINCT ?item) AS ?count)
 WHERE
 {{
@@ -348,7 +351,7 @@ WHERE
     def getItemText(self):
         # leads to 405 Method not allowed in SPARQLWrapper under certain circumstances
         # itemText=self.asText(long=True)
-        itemText=self.itemQid
+        itemText=f"{self.itemQid}:{self.item.qlabel}"
         return itemText
     
     @classmethod
@@ -368,43 +371,83 @@ WHERE
                 return qm
         return None
     
-    
-    def generateSparqlQuery(self,lang:str='en',naive:bool=True)->str:
+    def generateSparqlQuery(self,genMap:dict,listSeparator:str="⇹",naive:bool=True,lang:str='en')->str:
         '''
         generate a SPARQL Query
+        
+        Args:
+            genMap(dict): a dictionary of generation items aggregates/ignores/labels
+            listSeparator(str): the symbole to use as a list separator for GROUP_CONCAT
+            naive(bool): if True - generate a naive straight forward SPARQL query
+                if False generate a proper truly tabular aggregate query
+            lang(str): the language to generate for
+            
+        Returns:
+            str: the generated SPARQL Query
         '''
+        # The Wikidata item to generate the query for
         item=self.item
+        # the name of this script
         script=Path(__file__).name
+        # the mode of generation
         naiveText="naive" if naive else "aggregate"
+        # start with th preamble and PREFIX section
+        # select the item and it's label
         sparqlQuery=f"""# truly tabular {naiveText} query for 
 # {item.qid}:{item.qlabel}
-# generated by {script} on {self.isodate}
-PREFIX wd: <http://www.wikidata.org/entity/>
-PREFIX wdt: <http://www.wikidata.org/prop/direct/>
-PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+# generated by {script} version {Version.version} on {self.isodate}
+{WikidataItem.getPrefixes()}
 SELECT ?{item.varname} ?{item.varname}Label"""
+        # loop over all properties
         for wdProp in self.properties.values():
             if naive:
                 sparqlQuery+=f"\n  ?{wdProp.varname}"
             else:
-                sparqlQuery+=f"\n  (SAMPLE (?{wdProp.varname}) AS ?{wdProp.varname})"
+                if wdProp.pid in genMap:
+                    genList=genMap[wdProp.pid]
+                    for aggregate in genList:
+                        if not aggregate in ["ignore","label"]:
+                            distinct=""
+                            if aggregate=="list": 
+                                aggregateFunc="GROUP_CONCAT"
+                                aggregateParam=f';SEPARATOR="{listSeparator}"'
+                                distinct="DISTINCT "
+                            else:
+                                if aggregate=="count":
+                                    distinct="DISTINCT "
+                                aggregateFunc=aggregate.upper()
+                                aggregateParam=""
+                            sparqlQuery+=f"\n  ({aggregateFunc} ({distinct}?{wdProp.varname}{aggregateParam}) AS ?{wdProp.varname}_{aggregate})"
+                        elif aggregate=="label":
+                            sparqlQuery+=f"\n  ?{wdProp.varname}Label"
         sparqlQuery+=f"""
 WHERE {{
   # instanceof {item.qid}:{item.qlabel}
   ?{item.varname} wdt:P31 wd:{item.qid}.
   # label
-  ?{item.varname} rdfs:label ?{item.varname}Label  
+  ?{item.varname} rdfs:label ?{item.varname}Label.  
   FILTER (LANG(?{item.varname}Label) = "{lang}").
 """
         for wdProp in self.properties.values():
-                sparqlQuery+=f"""  # {wdProp}
-  OPTIONAL {{ ?{item.varname} wdt:{wdProp.pid} ?{wdProp.varname}. }}
-"""                
+            sparqlQuery+=f"""  # {wdProp}
+  OPTIONAL {{ 
+    ?{item.varname} wdt:{wdProp.pid} ?{wdProp.varname}. """
+            if wdProp.pid in genMap:
+                genList=genMap[wdProp.pid]
+                if "label" in genList:
+                    sparqlQuery+=f"""\n    ?{wdProp.varname} rdfs:label ?{wdProp.varname}Label."""
+                    sparqlQuery+=f"""\n    FILTER (LANG(?{wdProp.varname}Label) = "{lang}")."""
+            sparqlQuery+="\n  }\n"
         # close where Clause
         sparqlQuery+="""}\n"""
         # optionally add Aggregate
         if not naive:
             sparqlQuery+=f"GROUP BY ?{item.varname} ?{item.varname}Label"
+            for wdProp in self.properties.values():
+                if wdProp.pid in genMap:
+                    genList=genMap[wdProp.pid]
+                    if "ignore" in genList:
+                        sparqlQuery+=f"\nHAVING (COUNT(?{wdProp.varname})=1)"
         return sparqlQuery
     
     def mostFrequentPropertiesQuery(self,whereClause:str=None):
@@ -414,14 +457,39 @@ WHERE {{
         Args:
             whereClause(str): an extra WhereClause to use
         '''
-        query=self.queryManager.queriesByName["mostFrequentProperties"]
         if whereClause is None:
             whereClause=f"?item wdt:P31 wd:{self.itemQid};";
         else:
             whereClause+=";"     
         itemText=self.getItemText()
-        query.title=f"most frequently used properties for {self.item.asText(long=True)}"
-        query.query=query.query % (itemText,whereClause,self.lang)
+        if self.endpoint=="https://query.wikidata.org/sparql":
+            optimizerHint='      hint:Query hint:optimizer "None".\n'
+        else:
+            optimizerHint=''
+        sparqlQuery=f"""# get the most frequently used properties for
+# {itemText}
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX schema: <http://schema.org/>
+PREFIX wd: <http://www.wikidata.org/entity/>
+PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>
+SELECT ?prop ?propLabel ?count WHERE {{
+  {{
+    SELECT ?prop (COUNT(DISTINCT ?item) AS ?count) WHERE {{
+{optimizerHint}        
+      {whereClause}
+      ?p ?id.
+      ?prop wikibase:directClaim ?p.
+    }}
+    GROUP BY ?prop ?propLabel
+  }}
+  ?prop rdfs:label ?propLabel
+  FILTER(LANG(?propLabel) = "{self.lang}")      
+}}
+ORDER BY DESC (?count)
+"""
+        title=f"most frequently used properties for {self.item.asText(long=True)}"
+        query=Query(name=f"mostFrequentProperties for {itemText}",query=sparqlQuery,title=title)
         return query
     
     def noneTabularQuery(self,wdProperty:WikidataProperty,asFrequency:bool=True):

{pylodstorage-0.2.14 → pylodstorage-0.3.1}/lodstorage/version.py

@@ -9,7 +9,7 @@ class Version(object):
     Version handling for pyLoDStorage
     '''
     name="pylodstorage"
-    version='0.2.14'
+    version='0.3.1'
     date = '2020-09-10'
-    updated = '2022-07-27'
+    updated = '2022-07-28'
     description='python List of Dict (Table) Storage library'

{pylodstorage-0.2.14 → pylodstorage-0.3.1/pylodstorage.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pylodstorage
-Version: 0.2.14
+Version: 0.3.1
 Summary: python List of Dict (Table) Storage library
 Home-page: https://github.com/WolfgangFahl/pyLodStorage
 Author: Wolfgang Fahl

{pylodstorage-0.2.14 → pylodstorage-0.3.1}/sampledata/trulytabular.yaml

@@ -6,10 +6,15 @@
     sparql: |
         # get the most frequently used properties for
         # %s
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        PREFIX schema: <http://schema.org/>
+        PREFIX wd: <http://www.wikidata.org/entity/>
+        PREFIX wdt: <http://www.wikidata.org/prop/direct/>
+        PREFIX xsd:  <http://www.w3.org/2001/XMLSchema#>
         SELECT ?prop ?propLabel ?count WHERE {
           {
             SELECT ?prop (COUNT(DISTINCT ?item) AS ?count) WHERE {
-              hint:Query hint:optimizer "None".
+              #hint:Query hint:optimizer "None".
               %s
                 ?p ?id.
               ?prop wikibase:directClaim ?p.