mustrd 0.2.7a0__tar.gz → 0.3.1a0__tar.gz

mustrd-0.3.1a0/PKG-INFO

@@ -0,0 +1,96 @@
+Metadata-Version: 2.3
+Name: mustrd
+Version: 0.3.1a0
+Summary: A Spec By Example framework for RDF and SPARQL, Inspired by Cucumber.
+License: MIT
+Author: John Placek
+Author-email: john.placek@semanticpartners.com
+Requires-Python: >=3.11,<4.0
+Classifier: Framework :: Pytest
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Utilities
+Requires-Dist: Jinja2 (==3.1.5)
+Requires-Dist: beautifulsoup4 (>=4.11.1,<5.0.0)
+Requires-Dist: colorama (==0.4.6)
+Requires-Dist: colorlog (>=6.7.0,<7.0.0)
+Requires-Dist: coverage (==7.4.3)
+Requires-Dist: edn-format (>=0.7.5,<0.8.0)
+Requires-Dist: flake8 (==7.0.0)
+Requires-Dist: multimethods-py (>=0.5.3,<0.6.0)
+Requires-Dist: numpy (>=1.26.0,<1.27.0)
+Requires-Dist: openpyxl (>=3.1.2,<4.0.0)
+Requires-Dist: pandas (>=2.0,<3.0)
+Requires-Dist: pyshacl (>=0.30.0,<0.31.0)
+Requires-Dist: pytest (>=7.2.0,<8.0.0)
+Requires-Dist: rdflib (>=7.1.3,<8.0.0)
+Requires-Dist: requests (>=2.28.2,<3.0.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
+Requires-Dist: toml (>=0.10.2,<0.11.0)
+Requires-Dist: tomli (>=2.0.1,<3.0.0)
+Requires-Dist: urllib3 (==1.26.19)
+Project-URL: Repository, https://github.com/Semantic-partners/mustrd
+Description-Content-Type: text/markdown
+
+# mustrd
+
+**"MustRD: Validate your SPARQL queries and transformations with precision and confidence, using BDD and Given-When-Then principles."**
+
+[<img src="https://github.com/Semantic-partners/mustrd/raw/python-coverage-comment-action-data/badge.svg?sanitize=true" alt="coverage badge">](https://github.com/Semantic-partners/mustrd/tree/python-coverage-comment-action-data)
+
+### Why?
+
+SPARQL is a powerful query language for RDF data, but how can you ensure your queries and transformations are doing what you intend? Whether you're working on a pipeline or a standalone query, certainty is key.
+
+While RDF and SPARQL offer great flexibility, we noticed a gap in tooling to validate their behavior. We missed the robust testing frameworks available in imperative programming languages that help ensure your code works as expected.
+
+With MustRD, you can:
+
+* Define data scenarios and verify that queries produce the expected results.
+* Test edge cases to ensure your queries remain reliable.
+* Isolate small SPARQL enrichment or transformation steps and confirm you're only inserting what you intend.
+
+### What?
+
+MustRD is a Spec-By-Example ontology with a reference Python implementation, inspired by tools like Cucumber. It uses the Given-When-Then approach to define and validate SPARQL queries and transformations.
+
+MustRD is designed to be triplestore/SPARQL engine agnostic, leveraging open standards to ensure compatibility across different platforms.
+
+### What it is NOT
+
+MustRD is not an alternative to SHACL. While SHACL validates data structures, MustRD focuses on validating data transformations and query results.
+
+### How?
+
+You define your specs in Turtle (`.ttl`) or TriG (`.trig`) files using the Given-When-Then approach:
+
+* **Given**: Define the starting dataset.
+* **When**: Specify the action (e.g., a SPARQL query).
+* **Then**: Outline the expected results.
+
+Depending on the type of SPARQL query (CONSTRUCT, SELECT, INSERT/DELETE), MustRD runs the query and compares the results against the expectations defined in the spec.
+
+Expectations can also be defined as:
+
+* INSERT queries.
+* SELECT queries.
+* Higher-order expectation languages, similar to those used in various platforms.
+
+### When?
+
+MustRD is a work in progress, built to meet the needs of our projects across multiple clients and vendor stacks. While we find it useful, it may not meet your needs out of the box.
+
+We invite you to try it, raise issues, or contribute via pull requests. If you need custom features, contact us for consultancy rates, and we may prioritize your request.
+
+## Support
+
+Semantic Partners is a specialist consultancy in Semantic Technology. If you need more support, contact us at info@semanticpartners.com or mustrd@semanticpartners.com.
+
+

mustrd-0.3.1a0/README.md

@@ -0,0 +1,54 @@
+# mustrd
+
+**"MustRD: Validate your SPARQL queries and transformations with precision and confidence, using BDD and Given-When-Then principles."**
+
+[<img src="https://github.com/Semantic-partners/mustrd/raw/python-coverage-comment-action-data/badge.svg?sanitize=true" alt="coverage badge">](https://github.com/Semantic-partners/mustrd/tree/python-coverage-comment-action-data)
+
+### Why?
+
+SPARQL is a powerful query language for RDF data, but how can you ensure your queries and transformations are doing what you intend? Whether you're working on a pipeline or a standalone query, certainty is key.
+
+While RDF and SPARQL offer great flexibility, we noticed a gap in tooling to validate their behavior. We missed the robust testing frameworks available in imperative programming languages that help ensure your code works as expected.
+
+With MustRD, you can:
+
+* Define data scenarios and verify that queries produce the expected results.
+* Test edge cases to ensure your queries remain reliable.
+* Isolate small SPARQL enrichment or transformation steps and confirm you're only inserting what you intend.
+
+### What?
+
+MustRD is a Spec-By-Example ontology with a reference Python implementation, inspired by tools like Cucumber. It uses the Given-When-Then approach to define and validate SPARQL queries and transformations.
+
+MustRD is designed to be triplestore/SPARQL engine agnostic, leveraging open standards to ensure compatibility across different platforms.
+
+### What it is NOT
+
+MustRD is not an alternative to SHACL. While SHACL validates data structures, MustRD focuses on validating data transformations and query results.
+
+### How?
+
+You define your specs in Turtle (`.ttl`) or TriG (`.trig`) files using the Given-When-Then approach:
+
+* **Given**: Define the starting dataset.
+* **When**: Specify the action (e.g., a SPARQL query).
+* **Then**: Outline the expected results.
+
+Depending on the type of SPARQL query (CONSTRUCT, SELECT, INSERT/DELETE), MustRD runs the query and compares the results against the expectations defined in the spec.
+
+Expectations can also be defined as:
+
+* INSERT queries.
+* SELECT queries.
+* Higher-order expectation languages, similar to those used in various platforms.
+
+### When?
+
+MustRD is a work in progress, built to meet the needs of our projects across multiple clients and vendor stacks. While we find it useful, it may not meet your needs out of the box.
+
+We invite you to try it, raise issues, or contribute via pull requests. If you need custom features, contact us for consultancy rates, and we may prioritize your request.
+
+## Support
+
+Semantic Partners is a specialist consultancy in Semantic Technology. If you need more support, contact us at info@semanticpartners.com or mustrd@semanticpartners.com.
+

{mustrd-0.2.7a0 → mustrd-0.3.1a0}/mustrd/README.md

@@ -27,3 +27,5 @@ As the project is actually built from the requirements.txt file at the project r
 
 `poetry export -f requirements.txt --without-hashes > requirements.txt`
 
+We also recommend pairing MustRD with the VS Code plugin [faubulous.mentor](https://marketplace.visualstudio.com/items?itemName=faubulous.mentor) to enhance your development experience and streamline working with SPARQL and RDF specifications.
+

{mustrd-0.2.7a0 → mustrd-0.3.1a0}/mustrd/anzo_utils.py

@@ -31,16 +31,18 @@ from requests import Response, HTTPError, RequestException
 from bs4 import BeautifulSoup
 import logging
 
+logger = logging.getLogger()
+
 
 def query_azg(anzo_config: dict, query: str,
               format: str = "json", is_update: bool = False,
               data_layers: List[str] = None):
     params = {
-        'skipCache': True,
+        'skipCache': 'true',
         'format': format,
         'datasourceURI': anzo_config['gqe_uri'],
-        'default-graph-uri': data_layers,
-        'named-graph-uri': data_layers
+        'using-graph-uri' if is_update else 'default-graph-uri': data_layers,
+        'using-named-graph-uri' if is_update else 'named-graph-uri': data_layers
     }
     url = f"{anzo_config['url']}/sparql"
     return send_anzo_query(anzo_config, url=url, params=params, query=query, is_update=is_update)
@@ -52,7 +54,7 @@ def query_graphmart(anzo_config: dict,
                     format: str = "json",
                     data_layers: List[str] = None):
     params = {
-        'skipCache': True,
+        'skipCache': 'true',
         'format': format,
         'default-graph-uri': data_layers,
         'named-graph-uri': data_layers
@@ -87,7 +89,8 @@ def manage_anzo_response(response: Response) -> str:
 
 def send_anzo_query(anzo_config, url, params, query, is_update=False):
     headers = {"Content-Type": f"application/sparql-{'update' if is_update else 'query' }"}
-    return manage_anzo_response(requests.post(url=url, params=params, data=query,
+    logger.debug(f"send_anzo_query {url=} {query=} {is_update=}")
+    return manage_anzo_response(requests.post(url=url, params=params, data=query.encode('utf-8'),
                                               auth=(anzo_config['username'], anzo_config['password']),
                                               headers=headers, verify=False))
 

{mustrd-0.2.7a0 → mustrd-0.3.1a0}/mustrd/logger_setup.py

@@ -35,6 +35,7 @@ def setup_logger(name: str) -> logging.Logger:
     log = logging.getLogger(name)
     log.setLevel(LOG_LEVEL)
     
+
     stderr_handler = logging.StreamHandler(sys.stderr)
     stderr_handler.setLevel(logging.ERROR)
     log.addHandler(stderr_handler)
@@ -50,3 +51,5 @@ def setup_logger(name: str) -> logging.Logger:
 def flush():
     logging.shutdown()
     sys.stdout.flush()
+
+logging.getLogger("edn_format").setLevel(logging.WARNING)

{mustrd-0.2.7a0 → mustrd-0.3.1a0}/mustrd/model/mustrdShapes.ttl

@@ -140,10 +140,20 @@ must:OrderedTableDatasetShape
 must:FileDatasetShape
     a              sh:NodeShape ;
     sh:targetClass must:FileDataset ;
-    sh:property    [ sh:path     must:file ;
-                     sh:datatype xsd:string ;
-                     sh:minCount 1 ;
-                     sh:maxCount 1 ; ] .
+    sh:or (
+		[
+			sh:path must:file ;
+            sh:datatype xsd:string ;
+			sh:maxCount 1 ;
+		]
+		[
+			sh:path must:fileurl ;
+            sh:nodeKind sh:IRI ;
+			sh:minCount 1 ;
+			sh:maxCount 1 ;
+		]
+	)
+     .
 
 must:StatementShape
     a              sh:NodeShape ;
@@ -249,5 +259,14 @@ must:AnzoGraphmartQueryDrivenTemplatedStepSparqlSourceShape
                      sh:minCount    1 ;
                      sh:maxCount    1 ; ]  .
 
-
-
+must:SpadeEdnGroupSourceShape
+    a              sh:NodeShape ;
+    sh:targetClass must:SpadeEdnGroupSource ;
+    sh:property    [ sh:path     must:fileurl ;
+                     sh:message "A SpadeEdnGroupSource must have a fileurl property pointing to the spade.edn config." ;
+                     sh:minCount 1 ;
+                     sh:maxCount 1 ; ] ;
+    sh:property    [ sh:path     must:groupId ;
+                     sh:message "A SpadeEdnGroupSource must have a groupId property referencing the group in the EDN file." ;
+                     sh:minCount 1 ;
+                     sh:maxCount 1 ; ] .

{mustrd-0.2.7a0 → mustrd-0.3.1a0}/mustrd/model/ontology.ttl

@@ -146,7 +146,7 @@ sh:order rdf:type owl:DatatypeProperty ;
 
 ###  https://mustrd.com/model/file
 :file rdf:type owl:DatatypeProperty ;
-      rdfs:comment "Relative or absolute path to local file" ;
+      rdfs:comment "Relative or absolute path to local file as a string, or a file:// url" ;
       rdfs:label "file" .
 
 
@@ -158,7 +158,6 @@ sh:order rdf:type owl:DatatypeProperty ;
 
 ###  https://mustrd.com/model/fileurl
 :fileurl rdf:type owl:DatatypeProperty ;
-         rdfs:domain :FileSparqlSource ;
          rdfs:comment "a full or relatively qualified file:// url. Relative to what? We haven't thought that through, yet." ;
          rdfs:isDefinedBy : ;
          rdfs:label "fileUrl" .
@@ -461,6 +460,11 @@ sh:order rdf:type owl:DatatypeProperty ;
                          rdfs:isDefinedBy : ;
                          rdfs:label "AnzoGraphmartQueryDrivenTemplatedStepSparqlSource" .
 
+### https://mustrd.com/model/SpadeEdnGroupSource
+:SpadeEdnGroupSource rdf:type owl:Class ;
+    rdfs:subClassOf :SparqlSource ;
+    rdfs:comment "Allows reference to a spade.edn file, and a specific groupid (think Anzo layer), within that" ; 
+    rdfs:label "SpadeEdnGroupSource" .
 
 ###  https://mustrd.com/model/Then
 :Then rdf:type owl:Class ;