ogc-na 0.3.57__tar.gz → 0.4.1__tar.gz

{ogc_na-0.3.57 → ogc_na-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ogc_na
-Version: 0.3.57
+Version: 0.4.1
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/

{ogc_na-0.3.57 → ogc_na-0.4.1}/ogc/na/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.57'
-__version_tuple__ = version_tuple = (0, 3, 57)
+__version__ = version = '0.4.1'
+__version_tuple__ = version_tuple = (0, 4, 1)

{ogc_na-0.3.57 → ogc_na-0.4.1}/ogc/na/ingest_json.py

@@ -45,7 +45,7 @@ from typing import Union, Optional, Sequence, cast, Iterable, Any
 import jq
 from jsonpath_ng.ext import parse as json_path_parse
 from jsonschema import validate as json_validate
-from rdflib import Graph, DC, DCTERMS, SKOS, OWL, RDF, RDFS, XSD, DCAT
+from rdflib import Graph, DC, DCTERMS, SKOS, OWL, RDF, RDFS, XSD, DCAT, URIRef
 from rdflib.namespace import Namespace, DefinedNamespace
 
 from ogc.na import util, profile
@@ -418,7 +418,8 @@ def process_file(input_fn: str | Path,
                  provenance_base_uri: str | bool | None = None,
                  provenance_process_id: str | None = None,
                  fetch_url_whitelist: bool | Sequence[str] | None = None,
-                 transform_args: dict | None = None) -> UpliftResult | None:
+                 transform_args: dict | None = None,
+                 generated_provenance_classes: bool | list[str | URIRef] = False) -> UpliftResult | None:
     """
     Process input file and generate output RDF files.
 
@@ -439,6 +440,8 @@ def process_file(input_fn: str | Path,
         retrieving them. If None, it will not be used; if empty sequence or False, remote fetching operations will
         throw an exception
     :param transform_args: Additional arguments to pass as variables to the jq transform
+    :param generated_provenance_classes: List of classes whose instances will be included in the "generated"
+        provenance metadata. If `True`, all subjects in the output graph will be added.
     :return: List of output files created
     """
 
@@ -489,16 +492,16 @@ def process_file(input_fn: str | Path,
 
     provenance_metadata: ProvenanceMetadata | None = None
     if provenance_base_uri is not False:
-        used = [FileProvenanceMetadata(filename=input_fn, mime_type='application/json')]
-        used.extend(FileProvenanceMetadata(filename=c, mime_type='application/yaml') for c in provenance_contexts)
         provenance_metadata = ProvenanceMetadata(
-            used=used,
             batch_activity_id=provenance_process_id,
             base_uri=provenance_base_uri,
             root_directory=os.getcwd(),
             start=start_time,
             end_auto=True,
         )
+        provenance_metadata.add_used(FileProvenanceMetadata(filename=input_fn, mime_type='application/json'))
+        provenance_metadata.add_used(FileProvenanceMetadata(filename=c, mime_type='application/yaml')
+                                     for c in provenance_contexts)
 
     if transform_args is None:
         transform_args = {}
@@ -518,13 +521,29 @@ def process_file(input_fn: str | Path,
 
     uplift_result.input_file = input_fn
 
+    if provenance_metadata and generated_provenance_classes:
+        if generated_provenance_classes is True:
+            # add all subjects to "generated"
+            provenance_metadata.add_generated(
+                FileProvenanceMetadata(uri=str(s), use_bnode=False)
+                for s in uplift_result.graph.subjects()
+                if isinstance(s, URIRef)
+            )
+        elif generated_provenance_classes:
+            provenance_metadata.add_generated(
+                FileProvenanceMetadata(uri=str(s), use_bnode=False)
+                for cls in generated_provenance_classes
+                for s in uplift_result.graph.subjects(predicate=RDF.type, object=URIRef(cls))
+                if isinstance(s, URIRef)
+            )
+
     # False = do not generate
     # None = auto filename
     # - = stdout
     if ttl_fn is not False:
         if ttl_fn == '-':
             if provenance_metadata:
-                provenance_metadata.output = FileProvenanceMetadata(mime_type='text/turtle', use_bnode=False)
+                provenance_metadata.add_generated(FileProvenanceMetadata(mime_type='text/turtle', use_bnode=False))
                 generate_provenance(uplift_result.graph, provenance_metadata)
             print(uplift_result.graph.serialize(format='ttl'))
         else:
@@ -533,9 +552,9 @@ def process_file(input_fn: str | Path,
                     if input_fn.suffix != '.ttl' \
                     else input_fn.with_suffix(input_fn.suffix + '.ttl')
             if provenance_metadata:
-                provenance_metadata.output = FileProvenanceMetadata(filename=ttl_fn,
+                provenance_metadata.add_generated(FileProvenanceMetadata(filename=ttl_fn,
                                                                     mime_type='text/turtle',
-                                                                    use_bnode=False)
+                                                                    use_bnode=False))
                 generate_provenance(uplift_result.graph, provenance_metadata)
             uplift_result.graph.serialize(destination=ttl_fn, format='ttl')
             uplift_result.output_files.append(ttl_fn)
@@ -656,7 +675,8 @@ def process(input_files: str | Path | Sequence[str | Path],
             provenance_base_uri: Optional[Union[str, bool]] = None,
             fetch_url_whitelist: Optional[Union[Sequence, bool]] = None,
             transform_args: dict | None = None,
-            file_filter: str | re.Pattern = None) -> list[UpliftResult]:
+            file_filter: str | re.Pattern = None,
+            generated_provenance_classes: bool | list[str | URIRef] = False) -> list[UpliftResult]:
     """
     Performs the JSON-LD uplift process.
 
@@ -678,6 +698,8 @@ def process(input_files: str | Path | Sequence[str | Path],
         throw an exception
     :param transform_args: Additional arguments to pass as variables to the jq transform
     :param file_filter: Filename filter for input files
+    :param generated_provenance_classes: List of classes whose instances will be included in the "generated"
+        provenance metadata. If `True`, all subjects in the output graph will be added.
     :return: a list of JSON-LD and/or Turtle output files
     """
     result: list[UpliftResult] = []
@@ -729,6 +751,7 @@ def process(input_files: str | Path | Sequence[str | Path],
                     fetch_url_whitelist=fetch_url_whitelist,
                     domain_cfg=domain_cfg,
                     transform_args=transform_args,
+                    generated_provenance_classes=generated_provenance_classes,
                 ))
             except MissingContextException as e:
                 if skip_on_missing_context or batch:
@@ -751,6 +774,7 @@ def process(input_files: str | Path | Sequence[str | Path],
                     fetch_url_whitelist=fetch_url_whitelist,
                     domain_cfg=domain_cfg,
                     transform_args=transform_args,
+                    generated_provenance_classes=generated_provenance_classes,
                 ))
             except Exception as e:
                 if skip_on_missing_context:
@@ -879,6 +903,18 @@ def _process_cmdln():
         help='Regular expression to filter input filenames',
     )
 
+    parser.add_argument(
+        '--generated-provenance-classes',
+        nargs='*',
+        help='Classes whose subjects will be included in the "generated" provenance metadata',
+    )
+
+    parser.add_argument(
+        '--all-subjects-provenance',
+        action='store_true',
+        help='Add all subjects in the resulting graph to the "generated" provenance metadata',
+    )
+
     args = parser.parse_args()
 
     if args.domain_config:
@@ -904,6 +940,12 @@ def _process_cmdln():
     if args.transform_arg:
         transform_args = dict((e.split('=', 1) for e in args.transform_arg))
 
+    generated_provenance_classes = False
+    if args.all_subjects_provenance:
+        generated_provenance_classes = True
+    elif args.generated_provenance_classes:
+        generated_provenance_classes = args.generated_provenance_classes
+
     result = process(input_files,
                      context_fn=args.context,
                      domain_cfg=domain_cfg,
@@ -915,6 +957,7 @@ def _process_cmdln():
                      fetch_url_whitelist=args.url_whitelist,
                      transform_args=transform_args,
                      file_filter=args.file_filter,
+                     generated_provenance_classes=generated_provenance_classes,
              )
 
     if args.fs:

{ogc_na-0.3.57 → ogc_na-0.4.1}/ogc/na/provenance.py

@@ -4,7 +4,7 @@ import mimetypes
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Union, Optional, Sequence
+from typing import Union, Optional, Sequence, Iterable
 
 from rdflib.term import Node
 
@@ -40,6 +40,22 @@ class ProvenanceMetadata:
     activity_label: str = None
     comment: str = None
 
+    def _add_list(self, attr: str, item: FileProvenanceMetadata | Iterable[FileProvenanceMetadata]):
+        items = [item] if isinstance(item, FileProvenanceMetadata) else [*item]
+        cur: FileProvenanceMetadata | list[FileProvenanceMetadata] | None = getattr(self, attr, None)
+        if not cur:
+            setattr(self, attr, items)
+        elif isinstance(cur, FileProvenanceMetadata):
+            setattr(self, attr, [self.used, *items])
+        else:
+            setattr(self, attr, cur + items)
+
+    def add_used(self, used: FileProvenanceMetadata | Iterable[FileProvenanceMetadata]) -> None:
+        self._add_list('used', used)
+
+    def add_generated(self, generated: FileProvenanceMetadata | Iterable[FileProvenanceMetadata]) -> None:
+        self._add_list('generated', generated)
+
 
 def add_provenance_agent(g: Graph, module_name: str = None) -> Node:
     agent = BNode()
@@ -133,7 +149,7 @@ def generate_provenance(g: Graph = None,
             g.add((activity, PROV.used, used))
 
     if metadata.generated:
-        for generated in metadata.used if isinstance(metadata.generated, Sequence) else (metadata.generated,):
+        for generated in metadata.generated if isinstance(metadata.generated, Sequence) else (metadata.generated,):
             generated = add_provenance_entity(g, metadata=generated,
                                               root_directory=metadata.root_directory,
                                               base_uri=metadata.base_uri)

{ogc_na-0.3.57 → ogc_na-0.4.1}/ogc/na/update_vocabs.py

@@ -40,6 +40,7 @@ from rdflib import Graph, RDF, SKOS, URIRef
 
 from ogc.na import util
 from ogc.na.domain_config import DomainConfiguration, DomainConfigurationEntry
+from ogc.na.profile import ProfileRegistry
 from ogc.na.provenance import generate_provenance, ProvenanceMetadata, FileProvenanceMetadata
 
 logger = logging.getLogger('ogc.na.update_vocabs')
@@ -245,6 +246,7 @@ def _main():
 
     parser.add_argument(
         "domain_cfg",
+        nargs="?",
         metavar="domain-cfg",
         help=("Domain configuration (can be a local or remote RDF file, "
               "or a SPARQL endpoint in the form 'sparql:http://example.org/sparql')"),
@@ -362,6 +364,17 @@ def _main():
         help='Ignore errors when retrieving profile artifacts'
     )
 
+    parser.add_argument(
+        '--graph-uri',
+        help='Override graph URI that will be used for all resources',
+    )
+
+    parser.add_argument(
+        '--profile-uris',
+        nargs='*',
+        help='Override profile URIs that will be used for all resources',
+    )
+
     args = parser.parse_args()
 
     setup_logging(args.debug)
@@ -410,29 +423,52 @@ def _main():
                 raise Exception(f"Invalid local artifact mapping: {mappingstr}")
             local_artifacts_mappings[mapping[0]] = mapping[1]
 
-    domain_cfg = DomainConfiguration(args.domain_cfg, working_directory=args.working_directory,
-                                     profile_sources=args.profile_source,
-                                     ignore_artifact_errors=args.ignore_artifact_errors,
-                                     local_artifacts_mappings=local_artifacts_mappings)
-    cfg_entries = domain_cfg.entries
-    if not len(cfg_entries):
-        if args.domain:
-            logger.warning('No configuration found in %s for domain %s, exiting',
-                           args.domain_cfg, args.domain)
+    if not args.domain_cfg and not (args.profile_source and args.profile_uris):
+        logger.error('Either a domain configuration or a profile source and a set of '
+                     'profile URIs need to be provided')
+        sys.exit(2)
+
+    modified: dict[Path, DomainConfigurationEntry | None]
+    added: dict[Path, DomainConfigurationEntry | None]
+
+    if args.domain_cfg:
+        domain_cfg = DomainConfiguration(args.domain_cfg, working_directory=args.working_directory,
+                                         profile_sources=args.profile_source,
+                                         ignore_artifact_errors=args.ignore_artifact_errors,
+                                         local_artifacts_mappings=local_artifacts_mappings)
+        cfg_entries = domain_cfg.entries
+        if not len(cfg_entries):
+            if args.domain:
+                logger.warning('No configuration found in %s for domain %s, exiting',
+                               args.domain_cfg, args.domain)
+            else:
+                logger.warning('No configuration found in %s exiting', args.domain_cfg)
+            sys.exit(1)
+
+        profile_registry = domain_cfg.profile_registry
+
+        if args.batch:
+            modified = cfg_entries.find_all()
+            added = {}
         else:
-            logger.warning('No configuration found in %s exiting', args.domain_cfg)
-        sys.exit(1)
+            modified = cfg_entries.find_entries_for_files(mod_list)
+            added = cfg_entries.find_entries_for_files(add_list)
 
-    profile_registry = domain_cfg.profile_registry
+        root_directory = domain_cfg.working_directory
 
-    modified: dict[Path, DomainConfigurationEntry]
-    added: dict[Path, DomainConfigurationEntry]
-    if args.batch:
-        modified = cfg_entries.find_all()
-        added = {}
+    elif args.batch:
+        logger.error('--batch requires a domain configuration')
+        sys.exit(3)
     else:
-        modified = cfg_entries.find_entries_for_files(mod_list)
-        added = cfg_entries.find_entries_for_files(add_list)
+        logger.info('Loading profile sources and URIs from script parameters')
+        profile_registry = ProfileRegistry(args.profile_source,
+                                           ignore_artifact_errors=args.ignore_artifact_errors,
+                                           local_artifact_mappings=args.local_artifact_mappings)
+
+        modified = {Path(x): None for x in mod_list}
+        added = {Path(x): None for x in add_list}
+
+        root_directory = Path(args.working_directory or os.getcwd())
 
     output_path = Path(args.output_directory) if args.output_directory else None
 
@@ -448,21 +484,28 @@ def _main():
             if args.no_provenance:
                 provenance_metadata = None
             else:
+                used = [FileProvenanceMetadata(filename=doc)]
+                if args.domain_cfg:
+                    used.append(FileProvenanceMetadata(filename=args.domain_cfg))
                 provenance_metadata = ProvenanceMetadata(
-                    used=[FileProvenanceMetadata(filename=doc),
-                          FileProvenanceMetadata(filename=args.domain_cfg)],
+                    used=used,
                     start=datetime.now(),
                     end_auto=True,
-                    root_directory=domain_cfg.working_directory,
+                    root_directory=root_directory,
                     batch_activity_id=activity_id,
                     activity_label='Entailment and validation',
                     comment=cmdline,
                     base_uri=args.base_uri,
                 )
 
+            if cfg:
+                conforms_to = cfg.conforms_to
+            else:
+                conforms_to = args.profile_uris
+
             origg = Graph().parse(doc)
-            newg, entail_artifacts = profile_registry.entail(origg, cfg.conforms_to)
-            validation_result = profile_registry.validate(newg, cfg.conforms_to, log_artifact_errors=True)
+            newg, entail_artifacts = profile_registry.entail(origg, conforms_to)
+            validation_result = profile_registry.validate(newg, conforms_to, log_artifact_errors=True)
 
             if provenance_metadata:
                 def add_artifact(a: Union[str, Path]):
@@ -482,7 +525,7 @@ def _main():
             else:
                 entailment_dir = DEFAULT_ENTAILED_DIR
 
-            loadable_path = make_rdf(doc, newg, cfg.uri_root_filter,
+            loadable_path = make_rdf(doc, newg, cfg.uri_root_filter if cfg else None,
                                      entailment_dir, provenance_metadata)
             with open(loadable_path.with_suffix('.txt'), 'w') as validation_file:
                 validation_file.write(validation_result.text)
@@ -495,13 +538,20 @@ def _main():
                     if p != loadable_path:
                         loadables[p] = g
 
-                graphname = next(get_graph_uri_for_vocab(newg), None)
+                if args.graph_uri:
+                    graphname = args.graph_uri
+                else:
+                    graphname = next(get_graph_uri_for_vocab(newg), None)
+
                 if not graphname:
                     logger.warning("No graph name could be deduced from the vocabulary")
                     # Create graph name from a colon-separated list of
                     # path components relative to the working directory
-                    urnpart = ':'.join(p for p in docrelpath.parts if p and p != '..')
-                    graphname = "x-urn:{}".format(urnpart)
+                    urnparts = ['x-urn', 'ogc', 'na']
+                    if cfg and cfg.identifier:
+                        urnparts.append(str(cfg.identifier))
+                    urnparts.extend(p for p in docrelpath.parts if p and p != '..')
+                    graphname = ':'.join(urnparts)
                 append_data = graphname in uploaded_graphs
                 logger.info("Using graph name %s for %s", graphname, str(doc))
 
@@ -523,7 +573,7 @@ def _main():
                         raise e
                     versioned_gname = f'{graphname}{n + 1}'
 
-            report.setdefault(cfg.identifier, {}) \
+            report.setdefault(cfg.identifier if cfg else args.graph_uri, {}) \
                 .setdefault(report_cat, []).append(os.path.relpath(doc))
 
     for scope, scopereport in report.items():

{ogc_na-0.3.57 → ogc_na-0.4.1}/ogc_na.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: ogc_na
-Version: 0.3.57
+Version: 0.4.1
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/