ogc-na 0.1.2__tar.gz → 0.1.4__tar.gz

{ogc_na-0.1.2 → ogc_na-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ogc_na
-Version: 0.1.2
+Version: 0.1.4
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc/na/annotate_schema.py

@@ -116,16 +116,16 @@ The resulting context will be printed to the standard output.
 from __future__ import annotations
 import argparse
 import dataclasses
-import functools
 import json
 import logging
+import re
 import sys
 from pathlib import Path
-from typing import Any, AnyStr
+from typing import Any, AnyStr, Callable
 from urllib.parse import urlparse, urljoin
 import yaml
 import requests
-from ogc.na.util import is_url, merge_dicts, load_yaml
+from ogc.na.util import is_url, merge_dicts, load_yaml, LRUCache
 
 try:
     from yaml import CLoader as YamlLoader, CDumper as YamlDumper
@@ -137,8 +137,11 @@ logger = logging.getLogger(__name__)
 ANNOTATION_CONTEXT = 'x-jsonld-context'
 ANNOTATION_ID = 'x-jsonld-id'
 ANNOTATION_TYPE = 'x-jsonld-type'
+ANNOTATION_PREFIXES = 'x-jsonld-prefixes'
 REF_ROOT_MARKER = '$_ROOT_/'
 
+context_term_cache = LRUCache(maxsize=20)
+
 
 @dataclasses.dataclass
 class AnnotatedSchema:
@@ -218,63 +221,95 @@ def resolve_ref(ref: str, fn_from: str | Path | None = None, url_from: str | Non
         return ref, None
 
 
-@functools.lru_cache(maxsize=20)
-def read_context_terms(file: Path | str = None, url: str = None) -> dict[str, str]:
+def read_context_terms(ctx: Path | str | dict) -> tuple[dict[str, str], dict[str, str], dict[str, str]]:
     """
     Reads all the terms from a JSON-LD context document.
 
-    :param file: file path to load
-    :param url: URL to load
+    :param ctx: file path (Path), URL (str) or dictionary (dict) to load
     :return: a dictionary with term -> URI mappings
     """
+
+    cached = context_term_cache.get(ctx)
+    if cached:
+        return cached
+
     context: dict[str, Any] | None = None
 
-    if file:
-        with open(file) as f:
+    if isinstance(ctx, Path):
+        with open(ctx) as f:
             context = json.load(f).get('@context')
-    elif url:
-        r = requests.get(url)
+    elif isinstance(ctx, str):
+        r = requests.get(ctx)
         r.raise_for_status()
         context = r.json().get('@context')
+    elif ctx:
+        context = ctx.get('@context')
 
     if not context:
-        return {}
+        return {}, {}, {}
 
-    result: dict[str, str] = {}
-    pending: dict[str, list] = {}
+    result: dict[str, str | tuple[str, str]] = {}
+    types: dict[str, str | tuple[str, str]] = {}
 
     vocab = context.get('@vocab')
 
+    def expand_uri(uri: str) -> str | tuple[str, str] | None:
+        if not uri:
+            return None
+
+        if ':' in uri:
+            # either URI or prefix:suffix
+            pref, suf = uri.split(':', 1)
+            if suf.startswith('//') or pref == 'urn':
+                # assume full URI
+                return uri
+            else:
+                # prefix:suffix -> add to pending for expansion
+                return pref, suf
+        elif vocab:
+            # append term_val to vocab to get URI
+            return f"{vocab}{term_id}"
+        else:
+            return uri
+
     for term, term_val in context.items():
         if not term.startswith("@"):
             # assume term
+            term_type = None
             if isinstance(term_val, str):
                 term_id = term_val
             elif isinstance(term_val, dict):
                 term_id = term_val.get('@id')
+                term_type = term_val.get('@type')
             else:
                 term_id = None
 
-            if term_id:
-                if ':' in term_id:
-                    # either URI or prefix:suffix
-                    pref, suf = term_id.split(':', 1)
-                    if suf.startswith('//'):
-                        # assume URI -> add to result
-                        result[term] = term_id
-                    else:
-                        # prefix:suffix -> add to pending for expansion
-                        pending[term] = [pref, suf]
-                elif vocab:
-                    # append term_val to vocab to get URI
-                    result[term] = f"{vocab}{term_id}"
+            expanded_id = expand_uri(term_id)
+            if expanded_id:
+                result[term] = expanded_id
+            expanded_type = expand_uri(term_type)
+            if expanded_type:
+                types[term] = expanded_type
 
-    for term, term_val in pending.items():
-        pref, suf = term_val
-        if pref in result:
-            result[term] = f"{result[pref]}{suf}"
+    prefixes = {}
 
-    return result
+    def expand_result(d: dict[str, str | tuple[str, str]]) -> dict[str, str]:
+        r = {}
+        for term, term_val in d.items():
+            if isinstance(term_val, str):
+                r[term] = term_val
+            else:
+                pref, suf = term_val
+                if pref in result:
+                    r[term] = f"{result[pref]}{suf}"
+                    prefixes[pref] = result[pref]
+        return r
+
+    expanded_types = expand_result(types)
+    expanded_terms = expand_result(result)
+
+    context_term_cache[ctx] = expanded_terms, prefixes, expanded_types
+    return expanded_terms, prefixes, expanded_types
 
 
 class SchemaAnnotator:
@@ -287,7 +322,8 @@ class SchemaAnnotator:
     """
 
     def __init__(self, fn: Path | str | None = None, url: str | None = None,
-                 follow_refs: bool = True, ref_root: Path | str | None = None):
+                 follow_refs: bool = True, ref_root: Path | str | None = None,
+                 context: str | Path | dict | None = None):
         """
         :param fn: file path to load (root schema)
         :param url: URL to load (root schema)
@@ -297,6 +333,7 @@ class SchemaAnnotator:
         self.bundled_schema = None
         self.ref_root = Path(ref_root) if ref_root else None
         self._follow_refs = follow_refs
+        self._provided_context = context
 
         self._process_schema(fn, url)
 
@@ -309,19 +346,29 @@ class SchemaAnnotator:
 
         base_url = schema.get('$id', base_url)
 
-        if not context_fn:
-            terms = {}
-        elif base_url:
-            context_fn = urljoin(base_url, context_fn)
-            terms = read_context_terms(url=context_fn)
-        else:
-            context_fn = Path(fn).parent / context_fn
-            terms = read_context_terms(file=context_fn)
+        terms = {}
+        prefixes = {}
+        types = {}
+
+        if context_fn != self._provided_context or not (isinstance(context_fn, Path)
+                                                        and isinstance(self._provided_context, Path)
+                                                        and self._provided_context.resolve() == context_fn.resolve()):
+            # Only load the provided context if it's different from the schema-referenced one
+            terms, prefixes, types = read_context_terms(self._provided_context)
+
+        if context_fn:
+            if base_url:
+                context_fn = urljoin(base_url, str(context_fn))
+            else:
+                context_fn = Path(fn).parent / context_fn
+
+            for e in zip((terms, prefixes, types), read_context_terms(context_fn)):
+                e[0].update(e[1])
 
         def process_properties(obj: dict):
             properties: dict[str, dict] = obj.get('properties') if obj else None
-            if not properties:
-                return
+            if not isinstance(properties, dict):
+                raise ValueError('"properties" must be a dictionary')
 
             empty_properties = []
             for prop, prop_value in properties.items():
@@ -330,7 +377,9 @@ class SchemaAnnotator:
                     continue
                 if prop in terms:
                     prop_value[ANNOTATION_ID] = terms[prop]
-                if '$ref' in prop_value:
+                    if prop in types:
+                        prop_value[ANNOTATION_TYPE] = types[prop]
+                if '$ref' in prop_value and self._follow_refs:
 
                     ref_fn, ref_url = resolve_ref(prop_value['$ref'], fn, url, base_url)
                     ref = ref_fn or ref_url
@@ -346,13 +395,28 @@ class SchemaAnnotator:
 
             properties.update({p: {ANNOTATION_ID: terms[p]} for p in empty_properties if p in terms})
 
-        schema_type = schema.get('type')
+        def process_subschema(subschema):
+
+            schema_type = subschema.get('type')
+            if not schema_type and 'properties' in subschema:
+                schema_type = 'object'
+
+            if schema_type == 'object':
+                process_properties(subschema)
+            elif schema_type == 'array':
+                for k in ('prefixItems', 'items', 'contains'):
+                    process_properties(subschema.get(k))
 
-        if schema_type == 'object':
-            process_properties(schema)
-        elif schema_type == 'array':
-            for k in ('prefixItems', 'items', 'contains'):
-                process_properties(schema.get(k))
+            for defs_prop in ('$defs', 'definitions'):
+                defs_value = subschema.get(defs_prop)
+                if isinstance(defs_value, dict):
+                    for defs_entry in defs_value.values():
+                        process_subschema(defs_entry)
+
+        process_subschema(schema)
+
+        if prefixes:
+            schema[ANNOTATION_PREFIXES] = prefixes
 
         self.schemas[fn or url] = AnnotatedSchema(
             source=fn or url,
@@ -389,18 +453,43 @@ class ContextBuilder:
 
         base_url = schema.get('$id', base_url)
 
+        prefixes = schema.get(ANNOTATION_PREFIXES, {}).items()
+        rev_prefixes = {v: k for k, v in prefixes}
+
+        def compact_uri(uri: str) -> str:
+            if uri.startswith('@'):
+                # JSON-LD keyword
+                return uri
+            parts = urlparse(uri)
+            if parts.fragment:
+                pref, suf = uri.rsplit('#', 1)
+                pref += '#'
+            elif len(parts.path) > 1:
+                pref, suf = uri.rsplit('/', 1)
+                pref += '/'
+            else:
+                return uri
+
+            if pref in rev_prefixes:
+                return f"{rev_prefixes[pref]}:{suf}"
+            else:
+                return uri
+
         own_context = {}
 
+        if prefixes:
+            own_context.update(prefixes)
+
         def read_properties(where: dict):
             if not isinstance(where, dict):
                 return
             for prop, prop_val in where.get('properties', {}).items():
                 if isinstance(prop_val, dict) and ANNOTATION_ID in prop_val:
                     prop_context = {
-                        '@id': prop_val[ANNOTATION_ID]
+                        '@id': compact_uri(prop_val[ANNOTATION_ID])
                     }
                     if ANNOTATION_TYPE in prop_val:
-                        prop_context['@type'] = prop_val[ANNOTATION_TYPE]
+                        prop_context['@type'] = compact_uri(prop_val[ANNOTATION_TYPE])
 
                     if '$ref' in prop_val:
                         ref_fn, ref_url = resolve_ref(prop_val['$ref'], fn, url, base_url)
@@ -459,6 +548,7 @@ def dump_annotated_schemas(annotator: SchemaAnnotator, subdir: Path | str = 'ann
     :param annotator: a `SchemaAnnotator` with the annotated schemas to read
     :param subdir: a name for the mirror directory
     :param root_dir: root directory for computing relative paths to schemas
+    :param output_fn_transform: optional callable to transform the output path
     """
     wd = (Path(root_dir) if root_dir else Path()).resolve()
     subdir = subdir if isinstance(subdir, Path) else Path(subdir)

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc/na/ingest_json.py

@@ -50,6 +50,7 @@ from rdflib.namespace import Namespace, DefinedNamespace
 from ogc.na import util, profile
 from ogc.na.domain_config import UpliftConfigurationEntry, DomainConfiguration
 from ogc.na.provenance import ProvenanceMetadata, FileProvenanceMetadata, generate_provenance
+from ogc.na.input_filters import apply_input_filter
 
 logger = logging.getLogger(__name__)
 
@@ -393,7 +394,8 @@ def process_file(input_fn: str | Path,
         If False, no Turtle output will be generated.
     :param context_fn: YAML context filename. If None, will be autodetected:
         1. From a file with the same name but yml/yaml extension (test.json -> test.yml)
-        2. From a _json-context.yml/_json-context.yaml file in the same directory
+        2. From the domain_cfg
+        3. From a _json-context.yml/_json-context.yaml file in the same directory
     :param domain_cfg: domain configuration with uplift definition locations
     :param base: base URI for JSON-LD
     :param provenance_base_uri: base URI for provenance resources
@@ -436,8 +438,15 @@ def process_file(input_fn: str | Path,
     if not contexts:
         raise MissingContextException('No context file provided and one could not be discovered automatically')
 
-    with open(input_fn, 'r') as j:
-        input_data = json.load(j)
+    # Apply input filter of first context only (if any)
+    input_filters = contexts[0].get('input-filter')
+    if input_filters:
+        if not isinstance(input_filters, dict):
+            raise ValueError('input-filter must be an object')
+        input_data = apply_input_filter(input_fn, input_filters)
+    else:
+        with open(input_fn, 'r') as j:
+            input_data = json.load(j)
 
     provenance_metadata: ProvenanceMetadata | None = None
     if provenance_base_uri is not False:
@@ -635,7 +644,10 @@ def process(input_files: str | Path | Sequence[str | Path],
         logger.info("Input files: %s", input_files)
         remaining_fn: deque = deque()
         for input_file in input_files:
-            remaining_fn.extend(input_file.split(','))
+            if isinstance(input_file, str):
+                remaining_fn.extend(input_file.split(','))
+            else:
+                remaining_fn.append(input_file)
         while remaining_fn:
             fn = str(remaining_fn.popleft())
 
@@ -645,9 +657,6 @@ def process(input_files: str | Path | Sequence[str | Path],
                 remaining_fn.extend(filenames_from_context(fn, domain_config=domain_cfg) or [])
                 continue
 
-            if not re.match(r'.*\.json-?(ld)?$', fn):
-                logger.debug('File %s does not match, skipping', fn)
-                continue
             logger.info('File %s matches, processing', fn)
             try:
                 result.append(process_file(

ogc_na-0.1.4/ogc/na/input_filters/__init__.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from importlib import import_module
+from io import BytesIO
+from pathlib import Path
+from typing import Any, IO, TextIO
+
+
+def apply_input_filter(stream: IO | bytes | str | Path, filters: dict[str, dict]) -> dict[str, Any] | list:
+    filter_name, filter_conf = filters.popitem()
+    try:
+        filter_mod = import_module(f"ogc.na.input_filters.{filter_name}")
+    except ImportError:
+        raise ValueError(f'Cannot find input filter with name "{filter_name}"')
+
+    content: bytes | None = None
+    if isinstance(stream, Path) or isinstance(stream, str):
+        with open(stream, 'rb') as f:
+            content = f.read()
+    elif isinstance(stream, TextIO):
+        content = stream.read().encode('utf-8')
+    else:
+        content = stream.read()
+
+    return filter_mod.apply_filter(content, filter_conf)

ogc_na-0.1.4/ogc/na/input_filters/csv.py

@@ -0,0 +1,71 @@
+"""
+CSV Input filter for ingest_json.
+
+Returns CSV rows as a list. Values will always be strings (no type inference or coercion is performed).
+
+Configuration values:
+
+* `rows` (default: `dict`): type of elements in the result list:
+    * `dict`: elements will be dictionaries, with the keys taken from the `header-row`.
+    * `list`: each resulting row will be an array values.
+* `header-row` (default: `0`): if `rows` is `dict`, the (0-based) index of the header row. All rows before the
+    header row will be skipped.
+* `skip-rows` (default: `0`): number of rows to skip at the beginning of the file (apart from the header and pre-header
+    ones if `rows` is `dict`).
+* `delimiter` (default: `,`): field separator character
+* `quotechar` (default: `"`): char used to quote (enclose) field values
+* `skip-empty-rows` (default: `True`): whether to omit empty rows (i.e., those with no values) from the result
+* `trim-values` (default: `False`): whether to apply `.strip()` to the resulting values
+"""
+from __future__ import annotations
+
+import csv
+from io import BytesIO, TextIOWrapper, StringIO
+from typing import IO, Any
+
+from ogc.na import util
+
+DEFAULT_CONF = {
+    'rows': 'dict',
+    'header-row': 0,
+    'skip-rows': 0,
+    'delimiter': ',',
+    'quotechar': '"',
+    'skip-empty-rows': True,
+    'trim-values': False,
+}
+
+
+def apply_filter(content: bytes, conf: dict[str, Any] | None) -> dict[str, Any] | list:
+    conf = util.deep_update(DEFAULT_CONF, conf) if conf else DEFAULT_CONF
+
+    textio = StringIO(content.decode('utf-8'))
+    reader = csv.reader(textio, delimiter=conf['delimiter'], quotechar=conf['quotechar'])
+
+    headers = None
+    if conf['rows'] == 'dict':
+        header_row = max(conf['header-row'], 0)
+        # Skip to header row
+        for i in range(header_row):
+            next(reader, None)
+        headers = next(reader, [])
+        if not headers:
+            return []
+
+    # Skip requested rows
+    for i in range(conf['skip-rows']):
+        next(reader, None)
+
+    result = []
+    for row in reader:
+        if not row and conf['skip-empty-rows']:
+            # skip empty rows
+            continue
+        if conf['trim-values']:
+            row = [v.strip() for v in row]
+        if conf['rows'] == 'list':
+            result.append(row)
+        else:
+            result.append(dict(zip(headers, row)))
+
+    return result

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc/na/profile.py

@@ -202,7 +202,7 @@ class ProfileRegistry:
             if isinstance(src, str) and src.startswith('sparql:'):
                 endpoint = src[len('sparql:'):]
                 logger.info("Fetching profiles from SPARQL endpoint %s", endpoint)
-                assert util.isurl(endpoint)
+                assert util.is_url(endpoint, http_only=True)
                 s = g.query(PROFILES_QUERY.replace('__SERVICE__', f"SERVICE <{endpoint}>")).graph
                 util.copy_triples(s, g)
             else:

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc/na/util.py

@@ -7,7 +7,8 @@ import os.path
 import shlex
 from glob import glob
 from pathlib import Path
-from typing import Optional, Union, Any
+from time import time
+from typing import Optional, Union, Any, Mapping, Hashable
 
 import requests
 import rfc3987
@@ -18,6 +19,7 @@ from urllib.parse import urlparse
 from ogc.na.validation import ValidationReport
 
 import yaml
+
 try:
     from yaml import CLoader as YamlLoader, CSafeLoader as SafeYamlLoader, CDumper as YamlDumper
 except ImportError:
@@ -105,12 +107,12 @@ def validate(g: Graph, shacl_graph: Graph, extra: Optional[Graph] = None) -> Val
                                            advanced=True))
 
 
-def isurl(url: str, http_only: bool = False) -> bool:
+def is_url(url: str, http_only: bool = False) -> bool:
     """
     Checks whether a string is a valid URL.
 
     :param url: the input string
-    :param http_only: whether to only accept HTTP and HTTPS URL's as valid
+    :param http_only: whether to only accept HTTP and HTTPS URLs as valid
     :return: `True` if this is a valid URL, otherwise `False`
     """
     if not url:
@@ -159,7 +161,7 @@ def dump_yaml(content: Any, filename: str | Path | None = None,
 
     :param content: content to convert to YAML.
     :param filename: optional filename to dump the content into. If None, string content will be returned.
-    :param kwargs: other args to pass to yaml.dump
+    :param kwargs: other args to pass to `yaml.dump()`
     """
     kwargs.setdefault('sort_keys', False)
     if filename:
@@ -188,14 +190,6 @@ def merge_dicts(src: dict, dst: dict) -> dict:
     return dst
 
 
-def is_url(s: str) -> bool:
-    try:
-        url = urlparse(s)
-        return bool(url.scheme and url.netloc)
-    except ValueError:
-        return False
-
-
 def glob_list_split(s: str, exclude_dirs: bool = True, recursive: bool = False) -> list[str]:
     result = []
     for e in shlex.split(s):
@@ -206,3 +200,42 @@ def glob_list_split(s: str, exclude_dirs: bool = True, recursive: bool = False)
                 if not exclude_dirs or os.path.isfile(fn):
                     result.append(fn)
     return result
+
+
+class LRUCache:
+
+    def __init__(self, maxsize: int = 10):
+        self._cache: dict[Hashable, Any] = {}
+        self._last_access: dict[Hashable, float] = {}
+        self._maxsize = maxsize
+
+    def __contains__(self, item):
+        return item in self._cache
+
+    def __len__(self):
+        return len(self._cache)
+
+    def get(self, key, default=None):
+        if not isinstance(key, Hashable):
+            return default
+        return self._cache.get(key, default)
+
+    def __setitem__(self, key, value):
+        if not isinstance(key, Hashable):
+            return
+        if len(self._cache) >= self._maxsize and key not in self._cache:
+            key_to_remove = min(self._last_access, key=self._last_access.get)
+            del self._cache[key_to_remove]
+            del self._last_access[key_to_remove]
+        self._cache[key] = value
+        self._last_access[key] = time()
+
+
+def deep_update(orig_dict: dict, with_dict: dict, replace: bool = False) -> dict:
+    dest = orig_dict if replace else {**orig_dict}
+    for k, v in with_dict.items():
+        if isinstance(v, Mapping):
+            dest[k] = deep_update(orig_dict.get(k, {}), with_dict, replace)
+        else:
+            dest[k] = v
+    return dest

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc_na.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ogc-na
-Version: 0.1.2
+Version: 0.1.4
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/

{ogc_na-0.1.2 → ogc_na-0.1.4}/ogc_na.egg-info/SOURCES.txt

@@ -20,6 +20,8 @@ ogc/na/provenance.py
 ogc/na/update_vocabs.py
 ogc/na/util.py
 ogc/na/validation.py
+ogc/na/input_filters/__init__.py
+ogc/na/input_filters/csv.py
 ogc_na.egg-info/PKG-INFO
 ogc_na.egg-info/SOURCES.txt
 ogc_na.egg-info/dependency_links.txt
@@ -30,8 +32,14 @@ rdf/domaincfg.vocab.ttl
 test/__init__.py
 test/test_annotate_schema.py
 test/test_ingest_json.py
+test/test_input_filters_csv.py
 test/test_profile.py
 test/data/empty.ttl
+test/data/headers.csv
+test/data/no-headers.csv
 test/data/profile_tree.ttl
 test/data/profile_tree_cyclic.ttl
+test/data/sample-context.jsonld
+test/data/sample-schema-prop-c.yml
+test/data/sample-schema.yml
 test/data/uplift_context_valid.yml

ogc_na-0.1.4/test/data/headers.csv

@@ -0,0 +1,89 @@
+"Year","Score","Title"
+1968,86,"Greetings"
+1970,17,"Bloody Mama"
+1970,73,"Hi,Mom!"
+1971,40,"Born to Win"
+1973,98,"Mean Streets"
+1973,88,"Bang the Drum Slowly"
+1974,97,"The Godfather,Part II"
+1976,41,"The Last Tycoon"
+1976,99,"Taxi Driver"
+1977,47,"1900"
+1977,67,"New York,New York"
+1978,93,"The Deer Hunter"
+1980,97,"Raging Bull"
+1981,75,"True Confessions"
+1983,90,"The King of Comedy"
+1984,89,"Once Upon a Time in America"
+1984,60,"Falling in Love"
+1985,98,"Brazil"
+1986,65,"The Mission"
+1987,100,"Dear America: Letters Home From Vietnam"
+1987,80,"The Untouchables"
+1987,78,"Angel Heart"
+1988,96,"Midnight Run"
+1989,64,"Jacknife"
+1989,47,"We're No Angels"
+1990,88,"Awakenings"
+1990,29,"Stanley & Iris"
+1990,96,"Goodfellas"
+1991,76,"Cape Fear"
+1991,69,"Mistress"
+1991,65,"Guilty by Suspicion"
+1991,71,"Backdraft"
+1992,87,"Thunderheart"
+1992,67,"Night and the City"
+1993,75,"This Boy's Life"
+1993,78,"Mad Dog and Glory"
+1993,96,"A Bronx Tale"
+1994,39,"Mary Shelley's Frankenstein"
+1995,80,"Casino"
+1995,86,"Heat"
+1996,74,"Sleepers"
+1996,38,"The Fan"
+1996,80,"Marvin's Room"
+1997,85,"Wag the Dog"
+1997,87,"Jackie Brown"
+1997,72,"Cop Land"
+1998,68,"Ronin"
+1998,38,"Great Expectations"
+1999,69,"Analyze This"
+1999,43,"Flawless"
+2000,43,"The Adventures of Rocky & Bullwinkle"
+2000,84,"Meet the Parents"
+2000,41,"Men of Honor"
+2001,73,"The Score"
+2001,33,"15 Minutes"
+2002,48,"City by the Sea"
+2002,27,"Analyze That"
+2003,4,"Godsend"
+2004,35,"Shark Tale"
+2004,38,"Meet the Fockers"
+2005,4,"The Bridge of San Luis Rey"
+2005,46,"Rent"
+2005,13,"Hide and Seek"
+2006,54,"The Good Shepherd"
+2007,21,"Arthur and the Invisibles"
+2007,76,"Captain Shakespeare"
+2008,19,"Righteous Kill"
+2008,51,"What Just Happened?"
+2009,46,"Everybody's Fine"
+2010,72,"Machete"
+2010,10,"Little Fockers"
+2010,50,"Stone"
+2011,25,"Killer Elite"
+2011,7,"New Year's Eve"
+2011,70,"Limitless"
+2012,92,"Silver Linings Playbook"
+2012,51,"Being Flynn"
+2012,29,"Red Lights"
+2013,46,"Last Vegas"
+2013,7,"The Big Wedding"
+2013,29,"Grudge Match"
+2013,11,"Killing Season"
+2014,9,"The Bag Man"
+2015,60,"Joy"
+2015,26,"Heist"
+2015,61,"The Intern"
+2016,11,"Dirty Grandpa"
+

ogc_na-0.1.4/test/data/no-headers.csv

@@ -0,0 +1,6 @@
+John,Doe,120 jefferson st.,Riverside, NJ, 08075
+Jack,McGinnis,220 hobo Av.,Phila, PA,09119
+"John ""Da Man""",Repici,120 Jefferson St.,Riverside, NJ,08075
+Stephen,Tyler,"7452 Terrace ""At the Plaza"" road",SomeTown,SD, 91234
+,Blankman,,SomeTown, SD, 00298
+"Joan ""the bone"", Anne",Jet,"9th, at Terrace plc",Desert City,CO,00123

ogc_na-0.1.4/test/data/sample-context.jsonld

@@ -0,0 +1,13 @@
+{
+  "@context": {
+    "prop": "http://example.com/props/",
+    "propA": "prop:a",
+    "propB": {
+      "@id": "prop:b",
+      "@context": {
+        "propD": "prop:d-inner"
+      }
+    },
+    "propD": "prop:d"
+  }
+}

ogc_na-0.1.4/test/data/sample-schema-prop-c.yml

@@ -0,0 +1,7 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+description: Sensor, Observation, Sample, and Actuator (SOSA)
+x-jsonld-context: sample-context.jsonld
+type: object
+properties:
+  propA:
+    type: object

ogc_na-0.1.4/test/data/sample-schema.yml

@@ -0,0 +1,12 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+description: Sensor, Observation, Sample, and Actuator (SOSA)
+x-jsonld-context: sample-context.jsonld
+type: object
+properties:
+  propA: {}
+  propB:
+    type: object
+    propD: {}
+  propC:
+    $ref: sample-schema-prop-c.yml
+  propD: {}

ogc_na-0.1.4/test/test_annotate_schema.py

@@ -0,0 +1,108 @@
+import unittest
+from pathlib import Path
+
+from rich import json
+
+from ogc.na import annotate_schema
+from ogc.na.annotate_schema import SchemaAnnotator
+
+THIS_DIR = Path(__file__).parent
+DATA_DIR = THIS_DIR / 'data'
+
+
+def deep_get(dct, *keys):
+    for key in keys:
+        dct = dct.get(key)
+        if dct is None:
+            return None
+    return dct
+
+
+class AnnotateSchemaTest(unittest.TestCase):
+
+    def test_resolve_ref_url_full(self):
+        ref = 'http://www.example.com/path/to/ref'
+        self.assertEqual(annotate_schema.resolve_ref(ref), (None, ref))
+
+    def test_resolve_ref_url_relative(self):
+        ref = '/path/to/ref'
+        base_url = 'http://www.example.com/base/url'
+        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
+                         (None, 'http://www.example.com/path/to/ref'))
+
+        ref = 'relative/ref'
+        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
+                         (None, 'http://www.example.com/base/relative/ref'))
+
+        ref = '../relative/ref'
+        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
+                         (None, 'http://www.example.com/relative/ref'))
+
+    def test_resolve_ref_filename(self):
+        ref = '/tmp/relative/test'
+        fn_from = '/var/lib/from.yml'
+
+        self.assertEqual(annotate_schema.resolve_ref(ref, fn_from),
+                         (Path(ref), None))
+
+        ref = 'child/ref'
+        self.assertEqual(annotate_schema.resolve_ref(ref, fn_from),
+                         (Path(fn_from).parent / ref, None))
+
+        ref = '../child/ref2'
+        result = annotate_schema.resolve_ref(ref, fn_from)
+        self.assertEqual(result[0].resolve(), Path(fn_from).parent.joinpath(ref).resolve(), None)
+        self.assertIsNone(result[1])
+
+    def test_resolve_ref_root(self):
+        ref = f'{annotate_schema.REF_ROOT_MARKER}tmp/relative/test'
+        ref_root = Path('/var/lib/root')
+
+        self.assertEqual(annotate_schema.resolve_ref(ref, ref_root=ref_root),
+                         (ref_root / 'tmp/relative/test', None))
+
+        self.assertEqual(annotate_schema.resolve_ref(ref, ref_root=None),
+                         (Path() / 'tmp/relative/test', None))
+
+    def test_annotate_follow_refs(self):
+        annotator = SchemaAnnotator(fn=DATA_DIR / 'sample-schema.yml', follow_refs=True)
+        schemas = annotator.schemas
+        self.assertEqual({*schemas.keys()},
+                          {Path(DATA_DIR / 'sample-schema.yml'), Path(DATA_DIR / 'sample-schema-prop-c.yml')})
+        full = schemas[Path(DATA_DIR / 'sample-schema.yml')].schema
+        prop_c = schemas[Path(DATA_DIR / 'sample-schema-prop-c.yml')].schema
+
+        self.assertEqual(deep_get(full, 'properties', 'propA', 'x-jsonld-id'), 'http://example.com/props/a')
+        self.assertEqual(deep_get(full, 'properties', 'propB', 'x-jsonld-id'), 'http://example.com/props/b')
+        self.assertEqual(deep_get(full, 'properties', 'propC', 'x-jsonld-id'), None)
+        self.assertEqual(deep_get(full, 'properties', 'propD', 'x-jsonld-id'), 'http://example.com/props/d')
+
+        self.assertEqual(deep_get(prop_c, 'properties', 'propA', 'x-jsonld-id'), 'http://example.com/props/a')
+
+    def test_annotate_no_follow_refs(self):
+        annotator = SchemaAnnotator(fn=DATA_DIR / 'sample-schema.yml', follow_refs=False)
+        schemas = annotator.schemas
+        self.assertEqual({*schemas.keys()}, {Path(DATA_DIR / 'sample-schema.yml')})
+        full = schemas[Path(DATA_DIR / 'sample-schema.yml')].schema
+
+        self.assertEqual(deep_get(full, 'properties', 'propA', 'x-jsonld-id'), 'http://example.com/props/a')
+        self.assertEqual(deep_get(full, 'properties', 'propB', 'x-jsonld-id'), 'http://example.com/props/b')
+        self.assertEqual(deep_get(full, 'properties', 'propC', 'x-jsonld-id'), None)
+        self.assertEqual(deep_get(full, 'properties', 'propD', 'x-jsonld-id'), 'http://example.com/props/d')
+
+    def test_annotate_provided_context(self):
+        annotator = SchemaAnnotator(fn=DATA_DIR / 'sample-schema.yml',
+                                    follow_refs=False,
+                                    context={
+                                        '@context': {
+                                            'another': 'http://example.net/another/',
+                                            'propA': 'another:a',
+                                            'propC': 'another:c'
+                                        }
+                                    })
+        schemas = annotator.schemas
+        self.assertEqual({*schemas.keys()}, {Path(DATA_DIR / 'sample-schema.yml')})
+        full = schemas[Path(DATA_DIR / 'sample-schema.yml')].schema
+
+        self.assertEqual(deep_get(full, 'properties', 'propA', 'x-jsonld-id'), 'http://example.com/props/a')
+        self.assertEqual(deep_get(full, 'properties', 'propC', 'x-jsonld-id'), 'http://example.net/another/c')

ogc_na-0.1.4/test/test_input_filters_csv.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import unittest
+from io import BytesIO
+from pathlib import Path
+
+from ogc.na.input_filters import csv
+
+THIS_DIR = Path(__file__).parent
+DATA_DIR = THIS_DIR / 'data'
+
+with open(DATA_DIR / 'headers.csv', 'rb') as f:
+    WITH_HEADERS = f.read()
+with open(DATA_DIR / 'no-headers.csv', 'rb') as f:
+    NO_HEADERS = f.read()
+
+
+class InputFiltersCSVTest(unittest.TestCase):
+
+    def test_rows_objects(self):
+        cfg = {
+            'rows': 'dict',
+            'skip-rows': 0,
+            'header-row': 0,
+        }
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], {
+            'Year': '1968',
+            'Score': '86',
+            'Title': 'Greetings',
+        })
+        self.assertEqual(rows[10], {
+            'Year': '1977',
+            'Score': '67',
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 3
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 84)
+        self.assertEqual(rows[0], {
+            'Year': '1971',
+            'Score': '40',
+            'Title': 'Born to Win',
+        })
+        self.assertEqual(rows[7], {
+            'Year': '1977',
+            'Score': '67',
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 0
+        cfg['header-row'] = 2
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 85)
+        self.assertEqual(rows[0], {
+            '1970': '1970',
+            '17': '73',
+            'Bloody Mama': 'Hi,Mom!',
+        })
+
+    def test_rows_lists(self):
+        cfg = {
+            'rows': 'list',
+            'skip-rows': 0,
+        }
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 88)
+        self.assertEqual(rows[0], ['Year', 'Score', 'Title'])
+        self.assertEqual(rows[10], ['1977', '47', '1900'])
+        cfg['skip-rows'] = 1
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], ['1968', '86', "Greetings"])
+        self.assertEqual(rows[3], ['1971', '40', "Born to Win"])
+        cfg['header-row'] = 2  # should have no effect
+        self.assertEqual(rows, csv.apply_filter(WITH_HEADERS, cfg))
+
+        cfg['skip-rows'] = 0
+        rows = csv.apply_filter(NO_HEADERS, cfg)
+        self.assertEqual(len(rows), 6)
+        self.assertEqual(rows[0], ['John', 'Doe', '120 jefferson st.', 'Riverside', ' NJ', ' 08075'])
+        cfg['trim-values'] = True
+        rows = csv.apply_filter(NO_HEADERS, cfg)
+        self.assertEqual(rows[0], ['John', 'Doe', '120 jefferson st.', 'Riverside', 'NJ', '08075'])

ogc_na-0.1.2/test/test_annotate_schema.py

@@ -1,51 +0,0 @@
-import unittest
-from pathlib import Path
-
-from ogc.na import annotate_schema
-
-
-class AnnotateSchemaTest(unittest.TestCase):
-
-    def test_resolve_ref_url_full(self):
-        ref = 'http://www.example.com/path/to/ref'
-        self.assertEqual(annotate_schema.resolve_ref(ref), (None, ref))
-
-    def test_resolve_ref_url_relative(self):
-        ref = '/path/to/ref'
-        base_url = 'http://www.example.com/base/url'
-        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
-                         (None, 'http://www.example.com/path/to/ref'))
-
-        ref = 'relative/ref'
-        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
-                         (None, 'http://www.example.com/base/relative/ref'))
-
-        ref = '../relative/ref'
-        self.assertEqual(annotate_schema.resolve_ref(ref, base_url=base_url),
-                         (None, 'http://www.example.com/relative/ref'))
-
-    def test_resolve_ref_filename(self):
-        ref = '/tmp/relative/test'
-        fn_from = '/var/lib/from.yml'
-
-        self.assertEqual(annotate_schema.resolve_ref(ref, fn_from),
-                         (Path(ref), None))
-
-        ref = 'child/ref'
-        self.assertEqual(annotate_schema.resolve_ref(ref, fn_from),
-                         (Path(fn_from).parent / ref, None))
-
-        ref = '../child/ref2'
-        result = annotate_schema.resolve_ref(ref, fn_from)
-        self.assertEqual(result[0].resolve(), Path(fn_from).parent.joinpath(ref).resolve(), None)
-        self.assertIsNone(result[1])
-
-    def test_resolve_ref_root(self):
-        ref = f'{annotate_schema.REF_ROOT_MARKER}tmp/relative/test'
-        ref_root = Path('/var/lib/root')
-
-        self.assertEqual(annotate_schema.resolve_ref(ref, ref_root=ref_root),
-                         (ref_root / 'tmp/relative/test', None))
-
-        self.assertEqual(annotate_schema.resolve_ref(ref, ref_root=None),
-                         (Path() / 'tmp/relative/test', None))