ogc-na 0.3.55__tar.gz → 0.3.57__tar.gz

{ogc_na-0.3.55 → ogc_na-0.3.57}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ogc_na
-Version: 0.3.55
+Version: 0.3.57
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/
@@ -27,6 +27,7 @@ Requires-Dist: rfc3987
 Requires-Dist: requests-cache
 Requires-Dist: xmltodict
 Requires-Dist: jsonpointer~=2.4
+Requires-Dist: openpyxl~=3.1.5
 Requires-Dist: setuptools
 Provides-Extra: dev
 Requires-Dist: mkdocs>=1.4.2; extra == "dev"

{ogc_na-0.3.55 → ogc_na-0.3.57}/ogc/na/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.55'
-__version_tuple__ = version_tuple = (0, 3, 55)
+__version__ = version = '0.3.57'
+__version_tuple__ = version_tuple = (0, 3, 57)

{ogc_na-0.3.55 → ogc_na-0.3.57}/ogc/na/annotate_schema.py

@@ -123,6 +123,7 @@ import json
 import logging
 import re
 import sys
+from collections import deque
 from operator import attrgetter
 from pathlib import Path
 from typing import Any, AnyStr, Callable, Sequence, Iterable
@@ -177,6 +178,7 @@ class ReferencedSchema:
     chain: list = dataclasses.field(default_factory=list)
     ref: str | Path = None
     is_json: bool = False
+    anchors: dict[str, Any] = dataclasses.field(default_factory=dict)
 
 
 @dataclasses.dataclass
@@ -192,8 +194,27 @@ class SchemaResolver:
         self._schema_cache: dict[str | Path, Any] = {}
 
     @staticmethod
-    def _get_branch(schema: dict, ref: str):
-        return jsonpointer.resolve_pointer(schema, re.sub('^#', '', ref))
+    def _get_branch(schema: dict, ref: str, anchors: dict[str, Any] = None):
+        ref = re.sub('^#', '', ref)
+        if anchors and ref in anchors:
+            return anchors[ref]
+        return jsonpointer.resolve_pointer(schema, ref)
+
+    @staticmethod
+    def _find_anchors(schema: dict) -> dict[str, Any]:
+        anchors = {}
+
+        pending = deque((schema,))
+        while pending:
+            current = pending.popleft()
+            if isinstance(current, dict):
+                if '$anchor' in current:
+                    anchors[current['$anchor']] = current
+                pending.extend(current.values())
+            elif isinstance(current, list):
+                pending.extend(current)
+
+        return anchors
 
     def load_contents(self, s: str | Path) -> tuple[dict, bool]:
         """
@@ -252,11 +273,13 @@ class SchemaResolver:
                     raise ValueError('Local ref provided without an anchor: ' + ref)
                 return ReferencedSchema(location=from_schema.location,
                                         fragment=ref[1:],
-                                        subschema=SchemaResolver._get_branch(from_schema.full_contents, ref),
+                                        subschema=SchemaResolver._get_branch(from_schema.full_contents, ref,
+                                                                             from_schema.anchors),
                                         full_contents=from_schema.full_contents,
                                         chain=chain,
                                         ref=ref,
-                                        is_json=from_schema.is_json)
+                                        is_json=from_schema.is_json,
+                                        anchors=from_schema.anchors)
 
             if force_contents:
                 is_json = False
@@ -269,20 +292,23 @@ class SchemaResolver:
                     contents = force_contents
             else:
                 contents, is_json = self.load_contents(schema_source)
+            anchors = SchemaResolver._find_anchors(contents)
             if fragment:
                 return ReferencedSchema(location=schema_source, fragment=fragment,
-                                        subschema=SchemaResolver._get_branch(contents, fragment),
+                                        subschema=SchemaResolver._get_branch(contents, fragment, anchors),
                                         full_contents=contents,
                                         chain=chain,
                                         ref=ref,
-                                        is_json=is_json)
+                                        is_json=is_json,
+                                        anchors=anchors)
             else:
                 return ReferencedSchema(location=schema_source,
                                         subschema=contents,
                                         full_contents=contents,
                                         chain=chain,
                                         ref=ref,
-                                        is_json=is_json)
+                                        is_json=is_json,
+                                        anchors=anchors)
         except Exception as e:
             f = f" from {from_schema.location}" if from_schema else ''
             raise IOError(f"Error resolving reference {ref}{f}") from e
@@ -522,7 +548,7 @@ class SchemaAnnotator:
                         if vocab and ':' not in prop_ctx and prop_ctx not in JSON_LD_KEYWORDS:
                             prop_ctx = f"{vocab}{prop_ctx}"
                         return {'@id': prop_ctx}
-                    elif '@id' not in prop_ctx and not vocab:
+                    elif '@id' not in prop_ctx and '@reverse' not in prop_ctx and not vocab:
                         raise ValueError(f'Missing @id for property {prop} in context {json.dumps(ctx, indent=2)}')
                     else:
                         result = {k: v for k, v in prop_ctx.items() if k in JSON_LD_KEYWORDS}
@@ -751,10 +777,11 @@ class ContextBuilder:
                             self._missed_properties[full_property_path_str] = False
                         prop_context['@' + term[len(ANNOTATION_PREFIX):]] = term_val
 
-                if isinstance(prop_context.get('@id'), str):
-                    self.visited_properties[full_property_path_str] = prop_context['@id']
+                if isinstance(prop_context.get('@id'), str) or isinstance(prop_context.get('@reverse'), str):
+                    prop_id_value = prop_context.get('@id', prop_context.get('@reverse'))
+                    self.visited_properties[full_property_path_str] = prop_id_value
                     self._missed_properties[full_property_path_str] = False
-                    if prop_context['@id'] in ('@nest', '@graph'):
+                    if prop_id_value in ('@nest', '@graph'):
                         merge_contexts(onto_context, process_subschema(prop_val, from_schema, full_property_path))
                     else:
                         merge_contexts(prop_context['@context'],

ogc_na-0.3.57/ogc/na/input_filters/xlsx.py

@@ -0,0 +1,87 @@
+"""
+Excel (XLSX) Input filter for ingest_json.
+
+Processes Excel XLSX files with [openpyxl](https://openpyxl.readthedocs.io/en/stable/).
+
+Configuration values:
+
+* `worksheet` (default: `None`): The name of the worksheet to process. If `None`, the default one will be used.
+* `rows` (default: `dict`): type of elements in the result list:
+    * `dict`: elements will be dictionaries, with the keys taken from the `header-row`.
+    * `list`: each resulting row will be an array values.
+* `header-row` (default: `0`): if `rows` is `dict`, the (0-based) index of the header row. All rows before the
+    header row will be skipped.
+* `skip-rows` (default: `0`): number of rows to skip at the beginning of the file (apart from the header and pre-header
+    ones if `rows` is `dict`).
+* `skip-empty-rows` (default: `True`): whether to omit empty rows (i.e., those with no values) from the result.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+from io import BytesIO
+from typing import Any
+from openpyxl import load_workbook
+from openpyxl.cell import Cell
+
+from ogc.na import util
+
+DEFAULT_CONF = {
+    'worksheet': None,
+    'rows': 'dict',
+    'header-row': 0,
+    'skip-rows': 0,
+    'skip-empty-rows': True,
+}
+
+
+def _cell_to_json(c: Cell) -> str | float | int | None:
+    if isinstance(c.value, datetime):
+        return c.value.isoformat()
+    return c.value
+
+
+def apply_filter(content: bytes, conf: dict[str, Any] | None) -> tuple[dict[str, Any] | list, dict[str, Any] | None]:
+    conf = util.deep_update(DEFAULT_CONF, conf) if conf else DEFAULT_CONF
+
+    metadata = {
+        'filter': {
+            'conf': conf,
+        },
+    }
+
+    wb = load_workbook(filename=BytesIO(content), read_only=True)
+    if conf['worksheet']:
+        ws = wb[conf['worksheet']]
+    else:
+        ws = wb.worksheets[0]
+    rows = ws.rows
+    metadata['worksheet'] = ws.title
+
+    headers = None
+    if conf['rows'] == 'dict':
+        header_row = max(conf['header-row'], 0)
+        # Skip to header row
+        for i in range(header_row):
+            next(rows, None)
+        headers = next(rows, [])
+        if not headers:
+            return [], None
+        else:
+            headers = [_cell_to_json(c) for c in headers]
+        metadata['headers'] = headers
+
+    for i in range(conf['skip-rows']):
+        next(rows, None)
+
+    result = []
+    for row in rows:
+        row_values = [_cell_to_json(c) for c in row]
+        if conf['skip-empty-rows'] and not any(c is not None for c in row_values):
+            # skip empty rows
+            continue
+        if conf['rows'] == 'list':
+            result.append(row_values)
+        else:
+            result.append(dict(zip(headers, row_values)))
+
+    return result, metadata

{ogc_na-0.3.55 → ogc_na-0.3.57}/ogc_na.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: ogc_na
-Version: 0.3.55
+Version: 0.3.57
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/
@@ -27,6 +27,7 @@ Requires-Dist: rfc3987
 Requires-Dist: requests-cache
 Requires-Dist: xmltodict
 Requires-Dist: jsonpointer~=2.4
+Requires-Dist: openpyxl~=3.1.5
 Requires-Dist: setuptools
 Provides-Extra: dev
 Requires-Dist: mkdocs>=1.4.2; extra == "dev"

{ogc_na-0.3.55 → ogc_na-0.3.57}/ogc_na.egg-info/SOURCES.txt

@@ -28,6 +28,7 @@ ogc/na/util.py
 ogc/na/validation.py
 ogc/na/input_filters/__init__.py
 ogc/na/input_filters/csv.py
+ogc/na/input_filters/xlsx.py
 ogc/na/input_filters/xml.py
 ogc_na.egg-info/PKG-INFO
 ogc_na.egg-info/SOURCES.txt
@@ -40,14 +41,19 @@ test/__init__.py
 test/test_annotate_schema.py
 test/test_ingest_json.py
 test/test_input_filters_csv.py
+test/test_input_filters_xlsx.py
 test/test_profile.py
 test/data/empty.ttl
 test/data/headers.csv
+test/data/headers.xlsx
 test/data/no-headers.csv
+test/data/no-headers.xlsx
 test/data/profile_tree.ttl
 test/data/profile_tree_cyclic.ttl
 test/data/sample-context.jsonld
 test/data/sample-schema-prop-c.yml
 test/data/sample-schema.yml
+test/data/schema-anchors.json
 test/data/schema-vocab.yml
+test/data/two-sheets.xlsx
 test/data/uplift_context_valid.yml

{ogc_na-0.3.55 → ogc_na-0.3.57}/ogc_na.egg-info/requires.txt

@@ -12,6 +12,7 @@ rfc3987
 requests-cache
 xmltodict
 jsonpointer~=2.4
+openpyxl~=3.1.5
 setuptools
 
 [dev]

{ogc_na-0.3.55 → ogc_na-0.3.57}/requirements.txt

@@ -12,6 +12,7 @@ rfc3987
 requests-cache
 xmltodict
 jsonpointer~=2.4
+openpyxl~=3.1.5
 
 # to be removed once https://github.com/RDFLib/pySHACL/issues/212 is fixed
 setuptools

ogc_na-0.3.57/test/data/schema-anchors.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "properties": {
+    "name": { "$ref": "https://example.com/person/name#name" },
+    "age": { "$ref": "https://example.com/person/age#age" }
+  },
+  "required": [ "name", "age" ],
+  "$defs": {
+    "name": {
+      "$id": "https://example.com/person/name",
+      "$anchor": "name",
+      "type": "string"
+    },
+    "age": {
+      "$id": "https://example.com/person/age",
+      "$anchor": "age",
+      "type": "integer"
+    },
+    "deep": {
+      "type": "object",
+      "properties": {
+        "inner": {
+          "$anchor": "innerProp",
+          "type": "string"
+        }
+      }
+    }
+  }
+}

{ogc_na-0.3.55 → ogc_na-0.3.57}/test/test_annotate_schema.py

@@ -1,8 +1,9 @@
+import json
 import unittest
 from pathlib import Path
 
 from ogc.na import annotate_schema
-from ogc.na.annotate_schema import SchemaAnnotator, ContextBuilder
+from ogc.na.annotate_schema import SchemaAnnotator, ContextBuilder, ReferencedSchema, SchemaResolver
 
 THIS_DIR = Path(__file__).parent
 DATA_DIR = THIS_DIR / 'data'
@@ -109,3 +110,14 @@ class AnnotateSchemaTest(unittest.TestCase):
 
         self.assertEqual(deep_get(builder.context, '@context', '@vocab'), vocab)
         self.assertEqual(deep_get(builder.context, '@context', '@base'), base)
+
+    def test_schema_anchors(self):
+        with open(DATA_DIR / 'schema-anchors.json') as f:
+            schema = json.load(f)
+        anchors = SchemaResolver._find_anchors(schema)
+        self.assertSetEqual({'name', 'age', 'innerProp'}, set(anchors.keys()))
+
+        self.assertEqual(SchemaResolver._get_branch(schema, '#/$defs/name'), anchors.get('name'))
+        self.assertEqual(SchemaResolver._get_branch(schema, '#/$defs/age'), anchors.get('age'))
+        self.assertEqual(SchemaResolver._get_branch(schema, '#/$defs/deep/properties/inner'),
+                         anchors.get('innerProp'))

ogc_na-0.3.57/test/test_input_filters_xlsx.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+import unittest
+from io import BytesIO
+from pathlib import Path
+
+from ogc.na.input_filters import xlsx
+
+THIS_DIR = Path(__file__).parent
+DATA_DIR = THIS_DIR / 'data'
+
+with open(DATA_DIR / 'headers.xlsx', 'rb') as f:
+    WITH_HEADERS = f.read()
+with open(DATA_DIR / 'no-headers.xlsx', 'rb') as f:
+    NO_HEADERS = f.read()
+with open(DATA_DIR / 'two-sheets.xlsx', 'rb') as f:
+    TWO_SHEETS = f.read()
+
+
+class InputFiltersCSVTest(unittest.TestCase):
+
+    def test_rows_objects(self):
+        cfg = {
+            'rows': 'dict',
+            'skip-rows': 0,
+            'header-row': 0,
+        }
+        rows = xlsx.apply_filter(WITH_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], {
+            'Year': 1968,
+            'Score': 86,
+            'Title': 'Greetings',
+        })
+        self.assertEqual(rows[10], {
+            'Year': 1977,
+            'Score': 67,
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 3
+        rows = xlsx.apply_filter(WITH_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 84)
+        self.assertEqual(rows[0], {
+            'Year': 1971,
+            'Score': 40,
+            'Title': 'Born to Win',
+        })
+        self.assertEqual(rows[7], {
+            'Year': 1977,
+            'Score': 67,
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 0
+        cfg['header-row'] = 2
+        rows = xlsx.apply_filter(WITH_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 85)
+        self.assertEqual(rows[0], {
+            1970: 1970,
+            17: 73,
+            'Bloody Mama': 'Hi,Mom!',
+        })
+
+    def test_rows_lists(self):
+        cfg = {
+            'rows': 'list',
+            'skip-rows': 0,
+        }
+        rows = xlsx.apply_filter(WITH_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 88)
+        self.assertEqual(rows[0], ['Year', 'Score', 'Title'])
+        self.assertEqual(rows[10], [1977, 47, '1900'])
+        cfg['skip-rows'] = 1
+        rows = xlsx.apply_filter(WITH_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], [1968, 86, "Greetings"])
+        self.assertEqual(rows[3], [1971, 40, "Born to Win"])
+        cfg['header-row'] = 2  # should have no effect
+        self.assertEqual(rows, xlsx.apply_filter(WITH_HEADERS, cfg)[0])
+
+        cfg['skip-rows'] = 0
+        rows = xlsx.apply_filter(NO_HEADERS, cfg)[0]
+        self.assertEqual(len(rows), 6)
+        self.assertEqual(rows[0], ['John', 'Doe', '120 jefferson st.', 'Riverside', ' NJ', ' 08075'])
+
+    def test_several_sheets(self):
+        rows, metadata = xlsx.apply_filter(TWO_SHEETS, {})
+        self.assertEqual(metadata.get('worksheet'), 'FirstSheet')
+        self.assertEqual(rows[0], {'FirstCol1': 1, 'FirstCol2': 'DD37Cf93aecA6Dc'})
+
+        rows, metadata = xlsx.apply_filter(TWO_SHEETS, {'worksheet': 'SecondSheet'})
+        self.assertEqual(metadata.get('worksheet'), 'SecondSheet')
+        self.assertEqual(rows[0], {'SecondCol1': 11, 'SecondCol2': '216E205d6eBb815'})