ogc-na 0.1.3__tar.gz → 0.1.4__tar.gz

{ogc_na-0.1.3 → ogc_na-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ogc_na
-Version: 0.1.3
+Version: 0.1.4
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/

{ogc_na-0.1.3 → ogc_na-0.1.4}/ogc/na/ingest_json.py

@@ -50,6 +50,7 @@ from rdflib.namespace import Namespace, DefinedNamespace
 from ogc.na import util, profile
 from ogc.na.domain_config import UpliftConfigurationEntry, DomainConfiguration
 from ogc.na.provenance import ProvenanceMetadata, FileProvenanceMetadata, generate_provenance
+from ogc.na.input_filters import apply_input_filter
 
 logger = logging.getLogger(__name__)
 
@@ -393,7 +394,8 @@ def process_file(input_fn: str | Path,
         If False, no Turtle output will be generated.
     :param context_fn: YAML context filename. If None, will be autodetected:
         1. From a file with the same name but yml/yaml extension (test.json -> test.yml)
-        2. From a _json-context.yml/_json-context.yaml file in the same directory
+        2. From the domain_cfg
+        3. From a _json-context.yml/_json-context.yaml file in the same directory
     :param domain_cfg: domain configuration with uplift definition locations
     :param base: base URI for JSON-LD
     :param provenance_base_uri: base URI for provenance resources
@@ -436,8 +438,15 @@ def process_file(input_fn: str | Path,
     if not contexts:
         raise MissingContextException('No context file provided and one could not be discovered automatically')
 
-    with open(input_fn, 'r') as j:
-        input_data = json.load(j)
+    # Apply input filter of first context only (if any)
+    input_filters = contexts[0].get('input-filter')
+    if input_filters:
+        if not isinstance(input_filters, dict):
+            raise ValueError('input-filter must be an object')
+        input_data = apply_input_filter(input_fn, input_filters)
+    else:
+        with open(input_fn, 'r') as j:
+            input_data = json.load(j)
 
     provenance_metadata: ProvenanceMetadata | None = None
     if provenance_base_uri is not False:
@@ -635,7 +644,10 @@ def process(input_files: str | Path | Sequence[str | Path],
         logger.info("Input files: %s", input_files)
         remaining_fn: deque = deque()
         for input_file in input_files:
-            remaining_fn.extend(input_file.split(','))
+            if isinstance(input_file, str):
+                remaining_fn.extend(input_file.split(','))
+            else:
+                remaining_fn.append(input_file)
         while remaining_fn:
             fn = str(remaining_fn.popleft())
 
@@ -645,9 +657,6 @@ def process(input_files: str | Path | Sequence[str | Path],
                 remaining_fn.extend(filenames_from_context(fn, domain_config=domain_cfg) or [])
                 continue
 
-            if not re.match(r'.*\.json-?(ld)?$', fn):
-                logger.debug('File %s does not match, skipping', fn)
-                continue
             logger.info('File %s matches, processing', fn)
             try:
                 result.append(process_file(

ogc_na-0.1.4/ogc/na/input_filters/__init__.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from importlib import import_module
+from io import BytesIO
+from pathlib import Path
+from typing import Any, IO, TextIO
+
+
+def apply_input_filter(stream: IO | bytes | str | Path, filters: dict[str, dict]) -> dict[str, Any] | list:
+    filter_name, filter_conf = filters.popitem()
+    try:
+        filter_mod = import_module(f"ogc.na.input_filters.{filter_name}")
+    except ImportError:
+        raise ValueError(f'Cannot find input filter with name "{filter_name}"')
+
+    content: bytes | None = None
+    if isinstance(stream, Path) or isinstance(stream, str):
+        with open(stream, 'rb') as f:
+            content = f.read()
+    elif isinstance(stream, TextIO):
+        content = stream.read().encode('utf-8')
+    else:
+        content = stream.read()
+
+    return filter_mod.apply_filter(content, filter_conf)

ogc_na-0.1.4/ogc/na/input_filters/csv.py

@@ -0,0 +1,71 @@
+"""
+CSV Input filter for ingest_json.
+
+Returns CSV rows as a list. Values will always be strings (no type inference or coercion is performed).
+
+Configuration values:
+
+* `rows` (default: `dict`): type of elements in the result list:
+    * `dict`: elements will be dictionaries, with the keys taken from the `header-row`.
+    * `list`: each resulting row will be an array values.
+* `header-row` (default: `0`): if `rows` is `dict`, the (0-based) index of the header row. All rows before the
+    header row will be skipped.
+* `skip-rows` (default: `0`): number of rows to skip at the beginning of the file (apart from the header and pre-header
+    ones if `rows` is `dict`).
+* `delimiter` (default: `,`): field separator character
+* `quotechar` (default: `"`): char used to quote (enclose) field values
+* `skip-empty-rows` (default: `True`): whether to omit empty rows (i.e., those with no values) from the result
+* `trim-values` (default: `False`): whether to apply `.strip()` to the resulting values
+"""
+from __future__ import annotations
+
+import csv
+from io import BytesIO, TextIOWrapper, StringIO
+from typing import IO, Any
+
+from ogc.na import util
+
+DEFAULT_CONF = {
+    'rows': 'dict',
+    'header-row': 0,
+    'skip-rows': 0,
+    'delimiter': ',',
+    'quotechar': '"',
+    'skip-empty-rows': True,
+    'trim-values': False,
+}
+
+
+def apply_filter(content: bytes, conf: dict[str, Any] | None) -> dict[str, Any] | list:
+    conf = util.deep_update(DEFAULT_CONF, conf) if conf else DEFAULT_CONF
+
+    textio = StringIO(content.decode('utf-8'))
+    reader = csv.reader(textio, delimiter=conf['delimiter'], quotechar=conf['quotechar'])
+
+    headers = None
+    if conf['rows'] == 'dict':
+        header_row = max(conf['header-row'], 0)
+        # Skip to header row
+        for i in range(header_row):
+            next(reader, None)
+        headers = next(reader, [])
+        if not headers:
+            return []
+
+    # Skip requested rows
+    for i in range(conf['skip-rows']):
+        next(reader, None)
+
+    result = []
+    for row in reader:
+        if not row and conf['skip-empty-rows']:
+            # skip empty rows
+            continue
+        if conf['trim-values']:
+            row = [v.strip() for v in row]
+        if conf['rows'] == 'list':
+            result.append(row)
+        else:
+            result.append(dict(zip(headers, row)))
+
+    return result

{ogc_na-0.1.3 → ogc_na-0.1.4}/ogc/na/util.py

@@ -8,7 +8,7 @@ import shlex
 from glob import glob
 from pathlib import Path
 from time import time
-from typing import Optional, Union, Any, Hashable
+from typing import Optional, Union, Any, Mapping, Hashable
 
 import requests
 import rfc3987
@@ -112,7 +112,7 @@ def is_url(url: str, http_only: bool = False) -> bool:
     Checks whether a string is a valid URL.
 
     :param url: the input string
-    :param http_only: whether to only accept HTTP and HTTPS URL's as valid
+    :param http_only: whether to only accept HTTP and HTTPS URLs as valid
     :return: `True` if this is a valid URL, otherwise `False`
     """
     if not url:
@@ -161,7 +161,7 @@ def dump_yaml(content: Any, filename: str | Path | None = None,
 
     :param content: content to convert to YAML.
     :param filename: optional filename to dump the content into. If None, string content will be returned.
-    :param kwargs: other args to pass to yaml.dump
+    :param kwargs: other args to pass to `yaml.dump()`
     """
     kwargs.setdefault('sort_keys', False)
     if filename:
@@ -229,3 +229,13 @@ class LRUCache:
             del self._last_access[key_to_remove]
         self._cache[key] = value
         self._last_access[key] = time()
+
+
+def deep_update(orig_dict: dict, with_dict: dict, replace: bool = False) -> dict:
+    dest = orig_dict if replace else {**orig_dict}
+    for k, v in with_dict.items():
+        if isinstance(v, Mapping):
+            dest[k] = deep_update(orig_dict.get(k, {}), with_dict, replace)
+        else:
+            dest[k] = v
+    return dest

{ogc_na-0.1.3 → ogc_na-0.1.4}/ogc_na.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ogc-na
-Version: 0.1.3
+Version: 0.1.4
 Summary: OGC Naming Authority tools
 Author-email: Rob Atkinson <ratkinson@ogc.org>, Piotr Zaborowski <pzaborowski@ogc.org>, Alejandro Villar <avillar@ogc.org>
 Project-URL: Homepage, https://github.com/opengeospatial/ogc-na-tools/

{ogc_na-0.1.3 → ogc_na-0.1.4}/ogc_na.egg-info/SOURCES.txt

@@ -20,6 +20,8 @@ ogc/na/provenance.py
 ogc/na/update_vocabs.py
 ogc/na/util.py
 ogc/na/validation.py
+ogc/na/input_filters/__init__.py
+ogc/na/input_filters/csv.py
 ogc_na.egg-info/PKG-INFO
 ogc_na.egg-info/SOURCES.txt
 ogc_na.egg-info/dependency_links.txt
@@ -30,8 +32,11 @@ rdf/domaincfg.vocab.ttl
 test/__init__.py
 test/test_annotate_schema.py
 test/test_ingest_json.py
+test/test_input_filters_csv.py
 test/test_profile.py
 test/data/empty.ttl
+test/data/headers.csv
+test/data/no-headers.csv
 test/data/profile_tree.ttl
 test/data/profile_tree_cyclic.ttl
 test/data/sample-context.jsonld

ogc_na-0.1.4/test/data/headers.csv

@@ -0,0 +1,89 @@
+"Year","Score","Title"
+1968,86,"Greetings"
+1970,17,"Bloody Mama"
+1970,73,"Hi,Mom!"
+1971,40,"Born to Win"
+1973,98,"Mean Streets"
+1973,88,"Bang the Drum Slowly"
+1974,97,"The Godfather,Part II"
+1976,41,"The Last Tycoon"
+1976,99,"Taxi Driver"
+1977,47,"1900"
+1977,67,"New York,New York"
+1978,93,"The Deer Hunter"
+1980,97,"Raging Bull"
+1981,75,"True Confessions"
+1983,90,"The King of Comedy"
+1984,89,"Once Upon a Time in America"
+1984,60,"Falling in Love"
+1985,98,"Brazil"
+1986,65,"The Mission"
+1987,100,"Dear America: Letters Home From Vietnam"
+1987,80,"The Untouchables"
+1987,78,"Angel Heart"
+1988,96,"Midnight Run"
+1989,64,"Jacknife"
+1989,47,"We're No Angels"
+1990,88,"Awakenings"
+1990,29,"Stanley & Iris"
+1990,96,"Goodfellas"
+1991,76,"Cape Fear"
+1991,69,"Mistress"
+1991,65,"Guilty by Suspicion"
+1991,71,"Backdraft"
+1992,87,"Thunderheart"
+1992,67,"Night and the City"
+1993,75,"This Boy's Life"
+1993,78,"Mad Dog and Glory"
+1993,96,"A Bronx Tale"
+1994,39,"Mary Shelley's Frankenstein"
+1995,80,"Casino"
+1995,86,"Heat"
+1996,74,"Sleepers"
+1996,38,"The Fan"
+1996,80,"Marvin's Room"
+1997,85,"Wag the Dog"
+1997,87,"Jackie Brown"
+1997,72,"Cop Land"
+1998,68,"Ronin"
+1998,38,"Great Expectations"
+1999,69,"Analyze This"
+1999,43,"Flawless"
+2000,43,"The Adventures of Rocky & Bullwinkle"
+2000,84,"Meet the Parents"
+2000,41,"Men of Honor"
+2001,73,"The Score"
+2001,33,"15 Minutes"
+2002,48,"City by the Sea"
+2002,27,"Analyze That"
+2003,4,"Godsend"
+2004,35,"Shark Tale"
+2004,38,"Meet the Fockers"
+2005,4,"The Bridge of San Luis Rey"
+2005,46,"Rent"
+2005,13,"Hide and Seek"
+2006,54,"The Good Shepherd"
+2007,21,"Arthur and the Invisibles"
+2007,76,"Captain Shakespeare"
+2008,19,"Righteous Kill"
+2008,51,"What Just Happened?"
+2009,46,"Everybody's Fine"
+2010,72,"Machete"
+2010,10,"Little Fockers"
+2010,50,"Stone"
+2011,25,"Killer Elite"
+2011,7,"New Year's Eve"
+2011,70,"Limitless"
+2012,92,"Silver Linings Playbook"
+2012,51,"Being Flynn"
+2012,29,"Red Lights"
+2013,46,"Last Vegas"
+2013,7,"The Big Wedding"
+2013,29,"Grudge Match"
+2013,11,"Killing Season"
+2014,9,"The Bag Man"
+2015,60,"Joy"
+2015,26,"Heist"
+2015,61,"The Intern"
+2016,11,"Dirty Grandpa"
+

ogc_na-0.1.4/test/data/no-headers.csv

@@ -0,0 +1,6 @@
+John,Doe,120 jefferson st.,Riverside, NJ, 08075
+Jack,McGinnis,220 hobo Av.,Phila, PA,09119
+"John ""Da Man""",Repici,120 Jefferson St.,Riverside, NJ,08075
+Stephen,Tyler,"7452 Terrace ""At the Plaza"" road",SomeTown,SD, 91234
+,Blankman,,SomeTown, SD, 00298
+"Joan ""the bone"", Anne",Jet,"9th, at Terrace plc",Desert City,CO,00123

ogc_na-0.1.4/test/test_input_filters_csv.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import unittest
+from io import BytesIO
+from pathlib import Path
+
+from ogc.na.input_filters import csv
+
+THIS_DIR = Path(__file__).parent
+DATA_DIR = THIS_DIR / 'data'
+
+with open(DATA_DIR / 'headers.csv', 'rb') as f:
+    WITH_HEADERS = f.read()
+with open(DATA_DIR / 'no-headers.csv', 'rb') as f:
+    NO_HEADERS = f.read()
+
+
+class InputFiltersCSVTest(unittest.TestCase):
+
+    def test_rows_objects(self):
+        cfg = {
+            'rows': 'dict',
+            'skip-rows': 0,
+            'header-row': 0,
+        }
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], {
+            'Year': '1968',
+            'Score': '86',
+            'Title': 'Greetings',
+        })
+        self.assertEqual(rows[10], {
+            'Year': '1977',
+            'Score': '67',
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 3
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 84)
+        self.assertEqual(rows[0], {
+            'Year': '1971',
+            'Score': '40',
+            'Title': 'Born to Win',
+        })
+        self.assertEqual(rows[7], {
+            'Year': '1977',
+            'Score': '67',
+            'Title': 'New York,New York',
+        })
+
+        cfg['skip-rows'] = 0
+        cfg['header-row'] = 2
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 85)
+        self.assertEqual(rows[0], {
+            '1970': '1970',
+            '17': '73',
+            'Bloody Mama': 'Hi,Mom!',
+        })
+
+    def test_rows_lists(self):
+        cfg = {
+            'rows': 'list',
+            'skip-rows': 0,
+        }
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 88)
+        self.assertEqual(rows[0], ['Year', 'Score', 'Title'])
+        self.assertEqual(rows[10], ['1977', '47', '1900'])
+        cfg['skip-rows'] = 1
+        rows = csv.apply_filter(WITH_HEADERS, cfg)
+        self.assertEqual(len(rows), 87)
+        self.assertEqual(rows[0], ['1968', '86', "Greetings"])
+        self.assertEqual(rows[3], ['1971', '40', "Born to Win"])
+        cfg['header-row'] = 2  # should have no effect
+        self.assertEqual(rows, csv.apply_filter(WITH_HEADERS, cfg))
+
+        cfg['skip-rows'] = 0
+        rows = csv.apply_filter(NO_HEADERS, cfg)
+        self.assertEqual(len(rows), 6)
+        self.assertEqual(rows[0], ['John', 'Doe', '120 jefferson st.', 'Riverside', ' NJ', ' 08075'])
+        cfg['trim-values'] = True
+        rows = csv.apply_filter(NO_HEADERS, cfg)
+        self.assertEqual(rows[0], ['John', 'Doe', '120 jefferson st.', 'Riverside', 'NJ', '08075'])