csvpath 0.0.581__tar.gz → 0.0.584__tar.gz

{csvpath-0.0.581 → csvpath-0.0.584}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: csvpath
-Version: 0.0.581
+Version: 0.0.584
 Summary: A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers.
 Author: David Kershaw
 Author-email: info@csvpath.org

{csvpath-0.0.581 → csvpath-0.0.584}/assets/config/jenkins-local-sftp.ini

@@ -25,7 +25,7 @@ path = cache
 use_cache = no
 
 [functions]
-imports =
+imports = 
 
 [results]
 archive = tmp/sftp/archive
@@ -94,7 +94,7 @@ port = SFTPPLUS_PORT
 admin_username = SFTPPLUS_ADMIN_USERNAME
 admin_password = SFTPPLUS_ADMIN_PASSWORD
 api_url = https://. . . :10020/json
-scripts_dir =
+scripts_dir = 
 execute_timeout = 300
 
 [ckan]
@@ -109,5 +109,5 @@ timeout = 5
 verify = False
 
 [slack]
-webhook_url =
+webhook_url = 
 

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/csvpath.py

@@ -1234,6 +1234,13 @@ class CsvPath(ErrorCollector, Printer):  # pylint: disable=R0902, R0904
         )
         for line in reader.next():
             self.track_line(line=line)
+            #
+            # some formats embed headers in each line. JSONL headers are the dict keys in
+            # each line that uses a dict
+            #
+            if reader.updates_headers:
+                _ = reader.current_headers
+                self.headers = self.headers if _ is None else _
             yield line
         self.finalize()
 

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/matching/functions/json/jsonpath.py

@@ -1,7 +1,15 @@
 import json
 import pyjson5
 from typing import Any
-from jsonpath_ng import jsonpath, parse
+#
+# the ext supports a "full" JSONPath syntax; whereas, w/o .ext is too limiting
+# both should support single quotes in paths, which we need because atm csvpaths
+# require double quotes for terms. if that becomes a problem we either have to
+# allow single quoted strings (probably easily doable) or unwrapped jsonpaths,
+# same as we do with regexes; but this would probably be a pain.
+#
+from jsonpath_ng.ext import parse
+#from jsonpath_ng import jsonpath, parse
 from ..function_focus import ValueProducer
 from csvpath.matching.productions import Term, Variable, Header, Reference
 from ..function import Function
@@ -43,9 +51,9 @@ class JsonPath(ValueProducer):
         v = self._value_two(skip=skip)
         jpath = parse(v)
         r = jpath.find(j)
-        if r and len(r) == 1:
+        if r is not None and len(r) == 1:
             self.value = r[0].value
-        elif r:
+        elif r is not None:
             self.value = [_.value for _ in r]
         else:
             raise DataException("No value for jsonpath")

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/util/config.py

@@ -73,7 +73,7 @@ class Config:
     DEFAULT_CONFIG = f"""
 [extensions]
 csvpath_files = csvpath, csvpaths
-csv_files = csv, tsv, dat, tab, psv, ssv, xlsx
+csv_files = csv, tsv, dat, tab, psv, ssv, xlsx, jsonl
 
 [errors]
 csvpath = collect, fail, print

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/util/file_readers.py

@@ -23,6 +23,23 @@ class DataFileReader(ABC):
         self._encoding = None
         self.mode = mode
         self.encoding = encoding
+        self._current_headers = None
+        self._updates_headers = False
+
+    #
+    # some formats -- esp. JSONL -- embed their headers line-by-line
+    #
+    @property
+    def updates_headers(self) -> bool:
+        return self._updates_headers
+
+    @property
+    def current_headers(self) -> list[str]:
+        return self._current_headers
+
+    @current_headers.setter
+    def current_headers(self, headers: list[str]) -> None:
+        self._current_headers = headers
 
     @classmethod
     def register_data(cls, *, path, filelike) -> None:

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/util/json/json_data_reader.py

@@ -27,6 +27,7 @@ class JsonDataReader(DataFileReader):
         self.path = path
         self.mode = mode
         self.encoding = encoding
+        self._updates_headers = True
 
     def next(self) -> list[str]:
         with jsonlines.open(self.path) as reader:
@@ -35,10 +36,12 @@ class JsonDataReader(DataFileReader):
                 line = JsonReaderHelper.line_from_obj(obj, i)
                 if isinstance(line, tuple):
                     headers = line[0]
+                    self.current_headers = headers
                     line = line[1]
-                    yield headers
+                    # yield headers
                     yield line
                 else:
+                    self.current_headers = line[:]
                     yield line
                 i += 1
 

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/util/json/json_reader_helper.py

@@ -74,7 +74,13 @@ class JsonReaderHelper:
         if isinstance(obj, (tuple, list)):
             return ["" if o is None else str(o) for o in obj]
         elif isinstance(obj, dict):
-            obj = {key: obj[key] for key in sorted(obj)}
+            #
+            # new theory is that we don't sort the keys because headers change on a row-by-row
+            # so the positions in the header list don't have meaning and there is no other value
+            # to the keys being sorted, and, moreover, if the key order had any meaning to the
+            # data generator that information is lost if we sort.
+            #
+            # obj = {key: obj[key] for key in sorted(obj)}
             keys = list(obj.keys())
             #
             # we sort because a dict does require a certain order of its keys.
@@ -82,12 +88,14 @@ class JsonReaderHelper:
             # order on any given line.
             #
             line = [cls.string_or_date_string(obj[k]) for k in keys]
+            return (keys, line)
             #
             # make a header row if line number is 0 else just a data row
             #
             #
             # check if we have headers for values. i.e. keys == values in first line
             #
+            """
             same = True
             if line_number == 0:
                 for i, _ in enumerate(line):
@@ -97,6 +105,7 @@ class JsonReaderHelper:
             if line_number == 0 and not same:
                 return (keys, line)
             return line
+            """
         else:
             return cls.string_or_date_string(obj)
 

{csvpath-0.0.581 → csvpath-0.0.584}/csvpath/util/line_counter.py

@@ -36,7 +36,16 @@ class LineCounter:
                 lm.next_line(last_line=[], data=line)
                 if len(line) == 0 and self.csvpathx.skip_blank_lines:
                     continue
-                if (not headers or len(headers) == 0) and len(line) > 0:
+                #
+                # some data formats embed headers in each line -- e.g. JSONL.
+                # in that case, we never have a header line. we'll just take the
+                # first headers available. in JSONL headers are an even looser
+                # concept than in CSV. it is very possible that a JSONL file's
+                # headers never settle for more than one line.
+                #
+                if reader.updates_headers and reader.current_headers:
+                    line = reader.current_headers
+                if (not headers or len(headers) == 0) and line and len(line) > 0:
                     headers = line[:]
         if not headers:
             headers = []

{csvpath-0.0.581 → csvpath-0.0.584}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csvpath"
-version = "0.0.581"
+version = "0.0.584"
 description = "A data preboarding framework for ingesting, managing, and validating CSV, Excel, and other tabular data files using a Collect, Store, Validate, Publish architecture to create a trusted publisher for downstream data consumers."
 authors = ["David Kershaw <info@csvpath.org>"]
 readme = "README.md"