csvpath 0.0.454__tar.gz → 0.0.455__tar.gz

{csvpath-0.0.454 → csvpath-0.0.455}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: csvpath
-Version: 0.0.454
+Version: 0.0.455
 Summary: A declarative language for data extraction and validation of CSV files
 Author: David Kershaw
 Author-email: dk107dk@hotmail.com

csvpath-0.0.455/config/config.ini

@@ -0,0 +1,19 @@
+[csvpath_files]
+extensions = txt, csvpath, csvpaths
+
+[csv_files]
+extensions = txt, csv, tsv, dat, tab, psv, ssv
+
+[errors]
+csvpath = raise, collect, stop, fail
+csvpaths = raise, collect
+
+[logging]
+csvpath = info
+csvpaths = info
+log_file = logs/csvpath.log
+log_files_to_keep = 100
+log_file_size = 52428800
+
+[config]
+path = config/config.ini

{csvpath-0.0.454 → csvpath-0.0.455}/csvpath/managers/files_manager.py

@@ -68,7 +68,10 @@ class FilesManager(CsvPathsFilesManager):  # pylint: disable=C0115
                 path = os.path.join(base, p)
                 self.named_files[name] = path
             else:
-                pass
+                self.csvpaths.logger.debug(
+                    "Skipping %s because extension not in accept list",
+                    os.path.join(base, p),
+                )
 
     def add_named_file(self, *, name: str, path: str) -> None:
         self.named_files[name] = path

{csvpath-0.0.454 → csvpath-0.0.455}/csvpath/util/config.py

@@ -1,6 +1,7 @@
 from configparser import RawConfigParser
 from dataclasses import dataclass
 from os import path, environ
+import os
 from typing import Dict, List
 from enum import Enum
 import logging
@@ -57,17 +58,17 @@ class CsvPathConfig:
     def __init__(self, holder):
         self._holder = holder
         self._config = RawConfigParser()
+        self.log_file_handler = None
         self._configpath = environ.get(CsvPathConfig.CSVPATH_CONFIG_FILE_ENV)
         if self._configpath is None:
             self._configpath = CsvPathConfig.CONFIG
-        self.log_file_handler = None
         self._load_config()
 
     def reload(self):
+        self._config = RawConfigParser()
         self._load_config()
 
     def set_config_path_and_reload(self, path: str) -> None:
-        self._config = RawConfigParser()
         self._configpath = path
         self.reload()
 
@@ -76,9 +77,9 @@ class CsvPathConfig:
         return self._configpath
 
     def _get(self, section: str, name: str):
+        if self._config is None:
+            raise ConfigurationException("No config object available")
         try:
-            if self._config is None:
-                raise ConfigurationException("No config object available")
             s = self._config[section][name]
             s = s.strip()
             ret = None
@@ -92,31 +93,83 @@ class CsvPathConfig:
                 f"Check config at {self.config_path} for [{section}][{name}]"
             )
 
-    def _load_config(self):
-        if not path.isfile(self._configpath):
-            raise ConfigurationException(
-                "No config file at {self._configpath}"
-            )  # pragma: no cover
-        else:
-            self._config.read(self._configpath)
-            self.csvpath_file_extensions = self._get(
-                Sections.CSVPATH_FILES.value, "extensions"
-            )
-            self.csv_file_extensions = self._get(Sections.CSV_FILES.value, "extensions")
-
-            self.csvpath_errors_policy = self._get(Sections.ERRORS.value, "csvpath")
-            self.csvpaths_errors_policy = self._get(Sections.ERRORS.value, "csvpaths")
-
-            self.csvpath_log_level = self._get(Sections.LOGGING.value, "csvpath")
-            self.csvpaths_log_level = self._get(Sections.LOGGING.value, "csvpaths")
+    def _create_default_config(self) -> None:
+        if not path.exists("config"):
+            os.makedirs("config")
+            with open(CsvPathConfig.CONFIG, "w") as file:
+                c = """
+                    [csvpath_files]
+                    extensions = txt, csvpath, csvpaths
+                    [csv_files]
+                    extensions = txt, csv, tsv, dat, tab, psv, ssv
+                    [errors]
+                    csvpath = raise, collect, stop, fail
+                    csvpaths = raise, collect
+                    [logging]
+                    csvpath = info
+                    csvpaths = info
+                    log_file = logs/csvpath.log
+                    log_files_to_keep = 100
+                    log_file_size = 52428800
+                    [config]
+                    path =
+                """
+                file.write(c)
+            print(f"Creating a default config file at {CsvPathConfig.CONFIG}.")
+            print("If you want your config to be somewhere else remember to")
+            print("update the path in the default config.ini")
+
+    def _assure_logs_path(self) -> None:
+        filepath = self.log_file
+        if not filepath:
+            filepath = "logs/csvpath.log"
+            self.log_file = filepath
+        dirpath = self._get_dir_path(filepath)
+        if dirpath and not path.exists(dirpath):
+            os.makedirs(dirpath)
+
+    def _get_dir_path(self, filepath):
+        if filepath.find(os.sep) > -1:
+            dirpath = filepath[0 : filepath.rfind(os.sep)]
+            return dirpath
+        return None
+
+    def _assure_config_file_path(self) -> None:
+        if not self._configpath or not os.path.isfile(self._configpath):
+            self._configpath = CsvPathConfig.CONFIG
+            self._create_default_config()
 
-            self.log_file = self._get(Sections.LOGGING.value, LogFile.LOG_FILE.value)
-            self.log_files_to_keep = self._get(
-                Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value
-            )
-            self.log_file_size = self._get(
-                Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value
-            )
+    def _load_config(self, norecurse=False):
+        self._assure_config_file_path()
+        #
+        #
+        #
+        self._config.read(self._configpath)
+        self.csvpath_file_extensions = self._get(
+            Sections.CSVPATH_FILES.value, "extensions"
+        )
+        self.csv_file_extensions = self._get(Sections.CSV_FILES.value, "extensions")
+
+        self.csvpath_errors_policy = self._get(Sections.ERRORS.value, "csvpath")
+        self.csvpaths_errors_policy = self._get(Sections.ERRORS.value, "csvpaths")
+
+        self.csvpath_log_level = self._get(Sections.LOGGING.value, "csvpath")
+        self.csvpaths_log_level = self._get(Sections.LOGGING.value, "csvpaths")
+
+        self.log_file = self._get(Sections.LOGGING.value, LogFile.LOG_FILE.value)
+        self.log_files_to_keep = self._get(
+            Sections.LOGGING.value, LogFile.LOG_FILES_TO_KEEP.value
+        )
+        self.log_file_size = self._get(
+            Sections.LOGGING.value, LogFile.LOG_FILE_SIZE.value
+        )
+        path = self._get("config", "path")
+        if path:
+            path = path.strip().lower()
+        if path and path != "" and path != self._configpath.strip().lower():
+            self._configpath = path
+            self.reload()
+            return
         self.validate_config()
 
     def validate_config(self) -> None:
@@ -176,10 +229,14 @@ class CsvPathConfig:
         if self.csvpaths_log_level not in LogLevels:
             raise ConfigurationException(f"CsvPaths log level {_} is wrong")
         #
-        # log files
+        # log files config
         #
         if self.log_file is None or not isinstance(self.log_file, str):
             raise ConfigurationException(f"Log file path is wrong: {self.log_file}")
+        #
+        # make sure the log dir exists
+        #
+        self._assure_logs_path()
         if self.log_files_to_keep is None or not isinstance(
             self.log_files_to_keep, int
         ):

csvpath-0.0.455/docs/asbool.md

@@ -0,0 +1,22 @@
+
+# Asbool
+
+The `asbool` qualifier makes CsvPath consider the match component it is used on as a bool according to its value, rather than as an existence test.
+
+The difference is:
+
+- CsvPath evaluates "true" and "false" as their bool equivalents, `True` and `False` respectively
+- A match component used as an existence test without `asbool` evaluates to `True` or `False` based on any its `not None` condition, resulting in, for e.g., the value `False` == `True` because `False` exists
+
+As an example, the value of `not.asbool()` is assigned according to:
+
+| When                                        | Example             | Example's result    |
+|---------------------------------------------|---------------------|---------------------|
+| If used alone, as a boolean match condition | #a.asbool           | evaluated as a bool |
+| Assignment                                  | @a.asbool = #b      | True                |
+| With the `nocontrib` qualifier              | @a.nocontrib.asbool = no() | True. In this case `nocontrib` overrides the match value of `asbool`. |
+| With the `onchange` qualifier               | @a.onchange.asbool = @b | evaluated as a bool |
+| With the `latch` qualifier                  | @a.latch.asbool = @b| evaluated as a bool |
+| With the `onmatch` qualifier                | @a.onmatch.asbool = @b     | True for the purposes of the whole row matching and evaluated as a bool if/when it does |
+
+

csvpath-0.0.455/docs/assignment.md

@@ -0,0 +1,81 @@
+
+# The Role Of Qualifiers In Assignment
+
+Assignment is the process of setting a variable equal to a value from another variable or a header, function, or term. There is more to assignment than you might think.
+
+Assignments have this common `x = y` form:
+
+- `@x = #y`
+- `@x = "a value"`
+- `@x = @y`
+- `@x.y = @z`
+- `@x = count()`
+
+Where things get more interesting is qualifiers.
+
+There are a number of qualifiers. These are the ones that are involved with assignment:
+
+- `asbool`
+- `decrease`
+- `increase`
+- `latch`
+- `nocontrib`
+- `notnone`
+- `onchange`
+- `onmatch`
+
+These qualifiers are consistently applied to all variable assignments. This in contrast to the more case-by-case way qualifiers work in other match components.
+
+You can <a href='https://github.com/dk107dk/csvpath/blob/main/docs/qualifiers.md'>read about all the qualifiers here</a>.
+
+Qualifiers can go together in any order. CsvPath decides what to do in an assignment based on all the qualifiers it finds and which are applicable. There are three ways a qualifier relates to an assignment:
+
+- It may qualify the specific act of the assignment
+- It may qualify the assignment's relationship to the whole row
+- Or it can apply to just the actual value itself that is being assigned
+
+You can think of these happening in layers.
+
+## Match Values Of Assignments
+
+| Form                                               | Match value                                |
+|----------------------------------------------------|--------------------------------------------|
+| `@x = y`                                           | True                                       |
+| `@x.latch = y`                                     | True on first assignment, otherwise False  |
+| `@x.onchange = y`                                  | True if changed, otherwise False           |
+| `@x.onmatch = y`                                   | True if row matches, otherwise False       |
+| `@x.increase = y`                                  | True if y is > x, otherwise False          |
+| `@x.decrease = y`                                  | True if y is < x, otherwise False          |
+| `@x.notnone = y`                                   | True if y is not None, otherwise False     |
+| `@x.[any-qualifiers-except-nocontrib].asbool = y`  | True or False determined by the value of x |
+| `@x.[any-other-qualifiers].nocontrib = y`          | True
+
+A typical assignment doesn't contribute to the match decision for a row. On the other hand, `increase`, `decrease`, `latch`, `notnone`, and `onchange` assignments contribute to matching.
+
+A latched assignment is one where the variable is set once and then never changes. For any row, regardless if the latched variable has been set, the match value is True. Again, the idea is that the latched assignment is a non-consideration for matching.
+
+An `onchange` assignment is one that only happens when the variable's value changes. Obviously, if a variable's value is the same as as the new value it doesn't matter for that variable if the assignment happens. However, `onchange`'s effect is on the match, not the assignment. If a variable with `onchange` is assigned to a value that it already holds, the match fails for the whole row. Conversely, if the same variable is given a new value, the variable's contribution to the row matching is True.
+
+`onmatch` is similar. If a row matches in all other respects, its `onmatch` variable assignments happen. If the same row doesn't match in other respects, the `onmatch`ed variable concurs -- it doesn't make the assignment and returns False in the match.
+
+`notnone` simply blocks a variable assignment if the value to be assigned is None. In that case the match vote is negative.
+
+`increase` and `decreae` are similar to `notnone`. They block assignment and report False if the value to be assigned to the variable is less than or greater than the current value, respectively.
+
+`asbool` overrides the previously described variable qualifiers. If `asbool` is found, the assignment returns True or False in the match according to the value the variable is set to. The interpretation of the variable's new value as a bool is similar to Python's `bool(x)`, but with the addition of "true" and "false" being treated as the True and False value, respectively.
+
+Finally, `nocontrib` is superior to all the other variable qualifiers, from the perspective of match decision voting. If found, the assignment is not considered for matching.
+
+The primacy or order of consideration is:
+
+1. Assignments with no qualifiers always succeed
+2. `nocontrib` overrides other assignment qualifiers
+3. `asbool` overlays all other assignment qualifiers except nocontrib
+4. `onmatch` determines if any of the below qualifiers come into play
+4. `notnone` is the next highest priority
+5. `increase` and `decrease` are the next highest priority after `notnone`
+4. `onchange` or `latch` are the lowest priority, meaning that they can be overridden, blocked from consideration, or have their match vote (or absence of voting) modified by any of the other variable qualifiers.
+
+This may seem like a lot. And it is. The silver lining is that the qualifiers have a lot of expressive power that you can tap into when you need it, and ignore when you don't.
+
+

csvpath-0.0.455/docs/config.md

@@ -0,0 +1,69 @@
+
+# Config
+
+CsvPaths has a few config options. By default, the config options are in `./config/config.ini`. You can change the location of your .ini file in two ways:
+- Set a `CSVPATH_CONFIG_FILE` env var pointing to your file
+- Create an instance of CsvPathConfig, set its CONFIG property, and call the `reload()` method
+
+The config options, at this time, are about:
+- File extensions
+- Error handling
+- Logging
+
+There are two types of files you can set extensions for:
+- CSV files
+- CsvPath files
+
+## File Extensions
+
+The defaults for these are:
+
+```ini
+    [csvpath_files]
+    extensions = txt, csvpath
+
+    [csv_files]
+    extensions = txt, csv, tsv, dat, tab, psv, ssv
+```
+
+## Error Handling
+
+The error settings are for when CsvPath or CsvPaths instances encounter problems. The options are:
+- `stop` - Halt processing; the CsvPath stopped property is set to True
+- `fail` - Mark the currently running CsvPath as having failed
+- `raise` - Raise the exception in as noisy a way as possible
+- `quiet` - Do nothing that affects the system out; this protects command line redirection of `print()` output. Logging is also minimized such that errors that would release a lot of metadata are slimmed down.
+- `collect` - Collect the errors in the error results for the CsvPath. This option is available with and without a CsvPaths instance.
+
+Multiple of these settings can be configured together. Quiet and raise do not coexist well. Raise will win because seeing problems lets you fix them.
+
+## Logging
+
+Logging levels are set at the major-component level. The components are:
+- `csvpath`
+- `csvpaths`
+- `matcher`
+- `scanner`
+
+Four levels are available:
+- `error`
+- `warning`
+- `debug`
+- `info`
+
+The levels are intended for the same functionality as their Python equivalents.
+
+CsvPath logs are directed to a file. The log file settings are:
+- `log_file` - a path to the log
+- `log_files_to_keep` - a number of logs, 1 to 100, kept in rotation before being deleted
+- `log_file_size` - an indication of roughly when a log file will be rotated
+
+As an example:
+```ini
+    log_file = logs/csvpath.log
+    log_files_to_keep = 100
+    log_file_size = 52428800
+```
+
+
+

csvpath-0.0.455/docs/examples.md

@@ -0,0 +1,119 @@
+
+# Examples
+
+These are simple examples of csvpath match parts. Test them yourself before relying on them. See the unit test for more simple path ideas.
+
+1. Find a value
+
+```bash
+    [ ~A running average of ages from 5 and 85~
+        between(#age, 4, 86)
+        @average_age.onmatch = average(#age, "match")
+        last.nocontrib() -> print("The average age between 5 and 85 is $.variables.average_age")
+    ]
+```
+
+2. Create a file
+
+```bash
+    [ ~Create a new CSV file sampling sales greater than $2000 in a region~
+        #region == or( "emea", "us" )
+        @r = random(0,1)
+        @line = line_count()
+        gt(#sale, 2000)
+        @ave = average.test.onmatch(#sale, "line")
+
+        count_lines() == 1 ->
+                print("line, region, average, sale, salesperson")
+        @r == 1 ->
+                print("$.variables.line, $.headers.region, $.variables.ave, $.headers.sale, $.headers.seller")
+    ]
+```
+
+3. Validate a file
+
+```bash
+    [ ~Apply five rules to check if this file meets expectations~
+        @last_age.onchange = @current_age
+        @current_age = #age
+
+        length(#lastname)==30           -> print("$.csvpath.line_count: lastname $.headers.lastname is > 30")
+        not( column(2) == "firstname" ) -> print("$.csvpath.line_count: 3rd header must be firstname, not $headers.2")
+        not(any(header()))              -> print("$.csvpath.line_count: check for missing values")
+        not(in(#title, "ceo|minon"))    -> print("$.csvpath.line_count: title cannot be $.headers.title")
+        gt(@last_age, @current_age)     -> print("$.csvpath.line_count: check age, it went down!")
+    ]
+```
+
+4. Find a first value
+
+```bash
+    [ ~ Find the first times fruit were the most popular and the most recent popular fruit ~
+        @fruit = in( #food, "Apple|Pear|Blueberry")
+        exists( @fruit.asbool )
+        first.year.onmatch( #year )
+        @fruit.asbool -> print("$.headers.food was the most popular food for the first time in $.headers.year")
+        last.nocontrib() -> print("First years for a type of fruit: $.variables.year")
+    ]
+```
+
+5. Keep it simple
+
+This works:
+
+```bash
+    $/User/fred/some_dir/csvpaths/test.csv[*][
+        line_count() == 1 -> print("$.csvpath.headers")
+        not( line_count() == 1 ) -> stop()
+    ]
+```
+
+This is better:
+
+```bash
+    $test[*][
+        line_count() == 1 -> print("$.csvpath.headers")
+        not( line_count() == 1 ) -> stop()
+    ]
+```
+
+Still better:
+
+```bash
+    $test[*][
+        line_count() == 1 -> print("$.csvpath.headers")
+        stop()
+    ]
+```
+
+Moving on up:
+
+```bash
+    $test[*][
+        firstline() -> print("$.csvpath.headers")
+        stop()
+    ]
+```
+
+Getting there:
+
+```bash
+    $test[*][
+        print("$.csvpath.headers")
+        stop()
+    ]
+```
+
+Stop here?
+
+```bash
+    $test[0][ print("$.csvpath.headers") ]
+```
+
+Best:
+
+```bash
+    $[0][ print("$.csvpath.headers") ]
+```
+
+

csvpath-0.0.455/docs/files.md

@@ -0,0 +1,55 @@
+
+# Named Files
+
+The file identifier following the root `$` and preceding the scanning part of the csvpath can be:
+- A relative or absolute file path
+- A logical identifier that points indirectly to a physical file, as described below
+- The empty string, in which case the file association happens in CsvPaths on the fly
+
+Filenames must match this regular expression `[A-Z,a-z,0-9\._/\-\\#&]+`. I.e. they have:
+
+- alphanums
+- forward and backward slashes
+- dots
+- hash marks
+- dashes
+- underscores, and
+- ampersands.
+
+## Using CsvPaths To Work With Files
+
+You can use the `CsvPaths` class to set up a list of named files so that you can have more concise csvpaths. Named files can take the form of:
+
+- A JSON file with a dictionary of file system paths under name keys
+- A dict object passed into the CsvPaths object containing the same name-to-file-path structure
+- A file system path pointing to a directory that will be used to populate the named files with all contained files
+
+Using named files requires `CsvPaths`, but the configuration happens in a CsvPaths's <a href='https://github.com/dk107dk/csvpath/blob/main/csvpath/managers/files_manager.py'>FilesManager</a>.
+
+## Example
+
+```python
+    paths = CsvPaths()
+    paths.files_manager.add_named_file("test", "tests/test_resources/test.csv")
+    path = paths.csvpath()
+    path.parse( """$test[*][#firstname=="Fred"]""" )
+    rows = path.collect()
+```
+This csvpath will be applied to the file named `"test"` and match rows where the `firstname` is `"Fred"`. The matched rows will be returned from the `collect()` method.
+
+## FilesManager
+
+The FilesManager methods are:
+
+| Method                              | Description                                                         |
+|-------------------------------------|---------------------------------------------------------------------|
+| add_named_files_from_dir(dir_path)  | Adds all files to the named files set by names minus any extension  |
+| add_named_files_from_json(filename) | Adds named files paths from a dict JSON structure found in the file |
+| set_named_files(Dict[str, str])     | Replaces all named files with the contents of a dict                |
+| add_named_file(name, path)          | Adds a single named file path                                       |
+| get_named_file(name)                | Gets the full file path associated with the name                    |
+| remove_named_file(name)             | Removes a named file                                                |
+
+
+Using these methods you can setup a CsvPaths, like the example above, then use a csvpath like `$logical_name[*][yes()]` to apply the csvpath to the file named `logical_name` in your CsvPaths object's `files_manager`. This use is easy and nearly transparent.
+

csvpath-0.0.455/docs/functions/above.md

@@ -0,0 +1,34 @@
+
+# Above, Below
+
+The ordinal comparison functions have multiple aliases, but a single simple function. They are:
+
+- `above()`, `gt()`, `after()`
+- `below()`, `lt()`, `before()`
+
+The functionality is just what you would expect: they implement the `>` and `<` operators as functions.
+
+Above and below handle number, date, and string comparisons. At this time they don't have any settings that would refine their function for specific use cases. Comparison by the three types is attempted in this order:
+- Number
+- Date
+- String
+
+Keep in mind two important considerations:
+- A number compared with a stringified number is a viable comparison, no different than a number and a number. The conversion is the same as in the underlying Python.
+- These functions answer a question: _is this thing before or after this other thing?_ The answer to: _is None after 1048?_ is _no, it is not_. The result of comparing `None` or `nan` to a regular `int`, `float`, `date`, or `string` is `False`. The functions believe the question is valid and the answer is `False` because there is no ordinal relationship between the two things being compared.
+
+There are no differences between each function's three aliases. The reason to have them is just to suit the use case. For instance, in comparing two numbers `lt()` and `gt()` may feel more right; whereas, in finding a point in time between two dates `before()` and `after()` may feel like a better fit.
+
+## Examples
+
+```bash
+    $[*][ before(#graduation, 2003) ]
+```
+This path matches on year of graduation as an int.
+
+```bash
+    $[*][ before(@gradulation, date("2020-05-30", "%Y-%m-%d"))
+           after(@gradulation, date("2000-05-30", "%Y-%m-%d"))]
+
+This path uses `before()` and `after() to find if a date is between two dates.
+

csvpath-0.0.455/docs/functions/advance.md

@@ -0,0 +1,27 @@
+
+# Advance
+
+`advance()` allows your csvpath to skip ahead some number of rows. Skipping rows avoids the processing overhead of checking if the rows match. The rows skipped are, of course, iterated; however, depending on the content of your csvpath, substantial latency may be avoided. Advance may be most useful for sampling or spot-checking.
+
+Note that you cannot advance during an advance. This is because `advance()` is a match component and will not be activated during the skipped iterations of an existing call to `advance()`.
+
+## Examples
+
+```bash
+    $[1*][
+        ~ collect a sample of 1000 responders ~
+        below(count(), 1001) -> advance(random(1,50))
+    ]
+```
+
+This csvpath collects a random sample of 1000 rows, starting after the header row. The sampled rows are from 1 to 50 lines apart.
+
+One way to run it looks like this, using dict objects to identify the csvpaths and files. There are, of course, other simple ways to do it.
+
+```python
+    paths = CsvPaths()
+    paths.files_manager.set_named_files(nf)
+    paths.paths_manager.set_named_paths(np)
+    lines = paths.collect_paths(pathsname="sample", filename="survey")
+```
+

csvpath-0.0.455/docs/functions/after_blank.md

@@ -0,0 +1,18 @@
+
+# After Blank
+
+`after_blank()` matches when the current line was preceded by a line with no values.
+
+Bear in mind a few things:
+- By default CsvPath skips truly blank lines
+- Lines with delimiters but no data are not exactly blank
+- Lines with fewer than the expected number of headers aren't blank
+
+This function considers:
+- The physical lines in the file
+- The possibility of delimiters but no values
+
+Basically, if the preceding line had no data in any header or no characters at all (or only whitespace characters), `after_blank()` would return True.
+
+`after_blank()` takes no arguments and always returns a bool value.
+