csvpath 0.0.2__py3-none-any.whl

csvpath/scanning/scanner.py

@@ -0,0 +1,165 @@
+import ply.yacc as yacc
+from csvpath.scanning.scanning_lexer import ScanningLexer
+from csvpath.parser_utility import ParserUtility
+from typing import List
+
+
+class UnexpectedException(Exception):
+    pass
+
+
+class Scanner(object):
+    tokens = ScanningLexer.tokens
+
+    def __init__(self):
+        self.lexer = ScanningLexer()
+        self.parser = yacc.yacc(module=self, start="path")
+        self.these: List = []
+        self.all_lines = False
+        self.filename = None
+        self.from_line = None
+        self.to_line = None
+        self.path = None
+        self.quiet = True
+        self.block_print = True
+        self.print(f"initialized Scanner: {self}")
+
+    def __str__(self):
+        return f"""
+            path: {self.path}
+            parser: {self.parser}
+            lexer: {self.lexer}
+            filename: {self.filename}
+            from_line: {self.from_line}
+            to_line: {self.to_line}
+            all_lines: {self.all_lines}
+            these: {self.these}
+        """
+
+    def print(self, msg: str) -> None:
+        if not self.block_print:
+            print(msg)
+
+    # ===================
+    # parse
+    # ===================
+
+    def parse(self, data):
+        self.path = data
+        self.parser.parse(data, lexer=self.lexer.lexer)
+        return self.parser
+
+    # ===================
+    # productions
+    # ===================
+
+    def p_error(self, p):
+        ParserUtility().error(self.parser, p)
+
+    # we'll want to use $name={ name: path} but for now we're treating filename as a file path
+    def p_root(self, p):
+        "root : ROOT filename"
+
+    def p_filename(self, p):
+        "filename : FILENAME"
+        self.filename = p[1]
+
+    def p_path(self, p):
+        "path : root LEFT_BRACKET expression RIGHT_BRACKET"
+        p[0] = p[3]
+
+    # ===================
+
+    def p_expression(self, p):
+        """expression : expression PLUS term
+        | expression MINUS term
+        | term"""
+        if len(p) == 4:
+            if p[2] == "+":
+                self._add_two_lines(p)
+            elif p[2] == "-":
+                self._collect_a_line_range(p)
+        else:
+            self._collect_a_line_number(p)
+        p[0] = self.these if self.these else [self.from_line]
+
+    def p_term(self, p):
+        """term : NUMBER
+        | NUMBER ALL_LINES
+        | ALL_LINES"""
+
+        if len(p) == 3:
+            self.from_line = p[1]
+
+        if p[len(p) - 1] == "*":
+            self.all_lines = True
+        else:
+            p[0] = [p[1]]
+
+    # ===================
+    # production support
+    # ===================
+
+    def _add_two_lines(self, p):
+        self._move_range_to_these()
+        if p[1] and p[1][0] not in self.these:
+            self.these.extend(p[1])
+        if p[3] and p[3][0] not in self.these:
+            self.these.extend(p[3])
+
+    def _collect_a_line_range(self, p):
+        if not isinstance(p[1], list):
+            raise UnexpectedException("non array in p[1]")
+        #
+        # if we continue to not raise unexpected exception we should remove the array tests!
+        #
+        if self.from_line and self.to_line:
+            # we have a from and to range. we have to move the range into
+            # these, then add this new range to these too
+            self._move_range_to_these()
+            fline = p[1][0] if isinstance(p[1], list) else p[1]
+            tline = p[3][0] if isinstance(p[3], list) else p[3]
+            self._add_range_to_these(fline, tline)
+        else:
+            if isinstance(p[1], list) and len(p[1]) == 1:
+                self.from_line = p[1][0]
+                if len(self.these) == 1 and self.these[0] == self.from_line:
+                    self.these = []
+            elif isinstance(p[1], list):
+                pass  # this is a list of several items -- i.e. it is self.these
+            else:
+                raise UnexpectedException("non array in p[1]")
+                self.from_line = p[1]  # does this ever happen?
+
+            if isinstance(p[3], list):
+                self.to_line = p[3][0]
+            else:
+                raise UnexpectedException("non array in p[3]")
+                self.to_line = p[3]  # does this ever happen?
+            # if we have a multi-element list on the left we set a range
+            # using the last item in the list as the from_line and
+            # the right side in the to_line. then we clear the range into these
+            if isinstance(p[1], list) and len(p[1]) > 1:
+                self.from_line = p[1][len(p[1]) - 1]
+                self._move_range_to_these()
+
+    def _collect_a_line_number(self, p):
+        if isinstance(p[1], list):
+            if p[1] and p[1][0] not in self.these:
+                self.these.extend(p[1])
+        elif not self.from_line:
+            self.from_line = p[1]
+
+    def _move_range_to_these(self):
+        if not self.from_line or not self.to_line:
+            return
+        for i in range(self.from_line, self.to_line + 1):
+            if i not in self.these:
+                self.these.append(i)
+        self.from_line = None
+        self.to_line = None
+
+    def _add_range_to_these(self, fline, tline):
+        for i in range(fline, tline + 1):
+            if i not in self.these:
+                self.these.append(i)

csvpath/scanning/scanning_lexer.py

@@ -0,0 +1,47 @@
+import ply.lex as lex
+
+
+class ScanningLexer(object):
+    tokens = [
+        "NUMBER",
+        "PLUS",
+        "MINUS",
+        "LEFT_BRACKET",
+        "RIGHT_BRACKET",
+        "ROOT",
+        "ANY",
+        "NAME",
+        "FILENAME",
+        "ALL_LINES",
+    ]
+
+    t_ignore = " \t\n\r"
+    t_ROOT = r"\$"
+    t_PLUS = r"\+"
+    t_MINUS = r"-"
+    t_LEFT_BRACKET = r"\["
+    t_RIGHT_BRACKET = r"\]"
+    t_ANY = r"\*"  # not yet used
+    t_NAME = r"[A-Z,a-z,0-9\._]+"
+    t_FILENAME = r"[A-Z,a-z,0-9\._/]+"
+    t_ALL_LINES = r"\*"
+
+    def t_NUMBER(self, t):
+        r"\d+"
+        t.value = int(t.value)
+        return t
+
+    def t_error(self, t):
+        print(f"Illegal character '{t.value[0]}'")
+        t.lexer.skip(1)
+
+    def __init__(self):
+        self.lexer = lex.lex(module=self)
+
+    def tokenize(self, data):
+        self.lexer.input(data)
+        while True:
+            tok = self.lexer.token()
+            if not tok:
+                break
+            yield tok

csvpath-0.0.2.dist-info/METADATA

@@ -0,0 +1,184 @@
+Metadata-Version: 2.1
+Name: csvpath
+Version: 0.0.2
+Summary: 
+Author: David Kershaw
+Author-email: dk107dk@hotmail.com
+Requires-Python: >=3.12,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: ply (>=3.11,<4.0)
+Requires-Dist: pytest (>=8.2.2,<9.0.0)
+Requires-Dist: python-dateutil (>=2.9.0.post0,<3.0.0)
+Description-Content-Type: text/markdown
+
+
+# CsvPath
+
+CsvPath defines a declarative syntax for inspecting and updating CSV files. Though much simpler, it is similar to:
+- XPath: CsvPath is to a CSV file like XPath is to an XML file
+- Schematron: Schematron is basically XPath rules applied using XSLT. CsvPath paths can be used as validation rules.
+- CSS selectors: CsvPath picks out structured data in a similar way to how CSS selectors pick out HTML structures.
+
+CsvPath is intended to fit with other DataOps and data quality tools. Files are streamed. The interface is simple. Custom functions can be added.
+
+# Usage
+CsvPath paths have two parts, scanning and matching. For usage, see the unit tests in [tests/test_scanner.py](tests/test_scanner.py), [tests/test_matcher.py](tests/test_matcher.py) and [tests/test_functions.py](tests/test_functions.py).
+
+    path = CsvPath(delimiter=",")
+    path.parse("$test.csv[5-25][#0=="Frog" @lastname=="Bats" count()==2]")
+    for i, line in enumerate( path.next() ):
+        print(f"{i}: {line}")
+
+    print(f"path vars: {path.variables}")
+
+This scanning and matching path says:
+- Open test.csv
+- Scan lines 5 through 25
+- Match the second time we see a line where the first column equals "Frog" and set the variable called  "lastname" to "Bats"
+
+# Scanning
+The scanner enumerates lines. For each line returned, the line number, the scanned line count, and the match count are available. The set of line numbers scanned is also available.
+
+The scan part of the path starts with '$' to indicate the root, meaning the file from the top. After the '$' comes the file path. The scanning instructions are in a bracket. The rules are:
+- `[*]` means all
+- `[3*]` means starting from line 3 and going to the end of the file
+- `[3]` by itself means just line 3
+- `[1-3]` means lines 1 through 3
+- `[1+3]` means lines 1 and line 3
+- `[1+3-8]` means line 1 and lines 3 through eight
+
+# Matching
+The match part is also bracketed. Matches have space separated
+components that are ANDed together. A match component is one of several types:
+<table>
+<tr>
+<td>Type</td>
+<td>Returns</td>
+<td>Matches</td>
+<td>Description</td>
+<td>Examples</td>
+</tr>
+    <tr>
+        <td>Term </td><td> Value </td><td> True when used alone, otherwise calculated </td>
+        <td>A quoted string or date, optionally quoted number, or
+        regex. Regex features are limited. A regex is wrapped  in "/" characters.</td>
+        <td>
+            <li/> `"Massachusetts"`
+            <li/> `89.7`
+            <li/> `/[0-9a-zA-Z]+!/`
+        </td>
+    </tr>
+    <tr>
+        <td>Function </td><td> Calculated   </td><td> Calculated </td>
+        <td>A function name followed by parentheses. Functions can
+contain terms, variables, headers and other  functions. Some functions
+take a specific or  unlimited number of types as arguments.     </td>
+        <td>
+            <li/> `not(count()==2)`
+        </td>
+    </tr>
+    <tr>
+        <td>Variable </td>
+        <td>Value</td>
+        <td>True/False when value tested. True when set, True/False existence when used alone</td>
+        <td>An @ followed by a name. A variable is
+            set or tested depending on the usage. By itself, it is an existence test. When used as
+            the left hand side of an "=" its value is set.
+            When it is used on either side of an "==" it is an equality test.
+        <td>
+            <li/> `@weather="cloudy"`
+            <li/> `count(@weather=="sunny")`
+            <li/> `@weather`
+            <li/> `#summer==@weather`
+
+#1 is an assignment that sets the variable and returns True. #2 is an argument used as a test in a way that is specific to the function. #3 is an existence test. #4 is a test.
+        </td>
+    </tr>
+    <tr>
+        <td>Header   </td>
+        <td>Value     </td>
+        <td>True/False existence when used alone, otherwise calculated</td>
+        <td>A # followed by a name or integer. The name references a value in line 0, the header
+ row. A number references a column by the 0-based column order.   </td>
+        <td>
+            <li/> `#firstname`
+            <li/> `#3`
+        </td>
+    </tr>
+    <tr>
+        <td>Equality</td>
+        <td>Calculated   </td>
+        <td>True at assignment, otherwise calculated   </td>
+        <td>Two of the other types joined with an "=" or "==".</td>
+        <td>
+            <li/> `@type_of_tree="Oak"`
+            <li/> `#name == @type_of_tree`
+        </td>
+    </tr>
+<table>
+
+    [ #common_name #0=="field" @tail=end() not(in(@tail, 'short|medium')) ]
+
+In the path above, the rules applied are:
+- `#common_name` indicates a header named "common_name". Headers are the values in the 0th line. This component of the match is an existence test.
+- `#2` means the 3rd column, counting from 0
+- Functions and column references are ANDed together
+- `@tail` creates a variable named "tail" and sets it to the value of the last column
+- Functions can contain functions, equality tests, and/or literals
+
+Most of the work of matching is done in functions. The match functions are:
+
+| Function                      | What it does                                              |Done|
+|-------------------------------|-----------------------------------------------------------|----|
+| add(value, value, ...)        | adds numbers                                              | X  |
+| after(value)                  | finds things after a date, number, string                 | X  |
+| average(number, type)         | returns the average up to current "line", "scan", "match" | X  |
+| before(value)                 | finds things before a date, number, string                | X  |
+| concat(value, value)          | counts the number of matches                              | X  |
+| count()                       | counts the number of matches                              | X  |
+| count(value)                  | count matches of value                                    | X  |
+| count_lines()                 | count lines to this point in the file                     | X  |
+| count_scans()                 | count lines we checked for match                          | X  |
+| divide(value, value, ...)     | divides numbers                                           | X  |
+| end()                         | returns the value of the last column                      | X  |
+| every(value, number)          | match every Nth time a value is seen                      | X  |
+| first(value)                  | match the first occurrence and capture line               | X  |
+| in(value, list)               | match in a pipe-delimited list                            | X  |
+| increment(value, n)           | increments a variable by n each time seen                 |    |
+| isinstance(value, typestr)    | tests for "int","float","complex","bool","usd"            | X  |
+| length(value)                 | returns the length of the value                           | X  |
+| lower(value)                  | makes value lowercase                                     | X  |
+| max(value, type)              | largest value seen up to current "line", "scan", "match"  | X  |
+| median(value, type)           | median value up to current "line", "scan", "match"        | X  |
+| min(value, type)              | smallest value seen up to current "line", "scan", "match" | X  |
+| multiply(value, value, ...)   | multiplies numbers                                        | X  |
+| no()                          | always false                                              | X  |
+| not(value)                    | negates a value                                           | X  |
+| now(format)                   | a datetime, optionally formatted                          | X  |
+| or(value, value,...)          | match any one                                             | X  |
+| percent(type)                 | % of total lines for "scan", "match", "line"              | X  |
+| random(list)                  | pick from a list                                          |    |
+| random(starting, ending)      | generates a random int from starting to ending            | X  |
+| regex(regex-string)           | match on a regular expression                             | X  |
+| subtract(value, value, ...)   | subtracts numbers                                         | X  |
+| tally(value, value, ...)      | counts times values are seen, including as a set          | X  |
+| then(y,m,d,hh,mm,ss,format)   | a datetime, optionally formatted                          |    |
+| upper(value)                  | makes value uppercase                                     | X  |
+
+# Not Ready For Production
+Anything could change. This project is a hobby.
+
+
+
+
+
+
+
+
+
+
+
+
+
+

csvpath-0.0.2.dist-info/RECORD

@@ -0,0 +1,54 @@
+csvpath/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+csvpath/csvpath.py,sha256=uLDNlR7dALsp9_sKsLKPhTxbgbzNjUUllauLBznvKvM,11858
+csvpath/matching/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+csvpath/matching/expression_encoder.py,sha256=fLkRyRwXL6bfEHhrZM_44Hueu6AiCzs4f9QcPxVIqxo,3614
+csvpath/matching/expression_math.py,sha256=eugzV7culmqQ91Dvrmba5vI1K_w6QN8VAE0JRHz19eE,4970
+csvpath/matching/expression_utility.py,sha256=tDCFZ96ugIDdt3pAkZQ2r9KJNn1S85-Tt_Dsp6fYoFA,734
+csvpath/matching/functions/above.py,sha256=X0v69Tm7YMGUraNoeZkJ08ulHyTtLGdE7ZIq5H0YfpU,1267
+csvpath/matching/functions/add.py,sha256=JyrT4Xc34-E4DWzy_k9MrD8gCsSuA223ULCCqcTdYwA,838
+csvpath/matching/functions/below.py,sha256=hQt63ATF3TFLcBn5M63wbBUrSt4zhvIN4zqUYjg_7qI,1264
+csvpath/matching/functions/concat.py,sha256=wmTOsY8awVcTSBoz4FsiKBCyRdN13yPP76BkqHLvgoQ,930
+csvpath/matching/functions/count.py,sha256=PESUdB69GAcAkKKzcDaZzhRm3c97ZmrQWiEg1IN5yoM,1642
+csvpath/matching/functions/count_lines.py,sha256=N2XXVzPuI54szfR-hujH3O0vKWLQpahBakb-MW_l1aw,328
+csvpath/matching/functions/count_scans.py,sha256=OfA1PEwz-c17pXjfvbTHhSreAAaxAQpr1HO80l4DZyY,347
+csvpath/matching/functions/divide.py,sha256=f-aIubXnPYdPXJtULOO3v8AIQK0jrwzNCiU4Ve8V2Yg,1049
+csvpath/matching/functions/end.py,sha256=6qOktqew2tLisdRf1xRvK2mI9vrx0kYfjlzMjRH8t6M,592
+csvpath/matching/functions/every.py,sha256=NcaqEFQ_6Gcs3JQFwA8OndSyuvI5umabbf4XaVgoYVo,1285
+csvpath/matching/functions/first.py,sha256=VR7BW-svsP6ZJBlHlOfjo-TOzHk4x18YYffchHbLMu4,1699
+csvpath/matching/functions/function.py,sha256=PFeRonjMzM4aSsU06-HcL1_4-sSGVsvRYZ6-u71yOoU,949
+csvpath/matching/functions/function_factory.py,sha256=2hCni4XKPOEJtsH5XNqLrB06XGIJmJTFF_wrxiAz7fw,4408
+csvpath/matching/functions/inf.py,sha256=NfFaWrO8DybuQBEvATWu54xnvlE_EgwIBQ3bKS3luTY,1385
+csvpath/matching/functions/is_instance.py,sha256=yMwciRcdR2Hd81JVXWWDEOw0lkSVAkRr3Y1Ud-JgOhw,3026
+csvpath/matching/functions/length.py,sha256=0xactwv9npFhwUOVOEoJQzaVxD2TqJoQCWW_kUzKw40,1032
+csvpath/matching/functions/lower.py,sha256=D6O3IQa80mo6ljqnZClPy687WWjz7ogLUdRImnLT7qo,669
+csvpath/matching/functions/minf.py,sha256=CI3IXnm6u5eapU35hmjALmBairfuAgJuyC0o0WSKs1I,5572
+csvpath/matching/functions/multiply.py,sha256=DiZBf-WYlklAhgEINh7u4oNu48AMIGyLHvVS9T6h8Sc,924
+csvpath/matching/functions/no.py,sha256=qTMiRFRVEvTjt-pHXyymVDS-Wbpya86az16G3MPMmpU,231
+csvpath/matching/functions/notf.py,sha256=EswnL10BK207_UuaWmSyYqOYsSbnAbnn6uR3j9OjQxM,845
+csvpath/matching/functions/now.py,sha256=02MxK71U3ysoXtumWC0-2t-4s6pcuBFUXMyQZZYAyGA,1181
+csvpath/matching/functions/orf.py,sha256=yhmMLEDwnYs2z1zpuwwUY8bMKUoFsDpb69wFvy4sQXw,953
+csvpath/matching/functions/percent.py,sha256=1iHg0rza--Hy3T4t8-uPl_zSVaHVygRlrOUqr3N7HXg,1063
+csvpath/matching/functions/random.py,sha256=MToY8craHnTk1sW6fRWT67PAsGE8lve1x7XnbP3C6ig,1119
+csvpath/matching/functions/regex.py,sha256=pGQ8ylaOQd22igemhLsqTY3arZTiB8ijcEG1AMa020s,1130
+csvpath/matching/functions/subtract.py,sha256=2LaC5qaewj7oNpnxaTaaJ6eX5MbSFYpYhuSDMgMLYVU,1002
+csvpath/matching/functions/tally.py,sha256=8UAzzuTAn9pKNywgetAs-jrVDrJgUGkA0ZtjFl2diZM,1265
+csvpath/matching/functions/upper.py,sha256=k5VvBe9svWNZZVWG54IH1FGVizECx6ux6oXeNVGoFx4,669
+csvpath/matching/matcher.py,sha256=zZGzgwcHOR3C8xhTVQELc0wCaTV2KiZ6xKAuyg0HLRk,6342
+csvpath/matching/matching_lexer.py,sha256=WnxAseLhTtdWq3KCTWEZLJSIntHOV0YN5FNrm8moX_g,1492
+csvpath/matching/parser.out,sha256=SeCgDdmNv37pTrJgmadBg9aBmyox2F5aZniSXYZgfII,54699
+csvpath/matching/parsetab.py,sha256=hPPwG-reUC9GpEOrKpH5NmaGCVZqypvBH71QNlT_YVo,25834
+csvpath/matching/productions/equality.py,sha256=PEaRepqrt4R73g8jwnp2Pye2cecx-c_gne1QocraV-8,4710
+csvpath/matching/productions/expression.py,sha256=goqTN9ikrrM5GwdtOcvN6s6XSOGbzsy9WvCVKtKUw4U,467
+csvpath/matching/productions/header.py,sha256=vlzZbJMgITAJfhU53fOFNg517VlI0VUYlbEZRA3KgbE,1075
+csvpath/matching/productions/matchable.py,sha256=VTkq9KYlhQVe3bE-jYaPSH5GsNHmDbkzdFOSbEnDtN4,1233
+csvpath/matching/productions/term.py,sha256=RxXNhhVTD7yI2gGEgzMJD6BL80Gxt0cUPQxcX4ioO9M,277
+csvpath/matching/productions/variable.py,sha256=euXats1XX1lm50iRQMc0VSDbCylDv5309728-NfoKQg,436
+csvpath/parser_utility.py,sha256=kDh6YYYd3Fhs8iWGRvXCnO4iaNkbfIYGPQKH2x9EOWc,1153
+csvpath/scanning/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+csvpath/scanning/parser.out,sha256=AmVGteuwH6JwD5wL8_J3-el64Mc6pgNblKxi2byiQIQ,56
+csvpath/scanning/parsetab.py,sha256=i7UQ0jqyT3lITtfe8dmQYAxKIf9rBjK4Qle8sGZxMBs,4106
+csvpath/scanning/scanner.py,sha256=LDUbF-wmUuyHdRK76vEdnUR7qihLE9aJ9mKn_MIccXo,5246
+csvpath/scanning/scanning_lexer.py,sha256=N3o9VTVwQegvJmIxOEFrVpQb2aly3L2hY9uv6cLBxxI,978
+csvpath-0.0.2.dist-info/METADATA,sha256=yUW-_mEq1FWApH4FiBOKv6A73jbeRgVhQS2YqrnjUDY,9078
+csvpath-0.0.2.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+csvpath-0.0.2.dist-info/RECORD,,

csvpath-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.8.1
+Root-Is-Purelib: true
+Tag: py3-none-any