jsonata-python 0.1.0__py3-none-any.whl

jsonata/tokenizer.py

@@ -0,0 +1,306 @@
+#
+# Copyright Robert Yokota
+# 
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+#
+# Derived from the following code:
+#
+#   Project name: jsonata-java
+#   Copyright Dashjoin GmbH. https://dashjoin.com
+#   Licensed under the Apache License, Version 2.0 (the "License")
+#
+#   Project name: JSONata
+# © Copyright IBM Corp. 2016, 2018 All Rights Reserved
+#   This project is licensed under the MIT License, see LICENSE
+#
+
+import math
+import re
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from jsonata import jexception, utils
+
+
+class Tokenizer:
+    operators = {
+        '.': 75,
+        '[': 80,
+        ']': 0,
+        '{': 70,
+        '}': 0,
+        '(': 80,
+        ')': 0,
+        ',': 0,
+        '@': 80,
+        '#': 80,
+        ';': 80,
+        ':': 80,
+        '?': 20,
+        '+': 50,
+        '-': 50,
+        '*': 60,
+        '/': 60,
+        '%': 60,
+        '|': 20,
+        '=': 40,
+        '<': 40,
+        '>': 40,
+        '^': 40,
+        '**': 60,
+        '..': 20,
+        ':=': 10,
+        '!=': 40,
+        '<=': 40,
+        '>=': 40,
+        '~>': 40,
+        'and': 30,
+        'or': 25,
+        'in': 40,
+        '&': 50,
+        '!': 0,  # not an operator, but needed as a stop character for name tokens
+        '~': 0  # not an operator, but needed as a stop character for name tokens
+    }
+
+    escapes = {
+        '"': '"',
+        '\\': '\\',
+        '/': '/',
+        'b': '\b',
+        'f': '\f',
+        'n': '\n',
+        'r': '\r',
+        't': '\t'
+    }
+
+    # Tokenizer (lexer) - invoked by the parser to return one token at a time
+
+    position: int
+    depth: int
+    path: str
+    length: int
+
+    def __init__(self, path):
+        self.position = 0
+        self.depth = 0
+
+        self.path = path
+        self.length = len(path)
+
+    @dataclass
+    class Token:
+        type: Optional[str]
+        value: Optional[Any]
+        position: int
+        id: Optional[Any] = None
+
+    def create(self, type: Optional[str], value: Optional[Any]) -> Token:
+        return Tokenizer.Token(type, value, self.position)
+
+    def is_closing_slash(self, position: int) -> bool:
+        if self.path[position] == '/' and self.depth == 0:
+            backslash_count = 0
+            while self.path[position - (backslash_count + 1)] == '\\':
+                backslash_count += 1
+            if int(math.fmod(backslash_count, 2)) == 0:
+                return True
+        return False
+
+    def scan_regex(self) -> re.Pattern:
+        # the prefix '/' will have been previously scanned. Find the end of the regex.
+        # search for closing '/' ignoring any that are escaped, or within brackets
+        start = self.position
+        # int depth = 0
+        pattern = None
+        flags = None
+
+        while self.position < self.length:
+            current_char = self.path[self.position]
+            if self.is_closing_slash(self.position):
+                # end of regex found
+                pattern = self.path[start:self.position]
+                if pattern == "":
+                    raise jexception.JException("S0301", self.position)
+                self.position += 1
+                current_char = self.path[self.position]
+                # flags
+                start = self.position
+                while current_char == 'i' or current_char == 'm':
+                    self.position += 1
+                    current_char = self.path[self.position]
+                flags = self.path[start:self.position] + 'g'
+
+                # Convert flags to Java Pattern flags
+                _flags = 0
+                if "i" in flags:
+                    _flags |= re.I
+                if "m" in flags:
+                    _flags |= re.M
+                return re.compile(pattern, _flags)  # Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);
+            if (current_char == '(' or current_char == '[' or current_char == '{') and self.path[self.position - 1] != '\\':
+                self.depth += 1
+            if (current_char == ')' or current_char == ']' or current_char == '}') and self.path[self.position - 1] != '\\':
+                self.depth -= 1
+            self.position += 1
+        raise jexception.JException("S0302", self.position)
+
+    def next(self, prefix: bool) -> Optional[Token]:
+        if self.position >= self.length:
+            return None
+        current_char = self.path[self.position]
+        # skip whitespace
+        while self.position < self.length and " \t\n\r".find(current_char) > -1:
+            self.position += 1
+            if self.position >= self.length:
+                return None  # Uli: JS relies on charAt returns null
+            current_char = self.path[self.position]
+        # skip comments
+        if current_char == '/' and self.path[self.position + 1] == '*':
+            comment_start = self.position
+            self.position += 2
+            current_char = self.path[self.position]
+            while not (current_char == '*' and self.path[self.position + 1] == '/'):
+                self.position += 1
+                current_char = self.path[self.position]
+                if self.position >= self.length:
+                    # no closing tag
+                    raise jexception.JException("S0106", comment_start)
+            self.position += 2
+            current_char = self.path[self.position]
+            return self.next(prefix)  # need this to swallow any following whitespace
+        # test for regex
+        if not prefix and current_char == '/':
+            self.position += 1
+            return self.create("regex", self.scan_regex())
+        # handle double-char operators
+        have_more = self.position < len(self.path) - 1  # Java: position+1 is valid
+        if current_char == '.' and have_more and self.path[self.position + 1] == '.':
+            # double-dot .. range operator
+            self.position += 2
+            return self.create("operator", "..")
+        if current_char == ':' and have_more and self.path[self.position + 1] == '=':
+            # := assignment
+            self.position += 2
+            return self.create("operator", ":=")
+        if current_char == '!' and have_more and self.path[self.position + 1] == '=':
+            # !=
+            self.position += 2
+            return self.create("operator", "!=")
+        if current_char == '>' and have_more and self.path[self.position + 1] == '=':
+            # >=
+            self.position += 2
+            return self.create("operator", ">=")
+        if current_char == '<' and have_more and self.path[self.position + 1] == '=':
+            # <=
+            self.position += 2
+            return self.create("operator", "<=")
+        if current_char == '*' and have_more and self.path[self.position + 1] == '*':
+            # **  descendant wildcard
+            self.position += 2
+            return self.create("operator", "**")
+        if current_char == '~' and have_more and self.path[self.position + 1] == '>':
+            # ~>  chain function
+            self.position += 2
+            return self.create("operator", "~>")
+        # test for single char operators
+        if Tokenizer.operators.get(str(current_char)) is not None:
+            self.position += 1
+            return self.create("operator", current_char)
+        # test for string literals
+        if current_char == '"' or current_char == '\'':
+            quote_type = current_char
+            # double quoted string literal - find end of string
+            self.position += 1
+            qstr = ""
+            while self.position < self.length:
+                current_char = self.path[self.position]
+                if current_char == '\\':
+                    self.position += 1
+                    current_char = self.path[self.position]
+                    if Tokenizer.escapes.get(str(current_char)) is not None:
+                        qstr += Tokenizer.escapes[str(current_char)]
+                    elif current_char == 'u':
+                        #  u should be followed by 4 hex digits
+                        octets = self.path[self.position + 1:(self.position + 1) + 4]
+                        if re.match("^[0-9a-fA-F]+$", octets):
+                            codepoint = int(octets, 16)
+                            qstr += chr(codepoint)
+                            self.position += 4
+                        else:
+                            raise jexception.JException("S0104", self.position)
+                    else:
+                        # illegal escape sequence
+                        raise jexception.JException("S0301", self.position, current_char)
+
+                elif current_char == quote_type:
+                    self.position += 1
+                    return self.create("string", qstr)
+                else:
+                    qstr += current_char
+                self.position += 1
+            raise jexception.JException("S0101", self.position)
+        # test for numbers
+        numregex = re.compile("^-?(0|([1-9][0-9]*))(\\.[0-9]+)?([Ee][-+]?[0-9]+)?")
+        match_ = numregex.search(self.path[self.position:])
+        if match_ is not None:
+            num = float(match_.group(0))
+            if not math.isnan(num) and math.isfinite(num):
+                self.position += len(match_.group(0))
+                # If the number is integral, use long as type
+                return self.create("number", utils.Utils.convert_number(num))
+            else:
+                raise jexception.JException("S0102", self.position)  # , match.group[0]);
+
+        # test for quoted names (backticks)
+        name = None
+        if current_char == '`':
+            # scan for closing quote
+            self.position += 1
+            end = self.path.find('`', self.position)
+            if end != -1:
+                name = self.path[self.position:end]
+                self.position = end + 1
+                return self.create("name", name)
+            self.position = self.length
+            raise jexception.JException("S0105", self.position)
+        # test for names
+        i = self.position
+        while True:
+            # if (i>=length) return null; // Uli: JS relies on charAt returns null
+
+            ch = self.path[i] if i < self.length else chr(0)
+            if i == self.length or " \t\n\r".find(ch) > -1 or str(ch) in Tokenizer.operators:
+                if self.path[self.position] == '$':
+                    # variable reference
+                    _name = self.path[self.position + 1:i]
+                    self.position = i
+                    return self.create("variable", _name)
+                else:
+                    _name = self.path[self.position:i]
+                    self.position = i
+                    if _name == "or" or _name == "in" or _name == "and":
+                        return self.create("operator", _name)
+                    elif _name == "true":
+                        return self.create("value", True)
+                    elif _name == "false":
+                        return self.create("value", False)
+                    elif _name == "null":
+                        return self.create("value", None)
+                    else:
+                        if self.position == self.length and _name == "":
+                            # whitespace at end of input
+                            return None
+                        return self.create("name", _name)
+            else:
+                i += 1

jsonata/utils.py

@@ -0,0 +1,150 @@
+#
+# Copyright Robert Yokota
+# 
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Derived from the following code:
+#
+#   Project name: jsonata-java
+#   Copyright Dashjoin GmbH. https://dashjoin.com
+#   Licensed under the Apache License, Version 2.0 (the "License")
+#
+
+import math
+from typing import Any, MutableMapping, MutableSequence, Optional
+
+from jsonata import jexception
+
+
+class Utils:
+    class NullValue:
+        def __repr__(self):
+            return "null"
+
+    NULL_VALUE = NullValue()
+
+    @staticmethod
+    def is_numeric(v: Optional[Any]) -> bool:
+        if isinstance(v, bool):
+            return False
+        if isinstance(v, int):
+            return True
+        is_num = False
+        if isinstance(v, float):
+            is_num = not math.isnan(v)
+            if is_num and not math.isfinite(v):
+                raise jexception.JException("D1001", 0, v)
+        return is_num
+
+    @staticmethod
+    def is_array_of_strings(v: Optional[Any]) -> bool:
+        if isinstance(v, list):
+            for o in v:
+                if not isinstance(o, str):
+                    return False
+            return True
+        return False
+
+    @staticmethod
+    def is_array_of_numbers(v: Optional[Any]) -> bool:
+        if isinstance(v, list):
+            for o in v:
+                if not Utils.is_numeric(o):
+                    return False
+            return True
+        return False
+
+    @staticmethod
+    def is_function(o: Optional[Any]) -> bool:
+        from jsonata import jsonata
+        return isinstance(o, (jsonata.Jsonata.JFunction, jsonata.Jsonata.JFunctionCallable))
+
+    NONE = object()
+
+    @staticmethod
+    def create_sequence(el: Optional[Any] = NONE) -> list:
+        sequence = Utils.JList()
+        sequence.sequence = True
+        if el is not Utils.NONE:
+            if isinstance(el, list) and len(el) == 1:
+                sequence.append(el[0])
+            else:
+                # This case does NOT exist in Javascript! Why?
+                sequence.append(el)
+        return sequence
+
+    class JList(list):
+        sequence: bool
+        outer_wrapper: bool
+        tuple_stream: bool
+        keep_singleton: bool
+        cons: bool
+
+        def __init__(self, c=()):
+            super().__init__(c)
+            self.sequence = False
+            self.outer_wrapper = False
+            self.tuple_stream = False
+            self.keep_singleton = False
+            self.cons = False
+
+        # Jsonata specific flags
+
+    @staticmethod
+    def is_sequence(result: Optional[Any]) -> bool:
+        return isinstance(result, Utils.JList) and result.sequence
+
+    @staticmethod
+    def convert_number(n: float) -> Optional[float]:
+        # Use long if the number is not fractional
+        if not Utils.is_numeric(n):
+            return None
+        if int(n) == float(n):
+            v = int(n)
+            if int(v) == v:
+                return int(v)
+            else:
+                return v
+        return float(n)
+
+    @staticmethod
+    def convert_value(val: Optional[Any]) -> Optional[Any]:
+        return val if val is not Utils.NULL_VALUE else None
+
+    @staticmethod
+    def convert_dict_nulls(res: MutableMapping[str, Any]) -> None:
+        for key, val in res.items():
+            v = Utils.convert_value(val)
+            if v is not val:
+                res[key] = v
+            Utils.recurse(val)
+
+    @staticmethod
+    def convert_list_nulls(res: MutableSequence[Any]) -> None:
+        for i, val in enumerate(res):
+            v = Utils.convert_value(val)
+            if v is not val:
+                res[i] = v
+            Utils.recurse(val)
+
+    @staticmethod
+    def recurse(val: Optional[Any]) -> None:
+        if isinstance(val, dict):
+            Utils.convert_dict_nulls(val)
+        if isinstance(val, list):
+            Utils.convert_list_nulls(val)
+
+    @staticmethod
+    def convert_nulls(res: Optional[Any]) -> Optional[Any]:
+        Utils.recurse(res)
+        return Utils.convert_value(res)