polyfile-weave 0.5.5__py3-none-any.whl

polyfile/expressions.py

@@ -0,0 +1,346 @@
+from collections import deque
+from enum import Enum
+from io import StringIO
+import itertools
+
+
+OPERATORS_BY_NAME = {}
+
+
+def function_call(obj, function_name):
+    raise RuntimeError("TODO: Implement")
+
+
+def to_int(v, byteorder='big') -> int:
+    if isinstance(v, int):
+        return v
+    elif isinstance(v, bytes) or isinstance(v, str) or isinstance(v, bytearray):
+        if len(v) == 0:
+            return 0
+        elif len(v) == 1:
+            return int(v[0])
+        else:
+            # Assume big endian
+            return int.from_bytes(v, byteorder=byteorder)
+    raise ValueError(f"Cannot convert {v!r} to an integer")
+
+
+class Operator(Enum):
+    ENUM_ACCESSOR = ('::', 0, lambda a, b: a[b.name], True, 2, False, (True, False))
+    MEMBER_ACCESS = ('.', 1, lambda a, b: None, True, 2, False, (True, False))
+    UNARY_PLUS = ('+', 2, lambda a: a, False, 1, True)
+    UNARY_MINUS = ('-', 2, lambda a: -to_int(a), False, 1, True)
+    LOGICAL_NOT = ('not', 2, lambda a: not a, False, 1)
+    BITWISE_NOT = ('~', 2, lambda a: ~to_int(a), False, 1)
+    MULTIPLICATION = ('*', 3, lambda a, b: to_int(a) * to_int(b))
+    DIVISION = ('/', 3, lambda a, b: to_int(a) // to_int(b))
+    REMAINDER = ('%', 3, lambda a, b: to_int(a) % to_int(b))
+    ADDITION = ('+', 4, lambda a, b: a + b)
+    SUBTRACTION = ('-', 4, lambda a, b: to_int(a) - to_int(b))
+    BITWISE_LEFT_SHIFT = ('<<', 5, lambda a, b: to_int(a) << to_int(b))
+    BITWISE_RIGHT_SHIFT = ('>>', 5, lambda a, b: to_int(a) >> to_int(b))
+    LESS_THAN = ('<', 6, lambda a, b: a < b)
+    GREATER_THAN = ('>', 6, lambda a, b: a > b)
+    LESS_THAN_EQUAL = ('<=', 6, lambda a, b: a <= b)
+    GREATER_THAN_EQUAL = ('>=', 6, lambda a, b: a >= b)
+    EQUALS = ('==', 7, lambda a, b: a == b)
+    NOT_EQUAL = ('!=', 7, lambda a, b: a != b)
+    BITWISE_AND = ('&', 8, lambda a, b: to_int(a) & to_int(b))
+    BITWISE_XOR = ('^', 9, lambda a, b: to_int(a) ^ to_int(b))
+    BITWISE_OR = ('|', 10, lambda a, b: to_int(a) | to_int(b))
+    LOGICAL_AND = ('and', 11, lambda a, b: a and b)
+    LOGICAL_OR = ('or', 12, lambda a, b: a or b)
+    TERNARY_ELSE = (':', 13, lambda a, b: (a, b), False)
+    TERNARY_CONDITIONAL = ('?', 14, lambda a, b: b[bool(a)], False)
+
+    def __init__(self,
+                 token,
+                 priority,
+                 execute,
+                 is_left_associative=True,
+                 arity=2,
+                 multiple_arity=False,
+                 expand=None):
+        self.token = token
+        self.priority = priority
+        self.execute = execute
+        self.left_associative = is_left_associative
+        self.arity = arity
+        self.multiple_arity = multiple_arity
+        if expand is None:
+            self.expand = (True,) * self.arity
+        else:
+            self.expand = expand
+        if not multiple_arity:
+            OPERATORS_BY_NAME[self.token] = self
+
+
+IDENTIFIER_BYTES = {
+    chr(i) for i in range(ord('A'), ord('Z') + 1)
+} | {
+    chr(i) for i in range(ord('a'), ord('z') + 1)
+} | {
+    chr(i) for i in range(ord('0'), ord('9') + 1)
+} | {
+    '-', '_'
+}
+
+
+class Token:
+    def __init__(self, raw_text):
+        self._raw = raw_text
+
+    @property
+    def raw_token(self):
+        return self._raw
+
+    def __len__(self):
+        return len(self._raw)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self._raw!r})"
+
+
+class Parenthesis(Token):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}()"
+
+
+class OpenParen(Parenthesis):
+    def __init__(self):
+        super().__init__('(')
+
+
+class CloseParen(Parenthesis):
+    def __init__(self):
+        super().__init__(')')
+
+
+class OperatorToken(Token):
+    def __init__(self, op):
+        if isinstance(op, str):
+            op = OPERATORS_BY_NAME[op]
+        super().__init__(op.token)
+        self.op: Operator = op
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(op={self.op!r})"
+
+
+class IdentifierToken(Token):
+    def __init__(self, name):
+        super().__init__(name)
+        self.name = name
+
+
+class IntegerToken(Token):
+    def __init__(self, raw_str, value):
+        super().__init__(raw_str)
+        self.value = value
+
+    def __int__(self):
+        return self.value
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(raw_str={self.raw_token!r}, value={self.value!r})"
+
+
+class Tokenizer:
+    def __init__(self, stream):
+        if isinstance(stream, str):
+            stream = StringIO(stream)
+        self._stream = stream
+        self._buffer = deque()
+        self._next_token = None
+        self.prev_token = None
+
+    def _peek_byte(self, n=1):
+        bytes_needed = n - len(self._buffer)
+        if bytes_needed > 0:
+            b = self._stream.read(bytes_needed)
+            self._buffer.extend(b)
+        return ''.join(itertools.islice(self._buffer, n))
+
+    def _pop_byte(self, n=1):
+        if len(self._buffer) < n:
+            return ''.join(self._buffer) + self._stream.read(1)
+        else:
+            return ''.join(self._buffer.popleft() for _ in range(n))
+
+    def peek(self):
+        if self._next_token is not None:
+            return self._next_token
+        ret = None
+        operand = None
+        # ignore leading whitespace
+        while self._peek_byte() == ' ' or self._peek_byte() == '\t':
+            self._pop_byte()
+        while ret is None:
+            c = self._peek_byte(3)
+            if len(c) == 0:
+                break
+            elif operand is not None:
+                if c[0] in IDENTIFIER_BYTES:
+                    operand += self._pop_byte()
+                else:
+                    break
+            elif c in OPERATORS_BY_NAME:
+                ret = OperatorToken(c)
+            elif c[:2] in OPERATORS_BY_NAME:
+                ret = OperatorToken(c[:2])
+            elif c[0] in OPERATORS_BY_NAME:
+                if c[0] == '+':
+                    if self.prev_token is None or isinstance(self.prev_token, OperatorToken):
+                        ret = OperatorToken(Operator.UNARY_PLUS)
+                    else:
+                        ret = OperatorToken(Operator.ADDITION)
+                elif c[0] == '-':
+                    if self.prev_token is None or isinstance(self.prev_token, OperatorToken):
+                        ret = OperatorToken(Operator.UNARY_MINUS)
+                    else:
+                        ret = OperatorToken(Operator.SUBTRACTION)
+                else:
+                    ret = OperatorToken(c[0])
+            elif c[0] == '(':
+                ret = OpenParen()
+            elif c[0] == ')':
+                ret = CloseParen()
+            elif c[0] == ' ' or c[0] == '\t':
+                break
+            else:
+                operand = self._pop_byte()
+        if operand is not None:
+            if operand.startswith('0x'):
+                ret = IntegerToken(operand, int(operand, 16))
+            elif operand.startswith('0o'):
+                ret = IntegerToken(operand, int(operand, 8))
+            elif operand.startswith('0b'):
+                ret = IntegerToken(operand, int(operand, 2))
+            else:
+                # Is this an integer?
+                try:
+                    ret = IntegerToken(operand, int(operand))
+                except ValueError:
+                    ret = IdentifierToken(operand)
+        elif ret is not None:
+            self._pop_byte(len(ret))
+        return ret
+
+    def has_next(self) -> bool:
+        return self.peek() is not None
+
+    def next(self):
+        ret = self.peek()
+        self.prev_token = ret
+        self._next_token = None
+        return ret
+
+    def __iter__(self):
+        while True:
+            ret = self.next()
+            if ret is None:
+                break
+            yield ret
+
+
+def tokenize(stream_or_str):
+    yield from Tokenizer(stream_or_str)
+
+
+def infix_to_rpn(tokens):
+    """Converts an infix expression to reverse Polish notation using the Shunting Yard algorithm"""
+    operators = []
+
+    for token in tokens:
+        if isinstance(token, OpenParen):
+            operators.append(token)
+        elif isinstance(token, CloseParen):
+            while not isinstance(operators[-1], OpenParen):
+                yield operators.pop()
+            # TODO: Throw a nice mismatched parenthesis exception here instead of relying on operators[-1]
+            # to throw an index out of bounds exception
+            if operators and isinstance(operators[-1], OpenParen):
+                operators.pop()
+        elif isinstance(token, OperatorToken):
+            while operators and not isinstance(operators[-1], OpenParen) and \
+                    (operators[-1].op.priority < token.op.priority \
+                        or \
+                    (operators[-1].op.priority == token.op.priority and operators[-1].op.left_associative)):
+                yield operators.pop()
+            operators.append(token)
+        else:
+            yield token
+
+    while operators:
+        top = operators.pop()
+        if isinstance(top, Parenthesis):
+            raise RuntimeError("Mismatched parenthesis")
+        yield top
+
+
+class Expression:
+    def __init__(self, rpn):
+        self.tokens = tuple(rpn)
+
+    @staticmethod
+    def get_value(token: Token, assignments: dict):
+        if isinstance(token, IntegerToken):
+            return token.value
+        elif isinstance(token, IdentifierToken):
+            if token.name not in assignments:
+                raise KeyError(f'Unknown identifier {token.name}')
+            return assignments[token.name]
+        elif isinstance(token, int) or isinstance(token, str) or isinstance(token, bytes):
+            return token
+        else:
+            raise ValueError(f"Unexpected token {token!r}")
+
+    def interpret(self, assignments=None):
+        if assignments is None:
+            assignments = {}
+        values = []
+        for t in self.tokens:
+            if isinstance(t, OperatorToken):
+                args = []
+                for expand, v in zip(t.op.expand, values[-t.op.arity:]):
+                    if expand:
+                        args.append(self.get_value(v, assignments))
+                    else:
+                        args.append(v)
+                values = values[:-t.op.arity] + [t.op.execute(*args)]
+            else:
+                values.append(t)
+        if len(values) != 1:
+            raise RuntimeError(f"Unexpected extra tokens: {values[:-1]}")
+        if isinstance(values[0], IdentifierToken):
+            return self.get_value(values[0], assignments)
+        return values[0]
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}(rpn={self.tokens!r})"
+
+
+def parse(expression_str: str) -> Expression:
+    return Expression(infix_to_rpn(tokenize(expression_str)))
+
+
+if __name__ == '__main__':
+    assignments = {
+        'sampling_factors': 1234,
+        'thumbnail_x': 5,
+        'thumbnail_y': 7,
+        'marker': 1,
+        'marker_enum': {
+            'soi': 0,
+            'eoi': 3
+        }
+    }
+    for s in (
+        '(sampling_factors & -0xf0) >> 4',
+        'thumbnail_x * thumbnail_y * 3',
+        'marker != marker_enum::soi and marker != marker_enum::eoi'
+    ):
+        print(parse(s).interpret(assignments))

polyfile/fileutils.py

@@ -0,0 +1,343 @@
+from io import BytesIO, SEEK_END, UnsupportedOperation
+import mmap
+import os
+from pathlib import Path
+import tempfile as tf
+import shutil
+import sys
+from typing import AnyStr, ContextManager, IO, Iterator, Iterable, List, Optional, TextIO, Union
+
+
+Streamable = Union[str, Path, IO, "FileStream", bytes]
+
+
+def make_stream(path_or_stream: Streamable, mode: str = 'rb',
+                close_on_exit: Optional[bool] = None) -> "FileStream":
+    if isinstance(path_or_stream, FileStream):
+        return path_or_stream
+    else:
+        return FileStream(path_or_stream, mode=mode, close_on_exit=close_on_exit)
+
+
+class Tempfile:
+    def __init__(self, contents: bytes, prefix: Optional[str] = None, suffix: Optional[str] = None):
+        self._path: Optional[str] = None
+        self._data: bytes = contents
+        self._prefix: Optional[str] = prefix
+        self._suffix: Optional[str] = suffix
+
+    def __enter__(self) -> str:
+        tmp = tf.NamedTemporaryFile(prefix=self._prefix, suffix=self._suffix, delete=False)
+        tmp.write(self._data)
+        tmp.flush()
+        tmp.close()
+        self._path = tmp.name
+        return tmp.name
+
+    def __exit__(self, type, value, traceback):
+        if self._path is not None:
+            os.unlink(self._path)
+            self._path = None
+
+
+class ExactNamedTempfile(Tempfile):
+    def __init__(self, contents: bytes, name: str):
+        super().__init__(contents)
+        self._name: str = name
+
+    def __enter__(self) -> str:
+        tmpdir = Path(tf.mkdtemp())
+        file_path = tmpdir / self._name
+        with open(file_path, "wb") as f:
+            f.write(self._data)
+        self._path = str(tmpdir)
+        return str(file_path)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self._path is not None:
+            shutil.rmtree(self._path)
+            self._path = None
+
+
+class PathOrStdin:
+    def __init__(self, path: str):
+        self.path: str = path
+        if self.path == '-':
+            self._tempfile: Optional[ExactNamedTempfile] = ExactNamedTempfile(sys.stdin.buffer.read(), "STDIN")
+        elif not Path(path).exists():
+            raise FileNotFoundError(path)
+        else:
+            self._tempfile = None
+
+    def __enter__(self) -> str:
+        if self._tempfile is None:
+            return self.path
+        else:
+            return self._tempfile.__enter__()
+
+    def __exit__(self, *args, **kwargs):
+        if self._tempfile is not None:
+            return self._tempfile.__exit__(*args, **kwargs)
+
+
+class PathOrStdout(ContextManager[TextIO]):
+    def __init__(self, path: str):
+        self.path: str = path
+        if self.path == '-':
+            self._tempfile: Optional[TextIO] = sys.stdout
+        else:
+            self._tempfile = None
+
+    def __enter__(self) -> TextIO:
+        if self._tempfile is None:
+            self._tempfile = open(self.path, "w")
+        return self._tempfile
+
+    def __exit__(self, *args, **kwargs) -> None:  # type: ignore
+        if self._tempfile is not None:
+            if self._tempfile is not sys.stdout:
+                self._tempfile.close()
+                self._tempfile = None
+        return None
+
+
+class FileStream(IO):
+    def __init__(
+            self,
+            path_or_stream: Streamable,
+            start: int = 0,
+            length: Optional[int] = None,
+            mode: str = "rb",
+            close_on_exit: Optional[bool] = None
+    ):
+        if isinstance(path_or_stream, Path):
+            path_or_stream = str(path_or_stream)
+        if isinstance(path_or_stream, str):
+            self._stream: IO = open(path_or_stream, mode)
+            if close_on_exit is None:
+                close_on_exit = True
+        else:
+            if isinstance(path_or_stream, bytes):
+                path_or_stream = BytesIO(path_or_stream)
+                setattr(path_or_stream, "name", "bytes")
+            elif not path_or_stream.seekable():
+                raise ValueError('FileStream can only wrap streams that are seekable')
+            elif not path_or_stream.readable():
+                raise ValueError('FileStream can only wrap streams that are readable')
+            self._stream = path_or_stream
+        if isinstance(path_or_stream, FileStream):
+            if length is None:
+                self._length = len(path_or_stream) - start
+            else:
+                self._length = min(length, len(path_or_stream))
+        else:
+            if isinstance(path_or_stream, BytesIO):
+                orig_pos = path_or_stream.tell()
+                path_or_stream.seek(0, SEEK_END)
+                try:
+                    filesize = path_or_stream.tell()
+                finally:
+                    path_or_stream.seek(orig_pos)
+            else:
+                filesize = os.path.getsize(self._stream.name)
+            if length is None:
+                self._length = filesize - start
+            else:
+                self._length = min(filesize, length) - start
+        if close_on_exit is None:
+            close_on_exit = False
+        self._name = self._stream.name
+        self.start = start
+        self.close_on_exit = close_on_exit
+        self._entries = 0
+        self._root = None
+
+    def __len__(self):
+        return self._length
+
+    def seekable(self):
+        return True
+
+    def writable(self):
+        return False
+
+    def readable(self):
+        return True
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def root(self):
+        if self._root is None:
+            if isinstance(self._stream, FileStream):
+                self._root = self._stream.root
+            else:
+                self._root = self._stream
+        return self._root
+
+    def save_pos(self):
+        f = self
+
+        class SP:
+            def __init__(self):
+                self.pos = f.root.tell()
+
+            def __enter__(self, *args, **kwargs) -> FileStream:
+                return f
+
+            def __exit__(self, *args, **kwargs):
+                f.root.seek(self.pos)
+
+        return SP()
+
+    def fileno(self):
+        return self._stream.fileno()
+
+    def offset(self):
+        if isinstance(self._stream, FileStream):
+            return self._stream.offset() + self.start
+        else:
+            return self.start
+
+    def seek(self, offset, from_what=0):
+        if from_what == 1:
+            offset = self.tell() + offset
+        elif from_what == 2:
+            offset = len(self) + offset
+        if offset - self.start > self._length:
+            raise IndexError(f"{self!r} is {len(self)} bytes long, but seek was requested for byte {offset}")
+        self._stream.seek(self.start + offset)
+
+    def tell(self):
+        return min(max(self._stream.tell() - self.start, 0), self._length)
+
+    def read(self, n=None) -> bytes:
+        if self._stream.tell() - self.start < 0:
+            # another context moved the position, so move it back to our zero index:
+            self.seek(0)
+            pos = 0
+        else:
+            pos = self.tell()
+        ls = len(self)
+        if pos >= ls:
+            return b''
+        elif n is None:
+            return self._stream.read()[:ls - pos]
+        else:
+            return self._stream.read(min(n, ls - pos))
+
+    def contains_all(self, *args):
+        if args:
+            with mmap.mmap(self.fileno(), 0, access=mmap.ACCESS_READ) as filecontent:
+                for string in args:
+                    if filecontent.find(string, self.offset(), self.offset() + len(self)) < 0:
+                        return False
+        return True
+
+    def first_index_of(self, byte_sequence: bytes) -> int:
+        with mmap.mmap(self.fileno(), 0, access=mmap.ACCESS_READ) as filecontent:
+            start_offset = self.offset()
+            end_offset = start_offset + len(self)
+            index = filecontent.find(byte_sequence, start_offset, end_offset)
+            if start_offset <= index < end_offset:
+                return index - start_offset
+            else:
+                return -1
+
+    @property
+    def content(self) -> bytes:
+        with self.save_pos():
+            self.seek(0)
+            return self.read(len(self))
+
+    def __bytes__(self):
+        return self.content
+
+    def tempfile(self, prefix=None, suffix=None):
+        class FSTempfile:
+            def __init__(self, file_stream: FileStream):
+                self._temp = None
+                self._fs: FileStream = file_stream
+
+            def __enter__(self):
+                self._temp = tf.NamedTemporaryFile(prefix=prefix, suffix=suffix, delete=False)
+                with self._fs.save_pos():
+                    self._fs.seek(0)
+                    while True:
+                        b = self._fs.read(1048576)  # write 1 MiB at a time
+                        if not b:
+                            break
+                        self._temp.write(b)
+                self._temp.flush()
+                self._temp.close()
+                return self._temp.name
+
+            def __exit__(self, type, value, traceback):
+                if self._temp is not None:
+                    os.unlink(self._temp.name)
+                    self._temp = None
+        return FSTempfile(self)
+
+    def __getitem__(self, index) -> Union[bytes, "FileStream"]:
+        if isinstance(index, int):
+            with self.save_pos():
+                self.seek(index)
+                return self.read(1)
+        elif not isinstance(index, slice):
+            raise ValueError(f"unexpected argument {index}")
+        if index.step is not None and index.step != 1:
+            raise ValueError(f"Invalid slice step: {index}")
+        length=None
+        if index.stop is not None:
+            if index.stop < 0 and (
+                    (index.stop < index.start and index.start > 0) or (index.stop > index.start and index.start < 0)):
+                length = len(self) + index.stop - index.start
+            elif index.stop < index.start:
+                length = 0
+            else:
+                length = index.stop - index.start
+        return FileStream(self, start=index.start, length=length, close_on_exit=False)
+
+    def __enter__(self) -> "FileStream":
+        self._entries += 1
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self._entries -= 1
+        assert self._entries >= 0
+        if self._entries == 0 and self.close_on_exit:
+            self.close_on_exit = False
+            self._stream.close()
+
+    def close(self) -> None:
+        if self._entries == 0:
+            self._stream.close()
+
+    def flush(self):
+        self._stream.flush()
+
+    def isatty(self) -> bool:
+        return self._stream.isatty()
+
+    def readline(self, limit: int = ...) -> AnyStr:
+        raise UnsupportedOperation()
+
+    def readlines(self, hint: int = ...) -> List[AnyStr]:
+        raise UnsupportedOperation()
+
+    def truncate(self, size: int = ...) -> int:
+        raise UnsupportedOperation()
+
+    def write(self, s: AnyStr) -> int:
+        raise UnsupportedOperation()
+
+    def writelines(self, lines: Iterable[AnyStr]) -> None:
+        raise UnsupportedOperation()
+
+    def __next__(self) -> AnyStr:
+        raise UnsupportedOperation()
+
+    def __iter__(self) -> Iterator[AnyStr]:
+        raise UnsupportedOperation()