duqlang 0.1.0__tar.gz

duqlang-0.1.0/PKG-INFO

@@ -0,0 +1,8 @@
+Metadata-Version: 2.3
+Name: duqlang
+Version: 0.1.0
+Summary: Data-processing language and CLI
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: prompt-toolkit>=3.0.53
+Requires-Python: >=3.12

duqlang-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "duqlang"
+version = "0.1.0"
+description = "Data-processing language and CLI"
+requires-python = ">=3.12"
+dependencies = [
+    "bs4>=0.0.2",
+    "httpx>=0.28.1",
+    "prompt-toolkit>=3.0.53",
+]
+
+[dependency-groups]
+dev = [
+    "ipykernel>=7.3.0",
+    "ipywidgets>=8.1.9",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.19,<0.9.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "duq"

duqlang-0.1.0/src/duq/__init__.py

@@ -0,0 +1,3 @@
+from duq._evaluation import evaluate
+from duq._tui import run_tui
+from duq._preview import print_value, print_value_async

duqlang-0.1.0/src/duq/__main__.py

@@ -0,0 +1,30 @@
+import asyncio
+import sys
+import warnings
+
+import duq
+
+
+def catch_warning(message, category, filename, lineno, file=None, line=None):
+    pass
+
+
+warnings.showwarning = catch_warning
+# warnings.filterwarnings(
+#     "ignore", message=".*was never awaited.*", category=RuntimeWarning
+# )
+# warnings.filterwarnings(
+#     "ignore", message=".*invalid escape sequence.*", category=SyntaxWarning
+# )
+
+if len(sys.argv) == 1:
+    asyncio.run(duq.run_tui())
+    sys.exit(0)
+
+if len(sys.argv) != 2:
+    print("Usage: duq '<source>'", file=sys.stderr)
+    sys.exit(1)
+
+source = sys.argv[1]
+result = duq.evaluate(source)
+asyncio.run(duq.print_value_async(result))

duqlang-0.1.0/src/duq/_evaluation.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Callable, assert_never
+
+from duq._syntax import ChainExpr, Expr, OpExpr, parse
+from duq._type_check import DuqTypeError, OpSignature, type_check_value
+
+
+@dataclass
+class OpInfo:
+    name: str
+    signature: OpSignature
+    op_func: Callable
+
+
+class DuqContext:
+    def __init__(self) -> None:
+        self.ops: dict[str, OpInfo] = {}
+        self.cache: dict[str, Any] = {}
+        self.cursor: int | None = None
+
+    @staticmethod
+    def create() -> DuqContext:
+        import duq._std
+
+        context = DuqContext()
+        context.load_ops(duq._std.op_definitions)
+        return context
+
+    def load_op(self, name: str, op_func: Callable) -> None:
+        if name in self.ops:
+            raise Exception(f"redefined op: `{name}`")
+        signature = OpSignature.of(name, op_func)
+        self.ops[name] = OpInfo(name, signature, op_func)
+
+    def load_ops(self, ops: dict[str, Callable]) -> None:
+        for op_name, op_func in ops.items():
+            self.load_op(op_name, op_func)
+
+    def resolve_op(self, name: str, input: Any) -> OpInfo:
+        overloads = [op for op in self.ops.values() if op.signature.matches_name(name)]
+        if len(overloads) == 1:
+            return overloads[0]
+        if len(overloads) == 0:
+            raise Exception(f"unknown operation `{name}`")
+
+        matches = [op for op in overloads if op.signature.matches_input(input)]
+        if len(matches) == 1:
+            return matches[0]
+        if len(matches) == 0:
+            example_op = overloads[0]
+            assert (input_param := example_op.signature.input_param) is not None
+            try:
+                type_check_value(input, input_param.annotation)
+            except DuqTypeError as e:
+                raise DuqTypeError(
+                    f"{example_op.name} (+{len(overloads) - 1} overloads): {e.args[0]}"
+                )
+
+        op_names = ", ".join(f"`{op.name}`" for op in matches)
+        raise Exception(f"ambiguous operation `{name}`: {op_names}")
+
+    def evaluate_op_expr(self, expr: OpExpr, input: Any) -> Any:
+        args = []
+        if expr.name.text.startswith("."):
+            op = self.resolve_op("std.record.field", input)
+            args.append(expr.name.text.removeprefix("."))
+        elif expr.name.text == "{":
+            op = self.resolve_op("map", input)
+        else:
+            op = self.resolve_op(expr.name.text, input)
+
+        arg_exprs = () if expr.arg_list is None else expr.arg_list.args
+        if op.signature.is_macro:
+            args += arg_exprs
+        else:
+            args += [self.evaluate(arg_expr, input) for arg_expr in arg_exprs]
+
+        args = tuple(args)
+        call_args = op.signature.type_check_inputs(self, input, args)
+
+        return op.op_func(*call_args)
+
+    def evaluate_chain_expr(self, expr: ChainExpr, input: Any) -> Any:
+        contains_cursor = (
+            self.cursor is not None
+            and self.cursor >= expr.span[0]
+            and self.cursor <= expr.span[1]
+        )
+        for subexpr in expr.exprs:
+            if (
+                self.cursor is not None
+                and contains_cursor
+                and subexpr.span[0] >= self.cursor
+            ):
+                break
+            input = self.evaluate(subexpr, input)
+        return input
+
+    def evaluate(self, expr: Expr, input: Any) -> Any:
+        if expr.type == "literal":
+            return expr.value
+        elif expr.type == "op":
+            return self.evaluate_op_expr(expr, input)
+        elif expr.type == "chain":
+            return self.evaluate_chain_expr(expr, input)
+        else:
+            assert_never(expr)
+
+
+def evaluate(source: str, input: Any = None) -> Any:
+    context = DuqContext.create()
+    expr = parse(source)
+    return context.evaluate(expr, input)

duqlang-0.1.0/src/duq/_preview.py

@@ -0,0 +1,165 @@
+import asyncio
+import html
+import json
+import re
+from typing import Any, AsyncIterator, Awaitable, Callable, Iterable
+
+import bs4
+
+from duq._evaluation import DuqContext
+from duq._syntax import parse
+from duq._util import DuqFuture, DuqStream, Hinted, Reactive
+
+
+def render_items(
+    start: str, end: str, items: Iterable[str], indent: int, sep=","
+) -> str:
+    start = html.escape(start)
+    end = html.escape(end)
+    sep = html.escape(sep)
+    indent_str = " " * indent
+    if not items:
+        return start + end
+    elif len(result := start + (sep + " ").join(items) + end) < 40:
+        return result
+    else:
+        result = start
+        for item in items:
+            result += html.escape("\n") + indent_str + "  " + item + sep
+        result += html.escape("\n") + indent_str + end
+        return result
+
+
+def render_value(value: Any, indent=0) -> Reactive[str]:
+    if value is None or isinstance(value, (int, float, str, bool)):
+        return Reactive.of(html.escape(json.dumps(value)))
+    elif isinstance(value, list):
+        child_strs_rx = Reactive.from_list(
+            [render_value(child, indent + 2) for child in value]
+        )
+        return child_strs_rx.map(
+            lambda child_strs: render_items("[", "]", child_strs, indent)
+        )
+    elif isinstance(value, dict):
+        child_strs_rx = Reactive.from_list(
+            [
+                render_value(child, indent + 2).map(
+                    lambda s: f"{html.escape(json.dumps(key))}: {s}"
+                )
+                for key, child in value.items()
+            ]
+        )
+        return child_strs_rx.map(
+            lambda child_strs: render_items("{", "}", child_strs, indent)
+        )
+    elif isinstance(value, Awaitable):
+        return Reactive.from_awaitable(value).flat_map(
+            lambda child_maybe: (
+                render_value(child_maybe.value, indent + 2)
+                if child_maybe.is_some
+                else Reactive.of("...")
+            ).map(
+                lambda child_str: render_items(
+                    "future(", ")", [child_str], indent, sep=""
+                )
+            )
+        )
+    elif isinstance(value, DuqFuture):
+        return render_value(value.create(), indent)
+    elif isinstance(value, AsyncIterator):
+
+        async def child_strs_iter() -> AsyncIterator[Reactive[str]]:
+            async for child in value:
+                yield render_value(child, indent + 2)
+
+        return Reactive.collect_from_iter(child_strs_iter()).map(
+            lambda child_strs: render_items(
+                "stream[",
+                "]",
+                child_strs.items + ([] if child_strs.is_done else ["..."]),
+                indent,
+            )
+        )
+    elif isinstance(value, DuqStream):
+        return render_value(value.create(), indent)
+    elif isinstance(value, Hinted):
+        child_rx = render_value(value.value, indent)
+        if value.hint == "cursor":
+
+            def bold(s: str) -> str:
+                lines = s.split("\n")
+                lines = [f"<green>{line}</green>" for line in lines]
+                return "\n".join(lines)
+
+            return child_rx.map(bold)
+        else:
+            return child_rx
+    elif isinstance(value, bs4.Tag):
+        indented = re.sub(
+            r"^(\s*)",
+            (indent * " ") + r"\1\1",
+            value.prettify(),
+            flags=re.MULTILINE,
+        ).removeprefix(indent * " ")
+        output = re.sub(r"\s*$", "", indented)
+        return Reactive.of(html.escape(output))
+    else:
+        raise Exception(f"unimplemented: {type(value)}")
+
+
+class Preview:
+    def __init__(
+        self, set_output: Callable[[str], None], set_error: Callable[[str], None]
+    ) -> None:
+        self.ctx = DuqContext.create()
+        self.source = ""
+        self.cursor_position = 0
+        self.current_task: asyncio.Task | None = None
+        self.set_output = set_output
+        self.set_error = set_error
+        self.refresh()
+
+    def set_source(self, source: str) -> None:
+        self.source = source
+        self.refresh()
+
+    def set_cursor_position(self, cursor_position: int) -> None:
+        self.cursor_position = cursor_position
+        self.refresh()
+
+    def refresh(self) -> None:
+        try:
+            expr = parse(self.source, self.cursor_position)
+            self.ctx.cursor = self.cursor_position
+            result = self.ctx.evaluate(expr, None)
+        except Exception as e:
+            self.set_error(f"Error: {e}")
+        else:
+
+            async def set_output_task():
+                try:
+                    output_rx = render_value(result)
+                    self.set_output(output_rx.initial)
+                    async for output in output_rx.updates:
+                        self.set_output(output)
+                except Exception as e:
+                    while isinstance(e, ExceptionGroup):
+                        e = e.exceptions[0]
+                    self.set_error(f"Error: {e}")
+
+            if self.current_task != None:
+                self.current_task.cancel()
+            self.current_task = asyncio.create_task(set_output_task())
+            self.set_error("")
+
+
+async def print_value_async(value: Any) -> None:
+    value_str_rx = render_value(value)
+    print(html.unescape(value_str_rx.initial))
+    async for value_str in value_str_rx.updates:
+        print(html.unescape(value_str))
+
+
+def print_value(value: Any) -> None:
+    value_str_rx = render_value(value)
+    print(html.unescape(value_str_rx.initial))

duqlang-0.1.0/src/duq/_std/__init__.py

@@ -0,0 +1,37 @@
+from duq._std import (
+    _basic,
+    _csv,
+    _file,
+    _future,
+    _html,
+    _http,
+    _json,
+    _bool,
+    _list,
+    _null,
+    _number,
+    _record,
+    _str,
+    _stream,
+)
+
+op_definitions = {
+    op_name: op_func
+    for op_definitions in [
+        _basic.op_definitions,
+        _bool.op_definitions,
+        _csv.op_definitions,
+        _file.op_definitions,
+        _future.op_definitions,
+        _html.op_definitions,
+        _http.op_definitions,
+        _json.op_definitions,
+        _list.op_definitions,
+        _null.op_definitions,
+        _number.op_definitions,
+        _record.op_definitions,
+        _str.op_definitions,
+        _stream.op_definitions,
+    ]
+    for op_name, op_func in op_definitions.items()
+}

duqlang-0.1.0/src/duq/_std/_basic.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+from typing import Any
+
+from duq._evaluation import DuqContext
+from duq._syntax import Expr
+from duq._util import Hinted
+
+
+def op_id(input: Any) -> Any:
+    return input
+
+
+def op_eq(input: Any, arg: Any) -> bool:
+    return input == arg
+
+
+def op_hint(input: Any, hint: str) -> Hinted[Any]:
+    return Hinted(hint, input)
+
+
+def op_now() -> str:
+    return datetime.now().isoformat(timespec="milliseconds") + "Z"
+
+
+def op_try(ctx: DuqContext, input: Any, body: Expr) -> Any:
+    try:
+        return ctx.evaluate(body, input)
+    except:
+        return None
+
+
+op_definitions = {
+    "std.basic.id": op_id,
+    "std.basic.eq": op_eq,
+    "std.basic.hint": op_hint,
+    "std.time.now": op_now,
+    "std.basic.try": op_try,
+}

duqlang-0.1.0/src/duq/_std/_bool.py

@@ -0,0 +1,12 @@
+def op_not(input: bool) -> bool:
+    return not input
+
+
+def op_str(input: bool) -> str:
+    return str(input).lower()
+
+
+op_definitions = {
+    "std.bool.not": op_not,
+    "std.bool.str": op_str,
+}

duqlang-0.1.0/src/duq/_std/_csv.py

@@ -0,0 +1,20 @@
+from typing import Any
+
+
+def op_tableToRecords(input: list[list[str]]) -> list[dict[str, str]]:
+    if len(input) == 0:
+        raise Exception("missing header row")
+    headers = input[0]
+    rows = input[1:]
+    result = []
+    for row in rows:
+        record = {}
+        for i, header in enumerate(headers):
+            record[header] = None if i >= len(row) else row[i]
+        result.append(record)
+    return result
+
+
+op_definitions = {
+    "std.csv.tableToRecords": op_tableToRecords,
+}

duqlang-0.1.0/src/duq/_std/_file.py

@@ -0,0 +1,8 @@
+def op_read(input: str) -> str:
+    with open(input, "r") as f:
+        return f.read()
+
+
+op_definitions = {
+    "std.file.read": op_read,
+}

duqlang-0.1.0/src/duq/_std/_future.py

@@ -0,0 +1,33 @@
+import asyncio
+from typing import Any
+
+from duq._evaluation import DuqContext
+from duq._syntax import Expr
+from duq._util import DuqFuture
+
+
+def op_sleep(seconds: int | float) -> DuqFuture[None]:
+    return DuqFuture(lambda: asyncio.sleep(seconds))
+
+
+def op_map(ctx: DuqContext, input: DuqFuture[Any], body: Expr) -> DuqFuture[Any]:
+    async def task():
+        value = await input.create()
+        return ctx.evaluate(body, value)
+
+    return DuqFuture(task)
+
+
+def op_all(input: list[DuqFuture[Any]]) -> DuqFuture[list[Any]]:
+    async def task():
+        children = [child.create() for child in input]
+        return await asyncio.gather(*children)
+
+    return DuqFuture(task)
+
+
+op_definitions = {
+    "std.future.sleep": op_sleep,
+    "std.future.map": op_map,
+    "std.future.all": op_all,
+}

duqlang-0.1.0/src/duq/_std/_html.py

@@ -0,0 +1,20 @@
+import bs4
+
+
+def op_html(input: str) -> bs4.BeautifulSoup:
+    return bs4.BeautifulSoup(input, "html.parser")
+
+
+def op_select(input: bs4.Tag, query: str) -> list[bs4.Tag]:
+    return list(input.select(query))
+
+
+def op_text(input: bs4.Tag) -> str:
+    return input.text
+
+
+op_definitions = {
+    "std.html.html": op_html,
+    "std.html.select": op_select,
+    "std.html.text": op_text,
+}

duqlang-0.1.0/src/duq/_std/_http.py

@@ -0,0 +1,25 @@
+import httpx
+
+from duq._evaluation import DuqContext
+from duq._util import DuqFuture
+
+
+def op_fetch(ctx: DuqContext, input: str) -> DuqFuture[str]:
+    async def task() -> str:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                input,
+                headers={
+                    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/152.0.0.0 Safari/537.36"
+                },
+            )
+            if response.status_code != 200:
+                raise Exception(f"request error: {input} -> {response.status_code}")
+        return response.text
+
+    return DuqFuture(task).cached(ctx, ["std.http.fetch", input])
+
+
+op_definitions = {
+    "std.http.fetch": op_fetch,
+}

duqlang-0.1.0/src/duq/_std/_json.py

@@ -0,0 +1,16 @@
+import json
+from typing import Any
+
+
+def op_json(input: str) -> Any:
+    return json.loads(input)
+
+
+def op_pretty(input: Any) -> str:
+    return json.dumps(input, indent=2)
+
+
+op_definitions = {
+    "std.json.json": op_json,
+    "std.json.pretty": op_pretty,
+}