davy-jones-locker 0.1.0__tar.gz

davy_jones_locker-0.1.0/.dockerignore

@@ -0,0 +1,6 @@
+.env
+runner.env
+.venv
+__pycache__
+*.pyc
+README.md

davy_jones_locker-0.1.0/.env.example

@@ -0,0 +1,42 @@
+# The engine's settings. Copy to .env (which isn't committed) and fill in.
+# The runner has its own file (runner.env.example): nothing here reaches it.
+
+# Postgres, for saved graphs, stored files and run records, and for SQL nodes
+# that read your own tables. Off until you set DATABASE_URL: those endpoints
+# answer 503, and everything else works. With engine-shared (its .env has
+# these roles and passwords), start it, uncomment the `shared` network lines
+# in compose.yaml, and uncomment the two URLs below. The engine's tables go in
+# schema FLOW_SCHEMA, created the first time the database answers.
+# DATABASE_URL=postgresql://engine:engine-dev@postgres:5432/flow
+FLOW_SCHEMA=flow
+
+# The database server-side SQL reads (attached read-only as "db"), through a
+# read-only role (engine-shared/postgres/engine-roles.sql makes one).
+# Empty: DATABASE_URL, whose role can read the engine's own tables too.
+# QUERY_DATABASE_URL=postgresql://engine_reader:engine-reader-dev@postgres:5432/flow
+
+# What /health reports as the engine's address (the app shows it). The app's
+# compose.yaml usually sets it to the engine's service name.
+PUBLIC_HOST=engine
+
+# Limits for SQL on the server, and for uploads.
+QUERY_TIMEOUT_SECONDS=60
+QUERY_MEMORY_LIMIT=1GB
+MAX_UPLOAD_MB=200
+
+# How long the engine waits for the runner (it adds 30s for transfer).
+RUN_TIMEOUT_SECONDS=60
+
+# The AI assistant in Manage (SQLQuery, PythonScript, JavaScript nodes), through
+# a LiteLLM proxy. Off unless all three settings below are set. The key stays on
+# the engine: the browser never sees it, and the runner can't read it.
+# LITELLM_BASE_URL is the proxy's address (with /v1 or without); requests go to
+# its /chat/completions. ASSIST_MODEL is a model name the proxy offers
+# (there's no default: GET /models on the proxy, with the key, lists them).
+LITELLM_BASE_URL=
+LITELLM_API_KEY=
+ASSIST_MODEL=
+# Rows of each input sent with a request, as examples of its values (0: none,
+# just column names and types).
+ASSIST_SAMPLE_ROWS=3
+ASSIST_TIMEOUT_SECONDS=90

davy_jones_locker-0.1.0/.gitignore

@@ -0,0 +1,5 @@
+.env
+runner.env
+__pycache__/
+*.pyc
+.venv/

davy_jones_locker-0.1.0/Dockerfile

@@ -0,0 +1,23 @@
+# DAVY JONES' LOCKER's engine image: the API, listening on 3000. Built from
+# davy-jones-locker/engine/. An app's engine image is built on this one (FROM),
+# adding its entry point; run as it is, it's the engine with nothing added.
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+WORKDIR /srv
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir --require-hashes -r requirements.txt
+
+RUN useradd --system --uid 10001 --create-home --home-dir /home/engine engine
+USER engine
+# DuckDB's postgres extension, fetched now: at run time DuckDB is locked down
+# and can't install anything.
+RUN python -c "import duckdb; duckdb.sql('INSTALL postgres')"
+
+COPY davy_jones_locker ./davy_jones_locker
+# The packages Python scripts can import, which the AI assistant tells the model.
+COPY runner/requirements.in ./runner/requirements.in
+
+EXPOSE 3000
+CMD ["uvicorn", "davy_jones_locker.main:app", "--host", "0.0.0.0", "--port", "3000"]

davy_jones_locker-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

davy_jones_locker-0.1.0/PKG-INFO

@@ -0,0 +1,238 @@
+Metadata-Version: 2.5
+Name: davy-jones-locker
+Version: 0.1.0
+Summary: The Python engine that executes flowgraphs: SQL on a locked-down DuckDB, Python in a sandboxed runner, saved graphs, and an AI assistant. Its browser shell is davy-jones-locker on npm, on SEAMONSTER.
+Project-URL: Homepage, https://github.com/lukewilliams/davy-jones-locker/tree/main/davy-jones-locker/engine#readme
+Project-URL: Repository, https://github.com/lukewilliams/davy-jones-locker
+Project-URL: Issues, https://github.com/lukewilliams/davy-jones-locker/issues
+Author: Luke Williams
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: data-pipeline,duckdb,fastapi,flowgraph,sandbox
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: FastAPI
+Classifier: Operating System :: POSIX
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Database
+Requires-Python: >=3.12
+Requires-Dist: duckdb>=1.5
+Requires-Dist: fastapi>=0.141
+Requires-Dist: httpx>=0.28
+Requires-Dist: psycopg[binary]>=3.3
+Requires-Dist: pyarrow>=25
+Requires-Dist: uvicorn>=0.53
+Description-Content-Type: text/markdown
+
+# davy-jones-locker (engine)
+
+> **Early release.** This is 0.1: the API may change between minor versions
+> until 1.0, so pin an exact version if you depend on it.
+
+DAVY JONES' LOCKER's Python engine, which executes flowgraphs on the server
+(FastAPI, port 3000), and **the runner**, the sandbox it runs Python in. An
+app built on the framework runs everything it can in the browser; the engine
+is the second engine, for:
+
+- **PythonScript** nodes, which can't run in a browser.
+- **SQL over the backend database**: a SQL node whose query reads a table its
+  wired inputs don't supply runs here, over Postgres (attached read-only as
+  `db`) and the inputs the browser sends.
+- **Whole graphs on the server**, for schedulers (n8n, say) to trigger.
+
+There's no sign-in yet (identity is the next part): keep the engine off the
+public internet until there is. Until then, anyone who can reach it can use
+its AI assistant, and so its LiteLLM key's budget.
+
+## An app's engine
+
+An app builds its engine on this one. Its entry point makes the FastAPI app:
+
+```python
+from davy_jones_locker import create_app
+
+app = create_app(title="engine-myapp")
+```
+
+and its image starts from the framework's ([Dockerfile](Dockerfile)), adding
+that entry point:
+
+```dockerfile
+FROM davy-jones-locker-engine
+COPY main.py ./
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "3000"]
+```
+
+The runner is used as it is ([runner/Dockerfile](runner/Dockerfile)), until
+an app adds libraries of its own. [../compose.yaml](../compose.yaml) runs
+both, with the runner's safeguards; copy it into the app and name the
+services for it (`engine-myapp`, `engine-myapp-runner`): browsers reach the
+engine through the app's own origin at `/engines/<service>/`.
+
+The package is `davy-jones-locker` (Python 3.12+); `pip install -e .` here
+for development. The images don't install it: they copy the package and
+install the pinned, hashed versions in `requirements.txt` and
+`runner/requirements.txt`.
+
+## Settings
+
+[.env.example](.env.example) (the engine) and
+[runner.env.example](runner.env.example) (the runner's limits: never
+credentials, since scripts run beside it). The app's compose file usually
+sets `RUNNER_URL` (the runner's address, default `http://runner:3000`) and
+`PUBLIC_HOST` (what `/health` reports, default `engine`).
+
+## API
+
+| | |
+|---|---|
+| `GET /health` | `{ status: 'ok', host, port, assistant }`: the app's connection check (`assistant`: `{ model, sampleRows }`, or null when it isn't set up) |
+| `POST /query` | form data: `sql`, and an `input` file per input (Parquet, its filename the reference name: `sqlnode1_data`, or `dataingest1_data.sheet1` for one table of several). Answers with the result as Parquet |
+| `POST /run/python` | form data: `code`, and `input` files as above. Answers with form data: `result` (JSON) and `table` (Parquet) when the last expression was a table |
+| `POST /execute` | JSON `{ document, targets? }`: runs a graph given in full |
+| `POST /graphs/{id}/execute` | JSON `{ targets? }` (optional): runs a saved graph |
+| `GET /graphs`, `GET /graphs/{id}`, `PUT /graphs/{id}` | saved graphs; `PUT` takes `{ name, document }` and answers `{ id, version }` |
+| `PUT /files/{key}` | stores data a DataIngest node read (Parquet), for server runs |
+| `POST /assist` | JSON `{ kind, request, code, inputs }`: the AI assistant writes a node's code. Answers `{ code, note, model }` (see below) |
+
+Errors are JSON `{ error }`, their message meant for the user. `/run/python`'s
+`result` is `{ output, value, variables }` or `{ output, error }`:
+
+- `output`: what the script printed.
+- `value`: its last expression's value: `{ kind: 'table', rows }` (the table
+  is the `table` part), `{ kind: 'json', type, value }`,
+  `{ kind: 'text', type, text }` (anything JSON can't hold), or
+  `{ kind: 'none' }`.
+- `variables`: its top-level variables, `[{ name, type, summary }]` (for
+  promoting them to outputs of their own, later).
+
+A graph run (`/execute`, `/graphs/{id}/execute`, e.g. from n8n) runs `targets`
+(default: every node with Auto Run on) and everything upstream, as the browser
+does, and answers `{ runId, status, failed, results }`: per node `{ status,
+error? | rowCount, output?, value?, export? }`, an export's file as
+`{ filename, contentType, size, base64 }`. Runs are recorded in `runs` when
+there's a database. On the server:
+
+| Kind | |
+|---|---|
+| SQLQuery | on the server's DuckDB: its inputs and the query database |
+| PythonScript | in the runner |
+| DataIngest | from its tables stored with `PUT /files/{key}` (the keys in the node's `ingest.tables`) |
+| DataExport | to CSV, TSV, JSON or Parquet, returned in the result |
+| JavaScript | refused: browser only until the engine has safeguards for it |
+| anything else | can't run on the server yet |
+
+## AI assistant
+
+The Assistant field in SEAMONSTER's Manage panel (SQLQuery, PythonScript and
+JavaScript nodes) asks `POST /assist` ([assist.py](davy_jones_locker/assist.py)),
+which asks a LiteLLM proxy's `/chat/completions`, OpenAI-style. Set it up in
+`.env`:
+
+| | |
+|---|---|
+| `LITELLM_BASE_URL` | the proxy's address (with or without `/v1`) |
+| `LITELLM_API_KEY` | its key: stays on the engine. The browser never sees it, and the runner can't read it |
+| `ASSIST_MODEL` | a model name the proxy offers. No default: the assistant is off until it's set |
+| `ASSIST_SAMPLE_ROWS` | rows of each input sent as examples of its values (default 3; 0 sends column names and types only) |
+| `ASSIST_TIMEOUT_SECONDS` | how long to wait for an answer (default 90) |
+
+Off (and hidden in the app) unless all three of the first settings are set.
+What the model is sent: the request; the node's kind and current code; each
+input's reference name, columns (name and type), row count and first
+`ASSIST_SAMPLE_ROWS` rows (values cut to 200 characters), which the browser
+works out by running what's upstream; and for SQL, the query database's
+tables and columns, not their rows. For Python, it's told the packages in
+[runner/requirements.in](runner/requirements.in), so enabling one there tells
+it too. It answers by calling a `write_code` tool (code and a note), or with
+a fenced code block. The code only fills the node's editor: nothing runs
+until the user runs it. Prompts and answers aren't logged, only their size
+and time.
+
+A LiteLLM proxy on the Docker host rather than on the network: from inside
+the container, that's `host.docker.internal`, which needs
+`extra_hosts: ["host.docker.internal:host-gateway"]` on the engine's service.
+
+## Database
+
+The engine owns its tables, in schema `FLOW_SCHEMA` (`flow`), created the
+first time the database answers ([database.py](davy_jones_locker/database.py)):
+`graphs`, `files` (stored data by key), `runs`. Without `DATABASE_URL`, or
+while the database can't be reached, those endpoints answer 503 and graph runs
+aren't recorded; everything else still works, and the engine starts either
+way.
+
+SQL reads the query database (`QUERY_DATABASE_URL`, else `DATABASE_URL`),
+attached read-only as catalog `db`: `db.public.orders`, or just `orders`
+(plain names look in the inputs, then `db.public`). Its role should be
+read-only: engine-shared's reader role is, and
+[its script](../engine-shared/postgres/engine-roles.sql) makes the same roles
+on any Postgres.
+
+## Safeguards
+
+User code runs in two places here, and neither is trusted.
+
+**SQL** ([sql.py](davy_jones_locker/sql.py)). Each query or graph run gets its
+own in-memory DuckDB. Once the query database is attached, DuckDB is locked
+down: no file access, no new extensions or attachments, settings frozen. That
+doesn't stop `postgres_scan()` connecting to any server by connection string,
+so a query must also be one SELECT that calls none of the functions that open
+connections or run SQL given as text (`postgres_*`, `query`, `query_table`),
+checked with DuckDB's own parser. Queries stop after `QUERY_TIMEOUT_SECONDS`
+and are held to `QUERY_MEMORY_LIMIT`.
+
+**Python** ([sandbox/](davy_jones_locker/sandbox/)). The engine never runs
+it: the runner does, in a separate container that holds no credentials (its
+settings file is separate, and nothing secret goes in it) and can reach
+nothing but the engine. The container is read-only, unprivileged,
+capability-free and resource-limited ([../compose.yaml](../compose.yaml)).
+Each script runs in its own process with time, CPU, memory and file limits,
+in a scratch directory deleted afterwards, one at a time per container
+(concurrent runs share a user, so could read each other's files: scale with
+replicas). After each run, every process it started is killed, even ones
+that left its process group, so nothing a script leaves running sees the next
+run; the container's init reaps them, so they can't use up its process limit
+either. Python can't be sandboxed from inside itself, so these limits are the
+safeguard, not anything the script is told. In production, Azure Container
+Apps' dynamic sessions (Hyper-V isolated) could replace the runner behind the
+same API.
+
+**Libraries**: scripts get exactly what [runner/requirements.in](runner/requirements.in)
+lists (pandas, numpy, pyarrow), compiled to a pinned, hashed
+`runner/requirements.txt` and installed at image build; nothing installs at
+run time. To enable a library, add it there, recompile (the command is in the
+file), and rebuild. An app adds its own the same way, in a runner image
+built on this one.
+
+## Layout
+
+```
+davy_jones_locker/
+  main.py        the API, and create_app
+  settings.py    environment settings (.env.example)
+  database.py    the engine's tables in Postgres
+  sql.py         locked-down DuckDB sessions, query checks
+  runner.py      calls to the runner
+  executor.py    whole-graph runs
+  assist.py      the AI assistant (through LiteLLM)
+  formdata.py    multipart/form-data both ways (the runner uses it too)
+  sandbox/
+    server.py    the runner's API, limits, one process per run
+    worker.py    runs one script
+runner/          the runner's Dockerfile and libraries
+Dockerfile       the engine's image
+```
+
+## Developing without Docker
+
+With Python 3.12 and the packages from both lock files installed, from this
+folder:
+
+```
+RUNNER_URL=http://127.0.0.1:9002 uvicorn davy_jones_locker.main:app --port 9001
+uvicorn davy_jones_locker.sandbox.server:app --port 9002
+```
+
+(On Windows the runner can't set its process limits, and there's no container
+around it: development only.)