polymo 1.1.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {polymo-1.1.0 → polymo-1.3.0}/PKG-INFO +22 -26
- {polymo-1.1.0 → polymo-1.3.0}/README.md +16 -18
- {polymo-1.1.0 → polymo-1.3.0}/pyproject.toml +6 -6
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/preview.py +1 -0
- polymo-1.3.0/src/polymo/builder/static/main.js +80 -0
- polymo-1.3.0/src/polymo/cli.py +70 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/codegen/generator.py +6 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/codegen/templates/core.py.jinja +72 -31
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/codegen/templates/dp.py.jinja +42 -24
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/config.py +256 -18
- polymo-1.1.0/src/polymo/builder/static/main.js +0 -80
- polymo-1.1.0/src/polymo/cli.py +0 -70
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/__init__.py +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/__init__.py +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/app.py +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/examples/countries.polymo.json +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/examples/githubrepos.polymo.json +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/examples/jsonplaceholder.polymo.json +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/examples/jsonplaceholder_endpoints.polymo.json +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/examples/weather.polymo.json +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/favicon.ico +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/favicon.png +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/index.html +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/logo.png +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/logo192.png +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/main.css +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/static/mockServiceWorker.js +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/builder/templates/index.html +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/codegen/__init__.py +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/codegen/templating.py +0 -0
- {polymo-1.1.0 → polymo-1.3.0}/src/polymo/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: polymo
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder
|
|
5
5
|
Keywords: spark,pyspark,rest,api,ingestion,data-engineering,etl,http
|
|
6
6
|
Author: Daniel Tom
|
|
@@ -21,18 +21,16 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
21
21
|
Classifier: Operating System :: OS Independent
|
|
22
22
|
Classifier: Typing :: Typed
|
|
23
23
|
Requires-Dist: jinja2>=3.1.6
|
|
24
|
-
Requires-Dist: fastapi>=0.110
|
|
25
|
-
Requires-Dist: uvicorn>=0.24
|
|
26
|
-
Requires-Dist:
|
|
27
|
-
Requires-Dist:
|
|
28
|
-
Requires-Dist:
|
|
29
|
-
Requires-Dist: requests>=2.31 ; extra == 'builder'
|
|
24
|
+
Requires-Dist: fastapi>=0.110
|
|
25
|
+
Requires-Dist: uvicorn>=0.24
|
|
26
|
+
Requires-Dist: pyspark>=4
|
|
27
|
+
Requires-Dist: pyarrow>=13
|
|
28
|
+
Requires-Dist: requests>=2.31
|
|
30
29
|
Requires-Python: >=3.10
|
|
31
30
|
Project-URL: Documentation, https://dan1elt0m.github.io/polymo/
|
|
32
31
|
Project-URL: Homepage, https://github.com/dan1elt0m/polymo
|
|
33
32
|
Project-URL: Issues, https://github.com/dan1elt0m/polymo/issues
|
|
34
33
|
Project-URL: Repository, https://github.com/dan1elt0m/polymo.git
|
|
35
|
-
Provides-Extra: builder
|
|
36
34
|
Description-Content-Type: text/markdown
|
|
37
35
|
|
|
38
36
|
<p align="center">
|
|
@@ -65,9 +63,9 @@ Standalone Lakeflow Declarative Pipelines source. polymo is NOT needed
|
|
|
65
63
|
at runtime; edit this file freely.
|
|
66
64
|
"""
|
|
67
65
|
|
|
68
|
-
BASE_URL = "https://jsonplaceholder.typicode.com"
|
|
69
|
-
PATH = "/posts"
|
|
70
|
-
PARAMS: dict = {"_limit": 20}
|
|
66
|
+
BASE_URL: str = "https://jsonplaceholder.typicode.com"
|
|
67
|
+
PATH: str = "/posts"
|
|
68
|
+
PARAMS: dict[str, Any] = {"_limit": 20}
|
|
71
69
|
|
|
72
70
|
# … fetch/retry helpers elided …
|
|
73
71
|
|
|
@@ -114,30 +112,28 @@ The YAML runtime (`spark.read.format("polymo")`, `PolymoConfig`,
|
|
|
114
112
|
ready to move yet.
|
|
115
113
|
|
|
116
114
|
## How to start?
|
|
117
|
-
Locally you probably want to install polymo along with the Builder UI:
|
|
118
115
|
|
|
119
116
|
```bash
|
|
120
|
-
pip install
|
|
117
|
+
pip install polymo
|
|
121
118
|
```
|
|
122
119
|
|
|
123
|
-
This
|
|
120
|
+
This installs everything you need, Builder UI included — FastAPI, Uvicorn,
|
|
121
|
+
PySpark, PyArrow, and requests all come along, so there's nothing extra to
|
|
122
|
+
opt into.
|
|
123
|
+
|
|
124
|
+
Note that `polymo` is never a dependency of the scripts it generates —
|
|
125
|
+
nothing you export needs polymo installed to run.
|
|
126
|
+
|
|
127
|
+
## Launch the builder UI
|
|
124
128
|
|
|
125
|
-
Scripting connector generation directly in Python (calling `generate()` /
|
|
126
|
-
`parse_config()` without the UI — see the [API reference](docs/api.md))
|
|
127
|
-
doesn't need pyspark at all. In that case, just install the bare minimum
|
|
128
|
-
deps with:
|
|
129
129
|
```bash
|
|
130
|
-
|
|
130
|
+
polymo
|
|
131
131
|
```
|
|
132
132
|
|
|
133
|
-
|
|
134
|
-
`polymo` is never a dependency of the scripts it generates — nothing you
|
|
135
|
-
export needs polymo installed to run.
|
|
136
|
-
|
|
137
|
-
## Launch the builder UI
|
|
133
|
+
Or without installing it first:
|
|
138
134
|
|
|
139
|
-
```bash
|
|
140
|
-
polymo
|
|
135
|
+
```bash
|
|
136
|
+
uvx polymo
|
|
141
137
|
```
|
|
142
138
|
|
|
143
139
|
#### (Optional) Run the Builder in Docker
|
|
@@ -28,9 +28,9 @@ Standalone Lakeflow Declarative Pipelines source. polymo is NOT needed
|
|
|
28
28
|
at runtime; edit this file freely.
|
|
29
29
|
"""
|
|
30
30
|
|
|
31
|
-
BASE_URL = "https://jsonplaceholder.typicode.com"
|
|
32
|
-
PATH = "/posts"
|
|
33
|
-
PARAMS: dict = {"_limit": 20}
|
|
31
|
+
BASE_URL: str = "https://jsonplaceholder.typicode.com"
|
|
32
|
+
PATH: str = "/posts"
|
|
33
|
+
PARAMS: dict[str, Any] = {"_limit": 20}
|
|
34
34
|
|
|
35
35
|
# … fetch/retry helpers elided …
|
|
36
36
|
|
|
@@ -77,30 +77,28 @@ The YAML runtime (`spark.read.format("polymo")`, `PolymoConfig`,
|
|
|
77
77
|
ready to move yet.
|
|
78
78
|
|
|
79
79
|
## How to start?
|
|
80
|
-
Locally you probably want to install polymo along with the Builder UI:
|
|
81
80
|
|
|
82
81
|
```bash
|
|
83
|
-
pip install
|
|
82
|
+
pip install polymo
|
|
84
83
|
```
|
|
85
84
|
|
|
86
|
-
This
|
|
85
|
+
This installs everything you need, Builder UI included — FastAPI, Uvicorn,
|
|
86
|
+
PySpark, PyArrow, and requests all come along, so there's nothing extra to
|
|
87
|
+
opt into.
|
|
88
|
+
|
|
89
|
+
Note that `polymo` is never a dependency of the scripts it generates —
|
|
90
|
+
nothing you export needs polymo installed to run.
|
|
91
|
+
|
|
92
|
+
## Launch the builder UI
|
|
87
93
|
|
|
88
|
-
Scripting connector generation directly in Python (calling `generate()` /
|
|
89
|
-
`parse_config()` without the UI — see the [API reference](docs/api.md))
|
|
90
|
-
doesn't need pyspark at all. In that case, just install the bare minimum
|
|
91
|
-
deps with:
|
|
92
94
|
```bash
|
|
93
|
-
|
|
95
|
+
polymo
|
|
94
96
|
```
|
|
95
97
|
|
|
96
|
-
|
|
97
|
-
`polymo` is never a dependency of the scripts it generates — nothing you
|
|
98
|
-
export needs polymo installed to run.
|
|
99
|
-
|
|
100
|
-
## Launch the builder UI
|
|
98
|
+
Or without installing it first:
|
|
101
99
|
|
|
102
|
-
```bash
|
|
103
|
-
polymo
|
|
100
|
+
```bash
|
|
101
|
+
uvx polymo
|
|
104
102
|
```
|
|
105
103
|
|
|
106
104
|
#### (Optional) Run the Builder in Docker
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "polymo"
|
|
3
|
-
version = "1.
|
|
3
|
+
version = "1.3.0"
|
|
4
4
|
description = "Generate standalone Lakeflow Declarative Pipelines scripts from a point-and-click REST connector builder"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -27,13 +27,8 @@ classifiers = [
|
|
|
27
27
|
]
|
|
28
28
|
dependencies = [
|
|
29
29
|
"jinja2>=3.1.6",
|
|
30
|
-
]
|
|
31
|
-
|
|
32
|
-
[project.optional-dependencies]
|
|
33
|
-
builder = [
|
|
34
30
|
"fastapi>=0.110",
|
|
35
31
|
"uvicorn>=0.24",
|
|
36
|
-
"jinja2>=3.1.6",
|
|
37
32
|
"pyspark>=4",
|
|
38
33
|
"pyarrow>=13",
|
|
39
34
|
"requests>=2.31",
|
|
@@ -82,3 +77,8 @@ Issues = "https://github.com/dan1elt0m/polymo/issues"
|
|
|
82
77
|
# must not reformat them or the golden tests would drift from what the
|
|
83
78
|
# generator actually produces.
|
|
84
79
|
extend-exclude = ["tests/codegen/golden"]
|
|
80
|
+
|
|
81
|
+
[tool.pytest.ini_options]
|
|
82
|
+
markers = [
|
|
83
|
+
"spark: exercises a real local[1] PySpark session (slower: ~seconds of JVM startup).",
|
|
84
|
+
]
|
|
@@ -29,6 +29,7 @@ def run_preview(
|
|
|
29
29
|
if token:
|
|
30
30
|
namespace["API_TOKEN"] = token # bearer
|
|
31
31
|
namespace["CLIENT_SECRET"] = token # oauth2 (harmless if unused)
|
|
32
|
+
namespace["API_KEY"] = token # api_key (harmless if unused)
|
|
32
33
|
|
|
33
34
|
raw_pages: List[Dict[str, Any]] = []
|
|
34
35
|
original_request = namespace["_request"]
|