nextpipe 0.3.5.dev0__tar.gz → 0.4.0.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/PKG-INFO +2 -2
- nextpipe-0.4.0.dev0/docs/examples/multifile-workflow.md +186 -0
- nextpipe-0.4.0.dev0/docs/tutorials/echo-multi.md +154 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/mkdocs.yml +1 -0
- nextpipe-0.4.0.dev0/nextpipe/__about__.py +1 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/decorators.py +24 -8
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/flow.py +53 -13
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/schema.py +8 -4
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/pyproject.toml +1 -1
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/.gitignore +1 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/README.md +5 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/.gitignore +1 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/README.md +19 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/app.yaml +6 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/inputs/data.csv +3 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/inputs/input.xlsx +0 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/main.py +57 -0
- nextpipe-0.4.0.dev0/tests/apps/echo-multi/requirements.txt +1 -0
- nextpipe-0.4.0.dev0/tests/pipelines/appapp.json.golden +24 -0
- nextpipe-0.4.0.dev0/tests/pipelines/appapp.py +52 -0
- nextpipe-0.4.0.dev0/tests/pipelines/foreach.json +3 -0
- nextpipe-0.4.0.dev0/tests/pipelines/inputs/data.csv +3 -0
- nextpipe-0.4.0.dev0/tests/pipelines/inputs/input.xlsx +0 -0
- nextpipe-0.4.0.dev0/tests/pipelines/multifile.py +92 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_graph.py +6 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_integration.py +44 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_schema.py +6 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_threads.py +6 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_uplink.py +6 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_version.py +6 -0
- nextpipe-0.3.5.dev0/nextpipe/__about__.py +0 -1
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/lint.yml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/release.yml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/test.yml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.gitignore +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.markdownlint.jsonc +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.nextmv/bump_requirements.sh +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.prettierrc.yml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.readthedocs.yaml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/LICENSE.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/README.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/RELEASE.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/basic-chained-workflow.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/complex-workflow-csv.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/complex-workflow.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/ensemble-workflow.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/fanout-workflow.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/logo-180.png +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/nextmv-favicon.svg +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/routing-ortools-pyvroom-selected.png +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/routing-selected.png +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/index.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/config.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/decorators.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/flow.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/graph.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/schema.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/threads.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/uplink.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/utils.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/requirements.txt +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/tutorials/echo.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/tutorials/getting-started.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/__init__.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/config.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/graph.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/threads.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/uplink.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/utils.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe.code-workspace +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/__init__.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/.gitignore +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/README.md +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/app.yaml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/main.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/requirements.txt +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/app.yaml +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/main.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/requirements.txt +0 -0
- /nextpipe-0.3.5.dev0/tests/pipelines/chain.json → /nextpipe-0.4.0.dev0/tests/pipelines/appapp.json +0 -0
- /nextpipe-0.3.5.dev0/tests/pipelines/foreach-2-pred.json → /nextpipe-0.4.0.dev0/tests/pipelines/chain.json +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/chain.json.golden +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/chain.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.json +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.json.golden +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/fail.py +0 -0
- /nextpipe-0.3.5.dev0/tests/pipelines/foreach.json → /nextpipe-0.4.0.dev0/tests/pipelines/foreach-2-pred.json +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach-2-pred.json.golden +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach-2-pred.py +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach.json.golden +0 -0
- {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nextpipe
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0.dev0
|
|
4
4
|
Summary: Framework for Decision Pipeline modeling and execution
|
|
5
5
|
Project-URL: Homepage, https://www.nextmv.io
|
|
6
6
|
Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
|
|
@@ -108,7 +108,7 @@ Requires-Dist: dataclasses-json>=0.6.7
|
|
|
108
108
|
Requires-Dist: nextmv>=0.29.4
|
|
109
109
|
Requires-Dist: requests>=2.31.0
|
|
110
110
|
Provides-Extra: dev
|
|
111
|
-
Requires-Dist: goldie>=0.1.
|
|
111
|
+
Requires-Dist: goldie>=0.1.8; extra == 'dev'
|
|
112
112
|
Requires-Dist: ruff>=0.11.6; extra == 'dev'
|
|
113
113
|
Description-Content-Type: text/markdown
|
|
114
114
|
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
# multi-file Workflow Example
|
|
2
|
+
|
|
3
|
+
!!! tip
|
|
4
|
+
|
|
5
|
+
This example uses the [`echo` app](../tutorials/echo.md), make sure to complete
|
|
6
|
+
that tutorial first.
|
|
7
|
+
|
|
8
|
+
This example showcases how to use **multi-file** applications _within_ a
|
|
9
|
+
Nextpipe workflow. A multi-file application differs from a JSON-based
|
|
10
|
+
application in that it accepts a directory of files as input and produces a
|
|
11
|
+
directory of files as output. Note that the workflow itself is also a multi-file
|
|
12
|
+
application, however, this is a user choice (i.e., you could also create a
|
|
13
|
+
JSON-based workflow that uses multi-file sub-applications).
|
|
14
|
+
|
|
15
|
+
For demonstration purposes, we will use the simple [echo-multi application] as
|
|
16
|
+
the sub-application, which echoes the input files as output files.
|
|
17
|
+
|
|
18
|
+
Find the workflow code below (mind the comments explaining each step):
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import os
|
|
22
|
+
import shutil
|
|
23
|
+
|
|
24
|
+
import nextmv
|
|
25
|
+
import nextmv.cloud
|
|
26
|
+
|
|
27
|
+
from nextpipe import FlowSpec, app, log, needs, step
|
|
28
|
+
|
|
29
|
+
options = nextmv.Options(
|
|
30
|
+
nextmv.Option("input", str, "inputs/", "Path to input file.", False),
|
|
31
|
+
nextmv.Option("output", str, "outputs/", "Path to output file.", False),
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
# >>> Workflow definition
|
|
36
|
+
class Flow(FlowSpec):
|
|
37
|
+
# The first step receives the path to the input files directly (see main()) and
|
|
38
|
+
# automatically zips the directory and passes it to the 'echo-multi' sub-app.
|
|
39
|
+
@app(app_id="echo-multi")
|
|
40
|
+
@step
|
|
41
|
+
def solve1():
|
|
42
|
+
"""Runs a multi-file model."""
|
|
43
|
+
pass
|
|
44
|
+
|
|
45
|
+
# The second step receives the path to the output files from the first step. This path
|
|
46
|
+
# will point to a temporary directory containing the output files from the first step.
|
|
47
|
+
@needs(predecessors=[solve1])
|
|
48
|
+
@step
|
|
49
|
+
def transform(result_path: str):
|
|
50
|
+
"""Transforms the result for the next step."""
|
|
51
|
+
# Just list the content of the result directory.
|
|
52
|
+
log(f"Contents of result directory {result_path}:")
|
|
53
|
+
for file_name in os.listdir(result_path):
|
|
54
|
+
full_file_name = os.path.join(result_path, file_name)
|
|
55
|
+
if os.path.isfile(full_file_name):
|
|
56
|
+
log(f"- {file_name}")
|
|
57
|
+
|
|
58
|
+
# Add a new file to the result for demonstration purposes.
|
|
59
|
+
new_file_path = os.path.join(result_path, "additional_file.txt")
|
|
60
|
+
with open(new_file_path, "w") as f:
|
|
61
|
+
f.write("This is an additional file added in the transform step.\n")
|
|
62
|
+
log(f"Added new file: {new_file_path}")
|
|
63
|
+
|
|
64
|
+
return result_path
|
|
65
|
+
|
|
66
|
+
# The third step receives the (modified) directory from the transform step and runs
|
|
67
|
+
# another multi-file app on it.
|
|
68
|
+
@app(
|
|
69
|
+
app_id="echo-multi",
|
|
70
|
+
# We specify the content type explicitly here. This is normally done via the app's
|
|
71
|
+
# manifest, but we can do it explicitly like this too.
|
|
72
|
+
run_configuration=nextmv.RunConfiguration(
|
|
73
|
+
format=nextmv.Format(
|
|
74
|
+
format_input=nextmv.FormatInput(input_type=nextmv.InputFormat.MULTI_FILE),
|
|
75
|
+
format_output=nextmv.FormatOutput(output_type=nextmv.OutputFormat.MULTI_FILE),
|
|
76
|
+
)
|
|
77
|
+
),
|
|
78
|
+
full_result=True,
|
|
79
|
+
)
|
|
80
|
+
@needs(predecessors=[transform])
|
|
81
|
+
@step
|
|
82
|
+
def solve2(result: nextmv.cloud.RunResult):
|
|
83
|
+
"""Runs another multi-file model."""
|
|
84
|
+
pass
|
|
85
|
+
|
|
86
|
+
# The final step receives the output from 'solve2' as a full result object (see
|
|
87
|
+
# 'full_result=True' above). In this case, the path to the output files is available
|
|
88
|
+
# via 'result.output'.
|
|
89
|
+
@needs(predecessors=[solve2])
|
|
90
|
+
@step
|
|
91
|
+
def prepare_output(result: nextmv.cloud.RunResult):
|
|
92
|
+
"""Transforms the result for the next step."""
|
|
93
|
+
# Extract the path to the output files.
|
|
94
|
+
result_path = result.output
|
|
95
|
+
# Simply copy the files from the given directory to the expected output directory.
|
|
96
|
+
os.makedirs(options.output, exist_ok=True)
|
|
97
|
+
for file_name in os.listdir(result_path):
|
|
98
|
+
full_file_name = os.path.join(result_path, file_name)
|
|
99
|
+
if os.path.isfile(full_file_name):
|
|
100
|
+
shutil.copy(full_file_name, options.output)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def main():
|
|
104
|
+
# Run workflow (simply provide the path to the multi-file input)
|
|
105
|
+
flow = Flow("DecisionFlow", options.input)
|
|
106
|
+
flow.run()
|
|
107
|
+
# The last step of the flow already prepares the output in the requested directory,
|
|
108
|
+
# so no need to do anything here anymore.
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
main()
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Run the example:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
$ python main.py
|
|
119
|
+
[nextpipe] No application ID or run ID found, uplink is inactive.
|
|
120
|
+
[nextpipe] Flow: Flow
|
|
121
|
+
[nextpipe] nextpipe: v0.3.5
|
|
122
|
+
[nextpipe] nextmv: 0.33.0
|
|
123
|
+
[nextpipe] Flow graph steps:
|
|
124
|
+
[nextpipe] Step:
|
|
125
|
+
[nextpipe] Definition: Step(solve1, StepRun(echo-multi, , {}, False))
|
|
126
|
+
[nextpipe] Docstring: Runs a multi-file model.
|
|
127
|
+
[nextpipe] Step:
|
|
128
|
+
[nextpipe] Definition: Step(transform, StepNeeds(solve1))
|
|
129
|
+
[nextpipe] Docstring: Transforms the result for the next step.
|
|
130
|
+
[nextpipe] Step:
|
|
131
|
+
[nextpipe] Definition: Step(solve2, StepNeeds(transform), StepRun(echo-multi, , {}, True))
|
|
132
|
+
[nextpipe] Docstring: Runs another multi-file model.
|
|
133
|
+
[nextpipe] Step:
|
|
134
|
+
[nextpipe] Definition: Step(prepare_output, StepNeeds(solve2))
|
|
135
|
+
[nextpipe] Docstring: Transforms the result for the next step.
|
|
136
|
+
[nextpipe] Mermaid diagram:
|
|
137
|
+
[nextpipe] graph LR
|
|
138
|
+
solve1(solve1)
|
|
139
|
+
solve1 --> transform
|
|
140
|
+
transform(transform)
|
|
141
|
+
transform --> solve2
|
|
142
|
+
solve2(solve2)
|
|
143
|
+
solve2 --> prepare_output
|
|
144
|
+
prepare_output(prepare_output)
|
|
145
|
+
|
|
146
|
+
[nextpipe] Mermaid URL: https://mermaid.ink/svg/Z3JhcGggTFIKICBzb2x2ZTEoc29sdmUxKQogIHNvbHZlMSAtLT4gdHJhbnNmb3JtCiAgdHJhbnNmb3JtKHRyYW5zZm9ybSkKICB0cmFuc2Zvcm0gLS0+IHNvbHZlMgogIHNvbHZlMihzb2x2ZTIpCiAgc29sdmUyIC0tPiBwcmVwYXJlX291dHB1dAogIHByZXBhcmVfb3V0cHV0KHByZXBhcmVfb3V0cHV0KQo=?theme=dark
|
|
147
|
+
[nextpipe] Running node solve1_0
|
|
148
|
+
[nextpipe] Started app step solve1_0 run, find it at https://cloud.nextmv.io/app/echo-multi/run/latest-a-JAvuFgDR?view=details
|
|
149
|
+
/home/marius/.asdf/installs/python/3.13.7/lib/python3.13/shutil.py:1281: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
|
|
150
|
+
tarobj.extractall(extract_dir, filter=filter)
|
|
151
|
+
[nextpipe] Running node transform_0
|
|
152
|
+
[transform_0] Contents of result directory /tmp/nextpipe_output_igqsibzm:
|
|
153
|
+
[transform_0] - input.xlsx
|
|
154
|
+
[transform_0] - data.csv
|
|
155
|
+
[transform_0] Added new file: /tmp/nextpipe_output_igqsibzm/additional_file.txt
|
|
156
|
+
[nextpipe] Running node solve2_0
|
|
157
|
+
[nextpipe] Started app step solve2_0 run, find it at https://cloud.nextmv.io/app/echo-multi/run/latest-HIwvuFgDg?view=details
|
|
158
|
+
[nextpipe] Running node prepare_output_0
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Content of the output directory:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
tree outputs/
|
|
165
|
+
outputs/
|
|
166
|
+
├── additional_file.txt
|
|
167
|
+
├── data.csv
|
|
168
|
+
└── input.xlsx
|
|
169
|
+
|
|
170
|
+
1 directory, 3 files
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
The resulting Mermaid diagram for this flow looks like this:
|
|
174
|
+
|
|
175
|
+
```mermaid
|
|
176
|
+
graph LR
|
|
177
|
+
solve1(solve1)
|
|
178
|
+
solve1 --> transform
|
|
179
|
+
transform(transform)
|
|
180
|
+
transform --> solve2
|
|
181
|
+
solve2(solve2)
|
|
182
|
+
solve2 --> prepare_output
|
|
183
|
+
prepare_output(prepare_output)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
[echo-multi application]: ../tutorials/echo-multi.md
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# The `echo-multi` app
|
|
2
|
+
|
|
3
|
+
Several examples assume you have a Nextmv application called `echo-multi`. This
|
|
4
|
+
is just a simple application created for demonstration purposes. It takes the
|
|
5
|
+
input files and echoes them as output files.
|
|
6
|
+
|
|
7
|
+
Let's get set up with the `echo-multi` application. Before starting:
|
|
8
|
+
|
|
9
|
+
1. [Sign up][signup] for a Nextmv account.
|
|
10
|
+
2. Get your API key. Go to [Team > API Key][api-key].
|
|
11
|
+
|
|
12
|
+
Make sure that you have your API key set as an environment variable:
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
export NEXTMV_API_KEY="<YOUR-API-KEY>"
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Now that you have a valid Nextmv account and API key, let's create the
|
|
19
|
+
`echo-multi` Nextmv app (start in an empty directory).
|
|
20
|
+
|
|
21
|
+
1. Create a folder `inputs/` and add some sample input files to it. For example,
|
|
22
|
+
you can create two text files `input.csv` and `input.txt` with some sample
|
|
23
|
+
content.
|
|
24
|
+
1. In a new directory, create a file called `main.py` with the code for the
|
|
25
|
+
basic app that echoes the input.
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import glob
|
|
29
|
+
|
|
30
|
+
import os
|
|
31
|
+
import time
|
|
32
|
+
|
|
33
|
+
import nextmv
|
|
34
|
+
|
|
35
|
+
def main():
|
|
36
|
+
options = nextmv.Options(
|
|
37
|
+
nextmv.Option("input", str, "inputs/", "Path to input file.", False),
|
|
38
|
+
nextmv.Option("output", str, "outputs/", "Path to output file.", False),
|
|
39
|
+
nextmv.Option("duration", float, 1.0, "Runtime duration (in seconds).", False),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
# Read and prepare the input data.
|
|
43
|
+
input_data = read_input(options.input)
|
|
44
|
+
|
|
45
|
+
# Log information about the input files.
|
|
46
|
+
nextmv.log(f"Size of input files (count: {len(input_data)}):")
|
|
47
|
+
for file_path, content in input_data.items():
|
|
48
|
+
nextmv.log(f" {file_path}: {len(content)} bytes")
|
|
49
|
+
|
|
50
|
+
# Sleep for the specified duration.
|
|
51
|
+
nextmv.log(f"Sleeping for {options.duration} seconds...")
|
|
52
|
+
time.sleep(options.duration)
|
|
53
|
+
nextmv.log("Woke up from sleep.")
|
|
54
|
+
|
|
55
|
+
# Write the output.
|
|
56
|
+
write_output(options.output, input_data)
|
|
57
|
+
|
|
58
|
+
def read_input(input_path: str) -> dict[str, bytes]:
|
|
59
|
+
"""Reads the input files to memory."""
|
|
60
|
+
input_files = glob.glob(os.path.join(input_path, "**/*"), recursive=True)
|
|
61
|
+
content = {}
|
|
62
|
+
for file_path in input_files:
|
|
63
|
+
if os.path.isfile(file_path):
|
|
64
|
+
with open(file_path, "rb") as file:
|
|
65
|
+
nextmv.log(f"Reading file: {file_path}")
|
|
66
|
+
content[file_path] = file.read()
|
|
67
|
+
return content
|
|
68
|
+
|
|
69
|
+
def write_output(output_path: str, content: dict[str, bytes]) -> None:
|
|
70
|
+
"""Writes the given output files."""
|
|
71
|
+
if not os.path.exists(output_path):
|
|
72
|
+
os.makedirs(output_path)
|
|
73
|
+
|
|
74
|
+
for file_path, data in content.items():
|
|
75
|
+
output_file_path = os.path.join(output_path, os.path.basename(file_path))
|
|
76
|
+
with open(output_file_path, "wb") as file:
|
|
77
|
+
nextmv.log(f"Writing file: {output_file_path}")
|
|
78
|
+
file.write(data)
|
|
79
|
+
|
|
80
|
+
if __name__ == "__main__":
|
|
81
|
+
main()
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Note that the application uses the [`nextmv`][nextmv-docs] library. This
|
|
85
|
+
library is a dependency of `nextpipe` and should be installed automatically
|
|
86
|
+
when you install `nextpipe`.
|
|
87
|
+
|
|
88
|
+
You may run the app locally to test it:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
python main.py
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
1. Create a `requirements.txt` file with the following requirements for running
|
|
95
|
+
the app:
|
|
96
|
+
|
|
97
|
+
```requirements.txt
|
|
98
|
+
nextmv
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
1. Create an `app.yaml` file (the app manifest) with the following instructions:
|
|
102
|
+
|
|
103
|
+
```yaml
|
|
104
|
+
type: python
|
|
105
|
+
runtime: ghcr.io/nextmv-io/runtime/python:3.11
|
|
106
|
+
files:
|
|
107
|
+
- main.py
|
|
108
|
+
python:
|
|
109
|
+
pip-requirements: requirements.txt
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
1. Push the application to your Nextmv account. Create a `push.py` script in
|
|
113
|
+
the same directory with the following code:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
import os
|
|
117
|
+
|
|
118
|
+
from nextmv import cloud
|
|
119
|
+
|
|
120
|
+
client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
|
|
121
|
+
app = cloud.Application.new(client=client, name="echo-multi", id="echo-multi", description="Sample echo multi-file app.", exist_ok=True)
|
|
122
|
+
app.push(verbose=True)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
1. Execute the `push.py` script to push the app to your Nextmv account:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
$ python push.py
|
|
129
|
+
💽 Starting build for Nextmv application.
|
|
130
|
+
🐍 Bundling Python dependencies.
|
|
131
|
+
📋 Copied files listed in "app.yaml" manifest.
|
|
132
|
+
📦 Packaged application (588 files, 5.39 MiB).
|
|
133
|
+
🌟 Pushing to application: "echo-multi".
|
|
134
|
+
💥️ Successfully pushed to application: "echo-multi".
|
|
135
|
+
{
|
|
136
|
+
"app_id": "echo-multi",
|
|
137
|
+
"endpoint": "https://api.cloud.nextmv.io",
|
|
138
|
+
"instance_url": "v1/applications/echo-multi/runs?instance_id=devint"
|
|
139
|
+
}
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Alternatively, you can use the [Nextmv CLI][nextmv-cli] to create and push the app:
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
nextmv app create -a echo-multi -n echo-multi -d "Sample echo multi-file app."
|
|
146
|
+
nextmv app push -a echo-multi
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Now you are ready to run the examples.
|
|
150
|
+
|
|
151
|
+
[signup]: https://cloud.nextmv.io
|
|
152
|
+
[api-key]: https://cloud.nextmv.io/team/api-keys
|
|
153
|
+
[nextmv-docs]: https://nextmv-py.readthedocs.io/en/latest/nextmv/
|
|
154
|
+
[nextmv-cli]: https://docs.nextmv.io/docs/using-nextmv/reference/cli
|
|
@@ -12,6 +12,7 @@ nav:
|
|
|
12
12
|
- The echo app: tutorials/echo.md
|
|
13
13
|
- Examples:
|
|
14
14
|
- Basic chained workflow: examples/basic-chained-workflow.md
|
|
15
|
+
- Multi-file workflow: examples/multifile-workflow.md
|
|
15
16
|
- Fanout workflow: examples/fanout-workflow.md
|
|
16
17
|
- Ensemble workflow: examples/ensemble-workflow.md
|
|
17
18
|
- Complex workflow: examples/complex-workflow.md
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "v0.4.0.dev.0"
|
|
@@ -50,6 +50,7 @@ from collections.abc import Callable
|
|
|
50
50
|
from enum import Enum
|
|
51
51
|
from functools import wraps
|
|
52
52
|
|
|
53
|
+
import nextmv
|
|
53
54
|
from nextmv import cloud
|
|
54
55
|
from nextmv.deprecated import deprecated
|
|
55
56
|
|
|
@@ -772,10 +773,10 @@ class App:
|
|
|
772
773
|
The ID of the instance to run.
|
|
773
774
|
options : dict[str, Any]
|
|
774
775
|
The options to pass to the application.
|
|
775
|
-
input_type : InputType
|
|
776
|
-
The type of input to pass to the application (JSON or FILES).
|
|
777
776
|
full_result : bool
|
|
778
777
|
Whether to return the full result including metadata.
|
|
778
|
+
run_configuration : nextmv.RunConfiguration
|
|
779
|
+
The configuration to apply when running the app.
|
|
779
780
|
polling_options : Optional[cloud.PollingOptions]
|
|
780
781
|
Options for polling for the results of the app run.
|
|
781
782
|
"""
|
|
@@ -788,6 +789,7 @@ class App:
|
|
|
788
789
|
parameters: dict[str, typing.Any] = None,
|
|
789
790
|
options: dict[str, typing.Any] = None,
|
|
790
791
|
full_result: bool = False,
|
|
792
|
+
run_configuration: nextmv.RunConfiguration = None,
|
|
791
793
|
polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
|
|
792
794
|
):
|
|
793
795
|
"""
|
|
@@ -799,16 +801,22 @@ class App:
|
|
|
799
801
|
The ID of the Nextmv Application to run.
|
|
800
802
|
instance_id : str, optional
|
|
801
803
|
The ID of the instance to run. Default is defined by the app on Platform.
|
|
802
|
-
input_type : InputType, optional
|
|
803
|
-
The type of input to pass to the application, by default InputType.JSON.
|
|
804
804
|
options : dict[str, Any], optional
|
|
805
805
|
The options to pass to the application, by default None.
|
|
806
806
|
full_result : bool, optional
|
|
807
807
|
Whether to return the full result including metadata, by default False.
|
|
808
|
+
run_configuration : nextmv.RunConfiguration, optional
|
|
809
|
+
The configuration to apply when running the app, by default None.
|
|
808
810
|
polling_options : Optional[cloud.PollingOptions], optional
|
|
809
811
|
Options for polling for the results of the app run, by default _DEFAULT_POLLING_OPTIONS.
|
|
810
812
|
"""
|
|
811
813
|
|
|
814
|
+
if input_type:
|
|
815
|
+
deprecated(
|
|
816
|
+
"input_type",
|
|
817
|
+
"The 'input_type' argument is deprecated and will be removed in a future release.",
|
|
818
|
+
)
|
|
819
|
+
|
|
812
820
|
# Make sure only one of options or parameters is used.
|
|
813
821
|
if parameters and options:
|
|
814
822
|
raise ValueError("You can only use either 'parameters' or 'options', not both.")
|
|
@@ -824,6 +832,7 @@ class App:
|
|
|
824
832
|
self.options = options if options else {}
|
|
825
833
|
self.input_type = input_type
|
|
826
834
|
self.full_result = full_result
|
|
835
|
+
self.run_configuration = run_configuration
|
|
827
836
|
self.polling_options = polling_options
|
|
828
837
|
|
|
829
838
|
def __repr__(self):
|
|
@@ -835,7 +844,7 @@ class App:
|
|
|
835
844
|
A string representation of the app.
|
|
836
845
|
"""
|
|
837
846
|
|
|
838
|
-
return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.
|
|
847
|
+
return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.full_result})"
|
|
839
848
|
|
|
840
849
|
|
|
841
850
|
def app(
|
|
@@ -845,6 +854,7 @@ def app(
|
|
|
845
854
|
options: dict[str, typing.Any] = None,
|
|
846
855
|
input_type: InputType = InputType.JSON,
|
|
847
856
|
full_result: bool = False,
|
|
857
|
+
run_configuration: nextmv.RunConfiguration = None,
|
|
848
858
|
polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
|
|
849
859
|
):
|
|
850
860
|
"""
|
|
@@ -871,14 +881,13 @@ def app(
|
|
|
871
881
|
options : dict[str, Any]
|
|
872
882
|
The options to pass to the application. This is a dictionary of
|
|
873
883
|
parameter names and values. The values must be JSON serializable.
|
|
874
|
-
input_type : InputType
|
|
875
|
-
The type of input to pass to the application. This can be either
|
|
876
|
-
JSON or FILES. Default is JSON.
|
|
877
884
|
full_result : bool
|
|
878
885
|
Whether to return the full result of the application run. If this is
|
|
879
886
|
set to `True`, the full result (with metadata) will be returned. If
|
|
880
887
|
this is set to `False`, only the output of the application will be
|
|
881
888
|
returned.
|
|
889
|
+
run_configuration : nextmv.RunConfiguration
|
|
890
|
+
The configuration to apply when running the app.
|
|
882
891
|
polling_options : Optional[cloud.PollingOptions]
|
|
883
892
|
Options for polling for the results of the app run. This is used to
|
|
884
893
|
configure the polling behavior, such as the timeout and backoff
|
|
@@ -924,6 +933,12 @@ def app(
|
|
|
924
933
|
```
|
|
925
934
|
"""
|
|
926
935
|
|
|
936
|
+
if input_type:
|
|
937
|
+
deprecated(
|
|
938
|
+
"input_type",
|
|
939
|
+
"The 'input_type' argument is deprecated and will be removed in a future release.",
|
|
940
|
+
)
|
|
941
|
+
|
|
927
942
|
# Make sure only one of options or parameters is used.
|
|
928
943
|
if parameters and options:
|
|
929
944
|
raise ValueError("You can only use either 'parameters' or 'options', not both.")
|
|
@@ -950,6 +965,7 @@ def app(
|
|
|
950
965
|
options=converted_options,
|
|
951
966
|
input_type=input_type,
|
|
952
967
|
full_result=full_result,
|
|
968
|
+
run_configuration=run_configuration,
|
|
953
969
|
polling_options=polling_options,
|
|
954
970
|
)
|
|
955
971
|
wrapper.step.type = StepType.APP
|
|
@@ -27,14 +27,16 @@ import base64
|
|
|
27
27
|
import copy
|
|
28
28
|
import inspect
|
|
29
29
|
import io
|
|
30
|
+
import os
|
|
30
31
|
import random
|
|
32
|
+
import tempfile
|
|
31
33
|
import threading
|
|
32
34
|
import time
|
|
33
35
|
from importlib.metadata import version
|
|
34
36
|
from itertools import product
|
|
35
37
|
from typing import Any, Optional, Union
|
|
36
38
|
|
|
37
|
-
from nextmv.cloud import Application, Client
|
|
39
|
+
from nextmv.cloud import Application, Client, RunResult
|
|
38
40
|
|
|
39
41
|
from . import config, decorators, graph, schema, threads, uplink, utils
|
|
40
42
|
from .__about__ import __version__
|
|
@@ -907,6 +909,7 @@ class Runner:
|
|
|
907
909
|
# Run the step
|
|
908
910
|
if node.parent.definition.is_app():
|
|
909
911
|
app_step: decorators.App = node.parent.definition.app
|
|
912
|
+
|
|
910
913
|
# Prepare the input for the app
|
|
911
914
|
# TODO: We only support one predecessor for app steps for now. This may
|
|
912
915
|
# change in the future. We may want to support multiple predecessors for
|
|
@@ -923,12 +926,23 @@ class Runner:
|
|
|
923
926
|
# Merge the options from the app decorator with the options from the
|
|
924
927
|
# AppRunConfig. AppRunConfig options take precedence.
|
|
925
928
|
options = app_step.options | app_run_options
|
|
929
|
+
elif isinstance(inputs[0], RunResult):
|
|
930
|
+
# If the input is a RunResult, we use its output as input.
|
|
931
|
+
run_result: RunResult = inputs[0]
|
|
932
|
+
input = run_result.output
|
|
933
|
+
options = app_step.options
|
|
934
|
+
name = node.id
|
|
926
935
|
else:
|
|
927
936
|
# If the input is not AppRunConfig, we use it directly.
|
|
928
937
|
input = inputs[0]
|
|
929
938
|
options = app_step.options
|
|
930
939
|
name = node.id
|
|
931
940
|
|
|
941
|
+
# Detect dir mode / multi-file direct input
|
|
942
|
+
is_dir_mode = False
|
|
943
|
+
if isinstance(input, str) and os.path.isdir(input):
|
|
944
|
+
is_dir_mode = True
|
|
945
|
+
|
|
932
946
|
# Modify the polling options set for the step (by default or by the
|
|
933
947
|
# user) so that the initial delay is randomized and the stopping
|
|
934
948
|
# callback is configured as the node being cancelled if the user
|
|
@@ -942,13 +956,22 @@ class Runner:
|
|
|
942
956
|
run_args = (
|
|
943
957
|
[], # No nameless arguments
|
|
944
958
|
{ # We use the named arguments to pass the user arguments to the run function
|
|
945
|
-
"
|
|
946
|
-
"run_options": options,
|
|
947
|
-
"polling_options": polling_options,
|
|
959
|
+
"options": options,
|
|
948
960
|
"name": name,
|
|
949
961
|
},
|
|
950
962
|
)
|
|
951
963
|
|
|
964
|
+
# Prepare input argument. We need to use 'input_dir_path' when dealing with a
|
|
965
|
+
# directory input (e.g., multi-file input).
|
|
966
|
+
if is_dir_mode:
|
|
967
|
+
run_args[1]["input_dir_path"] = input
|
|
968
|
+
else:
|
|
969
|
+
run_args[1]["input"] = input
|
|
970
|
+
|
|
971
|
+
# Apply run configuration if given.
|
|
972
|
+
if app_step.run_configuration is not None:
|
|
973
|
+
run_args[1]["configuration"] = app_step.run_configuration
|
|
974
|
+
|
|
952
975
|
# Prepare the application itself.
|
|
953
976
|
app = Application(
|
|
954
977
|
client=client,
|
|
@@ -957,18 +980,35 @@ class Runner:
|
|
|
957
980
|
if app_step.instance_id is not None and app_step.instance_id != "":
|
|
958
981
|
app.default_instance_id = app_step.instance_id
|
|
959
982
|
|
|
960
|
-
#
|
|
961
|
-
|
|
962
|
-
*run_args[0],
|
|
963
|
-
**run_args[1],
|
|
964
|
-
)
|
|
965
|
-
run_id = result.id
|
|
966
|
-
node.run_id = run_id
|
|
967
|
-
utils.log_internal(f"Finished app step {node.id} run, find it at {result.console_url}")
|
|
983
|
+
# We always supply an output directory path in case of implicit multi-file output
|
|
984
|
+
temp_dir = tempfile.mkdtemp(prefix="nextpipe_output_")
|
|
968
985
|
|
|
969
|
-
#
|
|
986
|
+
# Run the application
|
|
987
|
+
try:
|
|
988
|
+
run_id = app.new_run(*run_args[0], **run_args[1])
|
|
989
|
+
console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
|
|
990
|
+
utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
|
|
991
|
+
result = app.run_result_with_polling(
|
|
992
|
+
run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
|
|
993
|
+
)
|
|
994
|
+
node.run_id = run_id
|
|
995
|
+
finally: # Make sure we clean up temp dir on failure too
|
|
996
|
+
# If the temp dir is empty, remove it
|
|
997
|
+
dir_result = False
|
|
998
|
+
if not os.listdir(temp_dir):
|
|
999
|
+
os.rmdir(temp_dir)
|
|
1000
|
+
else:
|
|
1001
|
+
dir_result = True
|
|
1002
|
+
|
|
1003
|
+
# Return result
|
|
1004
|
+
# - Do not unwrap if full result is requested
|
|
1005
|
+
# - If the output came in a directory, return the directory path
|
|
970
1006
|
if app_step.full_result:
|
|
1007
|
+
if dir_result:
|
|
1008
|
+
result.output = temp_dir
|
|
971
1009
|
return result
|
|
1010
|
+
if dir_result:
|
|
1011
|
+
return temp_dir
|
|
972
1012
|
return result.output
|
|
973
1013
|
|
|
974
1014
|
else:
|
|
@@ -70,8 +70,10 @@ class AppRunConfig:
|
|
|
70
70
|
|
|
71
71
|
Parameters
|
|
72
72
|
----------
|
|
73
|
-
input : dict[str, Any],
|
|
74
|
-
Input data for the app,
|
|
73
|
+
input : Union[dict[str, Any], str, Any]
|
|
74
|
+
Input data for the app. A JSON app can take a dictionary, multi-file apps can take
|
|
75
|
+
a directory path as a string. Other types will be passed to the underlying Python
|
|
76
|
+
SDK as-is (e.g., nextmv.Input).
|
|
75
77
|
options : Union[list[AppOption], dict[str, Any]], optional
|
|
76
78
|
Options for running the app, by default empty. These can be provided as a list of
|
|
77
79
|
`AppOption` instances, or, simply as a dictionary of key-value pairs.
|
|
@@ -88,8 +90,10 @@ class AppRunConfig:
|
|
|
88
90
|
... )
|
|
89
91
|
"""
|
|
90
92
|
|
|
91
|
-
input: dict[str, Any]
|
|
92
|
-
"""Input for the app.
|
|
93
|
+
input: Union[dict[str, Any], str, Any]
|
|
94
|
+
"""Input for the app. A JSON app can take a dictionary, multi-file apps can take a
|
|
95
|
+
directory path as a string. Other types will be passed to the underlying Python SDK
|
|
96
|
+
as-is (e.g., nextmv.Input)."""
|
|
93
97
|
options: Union[list[AppOption], dict[str, Any]] = field(default_factory=list)
|
|
94
98
|
"""Options for running the app."""
|
|
95
99
|
name: Optional[str] = None
|
|
@@ -12,6 +12,11 @@ cd apps/echo
|
|
|
12
12
|
nextmv app create -a echo -n "Echo" || true
|
|
13
13
|
nextmv app push -a echo
|
|
14
14
|
cd ../..
|
|
15
|
+
# Setup echo-multi app
|
|
16
|
+
cd apps/echo-multi
|
|
17
|
+
nextmv app create -a echo-multi -n "Echo Multi" || true
|
|
18
|
+
nextmv app push -a echo-multi
|
|
19
|
+
cd ../..
|
|
15
20
|
```
|
|
16
21
|
|
|
17
22
|
Furthermore, subscribe to the following marketplace apps and name them as follows:
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
outputs/
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# JSON echo
|
|
2
|
+
|
|
3
|
+
This is a sample app that reads input files and echoes them as output. This app
|
|
4
|
+
is meant to be used for the `multi-file` I/O format.
|
|
5
|
+
|
|
6
|
+
This app is used for testing.
|
|
7
|
+
|
|
8
|
+
## Usage
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
python main.py -threads 10 -duration 10
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Push to Nextmv
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
nextmv app create -a echo-multi -n "JSON echo multi-file" -d "This is a sample app that reads input files and echoes them as output."
|
|
18
|
+
nextmv app push -a echo-multi
|
|
19
|
+
```
|