nextpipe 0.3.5.dev0__tar.gz → 0.4.0.dev0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/PKG-INFO +2 -2
  2. nextpipe-0.4.0.dev0/docs/examples/multifile-workflow.md +186 -0
  3. nextpipe-0.4.0.dev0/docs/tutorials/echo-multi.md +154 -0
  4. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/mkdocs.yml +1 -0
  5. nextpipe-0.4.0.dev0/nextpipe/__about__.py +1 -0
  6. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/decorators.py +24 -8
  7. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/flow.py +53 -13
  8. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/schema.py +8 -4
  9. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/pyproject.toml +1 -1
  10. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/.gitignore +1 -0
  11. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/README.md +5 -0
  12. nextpipe-0.4.0.dev0/tests/apps/echo-multi/.gitignore +1 -0
  13. nextpipe-0.4.0.dev0/tests/apps/echo-multi/README.md +19 -0
  14. nextpipe-0.4.0.dev0/tests/apps/echo-multi/app.yaml +6 -0
  15. nextpipe-0.4.0.dev0/tests/apps/echo-multi/inputs/data.csv +3 -0
  16. nextpipe-0.4.0.dev0/tests/apps/echo-multi/inputs/input.xlsx +0 -0
  17. nextpipe-0.4.0.dev0/tests/apps/echo-multi/main.py +57 -0
  18. nextpipe-0.4.0.dev0/tests/apps/echo-multi/requirements.txt +1 -0
  19. nextpipe-0.4.0.dev0/tests/pipelines/appapp.json.golden +24 -0
  20. nextpipe-0.4.0.dev0/tests/pipelines/appapp.py +52 -0
  21. nextpipe-0.4.0.dev0/tests/pipelines/foreach.json +3 -0
  22. nextpipe-0.4.0.dev0/tests/pipelines/inputs/data.csv +3 -0
  23. nextpipe-0.4.0.dev0/tests/pipelines/inputs/input.xlsx +0 -0
  24. nextpipe-0.4.0.dev0/tests/pipelines/multifile.py +92 -0
  25. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_graph.py +6 -0
  26. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_integration.py +44 -0
  27. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_schema.py +6 -0
  28. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_threads.py +6 -0
  29. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_uplink.py +6 -0
  30. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/test_version.py +6 -0
  31. nextpipe-0.3.5.dev0/nextpipe/__about__.py +0 -1
  32. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/lint.yml +0 -0
  33. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/release.yml +0 -0
  34. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.github/workflows/test.yml +0 -0
  35. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.gitignore +0 -0
  36. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.markdownlint.jsonc +0 -0
  37. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.nextmv/bump_requirements.sh +0 -0
  38. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.prettierrc.yml +0 -0
  39. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/.readthedocs.yaml +0 -0
  40. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/LICENSE.md +0 -0
  41. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/README.md +0 -0
  42. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/RELEASE.md +0 -0
  43. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/basic-chained-workflow.md +0 -0
  44. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/complex-workflow-csv.md +0 -0
  45. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/complex-workflow.md +0 -0
  46. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/ensemble-workflow.md +0 -0
  47. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/examples/fanout-workflow.md +0 -0
  48. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/logo-180.png +0 -0
  49. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/nextmv-favicon.svg +0 -0
  50. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/routing-ortools-pyvroom-selected.png +0 -0
  51. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/images/routing-selected.png +0 -0
  52. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/index.md +0 -0
  53. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/config.md +0 -0
  54. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/decorators.md +0 -0
  55. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/flow.md +0 -0
  56. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/graph.md +0 -0
  57. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/schema.md +0 -0
  58. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/threads.md +0 -0
  59. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/uplink.md +0 -0
  60. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/reference/utils.md +0 -0
  61. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/requirements.txt +0 -0
  62. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/tutorials/echo.md +0 -0
  63. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/docs/tutorials/getting-started.md +0 -0
  64. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/__init__.py +0 -0
  65. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/config.py +0 -0
  66. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/graph.py +0 -0
  67. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/threads.py +0 -0
  68. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/uplink.py +0 -0
  69. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe/utils.py +0 -0
  70. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/nextpipe.code-workspace +0 -0
  71. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/__init__.py +0 -0
  72. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/.gitignore +0 -0
  73. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/README.md +0 -0
  74. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/app.yaml +0 -0
  75. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/main.py +0 -0
  76. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/apps/echo/requirements.txt +0 -0
  77. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/app.yaml +0 -0
  78. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/main.py +0 -0
  79. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/deploy/requirements.txt +0 -0
  80. /nextpipe-0.3.5.dev0/tests/pipelines/chain.json → /nextpipe-0.4.0.dev0/tests/pipelines/appapp.json +0 -0
  81. /nextpipe-0.3.5.dev0/tests/pipelines/foreach-2-pred.json → /nextpipe-0.4.0.dev0/tests/pipelines/chain.json +0 -0
  82. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/chain.json.golden +0 -0
  83. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/chain.py +0 -0
  84. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.json +0 -0
  85. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.json.golden +0 -0
  86. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/complex.py +0 -0
  87. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/fail.py +0 -0
  88. /nextpipe-0.3.5.dev0/tests/pipelines/foreach.json → /nextpipe-0.4.0.dev0/tests/pipelines/foreach-2-pred.json +0 -0
  89. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach-2-pred.json.golden +0 -0
  90. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach-2-pred.py +0 -0
  91. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach.json.golden +0 -0
  92. {nextpipe-0.3.5.dev0 → nextpipe-0.4.0.dev0}/tests/pipelines/foreach.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: nextpipe
3
- Version: 0.3.5.dev0
3
+ Version: 0.4.0.dev0
4
4
  Summary: Framework for Decision Pipeline modeling and execution
5
5
  Project-URL: Homepage, https://www.nextmv.io
6
6
  Project-URL: Documentation, https://nextpipe.docs.nextmv.io/en/latest/
@@ -108,7 +108,7 @@ Requires-Dist: dataclasses-json>=0.6.7
108
108
  Requires-Dist: nextmv>=0.29.4
109
109
  Requires-Dist: requests>=2.31.0
110
110
  Provides-Extra: dev
111
- Requires-Dist: goldie>=0.1.7; extra == 'dev'
111
+ Requires-Dist: goldie>=0.1.8; extra == 'dev'
112
112
  Requires-Dist: ruff>=0.11.6; extra == 'dev'
113
113
  Description-Content-Type: text/markdown
114
114
 
@@ -0,0 +1,186 @@
1
+ # multi-file Workflow Example
2
+
3
+ !!! tip
4
+
5
+ This example uses the [`echo` app](../tutorials/echo.md), make sure to complete
6
+ that tutorial first.
7
+
8
+ This example showcases how to use **multi-file** applications _within_ a
9
+ Nextpipe workflow. A multi-file application differs from a JSON-based
10
+ application in that it accepts a directory of files as input and produces a
11
+ directory of files as output. Note that the workflow itself is also a multi-file
12
+ application, however, this is a user choice (i.e., you could also create a
13
+ JSON-based workflow that uses multi-file sub-applications).
14
+
15
+ For demonstration purposes, we will use the simple [echo-multi application] as
16
+ the sub-application, which echoes the input files as output files.
17
+
18
+ Find the workflow code below (mind the comments explaining each step):
19
+
20
+ ```python
21
+ import os
22
+ import shutil
23
+
24
+ import nextmv
25
+ import nextmv.cloud
26
+
27
+ from nextpipe import FlowSpec, app, log, needs, step
28
+
29
+ options = nextmv.Options(
30
+ nextmv.Option("input", str, "inputs/", "Path to input file.", False),
31
+ nextmv.Option("output", str, "outputs/", "Path to output file.", False),
32
+ )
33
+
34
+
35
+ # >>> Workflow definition
36
+ class Flow(FlowSpec):
37
+ # The first step receives the path to the input files directly (see main()) and
38
+ # automatically zips the directory and passes it to the 'echo-multi' sub-app.
39
+ @app(app_id="echo-multi")
40
+ @step
41
+ def solve1():
42
+ """Runs a multi-file model."""
43
+ pass
44
+
45
+ # The second step receives the path to the output files from the first step. This path
46
+ # will point to a temporary directory containing the output files from the first step.
47
+ @needs(predecessors=[solve1])
48
+ @step
49
+ def transform(result_path: str):
50
+ """Transforms the result for the next step."""
51
+ # Just list the content of the result directory.
52
+ log(f"Contents of result directory {result_path}:")
53
+ for file_name in os.listdir(result_path):
54
+ full_file_name = os.path.join(result_path, file_name)
55
+ if os.path.isfile(full_file_name):
56
+ log(f"- {file_name}")
57
+
58
+ # Add a new file to the result for demonstration purposes.
59
+ new_file_path = os.path.join(result_path, "additional_file.txt")
60
+ with open(new_file_path, "w") as f:
61
+ f.write("This is an additional file added in the transform step.\n")
62
+ log(f"Added new file: {new_file_path}")
63
+
64
+ return result_path
65
+
66
+ # The third step receives the (modified) directory from the transform step and runs
67
+ # another multi-file app on it.
68
+ @app(
69
+ app_id="echo-multi",
70
+ # We specify the content type explicitly here. This is normally done via the app's
71
+ # manifest, but we can do it explicitly like this too.
72
+ run_configuration=nextmv.RunConfiguration(
73
+ format=nextmv.Format(
74
+ format_input=nextmv.FormatInput(input_type=nextmv.InputFormat.MULTI_FILE),
75
+ format_output=nextmv.FormatOutput(output_type=nextmv.OutputFormat.MULTI_FILE),
76
+ )
77
+ ),
78
+ full_result=True,
79
+ )
80
+ @needs(predecessors=[transform])
81
+ @step
82
+ def solve2(result: nextmv.cloud.RunResult):
83
+ """Runs another multi-file model."""
84
+ pass
85
+
86
+ # The final step receives the output from 'solve2' as a full result object (see
87
+ # 'full_result=True' above). In this case, the path to the output files is available
88
+ # via 'result.output'.
89
+ @needs(predecessors=[solve2])
90
+ @step
91
+ def prepare_output(result: nextmv.cloud.RunResult):
92
+ """Transforms the result for the next step."""
93
+ # Extract the path to the output files.
94
+ result_path = result.output
95
+ # Simply copy the files from the given directory to the expected output directory.
96
+ os.makedirs(options.output, exist_ok=True)
97
+ for file_name in os.listdir(result_path):
98
+ full_file_name = os.path.join(result_path, file_name)
99
+ if os.path.isfile(full_file_name):
100
+ shutil.copy(full_file_name, options.output)
101
+
102
+
103
+ def main():
104
+ # Run workflow (simply provide the path to the multi-file input)
105
+ flow = Flow("DecisionFlow", options.input)
106
+ flow.run()
107
+ # The last step of the flow already prepares the output in the requested directory,
108
+ # so no need to do anything here anymore.
109
+
110
+
111
+ if __name__ == "__main__":
112
+ main()
113
+ ```
114
+
115
+ Run the example:
116
+
117
+ ```bash
118
+ $ python main.py
119
+ [nextpipe] No application ID or run ID found, uplink is inactive.
120
+ [nextpipe] Flow: Flow
121
+ [nextpipe] nextpipe: v0.3.5
122
+ [nextpipe] nextmv: 0.33.0
123
+ [nextpipe] Flow graph steps:
124
+ [nextpipe] Step:
125
+ [nextpipe] Definition: Step(solve1, StepRun(echo-multi, , {}, False))
126
+ [nextpipe] Docstring: Runs a multi-file model.
127
+ [nextpipe] Step:
128
+ [nextpipe] Definition: Step(transform, StepNeeds(solve1))
129
+ [nextpipe] Docstring: Transforms the result for the next step.
130
+ [nextpipe] Step:
131
+ [nextpipe] Definition: Step(solve2, StepNeeds(transform), StepRun(echo-multi, , {}, True))
132
+ [nextpipe] Docstring: Runs another multi-file model.
133
+ [nextpipe] Step:
134
+ [nextpipe] Definition: Step(prepare_output, StepNeeds(solve2))
135
+ [nextpipe] Docstring: Transforms the result for the next step.
136
+ [nextpipe] Mermaid diagram:
137
+ [nextpipe] graph LR
138
+ solve1(solve1)
139
+ solve1 --> transform
140
+ transform(transform)
141
+ transform --> solve2
142
+ solve2(solve2)
143
+ solve2 --> prepare_output
144
+ prepare_output(prepare_output)
145
+
146
+ [nextpipe] Mermaid URL: https://mermaid.ink/svg/Z3JhcGggTFIKICBzb2x2ZTEoc29sdmUxKQogIHNvbHZlMSAtLT4gdHJhbnNmb3JtCiAgdHJhbnNmb3JtKHRyYW5zZm9ybSkKICB0cmFuc2Zvcm0gLS0+IHNvbHZlMgogIHNvbHZlMihzb2x2ZTIpCiAgc29sdmUyIC0tPiBwcmVwYXJlX291dHB1dAogIHByZXBhcmVfb3V0cHV0KHByZXBhcmVfb3V0cHV0KQo=?theme=dark
147
+ [nextpipe] Running node solve1_0
148
+ [nextpipe] Started app step solve1_0 run, find it at https://cloud.nextmv.io/app/echo-multi/run/latest-a-JAvuFgDR?view=details
149
+ /home/marius/.asdf/installs/python/3.13.7/lib/python3.13/shutil.py:1281: DeprecationWarning: Python 3.14 will, by default, filter extracted tar archives and reject files or modify their metadata. Use the filter argument to control this behavior.
150
+ tarobj.extractall(extract_dir, filter=filter)
151
+ [nextpipe] Running node transform_0
152
+ [transform_0] Contents of result directory /tmp/nextpipe_output_igqsibzm:
153
+ [transform_0] - input.xlsx
154
+ [transform_0] - data.csv
155
+ [transform_0] Added new file: /tmp/nextpipe_output_igqsibzm/additional_file.txt
156
+ [nextpipe] Running node solve2_0
157
+ [nextpipe] Started app step solve2_0 run, find it at https://cloud.nextmv.io/app/echo-multi/run/latest-HIwvuFgDg?view=details
158
+ [nextpipe] Running node prepare_output_0
159
+ ```
160
+
161
+ Content of the output directory:
162
+
163
+ ```bash
164
+ tree outputs/
165
+ outputs/
166
+ ├── additional_file.txt
167
+ ├── data.csv
168
+ └── input.xlsx
169
+
170
+ 1 directory, 3 files
171
+ ```
172
+
173
+ The resulting Mermaid diagram for this flow looks like this:
174
+
175
+ ```mermaid
176
+ graph LR
177
+ solve1(solve1)
178
+ solve1 --> transform
179
+ transform(transform)
180
+ transform --> solve2
181
+ solve2(solve2)
182
+ solve2 --> prepare_output
183
+ prepare_output(prepare_output)
184
+ ```
185
+
186
+ [echo-multi application]: ../tutorials/echo-multi.md
@@ -0,0 +1,154 @@
1
+ # The `echo-multi` app
2
+
3
+ Several examples assume you have a Nextmv application called `echo-multi`. This
4
+ is just a simple application created for demonstration purposes. It takes the
5
+ input files and echoes them as output files.
6
+
7
+ Let's get set up with the `echo-multi` application. Before starting:
8
+
9
+ 1. [Sign up][signup] for a Nextmv account.
10
+ 2. Get your API key. Go to [Team > API Key][api-key].
11
+
12
+ Make sure that you have your API key set as an environment variable:
13
+
14
+ ```bash
15
+ export NEXTMV_API_KEY="<YOUR-API-KEY>"
16
+ ```
17
+
18
+ Now that you have a valid Nextmv account and API key, let's create the
19
+ `echo-multi` Nextmv app (start in an empty directory).
20
+
21
+ 1. Create a folder `inputs/` and add some sample input files to it. For example,
22
+ you can create two text files `input.csv` and `input.txt` with some sample
23
+ content.
24
+ 1. In a new directory, create a file called `main.py` with the code for the
25
+ basic app that echoes the input.
26
+
27
+ ```python
28
+ import glob
29
+
30
+ import os
31
+ import time
32
+
33
+ import nextmv
34
+
35
+ def main():
36
+ options = nextmv.Options(
37
+ nextmv.Option("input", str, "inputs/", "Path to input file.", False),
38
+ nextmv.Option("output", str, "outputs/", "Path to output file.", False),
39
+ nextmv.Option("duration", float, 1.0, "Runtime duration (in seconds).", False),
40
+ )
41
+
42
+ # Read and prepare the input data.
43
+ input_data = read_input(options.input)
44
+
45
+ # Log information about the input files.
46
+ nextmv.log(f"Size of input files (count: {len(input_data)}):")
47
+ for file_path, content in input_data.items():
48
+ nextmv.log(f" {file_path}: {len(content)} bytes")
49
+
50
+ # Sleep for the specified duration.
51
+ nextmv.log(f"Sleeping for {options.duration} seconds...")
52
+ time.sleep(options.duration)
53
+ nextmv.log("Woke up from sleep.")
54
+
55
+ # Write the output.
56
+ write_output(options.output, input_data)
57
+
58
+ def read_input(input_path: str) -> dict[str, bytes]:
59
+ """Reads the input files to memory."""
60
+ input_files = glob.glob(os.path.join(input_path, "**/*"), recursive=True)
61
+ content = {}
62
+ for file_path in input_files:
63
+ if os.path.isfile(file_path):
64
+ with open(file_path, "rb") as file:
65
+ nextmv.log(f"Reading file: {file_path}")
66
+ content[file_path] = file.read()
67
+ return content
68
+
69
+ def write_output(output_path: str, content: dict[str, bytes]) -> None:
70
+ """Writes the given output files."""
71
+ if not os.path.exists(output_path):
72
+ os.makedirs(output_path)
73
+
74
+ for file_path, data in content.items():
75
+ output_file_path = os.path.join(output_path, os.path.basename(file_path))
76
+ with open(output_file_path, "wb") as file:
77
+ nextmv.log(f"Writing file: {output_file_path}")
78
+ file.write(data)
79
+
80
+ if __name__ == "__main__":
81
+ main()
82
+ ```
83
+
84
+ Note that the application uses the [`nextmv`][nextmv-docs] library. This
85
+ library is a dependency of `nextpipe` and should be installed automatically
86
+ when you install `nextpipe`.
87
+
88
+ You may run the app locally to test it:
89
+
90
+ ```bash
91
+ python main.py
92
+ ```
93
+
94
+ 1. Create a `requirements.txt` file with the following requirements for running
95
+ the app:
96
+
97
+ ```requirements.txt
98
+ nextmv
99
+ ```
100
+
101
+ 1. Create an `app.yaml` file (the app manifest) with the following instructions:
102
+
103
+ ```yaml
104
+ type: python
105
+ runtime: ghcr.io/nextmv-io/runtime/python:3.11
106
+ files:
107
+ - main.py
108
+ python:
109
+ pip-requirements: requirements.txt
110
+ ```
111
+
112
+ 1. Push the application to your Nextmv account. Create a `push.py` script in
113
+ the same directory with the following code:
114
+
115
+ ```python
116
+ import os
117
+
118
+ from nextmv import cloud
119
+
120
+ client = cloud.Client(api_key=os.getenv("NEXTMV_API_KEY"))
121
+ app = cloud.Application.new(client=client, name="echo-multi", id="echo-multi", description="Sample echo multi-file app.", exist_ok=True)
122
+ app.push(verbose=True)
123
+ ```
124
+
125
+ 1. Execute the `push.py` script to push the app to your Nextmv account:
126
+
127
+ ```bash
128
+ $ python push.py
129
+ 💽 Starting build for Nextmv application.
130
+ 🐍 Bundling Python dependencies.
131
+ 📋 Copied files listed in "app.yaml" manifest.
132
+ 📦 Packaged application (588 files, 5.39 MiB).
133
+ 🌟 Pushing to application: "echo-multi".
134
+ 💥️ Successfully pushed to application: "echo-multi".
135
+ {
136
+ "app_id": "echo-multi",
137
+ "endpoint": "https://api.cloud.nextmv.io",
138
+ "instance_url": "v1/applications/echo-multi/runs?instance_id=devint"
139
+ }
140
+ ```
141
+
142
+ Alternatively, you can use the [Nextmv CLI][nextmv-cli] to create and push the app:
143
+
144
+ ```bash
145
+ nextmv app create -a echo-multi -n echo-multi -d "Sample echo multi-file app."
146
+ nextmv app push -a echo-multi
147
+ ```
148
+
149
+ Now you are ready to run the examples.
150
+
151
+ [signup]: https://cloud.nextmv.io
152
+ [api-key]: https://cloud.nextmv.io/team/api-keys
153
+ [nextmv-docs]: https://nextmv-py.readthedocs.io/en/latest/nextmv/
154
+ [nextmv-cli]: https://docs.nextmv.io/docs/using-nextmv/reference/cli
@@ -12,6 +12,7 @@ nav:
12
12
  - The echo app: tutorials/echo.md
13
13
  - Examples:
14
14
  - Basic chained workflow: examples/basic-chained-workflow.md
15
+ - Multi-file workflow: examples/multifile-workflow.md
15
16
  - Fanout workflow: examples/fanout-workflow.md
16
17
  - Ensemble workflow: examples/ensemble-workflow.md
17
18
  - Complex workflow: examples/complex-workflow.md
@@ -0,0 +1 @@
1
+ __version__ = "v0.4.0.dev.0"
@@ -50,6 +50,7 @@ from collections.abc import Callable
50
50
  from enum import Enum
51
51
  from functools import wraps
52
52
 
53
+ import nextmv
53
54
  from nextmv import cloud
54
55
  from nextmv.deprecated import deprecated
55
56
 
@@ -772,10 +773,10 @@ class App:
772
773
  The ID of the instance to run.
773
774
  options : dict[str, Any]
774
775
  The options to pass to the application.
775
- input_type : InputType
776
- The type of input to pass to the application (JSON or FILES).
777
776
  full_result : bool
778
777
  Whether to return the full result including metadata.
778
+ run_configuration : nextmv.RunConfiguration
779
+ The configuration to apply when running the app.
779
780
  polling_options : Optional[cloud.PollingOptions]
780
781
  Options for polling for the results of the app run.
781
782
  """
@@ -788,6 +789,7 @@ class App:
788
789
  parameters: dict[str, typing.Any] = None,
789
790
  options: dict[str, typing.Any] = None,
790
791
  full_result: bool = False,
792
+ run_configuration: nextmv.RunConfiguration = None,
791
793
  polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
792
794
  ):
793
795
  """
@@ -799,16 +801,22 @@ class App:
799
801
  The ID of the Nextmv Application to run.
800
802
  instance_id : str, optional
801
803
  The ID of the instance to run. Default is defined by the app on Platform.
802
- input_type : InputType, optional
803
- The type of input to pass to the application, by default InputType.JSON.
804
804
  options : dict[str, Any], optional
805
805
  The options to pass to the application, by default None.
806
806
  full_result : bool, optional
807
807
  Whether to return the full result including metadata, by default False.
808
+ run_configuration : nextmv.RunConfiguration, optional
809
+ The configuration to apply when running the app, by default None.
808
810
  polling_options : Optional[cloud.PollingOptions], optional
809
811
  Options for polling for the results of the app run, by default _DEFAULT_POLLING_OPTIONS.
810
812
  """
811
813
 
814
+ if input_type:
815
+ deprecated(
816
+ "input_type",
817
+ "The 'input_type' argument is deprecated and will be removed in a future release.",
818
+ )
819
+
812
820
  # Make sure only one of options or parameters is used.
813
821
  if parameters and options:
814
822
  raise ValueError("You can only use either 'parameters' or 'options', not both.")
@@ -824,6 +832,7 @@ class App:
824
832
  self.options = options if options else {}
825
833
  self.input_type = input_type
826
834
  self.full_result = full_result
835
+ self.run_configuration = run_configuration
827
836
  self.polling_options = polling_options
828
837
 
829
838
  def __repr__(self):
@@ -835,7 +844,7 @@ class App:
835
844
  A string representation of the app.
836
845
  """
837
846
 
838
- return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.input_type}, {self.full_result})"
847
+ return f"StepRun({self.app_id}, {self.instance_id}, {self.options}, {self.full_result})"
839
848
 
840
849
 
841
850
  def app(
@@ -845,6 +854,7 @@ def app(
845
854
  options: dict[str, typing.Any] = None,
846
855
  input_type: InputType = InputType.JSON,
847
856
  full_result: bool = False,
857
+ run_configuration: nextmv.RunConfiguration = None,
848
858
  polling_options: typing.Optional[cloud.PollingOptions] = _DEFAULT_POLLING_OPTIONS,
849
859
  ):
850
860
  """
@@ -871,14 +881,13 @@ def app(
871
881
  options : dict[str, Any]
872
882
  The options to pass to the application. This is a dictionary of
873
883
  parameter names and values. The values must be JSON serializable.
874
- input_type : InputType
875
- The type of input to pass to the application. This can be either
876
- JSON or FILES. Default is JSON.
877
884
  full_result : bool
878
885
  Whether to return the full result of the application run. If this is
879
886
  set to `True`, the full result (with metadata) will be returned. If
880
887
  this is set to `False`, only the output of the application will be
881
888
  returned.
889
+ run_configuration : nextmv.RunConfiguration
890
+ The configuration to apply when running the app.
882
891
  polling_options : Optional[cloud.PollingOptions]
883
892
  Options for polling for the results of the app run. This is used to
884
893
  configure the polling behavior, such as the timeout and backoff
@@ -924,6 +933,12 @@ def app(
924
933
  ```
925
934
  """
926
935
 
936
+ if input_type:
937
+ deprecated(
938
+ "input_type",
939
+ "The 'input_type' argument is deprecated and will be removed in a future release.",
940
+ )
941
+
927
942
  # Make sure only one of options or parameters is used.
928
943
  if parameters and options:
929
944
  raise ValueError("You can only use either 'parameters' or 'options', not both.")
@@ -950,6 +965,7 @@ def app(
950
965
  options=converted_options,
951
966
  input_type=input_type,
952
967
  full_result=full_result,
968
+ run_configuration=run_configuration,
953
969
  polling_options=polling_options,
954
970
  )
955
971
  wrapper.step.type = StepType.APP
@@ -27,14 +27,16 @@ import base64
27
27
  import copy
28
28
  import inspect
29
29
  import io
30
+ import os
30
31
  import random
32
+ import tempfile
31
33
  import threading
32
34
  import time
33
35
  from importlib.metadata import version
34
36
  from itertools import product
35
37
  from typing import Any, Optional, Union
36
38
 
37
- from nextmv.cloud import Application, Client
39
+ from nextmv.cloud import Application, Client, RunResult
38
40
 
39
41
  from . import config, decorators, graph, schema, threads, uplink, utils
40
42
  from .__about__ import __version__
@@ -907,6 +909,7 @@ class Runner:
907
909
  # Run the step
908
910
  if node.parent.definition.is_app():
909
911
  app_step: decorators.App = node.parent.definition.app
912
+
910
913
  # Prepare the input for the app
911
914
  # TODO: We only support one predecessor for app steps for now. This may
912
915
  # change in the future. We may want to support multiple predecessors for
@@ -923,12 +926,23 @@ class Runner:
923
926
  # Merge the options from the app decorator with the options from the
924
927
  # AppRunConfig. AppRunConfig options take precedence.
925
928
  options = app_step.options | app_run_options
929
+ elif isinstance(inputs[0], RunResult):
930
+ # If the input is a RunResult, we use its output as input.
931
+ run_result: RunResult = inputs[0]
932
+ input = run_result.output
933
+ options = app_step.options
934
+ name = node.id
926
935
  else:
927
936
  # If the input is not AppRunConfig, we use it directly.
928
937
  input = inputs[0]
929
938
  options = app_step.options
930
939
  name = node.id
931
940
 
941
+ # Detect dir mode / multi-file direct input
942
+ is_dir_mode = False
943
+ if isinstance(input, str) and os.path.isdir(input):
944
+ is_dir_mode = True
945
+
932
946
  # Modify the polling options set for the step (by default or by the
933
947
  # user) so that the initial delay is randomized and the stopping
934
948
  # callback is configured as the node being cancelled if the user
@@ -942,13 +956,22 @@ class Runner:
942
956
  run_args = (
943
957
  [], # No nameless arguments
944
958
  { # We use the named arguments to pass the user arguments to the run function
945
- "input": input,
946
- "run_options": options,
947
- "polling_options": polling_options,
959
+ "options": options,
948
960
  "name": name,
949
961
  },
950
962
  )
951
963
 
964
+ # Prepare input argument. We need to use 'input_dir_path' when dealing with a
965
+ # directory input (e.g., multi-file input).
966
+ if is_dir_mode:
967
+ run_args[1]["input_dir_path"] = input
968
+ else:
969
+ run_args[1]["input"] = input
970
+
971
+ # Apply run configuration if given.
972
+ if app_step.run_configuration is not None:
973
+ run_args[1]["configuration"] = app_step.run_configuration
974
+
952
975
  # Prepare the application itself.
953
976
  app = Application(
954
977
  client=client,
@@ -957,18 +980,35 @@ class Runner:
957
980
  if app_step.instance_id is not None and app_step.instance_id != "":
958
981
  app.default_instance_id = app_step.instance_id
959
982
 
960
- # Run the application
961
- result = app.new_run_with_result(
962
- *run_args[0],
963
- **run_args[1],
964
- )
965
- run_id = result.id
966
- node.run_id = run_id
967
- utils.log_internal(f"Finished app step {node.id} run, find it at {result.console_url}")
983
+ # We always supply an output directory path in case of implicit multi-file output
984
+ temp_dir = tempfile.mkdtemp(prefix="nextpipe_output_")
968
985
 
969
- # Return result (do not unwrap if full result is requested)
986
+ # Run the application
987
+ try:
988
+ run_id = app.new_run(*run_args[0], **run_args[1])
989
+ console_url = f"{client.console_url}/app/{app_step.app_id}/run/{run_id}?view=details"
990
+ utils.log_internal(f"Started app step {node.id} run, find it at {console_url}")
991
+ result = app.run_result_with_polling(
992
+ run_id=run_id, polling_options=polling_options, output_dir_path=temp_dir
993
+ )
994
+ node.run_id = run_id
995
+ finally: # Make sure we clean up temp dir on failure too
996
+ # If the temp dir is empty, remove it
997
+ dir_result = False
998
+ if not os.listdir(temp_dir):
999
+ os.rmdir(temp_dir)
1000
+ else:
1001
+ dir_result = True
1002
+
1003
+ # Return result
1004
+ # - Do not unwrap if full result is requested
1005
+ # - If the output came in a directory, return the directory path
970
1006
  if app_step.full_result:
1007
+ if dir_result:
1008
+ result.output = temp_dir
971
1009
  return result
1010
+ if dir_result:
1011
+ return temp_dir
972
1012
  return result.output
973
1013
 
974
1014
  else:
@@ -70,8 +70,10 @@ class AppRunConfig:
70
70
 
71
71
  Parameters
72
72
  ----------
73
- input : dict[str, Any], optional
74
- Input data for the app, by default None.
73
+ input : Union[dict[str, Any], str, Any]
74
+ Input data for the app. A JSON app can take a dictionary, multi-file apps can take
75
+ a directory path as a string. Other types will be passed to the underlying Python
76
+ SDK as-is (e.g., nextmv.Input).
75
77
  options : Union[list[AppOption], dict[str, Any]], optional
76
78
  Options for running the app, by default empty. These can be provided as a list of
77
79
  `AppOption` instances, or, simply as a dictionary of key-value pairs.
@@ -88,8 +90,10 @@ class AppRunConfig:
88
90
  ... )
89
91
  """
90
92
 
91
- input: dict[str, Any] = None
92
- """Input for the app."""
93
+ input: Union[dict[str, Any], str, Any]
94
+ """Input for the app. A JSON app can take a dictionary, multi-file apps can take a
95
+ directory path as a string. Other types will be passed to the underlying Python SDK
96
+ as-is (e.g., nextmv.Input)."""
93
97
  options: Union[list[AppOption], dict[str, Any]] = field(default_factory=list)
94
98
  """Options for running the app."""
95
99
  name: Optional[str] = None
@@ -71,5 +71,5 @@ path = "nextpipe/__about__.py"
71
71
  [project.optional-dependencies]
72
72
  dev = [
73
73
  "ruff>=0.11.6",
74
- "goldie>=0.1.7",
74
+ "goldie>=0.1.8",
75
75
  ]
@@ -1,3 +1,4 @@
1
1
  go-nextroute/
2
2
  python-pyvroom-routing/
3
3
  python-ortools-routing/
4
+ outputs/
@@ -12,6 +12,11 @@ cd apps/echo
12
12
  nextmv app create -a echo -n "Echo" || true
13
13
  nextmv app push -a echo
14
14
  cd ../..
15
+ # Setup echo-multi app
16
+ cd apps/echo-multi
17
+ nextmv app create -a echo-multi -n "Echo Multi" || true
18
+ nextmv app push -a echo-multi
19
+ cd ../..
15
20
  ```
16
21
 
17
22
  Furthermore, subscribe to the following marketplace apps and name them as follows:
@@ -0,0 +1 @@
1
+ outputs/
@@ -0,0 +1,19 @@
1
+ # JSON echo
2
+
3
+ This is a sample app that reads input files and echoes them as output. This app
4
+ is meant to be used for the `multi-file` I/O format.
5
+
6
+ This app is used for testing.
7
+
8
+ ## Usage
9
+
10
+ ```bash
11
+ python main.py -threads 10 -duration 10
12
+ ```
13
+
14
+ ## Push to Nextmv
15
+
16
+ ```bash
17
+ nextmv app create -a echo-multi -n "JSON echo multi-file" -d "This is a sample app that reads input files and echoes them as output."
18
+ nextmv app push -a echo-multi
19
+ ```