create-dagster 1.10.18rc0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. create_dagster-1.10.18rc0/LICENSE +201 -0
  2. create_dagster-1.10.18rc0/MANIFEST.in +3 -0
  3. create_dagster-1.10.18rc0/PKG-INFO +24 -0
  4. create_dagster-1.10.18rc0/create_dagster/__init__.py +5 -0
  5. create_dagster-1.10.18rc0/create_dagster/cli/__init__.py +33 -0
  6. create_dagster-1.10.18rc0/create_dagster/cli/scaffold.py +234 -0
  7. create_dagster-1.10.18rc0/create_dagster/py.typed +1 -0
  8. create_dagster-1.10.18rc0/create_dagster/scaffold.py +242 -0
  9. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/.gitignore.jinja +4 -0
  10. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja +23 -0
  11. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/__init__.py +0 -0
  12. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/components/__init__.py +0 -0
  13. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja +8 -0
  14. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/defs/.gitkeep +0 -0
  15. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/defs/__init__.py +0 -0
  16. create_dagster-1.10.18rc0/create_dagster/templates/PROJECT_NAME_PLACEHOLDER/tests/__init__.py +0 -0
  17. create_dagster-1.10.18rc0/create_dagster/templates/WORKSPACE_NAME_PLACEHOLDER/dg.toml.jinja +3 -0
  18. create_dagster-1.10.18rc0/create_dagster/templates/WORKSPACE_NAME_PLACEHOLDER/projects/.gitkeep +0 -0
  19. create_dagster-1.10.18rc0/create_dagster/version.py +1 -0
  20. create_dagster-1.10.18rc0/create_dagster.egg-info/PKG-INFO +24 -0
  21. create_dagster-1.10.18rc0/create_dagster.egg-info/SOURCES.txt +27 -0
  22. create_dagster-1.10.18rc0/create_dagster.egg-info/dependency_links.txt +1 -0
  23. create_dagster-1.10.18rc0/create_dagster.egg-info/entry_points.txt +2 -0
  24. create_dagster-1.10.18rc0/create_dagster.egg-info/not-zip-safe +1 -0
  25. create_dagster-1.10.18rc0/create_dagster.egg-info/requires.txt +1 -0
  26. create_dagster-1.10.18rc0/create_dagster.egg-info/top_level.txt +1 -0
  27. create_dagster-1.10.18rc0/setup.cfg +14 -0
  28. create_dagster-1.10.18rc0/setup.py +46 -0
@@ -0,0 +1,201 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
+
7
+ 1. Definitions.
8
+
9
+ "License" shall mean the terms and conditions for use, reproduction,
10
+ and distribution as defined by Sections 1 through 9 of this document.
11
+
12
+ "Licensor" shall mean the copyright owner or entity authorized by
13
+ the copyright owner that is granting the License.
14
+
15
+ "Legal Entity" shall mean the union of the acting entity and all
16
+ other entities that control, are controlled by, or are under common
17
+ control with that entity. For the purposes of this definition,
18
+ "control" means (i) the power, direct or indirect, to cause the
19
+ direction or management of such entity, whether by contract or
20
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
21
+ outstanding shares, or (iii) beneficial ownership of such entity.
22
+
23
+ "You" (or "Your") shall mean an individual or Legal Entity
24
+ exercising permissions granted by this License.
25
+
26
+ "Source" form shall mean the preferred form for making modifications,
27
+ including but not limited to software source code, documentation
28
+ source, and configuration files.
29
+
30
+ "Object" form shall mean any form resulting from mechanical
31
+ transformation or translation of a Source form, including but
32
+ not limited to compiled object code, generated documentation,
33
+ and conversions to other media types.
34
+
35
+ "Work" shall mean the work of authorship, whether in Source or
36
+ Object form, made available under the License, as indicated by a
37
+ copyright notice that is included in or attached to the work
38
+ (an example is provided in the Appendix below).
39
+
40
+ "Derivative Works" shall mean any work, whether in Source or Object
41
+ form, that is based on (or derived from) the Work and for which the
42
+ editorial revisions, annotations, elaborations, or other modifications
43
+ represent, as a whole, an original work of authorship. For the purposes
44
+ of this License, Derivative Works shall not include works that remain
45
+ separable from, or merely link (or bind by name) to the interfaces of,
46
+ the Work and Derivative Works thereof.
47
+
48
+ "Contribution" shall mean any work of authorship, including
49
+ the original version of the Work and any modifications or additions
50
+ to that Work or Derivative Works thereof, that is intentionally
51
+ submitted to Licensor for inclusion in the Work by the copyright owner
52
+ or by an individual or Legal Entity authorized to submit on behalf of
53
+ the copyright owner. For the purposes of this definition, "submitted"
54
+ means any form of electronic, verbal, or written communication sent
55
+ to the Licensor or its representatives, including but not limited to
56
+ communication on electronic mailing lists, source code control systems,
57
+ and issue tracking systems that are managed by, or on behalf of, the
58
+ Licensor for the purpose of discussing and improving the Work, but
59
+ excluding communication that is conspicuously marked or otherwise
60
+ designated in writing by the copyright owner as "Not a Contribution."
61
+
62
+ "Contributor" shall mean Licensor and any individual or Legal Entity
63
+ on behalf of whom a Contribution has been received by Licensor and
64
+ subsequently incorporated within the Work.
65
+
66
+ 2. Grant of Copyright License. Subject to the terms and conditions of
67
+ this License, each Contributor hereby grants to You a perpetual,
68
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69
+ copyright license to reproduce, prepare Derivative Works of,
70
+ publicly display, publicly perform, sublicense, and distribute the
71
+ Work and such Derivative Works in Source or Object form.
72
+
73
+ 3. Grant of Patent License. Subject to the terms and conditions of
74
+ this License, each Contributor hereby grants to You a perpetual,
75
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76
+ (except as stated in this section) patent license to make, have made,
77
+ use, offer to sell, sell, import, and otherwise transfer the Work,
78
+ where such license applies only to those patent claims licensable
79
+ by such Contributor that are necessarily infringed by their
80
+ Contribution(s) alone or by combination of their Contribution(s)
81
+ with the Work to which such Contribution(s) was submitted. If You
82
+ institute patent litigation against any entity (including a
83
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
84
+ or a Contribution incorporated within the Work constitutes direct
85
+ or contributory patent infringement, then any patent licenses
86
+ granted to You under this License for that Work shall terminate
87
+ as of the date such litigation is filed.
88
+
89
+ 4. Redistribution. You may reproduce and distribute copies of the
90
+ Work or Derivative Works thereof in any medium, with or without
91
+ modifications, and in Source or Object form, provided that You
92
+ meet the following conditions:
93
+
94
+ (a) You must give any other recipients of the Work or
95
+ Derivative Works a copy of this License; and
96
+
97
+ (b) You must cause any modified files to carry prominent notices
98
+ stating that You changed the files; and
99
+
100
+ (c) You must retain, in the Source form of any Derivative Works
101
+ that You distribute, all copyright, patent, trademark, and
102
+ attribution notices from the Source form of the Work,
103
+ excluding those notices that do not pertain to any part of
104
+ the Derivative Works; and
105
+
106
+ (d) If the Work includes a "NOTICE" text file as part of its
107
+ distribution, then any Derivative Works that You distribute must
108
+ include a readable copy of the attribution notices contained
109
+ within such NOTICE file, excluding those notices that do not
110
+ pertain to any part of the Derivative Works, in at least one
111
+ of the following places: within a NOTICE text file distributed
112
+ as part of the Derivative Works; within the Source form or
113
+ documentation, if provided along with the Derivative Works; or,
114
+ within a display generated by the Derivative Works, if and
115
+ wherever such third-party notices normally appear. The contents
116
+ of the NOTICE file are for informational purposes only and
117
+ do not modify the License. You may add Your own attribution
118
+ notices within Derivative Works that You distribute, alongside
119
+ or as an addendum to the NOTICE text from the Work, provided
120
+ that such additional attribution notices cannot be construed
121
+ as modifying the License.
122
+
123
+ You may add Your own copyright statement to Your modifications and
124
+ may provide additional or different license terms and conditions
125
+ for use, reproduction, or distribution of Your modifications, or
126
+ for any such Derivative Works as a whole, provided Your use,
127
+ reproduction, and distribution of the Work otherwise complies with
128
+ the conditions stated in this License.
129
+
130
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
131
+ any Contribution intentionally submitted for inclusion in the Work
132
+ by You to the Licensor shall be under the terms and conditions of
133
+ this License, without any additional terms or conditions.
134
+ Notwithstanding the above, nothing herein shall supersede or modify
135
+ the terms of any separate license agreement you may have executed
136
+ with Licensor regarding such Contributions.
137
+
138
+ 6. Trademarks. This License does not grant permission to use the trade
139
+ names, trademarks, service marks, or product names of the Licensor,
140
+ except as required for reasonable and customary use in describing the
141
+ origin of the Work and reproducing the content of the NOTICE file.
142
+
143
+ 7. Disclaimer of Warranty. Unless required by applicable law or
144
+ agreed to in writing, Licensor provides the Work (and each
145
+ Contributor provides its Contributions) on an "AS IS" BASIS,
146
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147
+ implied, including, without limitation, any warranties or conditions
148
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149
+ PARTICULAR PURPOSE. You are solely responsible for determining the
150
+ appropriateness of using or redistributing the Work and assume any
151
+ risks associated with Your exercise of permissions under this License.
152
+
153
+ 8. Limitation of Liability. In no event and under no legal theory,
154
+ whether in tort (including negligence), contract, or otherwise,
155
+ unless required by applicable law (such as deliberate and grossly
156
+ negligent acts) or agreed to in writing, shall any Contributor be
157
+ liable to You for damages, including any direct, indirect, special,
158
+ incidental, or consequential damages of any character arising as a
159
+ result of this License or out of the use or inability to use the
160
+ Work (including but not limited to damages for loss of goodwill,
161
+ work stoppage, computer failure or malfunction, or any and all
162
+ other commercial damages or losses), even if such Contributor
163
+ has been advised of the possibility of such damages.
164
+
165
+ 9. Accepting Warranty or Additional Liability. While redistributing
166
+ the Work or Derivative Works thereof, You may choose to offer,
167
+ and charge a fee for, acceptance of support, warranty, indemnity,
168
+ or other liability obligations and/or rights consistent with this
169
+ License. However, in accepting such obligations, You may act only
170
+ on Your own behalf and on Your sole responsibility, not on behalf
171
+ of any other Contributor, and only if You agree to indemnify,
172
+ defend, and hold each Contributor harmless for any liability
173
+ incurred by, or claims asserted against, such Contributor by reason
174
+ of your accepting any such warranty or additional liability.
175
+
176
+ END OF TERMS AND CONDITIONS
177
+
178
+ APPENDIX: How to apply the Apache License to your work.
179
+
180
+ To apply the Apache License to your work, attach the following
181
+ boilerplate notice, with the fields enclosed by brackets "{}"
182
+ replaced with your own identifying information. (Don't include
183
+ the brackets!) The text should be enclosed in the appropriate
184
+ comment syntax for the file format. We also recommend that a
185
+ file or class name and description of purpose be included on the
186
+ same "printed page" as the copyright notice for easier
187
+ identification within third-party archives.
188
+
189
+ Copyright 2023 Dagster Labs, Inc.
190
+
191
+ Licensed under the Apache License, Version 2.0 (the "License");
192
+ you may not use this file except in compliance with the License.
193
+ You may obtain a copy of the License at
194
+
195
+ http://www.apache.org/licenses/LICENSE-2.0
196
+
197
+ Unless required by applicable law or agreed to in writing, software
198
+ distributed under the License is distributed on an "AS IS" BASIS,
199
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200
+ See the License for the specific language governing permissions and
201
+ limitations under the License.
@@ -0,0 +1,3 @@
1
+ include LICENSE
2
+ include create_dagster/py.typed
3
+ recursive-include create_dagster/templates *
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.4
2
+ Name: create-dagster
3
+ Version: 1.10.18rc0
4
+ Summary: CLI for creating a new dg project or workspace.
5
+ Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/create-dagster
6
+ Author: Dagster Labs
7
+ Author-email: hello@dagsterlabs.com
8
+ License: Apache-2.0
9
+ Classifier: Programming Language :: Python :: 3.9
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ License-File: LICENSE
16
+ Requires-Dist: dagster-dg-core==1.10.18rc0
17
+ Dynamic: author
18
+ Dynamic: author-email
19
+ Dynamic: classifier
20
+ Dynamic: home-page
21
+ Dynamic: license
22
+ Dynamic: license-file
23
+ Dynamic: requires-dist
24
+ Dynamic: summary
@@ -0,0 +1,5 @@
1
+ from dagster_shared.libraries import DagsterLibraryRegistry
2
+
3
+ from create_dagster.version import __version__ as __version__
4
+
5
+ DagsterLibraryRegistry.register("create-dagster", __version__)
@@ -0,0 +1,33 @@
1
+ import click
2
+
3
+ from create_dagster.cli.scaffold import scaffold_project_command, scaffold_workspace_command
4
+ from create_dagster.version import __version__
5
+
6
+ CREATE_DAGSTER_CLI_MAX_OUTPUT_WIDTH = 120
7
+
8
+
9
+ def get_create_dagster_cli():
10
+ @click.group(
11
+ name="create-dagster",
12
+ commands={
13
+ "project": scaffold_project_command,
14
+ "workspace": scaffold_workspace_command,
15
+ },
16
+ context_settings={
17
+ "max_content_width": CREATE_DAGSTER_CLI_MAX_OUTPUT_WIDTH,
18
+ "help_option_names": ["-h", "--help"],
19
+ },
20
+ )
21
+ @click.version_option(__version__, "--version", "-v")
22
+ def group():
23
+ """CLI for creating a new Dagster project or workspace."""
24
+
25
+ return group
26
+
27
+
28
+ ENV_PREFIX = "CREATE_DAGSTER"
29
+ cli = get_create_dagster_cli()
30
+
31
+
32
+ def main():
33
+ cli(auto_envvar_prefix=ENV_PREFIX)
@@ -0,0 +1,234 @@
1
+ import subprocess
2
+ from pathlib import Path
3
+ from typing import Optional
4
+
5
+ import click
6
+ from dagster_dg_core.config import (
7
+ DgProjectPythonEnvironment,
8
+ DgProjectPythonEnvironmentFlag,
9
+ DgRawWorkspaceConfig,
10
+ DgWorkspaceScaffoldProjectOptions,
11
+ normalize_cli_config,
12
+ )
13
+ from dagster_dg_core.context import DgContext
14
+ from dagster_dg_core.shared_options import dg_editable_dagster_options, dg_global_options
15
+ from dagster_dg_core.utils import (
16
+ DgClickCommand,
17
+ exit_with_error,
18
+ format_multiline_str,
19
+ get_shortest_path_repr,
20
+ get_venv_activation_cmd,
21
+ is_uv_installed,
22
+ pushd,
23
+ )
24
+ from dagster_dg_core.utils.telemetry import cli_telemetry_wrapper
25
+ from typing_extensions import get_args
26
+
27
+ from create_dagster.scaffold import scaffold_project, scaffold_workspace
28
+
29
+ DEFAULT_WORKSPACE_NAME = "dagster-workspace"
30
+
31
+
32
+ def _print_package_install_warning_message() -> None:
33
+ pip_install_cmd = "pip install --editable ."
34
+ uv_install_cmd = "uv sync"
35
+ click.secho(
36
+ format_multiline_str(
37
+ f"""
38
+ The environment used for your project must include an installation of your project
39
+ package. Please run `{uv_install_cmd}` (for uv) or `{pip_install_cmd}` (for pip) before
40
+ running `dg` commands against your project.
41
+ """,
42
+ ),
43
+ fg="yellow",
44
+ )
45
+
46
+
47
+ def _should_run_uv_sync(
48
+ python_environment: DgProjectPythonEnvironmentFlag,
49
+ venv_path: Path,
50
+ uv_sync_flag: Optional[bool],
51
+ ) -> bool:
52
+ # This already will have occurred during the scaffolding step
53
+ if python_environment == "uv_managed" or uv_sync_flag is False:
54
+ return False
55
+ # This can force running `uv sync` even if a venv already exists
56
+ elif uv_sync_flag is True:
57
+ return True
58
+ elif venv_path.exists():
59
+ _print_package_install_warning_message()
60
+ return False
61
+ elif is_uv_installed(): # uv_sync_flag is unset (None)
62
+ response = click.prompt(
63
+ format_multiline_str("""
64
+ Run uv sync? This will create the virtual environment you need to activate in
65
+ order to work on this project. (y/n)
66
+ """),
67
+ default="y",
68
+ ).lower()
69
+ if response not in ("y", "n"):
70
+ exit_with_error(f"Invalid response '{response}'. Please enter 'y' or 'n'.")
71
+ if response == "n":
72
+ _print_package_install_warning_message()
73
+ return response == "y"
74
+ else:
75
+ return False
76
+
77
+
78
+ # ########################
79
+ # ##### PROJECT
80
+ # ########################
81
+
82
+
83
+ @click.command(
84
+ name="project",
85
+ cls=DgClickCommand,
86
+ context_settings={"help_option_names": ["-h", "--help"]},
87
+ )
88
+ @click.argument("path", type=Path)
89
+ @click.option(
90
+ "--python-environment",
91
+ default="active",
92
+ type=click.Choice(get_args(DgProjectPythonEnvironmentFlag)),
93
+ help="Type of Python environment in which to launch subprocesses for this project.",
94
+ )
95
+ @click.option(
96
+ "--uv-sync/--no-uv-sync",
97
+ is_flag=True,
98
+ default=None,
99
+ help="""
100
+ Preemptively answer the "Run uv sync?" prompt presented after project initialization.
101
+ """.strip(),
102
+ )
103
+ @dg_editable_dagster_options
104
+ @dg_global_options
105
+ @cli_telemetry_wrapper
106
+ def scaffold_project_command(
107
+ path: Path,
108
+ use_editable_dagster: Optional[str],
109
+ python_environment: DgProjectPythonEnvironmentFlag,
110
+ uv_sync: Optional[bool],
111
+ **global_options: object,
112
+ ) -> None:
113
+ """Scaffold a new Dagster project at PATH. The name of the project will be the final component of PATH.
114
+
115
+ This command can be run inside or outside of a workspace directory. If run inside a workspace,
116
+ the project will be added to the workspace's list of project specs.
117
+
118
+ "." may be passed as PATH to create the new project inside the existing working directory.
119
+
120
+ Examples:
121
+ dagster-create project PROJECT_NAME
122
+ Scaffold a new project in new directory PROJECT_NAME. Automatically creates directory
123
+ and parent directories.
124
+ dagster-create project .
125
+ Scaffold a new project in the CWD. The project name is taken from the last component of the CWD.
126
+
127
+ Created projects will have the following structure:
128
+
129
+ ├── src
130
+ │ └── <project_name>
131
+ │ ├── __init__.py
132
+ │ ├── definitions.py
133
+ │ ├── defs
134
+ │ │ └── __init__.py
135
+ │ └── lib
136
+ │ └── __init__.py
137
+ ├── tests
138
+ │ └── __init__.py
139
+ └── pyproject.toml
140
+
141
+ The `src.<project_name>.defs` directory holds Python objects that can be targeted by the
142
+ `dg scaffold` command or have dg-inspectable metadata. Custom component types in the project
143
+ live in `src.<project_name>.lib`. These types can be created with `dg scaffold component`.
144
+ """
145
+ cli_config = normalize_cli_config(global_options, click.get_current_context())
146
+ dg_context = DgContext.from_file_discovery_and_command_line_config(Path.cwd(), cli_config)
147
+
148
+ if uv_sync is True and not is_uv_installed():
149
+ exit_with_error("""
150
+ uv is not installed. Please install uv to use the `--uv-sync` option.
151
+ See https://docs.astral.sh/uv/getting-started/installation/.
152
+ """)
153
+ elif uv_sync is False and python_environment == "uv_managed":
154
+ exit_with_error(
155
+ "The `--uv-sync` option cannot be set to False when using the `--python-environment uv_managed` option."
156
+ )
157
+ elif python_environment == "uv_managed" and not is_uv_installed():
158
+ exit_with_error("""
159
+ uv is not installed. Please install uv to use the `--python-environment uv_managed` option.
160
+ See https://docs.astral.sh/uv/getting-started/installation/.
161
+ """)
162
+
163
+ abs_path = path.resolve()
164
+ if dg_context.is_workspace and dg_context.has_project(
165
+ abs_path.relative_to(dg_context.workspace_root_path)
166
+ ):
167
+ exit_with_error(f"The current workspace already specifies a project at {abs_path}.")
168
+ elif str(path) != "." and abs_path.exists():
169
+ exit_with_error(f"A file or directory already exists at {abs_path}.")
170
+
171
+ scaffold_project(
172
+ abs_path,
173
+ dg_context,
174
+ use_editable_dagster=use_editable_dagster,
175
+ python_environment=DgProjectPythonEnvironment.from_flag(python_environment),
176
+ )
177
+
178
+ venv_path = path / ".venv"
179
+ if _should_run_uv_sync(python_environment, venv_path, uv_sync):
180
+ click.echo("Running `uv sync --group dev`...")
181
+ with pushd(path):
182
+ subprocess.run(["uv", "sync", "--group", "dev"], check=True)
183
+
184
+ click.echo("\nuv.lock and virtual environment created.")
185
+ display_venv_path = get_shortest_path_repr(venv_path)
186
+ click.echo(
187
+ f"""
188
+ Run `{get_venv_activation_cmd(display_venv_path)}` to activate your project's virtual environment.
189
+ """.strip()
190
+ )
191
+
192
+
193
+ # ########################
194
+ # ##### WORKSPACE
195
+ # ########################
196
+
197
+
198
+ @click.command(
199
+ name="workspace",
200
+ cls=DgClickCommand,
201
+ context_settings={"help_option_names": ["-h", "--help"]},
202
+ )
203
+ @click.argument("name", type=str, default=DEFAULT_WORKSPACE_NAME)
204
+ @dg_editable_dagster_options
205
+ @dg_global_options
206
+ @cli_telemetry_wrapper
207
+ def scaffold_workspace_command(
208
+ name: str,
209
+ use_editable_dagster: Optional[str],
210
+ **global_options: object,
211
+ ):
212
+ """Initialize a new Dagster workspace.
213
+
214
+ Examples:
215
+ dagster-create workspace WORKSPACE_NAME
216
+ Scaffold a new workspace in new directory WORKSPACE_NAME. Automatically creates directory
217
+ and parent directories.
218
+ dagster-create workspace .
219
+ Scaffold a new workspace in the CWD. The workspace name is the last component of the CWD.
220
+
221
+ The scaffolded workspace folder has the following structure:
222
+
223
+ ├── <workspace_name>
224
+ │ ├── projects
225
+ | | └── <Dagster projects go here>
226
+ │ └── dg.toml
227
+
228
+ """
229
+ workspace_config = DgRawWorkspaceConfig(
230
+ scaffold_project_options=DgWorkspaceScaffoldProjectOptions.get_raw_from_cli(
231
+ use_editable_dagster,
232
+ )
233
+ )
234
+ scaffold_workspace(name, workspace_config)
@@ -0,0 +1 @@
1
+ partial
@@ -0,0 +1,242 @@
1
+ import os
2
+ from collections.abc import Sequence
3
+ from dataclasses import asdict
4
+ from pathlib import Path
5
+ from typing import Optional
6
+
7
+ import click
8
+ from dagster_dg_core.config import (
9
+ DgProjectPythonEnvironment,
10
+ DgRawWorkspaceConfig,
11
+ DgWorkspaceScaffoldProjectOptions,
12
+ discover_workspace_root,
13
+ modify_dg_toml_config,
14
+ )
15
+ from dagster_dg_core.context import DgContext
16
+ from dagster_dg_core.utils import exit_with_error, get_toml_node, has_toml_node, set_toml_node
17
+ from dagster_shared.libraries import parse_package_version
18
+ from dagster_shared.scaffold import scaffold_subtree
19
+
20
+
21
+ def scaffold_workspace(
22
+ dirname: str,
23
+ workspace_config: Optional[DgRawWorkspaceConfig] = None,
24
+ ) -> Path:
25
+ # Can't create a workspace that is a child of another workspace
26
+ import tomlkit
27
+ import tomlkit.items
28
+
29
+ new_workspace_path = Path.cwd() if dirname == "." else Path.cwd() / dirname
30
+ existing_workspace_path = discover_workspace_root(new_workspace_path)
31
+ if existing_workspace_path:
32
+ exit_with_error(
33
+ f"Workspace already exists at {existing_workspace_path}. Run `dg scaffold project` to add a new project to that workspace."
34
+ )
35
+ elif dirname != "." and new_workspace_path.exists():
36
+ exit_with_error(f"Folder already exists at {new_workspace_path}.")
37
+
38
+ scaffold_subtree(
39
+ path=new_workspace_path,
40
+ name_placeholder="WORKSPACE_NAME_PLACEHOLDER",
41
+ project_template_path=Path(
42
+ os.path.join(os.path.dirname(__file__), "templates", "WORKSPACE_NAME_PLACEHOLDER")
43
+ ),
44
+ project_name=dirname,
45
+ )
46
+
47
+ if workspace_config is not None:
48
+ with modify_dg_toml_config(new_workspace_path / "dg.toml") as toml:
49
+ for k, v in workspace_config.items():
50
+ # Ignore empty collections and None, but not False
51
+ if v != {} and v != [] and v is not None:
52
+ get_toml_node(toml, ("workspace",), (tomlkit.items.Table)).add(tomlkit.nl())
53
+ set_toml_node(toml, ("workspace", k), v)
54
+
55
+ click.echo(f"Scaffolded files for Dagster workspace at {new_workspace_path}.")
56
+ return new_workspace_path
57
+
58
+
59
+ def scaffold_project(
60
+ path: Path,
61
+ dg_context: DgContext,
62
+ use_editable_dagster: Optional[str],
63
+ python_environment: Optional[DgProjectPythonEnvironment] = None,
64
+ ) -> None:
65
+ import tomlkit
66
+ import tomlkit.items
67
+
68
+ click.echo(f"Creating a Dagster project at {path}.")
69
+
70
+ cli_options = DgWorkspaceScaffoldProjectOptions.get_raw_from_cli(use_editable_dagster)
71
+ workspace_options = (
72
+ dg_context.config.workspace.scaffold_project_options
73
+ if dg_context.config.workspace
74
+ else None
75
+ )
76
+
77
+ final_use_editable_dagster = cli_options.get(
78
+ "use_editable_dagster",
79
+ workspace_options.use_editable_dagster if workspace_options else None,
80
+ )
81
+
82
+ if final_use_editable_dagster:
83
+ editable_dagster_root = (
84
+ _get_editable_dagster_from_env()
85
+ if final_use_editable_dagster is True
86
+ else final_use_editable_dagster
87
+ )
88
+ deps = EDITABLE_DAGSTER_DEPENDENCIES
89
+ dev_deps = EDITABLE_DAGSTER_DEV_DEPENDENCIES
90
+ sources = _gather_dagster_packages(Path(editable_dagster_root))
91
+ else:
92
+ editable_dagster_root = None
93
+ deps = PYPI_DAGSTER_DEPENDENCIES
94
+ dev_deps = PYPI_DAGSTER_DEV_DEPENDENCIES
95
+
96
+ from create_dagster.version import __version__
97
+
98
+ create_dagster_version = parse_package_version(__version__)
99
+ if create_dagster_version.release[0] >= 1:
100
+ # Pin scaffolded libraries to match create-dagster version
101
+ deps = [f"{dep}=={__version__}" for dep in deps]
102
+ dev_deps = [f"{dep}=={__version__}" for dep in dev_deps]
103
+
104
+ sources = []
105
+
106
+ dependencies_str = _get_pyproject_toml_dependencies(deps)
107
+ dev_dependencies_str = _get_pyproject_toml_dev_dependencies(dev_deps)
108
+ uv_sources_str = _get_pyproject_toml_uv_sources(sources) if sources else ""
109
+
110
+ scaffold_subtree(
111
+ path=path,
112
+ name_placeholder="PROJECT_NAME_PLACEHOLDER",
113
+ project_template_path=Path(
114
+ os.path.join(os.path.dirname(__file__), "templates", "PROJECT_NAME_PLACEHOLDER")
115
+ ),
116
+ dependencies=dependencies_str,
117
+ dev_dependencies=dev_dependencies_str,
118
+ uv_sources=uv_sources_str,
119
+ )
120
+ click.echo(f"Scaffolded files for Dagster project at {path}.")
121
+
122
+ if python_environment:
123
+ with modify_dg_toml_config(dg_context.with_root_path(path).config_file_path) as toml:
124
+ python_environment_dict = asdict(python_environment)
125
+ used_key = next((k for k, v in python_environment_dict.items() if v), None)
126
+ if used_key:
127
+ set_toml_node(toml, ("project", "python_environment"), {})
128
+ set_toml_node(
129
+ toml,
130
+ ("project", "python_environment", used_key),
131
+ python_environment_dict[used_key],
132
+ )
133
+
134
+ # Build the venv
135
+ cl_dg_context = dg_context.with_root_path(path)
136
+ if cl_dg_context.use_dg_managed_environment:
137
+ cl_dg_context.ensure_uv_lock()
138
+
139
+ # Update pyproject.toml
140
+ if cl_dg_context.is_workspace:
141
+ entry = {
142
+ "path": str(cl_dg_context.root_path.relative_to(cl_dg_context.workspace_root_path)),
143
+ }
144
+
145
+ with modify_dg_toml_config(dg_context.config_file_path) as toml:
146
+ if not has_toml_node(toml, ("workspace", "projects")):
147
+ projects = tomlkit.aot()
148
+ set_toml_node(toml, ("workspace", "projects"), projects)
149
+ item = tomlkit.table()
150
+ else:
151
+ projects = get_toml_node(
152
+ toml,
153
+ ("workspace", "projects"),
154
+ (tomlkit.items.AoT, tomlkit.items.Array),
155
+ )
156
+ if isinstance(projects, tomlkit.items.Array):
157
+ item = tomlkit.inline_table()
158
+ else:
159
+ item = tomlkit.table()
160
+
161
+ # item.trivia.indent is the preceding whitespace-- this ensures there is always a blank
162
+ # line before the new entry
163
+ item.trivia.indent = item.trivia.indent + "\n"
164
+ for key, value in entry.items():
165
+ item[key] = value
166
+ projects.append(item)
167
+
168
+
169
+ # Despite the fact that editable dependencies are resolved through tool.uv.sources, we need to set
170
+ # the dependencies themselves differently depending on whether we are using editable dagster or
171
+ # not. This is because `tool.uv.sources` only seems to apply to direct dependencies of the package,
172
+ # so any 2+-order Dagster dependency of our package needs to be listed as a direct dependency in the
173
+ # editable case. See: https://github.com/astral-sh/uv/issues/9446
174
+ EDITABLE_DAGSTER_DEPENDENCIES = [
175
+ "dagster",
176
+ "dagster-pipes",
177
+ "dagster-shared",
178
+ "dagster-test", # we include dagster-test for testing purposes
179
+ ]
180
+ EDITABLE_DAGSTER_DEV_DEPENDENCIES = [
181
+ "dagster-webserver",
182
+ "dagster-graphql",
183
+ "dagster-dg-core",
184
+ "dagster-dg-cli",
185
+ "dagster-cloud-cli",
186
+ ]
187
+
188
+
189
+ PYPI_DAGSTER_DEPENDENCIES = ["dagster"]
190
+ PYPI_DAGSTER_DEV_DEPENDENCIES = [
191
+ "dagster-webserver",
192
+ "dagster-dg-cli",
193
+ ]
194
+
195
+
196
+ def _get_editable_dagster_from_env() -> str:
197
+ if not os.environ.get("DAGSTER_GIT_REPO_DIR"):
198
+ exit_with_error(
199
+ "The `--use-editable-dagster` option "
200
+ "requires the `DAGSTER_GIT_REPO_DIR` environment variable to be set."
201
+ )
202
+ return os.environ["DAGSTER_GIT_REPO_DIR"]
203
+
204
+
205
+ def _get_pyproject_toml_dependencies(deps: Sequence[str]) -> str:
206
+ return "\n".join(
207
+ [
208
+ "dependencies = [",
209
+ *[f' "{dep}",' for dep in deps],
210
+ "]",
211
+ ]
212
+ )
213
+
214
+
215
+ def _get_pyproject_toml_dev_dependencies(deps: Sequence[str]) -> str:
216
+ return "\n".join(
217
+ [
218
+ "dev = [",
219
+ *[f' "{dep}",' for dep in deps],
220
+ "]",
221
+ ]
222
+ )
223
+
224
+
225
+ def _get_pyproject_toml_uv_sources(lib_paths: list[Path]) -> str:
226
+ lib_lines = [f"{path.name} = {{ path = '{path}', editable = true }}" for path in lib_paths]
227
+ return "\n".join(
228
+ [
229
+ "[tool.uv.sources]",
230
+ *lib_lines,
231
+ ]
232
+ )
233
+
234
+
235
+ def _gather_dagster_packages(editable_dagster_root: Path) -> list[Path]:
236
+ return [
237
+ p.parent
238
+ for p in (
239
+ *editable_dagster_root.glob("python_modules/dagster*/setup.py"),
240
+ *editable_dagster_root.glob("python_modules/libraries/dagster*/setup.py"),
241
+ )
242
+ ]
@@ -0,0 +1,4 @@
1
+ # Ignore any upstream gitignore pattern for "lib". This is included because "lib" is an important
2
+ # folder in scaffolded `dg` projects, but "lib" is also present in Github's default gitignore file
3
+ # for Python projects.
4
+ !lib
@@ -0,0 +1,23 @@
1
+ [project]
2
+ name = "{{ project_name }}"
3
+ requires-python = ">=3.9,<3.13"
4
+ version = "0.1.0"
5
+ {{ dependencies }}
6
+
7
+ [project.entry-points]
8
+ "dagster_dg.plugin" = { {{ project_name }} = "{{ project_name }}.components"}
9
+
10
+ [dependency-groups]
11
+ {{ dev_dependencies }}
12
+
13
+ [build-system]
14
+ requires = ["hatchling"]
15
+ build-backend = "hatchling.build"
16
+
17
+ [tool.dg]
18
+ directory_type = "project"
19
+
20
+ [tool.dg.project]
21
+ root_module = "{{ project_name }}"
22
+
23
+ {{ uv_sources }}
@@ -0,0 +1,8 @@
1
+ import {{ project_name }}.defs
2
+
3
+ from dagster import definitions, load_defs
4
+
5
+
6
+ @definitions
7
+ def defs():
8
+ return load_defs(defs_root={{ project_name }}.defs)
@@ -0,0 +1,3 @@
1
+ directory_type = "workspace"
2
+
3
+ [workspace]
@@ -0,0 +1 @@
1
+ __version__ = "1.10.18rc0"
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.4
2
+ Name: create-dagster
3
+ Version: 1.10.18rc0
4
+ Summary: CLI for creating a new dg project or workspace.
5
+ Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/create-dagster
6
+ Author: Dagster Labs
7
+ Author-email: hello@dagsterlabs.com
8
+ License: Apache-2.0
9
+ Classifier: Programming Language :: Python :: 3.9
10
+ Classifier: Programming Language :: Python :: 3.10
11
+ Classifier: Programming Language :: Python :: 3.11
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ License-File: LICENSE
16
+ Requires-Dist: dagster-dg-core==1.10.18rc0
17
+ Dynamic: author
18
+ Dynamic: author-email
19
+ Dynamic: classifier
20
+ Dynamic: home-page
21
+ Dynamic: license
22
+ Dynamic: license-file
23
+ Dynamic: requires-dist
24
+ Dynamic: summary
@@ -0,0 +1,27 @@
1
+ LICENSE
2
+ MANIFEST.in
3
+ setup.cfg
4
+ setup.py
5
+ create_dagster/__init__.py
6
+ create_dagster/py.typed
7
+ create_dagster/scaffold.py
8
+ create_dagster/version.py
9
+ create_dagster.egg-info/PKG-INFO
10
+ create_dagster.egg-info/SOURCES.txt
11
+ create_dagster.egg-info/dependency_links.txt
12
+ create_dagster.egg-info/entry_points.txt
13
+ create_dagster.egg-info/not-zip-safe
14
+ create_dagster.egg-info/requires.txt
15
+ create_dagster.egg-info/top_level.txt
16
+ create_dagster/cli/__init__.py
17
+ create_dagster/cli/scaffold.py
18
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/.gitignore.jinja
19
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/pyproject.toml.jinja
20
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/__init__.py
21
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/definitions.py.jinja
22
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/components/__init__.py
23
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/defs/.gitkeep
24
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/src/PROJECT_NAME_PLACEHOLDER/defs/__init__.py
25
+ create_dagster/templates/PROJECT_NAME_PLACEHOLDER/tests/__init__.py
26
+ create_dagster/templates/WORKSPACE_NAME_PLACEHOLDER/dg.toml.jinja
27
+ create_dagster/templates/WORKSPACE_NAME_PLACEHOLDER/projects/.gitkeep
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ create-dagster = create_dagster.cli:main
@@ -0,0 +1 @@
1
+ dagster-dg-core==1.10.18rc0
@@ -0,0 +1 @@
1
+ create_dagster
@@ -0,0 +1,14 @@
1
+ [metadata]
2
+ license_files = LICENSE
3
+
4
+ [check-manifest]
5
+ ignore =
6
+ .coveragerc
7
+ tox.ini
8
+ pytest.ini
9
+ create_dagster_tests/**
10
+
11
+ [egg_info]
12
+ tag_build =
13
+ tag_date = 0
14
+
@@ -0,0 +1,46 @@
1
+ from pathlib import Path
2
+
3
+ from setuptools import find_packages, setup
4
+
5
+
6
+ def get_version() -> str:
7
+ version: dict[str, str] = {}
8
+ with open(Path(__file__).parent / "create_dagster/version.py", encoding="utf8") as fp:
9
+ exec(fp.read(), version)
10
+
11
+ return version["__version__"]
12
+
13
+
14
+ ver = get_version()
15
+ # dont pin dev installs to avoid pip dep resolver issues
16
+ pin = "" if ver == "1!0+dev" else f"=={ver}"
17
+ setup(
18
+ name="create-dagster",
19
+ version=get_version(),
20
+ author="Dagster Labs",
21
+ author_email="hello@dagsterlabs.com",
22
+ license="Apache-2.0",
23
+ description="CLI for creating a new dg project or workspace.",
24
+ url=(
25
+ "https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/create-dagster"
26
+ ),
27
+ classifiers=[
28
+ "Programming Language :: Python :: 3.9",
29
+ "Programming Language :: Python :: 3.10",
30
+ "Programming Language :: Python :: 3.11",
31
+ "Programming Language :: Python :: 3.12",
32
+ "License :: OSI Approved :: Apache Software License",
33
+ "Operating System :: OS Independent",
34
+ ],
35
+ packages=find_packages(exclude=["create_dagster_tests*"]),
36
+ include_package_data=True,
37
+ install_requires=[
38
+ "dagster-dg-core==1.10.18rc0",
39
+ ],
40
+ zip_safe=False,
41
+ entry_points={
42
+ "console_scripts": [
43
+ "create-dagster = create_dagster.cli:main",
44
+ ]
45
+ },
46
+ )