metaflow 2.14.3__py2.py3-none-any.whl → 2.15.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/__init__.py +7 -0
- metaflow/cmd/code/__init__.py +230 -0
- metaflow/cmd/develop/stub_generator.py +5 -2
- metaflow/cmd/main_cli.py +1 -0
- metaflow/cmd/make_wrapper.py +16 -0
- metaflow/metaflow_config.py +2 -0
- metaflow/metaflow_environment.py +3 -1
- metaflow/mflog/__init__.py +4 -3
- metaflow/plugins/aws/batch/batch_cli.py +4 -4
- metaflow/plugins/aws/batch/batch_decorator.py +8 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +2 -1
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +2 -0
- metaflow/plugins/pypi/bootstrap.py +18 -27
- metaflow/plugins/pypi/conda_environment.py +8 -8
- metaflow/plugins/pypi/parsers.py +268 -0
- metaflow/plugins/pypi/utils.py +18 -0
- metaflow/runner/subprocess_manager.py +12 -6
- metaflow/version.py +1 -1
- metaflow-2.15.1.data/data/share/metaflow/devtools/Makefile +322 -0
- metaflow-2.15.1.data/data/share/metaflow/devtools/Tiltfile +620 -0
- metaflow-2.15.1.data/data/share/metaflow/devtools/pick_services.sh +104 -0
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/METADATA +3 -3
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/RECORD +27 -21
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/WHEEL +1 -1
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/entry_points.txt +1 -0
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/LICENSE +0 -0
- {metaflow-2.14.3.dist-info → metaflow-2.15.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,268 @@
|
|
1
|
+
# this file can be overridden by extensions as is (e.g. metaflow-nflx-extensions)
|
2
|
+
from metaflow.exception import MetaflowException
|
3
|
+
|
4
|
+
|
5
|
+
class ParserValueError(MetaflowException):
|
6
|
+
headline = "Value error"
|
7
|
+
|
8
|
+
|
9
|
+
def requirements_txt_parser(content: str):
|
10
|
+
"""
|
11
|
+
Parse non-comment lines from a requirements.txt file as strictly valid
|
12
|
+
PEP 508 requirements.
|
13
|
+
|
14
|
+
Recognizes direct references (e.g. "my_lib @ git+https://..."), extras
|
15
|
+
(e.g. "requests[security]"), and version specifiers (e.g. "==2.0"). If
|
16
|
+
the package name is "python", its specifier is stored in the "python"
|
17
|
+
key instead of "packages".
|
18
|
+
|
19
|
+
Parameters
|
20
|
+
----------
|
21
|
+
content : str
|
22
|
+
Contents of a requirements.txt file.
|
23
|
+
|
24
|
+
Returns
|
25
|
+
-------
|
26
|
+
dict
|
27
|
+
A dictionary with two keys:
|
28
|
+
- "packages": dict(str -> str)
|
29
|
+
Mapping from package name (plus optional extras/references) to a
|
30
|
+
version specifier string.
|
31
|
+
- "python": str or None
|
32
|
+
The Python version constraints if present, otherwise None.
|
33
|
+
|
34
|
+
Raises
|
35
|
+
------
|
36
|
+
ParserValueError
|
37
|
+
If a requirement line is invalid PEP 508 or if environment markers are
|
38
|
+
detected, or if multiple Python constraints are specified.
|
39
|
+
"""
|
40
|
+
import re
|
41
|
+
from metaflow._vendor.packaging.requirements import Requirement, InvalidRequirement
|
42
|
+
|
43
|
+
parsed = {"packages": {}, "python": None}
|
44
|
+
|
45
|
+
inline_comment_pattern = re.compile(r"\s+#.*$")
|
46
|
+
for line in content.splitlines():
|
47
|
+
line = line.strip()
|
48
|
+
|
49
|
+
# support Rye lockfiles by skipping lines not compliant with requirements
|
50
|
+
if line == "-e file:.":
|
51
|
+
continue
|
52
|
+
|
53
|
+
if not line or line.startswith("#"):
|
54
|
+
continue
|
55
|
+
|
56
|
+
line = inline_comment_pattern.sub("", line).strip()
|
57
|
+
if not line:
|
58
|
+
continue
|
59
|
+
|
60
|
+
try:
|
61
|
+
req = Requirement(line)
|
62
|
+
except InvalidRequirement:
|
63
|
+
raise ParserValueError(f"Not a valid PEP 508 requirement: '{line}'")
|
64
|
+
|
65
|
+
if req.marker is not None:
|
66
|
+
raise ParserValueError(
|
67
|
+
"Environment markers (e.g. 'platform_system==\"Linux\"') "
|
68
|
+
f"are not supported for line: '{line}'"
|
69
|
+
)
|
70
|
+
|
71
|
+
dep_key = req.name
|
72
|
+
if req.extras:
|
73
|
+
dep_key += f"[{','.join(req.extras)}]"
|
74
|
+
if req.url:
|
75
|
+
dep_key += f"@{req.url}"
|
76
|
+
|
77
|
+
dep_spec = str(req.specifier).lstrip(" =")
|
78
|
+
|
79
|
+
if req.name.lower() == "python":
|
80
|
+
if parsed["python"] is not None and dep_spec:
|
81
|
+
raise ParserValueError(
|
82
|
+
f"Multiple Python version specs not allowed: '{line}'"
|
83
|
+
)
|
84
|
+
parsed["python"] = dep_spec or None
|
85
|
+
else:
|
86
|
+
parsed["packages"][dep_key] = dep_spec
|
87
|
+
|
88
|
+
return parsed
|
89
|
+
|
90
|
+
|
91
|
+
def pyproject_toml_parser(content: str):
|
92
|
+
"""
|
93
|
+
Parse a pyproject.toml file per PEP 621.
|
94
|
+
|
95
|
+
Reads the 'requires-python' and 'dependencies' fields from the "[project]" section.
|
96
|
+
Each dependency line must be a valid PEP 508 requirement. If the package name is
|
97
|
+
"python", its specifier is stored in the "python" key instead of "packages".
|
98
|
+
|
99
|
+
Parameters
|
100
|
+
----------
|
101
|
+
content : str
|
102
|
+
Contents of a pyproject.toml file.
|
103
|
+
|
104
|
+
Returns
|
105
|
+
-------
|
106
|
+
dict
|
107
|
+
A dictionary with two keys:
|
108
|
+
- "packages": dict(str -> str)
|
109
|
+
Mapping from package name (plus optional extras/references) to a
|
110
|
+
version specifier string.
|
111
|
+
- "python": str or None
|
112
|
+
The Python version constraints if present, otherwise None.
|
113
|
+
|
114
|
+
Raises
|
115
|
+
------
|
116
|
+
RuntimeError
|
117
|
+
If no TOML library (tomllib in Python 3.11+ or tomli in earlier versions) is found.
|
118
|
+
ParserValueError
|
119
|
+
If a dependency is not valid PEP 508, if environment markers are used, or if
|
120
|
+
multiple Python constraints are specified.
|
121
|
+
"""
|
122
|
+
try:
|
123
|
+
import tomllib as toml # Python 3.11+
|
124
|
+
except ImportError:
|
125
|
+
try:
|
126
|
+
import tomli as toml # Python < 3.11 (requires "tomli" package)
|
127
|
+
except ImportError:
|
128
|
+
raise RuntimeError(
|
129
|
+
"Could not import a TOML library. For Python <3.11, please install 'tomli'."
|
130
|
+
)
|
131
|
+
from metaflow._vendor.packaging.requirements import Requirement, InvalidRequirement
|
132
|
+
|
133
|
+
data = toml.loads(content)
|
134
|
+
|
135
|
+
project = data.get("project", {})
|
136
|
+
requirements = project.get("dependencies", [])
|
137
|
+
requires_python = project.get("requires-python")
|
138
|
+
|
139
|
+
parsed = {"packages": {}, "python": None}
|
140
|
+
|
141
|
+
if requires_python is not None:
|
142
|
+
# If present, store verbatim; note that PEP 621 does not necessarily
|
143
|
+
# require "python" to be a dependency in the usual sense.
|
144
|
+
# Example: "requires-python" = ">=3.7,<4"
|
145
|
+
parsed["python"] = requires_python.lstrip("=").strip()
|
146
|
+
|
147
|
+
for dep_line in requirements:
|
148
|
+
dep_line_stripped = dep_line.strip()
|
149
|
+
try:
|
150
|
+
req = Requirement(dep_line_stripped)
|
151
|
+
except InvalidRequirement:
|
152
|
+
raise ParserValueError(
|
153
|
+
f"Not a valid PEP 508 requirement: '{dep_line_stripped}'"
|
154
|
+
)
|
155
|
+
|
156
|
+
if req.marker is not None:
|
157
|
+
raise ParserValueError(
|
158
|
+
f"Environment markers not supported for line: '{dep_line_stripped}'"
|
159
|
+
)
|
160
|
+
|
161
|
+
dep_key = req.name
|
162
|
+
if req.extras:
|
163
|
+
dep_key += f"[{','.join(req.extras)}]"
|
164
|
+
if req.url:
|
165
|
+
dep_key += f"@{req.url}"
|
166
|
+
|
167
|
+
dep_spec = str(req.specifier).lstrip("=")
|
168
|
+
|
169
|
+
if req.name.lower() == "python":
|
170
|
+
if parsed["python"] is not None and dep_spec:
|
171
|
+
raise ParserValueError(
|
172
|
+
f"Multiple Python version specs not allowed: '{dep_line_stripped}'"
|
173
|
+
)
|
174
|
+
parsed["python"] = dep_spec or None
|
175
|
+
else:
|
176
|
+
parsed["packages"][dep_key] = dep_spec
|
177
|
+
|
178
|
+
return parsed
|
179
|
+
|
180
|
+
|
181
|
+
def conda_environment_yml_parser(content: str):
|
182
|
+
"""
|
183
|
+
Parse a minimal environment.yml file under strict assumptions.
|
184
|
+
|
185
|
+
The file must contain a 'dependencies:' line, after which each dependency line
|
186
|
+
appears with a '- ' prefix. Python can appear as 'python=3.9', etc.; other
|
187
|
+
packages as 'numpy=1.21.2' or simply 'numpy'. Non-compliant lines raise ParserValueError.
|
188
|
+
|
189
|
+
Parameters
|
190
|
+
----------
|
191
|
+
content : str
|
192
|
+
Contents of a environment.yml file.
|
193
|
+
|
194
|
+
Returns
|
195
|
+
-------
|
196
|
+
dict
|
197
|
+
A dictionary with keys:
|
198
|
+
{
|
199
|
+
"packages": dict(str -> str),
|
200
|
+
"python": str or None
|
201
|
+
}
|
202
|
+
|
203
|
+
Raises
|
204
|
+
------
|
205
|
+
ParserValueError
|
206
|
+
If the file has malformed lines or unsupported sections.
|
207
|
+
"""
|
208
|
+
import re
|
209
|
+
|
210
|
+
packages = {}
|
211
|
+
python_version = None
|
212
|
+
|
213
|
+
inside_dependencies = False
|
214
|
+
|
215
|
+
# Basic pattern for lines like "numpy=1.21.2"
|
216
|
+
# Group 1: package name
|
217
|
+
# Group 2: optional operator + version (could be "=1.21.2", "==1.21.2", etc.)
|
218
|
+
line_regex = re.compile(r"^([A-Za-z0-9_\-\.]+)(\s*[=<>!~].+\s*)?$")
|
219
|
+
inline_comment_pattern = re.compile(r"\s+#.*$")
|
220
|
+
|
221
|
+
for line in content.splitlines():
|
222
|
+
line = line.strip()
|
223
|
+
if not line or line.startswith("#"):
|
224
|
+
continue
|
225
|
+
|
226
|
+
line = inline_comment_pattern.sub("", line).strip()
|
227
|
+
if not line:
|
228
|
+
continue
|
229
|
+
|
230
|
+
if line.lower().startswith("dependencies:"):
|
231
|
+
inside_dependencies = True
|
232
|
+
continue
|
233
|
+
|
234
|
+
if inside_dependencies and not line.startswith("-"):
|
235
|
+
inside_dependencies = False
|
236
|
+
continue
|
237
|
+
|
238
|
+
if not inside_dependencies:
|
239
|
+
continue
|
240
|
+
|
241
|
+
dep_line = line.lstrip("-").strip()
|
242
|
+
if dep_line.endswith(":"):
|
243
|
+
raise ParserValueError(
|
244
|
+
f"Unsupported subsection '{dep_line}' in environment.yml."
|
245
|
+
)
|
246
|
+
|
247
|
+
match = line_regex.match(dep_line)
|
248
|
+
if not match:
|
249
|
+
raise ParserValueError(
|
250
|
+
f"Line '{dep_line}' is not a valid conda package specifier."
|
251
|
+
)
|
252
|
+
|
253
|
+
pkg_name, pkg_version_part = match.groups()
|
254
|
+
version_spec = pkg_version_part.strip() if pkg_version_part else ""
|
255
|
+
|
256
|
+
if version_spec.startswith("="):
|
257
|
+
version_spec = version_spec.lstrip("=").strip()
|
258
|
+
|
259
|
+
if pkg_name.lower() == "python":
|
260
|
+
if python_version is not None and version_spec:
|
261
|
+
raise ParserValueError(
|
262
|
+
f"Multiple Python version specs detected: '{dep_line}'"
|
263
|
+
)
|
264
|
+
python_version = version_spec
|
265
|
+
else:
|
266
|
+
packages[pkg_name] = version_spec
|
267
|
+
|
268
|
+
return {"packages": packages, "python": python_version}
|
metaflow/plugins/pypi/utils.py
CHANGED
@@ -72,6 +72,24 @@ def pip_tags(python_version, mamba_platform):
|
|
72
72
|
)
|
73
73
|
]
|
74
74
|
platforms.append("linux_x86_64")
|
75
|
+
elif mamba_platform == "linux-aarch64":
|
76
|
+
platforms = [
|
77
|
+
"manylinux%s_aarch64" % s
|
78
|
+
for s in (
|
79
|
+
"2014",
|
80
|
+
"_2_17",
|
81
|
+
"_2_18",
|
82
|
+
"_2_19",
|
83
|
+
"_2_20",
|
84
|
+
"_2_21",
|
85
|
+
"_2_23",
|
86
|
+
"_2_24",
|
87
|
+
"_2_25",
|
88
|
+
"_2_26",
|
89
|
+
"_2_27",
|
90
|
+
)
|
91
|
+
]
|
92
|
+
platforms.append("linux_aarch64")
|
75
93
|
elif mamba_platform == "osx-64":
|
76
94
|
platforms = tags.mac_platforms(arch="x86_64")
|
77
95
|
elif mamba_platform == "osx-arm64":
|
@@ -78,13 +78,19 @@ class SubprocessManager(object):
|
|
78
78
|
self.commands: Dict[int, CommandManager] = {}
|
79
79
|
|
80
80
|
try:
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
81
|
+
try:
|
82
|
+
loop = asyncio.get_running_loop()
|
83
|
+
loop.add_signal_handler(
|
84
|
+
signal.SIGINT,
|
85
|
+
lambda: asyncio.create_task(self._async_handle_sigint()),
|
86
|
+
)
|
87
|
+
except RuntimeError:
|
88
|
+
signal.signal(signal.SIGINT, self._handle_sigint)
|
89
|
+
except ValueError:
|
90
|
+
sys.stderr.write(
|
91
|
+
"Warning: Unable to set signal handlers in non-main thread. "
|
92
|
+
"Interrupt handling will be limited.\n"
|
85
93
|
)
|
86
|
-
except RuntimeError:
|
87
|
-
signal.signal(signal.SIGINT, self._handle_sigint)
|
88
94
|
|
89
95
|
async def _async_handle_sigint(self):
|
90
96
|
pids = [
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.
|
1
|
+
metaflow_version = "2.15.1"
|
@@ -0,0 +1,322 @@
|
|
1
|
+
SHELL := /bin/bash
|
2
|
+
.SHELLFLAGS := -eu -o pipefail -c
|
3
|
+
|
4
|
+
help:
|
5
|
+
@echo "Available targets:"
|
6
|
+
@echo " up - Start the development environment"
|
7
|
+
@echo " shell - Switch to development environment's shell"
|
8
|
+
@echo " ui - Open Metaflow UI"
|
9
|
+
@echo " dashboard - Open Minikube dashboard"
|
10
|
+
@echo " down - Stop and clean up the environment"
|
11
|
+
@echo " help - Show this help message"
|
12
|
+
|
13
|
+
HELM_VERSION := v3.14.0
|
14
|
+
MINIKUBE_VERSION := v1.32.0
|
15
|
+
TILT_VERSION := v0.33.11
|
16
|
+
GUM_VERSION := v0.15.2
|
17
|
+
|
18
|
+
MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
|
19
|
+
MKFILE_DIR := $(dir $(MKFILE_PATH))
|
20
|
+
DEVTOOLS_DIR := $(MKFILE_DIR).devtools
|
21
|
+
PICK_SERVICES := $(MKFILE_DIR)pick_services.sh
|
22
|
+
MINIKUBE_DIR := $(DEVTOOLS_DIR)/minikube
|
23
|
+
MINIKUBE := $(MINIKUBE_DIR)/minikube
|
24
|
+
TILT_DIR := $(DEVTOOLS_DIR)/tilt
|
25
|
+
TILT := $(TILT_DIR)/tilt
|
26
|
+
TILTFILE := $(MKFILE_DIR)/Tiltfile
|
27
|
+
MAKE_CMD := $(MAKE) -C "$(MKFILE_DIR)"
|
28
|
+
|
29
|
+
MINIKUBE_CPUS ?= 4
|
30
|
+
MINIKUBE_MEMORY ?= 6000
|
31
|
+
MINIKUBE_DISK_SIZE ?= 20g
|
32
|
+
|
33
|
+
ifeq ($(shell uname), Darwin)
|
34
|
+
minikube_os = darwin
|
35
|
+
tilt_os = mac
|
36
|
+
else
|
37
|
+
minikube_os = linux
|
38
|
+
tilt_os = linux
|
39
|
+
endif
|
40
|
+
|
41
|
+
ifeq ($(shell uname -m), x86_64)
|
42
|
+
arch = amd64
|
43
|
+
tilt_arch = x86_64
|
44
|
+
else
|
45
|
+
arch = arm64
|
46
|
+
tilt_arch = arm64
|
47
|
+
endif
|
48
|
+
|
49
|
+
# TODO: Move scripts to a folder
|
50
|
+
|
51
|
+
install-helm:
|
52
|
+
@if ! command -v helm >/dev/null 2>&1; then \
|
53
|
+
echo "📥 Installing Helm $(HELM_VERSION) (may require sudo access)..."; \
|
54
|
+
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | HELM_INSTALL_VERSION=$(HELM_VERSION) bash; \
|
55
|
+
echo "✅ Helm installation complete"; \
|
56
|
+
fi
|
57
|
+
|
58
|
+
check-docker:
|
59
|
+
@if ! command -v docker >/dev/null 2>&1; then \
|
60
|
+
echo "❌ Docker is not installed. Please install Docker first: https://docs.docker.com/get-docker/"; \
|
61
|
+
exit 1; \
|
62
|
+
fi
|
63
|
+
@echo "🔍 Checking Docker daemon..."
|
64
|
+
@if [ "$(shell uname)" = "Darwin" ]; then \
|
65
|
+
open -a Docker || (echo "❌ Please start Docker Desktop" && exit 1); \
|
66
|
+
else \
|
67
|
+
systemctl is-active --quiet docker || (echo "❌ Docker daemon is not running. Start with 'sudo systemctl start docker'" && exit 1); \
|
68
|
+
fi
|
69
|
+
@echo "✅ Docker is running"
|
70
|
+
|
71
|
+
install-brew:
|
72
|
+
@if [ "$(shell uname)" = "Darwin" ] && ! command -v brew >/dev/null 2>&1; then \
|
73
|
+
echo "📥 Installing Homebrew..."; \
|
74
|
+
/bin/bash -c "$$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"; \
|
75
|
+
echo "✅ Homebrew installation complete"; \
|
76
|
+
fi
|
77
|
+
|
78
|
+
install-curl:
|
79
|
+
@if ! command -v curl >/dev/null 2>&1; then \
|
80
|
+
echo "📥 Installing curl..."; \
|
81
|
+
if [ "$(shell uname)" = "Darwin" ]; then \
|
82
|
+
HOMEBREW_NO_AUTO_UPDATE=1 brew install curl; \
|
83
|
+
elif command -v apt-get >/dev/null 2>&1; then \
|
84
|
+
sudo apt-get update && sudo apt-get install -y curl; \
|
85
|
+
elif command -v yum >/dev/null 2>&1; then \
|
86
|
+
sudo yum install -y curl; \
|
87
|
+
elif command -v dnf >/dev/null 2>&1; then \
|
88
|
+
sudo dnf install -y curl; \
|
89
|
+
else \
|
90
|
+
echo "❌ Could not install curl. Please install manually."; \
|
91
|
+
exit 1; \
|
92
|
+
fi; \
|
93
|
+
echo "✅ curl installation complete"; \
|
94
|
+
fi
|
95
|
+
|
96
|
+
install-gum:
|
97
|
+
@echo "🔍 Checking if gum is installed..."
|
98
|
+
@if ! command -v gum >/dev/null 2>&1; then \
|
99
|
+
echo "📥 Installing gum..."; \
|
100
|
+
if [ "$(shell uname)" = "Darwin" ]; then \
|
101
|
+
HOMEBREW_NO_AUTO_UPDATE=1 brew install gum; \
|
102
|
+
elif command -v apt-get >/dev/null 2>&1; then \
|
103
|
+
curl -fsSL -o /tmp/gum.deb \
|
104
|
+
"https://github.com/charmbracelet/gum/releases/download/$(GUM_VERSION)/gum_$(GUM_VERSION:v%=%)_$(arch).deb"; \
|
105
|
+
sudo apt-get update -qq; \
|
106
|
+
sudo apt-get install -y /tmp/gum.deb || sudo dpkg -i /tmp/gum.deb; \
|
107
|
+
rm -f /tmp/gum.deb; \
|
108
|
+
else \
|
109
|
+
echo "❌ Could not determine how to install gum for your platform. Please install manually."; \
|
110
|
+
exit 1; \
|
111
|
+
fi; \
|
112
|
+
echo "✅ gum installation complete"; \
|
113
|
+
else \
|
114
|
+
echo "✅ gum is already installed."; \
|
115
|
+
fi
|
116
|
+
|
117
|
+
setup-minikube:
|
118
|
+
@if [ ! -f "$(MINIKUBE)" ]; then \
|
119
|
+
echo "📥 Installing Minikube $(MINIKUBE_VERSION)"; \
|
120
|
+
mkdir -p $(MINIKUBE_DIR); \
|
121
|
+
curl -L --fail https://github.com/kubernetes/minikube/releases/download/$(MINIKUBE_VERSION)/minikube-$(minikube_os)-$(arch) -o $(MINIKUBE) || (echo "❌ Failed to download minikube" && exit 1); \
|
122
|
+
chmod +x $(MINIKUBE); \
|
123
|
+
echo "✅ Minikube $(MINIKUBE_VERSION) installed successfully"; \
|
124
|
+
fi
|
125
|
+
@echo "🔧 Setting up Minikube $(MINIKUBE_VERSION) cluster..."
|
126
|
+
@if ! $(MINIKUBE) status >/dev/null 2>&1; then \
|
127
|
+
echo "🚀 Starting new Minikube $(MINIKUBE_VERSION) cluster..."; \
|
128
|
+
$(MINIKUBE) start \
|
129
|
+
--cpus $(MINIKUBE_CPUS) \
|
130
|
+
--memory $(MINIKUBE_MEMORY) \
|
131
|
+
--disk-size $(MINIKUBE_DISK_SIZE) \
|
132
|
+
--driver docker; \
|
133
|
+
echo "🔌 Enabling metrics-server and dashboard (quietly)..."; \
|
134
|
+
$(MINIKUBE) addons enable metrics-server >/dev/null 2>&1; \
|
135
|
+
$(MINIKUBE) addons enable dashboard >/dev/null 2>&1; \
|
136
|
+
else \
|
137
|
+
echo "✅ Minikube $(MINIKUBE_VERSION) cluster is already running"; \
|
138
|
+
fi
|
139
|
+
@echo "🎉 Minikube $(MINIKUBE_VERSION) cluster is ready!"
|
140
|
+
|
141
|
+
setup-tilt:
|
142
|
+
@if [ ! -f "$(TILT)" ]; then \
|
143
|
+
echo "📥 Installing Tilt $(TILT_VERSION)"; \
|
144
|
+
mkdir -p $(TILT_DIR); \
|
145
|
+
(curl -L https://github.com/tilt-dev/tilt/releases/download/$(TILT_VERSION)/tilt.$(TILT_VERSION:v%=%).$(tilt_os).$(tilt_arch).tar.gz | tar -xz -C $(TILT_DIR)) && echo "✅ Tilt $(TILT_VERSION) installed successfully" || (echo "❌ Failed to install Tilt" && exit 1); \
|
146
|
+
fi
|
147
|
+
|
148
|
+
tunnel:
|
149
|
+
$(MINIKUBE) tunnel
|
150
|
+
|
151
|
+
teardown-minikube:
|
152
|
+
@echo "🛑 Stopping Minikube $(MINIKUBE_VERSION) cluster..."
|
153
|
+
-$(MINIKUBE) stop
|
154
|
+
@echo "🗑️ Deleting Minikube $(MINIKUBE_VERSION) cluster..."
|
155
|
+
-$(MINIKUBE) delete --all
|
156
|
+
@echo "🧹 Removing Minikube binary..."
|
157
|
+
-rm -rf $(MINIKUBE_DIR)
|
158
|
+
@echo "✅ Minikube $(MINIKUBE_VERSION) teardown complete"
|
159
|
+
|
160
|
+
dashboard:
|
161
|
+
@echo "🔗 Opening Minikube Dashboard..."
|
162
|
+
@$(MINIKUBE) dashboard
|
163
|
+
|
164
|
+
# make shell is symlinked to metaflow-dev shell by metaflow
|
165
|
+
up: install-brew check-docker install-curl install-gum setup-minikube install-helm setup-tilt
|
166
|
+
@echo "🚀 Starting up (may require sudo access)..."
|
167
|
+
@mkdir -p $(DEVTOOLS_DIR)
|
168
|
+
@echo '#!/bin/bash' > $(DEVTOOLS_DIR)/start.sh
|
169
|
+
@echo 'set -e' >> $(DEVTOOLS_DIR)/start.sh
|
170
|
+
@echo 'trap "exit" INT TERM' >> $(DEVTOOLS_DIR)/start.sh
|
171
|
+
@echo 'trap "kill 0" EXIT' >> $(DEVTOOLS_DIR)/start.sh
|
172
|
+
@echo 'eval $$($(MINIKUBE) docker-env)' >> $(DEVTOOLS_DIR)/start.sh
|
173
|
+
|
174
|
+
@echo 'if [ -n "$$SERVICES_OVERRIDE" ]; then' >> "$(DEVTOOLS_DIR)/start.sh"
|
175
|
+
@echo ' echo "🌐 Using user-provided list of services: $$SERVICES_OVERRIDE"' >> "$(DEVTOOLS_DIR)/start.sh"
|
176
|
+
@echo ' SERVICES="$$SERVICES_OVERRIDE"' >> "$(DEVTOOLS_DIR)/start.sh"
|
177
|
+
@echo 'else' >> "$(DEVTOOLS_DIR)/start.sh"
|
178
|
+
@echo ' echo "📝 Selecting services..."' >> "$(DEVTOOLS_DIR)/start.sh"
|
179
|
+
@echo ' SERVICES=$$($(PICK_SERVICES))' >> "$(DEVTOOLS_DIR)/start.sh"
|
180
|
+
@echo 'fi' >> "$(DEVTOOLS_DIR)/start.sh"
|
181
|
+
@echo 'PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" $(MINIKUBE) tunnel &' >> $(DEVTOOLS_DIR)/start.sh
|
182
|
+
@echo 'echo -e "🚀 Starting Tilt with selected services..."' >> $(DEVTOOLS_DIR)/start.sh
|
183
|
+
@echo 'echo -e "\033[1;38;5;46m\n🔥 \033[1;38;5;196mNext Steps:\033[0;38;5;46m Use \033[3mmetaflow-dev shell\033[23m to switch to the development\n environment'\''s shell and start executing your Metaflow flows.\n\033[0m"' >> "$(DEVTOOLS_DIR)/start.sh"
|
184
|
+
@echo 'PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" SERVICES="$$SERVICES" tilt up -f $(TILTFILE)' >> $(DEVTOOLS_DIR)/start.sh
|
185
|
+
@echo 'wait' >> $(DEVTOOLS_DIR)/start.sh
|
186
|
+
@chmod +x $(DEVTOOLS_DIR)/start.sh
|
187
|
+
@$(DEVTOOLS_DIR)/start.sh
|
188
|
+
|
189
|
+
down:
|
190
|
+
@echo "🛑 Stopping all services..."
|
191
|
+
@-pkill -f "$(MINIKUBE) tunnel" 2>/dev/null || true
|
192
|
+
@echo "⏹️ Stopping Tilt..."
|
193
|
+
-PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" tilt down -f $(TILTFILE)
|
194
|
+
@echo "🧹 Cleaning up Minikube..."
|
195
|
+
$(MAKE_CMD) teardown-minikube
|
196
|
+
@echo "🗑️ Removing Tilt binary and directory..."
|
197
|
+
-rm -rf $(TILT_DIR)
|
198
|
+
@echo "🧹 Removing temporary scripts..."
|
199
|
+
-rm -rf $(DEVTOOLS_DIR)
|
200
|
+
@echo "✨ All done!"
|
201
|
+
|
202
|
+
shell: setup-tilt
|
203
|
+
@echo "⏳ Checking if development environment is up..."
|
204
|
+
@set -e; \
|
205
|
+
for i in $$(seq 1 90); do \
|
206
|
+
if "$(TILT)" get session >/dev/null 2>&1; then \
|
207
|
+
found_session=1; \
|
208
|
+
break; \
|
209
|
+
else \
|
210
|
+
sleep 2; \
|
211
|
+
fi; \
|
212
|
+
done; \
|
213
|
+
if [ -z "$${found_session}" ]; then \
|
214
|
+
echo "❌ Development environment is not up."; \
|
215
|
+
echo " Please run 'metaflow-dev up' in another terminal, then re-run 'metaflow-dev shell'."; \
|
216
|
+
exit 1; \
|
217
|
+
fi
|
218
|
+
@echo "⏳ Waiting for development environment to be ready..."
|
219
|
+
@while true; do \
|
220
|
+
"$(TILT)" get uiresource generate-configs >/dev/null 2>&1; \
|
221
|
+
status=$$?; \
|
222
|
+
if [ $$status -eq 0 ]; then \
|
223
|
+
"$(TILT)" wait --for=condition=Ready uiresource/generate-configs; \
|
224
|
+
break; \
|
225
|
+
elif [ $$status -eq 127 ]; then \
|
226
|
+
echo "❌ Development environment is not up."; \
|
227
|
+
echo " Please run 'metaflow-dev up' in another terminal, then re-run 'metaflow-dev shell'."; \
|
228
|
+
exit 1; \
|
229
|
+
else \
|
230
|
+
sleep 1; \
|
231
|
+
fi; \
|
232
|
+
done
|
233
|
+
@echo "🔧 Starting a new shell for development environment..."
|
234
|
+
@bash -c '\
|
235
|
+
if [ -n "$$SHELL" ]; then \
|
236
|
+
user_shell="$$SHELL"; \
|
237
|
+
else \
|
238
|
+
user_shell="$(SHELL)"; \
|
239
|
+
fi; \
|
240
|
+
echo "🔎 Using $$user_shell for interactive session."; \
|
241
|
+
echo "🐍 If you installed Metaflow in a virtual environment, activate it now."; \
|
242
|
+
if [ -f "$(DEVTOOLS_DIR)/aws_config" ]; then \
|
243
|
+
env METAFLOW_HOME="$(DEVTOOLS_DIR)" \
|
244
|
+
METAFLOW_PROFILE=local \
|
245
|
+
AWS_CONFIG_FILE="$(DEVTOOLS_DIR)/aws_config" \
|
246
|
+
"$$user_shell" -i; \
|
247
|
+
else \
|
248
|
+
env METAFLOW_HOME="$(DEVTOOLS_DIR)" \
|
249
|
+
METAFLOW_PROFILE=local \
|
250
|
+
"$$user_shell" -i; \
|
251
|
+
fi'
|
252
|
+
|
253
|
+
|
254
|
+
# @echo '$(MAKE_CMD) create-dev-shell' >> $(DEVTOOLS_DIR)/start.sh
|
255
|
+
# @echo 'rm -f /tmp/metaflow-devshell-*' >> $(DEVTOOLS_DIR)/start.sh
|
256
|
+
create-dev-shell: setup-tilt
|
257
|
+
@bash -c '\
|
258
|
+
SHELL_PATH=/tmp/metaflow-dev-shell-$$$$ && \
|
259
|
+
echo "#!/bin/bash" > $$SHELL_PATH && \
|
260
|
+
echo "set -e" >> $$SHELL_PATH && \
|
261
|
+
echo "" >> $$SHELL_PATH && \
|
262
|
+
echo "echo \"⏳ Checking if development environment is up...\"" >> $$SHELL_PATH && \
|
263
|
+
echo "if ! $(TILT) get session >/dev/null 2>&1; then" >> $$SHELL_PATH && \
|
264
|
+
echo " echo \"❌ Development environment is not up.\"" >> $$SHELL_PATH && \
|
265
|
+
echo " echo \" Please run '\''make up'\'' in another terminal, then re-run this script.\"" >> $$SHELL_PATH && \
|
266
|
+
echo " exit 1" >> $$SHELL_PATH && \
|
267
|
+
echo "fi" >> $$SHELL_PATH && \
|
268
|
+
echo "" >> $$SHELL_PATH && \
|
269
|
+
echo "echo \"⏳ Waiting for development environment to be ready...\"" >> $$SHELL_PATH && \
|
270
|
+
echo "$(TILT) wait --for=condition=Ready uiresource/generate-configs" >> $$SHELL_PATH && \
|
271
|
+
echo "" >> $$SHELL_PATH && \
|
272
|
+
echo "echo \"🔧 Starting a new shell for development environment...\"" >> $$SHELL_PATH && \
|
273
|
+
echo "if [ -n \"\$$SHELL\" ]; then" >> $$SHELL_PATH && \
|
274
|
+
echo " user_shell=\"\$$SHELL\"" >> $$SHELL_PATH && \
|
275
|
+
echo "else" >> $$SHELL_PATH && \
|
276
|
+
echo " user_shell=\"$(SHELL)\"" >> $$SHELL_PATH && \
|
277
|
+
echo "fi" >> $$SHELL_PATH && \
|
278
|
+
echo "echo \"🔎 Using \$$user_shell for interactive session.\"" >> $$SHELL_PATH && \
|
279
|
+
echo "echo \"🐍 If you installed Metaflow in a virtual environment, activate it now.\"" >> $$SHELL_PATH && \
|
280
|
+
echo "if [ -f \"$(DEVTOOLS_DIR)/aws_config\" ]; then" >> $$SHELL_PATH && \
|
281
|
+
echo " env METAFLOW_HOME=\"$(DEVTOOLS_DIR)\" \\" >> $$SHELL_PATH && \
|
282
|
+
echo " METAFLOW_PROFILE=local \\" >> $$SHELL_PATH && \
|
283
|
+
echo " AWS_CONFIG_FILE=\"$(DEVTOOLS_DIR)/aws_config\" \\" >> $$SHELL_PATH && \
|
284
|
+
echo " \"\$$user_shell\" -i" >> $$SHELL_PATH && \
|
285
|
+
echo "else" >> $$SHELL_PATH && \
|
286
|
+
echo " env METAFLOW_HOME=\"$(DEVTOOLS_DIR)\" \\" >> $$SHELL_PATH && \
|
287
|
+
echo " METAFLOW_PROFILE=local \\" >> $$SHELL_PATH && \
|
288
|
+
echo " \"\$$user_shell\" -i" >> $$SHELL_PATH && \
|
289
|
+
echo "fi" >> $$SHELL_PATH && \
|
290
|
+
chmod +x $$SHELL_PATH && \
|
291
|
+
echo "✨ Created $$SHELL_PATH" && \
|
292
|
+
echo "🔑 Execute it from ANY directory to switch to development environment shell!" \
|
293
|
+
'
|
294
|
+
|
295
|
+
ui: setup-tilt
|
296
|
+
@echo "⏳ Checking if the development environment is up..."
|
297
|
+
@if ! $(TILT) get session >/dev/null 2>&1; then \
|
298
|
+
echo "❌ Development environment is not up."; \
|
299
|
+
echo " Please run 'metaflow-dev up' in another terminal, then re-run 'metaflow-dev ui'."; \
|
300
|
+
exit 1; \
|
301
|
+
fi
|
302
|
+
@echo "⏳ Waiting for Metaflow UI to be ready..."
|
303
|
+
@while true; do \
|
304
|
+
"$(TILT)" get uiresource metaflow-ui >/dev/null 2>&1; \
|
305
|
+
status=$$?; \
|
306
|
+
if [ $$status -eq 0 ]; then \
|
307
|
+
"$(TILT)" wait --for=condition=Ready uiresource/metaflow-ui; \
|
308
|
+
break; \
|
309
|
+
elif [ $$status -eq 127 ]; then \
|
310
|
+
echo "❌ Development environment is not up."; \
|
311
|
+
echo " Please run 'metaflow-dev up' in another terminal, then re-run 'metaflow-dev shell'."; \
|
312
|
+
exit 1; \
|
313
|
+
else \
|
314
|
+
sleep 1; \
|
315
|
+
fi; \
|
316
|
+
done
|
317
|
+
@echo "🔗 Opening Metaflow UI at http://localhost:3000"
|
318
|
+
@open http://localhost:3000
|
319
|
+
|
320
|
+
.PHONY: install-helm setup-minikube setup-tilt teardown-minikube tunnel up down check-docker install-curl install-gum install-brew up down dashboard shell ui help
|
321
|
+
|
322
|
+
.DEFAULT_GOAL := up
|