ob-metaflow 2.15.0.1__py2.py3-none-any.whl → 2.15.5.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow might be problematic. Click here for more details.
- metaflow/__init__.py +6 -0
- metaflow/cmd/code/__init__.py +230 -0
- metaflow/cmd/develop/stub_generator.py +5 -2
- metaflow/cmd/main_cli.py +1 -0
- metaflow/cmd/make_wrapper.py +35 -3
- metaflow/extension_support/plugins.py +1 -0
- metaflow/metaflow_config.py +2 -0
- metaflow/metaflow_environment.py +3 -1
- metaflow/mflog/__init__.py +4 -3
- metaflow/plugins/__init__.py +14 -0
- metaflow/plugins/argo/argo_client.py +9 -2
- metaflow/plugins/argo/argo_workflows.py +79 -28
- metaflow/plugins/argo/argo_workflows_cli.py +16 -25
- metaflow/plugins/argo/argo_workflows_deployer_objects.py +5 -2
- metaflow/plugins/cards/card_modules/main.js +52 -50
- metaflow/plugins/kubernetes/kubernetes_decorator.py +2 -1
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +2 -0
- metaflow/plugins/metadata_providers/service.py +16 -7
- metaflow/plugins/pypi/bootstrap.py +17 -26
- metaflow/plugins/pypi/conda_environment.py +8 -8
- metaflow/plugins/pypi/parsers.py +268 -0
- metaflow/plugins/pypi/utils.py +18 -0
- metaflow/runner/click_api.py +5 -1
- metaflow/runner/deployer.py +3 -2
- metaflow/version.py +1 -1
- {ob_metaflow-2.15.0.1.data → ob_metaflow-2.15.5.1.data}/data/share/metaflow/devtools/Makefile +36 -17
- {ob_metaflow-2.15.0.1.data → ob_metaflow-2.15.5.1.data}/data/share/metaflow/devtools/Tiltfile +29 -10
- ob_metaflow-2.15.5.1.dist-info/METADATA +87 -0
- {ob_metaflow-2.15.0.1.dist-info → ob_metaflow-2.15.5.1.dist-info}/RECORD +34 -32
- {ob_metaflow-2.15.0.1.dist-info → ob_metaflow-2.15.5.1.dist-info}/WHEEL +1 -1
- ob_metaflow-2.15.0.1.dist-info/METADATA +0 -94
- {ob_metaflow-2.15.0.1.data → ob_metaflow-2.15.5.1.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {ob_metaflow-2.15.0.1.dist-info → ob_metaflow-2.15.5.1.dist-info}/LICENSE +0 -0
- {ob_metaflow-2.15.0.1.dist-info → ob_metaflow-2.15.5.1.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.15.0.1.dist-info → ob_metaflow-2.15.5.1.dist-info}/top_level.txt +0 -0
{ob_metaflow-2.15.0.1.data → ob_metaflow-2.15.5.1.data}/data/share/metaflow/devtools/Makefile
RENAMED
|
@@ -8,6 +8,7 @@ help:
|
|
|
8
8
|
@echo " ui - Open Metaflow UI"
|
|
9
9
|
@echo " dashboard - Open Minikube dashboard"
|
|
10
10
|
@echo " down - Stop and clean up the environment"
|
|
11
|
+
@echo " all-up - Start the development environment with all services"
|
|
11
12
|
@echo " help - Show this help message"
|
|
12
13
|
|
|
13
14
|
HELM_VERSION := v3.14.0
|
|
@@ -21,13 +22,14 @@ DEVTOOLS_DIR := $(MKFILE_DIR).devtools
|
|
|
21
22
|
PICK_SERVICES := $(MKFILE_DIR)pick_services.sh
|
|
22
23
|
MINIKUBE_DIR := $(DEVTOOLS_DIR)/minikube
|
|
23
24
|
MINIKUBE := $(MINIKUBE_DIR)/minikube
|
|
25
|
+
HELM_DIR := $(DEVTOOLS_DIR)/helm
|
|
24
26
|
TILT_DIR := $(DEVTOOLS_DIR)/tilt
|
|
25
27
|
TILT := $(TILT_DIR)/tilt
|
|
26
28
|
TILTFILE := $(MKFILE_DIR)/Tiltfile
|
|
27
|
-
MAKE_CMD := $(MAKE) -
|
|
29
|
+
MAKE_CMD := $(MAKE) -f "$(MKFILE_PATH)"
|
|
28
30
|
|
|
29
31
|
MINIKUBE_CPUS ?= 4
|
|
30
|
-
MINIKUBE_MEMORY ?=
|
|
32
|
+
MINIKUBE_MEMORY ?= 6144
|
|
31
33
|
MINIKUBE_DISK_SIZE ?= 20g
|
|
32
34
|
|
|
33
35
|
ifeq ($(shell uname), Darwin)
|
|
@@ -50,9 +52,18 @@ endif
|
|
|
50
52
|
|
|
51
53
|
install-helm:
|
|
52
54
|
@if ! command -v helm >/dev/null 2>&1; then \
|
|
53
|
-
echo "📥 Installing Helm $(HELM_VERSION)
|
|
54
|
-
|
|
55
|
+
echo "📥 Installing Helm $(HELM_VERSION)..."; \
|
|
56
|
+
mkdir -p "$(HELM_DIR)"; \
|
|
57
|
+
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 \
|
|
58
|
+
| HELM_INSTALL_VERSION="$(HELM_VERSION)" \
|
|
59
|
+
USE_SUDO="false" \
|
|
60
|
+
PATH="$(HELM_DIR):$$PATH" \
|
|
61
|
+
HELM_INSTALL_DIR="$(HELM_DIR)" \
|
|
62
|
+
bash; \
|
|
63
|
+
chmod +x "$(HELM_DIR)/helm"; \
|
|
55
64
|
echo "✅ Helm installation complete"; \
|
|
65
|
+
else \
|
|
66
|
+
echo "✅ Helm is already installed at $$(command -v helm)"; \
|
|
56
67
|
fi
|
|
57
68
|
|
|
58
69
|
check-docker:
|
|
@@ -98,7 +109,7 @@ install-gum:
|
|
|
98
109
|
@if ! command -v gum >/dev/null 2>&1; then \
|
|
99
110
|
echo "📥 Installing gum..."; \
|
|
100
111
|
if [ "$(shell uname)" = "Darwin" ]; then \
|
|
101
|
-
HOMEBREW_NO_AUTO_UPDATE=1 brew install gum; \
|
|
112
|
+
HOMEBREW_NO_AUTO_UPDATE=1 brew install gum|| { echo "❌ Failed to install gum via Homebrew"; exit 1; }; \
|
|
102
113
|
elif command -v apt-get >/dev/null 2>&1; then \
|
|
103
114
|
curl -fsSL -o /tmp/gum.deb \
|
|
104
115
|
"https://github.com/charmbracelet/gum/releases/download/$(GUM_VERSION)/gum_$(GUM_VERSION:v%=%)_$(arch).deb"; \
|
|
@@ -129,7 +140,8 @@ setup-minikube:
|
|
|
129
140
|
--cpus $(MINIKUBE_CPUS) \
|
|
130
141
|
--memory $(MINIKUBE_MEMORY) \
|
|
131
142
|
--disk-size $(MINIKUBE_DISK_SIZE) \
|
|
132
|
-
--driver docker
|
|
143
|
+
--driver docker \
|
|
144
|
+
|| { echo "❌ Failed to start Minikube (check if Docker is running)"; exit 1; }; \
|
|
133
145
|
echo "🔌 Enabling metrics-server and dashboard (quietly)..."; \
|
|
134
146
|
$(MINIKUBE) addons enable metrics-server >/dev/null 2>&1; \
|
|
135
147
|
$(MINIKUBE) addons enable dashboard >/dev/null 2>&1; \
|
|
@@ -170,7 +182,6 @@ up: install-brew check-docker install-curl install-gum setup-minikube install-he
|
|
|
170
182
|
@echo 'trap "exit" INT TERM' >> $(DEVTOOLS_DIR)/start.sh
|
|
171
183
|
@echo 'trap "kill 0" EXIT' >> $(DEVTOOLS_DIR)/start.sh
|
|
172
184
|
@echo 'eval $$($(MINIKUBE) docker-env)' >> $(DEVTOOLS_DIR)/start.sh
|
|
173
|
-
|
|
174
185
|
@echo 'if [ -n "$$SERVICES_OVERRIDE" ]; then' >> "$(DEVTOOLS_DIR)/start.sh"
|
|
175
186
|
@echo ' echo "🌐 Using user-provided list of services: $$SERVICES_OVERRIDE"' >> "$(DEVTOOLS_DIR)/start.sh"
|
|
176
187
|
@echo ' SERVICES="$$SERVICES_OVERRIDE"' >> "$(DEVTOOLS_DIR)/start.sh"
|
|
@@ -181,16 +192,19 @@ up: install-brew check-docker install-curl install-gum setup-minikube install-he
|
|
|
181
192
|
@echo 'PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" $(MINIKUBE) tunnel &' >> $(DEVTOOLS_DIR)/start.sh
|
|
182
193
|
@echo 'echo -e "🚀 Starting Tilt with selected services..."' >> $(DEVTOOLS_DIR)/start.sh
|
|
183
194
|
@echo 'echo -e "\033[1;38;5;46m\n🔥 \033[1;38;5;196mNext Steps:\033[0;38;5;46m Use \033[3mmetaflow-dev shell\033[23m to switch to the development\n environment'\''s shell and start executing your Metaflow flows.\n\033[0m"' >> "$(DEVTOOLS_DIR)/start.sh"
|
|
184
|
-
@echo 'PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" SERVICES="$$SERVICES" tilt up -f $(TILTFILE)' >> $(DEVTOOLS_DIR)/start.sh
|
|
195
|
+
@echo 'PATH="$(HELM_DIR):$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" SERVICES="$$SERVICES" tilt up -f $(TILTFILE)' >> $(DEVTOOLS_DIR)/start.sh
|
|
185
196
|
@echo 'wait' >> $(DEVTOOLS_DIR)/start.sh
|
|
186
197
|
@chmod +x $(DEVTOOLS_DIR)/start.sh
|
|
187
198
|
@$(DEVTOOLS_DIR)/start.sh
|
|
188
199
|
|
|
200
|
+
all-up:
|
|
201
|
+
@echo "🚀 Starting up all services..."
|
|
202
|
+
SERVICES_OVERRIDE=all $(MAKE_CMD) up
|
|
203
|
+
|
|
189
204
|
down:
|
|
190
205
|
@echo "🛑 Stopping all services..."
|
|
191
206
|
@-pkill -f "$(MINIKUBE) tunnel" 2>/dev/null || true
|
|
192
207
|
@echo "⏹️ Stopping Tilt..."
|
|
193
|
-
-PATH="$(MINIKUBE_DIR):$(TILT_DIR):$$PATH" tilt down -f $(TILTFILE)
|
|
194
208
|
@echo "🧹 Cleaning up Minikube..."
|
|
195
209
|
$(MAKE_CMD) teardown-minikube
|
|
196
210
|
@echo "🗑️ Removing Tilt binary and directory..."
|
|
@@ -201,7 +215,7 @@ down:
|
|
|
201
215
|
|
|
202
216
|
shell: setup-tilt
|
|
203
217
|
@echo "⏳ Checking if development environment is up..."
|
|
204
|
-
@set -
|
|
218
|
+
@set -eu; \
|
|
205
219
|
for i in $$(seq 1 90); do \
|
|
206
220
|
if "$(TILT)" get session >/dev/null 2>&1; then \
|
|
207
221
|
found_session=1; \
|
|
@@ -210,7 +224,7 @@ shell: setup-tilt
|
|
|
210
224
|
sleep 2; \
|
|
211
225
|
fi; \
|
|
212
226
|
done; \
|
|
213
|
-
if [ -z "$${found_session}" ]; then \
|
|
227
|
+
if [ -z "$${found_session:-}" ]; then \
|
|
214
228
|
echo "❌ Development environment is not up."; \
|
|
215
229
|
echo " Please run 'metaflow-dev up' in another terminal, then re-run 'metaflow-dev shell'."; \
|
|
216
230
|
exit 1; \
|
|
@@ -220,7 +234,10 @@ shell: setup-tilt
|
|
|
220
234
|
"$(TILT)" get uiresource generate-configs >/dev/null 2>&1; \
|
|
221
235
|
status=$$?; \
|
|
222
236
|
if [ $$status -eq 0 ]; then \
|
|
223
|
-
"$(TILT)" wait --for=condition=Ready uiresource/generate-configs; \
|
|
237
|
+
if ! "$(TILT)" wait --for=condition=Ready uiresource/generate-configs --timeout=300s; then \
|
|
238
|
+
echo "❌ Timed out waiting for development environment to be ready."; \
|
|
239
|
+
exit 1; \
|
|
240
|
+
fi; \
|
|
224
241
|
break; \
|
|
225
242
|
elif [ $$status -eq 127 ]; then \
|
|
226
243
|
echo "❌ Development environment is not up."; \
|
|
@@ -250,8 +267,7 @@ shell: setup-tilt
|
|
|
250
267
|
"$$user_shell" -i; \
|
|
251
268
|
fi'
|
|
252
269
|
|
|
253
|
-
|
|
254
|
-
#
|
|
270
|
+
|
|
255
271
|
# @echo '$(MAKE_CMD) create-dev-shell' >> $(DEVTOOLS_DIR)/start.sh
|
|
256
272
|
# @echo 'rm -f /tmp/metaflow-devshell-*' >> $(DEVTOOLS_DIR)/start.sh
|
|
257
273
|
create-dev-shell: setup-tilt
|
|
@@ -268,7 +284,10 @@ create-dev-shell: setup-tilt
|
|
|
268
284
|
echo "fi" >> $$SHELL_PATH && \
|
|
269
285
|
echo "" >> $$SHELL_PATH && \
|
|
270
286
|
echo "echo \"⏳ Waiting for development environment to be ready...\"" >> $$SHELL_PATH && \
|
|
271
|
-
echo "$(TILT) wait --for=condition=Ready uiresource/generate-configs" >> $$SHELL_PATH && \
|
|
287
|
+
echo "if ! $(TILT) wait --for=condition=Ready uiresource/generate-configs --timeout=300s; then" >> $$SHELL_PATH && \
|
|
288
|
+
echo " echo \"❌ Timed out waiting for development environment to be ready.\"" >> $$SHELL_PATH && \
|
|
289
|
+
echo " exit 1" >> $$SHELL_PATH && \
|
|
290
|
+
echo "fi" >> $$SHELL_PATH && \
|
|
272
291
|
echo "" >> $$SHELL_PATH && \
|
|
273
292
|
echo "echo \"🔧 Starting a new shell for development environment...\"" >> $$SHELL_PATH && \
|
|
274
293
|
echo "if [ -n \"\$$SHELL\" ]; then" >> $$SHELL_PATH && \
|
|
@@ -318,6 +337,6 @@ ui: setup-tilt
|
|
|
318
337
|
@echo "🔗 Opening Metaflow UI at http://localhost:3000"
|
|
319
338
|
@open http://localhost:3000
|
|
320
339
|
|
|
321
|
-
.PHONY: install-helm setup-minikube setup-tilt teardown-minikube tunnel up down check-docker install-curl install-gum install-brew up down dashboard shell ui help
|
|
340
|
+
.PHONY: install-helm setup-minikube setup-tilt teardown-minikube tunnel up down check-docker install-curl install-gum install-brew up down dashboard shell ui all-up help
|
|
322
341
|
|
|
323
|
-
.DEFAULT_GOAL :=
|
|
342
|
+
.DEFAULT_GOAL := help
|
{ob_metaflow-2.15.0.1.data → ob_metaflow-2.15.5.1.data}/data/share/metaflow/devtools/Tiltfile
RENAMED
|
@@ -23,8 +23,13 @@ components = {
|
|
|
23
23
|
"argo-events": ["argo-workflows"],
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
services_env = os.getenv("SERVICES", "").strip().lower()
|
|
27
|
+
|
|
28
|
+
if services_env:
|
|
29
|
+
if services_env == "all":
|
|
30
|
+
requested_components = list(components.keys())
|
|
31
|
+
else:
|
|
32
|
+
requested_components = services_env.split(",")
|
|
28
33
|
else:
|
|
29
34
|
requested_components = list(components.keys())
|
|
30
35
|
|
|
@@ -78,7 +83,7 @@ for component in requested_components:
|
|
|
78
83
|
if result not in enabled_components:
|
|
79
84
|
enabled_components.append(result)
|
|
80
85
|
|
|
81
|
-
# Print a friendly summary when running `tilt up`.
|
|
86
|
+
# Print a friendly summary when running `tilt up`.
|
|
82
87
|
if config.tilt_subcommand == 'up':
|
|
83
88
|
print("\n📦 Components to install:")
|
|
84
89
|
for component in enabled_components:
|
|
@@ -99,6 +104,7 @@ if "minio" in enabled_components:
|
|
|
99
104
|
set=[
|
|
100
105
|
'rootUser=rootuser',
|
|
101
106
|
'rootPassword=rootpass123',
|
|
107
|
+
# TODO: perturb the bucket name to avoid conflicts
|
|
102
108
|
'buckets[0].name=metaflow-test',
|
|
103
109
|
'buckets[0].policy=none',
|
|
104
110
|
'buckets[0].purge=false',
|
|
@@ -498,13 +504,14 @@ if "argo-events" in enabled_components:
|
|
|
498
504
|
'argo-events-controller-manager',
|
|
499
505
|
labels=['argo-events'],
|
|
500
506
|
)
|
|
501
|
-
|
|
507
|
+
|
|
502
508
|
metaflow_config["METAFLOW_ARGO_EVENTS_EVENT"] = "metaflow-event"
|
|
503
509
|
metaflow_config["METAFLOW_ARGO_EVENTS_EVENT_BUS"] = "default"
|
|
504
510
|
metaflow_config["METAFLOW_ARGO_EVENTS_EVENT_SOURCE"] = "argo-events-webhook"
|
|
505
511
|
metaflow_config["METAFLOW_ARGO_EVENTS_SERVICE_ACCOUNT"] = "operate-workflow-sa"
|
|
506
512
|
metaflow_config["METAFLOW_ARGO_EVENTS_WEBHOOK_AUTH"] = "service"
|
|
507
|
-
metaflow_config["
|
|
513
|
+
metaflow_config["METAFLOW_ARGO_EVENTS_INTERNAL_WEBHOOK_URL"] = "http://argo-events-webhook-eventsource-svc:12000/metaflow-event"
|
|
514
|
+
metaflow_config["METAFLOW_ARGO_EVENTS_WEBHOOK_URL"] = "http://localhost:12000/metaflow-event"
|
|
508
515
|
|
|
509
516
|
config_resources.append('argo-events-controller-manager')
|
|
510
517
|
config_resources.append('argo-events-webhook-eventsource-svc')
|
|
@@ -522,8 +529,8 @@ if "metadata-service" in enabled_components:
|
|
|
522
529
|
'metadatadb.password=metaflow123',
|
|
523
530
|
'metadatadb.database=metaflow',
|
|
524
531
|
'metadatadb.host=postgresql',
|
|
525
|
-
'image.repository=public.ecr.aws/
|
|
526
|
-
'image.tag=2.4.13-
|
|
532
|
+
'image.repository=public.ecr.aws/outerbounds/metaflow_metadata_service',
|
|
533
|
+
'image.tag=2.4.13-2-g70af4ed',
|
|
527
534
|
'resources.requests.cpu=25m',
|
|
528
535
|
'resources.requests.memory=64Mi',
|
|
529
536
|
'resources.limits.cpu=50m',
|
|
@@ -560,8 +567,8 @@ if "ui" in enabled_components:
|
|
|
560
567
|
'uiBackend.metadatadb.host=postgresql',
|
|
561
568
|
'uiBackend.metaflowDatastoreSysRootS3=s3://metaflow-test',
|
|
562
569
|
'uiBackend.metaflowS3EndpointURL=http://minio.default.svc.cluster.local:9000',
|
|
563
|
-
'uiBackend.image.name=public.ecr.aws/
|
|
564
|
-
'uiBackend.image.tag=2.4.13-
|
|
570
|
+
'uiBackend.image.name=public.ecr.aws/outerbounds/metaflow_metadata_service',
|
|
571
|
+
'uiBackend.image.tag=2.4.13-2-g70af4ed',
|
|
565
572
|
'uiBackend.env[0].name=AWS_ACCESS_KEY_ID',
|
|
566
573
|
'uiBackend.env[0].value=rootuser',
|
|
567
574
|
'uiBackend.env[1].name=AWS_SECRET_ACCESS_KEY',
|
|
@@ -600,8 +607,20 @@ if "ui" in enabled_components:
|
|
|
600
607
|
config_resources.append('metaflow-ui')
|
|
601
608
|
config_resources.append('metaflow-ui-static')
|
|
602
609
|
|
|
610
|
+
cmd = '''
|
|
611
|
+
ARCH=$(kubectl get nodes -o jsonpath='{.items[0].status.nodeInfo.architecture}')
|
|
612
|
+
case "$ARCH" in
|
|
613
|
+
arm64) echo linux-aarch64 ;;
|
|
614
|
+
amd64) echo linux-64 ;;
|
|
615
|
+
*) echo linux-64 ;;
|
|
616
|
+
esac
|
|
617
|
+
'''
|
|
618
|
+
|
|
619
|
+
# For @conda/@pypi emulation
|
|
620
|
+
metaflow_config["METAFLOW_KUBERNETES_CONDA_ARCH"] = str(local(cmd)).strip()
|
|
621
|
+
|
|
603
622
|
local_resource(
|
|
604
623
|
name="generate-configs",
|
|
605
624
|
cmd=write_config_files(),
|
|
606
625
|
resource_deps=config_resources,
|
|
607
|
-
)
|
|
626
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: ob-metaflow
|
|
3
|
+
Version: 2.15.5.1
|
|
4
|
+
Summary: Metaflow: More AI and ML, Less Engineering
|
|
5
|
+
Author: Netflix, Outerbounds & the Metaflow Community
|
|
6
|
+
Author-email: help@outerbounds.co
|
|
7
|
+
License: Apache License 2.0
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: requests
|
|
11
|
+
Requires-Dist: boto3
|
|
12
|
+
Requires-Dist: pylint
|
|
13
|
+
Requires-Dist: kubernetes
|
|
14
|
+
Provides-Extra: stubs
|
|
15
|
+
Requires-Dist: metaflow-stubs==2.15.5.1; extra == "stubs"
|
|
16
|
+
Dynamic: author
|
|
17
|
+
Dynamic: author-email
|
|
18
|
+
Dynamic: description
|
|
19
|
+
Dynamic: description-content-type
|
|
20
|
+
Dynamic: license
|
|
21
|
+
Dynamic: provides-extra
|
|
22
|
+
Dynamic: requires-dist
|
|
23
|
+
Dynamic: summary
|
|
24
|
+
|
|
25
|
+

|
|
26
|
+
|
|
27
|
+
# Metaflow
|
|
28
|
+
|
|
29
|
+
[Metaflow](https://metaflow.org) is a human-centric framework designed to help scientists and engineers **build and manage real-life AI and ML systems**. Serving teams of all sizes and scale, Metaflow streamlines the entire development lifecycle—from rapid prototyping in notebooks to reliable, maintainable production deployments—enabling teams to iterate quickly and deliver robust systems efficiently.
|
|
30
|
+
|
|
31
|
+
Originally developed at [Netflix](https://netflixtechblog.com/open-sourcing-metaflow-a-human-centric-framework-for-data-science-fa72e04a5d9) and now supported by [Outerbounds](https://outerbounds.com), Metaflow is designed to boost the productivity for research and engineering teams working on [a wide variety of projects](https://netflixtechblog.com/supporting-diverse-ml-systems-at-netflix-2d2e6b6d205d), from classical statistics to state-of-the-art deep learning and foundation models. By unifying code, data, and compute at every stage, Metaflow ensures seamless, end-to-end management of real-world AI and ML systems.
|
|
32
|
+
|
|
33
|
+
Today, Metaflow powers thousands of AI and ML experiences across a diverse array of companies, large and small, including Amazon, Doordash, Dyson, Goldman Sachs, Ramp, and [many others](ADOPTERS.md). At Netflix alone, Metaflow supports over 3000 AI and ML projects, executes hundreds of millions of data-intensive high-performance compute jobs processing petabytes of data and manages tens of petabytes of models and artifacts for hundreds of users across its AI, ML, data science, and engineering teams.
|
|
34
|
+
|
|
35
|
+
## From prototype to production (and back)
|
|
36
|
+
|
|
37
|
+
Metaflow provides a simple and friendly pythonic [API](https://docs.metaflow.org) that covers foundational needs of AI and ML systems:
|
|
38
|
+
<img src="./docs/prototype-to-prod.png" width="800px">
|
|
39
|
+
|
|
40
|
+
1. [Rapid local prototyping](https://docs.metaflow.org/metaflow/basics), [support for notebooks](https://docs.metaflow.org/metaflow/managing-flows/notebook-runs), and built-in support for [experiment tracking, versioning](https://docs.metaflow.org/metaflow/client) and [visualization](https://docs.metaflow.org/metaflow/visualizing-results).
|
|
41
|
+
2. [Effortlessly scale horizontally and vertically in your cloud](https://docs.metaflow.org/scaling/remote-tasks/introduction), utilizing both CPUs and GPUs, with [fast data access](https://docs.metaflow.org/scaling/data) for running [massive embarrassingly parallel](https://docs.metaflow.org/metaflow/basics#foreach) as well as [gang-scheduled](https://docs.metaflow.org/scaling/remote-tasks/distributed-computing) compute workloads [reliably](https://docs.metaflow.org/scaling/failures) and [efficiently](https://docs.metaflow.org/scaling/checkpoint/introduction).
|
|
42
|
+
3. [Easily manage dependencies](https://docs.metaflow.org/scaling/dependencies) and [deploy with one-click](https://docs.metaflow.org/production/introduction) to highly available production orchestrators with built in support for [reactive orchestration](https://docs.metaflow.org/production/event-triggering).
|
|
43
|
+
|
|
44
|
+
For full documentation, check out our [API Reference](https://docs.metaflow.org/api) or see our [Release Notes](https://github.com/Netflix/metaflow/releases) for the latest features and improvements.
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
## Getting started
|
|
48
|
+
|
|
49
|
+
Getting up and running is easy. If you don't know where to start, [Metaflow sandbox](https://outerbounds.com/sandbox) will have you running and exploring in seconds.
|
|
50
|
+
|
|
51
|
+
### Installing Metaflow
|
|
52
|
+
|
|
53
|
+
To install Metaflow in your Python environment from [PyPI](https://pypi.org/project/metaflow/):
|
|
54
|
+
|
|
55
|
+
```sh
|
|
56
|
+
pip install metaflow
|
|
57
|
+
```
|
|
58
|
+
Alternatively, using [conda-forge](https://anaconda.org/conda-forge/metaflow):
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
conda install -c conda-forge metaflow
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Once installed, a great way to get started is by following our [tutorial](https://docs.metaflow.org/getting-started/tutorials). It walks you through creating and running your first Metaflow flow step by step.
|
|
65
|
+
|
|
66
|
+
For more details on Metaflow’s features and best practices, check out:
|
|
67
|
+
- [How Metaflow works](https://docs.metaflow.org/metaflow/basics)
|
|
68
|
+
- [Additional resources](https://docs.metaflow.org/introduction/metaflow-resources)
|
|
69
|
+
|
|
70
|
+
If you need help, don’t hesitate to reach out on our [Slack community](http://slack.outerbounds.co/)!
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
### Deploying infrastructure for Metaflow in your cloud
|
|
74
|
+
<img src="./docs/multicloud.png" width="800px">
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
While you can get started with Metaflow easily on your laptop, the main benefits of Metaflow lie in its ability to [scale out to external compute clusters](https://docs.metaflow.org/scaling/remote-tasks/introduction)
|
|
78
|
+
and to [deploy to production-grade workflow orchestrators](https://docs.metaflow.org/production/introduction). To benefit from these features, follow this [guide](https://outerbounds.com/engineering/welcome/) to
|
|
79
|
+
configure Metaflow and the infrastructure behind it appropriately.
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
## Get in touch
|
|
83
|
+
We'd love to hear from you. Join our community [Slack workspace](http://slack.outerbounds.co/)!
|
|
84
|
+
|
|
85
|
+
## Contributing
|
|
86
|
+
We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
|
|
87
|
+
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
|
2
|
-
metaflow/__init__.py,sha256=
|
|
2
|
+
metaflow/__init__.py,sha256=cuNzIhOJ9ivGXgTGMuaVfdhtWpUMXbnsINVahBk4O4E,6306
|
|
3
3
|
metaflow/cards.py,sha256=IbRmredvmFEU0V6JL7DR8wCESwVmmZJubr6x24bo7U4,442
|
|
4
4
|
metaflow/cli.py,sha256=RU-yXpT-Lfl3xGyFNtL742e9KEqcRxEnQ-4mwXrXhvo,20928
|
|
5
5
|
metaflow/cli_args.py,sha256=hDsdWdRmfXYifVGq6b6FDfgoWxtIG2nr_lU6EBV0Pnk,3584
|
|
@@ -16,10 +16,10 @@ metaflow/includefile.py,sha256=kWKDSlzVcRVNGG9PV5eB3o2ynrzqhVsfaLtkqjshn7Q,20948
|
|
|
16
16
|
metaflow/info_file.py,sha256=wtf2_F0M6dgiUu74AFImM8lfy5RrUw5Yj7Rgs2swKRY,686
|
|
17
17
|
metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
|
|
18
18
|
metaflow/lint.py,sha256=x4p6tnRzYqNNniCGXyrUW0WuYfTUgnaOMRivxvnxask,11661
|
|
19
|
-
metaflow/metaflow_config.py,sha256=
|
|
19
|
+
metaflow/metaflow_config.py,sha256=i-7nDYXWTF9fTgtlD_AavTFsNfUma-y4Z0DFKp6ybkM,23732
|
|
20
20
|
metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
|
|
21
21
|
metaflow/metaflow_current.py,sha256=pfkXmkyHeMJhxIs6HBJNBEaBDpcl5kz9Wx5mW6F_3qo,7164
|
|
22
|
-
metaflow/metaflow_environment.py,sha256=
|
|
22
|
+
metaflow/metaflow_environment.py,sha256=e5BOkA7VdpjseI4HUkm_pR74NVJRNADL20LIQL4W1vU,8139
|
|
23
23
|
metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
|
|
24
24
|
metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,7491
|
|
25
25
|
metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
|
|
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
|
36
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
|
37
37
|
metaflow/util.py,sha256=mJBkV5tShIyCsLDeM1zygQGeciQVMrVPm_qI8Oi33G0,14656
|
|
38
38
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
|
39
|
-
metaflow/version.py,sha256=
|
|
39
|
+
metaflow/version.py,sha256=939d3EZaeQalkOTgDK0URMj9N-aHGtFRQyyWj9y33ks,30
|
|
40
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
|
41
41
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
|
42
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
|
@@ -121,12 +121,13 @@ metaflow/client/core.py,sha256=Cca6HbK-UBO72aELfFJxsl85ylYHHlCAd-uJP-lEepQ,83689
|
|
|
121
121
|
metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
|
|
122
122
|
metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
123
123
|
metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
|
|
124
|
-
metaflow/cmd/main_cli.py,sha256=
|
|
125
|
-
metaflow/cmd/make_wrapper.py,sha256=
|
|
124
|
+
metaflow/cmd/main_cli.py,sha256=LSehmMjkWojAN1XTtqW6S51ZpGNAdW4_VK5S7qH8-Ts,2982
|
|
125
|
+
metaflow/cmd/make_wrapper.py,sha256=N8L4u8QZAryH0sAjRsdEqG-gTj2S4LUsfDizOemrTR0,1604
|
|
126
126
|
metaflow/cmd/tutorials_cmd.py,sha256=8FdlKkicTOhCIDKcBR5b0Oz6giDvS-EMY3o9skIrRqw,5156
|
|
127
127
|
metaflow/cmd/util.py,sha256=jS_0rUjOnGGzPT65fzRLdGjrYAOOLA4jU2S0HJLV0oc,406
|
|
128
|
+
metaflow/cmd/code/__init__.py,sha256=VO4dNM9M9LHYy5nTgEiJvCV1RBl8lpDlYGJm6GIcaBA,7413
|
|
128
129
|
metaflow/cmd/develop/__init__.py,sha256=p1Sy8yU1MEKSrH5ttOWOZvNcI1qYu6J6jghdTHwPgOw,689
|
|
129
|
-
metaflow/cmd/develop/stub_generator.py,sha256=
|
|
130
|
+
metaflow/cmd/develop/stub_generator.py,sha256=bo2yWe0kvCZ-3arEFe9eAnPN-h8oNNPcQsDwsL350UM,65217
|
|
130
131
|
metaflow/cmd/develop/stubs.py,sha256=JX2qNZDvG0upvPueAcLhoR_zyLtRranZMwY05tLdpRQ,11884
|
|
131
132
|
metaflow/datastore/__init__.py,sha256=VxP6ddJt3rwiCkpiSfAhyVkUCOe1pgZZsytVEJzFmSQ,155
|
|
132
133
|
metaflow/datastore/content_addressed_store.py,sha256=6T7tNqL29kpmecyMLHF35RhoSBOb-OZcExnsB65AvnI,7641
|
|
@@ -140,17 +141,17 @@ metaflow/extension_support/__init__.py,sha256=2z0c4R8zsVmEFOMGT2Jujsl6xveDVa9KLl
|
|
|
140
141
|
metaflow/extension_support/_empty_file.py,sha256=HENjnM4uAfeNygxMB_feCCWORFoSat9n_QwzSx2oXPw,109
|
|
141
142
|
metaflow/extension_support/cmd.py,sha256=hk8iBUUINqvKCDxInKgWpum8ThiRZtHSJP7qBASHzl8,5711
|
|
142
143
|
metaflow/extension_support/integrations.py,sha256=AWAh-AZ-vo9IxuAVEjGw3s8p_NMm2DKHYx10oC51gPU,5506
|
|
143
|
-
metaflow/extension_support/plugins.py,sha256=
|
|
144
|
+
metaflow/extension_support/plugins.py,sha256=gl7NbIJLJyLTb5LELsj1D9paQip6t6Lqz6Rhmvqvyrw,11286
|
|
144
145
|
metaflow/metadata_provider/__init__.py,sha256=FZNSnz26VB_m18DQG8mup6-Gfl7r1U6lRMljJBp3VAM,64
|
|
145
146
|
metaflow/metadata_provider/heartbeat.py,sha256=42mQo6wOHdFuaCh426uV6Kn8swe7e5I3gqA_G7cI_LA,3127
|
|
146
147
|
metaflow/metadata_provider/metadata.py,sha256=meO4Fhxu7tbMUGwasYb9_AtL06fwrrXKKjIK7KRWZDs,27093
|
|
147
148
|
metaflow/metadata_provider/util.py,sha256=lYoQKbqoTM1iZChgyVWN-gX-HyM9tt9bXEMJexY9XmM,1723
|
|
148
|
-
metaflow/mflog/__init__.py,sha256=
|
|
149
|
+
metaflow/mflog/__init__.py,sha256=TkR9ny_JYvNCWJTdLiHsbLSLc9cUvzAzpDuHLdG8nkA,6020
|
|
149
150
|
metaflow/mflog/mflog.py,sha256=VebXxqitOtNAs7VJixnNfziO_i_urG7bsJ5JiB5IXgY,4370
|
|
150
151
|
metaflow/mflog/save_logs.py,sha256=4p1OwozsHJBslOzAf0wUq2XPMNpEOZWM68MgWzh_jJY,2330
|
|
151
152
|
metaflow/mflog/save_logs_periodically.py,sha256=2Uvk9hi-zlCqXxOQoXmmjH1SCugfw6eG6w70WgfI-ho,1256
|
|
152
153
|
metaflow/mflog/tee.py,sha256=wTER15qeHuiRpCkOqo-bd-r3Gj-EVlf3IvWRCA4beW4,887
|
|
153
|
-
metaflow/plugins/__init__.py,sha256=
|
|
154
|
+
metaflow/plugins/__init__.py,sha256=siqE9Zj_b9zKgMhll3f5L2m1gcAKxp_e4qMRTGJ65xY,8460
|
|
154
155
|
metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn2GUev0,4061
|
|
155
156
|
metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
|
|
156
157
|
metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
|
|
@@ -180,13 +181,13 @@ metaflow/plugins/airflow/sensors/base_sensor.py,sha256=s-OQBfPWZ_T3wn96Ua59CCEj1
|
|
|
180
181
|
metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=zhYlrZnXT20KW8-fVk0fCNtTyNiKJB5PMVASacu30r0,6034
|
|
181
182
|
metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqNA90nARrjjjEEk_x4,3275
|
|
182
183
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
183
|
-
metaflow/plugins/argo/argo_client.py,sha256=
|
|
184
|
+
metaflow/plugins/argo/argo_client.py,sha256=A1kI9rjVjCadDsBscZ2Wk8xRBI6GNgWV6SU7TyrdfrI,16530
|
|
184
185
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
|
185
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
|
186
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
|
186
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=TBo6hoFa-XWB8kA4K6rEDVzngFKWbQUkq7z5NFviMzM,185425
|
|
187
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=27eLtcp5N5plapP-uIJqR41B0zDfXOV39AGM0nchymo,37952
|
|
187
188
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=ogCSBmwsC2C3eusydrgjuAJd4qK18f1sI4jJwA4Fd-o,7800
|
|
188
189
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
|
189
|
-
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=
|
|
190
|
+
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=oumYqwW7CsIsUuCIoP7Eo6VGB5R126CAqcCq2M57Cxw,14072
|
|
190
191
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
|
191
192
|
metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
|
|
192
193
|
metaflow/plugins/argo/jobset_input_paths.py,sha256=-h0E_e0w6FMiBUod9Rf_XOSCtZv_C0exacw4q1SfIfg,501
|
|
@@ -237,7 +238,7 @@ metaflow/plugins/cards/card_modules/bundle.css,sha256=ms2wOKftlPM_i6bC_4BkrmqCOj
|
|
|
237
238
|
metaflow/plugins/cards/card_modules/card.py,sha256=6sbqP5mwf7QWvQvX2N_bC78H9ixuI5sQ8612Q5islys,4627
|
|
238
239
|
metaflow/plugins/cards/card_modules/components.py,sha256=Y-Yo7UgSOyTB3zs-wDfUqUKl0PI_BzeY1QGcy2fqE2M,26752
|
|
239
240
|
metaflow/plugins/cards/card_modules/convert_to_native_type.py,sha256=eGaQ8BabJ489a8AiyhXFy1pWJZl99gH0FQzwsNVxAGQ,15782
|
|
240
|
-
metaflow/plugins/cards/card_modules/main.js,sha256=
|
|
241
|
+
metaflow/plugins/cards/card_modules/main.js,sha256=72mKcMEjrne7TlLxc-LkxTJVoM4z2pLWGtU1NzHtlWM,1043628
|
|
241
242
|
metaflow/plugins/cards/card_modules/renderer_tools.py,sha256=uiTdKHWFInWgtfWArOUSQLnTwqVd4hAw49jfOfg8rGA,1642
|
|
242
243
|
metaflow/plugins/cards/card_modules/test_cards.py,sha256=74H0JdWrcoc5eITxCLSUwwa1kJZ3YMteynaKsCBihXE,5361
|
|
243
244
|
metaflow/plugins/cards/card_modules/chevron/__init__.py,sha256=SicpH_1eT76HUTA8aMuiGLRNKTqgYD1Qfutt2NdTL0E,156
|
|
@@ -292,29 +293,30 @@ metaflow/plugins/kubernetes/kube_utils.py,sha256=jdFMGbEmIow-oli26v31W9CmbZXigx0
|
|
|
292
293
|
metaflow/plugins/kubernetes/kubernetes.py,sha256=4WHVs421w5JfFSRAdpiZ6X9w7xEK8UPYGNcc9e0JOFc,30420
|
|
293
294
|
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=PY-aMaVelHddHq3jqEEu9cvNl7xEjT2lFFADN9dXMkw,13918
|
|
294
295
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
|
|
295
|
-
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=
|
|
296
|
+
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=vGzVmVsaid2sgM1w9NIWqrK2aCpBe9K4aYfr2r4KzIM,31736
|
|
296
297
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=0PjcVgJkmSqjBNEkibk8y0xxBrgu99D8ar9RsXnzCeY,32833
|
|
297
|
-
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=
|
|
298
|
+
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=XjduAYY_H-jX3lGK6b-jYt6QnDvC2ac3qIAgr-iDsCQ,42543
|
|
298
299
|
metaflow/plugins/kubernetes/spot_metadata_cli.py,sha256=an0nWCxgflmqIPBCBrlb4m3DereDFFJBLt-KKhqcHc8,1670
|
|
299
300
|
metaflow/plugins/kubernetes/spot_monitor_sidecar.py,sha256=zrWU-smQwPnL6MBHmzTxWyEA00R6iKKQbhhy50xFwQ8,3832
|
|
300
301
|
metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
301
302
|
metaflow/plugins/metadata_providers/local.py,sha256=Z0CXaGZJbAkj4II3WspJi-uCCtShH64yaXZQ5i9Ym7g,24390
|
|
302
|
-
metaflow/plugins/metadata_providers/service.py,sha256=
|
|
303
|
+
metaflow/plugins/metadata_providers/service.py,sha256=9j0db_EOGFdb49YTgr9q4EWqVAm1YUaW-Baj5gyRqIo,22809
|
|
303
304
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
|
304
|
-
metaflow/plugins/pypi/bootstrap.py,sha256=
|
|
305
|
+
metaflow/plugins/pypi/bootstrap.py,sha256=SNONquX6QnTbu7htmhaQeVeZ2ofaFaUCDScRIrTTERc,14718
|
|
305
306
|
metaflow/plugins/pypi/conda_decorator.py,sha256=piFcE4uGmWhhbGlxMK0GHd7BGEyqy6r9BFy8Mjoi80Q,15937
|
|
306
|
-
metaflow/plugins/pypi/conda_environment.py,sha256=
|
|
307
|
+
metaflow/plugins/pypi/conda_environment.py,sha256=kQeX1cHJZk1oY4d-5bH8fhVrzNaFBvOCQgaszNAa5ew,22459
|
|
307
308
|
metaflow/plugins/pypi/micromamba.py,sha256=LLJ2dGGOEyld07W8iI6dtE01h2Y1PQnBhU-dMBssZ3c,16502
|
|
309
|
+
metaflow/plugins/pypi/parsers.py,sha256=gpOOG2Ph95wI73MWCAi7XjpK0gYhv5k5YIGBs73QPuE,8556
|
|
308
310
|
metaflow/plugins/pypi/pip.py,sha256=H0cIy8odpZ-JTn4SwF0b74tuC3uRU7X8TdAQJ2kODG8,13971
|
|
309
311
|
metaflow/plugins/pypi/pypi_decorator.py,sha256=ybNgo-T5Z_0W2KNuED0pdjyI0qygZ4a1MXAzKqdHt_E,7250
|
|
310
312
|
metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
|
|
311
|
-
metaflow/plugins/pypi/utils.py,sha256=
|
|
313
|
+
metaflow/plugins/pypi/utils.py,sha256=glfXN0Do8HleB8YE9LE9Pb1tIBivqdPGmS0sUtO1e7k,3451
|
|
312
314
|
metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
|
|
313
315
|
metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
|
|
314
316
|
metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
|
|
315
317
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
316
|
-
metaflow/runner/click_api.py,sha256=
|
|
317
|
-
metaflow/runner/deployer.py,sha256=
|
|
318
|
+
metaflow/runner/click_api.py,sha256=4GbLSTA0Ju2kfjcslNn4yUOzGFcoMPFK00N_Q-q3IuY,23359
|
|
319
|
+
metaflow/runner/deployer.py,sha256=6ixXonNyLB1dfTNl1HVGVT5M8CybXDUa3oFTabn9Sp8,9099
|
|
318
320
|
metaflow/runner/deployer_impl.py,sha256=Kab9rLoA3EiBJDtTTulhPCeKzqiljW366nx2Tm0LYy0,6143
|
|
319
321
|
metaflow/runner/metaflow_runner.py,sha256=L302ew_BPBPs-NnW8n92dqqbqmHwrwGL5D6kTZvl5vY,16074
|
|
320
322
|
metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
|
|
@@ -361,12 +363,12 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
|
361
363
|
metaflow/user_configs/config_decorators.py,sha256=qCKVAvd0NKgaCxQ2OThes5-DYHXq6A1HqURubYNeFdw,20481
|
|
362
364
|
metaflow/user_configs/config_options.py,sha256=m6jccSpzI4qUJ7vyYkYBIf8G3V0Caunxg_k7zg4Zlqg,21067
|
|
363
365
|
metaflow/user_configs/config_parameters.py,sha256=oeJGVKu1ao_YQX6Lg6P2FEv5k5-_F4sARLlVpTW9ezM,15502
|
|
364
|
-
ob_metaflow-2.15.
|
|
365
|
-
ob_metaflow-2.15.
|
|
366
|
-
ob_metaflow-2.15.
|
|
367
|
-
ob_metaflow-2.15.
|
|
368
|
-
ob_metaflow-2.15.
|
|
369
|
-
ob_metaflow-2.15.
|
|
370
|
-
ob_metaflow-2.15.
|
|
371
|
-
ob_metaflow-2.15.
|
|
372
|
-
ob_metaflow-2.15.
|
|
366
|
+
ob_metaflow-2.15.5.1.data/data/share/metaflow/devtools/Makefile,sha256=l0dYh6E_qdNjda6mwEHm5isoG9wsFh77i3CZQ5P53ZQ,13665
|
|
367
|
+
ob_metaflow-2.15.5.1.data/data/share/metaflow/devtools/Tiltfile,sha256=P5_rn_F3xYLN1_cEAQ9mNeS22HG2rb8beKIz2RIK6fU,20634
|
|
368
|
+
ob_metaflow-2.15.5.1.data/data/share/metaflow/devtools/pick_services.sh,sha256=DCnrMXwtApfx3B4S-YiZESMyAFHbXa3VuNL0MxPLyiE,2196
|
|
369
|
+
ob_metaflow-2.15.5.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
|
370
|
+
ob_metaflow-2.15.5.1.dist-info/METADATA,sha256=cBdX7LRaXzST0EUj56Fe_X1AsSG7Klo0AY5wNTx9kqg,5913
|
|
371
|
+
ob_metaflow-2.15.5.1.dist-info/WHEEL,sha256=SrDKpSbFN1G94qcmBqS9nyHcDMp9cUS9OC06hC0G3G0,109
|
|
372
|
+
ob_metaflow-2.15.5.1.dist-info/entry_points.txt,sha256=RvEq8VFlgGe_FfqGOZi0D7ze1hLD0pAtXeNyGfzc_Yc,103
|
|
373
|
+
ob_metaflow-2.15.5.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
|
374
|
+
ob_metaflow-2.15.5.1.dist-info/RECORD,,
|
|
@@ -1,94 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.2
|
|
2
|
-
Name: ob-metaflow
|
|
3
|
-
Version: 2.15.0.1
|
|
4
|
-
Summary: Metaflow: More AI and ML, Less Engineering
|
|
5
|
-
Author: Netflix, Outerbounds & the Metaflow Community
|
|
6
|
-
Author-email: help@outerbounds.co
|
|
7
|
-
License: Apache License 2.0
|
|
8
|
-
Description-Content-Type: text/markdown
|
|
9
|
-
License-File: LICENSE
|
|
10
|
-
Requires-Dist: requests
|
|
11
|
-
Requires-Dist: boto3
|
|
12
|
-
Requires-Dist: pylint
|
|
13
|
-
Requires-Dist: kubernetes
|
|
14
|
-
Provides-Extra: stubs
|
|
15
|
-
Requires-Dist: metaflow-stubs==2.15.0.1; extra == "stubs"
|
|
16
|
-
Dynamic: author
|
|
17
|
-
Dynamic: author-email
|
|
18
|
-
Dynamic: description
|
|
19
|
-
Dynamic: description-content-type
|
|
20
|
-
Dynamic: license
|
|
21
|
-
Dynamic: provides-extra
|
|
22
|
-
Dynamic: requires-dist
|
|
23
|
-
Dynamic: summary
|
|
24
|
-
|
|
25
|
-

|
|
26
|
-
|
|
27
|
-
# Metaflow
|
|
28
|
-
|
|
29
|
-
Metaflow is a human-friendly library that helps scientists and engineers build and manage real-life data science projects. Metaflow was [originally developed at Netflix](https://netflixtechblog.com/open-sourcing-metaflow-a-human-centric-framework-for-data-science-fa72e04a5d9) to boost productivity of data scientists who work on a wide variety of projects from classical statistics to state-of-the-art deep learning.
|
|
30
|
-
|
|
31
|
-
For more information, see [Metaflow's website](https://metaflow.org) and [documentation](https://docs.metaflow.org).
|
|
32
|
-
|
|
33
|
-
## From prototype to production (and back)
|
|
34
|
-
|
|
35
|
-
Metaflow provides a simple, friendly API that covers foundational needs of ML, AI, and data science projects:
|
|
36
|
-
<img src="./docs/prototype-to-prod.png" width="800px">
|
|
37
|
-
|
|
38
|
-
1. [Rapid local prototyping](https://docs.metaflow.org/metaflow/basics), [support for notebooks](https://docs.metaflow.org/metaflow/visualizing-results), and [built-in experiment tracking and versioning](https://docs.metaflow.org/metaflow/client).
|
|
39
|
-
2. [Horizontal and vertical scalability to the cloud](https://docs.metaflow.org/scaling/remote-tasks/introduction), utilizing both CPUs and GPUs, and [fast data access](https://docs.metaflow.org/scaling/data).
|
|
40
|
-
3. [Managing dependencies](https://docs.metaflow.org/scaling/dependencies) and [one-click deployments to highly available production orchestrators](https://docs.metaflow.org/production/introduction).
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
## Getting started
|
|
44
|
-
|
|
45
|
-
Getting up and running is easy. If you don't know where to start, [Metaflow sandbox](https://outerbounds.com/sandbox) will have you running and exploring Metaflow in seconds.
|
|
46
|
-
|
|
47
|
-
### Installing Metaflow in your Python environment
|
|
48
|
-
|
|
49
|
-
To install Metaflow in your local environment, you can install from [PyPi](https://pypi.org/project/metaflow/):
|
|
50
|
-
|
|
51
|
-
```sh
|
|
52
|
-
pip install metaflow
|
|
53
|
-
```
|
|
54
|
-
Alternatively, you can also install from [conda-forge](https://anaconda.org/conda-forge/metaflow):
|
|
55
|
-
|
|
56
|
-
```sh
|
|
57
|
-
conda install -c conda-forge metaflow
|
|
58
|
-
```
|
|
59
|
-
If you are eager to try out Metaflow in practice, you can start with the [tutorial](https://docs.metaflow.org/getting-started/tutorials). After the tutorial, you can learn more about how Metaflow works [here](https://docs.metaflow.org/metaflow/basics).
|
|
60
|
-
|
|
61
|
-
### Deploying infrastructure for Metaflow in your cloud
|
|
62
|
-
<img src="./docs/multicloud.png" width="800px">
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
While you can get started with Metaflow easily on your laptop, the main benefits of Metaflow lie in its ability to [scale out to external compute clusters](https://docs.metaflow.org/scaling/remote-tasks/introduction)
|
|
66
|
-
and to [deploy to production-grade workflow orchestrators](https://docs.metaflow.org/production/introduction). To benefit from these features, follow this [guide](https://outerbounds.com/engineering/welcome/) to
|
|
67
|
-
configure Metaflow and the infrastructure behind it appropriately.
|
|
68
|
-
|
|
69
|
-
## [Resources](https://docs.metaflow.org/introduction/metaflow-resources)
|
|
70
|
-
|
|
71
|
-
### [Slack Community](http://slack.outerbounds.co/)
|
|
72
|
-
An active [community](http://slack.outerbounds.co/) of thousands of data scientists and ML engineers discussing the ins-and-outs of applied machine learning.
|
|
73
|
-
|
|
74
|
-
### [Tutorials](https://outerbounds.com/docs/tutorials-index/)
|
|
75
|
-
- [Introduction to Metaflow](https://outerbounds.com/docs/intro-tutorial-overview/)
|
|
76
|
-
- [Natural Language Processing with Metaflow](https://outerbounds.com/docs/nlp-tutorial-overview/)
|
|
77
|
-
- [Computer Vision with Metaflow](https://outerbounds.com/docs/cv-tutorial-overview/)
|
|
78
|
-
- [Recommender Systems with Metaflow](https://outerbounds.com/docs/recsys-tutorial-overview/)
|
|
79
|
-
- And more advanced content [here](https://outerbounds.com/docs/tutorials-index/)
|
|
80
|
-
|
|
81
|
-
### [Generative AI and LLM use cases](https://outerbounds.com/blog/?category=Foundation%20Models)
|
|
82
|
-
- [Infrastructure Stack for Large Language Models](https://outerbounds.com/blog/llm-infrastructure-stack/)
|
|
83
|
-
- [Parallelizing Stable Diffusion for Production Use Cases](https://outerbounds.com/blog/parallelizing-stable-diffusion-production-use-cases/)
|
|
84
|
-
- [Whisper with Metaflow on Kubernetes](https://outerbounds.com/blog/whisper-kubernetes/)
|
|
85
|
-
- [Training a Large Language Model With Metaflow, Featuring Dolly](https://outerbounds.com/blog/train-dolly-metaflow/)
|
|
86
|
-
|
|
87
|
-
## Get in touch
|
|
88
|
-
There are several ways to get in touch with us:
|
|
89
|
-
- [Slack Community](http://slack.outerbounds.co/)
|
|
90
|
-
- [Github Issues](https://github.com/Netflix/metaflow/issues)
|
|
91
|
-
|
|
92
|
-
## Contributing
|
|
93
|
-
We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details.
|
|
94
|
-
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|