omni-reason 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omni_reason-1.0.1/PKG-INFO +48 -0
- omni_reason-1.0.1/README.md +232 -0
- omni_reason-1.0.1/omni_reason/__init__.py +19 -0
- omni_reason-1.0.1/omni_reason/_vendor/LICENSE-LTX +381 -0
- omni_reason-1.0.1/omni_reason/_vendor/NOTICE.md +17 -0
- omni_reason-1.0.1/omni_reason/_vendor/__init__.py +1 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/__init__.py +1 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/guidance/__init__.py +15 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/guidance/perturbations.py +79 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/__init__.py +56 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/fuse_loras.py +178 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/helpers.py +61 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/module_ops.py +14 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/primitives.py +166 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/registry.py +84 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/sd_ops.py +145 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/sft_loader.py +66 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/loader/single_gpu_model_builder.py +236 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/__init__.py +8 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/common/__init__.py +9 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/common/normalization.py +59 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/model_protocol.py +39 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/__init__.py +22 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/adaln.py +45 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/attention.py +542 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/feed_forward.py +15 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/gelu_approx.py +10 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/modality.py +69 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/model.py +523 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/model_configurator.py +218 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/ops.py +106 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/rope.py +222 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/text_projection.py +38 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/timestep_embedding.py +143 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/transformer.py +442 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/transformer/transformer_args.py +351 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/__init__.py +25 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/convolution.py +317 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/enums.py +20 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/memory_efficient_decode.py +662 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/model_configurator.py +79 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/ops.py +82 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/resnet.py +277 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/sampling.py +123 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/tiling.py +69 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/model/video_vae/video_vae.py +1112 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/__init__.py +1 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/__init__.py +33 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/config.py +75 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/embeddings_connector.py +250 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/embeddings_processor.py +116 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/encoders/__init__.py +1 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/encoders/base_encoder.py +212 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/encoders/encoder_configurator.py +264 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/feature_extractor.py +129 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/text_encoders/gemma/tokenizer.py +71 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/tiling.py +464 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/types.py +211 -0
- omni_reason-1.0.1/omni_reason/_vendor/ltx/utils.py +62 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/__init__.py +1 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/data/__init__.py +28 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/data/encode.py +104 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/data/media.py +173 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/data/packing.py +583 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/inference/__init__.py +5 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/inference/pipeline.py +230 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/inference/sample.py +111 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/__init__.py +10 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/diffusion.py +160 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/model.py +210 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/structural_embedding.py +87 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/text_connector.py +19 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/text_pad.py +151 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/text_refine.py +44 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/__init__.py +26 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/adaln.py +44 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/attention.py +34 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/patchify.py +44 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/rope.py +61 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/timestep.py +37 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/transformer/transformer.py +229 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/vae/__init__.py +3 -0
- omni_reason-1.0.1/omni_reason/_vendor/reason/model/vae/video_vae.py +45 -0
- omni_reason-1.0.1/omni_reason/assets.py +141 -0
- omni_reason-1.0.1/omni_reason/cli.py +173 -0
- omni_reason-1.0.1/omni_reason/components.py +71 -0
- omni_reason-1.0.1/omni_reason/diagnostics.py +47 -0
- omni_reason-1.0.1/omni_reason/media_plan.py +24 -0
- omni_reason-1.0.1/omni_reason/neuron/__init__.py +2 -0
- omni_reason-1.0.1/omni_reason/neuron/cfg_parallel.py +72 -0
- omni_reason-1.0.1/omni_reason/neuron/compute.py +82 -0
- omni_reason-1.0.1/omni_reason/neuron/conditioning.py +112 -0
- omni_reason-1.0.1/omni_reason/neuron/engine.py +127 -0
- omni_reason-1.0.1/omni_reason/neuron/interface.py +54 -0
- omni_reason-1.0.1/omni_reason/neuron/progress.py +25 -0
- omni_reason-1.0.1/omni_reason/neuron/resident.py +39 -0
- omni_reason-1.0.1/omni_reason/neuron/runner.py +302 -0
- omni_reason-1.0.1/omni_reason/neuron/sequence_parallel.py +69 -0
- omni_reason-1.0.1/omni_reason/neuron/topology.py +78 -0
- omni_reason-1.0.1/omni_reason/notices.py +38 -0
- omni_reason-1.0.1/omni_reason/optimization.py +194 -0
- omni_reason-1.0.1/omni_reason/pipeline.py +219 -0
- omni_reason-1.0.1/omni_reason/resources/assets.json +394 -0
- omni_reason-1.0.1/omni_reason/resources/examples.json +36 -0
- omni_reason-1.0.1/omni_reason/resources/licenses/GEMMA-PROHIBITED-USE.txt +66 -0
- omni_reason-1.0.1/omni_reason/resources/licenses/LICENSE-APACHE-2.0.txt +202 -0
- omni_reason-1.0.1/omni_reason/resources/licenses/LICENSE-GEMMA.txt +158 -0
- omni_reason-1.0.1/omni_reason/resources/licenses/LICENSE-LTX.txt +381 -0
- omni_reason-1.0.1/omni_reason/resources/licenses/NOTICE.txt +30 -0
- omni_reason-1.0.1/omni_reason/resources/licenses.json +35 -0
- omni_reason-1.0.1/omni_reason/resources/weights.json +11 -0
- omni_reason-1.0.1/omni_reason/settings.py +24 -0
- omni_reason-1.0.1/omni_reason.egg-info/PKG-INFO +48 -0
- omni_reason-1.0.1/omni_reason.egg-info/SOURCES.txt +128 -0
- omni_reason-1.0.1/omni_reason.egg-info/dependency_links.txt +1 -0
- omni_reason-1.0.1/omni_reason.egg-info/entry_points.txt +2 -0
- omni_reason-1.0.1/omni_reason.egg-info/requires.txt +15 -0
- omni_reason-1.0.1/omni_reason.egg-info/top_level.txt +1 -0
- omni_reason-1.0.1/pyproject.toml +51 -0
- omni_reason-1.0.1/setup.cfg +4 -0
- omni_reason-1.0.1/tests/test_cli.py +192 -0
- omni_reason-1.0.1/tests/test_media_plan.py +30 -0
- omni_reason-1.0.1/tests/test_neuron_compute.py +49 -0
- omni_reason-1.0.1/tests/test_neuron_health.py +92 -0
- omni_reason-1.0.1/tests/test_neuron_interface.py +60 -0
- omni_reason-1.0.1/tests/test_notices.py +51 -0
- omni_reason-1.0.1/tests/test_optimization.py +56 -0
- omni_reason-1.0.1/tests/test_pipeline.py +81 -0
- omni_reason-1.0.1/tests/test_readonly_assets.py +22 -0
- omni_reason-1.0.1/tests/test_request_compute.py +47 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: omni-reason
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: Omni-Reason video task inference
|
|
5
|
+
Author: Omni-Reason Team
|
|
6
|
+
License: Original contributions: Apache-2.0; inherited LTX components: LTX-2 Community License
|
|
7
|
+
Project-URL: Homepage, https://omni-reason.com/
|
|
8
|
+
Project-URL: Model, https://huggingface.co/Omni-Reason/Omni-Reason-1
|
|
9
|
+
Project-URL: Hosted API and CLI, https://platform.omni-reason.com/docs
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: omni_reason/resources/licenses/GEMMA-PROHIBITED-USE.txt
|
|
13
|
+
License-File: omni_reason/resources/licenses/LICENSE-APACHE-2.0.txt
|
|
14
|
+
License-File: omni_reason/resources/licenses/LICENSE-GEMMA.txt
|
|
15
|
+
License-File: omni_reason/resources/licenses/LICENSE-LTX.txt
|
|
16
|
+
License-File: omni_reason/resources/licenses/NOTICE.txt
|
|
17
|
+
License-File: omni_reason/_vendor/NOTICE.md
|
|
18
|
+
Requires-Dist: diffusers==0.36.0
|
|
19
|
+
Requires-Dist: transformers==4.57.6
|
|
20
|
+
Requires-Dist: huggingface-hub==0.36.2
|
|
21
|
+
Requires-Dist: safetensors==0.7.0
|
|
22
|
+
Requires-Dist: accelerate==1.12.0
|
|
23
|
+
Requires-Dist: torch>=2.6
|
|
24
|
+
Requires-Dist: torchvision
|
|
25
|
+
Requires-Dist: einops
|
|
26
|
+
Requires-Dist: numpy
|
|
27
|
+
Requires-Dist: pillow
|
|
28
|
+
Requires-Dist: PyYAML
|
|
29
|
+
Requires-Dist: av
|
|
30
|
+
Requires-Dist: sentencepiece
|
|
31
|
+
Requires-Dist: pydantic
|
|
32
|
+
Requires-Dist: rich
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# Omni-Reason model runtime
|
|
36
|
+
|
|
37
|
+
Run Omni-Reason-1 on your own CUDA or Trainium hardware: provide a video, an
|
|
38
|
+
instruction and optional references, then generate a video answer. This code
|
|
39
|
+
package contains the runtime, not the 13B model weights or hardware drivers.
|
|
40
|
+
|
|
41
|
+
Start with the model card and its hardware-specific setup guides:
|
|
42
|
+
https://huggingface.co/Omni-Reason/Omni-Reason-1.
|
|
43
|
+
Prepare that environment before installing this package. The optional
|
|
44
|
+
Omni-Reason client exposes the same runner as `omni-reason model`.
|
|
45
|
+
|
|
46
|
+
Original contributions use Apache-2.0. Inherited LTX code and model components
|
|
47
|
+
retain their conditions; bundled license files and modification notices describe
|
|
48
|
+
the scope. Installing this package does not grant hosted service or data access.
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# Omni-Reason-1
|
|
2
|
+
|
|
3
|
+
**“Provide a video, an instruction and optional references; get a video answer on your own hardware.”**
|
|
4
|
+
|
|
5
|
+
Omni-Reason-1 is a **13B video-to-video model**. Download its complete components
|
|
6
|
+
from [Hugging Face](https://huggingface.co/Omni-Reason/Omni-Reason-1).
|
|
7
|
+
Both Trainium and CUDA use the same released components and example inputs.
|
|
8
|
+
Choose one environment below; no Data or Benchmark download is needed.
|
|
9
|
+
|
|
10
|
+
Prefer an installed command? The [terminal guide](../docs/cli.md) explains
|
|
11
|
+
`omni-reason model` for local inference and `omni-reason tv2v` for hosted use.
|
|
12
|
+
Both keep their own runtime and access requirements; the source commands below
|
|
13
|
+
remain supported.
|
|
14
|
+
|
|
15
|
+
## Choose how to run
|
|
16
|
+
|
|
17
|
+
| Your setup | Start here | First result |
|
|
18
|
+
| --- | --- | --- |
|
|
19
|
+
| Trainium2 host with Neuron driver and Docker | [Trainium tutorial](trainium.md) | A generated MP4, its input/settings record and an execution receipt |
|
|
20
|
+
| NVIDIA GPU, at least 48 GB VRAM and 64 GiB host RAM for the examples | [CUDA tutorial](cuda.md) | A generated MP4 and its input/settings record |
|
|
21
|
+
| Browser, without your own accelerator | [Platform preview](#platform-preview) | Generate and download videos after invitation registration |
|
|
22
|
+
|
|
23
|
+
The Trainium wrapper selects `--backend trainium` explicitly;
|
|
24
|
+
the local `python omni model` command defaults to CUDA.
|
|
25
|
+
Trainium names the hardware; Neuron is its runtime, not a separate model.
|
|
26
|
+
|
|
27
|
+
<a id="studio-preview"></a>
|
|
28
|
+
|
|
29
|
+
## Platform preview
|
|
30
|
+
|
|
31
|
+
**[Omni-Reason Platform](https://platform.omni-reason.com/) — Research Invitation Only.**
|
|
32
|
+
The hosted service is free and for research use only.
|
|
33
|
+
[Request an access code ↗](https://platform.omni-reason.com/request-access),
|
|
34
|
+
then register and sign in.
|
|
35
|
+
Upload a video, enter an instruction and optionally add a reference; submit,
|
|
36
|
+
follow the queue, then preview and download the generated video.
|
|
37
|
+
|
|
38
|
+
Web, API and CLI share your account and private Gallery. The public
|
|
39
|
+
[API and CLI guide](https://platform.omni-reason.com/docs) covers authentication,
|
|
40
|
+
submission and download commands. Access and usage are subject to available
|
|
41
|
+
research capacity. For self-hosting, use the independent Trainium or CUDA
|
|
42
|
+
workflow under the model's component licenses.
|
|
43
|
+
|
|
44
|
+
To try the hosted model from a terminal, open the public developer guide,
|
|
45
|
+
sign in to download the client wheel, and use Python 3.11 or newer:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
python -m pip install omni_reason_client-1.0.1-py3-none-any.whl
|
|
49
|
+
omni-reason login
|
|
50
|
+
omni-reason example robot --output robot-result.mp4
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Approve the login in your browser. The example command obtains and checks its
|
|
54
|
+
input, submits the task, waits and downloads the video. The client needs no
|
|
55
|
+
local accelerator, model weights or PyTorch; it is distributed through Platform,
|
|
56
|
+
not PyPI. Use `omni-reason doctor --frames 25` to check availability without
|
|
57
|
+
submitting a task. After the example, use your own video:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
omni-reason tv2v "Your task instruction" --input input.mp4 --output answer.mp4
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Add `--reference reference.png` when needed. API examples, queue status, recovery
|
|
64
|
+
and download options are in the same developer guide.
|
|
65
|
+
|
|
66
|
+
## See the example
|
|
67
|
+
|
|
68
|
+
Try a real task: **highlight the visible robot in green while preserving the
|
|
69
|
+
surrounding scene and motion.** This Mini example, `tv2v:Task_264`, uses its
|
|
70
|
+
complete instruction. The model receives Input; Target is for comparison only.
|
|
71
|
+
|
|
72
|
+
<!-- media:robot-highlight:start -->
|
|
73
|
+
<a href="https://huggingface.co/Omni-Reason/Omni-Reason-1/resolve/main/media/showcase/robot-highlight.mp4"><picture><source media="(prefers-reduced-motion: reduce) and (prefers-color-scheme: dark)" srcset="../assets/showcase/robot-highlight-dark-poster.png"><source media="(prefers-reduced-motion: reduce)" srcset="../assets/showcase/robot-highlight-light-poster.png"><source media="(prefers-color-scheme: dark)" srcset="../assets/showcase/robot-highlight-dark.webp"><img src="../assets/showcase/robot-highlight-light.webp" alt="Follow the visible robot" width="100%"></picture></a>
|
|
74
|
+
|
|
75
|
+
[Media credits](../assets/CREDITS.md)
|
|
76
|
+
|
|
77
|
+
<!-- media:robot-highlight:end -->
|
|
78
|
+
|
|
79
|
+
The recorded example lasts 6.6 seconds. Both first-run tutorials generate a
|
|
80
|
+
shorter window: 25 frames at 24 fps, about one second.
|
|
81
|
+
[View the CUDA tutorial output](../assets/showcase/runtime-robot.mp4).
|
|
82
|
+
A longer input is resampled and cropped to the requested window; generation
|
|
83
|
+
does not automatically process its full duration.
|
|
84
|
+
|
|
85
|
+
## Try the reference example
|
|
86
|
+
|
|
87
|
+
Use `--example colorization` after the first run. It colorizes grayscale
|
|
88
|
+
*Big Buck Bunny* footage using an independent RGB reference frame.
|
|
89
|
+
This licensed-film demonstration is separate from the Benchmark.
|
|
90
|
+
|
|
91
|
+
<!-- media:runtime-colorization:start -->
|
|
92
|
+
<a href="https://huggingface.co/Omni-Reason/Omni-Reason-1/resolve/main/media/showcase/runtime-colorization.mp4"><picture><source media="(prefers-reduced-motion: reduce) and (prefers-color-scheme: dark)" srcset="../assets/showcase/runtime-colorization-dark-poster.png"><source media="(prefers-reduced-motion: reduce)" srcset="../assets/showcase/runtime-colorization-light-poster.png"><source media="(prefers-color-scheme: dark)" srcset="../assets/showcase/runtime-colorization-dark.webp"><img src="../assets/showcase/runtime-colorization-light.webp" alt="Try reference-guided colorization" width="100%"></picture></a>
|
|
93
|
+
|
|
94
|
+
[Media credits](../assets/CREDITS.md)
|
|
95
|
+
|
|
96
|
+
<!-- media:runtime-colorization:end -->
|
|
97
|
+
|
|
98
|
+
A completed command confirms that the model runs. Inspect the answer for task
|
|
99
|
+
correctness, scene detail and motion. [Paper results](../paper/README.md) describe
|
|
100
|
+
the fixed evaluation; [Benchmark](../benchmark/README.md) evaluates your own outputs.
|
|
101
|
+
|
|
102
|
+
## Continue with your own work
|
|
103
|
+
|
|
104
|
+
- **Own videos and references:** follow the custom-input step in your hardware tutorial.
|
|
105
|
+
- **Repeated requests:** use the [Trainium Engine](trainium.md#keep-the-model-loaded)
|
|
106
|
+
or the [CUDA Python pipeline](cuda.md#run-multiple-videos).
|
|
107
|
+
- **Faster execution:** consult [performance choices](performance.md) after the first run.
|
|
108
|
+
- **Your trained model:** [train and export](../training/README.md), then
|
|
109
|
+
[run the export on CUDA](cuda.md#use-a-model-you-trained).
|
|
110
|
+
|
|
111
|
+
Training and inference do not need to use the same hardware. The supported
|
|
112
|
+
fine-tuning workflow trains on Trainium and exports model components for CUDA
|
|
113
|
+
inference. This release does not supply GPU training. The Trainium inference
|
|
114
|
+
loader currently validates the published model's fixed component hashes;
|
|
115
|
+
custom training exports are supported by the CUDA loader only.
|
|
116
|
+
|
|
117
|
+
## Select independent media windows
|
|
118
|
+
|
|
119
|
+
For custom inputs on either backend, `--frames` controls the answer,
|
|
120
|
+
`--source-frames` controls the observed video, and `--reference-frames`
|
|
121
|
+
controls a video reference. Each frame count is `8n+1`; images retain one frame.
|
|
122
|
+
`--source-start` and `--reference-start` are integer indices on the 24-fps
|
|
123
|
+
timeline (24 means one second). Omitted lengths follow `--frames`, preserving
|
|
124
|
+
the original first-run example settings.
|
|
125
|
+
|
|
126
|
+
For example, add `--source-frames 25 --frames 49 --source-start 24` to a
|
|
127
|
+
custom-input command to condition a 49-frame answer on 25 observed frames
|
|
128
|
+
starting one second into the input. Describe the task, including any future
|
|
129
|
+
prediction, in the instruction. These arguments select media windows; they
|
|
130
|
+
do not classify or rewrite the task.
|
|
131
|
+
|
|
132
|
+
The CUDA Python API accepts the same four names with underscores; Trainium
|
|
133
|
+
Engine cases accept `source_frames`, `reference_frames`, `source_start`
|
|
134
|
+
and `reference_start`. The packed input, reference and answer must fit the
|
|
135
|
+
model's token buckets. At 256 × 448, each temporal latent frame occupies 112
|
|
136
|
+
tokens; the largest bucket is 3,696 tokens. If no reference is supplied, the
|
|
137
|
+
NULL reference occupies 112 tokens. Short source videos repeat their final
|
|
138
|
+
frame when the requested window extends beyond the available footage.
|
|
139
|
+
The hosted Platform validates bounded selections and reports their exact
|
|
140
|
+
timing before generation.
|
|
141
|
+
|
|
142
|
+
See the concrete window-selection examples in the [Trainium guide](trainium.md#select-input-and-reference-windows)
|
|
143
|
+
and [CUDA guide](cuda.md#select-input-and-reference-windows). The new window
|
|
144
|
+
controls have been exercised in eight full-model single-chip Trainium requests;
|
|
145
|
+
CUDA has shared conditioning and API tests, with no new GPU hardware run for
|
|
146
|
+
this addition. Existing backend acceptance remains recorded separately.
|
|
147
|
+
|
|
148
|
+
## Model size and downloads
|
|
149
|
+
|
|
150
|
+
| Item | Value |
|
|
151
|
+
| --- | --- |
|
|
152
|
+
| Generation transformer | **13B** — 13,124,889,728 parameters (approximately 13.125 billion) |
|
|
153
|
+
| Released precision | bfloat16 |
|
|
154
|
+
| Complete runtime download | **56.04 GB** — 56,040,245,137 bytes across 27 files |
|
|
155
|
+
|
|
156
|
+
The 13B label counts the generation transformer. Fine-tuning updates these
|
|
157
|
+
parameters; the checkpoint contains their trained values. The complete download
|
|
158
|
+
also includes the text encoder, video VAE, connectors, tokenizer and configuration
|
|
159
|
+
files. Component notices and example inputs are supplied separately.
|
|
160
|
+
|
|
161
|
+
## Use the S3 mirror
|
|
162
|
+
|
|
163
|
+
HF is the recommended model source. Its S3 mirror has identical component paths,
|
|
164
|
+
sizes and SHA-256; choose either source. Keep the model's `NOTICE` and `licenses/`
|
|
165
|
+
files with it; see [component terms](../LICENSES.md).
|
|
166
|
+
|
|
167
|
+
After obtaining [S3 access](../docs/access.md), install AWS CLI on the host and
|
|
168
|
+
prepare a small CPU download environment from the repository root:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
python3 -m venv work/download-env
|
|
172
|
+
work/download-env/bin/pip install boto3
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Choose the directory for your backend, then download:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
# Trainium:
|
|
179
|
+
model_assets="$PWD/work/trainium/model-assets"
|
|
180
|
+
# For CUDA, use this instead:
|
|
181
|
+
# model_assets="$PWD/work/model-assets"
|
|
182
|
+
|
|
183
|
+
work/download-env/bin/python omni fetch model/omni_reason/resources/assets.json \
|
|
184
|
+
--output "$model_assets" --profile omni-release --max-bytes 60000000000
|
|
185
|
+
aws s3 cp s3://omni-reason-1/releases/v1.0.0/models/omni-reason-1/examples/ \
|
|
186
|
+
"$model_assets/examples/" --recursive --exclude "*" \
|
|
187
|
+
--include "robot/*" --include "colorization-bbb/*" --profile omni-release
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
Use your granted profile; on an authorized EC2 role, omit both `--profile`
|
|
191
|
+
options. The model manifest pins 27 runtime files (56.04 GB) and five component
|
|
192
|
+
notices. The three example inputs total 6.49 MB and are verified against fixed
|
|
193
|
+
hashes by the runner before generation. Add `--offline` to the generation
|
|
194
|
+
command in your selected tutorial. Trainium sees its directory as
|
|
195
|
+
`/work/model-assets` inside the container.
|
|
196
|
+
|
|
197
|
+
## Code layout
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
model/
|
|
201
|
+
├── README.md Model entry, examples and shared asset access
|
|
202
|
+
├── trainium.md Complete Trainium first-run and Engine guide
|
|
203
|
+
├── cuda.md Complete CUDA first-run and Python guide
|
|
204
|
+
├── performance.md Optional modes, measurements and fidelity
|
|
205
|
+
├── cuda-requirements.txt CUDA dependencies
|
|
206
|
+
├── Dockerfile.neuron Separate Neuron environment
|
|
207
|
+
├── neuron.sh Container launcher with explicit chip assignment
|
|
208
|
+
├── pyproject.toml Installable model package
|
|
209
|
+
├── run.py Compatible direct CLI entry
|
|
210
|
+
├── omni_reason/
|
|
211
|
+
│ ├── cli.py Shared arguments and backend dispatch
|
|
212
|
+
│ ├── assets.py Verified downloads and local reuse
|
|
213
|
+
│ ├── settings.py Shared generation defaults
|
|
214
|
+
│ ├── pipeline.py CUDA pipeline and public Python API
|
|
215
|
+
│ ├── optimization.py Optional CUDA execution paths
|
|
216
|
+
│ ├── components.py Component save/load interfaces
|
|
217
|
+
│ ├── diagnostics.py CUDA environment checks
|
|
218
|
+
│ ├── notices.py Component agreements
|
|
219
|
+
│ ├── neuron/ Trainium runtime, Engine and chip allocation
|
|
220
|
+
│ ├── resources/ Fixed model/example manifests and notices
|
|
221
|
+
│ └── _vendor/ Required architecture and upstream helpers
|
|
222
|
+
└── tests/ CLI, assets and runtime contract tests
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Model owns inference; Platform consumes a fixed version of this package.
|
|
226
|
+
The Trainium container also installs `training/` at the same source revision
|
|
227
|
+
for shared weight-loading and parallel helpers. A CUDA install does not need it.
|
|
228
|
+
Keep those shared implementations in one place.
|
|
229
|
+
|
|
230
|
+
Use `python omni model`, the Python API or compatible `python model/run.py`.
|
|
231
|
+
Tests are not installed into the runtime package. After installing the selected
|
|
232
|
+
environment, run `python -m unittest discover -s model/tests -v`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Omni-Reason component model and reusable video inference pipeline."""
|
|
2
|
+
|
|
3
|
+
__all__ = ["OmniReasonPipeline", "OmniReasonOutput", "OmniReasonTransformer",
|
|
4
|
+
"OmniReasonVAE", "OmniReasonConnectors", "OmniReasonScheduler"]
|
|
5
|
+
|
|
6
|
+
def __getattr__(name):
|
|
7
|
+
if name in {"ModelMixin", "SchedulerMixin"}:
|
|
8
|
+
import diffusers
|
|
9
|
+
value = getattr(diffusers, name)
|
|
10
|
+
elif name in {"OmniReasonPipeline", "OmniReasonOutput"}:
|
|
11
|
+
from . import pipeline
|
|
12
|
+
value = getattr(pipeline, name)
|
|
13
|
+
elif name in __all__:
|
|
14
|
+
from . import components
|
|
15
|
+
value = getattr(components, name)
|
|
16
|
+
else:
|
|
17
|
+
raise AttributeError(name)
|
|
18
|
+
globals()[name] = value
|
|
19
|
+
return value
|