magic123 0.1.0__tar.gz

magic123-0.1.0/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

magic123-0.1.0/MANIFEST.in

@@ -0,0 +1,16 @@
+include LICENSE
+include readme.md
+include requirements.txt
+include requirements-torch201-cuda117.txt
+include setup_cuda_ext.py
+recursive-include assets *
+recursive-include data/demo *
+recursive-include docs/static *
+recursive-include pretrained/zero123 *.yaml
+recursive-include raymarching/src *.cpp *.cu *.h
+recursive-include shencoder/src *.cpp *.cu *.h
+recursive-include freqencoder/src *.cpp *.cu *.h
+recursive-include gridencoder/src *.cpp *.cu *.h
+recursive-include render/renderutils/c_src *.cpp *.cu *.h
+recursive-include scripts *.sh *.py
+global-exclude __pycache__ *.py[cod] .DS_Store

magic123-0.1.0/PKG-INFO

@@ -0,0 +1,426 @@
+Metadata-Version: 2.4
+Name: magic123
+Version: 0.1.0
+Summary: Magic123: one image to high-quality 3D object generation using 2D and 3D diffusion priors.
+Author: Guocheng Qian
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://guochengqian.github.io/project/magic123/
+Project-URL: Repository, https://github.com/guochengqian/Magic123
+Project-URL: Issues, https://github.com/guochengqian/Magic123/issues
+Keywords: 3d-generation,diffusion,nerf,cuda,pytorch
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: tqdm
+Requires-Dist: rich
+Requires-Dist: ninja
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: opencv-python
+Requires-Dist: imageio
+Requires-Dist: imageio-ffmpeg
+Requires-Dist: pandas
+Requires-Dist: torch
+Requires-Dist: torchvision
+Requires-Dist: torch-ema
+Requires-Dist: einops
+Requires-Dist: tensorboard
+Requires-Dist: tensorboardX
+Requires-Dist: dearpygui
+Requires-Dist: huggingface_hub
+Requires-Dist: diffusers>=0.9.0
+Requires-Dist: accelerate
+Requires-Dist: transformers
+Requires-Dist: xatlas
+Requires-Dist: PyMCubes
+Requires-Dist: rembg
+Requires-Dist: omegaconf
+Requires-Dist: pytorch-lightning
+Requires-Dist: taming-transformers-rom1504
+Requires-Dist: kornia
+Requires-Dist: gdown
+Requires-Dist: trimesh
+Requires-Dist: pymeshlab==2022.2.post4
+Requires-Dist: timm==0.6.7
+Requires-Dist: easydict
+Requires-Dist: wandb
+Requires-Dist: termcolor
+Requires-Dist: sentencepiece
+Requires-Dist: psutil
+Requires-Dist: lpips
+Provides-Extra: cuda
+Provides-Extra: dev
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# Magic123: One Image to High-Quality 3D Object Generation Using Both 2D and 3D Diffusion Priors [ICLR 2024]
+
+[arXiv](https://arxiv.org/abs/2306.17843) | [webpage](https://guochengqian.github.io/project/magic123/)
+
+<img src="docs/static/magic123.gif" width="800" />
+
+[Guocheng Qian](https://guochengqian.github.io/) <sup>1,2</sup>, [Jinjie Mai](https://cemse.kaust.edu.sa/people/person/jinjie-mai) <sup>1</sup>, [Abdullah Hamdi](https://abdullahamdi.com/) <sup>3</sup>, [Jian Ren](https://alanspike.github.io/) <sup>2</sup>, [Aliaksandr Siarohin](https://aliaksandrsiarohin.github.io/aliaksandr-siarohin-website/) <sup>2</sup>, [Bing Li](https://cemse.kaust.edu.sa/people/person/bing-li) <sup>1</sup>, [Hsin-Ying Lee](http://hsinyinglee.com/) <sup>2</sup>, [Ivan Skorokhodov](https://universome.github.io/) <sup>1,2</sup>, [Peter Wonka](https://peterwonka.net/) <sup>1</sup>, [Sergey Tulyakov](http://www.stulyakov.com/) <sup>2</sup>, [Bernard Ghanem](https://www.bernardghanem.com/) <sup>1</sup>
+
+<sup>1</sup> [King Abdullah University of Science and Technology (KAUST)](https://www.kaust.edu.sa/),
+<sup>2</sup> [Snap Inc.](https://www.snap.com/),
+<sup>3</sup> [Visual Geometry Group, University of Oxford](http://www.robots.ox.ac.uk/~vgg/)
+
+
+Training convergence of a demo example:
+<img src="docs/static/ironman-val-magic123.gif" width="800" />
+
+
+Compare Magic123 without textual inversion with abaltions using only 2D prior (SDS) or using only 3D prior (Zero123):
+
+
+https://github.com/guochengqian/Magic123/assets/48788073/c91f4c81-8c2c-4f84-8ce1-420c12f7e886
+
+
+Effects of Joint Prior. Increasing the strength of 2D prior leads to more imagination, more details, and less 3D consistencies. 
+
+<img src="docs/static/2d_3d.png" width="800" />
+
+
+
+
+https://github.com/guochengqian/Magic123/assets/48788073/98cb4dd7-7bf3-4179-9b6d-e8b47d928a68
+
+
+Official PyTorch Implementation of Magic123: One Image to High-Quality 3D Object Generation Using Both 2D and 3D Diffusion Priors. Code is built upon [Stable-DreamFusion](https://github.com/ashawkey/stable-dreamfusion) repo.
+
+
+# NEWS:
+- [2024/01/16] Magic123 gets accepted to ICLR24
+- [2023/07/25] Code is available at [GitHub](https://github.com/guochengqian/Magic123) 
+- [2023/07/03] Paper is available at [arXiv](https://arxiv.org/abs/2306.17843) 
+- [2023/06/25] Much better performance than the submitted version is achieved by 1）reimplementing Magic123 using [Stable DreamFusion code](https://github.com/ashawkey/stable-dreamfusion), 2）fixing some gradient issues, 3）leveraging the [tricks](#tips-and-tricks)
+- [2023] Initial version of Magic123 submitted to conference
+
+
+# Install
+We only test on Ubuntu system. Make sure git, wget, Eigen are installed.  
+```
+apt update && apt upgrade
+apt install git wget libeigen3-dev python3-dev python3-venv -y
+```
+
+### Install from PyPI
+
+```bash
+python -m venv venv_magic123
+source venv_magic123/bin/activate
+python -m pip install --upgrade pip setuptools wheel
+python -m pip install magic123
+
+# Build the CUDA extensions after installing PyTorch. 4090/4090 Ti uses SM 8.9.
+export CUDA_HOME=${CUDA_HOME:-/usr/local/cuda-12.9}
+export PATH=$CUDA_HOME/bin:$PATH
+export TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST:-8.9}
+git clone https://github.com/guochengqian/Magic123.git
+cd Magic123
+bash scripts/install_ext.sh
+```
+
+The PyPI package ships the Python project and source code for the CUDA extensions. The extension modules are compiled locally because PyTorch CUDA extension wheels must match the local Python, PyTorch, CUDA toolkit, compiler, and GPU architecture.
+
+### CUDA and Windows notes
+
+- NVIDIA RTX 4090 and 4090 Ti builds should set `TORCH_CUDA_ARCH_LIST=8.9`.
+- The CUDA toolkit must provide `nvcc`; check with `python scripts/check_cuda_build_env.py`.
+- Building the PyTorch CUDA extensions also requires Python development headers. On Ubuntu/Debian install `python3-dev` or the version-specific package such as `python3.10-dev`.
+- On Windows, install Visual Studio Build Tools with the "Desktop development with C++" workload and run installation from a Developer PowerShell or Developer Command Prompt. The setup scripts use `vswhere`/Visual Studio discovery to locate `cl.exe`, but CUDA still requires a Visual Studio version supported by the installed CUDA toolkit.
+- On Colab, install PyTorch and the matching CUDA toolkit first, then run `bash scripts/install_ext.sh`. If Colab changes its compiler or CUDA image, rebuild the extensions with `--no-cache-dir`.
+
+### Install Environment 
+
+```bash
+source install.sh
+```
+Note: in this `install.sh`, we use python venv by default. If you prefer conda, uncomment the conda and comment venv in the file and run the same command. 
+
+
+### Download pre-trained models
+
+* [Zero-1-to-3](https://github.com/cvlab-columbia/zero123) for 3D diffusion prior.
+    We use `105000.ckpt` by default, reimplementation borrowed from Stable Diffusion repo, and is available in `guidance/zero123_utils.py`.
+    ```bash
+    cd pretrained/zero123
+    wget https://huggingface.co/cvlab/zero123-weights/resolve/main/105000.ckpt
+    cd ../../
+    ```
+
+* [MiDaS](https://github.com/isl-org/MiDaS) for depth estimation.
+    We use `dpt_beit_large_512.pt`. Put it in folder `pretrained/midas/`
+    ```bash
+    mkdir -p pretrained/midas
+    cd pretrained/midas
+    wget https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt
+    cd ../../
+    ```
+
+# Usage
+## Preprocess [Optional]
+We have included all preprocessed files in `./data` directory. Preprocessing is only necessary if you want to test on your own examples. Takes seconds.  
+
+### Step1: Extract depth 
+```
+python preprocess_image.py --path /path/to/image 
+```
+
+
+### Step 2: textual inversion [Optional]
+Magic123 uses the default [textual inversion](https://huggingface.co/docs/diffusers/training/text_inversion) from diffuers, which consumes around 2 hours on a 32G V100. If you do not want to spend time in this textual inversion, you can: (1) study whether there is other faster textual inversion; or (2) do not use textual inversion in the loss of texture and shape consistencies.  To run textual inversion: 
+
+```
+bash scripts/textual_inversion/textual_inversion.sh $GPU_IDX runwayml/stable-diffusion-v1-5 /path/to/example/rgba.png /path/to/save $token_name $init_token --max_train_steps 5000
+```
+$token_name is a the special token, usually name that by _examplename_
+$init_token is a single token to describe the image using natural language
+
+For example:
+```bash
+bash scripts/textual_inversion/textual_inversion.sh runwayml/stable-diffusion-v1-5 data/demo/a-full-body-ironman/rgba.png out/textual_inversion/ironman _ironman_ ironman --max_train_steps 3000
+```
+Don't forget to move the final `learned_embeds.bin` under data/demo/a-full-body-ironman/
+
+
+## Run 
+### Run Magic123 for a single example
+Takes ~40 mins for the coarse stage and ~20 mins for the second stage on a 32G V100. 
+```bash
+bash scripts/magic123/run_both_priors.sh $GPU_NO $JOBNAME_First_Stage $JOBNAME_Second_Stage $PATH_to_Example_Directory $IMAGE_BASE_NAME $Enable_First_Stage $Enable_Second_Stage {More_Arugments}
+```
+
+As an example, run Magic123 in the dragon example using both stages in GPU 0 and set the jobname for the first stage as `nerf` and the jobname for the second stage as `dmtet`, by the following command:
+```bash
+bash scripts/magic123/run_both_priors.sh 0 nerf dmtet data/realfusion15/metal_dragon_statue 1 1 
+```
+
+More arguments (e.g. `--lambda_guidance 1 40`) can be appended to the command line such as:
+```bash
+bash scripts/magic123/run_both_priors.sh 0 nerf dmtet data/realfusion15/metal_dragon_statue 1 1 --lambda_guidance 1 40
+```
+
+### Run Magic123 for a group of examples
+- Run all examples in a folder, check the scripts `scripts/magic123/run_folder_both_priors.sh`  
+- Run all examples in a given list, check the scripts `scripts/magic123/run_list_both_priors.sh` 
+
+
+### Run Magic123 on a single example without textual inversion
+textual inversion is tedious (requires ~2.5 hours optimization), if you want to test Magic123 quickly on your own example without textual inversion (might degrade the performance), try the following:
+
+- first, foreground and depth estimation
+    ```
+    python preprocess_image.py --path data/demo/a-full-body-ironman/main.png
+    ```
+
+- Run Magic123 coarse stage without textual inversion, takes ~40 mins
+    ```
+    export RUN_ID='default-a-full-body-ironman'
+    export DATA_DIR='data/demo/a-full-body-ironman'
+    export IMAGE_NAME='rgba.png'
+    export FILENAME=$(basename $DATA_DIR)
+    export dataset=$(basename $(dirname $DATA_DIR))
+    CUDA_VISIBLE_DEVICES=0 python main.py -O \
+    --text "A high-resolution DSLR image of a full body ironman" \
+    --sd_version 1.5 \
+    --image ${DATA_DIR}/${IMAGE_NAME} \
+    --workspace out/magic123-${RUN_ID}-coarse/$dataset/magic123_${FILENAME}_${RUN_ID}_coarse \
+    --optim adam \
+    --iters 5000 \
+    --guidance SD zero123 \
+    --lambda_guidance 1.0 40 \
+    --guidance_scale 100 5 \
+    --latent_iter_ratio 0 \
+    --normal_iter_ratio 0.2 \
+    --t_range 0.2 0.6 \
+    --bg_radius -1 \
+    --save_mesh
+    ```
+
+- Run Magic123 fine stage without textual inversion, takes around ~20 mins 
+    ```
+    export RUN_ID='default-a-full-body-ironman'
+    export RUN_ID2='dmtet'
+    export DATA_DIR='data/demo/a-full-body-ironman'
+    export IMAGE_NAME='rgba.png'
+    export FILENAME=$(basename $DATA_DIR)
+    export dataset=$(basename $(dirname $DATA_DIR))
+    CUDA_VISIBLE_DEVICES=0 python main.py -O \
+    --text "A high-resolution DSLR image of a full body ironman" \
+    --sd_version 1.5 \
+    --image ${DATA_DIR}/${IMAGE_NAME} \
+    --workspace out/magic123-${RUN_ID}-${RUN_ID2}/$dataset/magic123_${FILENAME}_${RUN_ID}_${RUN_ID2} \
+    --dmtet --init_ckpt out/magic123-${RUN_ID}-coarse/$dataset/magic123_${FILENAME}_${RUN_ID}_coarse/checkpoints/magic123_${FILENAME}_${RUN_ID}_coarse.pth \
+    --iters 5000 \
+    --optim adam \
+    --known_view_interval 4 \
+    --latent_iter_ratio 0 \
+    --guidance SD zero123 \
+    --lambda_guidance 1e-3 0.01 \
+    --guidance_scale 100 5 \
+    --rm_edge \
+    --bg_radius -1 \
+    --save_mesh 
+    ```
+
+### 10-step CUDA smoke test on the Ironman example
+
+After installing dependencies and CUDA extensions, run a short 4090-class smoke test:
+
+```bash
+bash scripts/magic123/smoke_train_ironman_10_steps.sh 0 out/smoke-ironman-10-steps
+```
+
+The script sets `TORCH_CUDA_ARCH_LIST=8.9`, runs `scripts/check_cuda_build_env.py`, trains for 10 iterations, and saves outputs under the workspace argument. The default workspace is `out/smoke-ironman-10-steps`.
+
+### Outputs and mesh export
+
+Use `--save_mesh` to export the object. Coarse NeRF runs save mesh files under the run workspace, typically in a `mesh/` subdirectory, including `mesh.obj`, material, and texture files. DMTet fine-stage runs also write the final textured `.obj` assets under the fine-stage workspace. The `.mp4` files are render previews; the `.obj` plus its `.mtl` and texture images are the 3D object files to open in Blender, MeshLab, or another mesh viewer.
+
+### Run ablation studies
+- Run Magic123 with only 2D prior *with* textual inversion (Like RealFusion but we achieve much better performance through training stragies and the coarse-to-fine pipeline)
+    ```
+    bash scripts/magic123/run_2dprior.sh 0 nerf dmtet data/realfusion15/metal_dragon_statue 1 1
+    ```
+
+- Run Magic123 with only 2D prior *without* textual inversion (Like RealFusion but we achieve much better performance through training stragies and the coarse-to-fine pipeline)
+    ```
+    bash scripts/magic123/run_2dprior_notextinv_ironman.sh 0 default 1 1
+    ```
+    note: change the path and the text prompt inside the script if you wana test another example. 
+
+- Run Magic123 with only 3D prior (Like Zero-1-to-3 but we achieve much better performance through training stragies and the coarse-to-fine pipeline)
+    ```
+    bash scripts/magic123/run_3dprior.sh 0 nerf dmtet data/demo/a-full-body-ironman 1 1
+    ```
+
+
+# Tips and Tricks
+1. Fix camera distance (*radius_range*) and FOV (*fovy_range*) and tune the camera polar range (*theta_range*). Note it is better to keep camera jittering to reduce grid artifacts. 
+2. Smaller range of time steps for the defusion noise (t_range). We find *[0.2, 0.6]* gives better performance for image-to-3D tasks. 
+3. Using normals as latent in the first 2000 improves generated geometry a bit gernerally (but not always). We turn on this for Magic123 corase stage in the script `--normal_iter_ratio 0.2` 
+4. We erode segmentation edges (makes the segmentation map 2 pixels shrinked towards internal side) to remove artifacts due to segmentation erros. This is turned on in the fine stage in magic123 in the script through `--rm_edge`
+5. Other general tricks such as improved textual inversion, advanced diffusion prior (DeepFloyd, SD-XL), stronger 3D prior (Zero123-XL), and larger batch size can be adopted as well but not studied in this work.
+6. textual inversion is not very necessary for well-known things (e.g. ironman) and easily described textures and geoemtries, since pure texts contains these texture information and will be understood by diffusion models. We use textual inversion by default in all experiments.
+
+
+# Some Projects that use Magic123
+1. [Threestudio](https://github.com/threestudio-project/threestudio)
+2. [DreamCraft3D](https://mrtornado24.github.io/DreamCraft3D/)
+
+# Acknowledgement
+This work is build upon Stable DreamFusion, many thanks to the author [Kiui Jiaxiang Tang](https://github.com/ashawkey) and many other contributors. 
+
+* [Stable DreamFusion](https://github.com/ashawkey/stable-dreamfusion)
+
+```
+@misc{stable-dreamfusion,
+    Author = {Jiaxiang Tang},
+    Year = {2022},
+    Note = {https://github.com/ashawkey/stable-dreamfusion},
+    Title = {Stable-dreamfusion: Text-to-3D with Stable-diffusion}
+}
+```
+
+
+We also get inspirations from a list of amazing research works and open-source projects, thanks a lot to all the authors for sharing!
+
+* [DreamFusion: Text-to-3D using 2D Diffusion](https://dreamfusion3d.github.io/)
+    ```
+    @article{poole2022dreamfusion,
+        author = {Poole, Ben and Jain, Ajay and Barron, Jonathan T. and Mildenhall, Ben},
+        title = {DreamFusion: Text-to-3D using 2D Diffusion},
+        journal = {arXiv},
+        year = {2022},
+    }
+    ```
+
+* [Magic3D: High-Resolution Text-to-3D Content Creation](https://research.nvidia.com/labs/dir/magic3d/)
+   ```
+   @inproceedings{lin2023magic3d,
+      title={Magic3D: High-Resolution Text-to-3D Content Creation},
+      author={Lin, Chen-Hsuan and Gao, Jun and Tang, Luming and Takikawa, Towaki and Zeng, Xiaohui and Huang, Xun and Kreis, Karsten and Fidler, Sanja and Liu, Ming-Yu and Lin, Tsung-Yi},
+      booktitle={IEEE Conference on Computer Vision and Pattern Recognition ({CVPR})},
+      year={2023}
+    }
+   ```
+
+* [Zero-1-to-3: Zero-shot One Image to 3D Object](https://github.com/cvlab-columbia/zero123)
+    ```
+    @misc{liu2023zero1to3,
+        title={Zero-1-to-3: Zero-shot One Image to 3D Object},
+        author={Ruoshi Liu and Rundi Wu and Basile Van Hoorick and Pavel Tokmakov and Sergey Zakharov and Carl Vondrick},
+        year={2023},
+        eprint={2303.11328},
+        archivePrefix={arXiv},
+        primaryClass={cs.CV}
+    }
+    ```
+
+* [RealFusion: 360° Reconstruction of Any Object from a Single Image](https://github.com/lukemelas/realfusion)
+    ```
+    @inproceedings{melaskyriazi2023realfusion,
+        author = {Melas-Kyriazi, Luke and Rupprecht, Christian and Laina, Iro and Vedaldi, Andrea},
+        title = {RealFusion: 360 Reconstruction of Any Object from a Single Image},
+        booktitle={CVPR}
+        year = {2023},
+        url = {https://arxiv.org/abs/2302.10663},
+    }
+    ```
+
+* [Make-it-3d: High-fidelity 3d creation from a single image with diffusion prior](https://arxiv.org/abs/2303.14184)
+    ```
+    @article{tang2023make-it-3d,
+        title={Make-it-3d: High-fidelity 3d creation from a single image with diffusion prior},
+        author={Tang, Junshu and Wang, Tengfei and Zhang, Bo and Zhang, Ting and Yi, Ran and Ma, Lizhuang and Chen, Dong},
+        journal={arXiv preprint arXiv:2303.14184},
+        year={2023}
+    }
+    ```
+
+* [Stable Diffusion](https://github.com/CompVis/stable-diffusion) and the [diffusers](https://github.com/huggingface/diffusers) library.
+
+    ```
+    @misc{rombach2021highresolution,
+        title={High-Resolution Image Synthesis with Latent Diffusion Models},
+        author={Robin Rombach and Andreas Blattmann and Dominik Lorenz and Patrick Esser and Björn Ommer},
+        year={2021},
+        eprint={2112.10752},
+        archivePrefix={arXiv},
+        primaryClass={cs.CV}
+    }
+
+    @misc{von-platen-etal-2022-diffusers,
+        author = {Patrick von Platen and Suraj Patil and Anton Lozhkov and Pedro Cuenca and Nathan Lambert and Kashif Rasul and Mishig Davaadorj and Thomas Wolf},
+        title = {Diffusers: State-of-the-art diffusion models},
+        year = {2022},
+        publisher = {GitHub},
+        journal = {GitHub repository},
+        howpublished = {\url{https://github.com/huggingface/diffusers}}
+    }
+    ```
+
+
+# Cite
+If you find this work useful, a citation will be appreciated via:
+```
+@inproceedings{
+Magic123,
+title={Magic123: One Image to High-Quality 3D Object Generation Using Both 2D and 3D Diffusion Priors},
+author={Qian, Guocheng and Mai, Jinjie and Hamdi, Abdullah and Ren, Jian and Siarohin, Aliaksandr and Li, Bing and Lee, Hsin-Ying and Skorokhodov, Ivan and Wonka, Peter and Tulyakov, Sergey and Ghanem, Bernard},
+booktitle={The Twelfth International Conference on Learning Representations (ICLR)},
+year={2024},
+url={https://openreview.net/forum?id=0jHkUDyEO9}
+}
+```

magic123-0.1.0/activation.py

@@ -0,0 +1,21 @@
+import torch
+from torch.autograd import Function
+from torch.cuda.amp import custom_bwd, custom_fwd 
+
+class _trunc_exp(Function):
+    @staticmethod
+    @custom_fwd(cast_inputs=torch.float)
+    def forward(ctx, x):
+        ctx.save_for_backward(x)
+        return torch.exp(x)
+
+    @staticmethod
+    @custom_bwd
+    def backward(ctx, g):
+        x = ctx.saved_tensors[0]
+        return g * torch.exp(x.clamp(max=15))
+
+trunc_exp = _trunc_exp.apply
+
+def biased_softplus(x, bias=0):
+    return torch.nn.functional.softplus(x - bias)