app.py

import os
import sys
import subprocess
import tempfile

import spaces
import torch
import ctypes

# Monkey-patch xformers.ops.memory_efficient_attention to fall back to
# torch SDPA on Blackwell (sm_120). On the new ZeroGPU GPUs, neither
# the FA3 nor the cutlass xformers kernel supports compute capability
# 12.0, so any call into xformers' MEA raises NotImplementedError.
# imagedream's mv_unet imports xformers.ops directly, so we have to
# patch it before that import happens.
import xformers
import xformers.ops as _xops


def _xformers_mea_sdpa(query, key, value, attn_bias=None, p=0.0, scale=None,
                      op=None, **kwargs):
    # xformers MEA accepts (B, M, K) or (B, M, H, K). Torch SDPA wants
    # (B, H, M, K). Reshape appropriately and reverse on output.
    if query.dim() == 3:
        # Single-head: add an H=1 axis.
        q = query.unsqueeze(1)
        k = key.unsqueeze(1)
        v = value.unsqueeze(1)
        squeeze_out = True
    else:
        # (B, M, H, K) -> (B, H, M, K)
        q = query.transpose(1, 2)
        k = key.transpose(1, 2)
        v = value.transpose(1, 2)
        squeeze_out = False
    attn_mask = attn_bias
    if hasattr(attn_mask, "materialize"):
        try:
            attn_mask = attn_mask.materialize(
                shape=(q.shape[0], q.shape[1], q.shape[2], k.shape[2]),
                dtype=q.dtype,
                device=q.device,
            )
        except Exception:
            attn_mask = None
    out = torch.nn.functional.scaled_dot_product_attention(
        q, k, v, attn_mask=attn_mask, dropout_p=p, scale=scale,
    )
    if squeeze_out:
        return out.squeeze(1)
    return out.transpose(1, 2)


_xops.memory_efficient_attention = _xformers_mea_sdpa
xformers.ops.memory_efficient_attention = _xformers_mea_sdpa

import gradio as gr
import numpy as np
from diffusers import DiffusionPipeline


CUDA_HOME = "/cuda-image/usr/local/cuda-13.0"
CUDA_LIBDIR = os.path.join(CUDA_HOME, "lib64")


@spaces.GPU(duration=600)
def _first_gpu_setup():
    try:
        import diff_gaussian_rasterization  # noqa
        return
    except ImportError:
        pass

    patch_dir = tempfile.mkdtemp(prefix="torch_cuda_patch_")
    with open(os.path.join(patch_dir, "sitecustomize.py"), "w") as f:
        f.write(
            "try:\n"
            "    import torch.utils.cpp_extension as _c\n"
            "    _c._check_cuda_version = lambda *a, **k: None\n"
            "except Exception:\n"
            "    pass\n"
        )

    env = os.environ.copy()
    env["CUDA_HOME"] = CUDA_HOME
    env["CUDA_PATH"] = CUDA_HOME
    env["PATH"] = os.path.join(CUDA_HOME, "bin") + os.pathsep + env.get("PATH", "")
    env["PYTHONPATH"] = patch_dir + os.pathsep + env.get("PYTHONPATH", "")
    env["TORCH_CUDA_ARCH_LIST"] = "12.0"

    subprocess.check_call(
        [sys.executable, "-m", "pip", "install", "--no-deps",
         "setuptools", "wheel", "ninja", "packaging"],
    )

    # The PyPI release of `diff-gaussian-rasterization` is the original
    # graphdeco-inria version; the LGM model uses that one.
    subprocess.check_call(
        [sys.executable, "-m", "pip", "install",
         "--no-build-isolation", "--no-deps",
         "git+https://github.com/graphdeco-inria/diff-gaussian-rasterization.git"],
        env=env,
    )


_first_gpu_setup()
try:
    ctypes.CDLL(os.path.join(CUDA_LIBDIR, "libcudart.so.13"), mode=ctypes.RTLD_GLOBAL)
    os.environ["LD_LIBRARY_PATH"] = CUDA_LIBDIR + os.pathsep + os.environ.get("LD_LIBRARY_PATH", "")
except OSError:
    pass


TMP_DIR = "/tmp"
os.makedirs(TMP_DIR, exist_ok=True)


image_pipeline = DiffusionPipeline.from_pretrained(
    "dylanebert/imagedream",
    custom_pipeline="dylanebert/multi-view-diffusion",
    torch_dtype=torch.float16,
    trust_remote_code=True,
).to("cuda")


splat_pipeline = DiffusionPipeline.from_pretrained(
    "dylanebert/LGM",
    custom_pipeline="dylanebert/LGM",
    torch_dtype=torch.float16,
    trust_remote_code=True,
).to("cuda")


@spaces.GPU
def run(input_image, seed):
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    input_image = input_image.astype("float32") / 255.0
    images = image_pipeline(
        "", input_image, guidance_scale=5, num_inference_steps=30, elevation=0
    )
    gaussians = splat_pipeline(images)
    output_ply_path = os.path.join(TMP_DIR, "output.ply")
    splat_pipeline.save_ply(gaussians, output_ply_path)
    return output_ply_path


_TITLE = """LGM Mini"""

_DESCRIPTION = """
<div>
A lightweight version of <a href="https://huggingface.co/spaces/ashawkey/LGM">LGM: Large Multi-View Gaussian Model for High-Resolution 3D Content Creation</a>.

To convert to mesh, download the output splat and visit [splat-to-mesh](https://huggingface.co/spaces/dylanebert/splat-to-mesh).
</div>
"""

css = """
#duplicate-button {
    margin: auto;
    color: white;
    background: #1565c0;
    border-radius: 100vh;
}
"""

block = gr.Blocks(title=_TITLE, css=css)
with block:
    gr.DuplicateButton(
        value="Duplicate Space for private use", elem_id="duplicate-button"
    )

    with gr.Row():
        with gr.Column(scale=1):
            gr.Markdown("# " + _TITLE)
    gr.Markdown(_DESCRIPTION)

    with gr.Row(variant="panel"):
        with gr.Column(scale=1):
            input_image = gr.Image(label="image", type="numpy")
            seed_input = gr.Number(label="seed", value=42)
            button_gen = gr.Button("Generate")

        with gr.Column(scale=1):
            output_splat = gr.Model3D(label="3D Gaussians")

        button_gen.click(
            fn=run, inputs=[input_image, seed_input], outputs=[output_splat]
        )

    gr.Examples(
        examples=[
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_cat_statue.jpg",
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_baby_penguin.jpg",
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/A_cartoon_house_with_red_roof.jpg",
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/a_hat.jpg",
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/an_antique_chest.jpg",
            "https://huggingface.co/datasets/dylanebert/iso3d/resolve/main/jpg@512/metal.jpg",
        ],
        inputs=[input_image],
        outputs=[output_splat],
        fn=lambda x: run(input_image=x, seed=42),
        cache_examples=True,
        label="Image-to-3D Examples",
    )

block.queue().launch(debug=True, share=True)