"""The LoRA stack: a catalog, arbitrary URLs, and the adapters they turn into on the transformer. diffusers carries the MiniMax-H3 LoRA loader itself (`MiniMaxH3LoraLoaderMixin`), including the conversion of the ComfyUI / musubi layouts onto `MiniMaxH3Transformer3DModel`'s module names, so a slot is one `load_lora_weights` call and the whole stack is one `set_adapters`. Alpha-less files load at `alpha == rank`; a `__metadata__` alpha is honored instead. Weights are per slot and change for free - nothing is folded into the checkpoint. A slot resolves in the main process (`LoraSpec.fetch`, which downloads) and is applied inside the GPU worker (`apply`), so no file is ever pulled off the network on booked GPU time. """ from __future__ import annotations import faulthandler import functools import hashlib import os import re import sys import tempfile import threading import time import traceback import urllib.request from urllib.parse import unquote from dataclasses import dataclass, replace NONE_CHOICE = "None" CUSTOM_CHOICE = "Custom URL" CACHE_DIR = os.path.join(tempfile.gettempdir(), "h3-lora-cache") # How many adapters may stay resident on the card before the ones outside the current stack are dropped. MAX_RESIDENT = int(os.getenv("H3_MAX_ADAPTERS", "8")) # Identical requests in a row before the rack is folded into the weights. 1 folds immediately, 0 never folds — the # fold makes every later request with that rack free, and makes the next change to it cost one unfold plus one fold. FUSE_AFTER = int(os.getenv("H3_FUSE_AFTER", "2")) # Every entry was checked against the released `transformer/` partition. LoRAs trained on a *pruned* checkpoint # (joyfox/MiniMax-H3-Turbo is the known one) fail with a size mismatch and are deliberately not listed; the same # repositories' `*_full*` files are, where they exist. Jojocodex's `*_pruned` files are the opposite case and do # load: there `pruned` names the fix rather than the base - `convert_h3_lora.py` strips the 100 AdaLN adapters that # targeted the pruned model's `adaln_proj` input dim of 8, which is the key that raised the mismatch to begin with. CATALOG: dict[str, dict] = { "Turbo 4-step · larryvrh": { "repo": "larryvrh/MiniMax-H3-Turbo-Lora", "weights": "minimax_h3_turbo_4step_ema_ckpt850.safetensors", "adapter": "turbo_larry", "default_weight": 1.0, "suggested_steps": 6, "url": "https://huggingface.co/larryvrh/MiniMax-H3-Turbo-Lora", "about": "4-step distillation of the joint video+audio path. The usual starting point.", "hint": "Drop steps to 4-6. Above ~1.2 it sharpens into artifacts, below ~0.8 it goes soft.", }, "Turbo 4-step · lightx2v": { "repo": "lightx2v/Minimax-h3-Turbo", "weights": "minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors", "adapter": "turbo_lightx_4", "default_weight": 1.0, "suggested_steps": 5, "url": "https://huggingface.co/lightx2v/Minimax-h3-Turbo", "about": "Alternative 4-step distillation, trained at 768p on the first/last-frame branch.", "hint": "Trained for keyframed requests; holds up on text-only ones too.", }, "Turbo 8-step · lightx2v": { "repo": "lightx2v/Minimax-h3-Turbo", "weights": "minimax_h3_fl2v_turbo_8step_v1.0_bf16.safetensors", "adapter": "turbo_lightx_8", "default_weight": 1.0, "suggested_steps": 8, "url": "https://huggingface.co/lightx2v/Minimax-h3-Turbo", "about": "8-step sibling: slower than the 4-step files, steadier on motion.", "hint": "Give it 8 steps - it is not distilled down to 4.", }, "Turbo 4-step · InstantX": { "repo": "InstantX/MiniMax-H3-Turbo-Lora-Diffusers", "weights": "minimax_h3_turbo_4step_ema_ckpt500_diffusers.safetensors", "adapter": "turbo_instantx", "default_weight": 1.0, "suggested_steps": 5, "url": "https://huggingface.co/InstantX/MiniMax-H3-Turbo-Lora-Diffusers", "about": "The ckpt500 EMA turbo LoRA, already converted to diffusers module names.", "hint": "An earlier checkpoint of the larryvrh round - useful for A/B against it.", }, # Pika's H3 Flex style fused with lightx2v's 4-step v1.2 turbo into one plain LoRA (rank-concatenated, Flex # baked in at 0.5). The FL2VA files are the ones for text and first/last-frame video; the Ref2VA ones are for # reference workflows. Pika runs them at strength 1.0, CFG 1, exactly 4 steps, video shift 6 - the same 768p # lightx2v distillation this Space already drives. Not yet run here as of 2026-11-07. "Turbo 4-step · Pika Flex": { "repo": "Pika-Labs/Pika-H3-Flex-Turbo-LoRA", "weights": "pika_h3_flex_turbo_4step_fl2va.safetensors", "adapter": "turbo_pika_flex", "default_weight": 1.0, "suggested_steps": 4, # 3.9 GB, the size of the Regenerate refiner, which read in about a second warm. "load_seconds": 5.0, "url": "https://huggingface.co/Pika-Labs/Pika-H3-Flex-Turbo-LoRA", "about": "Pika's H3 Flex look fused with a 4-step turbo - one LoRA, quality at 4 steps.", "hint": "Exactly 4 steps, weight 1.0. Don't stack other strong style or motion LoRAs high on it; sound thins below ~6 steps.", }, "Turbo 4-step · Pika Flex lite": { "repo": "Pika-Labs/Pika-H3-Flex-Turbo-LoRA", "weights": "pika_h3_flex_turbo_4step_fl2va_lite_r32.safetensors", "adapter": "turbo_pika_flex_lite", "default_weight": 1.0, "suggested_steps": 4, "url": "https://huggingface.co/Pika-Labs/Pika-H3-Flex-Turbo-LoRA", "about": "The same Pika Flex turbo compressed to rank 32: 310 MB instead of 3.9 GB, 96% of it kept.", "hint": "Exactly 4 steps, weight 1.0. The quick one to try first.", }, "Realism · People": { "repo": "fal/MiniMax-H3-Realism-People-LoRA", "weights": "h3-realism-people-t2v-i2v-r2v.safetensors", "adapter": "realism_people", "default_weight": 0.8, "url": "https://huggingface.co/fal/MiniMax-H3-Realism-People-LoRA", "about": "Photographic skin, hair and lighting on human subjects.", "hint": "Stacks well under a turbo LoRA. Back off toward 0.6 if faces start looking waxy.", }, "Motion Adapter": { "repo": "MATLOWAI/MiniMax-H3-Motion-Adapter", "weights": "minimax_h3_motion_adapter_pilot_r16.safetensors", "adapter": "motion_adapter", "default_weight": 0.8, "url": "https://huggingface.co/MATLOWAI/MiniMax-H3-Motion-Adapter", "about": "Pushes the amount of movement in the shot up.", "hint": "Rank 16 pilot - subtle by design. Raise it before you blame the prompt.", }, "Spatial Physics": { "repo": "Jojocodex/minimax-h3-spatial-physics-lora", "weights": "wushu_spatial_physics_clean_3000_pruned.safetensors", "adapter": "spatial_physics", "default_weight": 0.4, "url": "https://huggingface.co/Jojocodex/minimax-h3-spatial-physics-lora", "about": "Steadier object motion, contact and weight.", "hint": "Pairs with Wushu Action for fight choreography that keeps its footing. Trained to sit at 0.3-0.5.", }, "Wushu Action": { "repo": "Jojocodex/minimax-h3-wushu-action-lora", "weights": "wushu_action_h3_lora_v5_3000_pruned.safetensors", "adapter": "wushu_action", "default_weight": 0.5, "url": "https://huggingface.co/Jojocodex/minimax-h3-wushu-action-lora", "about": "Martial-arts choreography: strikes, spins, weapon work.", "hint": "Name the move in the prompt ('spinning hook kick'), not just the style. 0.5 over Spatial Physics at 0.3 is the pairing it was tuned against.", }, "Lineart Anime": { "repo": "DiffSynth-Studio/MiniMax-H3-LoRA-LineartAnime", "weights": "model.safetensors", "adapter": "lineart_anime", "default_weight": 1.0, "url": "https://huggingface.co/DiffSynth-Studio/MiniMax-H3-LoRA-LineartAnime", "about": "Flat-shaded lineart anime, trained against the raw fused-QKV checkpoint.", "hint": "Its factors are fp32; the transformer is cast back to bf16 after loading.", }, } CHOICES = [NONE_CHOICE, *CATALOG, CUSTOM_CHOICE] def default_weight(key: str) -> float: return CATALOG.get(key, {}).get("default_weight", 1.0) def suggested_steps(keys) -> int | None: """The step count the fastest turbo LoRA in a stack was distilled for, if any.""" steps = [CATALOG[key]["suggested_steps"] for key in keys if key in CATALOG and "suggested_steps" in CATALOG[key]] return min(steps) if steps else None def distilled_for(steps: int) -> str | None: """The catalog key of a turbo LoRA distilled for this many steps, if one is listed.""" for key, entry in CATALOG.items(): if entry.get("suggested_steps") == steps: return key return None def about(key: str) -> tuple[str, str, str]: """`(about, hint, url)` of a catalog entry, empty strings for anything else.""" entry = CATALOG.get(key, {}) return entry.get("about", ""), entry.get("hint", ""), entry.get("url", "") # --------------------------------------------------------------------------- # Sources # --------------------------------------------------------------------------- # Directories a `file:` source may point into: the Space's own outputs, where a LoRA trained here is written. # Registered by the engine, which owns that directory. Anything outside is refused - a slot is visitor input. LOCAL_ROOTS: list[str] = [] # Where a LoRA load that takes too long leaves its stacks. A file, not stderr: after b21f6a7 the loads still hung now # and then, and the faulthandler dump armed for them never reached the log. The main process prints what is here # when a GPU call ends badly (app.py), since the worker that wrote it is gone by then. STACK_DIR = os.path.join(tempfile.gettempdir(), "h3-stacks") def _watch_load(label: str): """Two independent watchers over a load: faulthandler (C level, needs no GIL) every 30 s, and a Python thread (needs the GIL - if only the first writes, the stuck code holds it) every 20 s. Returns a stop function.""" os.makedirs(STACK_DIR, exist_ok=True) path = os.path.join(STACK_DIR, f"{int(time.time())}-{os.getpid()}.txt") handle = open(path, "w", buffering=1) handle.write(f"[lora] watching the load of {label} in pid {os.getpid()}\n") handle.flush() # A fatal signal (a crash in C) during the load writes its traceback here too, where the main process reads it. faulthandler.enable(file=handle, all_threads=True) faulthandler.dump_traceback_later(30, repeat=True, file=handle) stop = threading.Event() def python_watch(): started = time.time() while not stop.wait(20): lines = [f"--- python watcher, {time.time() - started:.0f}s into the load"] for ident, frame in sys._current_frames().items(): lines.append(f"thread {ident}:") lines.extend(line.rstrip() for line in traceback.format_stack(frame)) try: handle.write("\n".join(lines) + "\n") handle.flush() except ValueError: # closed: the load finished between the wait and the write return threading.Thread(target=python_watch, daemon=True, name="lora-watch").start() def done(): stop.set() faulthandler.cancel_dump_traceback_later() faulthandler.enable(all_threads=True) # back to stderr before the file closes handle.close() try: os.remove(path) except OSError: pass return done def _read(path: str, limit: int = 4000) -> str: try: with open(path, encoding="utf-8", errors="replace") as handle: return handle.read(limit) except OSError as error: return f"<{type(error).__name__}>" def probe_stuck_load() -> None: """Look at a stuck load from outside, from the main process. The watchers inside it wrote nothing on three hangs (2026-11-08 18:07-18:12), which only a dead or wholly frozen process explains. This says which: whether the pid lives, each thread's state and kernel wait, the container's OOM kills, and a py-spy dump (no GIL needed).""" try: name = sorted(os.listdir(STACK_DIR))[-1] pid = int(name.rsplit("-", 1)[1].split(".")[0]) except (OSError, IndexError, ValueError): print("[lora] probe: no watched load to look at", flush=True) return lines = [f"[lora] probe of the stuck load, pid {pid} ({name}):"] status = _read(f"/proc/{pid}/status") alive = not status.startswith("<") if not alive: lines.append(f" the process is gone {status} - it died during the load (a crash or the OOM killer)") else: keep = ("State", "VmRSS", "VmSwap", "Threads", "SigPnd", "ShdPnd") lines += [" " + line for line in status.splitlines() if line.split(":")[0] in keep] try: for tid in sorted(os.listdir(f"/proc/{pid}/task"), key=int)[:40]: stat = _read(f"/proc/{pid}/task/{tid}/stat", 400) comm = stat[stat.find("(") + 1:stat.rfind(")")] state = stat[stat.rfind(")") + 2:stat.rfind(")") + 3] wchan = _read(f"/proc/{pid}/task/{tid}/wchan", 100) lines.append(f" thread {tid} {comm!r} state {state} wchan {wchan}") except OSError as error: lines.append(f" threads unreadable: {error}") events = [line for line in _read("/sys/fs/cgroup/memory.events").splitlines() if line.startswith(("oom", "max"))] available = next((line.split(":")[1].strip() for line in _read("/proc/meminfo").splitlines() if line.startswith("MemAvailable")), "?") lines.append(f" container memory: {' · '.join(events) or 'no memory.events'} · MemAvailable {available}") print(NEWLINE.join(lines), flush=True) if alive: import subprocess try: dump = subprocess.run(["py-spy", "dump", "--pid", str(pid), "--native"], capture_output=True, text=True, timeout=30) print(f"[lora] py-spy dump of pid {pid}:{NEWLINE}{(dump.stdout or dump.stderr)[-15000:]}", flush=True) except (OSError, subprocess.TimeoutExpired) as error: print(f"[lora] py-spy dump failed: {type(error).__name__}: {error}", flush=True) NEWLINE = chr(10) def print_load_stacks(since: float) -> None: """Print, then delete, the stacks a stuck LoRA load left (newer than `since`).""" try: names = sorted(os.listdir(STACK_DIR)) except OSError: return for name in names: path = os.path.join(STACK_DIR, name) try: if os.path.getmtime(path) >= since - 5: with open(path, encoding="utf-8", errors="replace") as handle: text = handle.read() print(f"[lora] stacks left by a stuck load ({name}):\n{text[-20000:]}", flush=True) os.remove(path) except OSError: pass def preload_modules() -> None: """Import, in the main process, everything `load_lora_weights` would otherwise import on first use. ZeroGPU forks the main process for each GPU call. A module another thread of the main process was halfway through importing at that instant is left locked in the fork, and the fork's first import of it waits forever: loads hung that way with nothing in the log (2026-11-07). Imported here, the fork finds them all done. """ for name in ( "peft", "peft.tuners.tuners_utils", "peft.tuners.lora", "peft.utils", "peft.utils.hotswap", "diffusers.loaders.peft", "diffusers.loaders.lora_pipeline", "diffusers.loaders.lora_base", "diffusers.loaders.lora_conversion_utils", "diffusers.utils.peft_utils", "diffusers.hooks.group_offloading", "safetensors.torch", ): try: __import__(name) except Exception as error: # a module this diffusers build does not have is not worth failing over print(f"[lora] could not preload {name}: {type(error).__name__}", flush=True) preload_modules() def parse_source(raw: str) -> tuple[str, str, str | None]: """Any LoRA reference into `(kind, source, weight_name)`, `kind` being hf / civitai / direct / local.""" raw = raw.strip() if raw.startswith("file:"): return "local", raw[len("file:") :], None # A file name is unquoted before it reaches the hub: one copied out of the address bar arrives as `%20`, and # `hf_hub_download` quotes it again into a `%2520` that is never found. match = re.match(r"https?://huggingface\.co/([^/]+/[^/?#]+)(?:/(?:blob|resolve)/[^/]+/([^?#]+))?", raw) if match: return "hf", match.group(1), unquote(match.group(2)) if match.group(2) else None if "civitai.com" in raw: return "civitai", raw, None if raw.startswith("http"): return "direct", raw, None # `user/repo`, `user/repo/inside/file.safetensors`, or the same with the `blob/main/` of a copied page path if re.match(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]", raw): owner, name, *rest = raw.split("/", 2) inside = re.sub(r"^(?:blob|resolve)/[^/]+/", "", rest[0]) if rest else "" return "hf", f"{owner}/{name}", unquote(inside) if inside else None raise ValueError(f"Not a LoRA source: {raw!r}") # Query parameters that carry a credential. A CivitAI download link has its API key in `token=`, and the URL is # what a Custom slot is named by - until 2026-11-04 the key went into the Space log on every load, into the adapter # name, into the report, and into every context and draft file written from that rack. _SECRET_PARAMS = r"token|api_key|apikey|key|access_token|auth|signature|sig" def redact_url(text: str) -> str: """`text` with every credential in a URL query masked - for anything printed or shown.""" return re.sub(rf"([?&](?:{_SECRET_PARAMS})=)[^&#\s]+", r"\1***", str(text), flags=re.IGNORECASE) def strip_secrets(url: str) -> str: """`url` with its credential parameters removed - for anything written to a file someone may be handed.""" stripped = re.sub(rf"(?<=[?&])(?:{_SECRET_PARAMS})=[^&#]*&?", "", url, flags=re.IGNORECASE) return re.sub(r"[?&]$", "", stripped) def _civitai_download_url(url: str) -> str: """Any CivitAI URL into a direct download URL.""" if "/api/download/models/" in url: # A link restored from a file has had its key stripped; the Space's own key stands in, when it has one. token = os.getenv("CIVITAI_API_KEY", "") if token and not re.search(r"[?&]token=", url): url += ("&" if "?" in url else "?") + f"token={token}" return url match = re.search(r"[?&]modelVersionId=(\d+)", url) if not match: raise ValueError( "A CivitAI URL needs its modelVersionId, e.g. https://civitai.com/models/123?modelVersionId=456" ) token = os.getenv("CIVITAI_API_KEY", "") url = f"https://civitai.com/api/download/models/{match.group(1)}" return f"{url}?token={token}" if token else url def _download(url: str) -> str: """Fetch a LoRA file to the local cache once, and return its path.""" os.makedirs(CACHE_DIR, exist_ok=True) name = url.split("?")[0].rstrip("/").split("/")[-1] if not name.endswith((".safetensors", ".sft")): # Named by the link without its key, so a restored link finds the file its keyed original fetched. name = "lora_" + hashlib.sha1(strip_secrets(url).encode("utf-8")).hexdigest()[:16] + ".safetensors" path = os.path.join(CACHE_DIR, name) if not os.path.exists(path): request = urllib.request.Request(url, headers={"User-Agent": "minimax-h3-turbo-loras"}) with urllib.request.urlopen(request, timeout=300) as response, open(path + ".part", "wb") as handle: while chunk := response.read(1 << 22): handle.write(chunk) os.replace(path + ".part", path) return path @dataclass(frozen=True) class LoraSpec: """One filled slot, on its way to the GPU worker. Picklable on purpose - it crosses a process boundary.""" label: str weight: float adapter: str source: str # a hub id, or a URL weights_file: str | None = None path: str | None = None # filled by `fetch` # A distillation of the whole denoising trajectory, which is what `suggested_steps` marks in the catalog. It # matters to anything that runs the transformer over part of a trajectory rather than all of it: the shortcut # such an adapter learned still aims at the endpoint it was trained to reach. distilled: bool = False # Seconds the GPU worker spends reading and injecting this file, when it is far off the small-adapter norm the # booking otherwise assumes. None means that norm. load_seconds: float | None = None # Shipped in float32 and worth halving on disk before the worker reads it: see `_bf16_copy`. bf16: bool = False def fetch(self) -> "LoraSpec": """Put the file on local disk. Runs in the main process, off booked GPU time.""" if self.path: return self kind, source, weight_name = parse_source(self.source) if kind == "local": path = os.path.realpath(source) roots = [os.path.realpath(root) for root in LOCAL_ROOTS] if not any(path.startswith(root + os.sep) for root in roots): raise ValueError("A file: LoRA has to be one this Space wrote or was handed in its Train LoRA tab.") if not os.path.isfile(path): raise FileNotFoundError( "That LoRA file is gone - the Space restarted since. Load the downloaded copy in the Train LoRA tab." ) return replace(self, path=path) if kind == "hf": from huggingface_hub import hf_hub_download name = self.weights_file or weight_name if name is None: name = _only_safetensors(source) path = hf_hub_download(source, name) return replace(self, path=_bf16_copy(path) if self.bf16 else path, weights_file=name) url = _civitai_download_url(source) if kind == "civitai" else source return replace(self, path=_download(url)) def _bf16_copy(path: str) -> str: """A bfloat16 copy of a float32 LoRA file, made once beside the cache. The worker casts every adapter to bfloat16 after loading it anyway; doing it here, in the main process and off booked time, halves what the worker reads. Worth it only for the large files - the Regenerate refiner is 7.9 GB in float32 - which is why it is a flag on the spec rather than a rule. """ import torch from safetensors import safe_open from safetensors.torch import save_file os.makedirs(CACHE_DIR, exist_ok=True) target = os.path.join(CACHE_DIR, "bf16-" + os.path.basename(path)) if os.path.exists(target): return target tensors = {} with safe_open(path, framework="pt") as handle: metadata = handle.metadata() for key in handle.keys(): tensor = handle.get_tensor(key) tensors[key] = tensor.to(torch.bfloat16) if tensor.is_floating_point() else tensor save_file(tensors, target + ".part", metadata=metadata) os.replace(target + ".part", target) return target # The adapters NVIDIA's Open-H3-Regenerate-2K refine runs with, added by the engine when its ladder is picked and # never shown in the rack. The reference recipe is three: the refiner, LightX2V's FL2VA turbo v1.2 and LMS Sharp. # Sharp is left out - beside its LoRA it adds 50 AdaLN branches driven by a timestep curve, which are modules of # their own and not something `load_lora_weights` can put on the transformer. REGENERATE_ADAPTERS = ( LoraSpec( label="Regenerate refiner · NVIDIA", weight=1.0, adapter="regenerate_refiner", source="Efficient-Large-Model/Open-H3-Regenerate-2K-LoRA", weights_file="open_h3_regenerate_2k_refiner_lora.safetensors", # Rank 384 on 312 linears: 3.9 GB once in bfloat16, against tens of MB for an ordinary adapter. Measured on # the first run, 2026-11-04: loaded in 1.1 s, read from the page cache the main process had just written it # through. Booked at the ordinary adapter's figure, which already covers a colder read several times over. load_seconds=5.0, bf16=True, ), LoraSpec( label="Turbo 4-step v1.2 · lightx2v", weight=1.0, adapter="turbo_lightx_4_v12", source="lightx2v/Minimax-h3-Turbo", weights_file="minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", ), ) def regenerate_rack(specs) -> list: """The rack a Regenerate refine runs: the visitor's style adapters, and the recipe's own in place of any turbo. A turbo from the rack would be a second distillation stacked on the one inside the recipe. Works on specs and on the rack's catalog keys alike, so the UI can price the same rack the engine will run. """ def distilled(item): if isinstance(item, str): return "suggested_steps" in CATALOG.get(item, {}) return getattr(item, "distilled", False) return [item for item in specs if not distilled(item)] + list(REGENERATE_ADAPTERS) def _only_safetensors(repo: str) -> str: """The single `.safetensors` in a repo, when the user did not name one.""" from huggingface_hub import list_repo_files files = [f for f in list_repo_files(repo) if f.endswith(".safetensors")] if not files: raise ValueError(f"`{repo}` holds no .safetensors file.") if len(files) > 1: raise ValueError(f"`{repo}` holds {len(files)} .safetensors files - name one, e.g. {repo}/{files[0]}") return files[0] def slot_spec(key: str, url: str, weight: float) -> LoraSpec | None: """A UI slot into a spec, or None when the slot is empty.""" if not key or key == NONE_CHOICE: return None if key == CUSTOM_CHOICE: url = (url or "").strip() if not url: return None parse_source(url) # fail here, where the slot is still nameable # The key stays in `source`, which only the main process reads, to download with. The label and the adapter # name are what get printed, shown and logged, so neither carries it. return LoraSpec( label=redact_url(url), weight=float(weight), adapter="custom_" + hashlib.sha1(strip_secrets(url).encode("utf-8")).hexdigest()[:12], source=url, ) entry = CATALOG[key] return LoraSpec( label=key, weight=float(weight), adapter=entry["adapter"], source=entry["repo"], weights_file=entry["weights"], distilled="suggested_steps" in entry, load_seconds=entry.get("load_seconds"), ) def prepare(url: str) -> str: """Fetch a Custom URL slot ahead of Generate, and say what landed or why nothing did. Only the download moves: injecting the adapter needs the GPU worker, which a press here would have to book. The file stays in the cache that `fetch` reads, so the request that follows finds it there - until the Space restarts and the disk goes with it. """ import time spec = slot_spec(CUSTOM_CHOICE, url, 1.0) if spec is None: raise ValueError("Paste a LoRA URL first.") started = time.time() path = spec.fetch().path return f"{os.path.basename(path)} · {os.path.getsize(path) / 2**20:.0f} MB · ready in {time.time() - started:.0f}s" def build_stack(slots) -> list[LoraSpec]: """`[(key, url, weight), ...]` into the specs to apply, first occurrence of an adapter winning.""" stack: list[LoraSpec] = [] seen: set[str] = set() for key, url, weight in slots: spec = slot_spec(key, url, weight) if spec is None or spec.adapter in seen: continue seen.add(spec.adapter) stack.append(spec) return stack # --------------------------------------------------------------------------- # Application - inside the GPU worker # --------------------------------------------------------------------------- def _resident(pipe) -> set[str]: """Adapters already injected into the transformer. The model is the source of truth: a fresh worker has none.""" return set(getattr(pipe.transformer, "peft_config", {}) or {}) def stack_key(specs) -> tuple: """What makes two racks the same rack: the adapters and the weights they run at.""" return tuple(sorted((spec.adapter, round(float(spec.weight), 4)) for spec in specs)) def _state(pipe) -> dict: """The fold bookkeeping, kept on the transformer itself. A recycled ZeroGPU worker re-forks the transformer from the startup copy, whose weights carry no fold and whose `peft_config` is empty — and with the bookkeeping on that same object, the state re-derives instead of lying. """ transformer = pipe.transformer if not hasattr(transformer, "_stack_state"): transformer._stack_state = {"fused": None, "key": None, "repeats": 0, "active": ([], [])} return transformer._stack_state def apply(pipe, specs, allow_fold: bool = True, fold_now: bool = False, on_stage=None) -> str: """Load whatever is missing, activate the rack with its weights, fold it once it settles, and report. Unfused adapters cost an extra pair of matmuls on every linear layer of every step, which is the price of a weight slider that moves for free. So the rack is left unfused while it is being tuned, and folded into the bfloat16 weights once the same rack comes back `FUSE_AFTER` requests running: the fold is `weight * (B @ A)` added to the weights once, and every later request with that same rack pays nothing at all for its LoRAs. A slot that fails to load is reported and skipped - one unusable URL does not cost the whole request. `allow_fold=False` keeps the rack out of the weights. A folded rack cannot be re-weighted without unfusing it, so anything that wants to change the weights mid-request - a second pass at a different strength - has to ask for the rack to stay where `set_adapters` can still reach it. `on_stage(text)` hears each piece of work before it starts - which adapter is loading, the activation, the fold - so a run that stops inside the rack shows where. A generation hung on 2026-11-06 after one adapter loaded, and the log could not tell whether the next load or the activation was the one that never returned. `fold_now` folds on this request instead of waiting for the rack to repeat - for a caller that knows the weights will not move again before the request ends, and runs enough steps that the fold pays for itself at once. """ import torch active: list[str] = [] weights: list[float] = [] notes: list[str] = [] stage = on_stage or (lambda text: None) state = _state(pipe) requested = stack_key(specs) # A fold that no longer matches the request comes out of the weights before anything else touches them - in # particular before a new adapter is injected into layers that still carry a merged one. if state["fused"] is not None and state["fused"] != requested: print(f"[lora] unfolding the previous rack ({len(state['fused'])} adapter(s))", flush=True) pipe.unfuse_lora() state["fused"] = None for spec in specs: try: if spec.path is None: raise ValueError("this slot was never fetched") if spec.adapter not in _resident(pipe): # Loads hung inside the GPU worker until ZeroGPU aborted the call, regularly, on any adapter # (2026-11-07). The one stack caught sat in diffusers' `load_lora_adapter` calling # `self.state_dict()` over the whole transformer, stuck in a `param.detach()` - for a dict whose # values this diffusers build never reads (`_create_lora_config` takes it and ignores it). The # transformer hands out its parameters as they are for the length of the load, with no tensor op # per parameter. Loads still hung after that (5 of 7 at 22:09-22:29) with no dump in the log, so the # stacks now go to a file the main process prints once the call has failed (`_watch_load`). watch_done = _watch_load(spec.label[:60]) stage(f"loading LoRA {spec.label[:60]}") started = time.time() directory, filename = os.path.split(spec.path) transformer = pipe.transformer transformer.state_dict = functools.partial(type(transformer).state_dict, transformer, keep_vars=True) try: pipe.load_lora_weights(directory, weight_name=filename, adapter_name=spec.adapter) print(f"[lora] read {spec.adapter} in {time.time() - started:.1f}s", flush=True) # DiffSynth-Studio's fp32 factors would otherwise drag the unfused LoRA path into fp32. pipe.transformer.to(torch.bfloat16) finally: transformer.__dict__.pop("state_dict", None) watch_done() # Timed because the booking prices a load: the figure the next large adapter is booked by. print(f"[lora] loaded {spec.label} as `{spec.adapter}` in {time.time() - started:.1f}s", flush=True) active.append(spec.adapter) weights.append(float(spec.weight)) except Exception as error: print(f"[lora] {spec.label} failed: {type(error).__name__}: {redact_url(error)}", flush=True) notes.append(f"{spec.label} failed ({type(error).__name__})") # The marker names what actually made it onto the card, which is the requested rack unless a slot failed. key = stack_key(spec for spec in specs if spec.adapter in active) if state["fused"] is not None and state["fused"] != key: pipe.unfuse_lora() state["fused"] = None state["repeats"] = state["repeats"] + 1 if key == state["key"] else 1 state["key"] = key if not active: pipe.disable_lora() return "; ".join(["none", *notes]) if notes else "none" if state["fused"] == key and not allow_fold: # The right rack, in the wrong form. A caller that passes `allow_fold=False` is going to re-weight part # of it partway through the request, and a folded adapter carries its weight in the checkpoint where # `set_adapters` cannot reach it - so the fold comes out rather than the re-weighting silently doing # nothing. Only a request that changes weights mid-flight pays this, and only when the previous one # happened to leave the same rack folded. print("[lora] unfolding: this request re-weights the rack between passes", flush=True) pipe.unfuse_lora() state["fused"] = None if state["fused"] == key: # The weights already carry this rack. Nothing to activate, nothing to pay. applied = ", ".join(f"{spec.label} @ {spec.weight:g}" for spec in specs if spec.adapter in active) return "; ".join([f"{applied} (folded)", *notes]) stage(f"activating {len(active)} LoRA") _evict(pipe, keep=set(active), state=state) pipe.set_adapters(active, adapter_weights=weights) pipe.enable_lora() state["active"] = (list(active), list(weights)) folded = "" if allow_fold and (fold_now or (FUSE_AFTER and state["repeats"] >= FUSE_AFTER)): # `set_adapters` above put each slot's weight on its adapter, and PEFT merges with exactly that scaling — # so the pipeline-level scale stays at 1.0. Merged adapters are skipped by the LoRA path, not applied twice. print(f"[lora] folding {len(active)} adapter(s) into the weights", flush=True) stage(f"folding {len(active)} LoRA into the weights") pipe.fuse_lora(adapter_names=active, lora_scale=1.0) state["fused"] = key folded = " (folded)" applied = ", ".join(f"{spec.label} @ {spec.weight:g}" for spec in specs if spec.adapter in active) return "; ".join([applied + folded, *notes]) def set_scale(pipe, factor: float, adapters=None) -> str: """Re-weight part of the rack by `factor` for whatever runs next, and say what happened. `adapters` is the subset to touch - everything else keeps the weight the request asked for. That distinction is the point: a distilled adapter is what misbehaves on a partial trajectory, and turning the whole rack off to be rid of it would take the look with it. Free while the rack is unfused, since `set_adapters` only rewrites a scalar per adapter, and impossible once it is folded, because the weights then carry the rack rather than reference it. """ state = _state(pipe) active, weights = state.get("active", ([], [])) if not active: return "no rack" if state["fused"] is not None: return "rack is folded, left as it is" touched = set(active) if adapters is None else set(adapters) & set(active) scaled = [w * float(factor) if name in touched else w for name, w in zip(active, weights)] if not any(scaled): pipe.disable_lora() return "rack off" pipe.set_adapters(active, adapter_weights=scaled) pipe.enable_lora() changed = [f"{name} @ {w:g}" for name, w in zip(active, scaled) if name in touched] kept = len(active) - len(changed) return ", ".join(changed) + (f" · {kept} left as asked" if kept else "") def _evict(pipe, keep: set[str], state: dict) -> None: """Drop resident adapters outside the current rack once too many have piled up on the card. Never while a fold is standing: deleting an adapter whose delta sits in the weights would strand it there. """ if state["fused"] is not None: return resident = _resident(pipe) if len(resident) <= MAX_RESIDENT: return stale = sorted(resident - keep) if stale: print(f"[lora] evicting {len(stale)} adapter(s): {', '.join(stale)}", flush=True) pipe.delete_adapters(stale)