BoxOfColors commited on
Commit
922310d
·
1 Parent(s): 58e91f5

Recalibrate GPU duration reservations, drop deprecated warnings

Browse files

Live logs showed all three models finishing (warm) in 18-39s while
every call reserved a blanket 120s of ZeroGPU quota — the hard-coded
120s floor plus generous load_overhead guesses were never tuned
against real GPU timings. ZeroGPU's admission check gates on the
requested duration against remaining quota, so over-reserving causes
premature "quota exceeded" errors well before quota is actually
exhausted.

- Lower the global floor from 120s to 45s.
- Recalibrate each model's load_overhead to ~1.5-2x its measured warm
end-to-end time (TARO 120->35, MMAudio 30->35, HunyuanFoley 90->55,
with extra margin kept for HunyuanFoley's larger cold-disk-read risk).
- Drop the deprecated local_dir_use_symlinks kwarg from hf_hub_download
calls.
- Silence the harmless weight_norm FutureWarning from vendored
HunyuanVideo-Foley DAC VAE loading code.

Files changed (1) hide show
  1. app.py +29 -13
app.py CHANGED
@@ -20,9 +20,18 @@ import tempfile
20
  import random
21
  import threading
22
  import time
 
23
  from concurrent.futures import ThreadPoolExecutor, as_completed
24
  from pathlib import Path
25
 
 
 
 
 
 
 
 
 
26
  import torch
27
  import numpy as np
28
  import torchaudio
@@ -121,22 +130,22 @@ def _dl_taro():
121
  def _dl_mmaudio():
122
  """Download MMAudio .pth files and return their local paths."""
123
  m = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/mmaudio_large_44k_v2.pth",
124
- cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_WEIGHTS_DIR), local_dir_use_symlinks=False)
125
  v = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/v1-44.pth",
126
- cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR), local_dir_use_symlinks=False)
127
  s = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/synchformer_state_dict.pth",
128
- cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR), local_dir_use_symlinks=False)
129
  print("MMAudio checkpoints downloaded.")
130
  return m, v, s
131
 
132
  def _dl_hunyuan():
133
  """Download HunyuanVideoFoley .pth files."""
134
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/hunyuanvideo_foley.pth",
135
- cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR), local_dir_use_symlinks=False)
136
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/vae_128d_48k.pth",
137
- cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR), local_dir_use_symlinks=False)
138
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/synchformer_state_dict.pth",
139
- cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR), local_dir_use_symlinks=False)
140
  print("HunyuanVideoFoley checkpoints downloaded.")
141
 
142
  def _populate_hf_cache_from_mirror(orig_repo_id, mirror_subpath):
@@ -743,7 +752,8 @@ MODEL_CONFIGS = {
743
  "window_s": TARO_MODEL_DUR, # 8.192 s
744
  "sr": TARO_SR, # 16000 (output resampled to TARGET_SR)
745
  "secs_per_step": 0.025, # measured 0.023 s/step on H200
746
- "load_overhead": 120, # CAVP+onset+MMDiT+VAE+vocoder load + feature extraction
 
747
  "tab_prefix": "taro",
748
  "label": "TARO",
749
  "regen_fn": None, # set after function definitions (avoids forward-ref)
@@ -752,7 +762,8 @@ MODEL_CONFIGS = {
752
  "window_s": 8.0, # MMAudio's fixed generation window
753
  "sr": 48000, # resampled from 44100 in post-processing
754
  "secs_per_step": 0.25, # measured 0.230 s/step on H200
755
- "load_overhead": 30, # 15s warm + 15s model init
 
756
  "tab_prefix": "mma",
757
  "label": "MMAudio",
758
  "regen_fn": None,
@@ -761,7 +772,10 @@ MODEL_CONFIGS = {
761
  "window_s": 15.0, # HunyuanFoley max video duration
762
  "sr": 48000,
763
  "secs_per_step": 0.50, # wall-time ~0.607s/step (incl. VAE decode + feature extraction)
764
- "load_overhead": 90, # cold disk: ~73s for 10 GB weights + ~8s aux models
 
 
 
765
  "tab_prefix": "hf",
766
  "label": "HunyuanFoley",
767
  "regen_fn": None,
@@ -813,10 +827,12 @@ def _catch_oom(fn):
813
 
814
 
815
  def _clamp_duration(secs: float, label: str) -> int:
816
- """Clamp a raw GPU-seconds estimate to [120, GPU_DURATION_CAP] and log it.
817
- ZeroGPU Pro users get up to 300 s per call; 120 s floor covers cold-disk
818
- model loads (e.g. HunyuanFoley XXL ~73 s on first access)."""
819
- result = min(GPU_DURATION_CAP, max(120, int(secs)))
 
 
820
  print(f"[duration] {label}: {secs:.0f}s raw → {result}s reserved")
821
  return result
822
 
 
20
  import random
21
  import threading
22
  import time
23
+ import warnings
24
  from concurrent.futures import ThreadPoolExecutor, as_completed
25
  from pathlib import Path
26
 
27
+ # HunyuanVideo-Foley's DAC VAE loads its weights via the pre-parametrization
28
+ # torch.nn.utils.weight_norm API — harmless, but noisy on every model load.
29
+ warnings.filterwarnings(
30
+ "ignore",
31
+ message="`torch.nn.utils.weight_norm` is deprecated",
32
+ category=FutureWarning,
33
+ )
34
+
35
  import torch
36
  import numpy as np
37
  import torchaudio
 
130
  def _dl_mmaudio():
131
  """Download MMAudio .pth files and return their local paths."""
132
  m = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/mmaudio_large_44k_v2.pth",
133
+ cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_WEIGHTS_DIR))
134
  v = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/v1-44.pth",
135
+ cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR))
136
  s = hf_hub_download(repo_id=CKPT_REPO_ID, filename="MMAudio/synchformer_state_dict.pth",
137
+ cache_dir=CACHE_DIR, local_dir=str(MMAUDIO_EXT_DIR))
138
  print("MMAudio checkpoints downloaded.")
139
  return m, v, s
140
 
141
  def _dl_hunyuan():
142
  """Download HunyuanVideoFoley .pth files."""
143
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/hunyuanvideo_foley.pth",
144
+ cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
145
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/vae_128d_48k.pth",
146
+ cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
147
  hf_hub_download(repo_id=CKPT_REPO_ID, filename="HunyuanVideo-Foley/synchformer_state_dict.pth",
148
+ cache_dir=CACHE_DIR, local_dir=str(HUNYUAN_MODEL_DIR))
149
  print("HunyuanVideoFoley checkpoints downloaded.")
150
 
151
  def _populate_hf_cache_from_mirror(orig_repo_id, mirror_subpath):
 
752
  "window_s": TARO_MODEL_DUR, # 8.192 s
753
  "sr": TARO_SR, # 16000 (output resampled to TARGET_SR)
754
  "secs_per_step": 0.025, # measured 0.023 s/step on H200
755
+ "load_overhead": 35, # CAVP+onset+MMDiT+VAE+vocoder load + feature extraction
756
+ # (measured ~18s warm end-to-end on RTX Pro 6000 Blackwell; ~2x margin)
757
  "tab_prefix": "taro",
758
  "label": "TARO",
759
  "regen_fn": None, # set after function definitions (avoids forward-ref)
 
762
  "window_s": 8.0, # MMAudio's fixed generation window
763
  "sr": 48000, # resampled from 44100 in post-processing
764
  "secs_per_step": 0.25, # measured 0.230 s/step on H200
765
+ "load_overhead": 35, # 15s warm + 15s model init
766
+ # (measured ~23s warm end-to-end on RTX Pro 6000 Blackwell; ~1.5x margin)
767
  "tab_prefix": "mma",
768
  "label": "MMAudio",
769
  "regen_fn": None,
 
772
  "window_s": 15.0, # HunyuanFoley max video duration
773
  "sr": 48000,
774
  "secs_per_step": 0.50, # wall-time ~0.607s/step (incl. VAE decode + feature extraction)
775
+ "load_overhead": 55, # SigLIP2 + Synchformer + CLAP + main model + DAC VAE load
776
+ # (measured ~39s warm end-to-end on RTX Pro 6000 Blackwell; extra
777
+ # margin kept vs. TARO/MMAudio for first-load cold-disk-cache risk
778
+ # on the ~10 GB XXL checkpoint)
779
  "tab_prefix": "hf",
780
  "label": "HunyuanFoley",
781
  "regen_fn": None,
 
827
 
828
 
829
  def _clamp_duration(secs: float, label: str) -> int:
830
+ """Clamp a raw GPU-seconds estimate to [45, GPU_DURATION_CAP] and log it.
831
+ ZeroGPU Pro users get up to 300 s per call; 45 s floor covers per-call
832
+ scheduling/queue overhead without over-reserving quota on every warm
833
+ call (each model's own load_overhead in MODEL_CONFIGS covers its
834
+ model-load cost above this floor)."""
835
+ result = min(GPU_DURATION_CAP, max(45, int(secs)))
836
  print(f"[duration] {label}: {secs:.0f}s raw → {result}s reserved")
837
  return result
838