thankfulcarp's picture
Fixed dimension bug
1ab10dc
import spaces
import torch
from diffusers import AutoencoderKLWan, WanImageToVideoPipeline, UniPCMultistepScheduler, WanTransformer3DModel, AutoModel, DiffusionPipeline
from diffusers.utils import export_to_video
from transformers import CLIPVisionModel, UMT5EncoderModel, CLIPTextModel, CLIPImageProcessor
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
import tempfile
import re
import os
import traceback
from huggingface_hub import list_repo_files
from huggingface_hub import hf_hub_download
import numpy as np
from PIL import Image
import gradio as gr
import json
import random
# --- I2V (Image-to-Video) Configuration ---
I2V_BASE_MODEL_ID = "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers" # Used for VAE/encoder components
I2V_FUSIONX_REPO_ID = "vrgamedevgirl84/Wan14BT2VFusioniX"
I2V_FUSIONX_FILENAME = "Wan14Bi2vFusioniX.safetensors"
# --- I2V LoRA Configuration ---
I2V_LORA_REPO_ID = "DeepBeepMeep/Wan2.1"
I2V_LORA_SUBFOLDER = "loras_i2v"
# --- Load Pipelines ---
print("πŸš€ Loading I2V pipeline from single file...")
i2v_pipe = None
try:
# Load ALL components needed for the pipeline from the base model repo
i2v_image_encoder = CLIPVisionModel.from_pretrained(I2V_BASE_MODEL_ID, subfolder="image_encoder", torch_dtype=torch.float32)
i2v_vae = AutoencoderKLWan.from_pretrained(I2V_BASE_MODEL_ID, subfolder="vae", torch_dtype=torch.float32)
i2v_text_encoder = UMT5EncoderModel.from_pretrained(I2V_BASE_MODEL_ID, subfolder="text_encoder", torch_dtype=torch.bfloat16)
i2v_tokenizer = AutoTokenizer.from_pretrained(I2V_BASE_MODEL_ID, subfolder="tokenizer")
i2v_image_processor = CLIPImageProcessor.from_pretrained(I2V_BASE_MODEL_ID, subfolder="image_processor")
# Create scheduler with custom flow_shift
scheduler_config = UniPCMultistepScheduler.load_config(I2V_BASE_MODEL_ID, subfolder="scheduler")
scheduler_config['flow_shift'] = 8.0
i2v_scheduler = UniPCMultistepScheduler.from_config(scheduler_config)
# Load the main transformer from the repo and filename
i2v_transformer = WanTransformer3DModel.from_single_file(
"https://huggingface.co/vrgamedevgirl84/Wan14BT2VFusioniX/blob/main/Wan14Bi2vFusioniX.safetensors",
torch_dtype=torch.bfloat16
)
# Manually assemble the pipeline with the custom transformer
i2v_pipe = WanImageToVideoPipeline(
vae=i2v_vae,
text_encoder=i2v_text_encoder,
tokenizer=i2v_tokenizer,
image_encoder=i2v_image_encoder,
image_processor=i2v_image_processor,
scheduler=i2v_scheduler,
transformer=i2v_transformer
)
i2v_pipe.to("cuda")
print("βœ… I2V pipeline loaded successfully from single file.")
except Exception as e:
print(f"❌ Critical Error: Failed to load I2V pipeline from single file.")
traceback.print_exc()
# --- LoRA Discovery ---
def get_available_presets(repo_id, subfolder):
"""
Fetches the list of available LoRA presets by looking for .lset files.
This is more robust as it ensures a preset and prompt info exists.
"""
try:
# Fetch all files from the repo to maintain compatibility with older library versions.
all_files = list_repo_files(repo_id=repo_id, repo_type='model')
# Manually filter for .lset files and get their names without the extension.
subfolder_path = f"{subfolder}/"
lset_files = [
os.path.splitext(f.split('/')[-1])[0] # Get filename without extension
for f in all_files
if f.startswith(subfolder_path) and f.endswith('.lset')
]
print(f"βœ… Discovered {len(lset_files)} LoRA presets in {repo_id}/{subfolder}")
return ["None"] + sorted(lset_files)
except Exception as e:
print(f"⚠️ Warning: Could not fetch LoRA presets from {repo_id}. LoRA selection will be disabled. Error: {e}")
return ["None"]
available_i2v_presets = get_available_presets(I2V_LORA_REPO_ID, I2V_LORA_SUBFOLDER) if i2v_pipe else ["None"]
# --- Constants and Configuration ---
MOD_VALUE = 16 # Changed to 16 for model compatibility
DEFAULT_H_SLIDER_VALUE = 480 # Default to 480p height
DEFAULT_W_SLIDER_VALUE = 640 # Default to 640p width
NEW_FORMULA_MAX_AREA = 640.0 * 480.0 # Default area for new images
LORA_MAX_AREA = 640.0 * 480.0 # Max area when using a LoRA
SLIDER_MIN_H, SLIDER_MAX_H = 128, 1024
SLIDER_MIN_W, SLIDER_MAX_W = 128, 1024
MAX_SEED = np.iinfo(np.int32).max
FIXED_FPS = 16
MIN_FRAMES_MODEL = 8
MAX_FRAMES_MODEL = 81
# --- Default Prompts ---
default_prompt_i2v = "Cinematic motion, smooth animation, detailed textures, dynamic lighting, professional cinematography"
default_negative_prompt = "Static image, no motion, blurred details, overexposed, underexposed, low quality, worst quality, JPEG artifacts, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, watermark, text, signature, three legs, many people in the background, walking backwards"
# --- LoRA Preset Helper Functions ---
def parse_lset_prompt(lset_prompt):
"""Parses a .lset prompt, resolving variables and highlighting them."""
# Find all variable declarations like ! {Subject}="woman"
variables = dict(re.findall(r'! \{(\w+)\}="([^"]+)"', lset_prompt))
# Remove the declaration lines to get the clean prompt template
prompt_template = re.sub(r'! \{\w+\}="[^"]+"\n?', '', lset_prompt).strip()
# Replace placeholders with their default values, highlighted with markdown
resolved_prompt = prompt_template
for key, value in variables.items():
# Highlight the default value to indicate it's a replaceable variable
highlighted_value = f"__{value}__"
resolved_prompt = resolved_prompt.replace(f"{{{key}}}", highlighted_value)
return resolved_prompt
def handle_lora_selection_change(preset_name, current_prompt, current_h, current_w, aspect_ratio):
"""
When a preset is selected, this function finds the corresponding .lset file,
parses it, appends the prompt, and resizes dimensions if they are too large.
"""
# Initialize updates to avoid changing UI elements unnecessarily
prompt_update = gr.update()
h_update = gr.update()
w_update = gr.update()
if not preset_name or preset_name == "None":
return prompt_update, h_update, w_update
# --- Handle Prompt ---
try:
lset_filename = f"{preset_name}.lset"
lset_path = hf_hub_download(
repo_id=I2V_LORA_REPO_ID, filename=lset_filename,
subfolder=I2V_LORA_SUBFOLDER, repo_type='model'
)
with open(lset_path, 'r', encoding='utf-8') as f:
lset_data = json.load(f)
if lset_prompt_raw := lset_data.get("prompt"):
resolved_prompt = parse_lset_prompt(lset_prompt_raw)
new_prompt = f"{current_prompt}\n\n{resolved_prompt}".strip()
gr.Info(f"βœ… Appended prompt from '{lset_filename}'. Replace highlighted text like __this__.")
prompt_update = gr.update(value=new_prompt)
except Exception as e:
print(f"Info: Could not process .lset for '{preset_name}'. Reason: {e}")
gr.Info(f"ℹ️ Error processing preset '{preset_name}'.")
# --- Handle Resolution ---
if current_h * current_w > LORA_MAX_AREA:
gr.Info(f"Resolution too high for LoRA. Scaling down to a 640x480 equivalent area.")
# aspect_ratio is W/H
if aspect_ratio > 0:
# Calculate ideal dimensions based on area, without premature rounding
calc_w = np.sqrt(LORA_MAX_AREA * aspect_ratio)
calc_h = np.sqrt(LORA_MAX_AREA / aspect_ratio)
# Round to the nearest multiple of MOD_VALUE
new_h = max(MOD_VALUE, round(calc_h / MOD_VALUE) * MOD_VALUE)
new_w = max(MOD_VALUE, round(calc_w / MOD_VALUE) * MOD_VALUE)
h_update = gr.update(value=new_h)
w_update = gr.update(value=new_w)
else: # Fallback if aspect ratio is invalid
h_update = gr.update(value=480)
w_update = gr.update(value=640)
return prompt_update, h_update, w_update
# --- Helper Functions ---
def sanitize_prompt_for_filename(prompt: str, max_len: int = 60) -> str:
"""Sanitizes a prompt string to be used as a valid filename."""
if not prompt:
prompt = "video"
sanitized = re.sub(r'[^\w\s_-]', '', prompt).strip()
sanitized = re.sub(r'[\s_-]+', '_', sanitized)
return sanitized[:max_len]
def update_linked_dimension(driving_value, other_value, aspect_ratio, mod_val, mode):
"""Updates a dimension slider based on the other, maintaining aspect ratio."""
# aspect_ratio is stored as W/H
if aspect_ratio is None or aspect_ratio == 0:
return gr.update() # Do nothing if aspect ratio is not set
if mode == 'h_drives_w':
# new_w = h * (W/H)
new_other_value = driving_value * aspect_ratio
else: # 'w_drives_h'
# new_h = w / (W/H)
new_other_value = driving_value / aspect_ratio
# Round to the nearest multiple of mod_val
new_other_value = max(mod_val, round(new_other_value / mod_val) * mod_val)
# Return an update only if the value has changed to prevent infinite loops
return gr.update(value=new_other_value) if int(new_other_value) != int(other_value) else gr.update()
def _calculate_new_dimensions_wan(pil_image, mod_val, calculation_max_area,
min_slider_h, max_slider_h,
min_slider_w, max_slider_w,
default_h, default_w):
orig_w, orig_h = pil_image.size
if orig_w <= 0 or orig_h <= 0:
return default_h, default_w
aspect_ratio = orig_h / orig_w
# Calculate ideal dimensions based on area, without premature rounding
calc_h = np.sqrt(calculation_max_area * aspect_ratio)
calc_w = np.sqrt(calculation_max_area / aspect_ratio)
# Round to the nearest multiple of mod_val
calc_h = max(mod_val, round(calc_h / mod_val) * mod_val)
calc_w = max(mod_val, round(calc_w / mod_val) * mod_val)
new_h = int(np.clip(calc_h, min_slider_h, (max_slider_h // mod_val) * mod_val))
new_w = int(np.clip(calc_w, min_slider_w, (max_slider_w // mod_val) * mod_val))
return new_h, new_w
def handle_image_upload_for_dims_wan(uploaded_pil_image):
default_aspect = DEFAULT_W_SLIDER_VALUE / DEFAULT_H_SLIDER_VALUE
if uploaded_pil_image is None:
return gr.update(value=DEFAULT_H_SLIDER_VALUE), gr.update(value=DEFAULT_W_SLIDER_VALUE), default_aspect
try:
# This function calculates initial slider positions based on a max area
new_h, new_w = _calculate_new_dimensions_wan(
uploaded_pil_image, MOD_VALUE, NEW_FORMULA_MAX_AREA,
SLIDER_MIN_H, SLIDER_MAX_H, SLIDER_MIN_W, SLIDER_MAX_W,
DEFAULT_H_SLIDER_VALUE, DEFAULT_W_SLIDER_VALUE
)
# We need the original image's true aspect ratio (W/H) for locking the sliders
orig_w, orig_h = uploaded_pil_image.size
aspect_ratio = orig_w / orig_h if orig_h > 0 else default_aspect
return gr.update(value=new_h), gr.update(value=new_w), aspect_ratio
except Exception as e:
gr.Warning("Error calculating new dimensions. Resetting to default.")
return gr.update(value=DEFAULT_H_SLIDER_VALUE), gr.update(value=DEFAULT_W_SLIDER_VALUE), default_aspect
# --- GPU Duration Estimators for @spaces.GPU ---
def get_i2v_duration(steps, duration_seconds):
"""Estimates GPU time for Image-to-Video generation."""
if steps > 8 and duration_seconds > 3: return 600
elif steps > 8 or duration_seconds > 3: return 300
else: return 150
def get_t2v_duration(steps, duration_seconds):
"""Estimates GPU time for Text-to-Video generation."""
if steps > 15 and duration_seconds > 4: return 700
elif steps > 15 or duration_seconds > 4: return 400
else: return 200
# --- Core Generation Functions ---
@spaces.GPU(duration_from_args=get_i2v_duration)
def generate_i2v_video(input_image, prompt, height, width,
negative_prompt, duration_seconds,
guidance_scale, steps, seed, randomize_seed,
preset_name, lora_weight,
progress=gr.Progress(track_tqdm=True)):
"""Generates a video from an initial image and a prompt."""
if input_image is None:
raise gr.Error("Please upload an input image for Image-to-Video generation.")
if i2v_pipe is None:
raise gr.Error("Image-to-Video pipeline is not available due to a loading error.")
target_h = max(MOD_VALUE, (int(height) // MOD_VALUE) * MOD_VALUE)
target_w = max(MOD_VALUE, (int(width) // MOD_VALUE) * MOD_VALUE)
# If a LoRA is used, enforce max resolution as a safety net
if preset_name and preset_name != "None":
if target_h * target_w > LORA_MAX_AREA:
print(f"⚠️ Warning: Resolution {target_w}x{target_h} is too high for LoRA. Rescaling to fit max area.")
aspect_ratio = target_w / target_h if target_h > 0 else 1.0
# Re-calculate w and h based on max area, without premature rounding
calc_w = np.sqrt(LORA_MAX_AREA * aspect_ratio)
calc_h = np.sqrt(LORA_MAX_AREA / aspect_ratio)
# Snap to MOD_VALUE by rounding to the nearest multiple
target_h = max(MOD_VALUE, round(calc_h / MOD_VALUE) * MOD_VALUE)
target_w = max(MOD_VALUE, round(calc_w / MOD_VALUE) * MOD_VALUE)
print(f" - Rescaled to: {target_w}x{target_h}")
# Calculate and adjust num_frames to be compatible with video codecs
target_frames = int(round(duration_seconds * FIXED_FPS))
adjusted_frames = 4 * round((target_frames - 1) / 4) + 1
num_frames = int(np.clip(adjusted_frames, MIN_FRAMES_MODEL, MAX_FRAMES_MODEL))
current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
resized_image = input_image.resize((target_w, target_h))
enhanced_prompt = f"{prompt}, cinematic quality, smooth motion, detailed animation, dynamic lighting"
lora_filename = None # Will be extracted from the .lset file
adapter_name = "i2v_lora"
try:
# If a preset is selected, load the corresponding LoRA
if preset_name and preset_name != "None":
lset_filename = f"{preset_name}.lset"
print(f"πŸš€ Processing preset: {preset_name}")
try:
lset_path = hf_hub_download(
repo_id=I2V_LORA_REPO_ID,
filename=lset_filename,
subfolder=I2V_LORA_SUBFOLDER,
repo_type='model'
)
with open(lset_path, 'r', encoding='utf-8') as f:
lset_data = json.load(f)
# Extract the LoRA filename from the .lset file
loras_list = lset_data.get("loras")
if not loras_list or not isinstance(loras_list, list) or len(loras_list) == 0:
raise gr.Error(f"Preset file '{lset_filename}' is invalid or does not specify a LoRA file.")
lora_filename = loras_list[0] # Use the first LoRA in the list
print(f" - Found LoRA file: {lora_filename}")
i2v_pipe.load_lora_weights(
I2V_LORA_REPO_ID,
weight_name=lora_filename,
adapter_name=adapter_name,
subfolder=I2V_LORA_SUBFOLDER
)
i2v_pipe.set_adapters([adapter_name], adapter_weights=[float(lora_weight)])
print(f" - LoRA '{lora_filename}' loaded successfully with weight {lora_weight}.")
except Exception as e:
raise gr.Error(f"Failed to load LoRA for preset '{preset_name}'. Reason: {e}")
with torch.inference_mode():
output_frames_list = i2v_pipe(
image=resized_image,
prompt=enhanced_prompt,
negative_prompt=negative_prompt,
height=target_h,
width=target_w,
num_frames=num_frames,
guidance_scale=float(guidance_scale),
num_inference_steps=int(steps),
generator=torch.Generator(device="cuda").manual_seed(current_seed)
).frames[0]
finally:
# Unload the LoRA to ensure a clean state for the next run
if lora_filename and hasattr(i2v_pipe, "unload_lora_weights"):
print(f"🧹 Unloading LoRA: {lora_filename}")
i2v_pipe.unload_lora_weights()
# Clear GPU cache to free up memory for the next run
if torch.cuda.is_available():
torch.cuda.empty_cache()
sanitized_prompt = sanitize_prompt_for_filename(prompt)
filename = f"i2v_{sanitized_prompt}_{current_seed}.mp4"
temp_dir = tempfile.mkdtemp()
video_path = os.path.join(temp_dir, filename)
export_to_video(output_frames_list, video_path, fps=FIXED_FPS)
return video_path, current_seed, gr.File(value=video_path, visible=True, label=f"πŸ“₯ Download: {filename}")
# --- Gradio UI Layout ---
with gr.Blocks() as demo:
with gr.Column(elem_classes=["main-container"]):
i2v_aspect_ratio = gr.State(value=DEFAULT_W_SLIDER_VALUE / DEFAULT_H_SLIDER_VALUE)
gr.Markdown("# Wan 2.1 Video Suite with Dynamic LoRA Presets")
gr.Markdown(
"""
Welcome! This space allows you to generate videos from images using the powerful Wan 2.1 model, enhanced with dynamic LoRA presets.
**How to use:**
1. Start in the **Image-to-Video** tab and upload your starting image.
2. Select a **LoRA Preset** from the dropdown to apply a unique style and automatically add a suggested prompt.
3. Customize the prompt, adjust settings like duration and resolution, and click **Generate I2V**!
"""
)
with gr.Tabs(elem_classes=["gr-tabs"]):
# --- Image-to-Video Tab ---
with gr.TabItem("πŸ–ΌοΈ Image-to-Video", id="i2v_tab"):
with gr.Row():
with gr.Column(elem_classes=["input-container"]):
i2v_input_image = gr.Image(
type="pil",
label="πŸ–ΌοΈ Input Image (auto-resizes H/W sliders)",
elem_classes=["image-upload"]
)
i2v_preset_name = gr.Dropdown(label="🎨 LoRA Preset", choices=available_i2v_presets, value="None", info="Select a preset to apply a LoRA and a suggested prompt.", interactive=len(available_i2v_presets) > 1)
i2v_prompt = gr.Textbox(
label="✏️ Prompt",
value=default_prompt_i2v, lines=3
)
i2v_duration = gr.Slider(
minimum=round(MIN_FRAMES_MODEL/FIXED_FPS,1),
maximum=round(MAX_FRAMES_MODEL/FIXED_FPS,1),
step=0.1, value=2, label="⏱️ Duration (seconds)",
info=f"Generates {MIN_FRAMES_MODEL}-{MAX_FRAMES_MODEL} frames at {FIXED_FPS}fps."
)
with gr.Accordion("βš™οΈ Advanced Settings", open=False):
i2v_neg_prompt = gr.Textbox(label="❌ Negative Prompt", value=default_negative_prompt, lines=4)
i2v_seed = gr.Slider(label="🎲 Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True)
i2v_rand_seed = gr.Checkbox(label="πŸ”€ Randomize seed", value=True, interactive=True)
i2v_lora_weight = gr.Slider(label="πŸ’ͺ LoRA Weight", minimum=0.0, maximum=2.0, step=0.1, value=0.8, interactive=True)
with gr.Row():
i2v_height = gr.Slider(minimum=SLIDER_MIN_H, maximum=SLIDER_MAX_H, step=MOD_VALUE, value=DEFAULT_H_SLIDER_VALUE, label=f"πŸ“ Height ({MOD_VALUE}px steps)")
i2v_width = gr.Slider(minimum=SLIDER_MIN_W, maximum=SLIDER_MAX_W, step=MOD_VALUE, value=DEFAULT_W_SLIDER_VALUE, label=f"πŸ“ Width ({MOD_VALUE}px steps)")
gr.Markdown("<p style='color: #ffcc00; font-size: 0.9em;'>⚠️ High resolutions can lead to out-of-memory errors. If generation fails, try a smaller size.</p>")
i2v_steps = gr.Slider(minimum=1, maximum=20, step=1, value=8, label="πŸš€ Inference Steps", info="8-10 recommended for great results.")
i2v_guidance = gr.Slider(minimum=0.0, maximum=20.0, step=0.5, value=1.0, label="🎯 Guidance Scale", visible=False)
i2v_generate_btn = gr.Button("🎬 Generate I2V", variant="primary", elem_classes=["generate-btn"])
with gr.Column(elem_classes=["output-container"]):
i2v_output_video = gr.Video(label="πŸŽ₯ Generated Video", autoplay=True, interactive=False)
i2v_download = gr.File(label="πŸ“₯ Download Video", visible=False)
# --- Event Handlers ---
# I2V Handlers
i2v_preset_name.change(
fn=handle_lora_selection_change,
inputs=[i2v_preset_name, i2v_prompt, i2v_height, i2v_width, i2v_aspect_ratio],
outputs=[i2v_prompt, i2v_height, i2v_width]
)
i2v_input_image.upload(
fn=handle_image_upload_for_dims_wan,
inputs=[i2v_input_image],
outputs=[i2v_height, i2v_width, i2v_aspect_ratio]
)
i2v_input_image.clear(
fn=lambda: (DEFAULT_H_SLIDER_VALUE, DEFAULT_W_SLIDER_VALUE, DEFAULT_W_SLIDER_VALUE / DEFAULT_H_SLIDER_VALUE),
inputs=[],
outputs=[i2v_height, i2v_width, i2v_aspect_ratio]
)
i2v_generate_btn.click(
fn=generate_i2v_video,
inputs=[i2v_input_image, i2v_prompt, i2v_height, i2v_width, i2v_neg_prompt, i2v_duration, i2v_guidance, i2v_steps, i2v_seed, i2v_rand_seed, i2v_preset_name, i2v_lora_weight],
outputs=[i2v_output_video, i2v_seed, i2v_download]
)
i2v_height.release(
fn=update_linked_dimension,
inputs=[i2v_height, i2v_width, i2v_aspect_ratio, gr.State(MOD_VALUE), gr.State('h_drives_w')],
outputs=[i2v_width]
)
i2v_width.release(
fn=update_linked_dimension,
inputs=[i2v_width, i2v_height, i2v_aspect_ratio, gr.State(MOD_VALUE), gr.State('w_drives_h')],
outputs=[i2v_height]
)
if __name__ == "__main__":
demo.queue().launch()