Spaces:
Running
on
Zero
Running
on
Zero
Commit
Β·
94c0532
1
Parent(s):
efa664c
Fixed i2v
Browse files
app.py
CHANGED
@@ -2,8 +2,8 @@ import spaces
|
|
2 |
import torch
|
3 |
from diffusers import AutoencoderKLWan, WanImageToVideoPipeline, UniPCMultistepScheduler, WanTransformer3DModel, AutoModel, DiffusionPipeline
|
4 |
from diffusers.utils import export_to_video
|
5 |
-
from transformers import CLIPVisionModel, UMT5EncoderModel
|
6 |
-
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
|
7 |
import tempfile
|
8 |
import re
|
9 |
import os
|
@@ -29,9 +29,17 @@ T2V_LORA_FILENAME = "FusionX_LoRa/Wan2.1_T2V_14B_FusionX_LoRA.safetensors"
|
|
29 |
print("π Loading I2V pipeline from single file...")
|
30 |
i2v_pipe = None
|
31 |
try:
|
32 |
-
# Load components needed for the pipeline from the base model repo
|
33 |
i2v_image_encoder = CLIPVisionModel.from_pretrained(I2V_BASE_MODEL_ID, subfolder="image_encoder", torch_dtype=torch.float32)
|
34 |
i2v_vae = AutoencoderKLWan.from_pretrained(I2V_BASE_MODEL_ID, subfolder="vae", torch_dtype=torch.float32)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
|
36 |
# Load the main transformer from the repo and filename
|
37 |
i2v_transformer = WanTransformer3DModel.from_single_file(
|
@@ -42,10 +50,13 @@ try:
|
|
42 |
# Manually assemble the pipeline with the custom transformer
|
43 |
i2v_pipe = WanImageToVideoPipeline(
|
44 |
vae=i2v_vae,
|
|
|
|
|
45 |
image_encoder=i2v_image_encoder,
|
|
|
|
|
46 |
transformer=i2v_transformer
|
47 |
)
|
48 |
-
i2v_pipe.scheduler = UniPCMultistepScheduler.from_config(i2v_pipe.scheduler.config, flow_shift=8.0)
|
49 |
i2v_pipe.to("cuda")
|
50 |
print("β
I2V pipeline loaded successfully from single file.")
|
51 |
except Exception as e:
|
|
|
2 |
import torch
|
3 |
from diffusers import AutoencoderKLWan, WanImageToVideoPipeline, UniPCMultistepScheduler, WanTransformer3DModel, AutoModel, DiffusionPipeline
|
4 |
from diffusers.utils import export_to_video
|
5 |
+
from transformers import CLIPVisionModel, UMT5EncoderModel, CLIPTokenizer, CLIPTextModel, CLIPImageProcessor
|
6 |
+
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
|
7 |
import tempfile
|
8 |
import re
|
9 |
import os
|
|
|
29 |
print("π Loading I2V pipeline from single file...")
|
30 |
i2v_pipe = None
|
31 |
try:
|
32 |
+
# Load ALL components needed for the pipeline from the base model repo
|
33 |
i2v_image_encoder = CLIPVisionModel.from_pretrained(I2V_BASE_MODEL_ID, subfolder="image_encoder", torch_dtype=torch.float32)
|
34 |
i2v_vae = AutoencoderKLWan.from_pretrained(I2V_BASE_MODEL_ID, subfolder="vae", torch_dtype=torch.float32)
|
35 |
+
i2v_text_encoder = CLIPTextModel.from_pretrained(I2V_BASE_MODEL_ID, subfolder="text_encoder", torch_dtype=torch.bfloat16)
|
36 |
+
i2v_tokenizer = CLIPTokenizer.from_pretrained(I2V_BASE_MODEL_ID, subfolder="tokenizer")
|
37 |
+
i2v_image_processor = CLIPImageProcessor.from_pretrained(I2V_BASE_MODEL_ID, subfolder="image_processor")
|
38 |
+
|
39 |
+
# Create scheduler with custom flow_shift
|
40 |
+
scheduler_config = UniPCMultistepScheduler.load_config(I2V_BASE_MODEL_ID, subfolder="scheduler")
|
41 |
+
scheduler_config['flow_shift'] = 8.0
|
42 |
+
i2v_scheduler = UniPCMultistepScheduler.from_config(scheduler_config)
|
43 |
|
44 |
# Load the main transformer from the repo and filename
|
45 |
i2v_transformer = WanTransformer3DModel.from_single_file(
|
|
|
50 |
# Manually assemble the pipeline with the custom transformer
|
51 |
i2v_pipe = WanImageToVideoPipeline(
|
52 |
vae=i2v_vae,
|
53 |
+
text_encoder=i2v_text_encoder,
|
54 |
+
tokenizer=i2v_tokenizer,
|
55 |
image_encoder=i2v_image_encoder,
|
56 |
+
image_processor=i2v_image_processor,
|
57 |
+
scheduler=i2v_scheduler,
|
58 |
transformer=i2v_transformer
|
59 |
)
|
|
|
60 |
i2v_pipe.to("cuda")
|
61 |
print("β
I2V pipeline loaded successfully from single file.")
|
62 |
except Exception as e:
|