remove unused files

demo/app.py

@@ -1,224 +0,0 @@
-import gradio as gr
-import torch
-from transformers import AutoConfig, AutoModelForCausalLM
-from janus.models import MultiModalityCausalLM, VLChatProcessor
-from PIL import Image
-
-import numpy as np
-
-
-# Load model and processor
-model_path = "deepseek-ai/Janus-1.3B"
-config = AutoConfig.from_pretrained(model_path)
-language_config = config.language_config
-language_config._attn_implementation = 'eager'
-vl_gpt = AutoModelForCausalLM.from_pretrained(model_path,
-                                             language_config=language_config,
-                                             trust_remote_code=True)
-vl_gpt = vl_gpt.to(torch.bfloat16).cuda()
-
-vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
-tokenizer = vl_chat_processor.tokenizer
-cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'
-# Multimodal Understanding function
-@torch.inference_mode()
-# Multimodal Understanding function
-def multimodal_understanding(image, question, seed, top_p, temperature):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-    
-    # set seed
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed)
-    
-    conversation = [
-        {
-            "role": "User",
-            "content": f"<image_placeholder>\n{question}",
-            "images": [image],
-        },
-        {"role": "Assistant", "content": ""},
-    ]
-    
-    pil_images = [Image.fromarray(image)]
-    prepare_inputs = vl_chat_processor(
-        conversations=conversation, images=pil_images, force_batchify=True
-    ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
-    
-    
-    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
-    
-    outputs = vl_gpt.language_model.generate(
-        inputs_embeds=inputs_embeds,
-        attention_mask=prepare_inputs.attention_mask,
-        pad_token_id=tokenizer.eos_token_id,
-        bos_token_id=tokenizer.bos_token_id,
-        eos_token_id=tokenizer.eos_token_id,
-        max_new_tokens=512,
-        do_sample=False if temperature == 0 else True,
-        use_cache=True,
-        temperature=temperature,
-        top_p=top_p,
-    )
-    
-    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
-    return answer
-
-
-def generate(input_ids,
-             width,
-             height,
-             temperature: float = 1,
-             parallel_size: int = 5,
-             cfg_weight: float = 5,
-             image_token_num_per_image: int = 576,
-             patch_size: int = 16):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-    
-    tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device)
-    for i in range(parallel_size * 2):
-        tokens[i, :] = input_ids
-        if i % 2 != 0:
-            tokens[i, 1:-1] = vl_chat_processor.pad_id
-    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
-    generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).to(cuda_device)
-
-    pkv = None
-    for i in range(image_token_num_per_image):
-        outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds,
-                                             use_cache=True,
-                                             past_key_values=pkv)
-        pkv = outputs.past_key_values
-        hidden_states = outputs.last_hidden_state
-        logits = vl_gpt.gen_head(hidden_states[:, -1, :])
-        logit_cond = logits[0::2, :]
-        logit_uncond = logits[1::2, :]
-        logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
-        probs = torch.softmax(logits / temperature, dim=-1)
-        next_token = torch.multinomial(probs, num_samples=1)
-        generated_tokens[:, i] = next_token.squeeze(dim=-1)
-        next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)
-        img_embeds = vl_gpt.prepare_gen_img_embeds(next_token)
-        inputs_embeds = img_embeds.unsqueeze(dim=1)
-    patches = vl_gpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int),
-                                                 shape=[parallel_size, 8, width // patch_size, height // patch_size])
-
-    return generated_tokens.to(dtype=torch.int), patches
-
-def unpack(dec, width, height, parallel_size=5):
-    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
-    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
-
-    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
-    visual_img[:, :, :] = dec
-
-    return visual_img
-
-
-
-@torch.inference_mode()
-def generate_image(prompt,
-                   seed=None,
-                   guidance=5):
-    # Clear CUDA cache and avoid tracking gradients
-    torch.cuda.empty_cache()
-    # Set the seed for reproducible results
-    if seed is not None:
-        torch.manual_seed(seed)
-        torch.cuda.manual_seed(seed)
-        np.random.seed(seed)
-    width = 384
-    height = 384
-    parallel_size = 5
-    
-    with torch.no_grad():
-        messages = [{'role': 'User', 'content': prompt},
-                    {'role': 'Assistant', 'content': ''}]
-        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
-                                                                   sft_format=vl_chat_processor.sft_format,
-                                                                   system_prompt='')
-        text = text + vl_chat_processor.image_start_tag
-        input_ids = torch.LongTensor(tokenizer.encode(text))
-        output, patches = generate(input_ids,
-                                   width // 16 * 16,
-                                   height // 16 * 16,
-                                   cfg_weight=guidance,
-                                   parallel_size=parallel_size)
-        images = unpack(patches,
-                        width // 16 * 16,
-                        height // 16 * 16)
-
-        return [Image.fromarray(images[i]).resize((1024, 1024), Image.LANCZOS) for i in range(parallel_size)]
-
-        
-
-# Gradio interface
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    # with gr.Row():
-    with gr.Row():
-        image_input = gr.Image()
-        with gr.Column():
-            question_input = gr.Textbox(label="Question")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-        
-    understanding_button = gr.Button("Chat")
-    understanding_output = gr.Textbox(label="Response")
-
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-            [
-                "explain this meme",
-                "images/doge.png",
-            ],
-            [
-                "Convert the formula into latex code.",
-                "images/equation.png",
-            ],
-        ],
-        inputs=[question_input, image_input],
-    )
-    
-        
-    gr.Markdown(value="# Text-to-Image Generation")
-
-    
-    
-    with gr.Row():
-        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight")
-
-    prompt_input = gr.Textbox(label="Prompt")
-    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345)
-
-    generation_button = gr.Button("Generate Images")
-
-    image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
-
-    examples_t2i = gr.Examples(
-        label="Text to image generation examples. (Tips for designing prompts: Adding description like 'digital art' at the end of the prompt or writing the prompt in more detail can help produce better images!)",
-        examples=[
-            "Master shifu racoon wearing drip attire as a street gangster.",
-            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
-            "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
-        ],
-        inputs=prompt_input,
-    )
-    
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[image_input, question_input, und_seed_input, top_p, temperature],
-        outputs=understanding_output
-    )
-    
-    generation_button.click(
-        fn=generate_image,
-        inputs=[prompt_input, seed_input, cfg_weight_input],
-        outputs=image_output
-    )
-
-demo.launch(share=True)

demo/app_janusflow.py

@@ -1,247 +0,0 @@
-import gradio as gr
-import torch
-from janus.janusflow.models import MultiModalityCausalLM, VLChatProcessor
-from PIL import Image
-from diffusers.models import AutoencoderKL
-import numpy as np
-
-cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
-# Load model and processor
-model_path = "deepseek-ai/JanusFlow-1.3B"
-vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
-tokenizer = vl_chat_processor.tokenizer
-
-vl_gpt = MultiModalityCausalLM.from_pretrained(model_path)
-vl_gpt = vl_gpt.to(torch.bfloat16).to(cuda_device).eval()
-
-# remember to use bfloat16 dtype, this vae doesn't work with fp16
-vae = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae")
-vae = vae.to(torch.bfloat16).to(cuda_device).eval()
-
-# Multimodal Understanding function
-@torch.inference_mode()
-# Multimodal Understanding function
-def multimodal_understanding(image, question, seed, top_p, temperature):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-    
-    # set seed
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed)
-    
-    conversation = [
-        {
-            "role": "User",
-            "content": f"<image_placeholder>\n{question}",
-            "images": [image],
-        },
-        {"role": "Assistant", "content": ""},
-    ]
-    
-    pil_images = [Image.fromarray(image)]
-    prepare_inputs = vl_chat_processor(
-        conversations=conversation, images=pil_images, force_batchify=True
-    ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
-    
-    
-    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
-    
-    outputs = vl_gpt.language_model.generate(
-        inputs_embeds=inputs_embeds,
-        attention_mask=prepare_inputs.attention_mask,
-        pad_token_id=tokenizer.eos_token_id,
-        bos_token_id=tokenizer.bos_token_id,
-        eos_token_id=tokenizer.eos_token_id,
-        max_new_tokens=512,
-        do_sample=False if temperature == 0 else True,
-        use_cache=True,
-        temperature=temperature,
-        top_p=top_p,
-    )
-    
-    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
-
-    return answer
-
-
-@torch.inference_mode()
-def generate(
-    input_ids,
-    cfg_weight: float = 2.0,
-    num_inference_steps: int = 30
-):
-    # we generate 5 images at a time, *2 for CFG
-    tokens = torch.stack([input_ids] * 10).cuda()
-    tokens[5:, 1:] = vl_chat_processor.pad_id
-    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
-    print(inputs_embeds.shape)
-
-    # we remove the last <bog> token and replace it with t_emb later
-    inputs_embeds = inputs_embeds[:, :-1, :] 
-    
-    # generate with rectified flow ode
-    # step 1: encode with vision_gen_enc
-    z = torch.randn((5, 4, 48, 48), dtype=torch.bfloat16).cuda()
-    
-    dt = 1.0 / num_inference_steps
-    dt = torch.zeros_like(z).cuda().to(torch.bfloat16) + dt
-    
-    # step 2: run ode
-    attention_mask = torch.ones((10, inputs_embeds.shape[1]+577)).to(vl_gpt.device)
-    attention_mask[5:, 1:inputs_embeds.shape[1]] = 0
-    attention_mask = attention_mask.int()
-    for step in range(num_inference_steps):
-        # prepare inputs for the llm
-        z_input = torch.cat([z, z], dim=0) # for cfg
-        t = step / num_inference_steps * 1000.
-        t = torch.tensor([t] * z_input.shape[0]).to(dt)
-        z_enc = vl_gpt.vision_gen_enc_model(z_input, t)
-        z_emb, t_emb, hs = z_enc[0], z_enc[1], z_enc[2]
-        z_emb = z_emb.view(z_emb.shape[0], z_emb.shape[1], -1).permute(0, 2, 1)
-        z_emb = vl_gpt.vision_gen_enc_aligner(z_emb)
-        llm_emb = torch.cat([inputs_embeds, t_emb.unsqueeze(1), z_emb], dim=1)
-
-        # input to the llm
-        # we apply attention mask for CFG: 1 for tokens that are not masked, 0 for tokens that are masked.
-        if step == 0:
-            outputs = vl_gpt.language_model.model(inputs_embeds=llm_emb, 
-                                             use_cache=True, 
-                                             attention_mask=attention_mask,
-                                             past_key_values=None)
-            past_key_values = []
-            for kv_cache in past_key_values:
-                k, v = kv_cache[0], kv_cache[1]
-                past_key_values.append((k[:, :, :inputs_embeds.shape[1], :], v[:, :, :inputs_embeds.shape[1], :]))
-            past_key_values = tuple(past_key_values)
-        else:
-            outputs = vl_gpt.language_model.model(inputs_embeds=llm_emb, 
-                                             use_cache=True, 
-                                             attention_mask=attention_mask,
-                                             past_key_values=past_key_values)
-        hidden_states = outputs.last_hidden_state
-        
-        # transform hidden_states back to v
-        hidden_states = vl_gpt.vision_gen_dec_aligner(vl_gpt.vision_gen_dec_aligner_norm(hidden_states[:, -576:, :]))
-        hidden_states = hidden_states.reshape(z_emb.shape[0], 24, 24, 768).permute(0, 3, 1, 2)
-        v = vl_gpt.vision_gen_dec_model(hidden_states, hs, t_emb)
-        v_cond, v_uncond = torch.chunk(v, 2)
-        v = cfg_weight * v_cond - (cfg_weight-1.) * v_uncond
-        z = z + dt * v
-        
-    # step 3: decode with vision_gen_dec and sdxl vae
-    decoded_image = vae.decode(z / vae.config.scaling_factor).sample
-    
-    images = decoded_image.float().clip_(-1., 1.).permute(0,2,3,1).cpu().numpy()
-    images = ((images+1) / 2. * 255).astype(np.uint8)
-    
-    return images
-    
-def unpack(dec, width, height, parallel_size=5):
-    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
-    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
-
-    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
-    visual_img[:, :, :] = dec
-
-    return visual_img
-
-
-@torch.inference_mode()
-def generate_image(prompt,
-                   seed=None,
-                   guidance=5,
-                   num_inference_steps=30):
-    # Clear CUDA cache and avoid tracking gradients
-    torch.cuda.empty_cache()
-    # Set the seed for reproducible results
-    if seed is not None:
-        torch.manual_seed(seed)
-        torch.cuda.manual_seed(seed)
-        np.random.seed(seed)
-    
-    with torch.no_grad():
-        messages = [{'role': 'User', 'content': prompt},
-                    {'role': 'Assistant', 'content': ''}]
-        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
-                                                                   sft_format=vl_chat_processor.sft_format,
-                                                                   system_prompt='')
-        text = text + vl_chat_processor.image_start_tag
-        input_ids = torch.LongTensor(tokenizer.encode(text))
-        images = generate(input_ids,
-                                   cfg_weight=guidance,
-                                   num_inference_steps=num_inference_steps)
-        return [Image.fromarray(images[i]).resize((1024, 1024), Image.LANCZOS) for i in range(images.shape[0])]
-
-        
-
-# Gradio interface
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    # with gr.Row():
-    with gr.Row():
-        image_input = gr.Image()
-        with gr.Column():
-            question_input = gr.Textbox(label="Question")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-        
-    understanding_button = gr.Button("Chat")
-    understanding_output = gr.Textbox(label="Response")
-
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-            [
-                "explain this meme",
-                "./images/doge.png",
-            ],
-            [
-                "Convert the formula into latex code.",
-                "./images/equation.png",
-            ],
-        ],
-        inputs=[question_input, image_input],
-    )
-    
-        
-    gr.Markdown(value="# Text-to-Image Generation")
-
-    
-    
-    with gr.Row():
-        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=2, step=0.5, label="CFG Weight")
-        step_input = gr.Slider(minimum=1, maximum=50, value=30, step=1, label="Number of Inference Steps")
-
-    prompt_input = gr.Textbox(label="Prompt")
-    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345)
-
-    generation_button = gr.Button("Generate Images")
-
-    image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
-
-    examples_t2i = gr.Examples(
-        label="Text to image generation examples.",
-        examples=[
-            "Master shifu racoon wearing drip attire as a street gangster.",
-            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
-            "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
-        ],
-        inputs=prompt_input,
-    )
-    
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[image_input, question_input, und_seed_input, top_p, temperature],
-        outputs=understanding_output
-    )
-    
-    generation_button.click(
-        fn=generate_image,
-        inputs=[prompt_input, seed_input, cfg_weight_input, step_input],
-        outputs=image_output
-    )
-
-demo.launch(share=True)

demo/app_januspro.py

@@ -1,294 +0,0 @@
-import gradio as gr
-import torch
-from transformers import AutoConfig, AutoModelForCausalLM
-from janus.models import MultiModalityCausalLM, VLChatProcessor
-from janus.utils.io import load_pil_images
-from demo.cam import generate_gradcam, GradCAM, AttentionGuidedCAM
-from PIL import Image
-from einops import rearrange
-
-import numpy as np
-import os
-import time
-# import spaces  # Import spaces for ZeroGPU compatibility
-
-
-# Load model and processor
-# model_path = "deepseek-ai/Janus-Pro-7B"
-model_path = "deepseek-ai/Janus-Pro-1B"
-config = AutoConfig.from_pretrained(model_path)
-language_config = config.language_config
-language_config._attn_implementation = 'eager'
-vl_gpt = AutoModelForCausalLM.from_pretrained(model_path,
-                                             language_config=language_config,
-                                             trust_remote_code=True)
-
-dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float16
-# dtype = torch.bfloat32 if torch.cuda.is_available() else torch.float32
-
-if torch.cuda.is_available():
-    vl_gpt = vl_gpt.to(dtype).cuda()
-else:
-    # vl_gpt = vl_gpt.to(torch.float16)
-    torch.set_default_device("mps")
-    vl_gpt = vl_gpt.to(dtype)
-
-vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
-tokenizer = vl_chat_processor.tokenizer
-cuda_device = 'cuda' if torch.cuda.is_available() else 'mps'
-
-
-
-# @torch.inference_mode() # cancel inference, for gradcam
-# @spaces.GPU(duration=120) 
-# Multimodal Understanding function
-def multimodal_understanding(image, question, seed, top_p, temperature, target_token_idx):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-
-
-    for param in vl_gpt.parameters():
-        param.requires_grad = True
-    
-    # set seed
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed)
-    
-
-    # Get the last transformer block of the Vision Transformer (ViT)
-
-
-    conversation = [
-        {
-            "role": "<|User|>",
-            "content": f"<image_placeholder>\n{question}",
-            "images": [image],
-        },
-        {"role": "<|Assistant|>", "content": ""},
-    ]
-    
-    pil_images = [Image.fromarray(image)]
-    prepare_inputs = vl_chat_processor(
-        conversations=conversation, images=pil_images, force_batchify=True
-    ).to(cuda_device, dtype=dtype)
-    
-
-
-    
-    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
-
-    # print("prepared inputs", prepare_inputs)
-    
-
-    outputs = vl_gpt.language_model.generate(
-        inputs_embeds=inputs_embeds,
-        attention_mask=prepare_inputs.attention_mask,
-        pad_token_id=tokenizer.eos_token_id,
-        bos_token_id=tokenizer.bos_token_id,
-        eos_token_id=tokenizer.eos_token_id,
-        max_new_tokens=512,
-        do_sample=False if temperature == 0 else True,
-        use_cache=True,
-        temperature=temperature,
-        top_p=top_p,
-    )
-
-
-    
-    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
-    print("answer generated")
-
-
-    target_layer = vl_gpt.vision_model.vision_tower.blocks
-
-    gradcam = AttentionGuidedCAM(vl_gpt, target_layer)
-    cam_tensor, output, grid_size = gradcam.generate_cam(prepare_inputs, tokenizer, temperature, top_p, target_token_idx)
-    cam_grid = cam_tensor.reshape(grid_size, grid_size)
-    cam = generate_gradcam(cam_grid, image)
-
-    output_arr = output.logits.detach().to(float).to("cpu").numpy()
-    predicted_ids = np.argmax(output_arr, axis=-1) # [1, num_tokens]
-    predicted_ids = predicted_ids.squeeze(0) # [num_tokens]
-    target_token_decoded = tokenizer.decode(predicted_ids[target_token_idx].tolist())
-
-    return answer, [cam], target_token_decoded
-
-
-def generate(input_ids,
-             width,
-             height,
-             temperature: float = 1,
-             parallel_size: int = 5,
-             cfg_weight: float = 5,
-             image_token_num_per_image: int = 576,
-             patch_size: int = 16):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-    
-    tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device)
-    for i in range(parallel_size * 2):
-        tokens[i, :] = input_ids
-        if i % 2 != 0:
-            tokens[i, 1:-1] = vl_chat_processor.pad_id
-    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
-    generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).to(cuda_device)
-
-    pkv = None
-    for i in range(image_token_num_per_image):
-        with torch.no_grad():
-            outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds,
-                                                use_cache=True,
-                                                past_key_values=pkv)
-            pkv = outputs.past_key_values
-            hidden_states = outputs.last_hidden_state
-            logits = vl_gpt.gen_head(hidden_states[:, -1, :])
-            logit_cond = logits[0::2, :]
-            logit_uncond = logits[1::2, :]
-            logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
-            probs = torch.softmax(logits / temperature, dim=-1)
-            next_token = torch.multinomial(probs, num_samples=1)
-            generated_tokens[:, i] = next_token.squeeze(dim=-1)
-            next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)
-
-            img_embeds = vl_gpt.prepare_gen_img_embeds(next_token)
-            inputs_embeds = img_embeds.unsqueeze(dim=1)
-
-    
-
-    patches = vl_gpt.gen_vision_model.decode_code(generated_tokens.to(dtype=torch.int),
-                                                 shape=[parallel_size, 8, width // patch_size, height // patch_size])
-
-    return generated_tokens.to(dtype=torch.int), patches
-
-def unpack(dec, width, height, parallel_size=5):
-    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
-    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
-
-    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
-    visual_img[:, :, :] = dec
-
-    return visual_img
-
-
-
-@torch.inference_mode()
-# @spaces.GPU(duration=120)  # Specify a duration to avoid timeout
-def generate_image(prompt,
-                   seed=None,
-                   guidance=5,
-                   t2i_temperature=1.0):
-    # Clear CUDA cache and avoid tracking gradients
-    torch.cuda.empty_cache()
-    # Set the seed for reproducible results
-    if seed is not None:
-        torch.manual_seed(seed)
-        torch.cuda.manual_seed(seed)
-        np.random.seed(seed)
-    width = 384
-    height = 384
-    parallel_size = 5
-    
-    with torch.no_grad():
-        messages = [{'role': '<|User|>', 'content': prompt},
-                    {'role': '<|Assistant|>', 'content': ''}]
-        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(conversations=messages,
-                                                                   sft_format=vl_chat_processor.sft_format,
-                                                                   system_prompt='')
-        text = text + vl_chat_processor.image_start_tag
-        
-        input_ids = torch.LongTensor(tokenizer.encode(text))
-        output, patches = generate(input_ids,
-                                   width // 16 * 16,
-                                   height // 16 * 16,
-                                   cfg_weight=guidance,
-                                   parallel_size=parallel_size,
-                                   temperature=t2i_temperature)
-        images = unpack(patches,
-                        width // 16 * 16,
-                        height // 16 * 16,
-                        parallel_size=parallel_size)
-
-        return [Image.fromarray(images[i]).resize((768, 768), Image.LANCZOS) for i in range(parallel_size)]
-        
-
-# Gradio interface
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    with gr.Row():
-        with gr.Column():
-            image_input = gr.Image()
-            saliency_map_output = gr.Gallery(label="Saliency Map", columns=1, rows=1, height=300)
-
-        with gr.Column():
-            question_input = gr.Textbox(label="Question")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-            target_token_idx = gr.Number(label="target_token_idx", precision=0, value=300)
-        
-    understanding_button = gr.Button("Chat")
-    understanding_output = gr.Textbox(label="Response")
-    understanding_target_token_decoded_output = gr.Textbox(label="Target Token Decoded")
-
-
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-            [
-                "explain this meme",
-                "images/doge.png",
-            ],
-            [
-                "Convert the formula into latex code.",
-                "images/equation.png",
-            ],
-        ],
-        inputs=[question_input, image_input],
-    )
-    
-
-
-        
-    gr.Markdown(value="# Text-to-Image Generation")
-
-    
-    
-    with gr.Row():
-        cfg_weight_input = gr.Slider(minimum=1, maximum=10, value=5, step=0.5, label="CFG Weight")
-        t2i_temperature = gr.Slider(minimum=0, maximum=1, value=1.0, step=0.05, label="temperature")
-
-    prompt_input = gr.Textbox(label="Prompt. (Prompt in more detail can help produce better images!)")
-    seed_input = gr.Number(label="Seed (Optional)", precision=0, value=12345)
-
-    generation_button = gr.Button("Generate Images")
-
-    image_output = gr.Gallery(label="Generated Images", columns=2, rows=2, height=300)
-
-    examples_t2i = gr.Examples(
-        label="Text to image generation examples.",
-        examples=[
-            "Master shifu racoon wearing drip attire as a street gangster.",
-            "The face of a beautiful girl",
-            "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k",
-            "A glass of red wine on a reflective surface.",
-            "A cute and adorable baby fox with big brown eyes, autumn leaves in the background enchanting,immortal,fluffy, shiny mane,Petals,fairyism,unreal engine 5 and Octane Render,highly detailed, photorealistic, cinematic, natural colors.",
-            "The image features an intricately designed eye set against a circular backdrop adorned with ornate swirl patterns that evoke both realism and surrealism. At the center of attention is a strikingly vivid blue iris surrounded by delicate veins radiating outward from the pupil to create depth and intensity. The eyelashes are long and dark, casting subtle shadows on the skin around them which appears smooth yet slightly textured as if aged or weathered over time.\n\nAbove the eye, there's a stone-like structure resembling part of classical architecture, adding layers of mystery and timeless elegance to the composition. This architectural element contrasts sharply but harmoniously with the organic curves surrounding it. Below the eye lies another decorative motif reminiscent of baroque artistry, further enhancing the overall sense of eternity encapsulated within each meticulously crafted detail. \n\nOverall, the atmosphere exudes a mysterious aura intertwined seamlessly with elements suggesting timelessness, achieved through the juxtaposition of realistic textures and surreal artistic flourishes. Each component\u2014from the intricate designs framing the eye to the ancient-looking stone piece above\u2014contributes uniquely towards creating a visually captivating tableau imbued with enigmatic allure.",
-        ],
-        inputs=prompt_input,
-    )
-    
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[image_input, question_input, und_seed_input, top_p, temperature, target_token_idx],
-        outputs=[understanding_output, saliency_map_output, understanding_target_token_decoded_output]
-    )
-    
-    generation_button.click(
-        fn=generate_image,
-        inputs=[prompt_input, seed_input, cfg_weight_input, t2i_temperature],
-        outputs=image_output
-    )
-
-demo.launch(share=True)
-# demo.queue(concurrency_count=1, max_size=10).launch(server_name="0.0.0.0", server_port=37906, root_path="/path")

demo/app_vqa.py

@@ -1,333 +0,0 @@
-import gradio as gr
-import torch
-from transformers import AutoConfig, AutoModelForCausalLM
-from janus.models import MultiModalityCausalLM, VLChatProcessor
-from janus.utils.io import load_pil_images
-from demo.cam import generate_gradcam, AttentionGuidedCAMJanus, AttentionGuidedCAMClip
-from demo.model_utils import Clip_Utils, Janus_Utils, add_title_to_image
-
-import numpy as np
-import matplotlib.pyplot as plt
-import gc
-from PIL import Image
-
-model_seed = 42
-torch.manual_seed(model_seed)
-np.random.seed(model_seed)
-torch.cuda.manual_seed(model_seed)
-
-model_type = "Janus-1B"
-janus_utils = Janus_Utils()
-vl_gpt, tokenizer = janus_utils.init_Janus(model_type.split('-')[-1])
-
-clip_utils = Clip_Utils()
-clip_utils.init_Clip()
-
-# @torch.inference_mode() # cancel inference, for gradcam
-# @spaces.GPU(duration=120) 
-# Multimodal Understanding function
-def multimodal_understanding(model_type, 
-                             saliency_map_method, 
-                             visual_pooling_method, 
-                             image, question, seed, top_p, temperature, target_token_idx,
-                             visualization_layer_min, visualization_layer_max, focus):
-    # Clear CUDA cache before generating
-    torch.cuda.empty_cache()
-
-    # set seed
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed)
-
-    input_text_decoded = ""
-    if model_type == "Clip":
-        
-        inputs = clip_utils.prepare_inputs([question], image)
-
-
-        if saliency_map_method == "GradCAM":
-            # Generate Grad-CAM
-            all_layers = [layer.layer_norm1 for layer in clip_utils.model.vision_model.encoder.layers]
-            if visualization_layers_min.value != visualization_layers_max.value:
-                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max-1]
-            else:
-                target_layers = [all_layers[visualization_layer_min-1]]
-            grad_cam = AttentionGuidedCAMClip(clip_utils.model, target_layers)
-            cam, outputs, grid_size = grad_cam.generate_cam(inputs, class_idx=0, visual_pooling_method=visual_pooling_method)
-            cam = [generate_gradcam(cam, image, size=(224, 224))]
-            grad_cam.remove_hooks()
-            target_token_decoded = ""
-            answer = ""
-    
-
-    elif model_type == "Janus-1B":
-        
-        for param in vl_gpt.parameters():
-            param.requires_grad = True
-
-
-        prepare_inputs = janus_utils.prepare_inputs(question, image)
-        inputs_embeds = janus_utils.generate_inputs_embeddings(prepare_inputs)
-        outputs = janus_utils.generate_outputs(inputs_embeds, prepare_inputs, temperature, top_p)
-
-        sequences = outputs.sequences.cpu().tolist()
-        answer = tokenizer.decode(sequences[0], skip_special_tokens=True)
-        attention_raw = outputs.attentions
-        print("answer generated")
-
-        input_ids = prepare_inputs.input_ids[0].cpu().tolist()
-        input_ids_decoded = [tokenizer.decode([input_ids[i]]) for i in range(len(input_ids))]
-        start=620
-
-        if saliency_map_method == "GradCAM":
-            # target_layers = vl_gpt.vision_model.vision_tower.blocks
-            if focus == "Visual Encoder":
-                all_layers = [block.norm1 for block in vl_gpt.vision_model.vision_tower.blocks]
-            else:
-                all_layers = [layer.self_attn for layer in vl_gpt.language_model.model.layers]
-
-            if visualization_layers_min.value != visualization_layers_max.value:
-                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max-1]
-            else:
-                target_layers = [all_layers[visualization_layer_min-1]]
-
-            gradcam = AttentionGuidedCAMJanus(vl_gpt, target_layers)
-            cam_tensors, grid_size = gradcam.generate_cam(prepare_inputs, tokenizer, temperature, top_p, target_token_idx, visual_pooling_method, focus)
-            if focus == "Visual Encoder":
-                cam_grid = cam_tensors.reshape(grid_size, grid_size)
-                cam = [generate_gradcam(cam_grid, image)]
-            else:
-                if target_token_idx != -1:
-                    input_text_decoded = input_ids_decoded[start + target_token_idx]
-                    for i, cam_tensor in enumerate(cam_tensors):
-                        if i == target_token_idx:
-                            cam_grid = cam_tensor.reshape(grid_size, grid_size)
-                            cam_i = generate_gradcam(cam_grid, image)
-                            cam = [add_title_to_image(cam_i, input_text_decoded)]
-                            break
-                else:
-                    cam = []
-                    for i, cam_tensor in enumerate(cam_tensors):
-                        cam_grid = cam_tensor.reshape(24, 24)
-                        cam_i = generate_gradcam(cam_grid, image)
-                        cam_i = add_title_to_image(cam_i, input_ids_decoded[start + i])
-
-                        cam.append(cam_i)
-                        
-                    # widths, heights = zip(*(img.size for img in heatmaps))
-                    # total_height = sum(heights)
-                    # max_width = max(widths)
-
-                    # combined_img = Image.new("RGB", (max_width, total_height))
-
-                    # y_offset = 0
-                    # for img in heatmaps:
-                    #     combined_img.paste(img, (0, y_offset))  # Stack vertically
-                    #     y_offset += img.height
-                    # cam = combined_img
-                    
-                
-                    
-
-        elif saliency_map_method == "Attention_Map":
-            attn_m_token = attention_raw[target_token_idx]
-            img_token_positions = prepare_inputs.images_seq_mask
-            mask = img_token_positions[0]
-
-            tg = attn_m_token[1][:, :, :, :len(mask)]
-            tg = tg[:, :, :, mask]
-            head = 0
-
-            # res = tg[0, head, 0].to(torch.float32)
-            res, _ = tg.max(dim=1)
-            # res = tg.sum(dim=1)
-            res = res.to(torch.float32)
-            grid_size = (int)(res.shape[-1] ** 0.5)
-            res = res.view(grid_size, grid_size)
-            cam = [generate_gradcam(res, image)]
-        
-        
-        # output_arr = output.logits.detach().to(float).to("cpu").numpy()
-        # predicted_ids = np.argmax(output_arr, axis=-1) # [1, num_tokens]
-        # predicted_ids = predicted_ids.squeeze(0) # [num_tokens]
-        # target_token_decoded = tokenizer.decode(predicted_ids[target_token_idx].tolist())
-
-
-    return answer, cam, input_text_decoded
-
-
-
-
-# Gradio interface
-
-def update_sliders(model):
-    if model == "Clip":
-        res = (
-            gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers min"), 
-            gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus")
-            )
-        return res
-    else:
-        res = (
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Visual Encoder", "Language Model"], value="Visual Encoder", label="focus")
-        )
-        return res
-
-def update_visualization_layers_sliders(focus):
-    if focus == "Visual Encoder":
-        res = (
-            gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="saliency map type"),
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers min"), 
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers max")
-        )
-        return res
-    else:
-        res = (
-            gr.Dropdown(choices=["GradCAM", "Attention_Map"], value="GradCAM", label="saliency map type"),
-            gr.Slider(minimum=1, maximum=24, value=9, step=1, label="visualization layers min"), 
-            gr.Slider(minimum=1, maximum=24, value=9, step=1, label="visualization layers max")
-        )
-        return res
-
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    with gr.Row():
-        with gr.Column():
-            image_input = gr.Image()
-            saliency_map_output = gr.Gallery(label="Saliency Map", columns=1)
-
-        with gr.Column():
-            model_selector = gr.Dropdown(choices=["Clip", "Janus-1B"], value="Clip", label="model")
-            focus = gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus")
-            saliency_map_method = gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="saliency map type")
-            visual_pooling_method = gr.Dropdown(choices=["CLS", "max", "avg"], value="CLS", label="visual pooling method")
-            
-
-            visualization_layers_min = gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers min")
-            visualization_layers_max = gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers max")
-        
-            question_input = gr.Textbox(label="Question")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-            target_token_idx = gr.Number(label="target_token_idx (-1 means all)", precision=0, value=-1)
-        
-
-
-        model_selector.change(
-            fn=update_sliders, 
-            inputs=model_selector, 
-            outputs=[ 
-                visualization_layers_min,
-                visualization_layers_max,
-                focus
-            ]
-        )
-        
-        focus.change(
-            fn = update_visualization_layers_sliders,
-            inputs = focus,
-            outputs=[
-                saliency_map_method,
-                visualization_layers_min,
-                visualization_layers_max,
-            ]
-        )
-
-        
-
-    understanding_button = gr.Button("Chat")
-    understanding_output = gr.Textbox(label="Response")
-    understanding_target_token_decoded_output = gr.Textbox(label="Target Token Decoded")
-
-
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-            
-            [
-                "What is the approximate global smartphone market share of Samsung?",
-                "images/PieChart.png"
-            ],
-            [
-                "What is the average internet speed in Japan?",
-                "images/BarChart.png"
-            ],
-            [
-                "What was the average price of coffee beans in October 2019?",
-                "images/AreaChart.png"
-            ],
-            [
-                "Which city's metro system has the largest number of stations?", 
-                "images/BubbleChart.png"
-            ],
-
-            [ 
-                "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
-                "images/Choropleth_New.png"
-            ],
-
-            [ 
-                "What distance have customers traveled in the taxi the most?", 
-                "images/Histogram.png"
-            ],
-
-            [
-                "What was the price of a barrel of oil in February 2020?", 
-                "images/LineChart.png" 
-            ],
-
-            [
-                "True/False: eBay is nested in the Software category.", 
-                "images/Treemap.png"
-            ],
-
-            [
-                "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
-                "images/Scatterplot.png"
-            ],
-            
-            [ 
-                "Which country has the lowest proportion of Gold medals?", 
-                "images/Stacked100.png"
-            ],
-
-            [
-                "What was the ratio of girls named 'Isla' to girls named 'Amelia' in 2012 in the UK?", 
-                "images/StackedArea.png"
-            ],
-
-            [
-                "What is the cost of peanuts in Seoul?", 
-                "images/StackedBar.png"
-            ],
-            
-
-            # [
-            #     "explain this meme",
-            #     "images/doge.png",
-            # ],
-            # [
-            #     "Convert the formula into latex code.",
-            #     "images/equation.png",
-            # ],
-            
-        ],
-        inputs=[question_input, image_input],
-    )
-    
-
-
-        
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[model_selector, saliency_map_method, visual_pooling_method, image_input, question_input, und_seed_input, top_p, temperature, target_token_idx, 
-                visualization_layers_min, visualization_layers_max, focus],
-        outputs=[understanding_output, saliency_map_output, understanding_target_token_decoded_output]
-    )
-    
-demo.launch(share=True)
-# demo.queue(concurrency_count=1, max_size=10).launch(server_name="0.0.0.0", server_port=37906, root_path="/path")

demo/fastapi_app.py

@@ -1,178 +0,0 @@
-from fastapi import FastAPI, File, Form, UploadFile, HTTPException
-from fastapi.responses import JSONResponse, StreamingResponse
-import torch
-from transformers import AutoConfig, AutoModelForCausalLM
-from janus.models import MultiModalityCausalLM, VLChatProcessor
-from PIL import Image
-import numpy as np
-import io
-
-app = FastAPI()
-
-# Load model and processor
-model_path = "deepseek-ai/Janus-1.3B"
-config = AutoConfig.from_pretrained(model_path)
-language_config = config.language_config
-language_config._attn_implementation = 'eager'
-vl_gpt = AutoModelForCausalLM.from_pretrained(model_path,
-                                              language_config=language_config,
-                                              trust_remote_code=True)
-vl_gpt = vl_gpt.to(torch.bfloat16).cuda()
-
-vl_chat_processor = VLChatProcessor.from_pretrained(model_path)
-tokenizer = vl_chat_processor.tokenizer
-cuda_device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
-
-@torch.inference_mode()
-def multimodal_understanding(image_data, question, seed, top_p, temperature):
-    torch.cuda.empty_cache()
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed)
-
-    conversation = [
-        {
-            "role": "User",
-            "content": f"<image_placeholder>\n{question}",
-            "images": [image_data],
-        },
-        {"role": "Assistant", "content": ""},
-    ]
-
-    pil_images = [Image.open(io.BytesIO(image_data))]
-    prepare_inputs = vl_chat_processor(
-        conversations=conversation, images=pil_images, force_batchify=True
-    ).to(cuda_device, dtype=torch.bfloat16 if torch.cuda.is_available() else torch.float16)
-    
-    inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
-    outputs = vl_gpt.language_model.generate(
-        inputs_embeds=inputs_embeds,
-        attention_mask=prepare_inputs.attention_mask,
-        pad_token_id=tokenizer.eos_token_id,
-        bos_token_id=tokenizer.bos_token_id,
-        eos_token_id=tokenizer.eos_token_id,
-        max_new_tokens=512,
-        do_sample=False if temperature == 0 else True,
-        use_cache=True,
-        temperature=temperature,
-        top_p=top_p,
-    )
-    
-    answer = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
-    return answer
-
-
-@app.post("/understand_image_and_question/")
-async def understand_image_and_question(
-    file: UploadFile = File(...),
-    question: str = Form(...),
-    seed: int = Form(42),
-    top_p: float = Form(0.95),
-    temperature: float = Form(0.1)
-):
-    image_data = await file.read()
-    response = multimodal_understanding(image_data, question, seed, top_p, temperature)
-    return JSONResponse({"response": response})
-
-
-def generate(input_ids,
-             width,
-             height,
-             temperature: float = 1,
-             parallel_size: int = 5,
-             cfg_weight: float = 5,
-             image_token_num_per_image: int = 576,
-             patch_size: int = 16):
-    torch.cuda.empty_cache()
-    tokens = torch.zeros((parallel_size * 2, len(input_ids)), dtype=torch.int).to(cuda_device)
-    for i in range(parallel_size * 2):
-        tokens[i, :] = input_ids
-        if i % 2 != 0:
-            tokens[i, 1:-1] = vl_chat_processor.pad_id
-    inputs_embeds = vl_gpt.language_model.get_input_embeddings()(tokens)
-    generated_tokens = torch.zeros((parallel_size, image_token_num_per_image), dtype=torch.int).to(cuda_device)
-
-    pkv = None
-    for i in range(image_token_num_per_image):
-        outputs = vl_gpt.language_model.model(inputs_embeds=inputs_embeds, use_cache=True, past_key_values=pkv)
-        pkv = outputs.past_key_values
-        hidden_states = outputs.last_hidden_state
-        logits = vl_gpt.gen_head(hidden_states[:, -1, :])
-        logit_cond = logits[0::2, :]
-        logit_uncond = logits[1::2, :]
-        logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
-        probs = torch.softmax(logits / temperature, dim=-1)
-        next_token = torch.multinomial(probs, num_samples=1)
-        generated_tokens[:, i] = next_token.squeeze(dim=-1)
-        next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)
-        img_embeds = vl_gpt.prepare_gen_img_embeds(next_token)
-        inputs_embeds = img_embeds.unsqueeze(dim=1)
-    patches = vl_gpt.gen_vision_model.decode_code(
-        generated_tokens.to(dtype=torch.int), 
-        shape=[parallel_size, 8, width // patch_size, height // patch_size]
-    )
-
-    return generated_tokens.to(dtype=torch.int), patches
-
-
-def unpack(dec, width, height, parallel_size=5):
-    dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)
-    dec = np.clip((dec + 1) / 2 * 255, 0, 255)
-
-    visual_img = np.zeros((parallel_size, width, height, 3), dtype=np.uint8)
-    visual_img[:, :, :] = dec
-
-    return visual_img
-
-
-@torch.inference_mode()
-def generate_image(prompt, seed, guidance):
-    torch.cuda.empty_cache()
-    seed = seed if seed is not None else 12345
-    torch.manual_seed(seed)
-    torch.cuda.manual_seed(seed)
-    np.random.seed(seed)
-    width = 384
-    height = 384
-    parallel_size = 5
-    
-    with torch.no_grad():
-        messages = [{'role': 'User', 'content': prompt}, {'role': 'Assistant', 'content': ''}]
-        text = vl_chat_processor.apply_sft_template_for_multi_turn_prompts(
-            conversations=messages,
-            sft_format=vl_chat_processor.sft_format,
-            system_prompt=''
-        )
-        text = text + vl_chat_processor.image_start_tag
-        input_ids = torch.LongTensor(tokenizer.encode(text))
-        _, patches = generate(input_ids, width // 16 * 16, height // 16 * 16, cfg_weight=guidance, parallel_size=parallel_size)
-        images = unpack(patches, width // 16 * 16, height // 16 * 16)
-
-        return [Image.fromarray(images[i]).resize((1024, 1024), Image.LANCZOS) for i in range(parallel_size)]
-
-
-@app.post("/generate_images/")
-async def generate_images(
-    prompt: str = Form(...),
-    seed: int = Form(None),
-    guidance: float = Form(5.0),
-):
-    try:
-        images = generate_image(prompt, seed, guidance)
-        def image_stream():
-            for img in images:
-                buf = io.BytesIO()
-                img.save(buf, format='PNG')
-                buf.seek(0)
-                yield buf.read()
-
-        return StreamingResponse(image_stream(), media_type="multipart/related")
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Image generation failed: {str(e)}")
-
-
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)

demo/fastapi_client.py

@@ -1,78 +0,0 @@
-import requests
-from PIL import Image
-import io
-# Endpoint URLs
-understand_image_url = "http://localhost:8000/understand_image_and_question/"
-generate_images_url = "http://localhost:8000/generate_images/"
-
-# Use your image file path here
-image_path = "images/equation.png"
-
-# Function to call the image understanding endpoint
-def understand_image_and_question(image_path, question, seed=42, top_p=0.95, temperature=0.1):
-    files = {'file': open(image_path, 'rb')}
-    data = {
-        'question': question,
-        'seed': seed,
-        'top_p': top_p,
-        'temperature': temperature
-    }
-    response = requests.post(understand_image_url, files=files, data=data)
-    response_data = response.json()
-    print("Image Understanding Response:", response_data['response'])
-
-
-# Function to call the text-to-image generation endpoint
-def generate_images(prompt, seed=None, guidance=5.0):
-    data = {
-        'prompt': prompt,
-        'seed': seed,
-        'guidance': guidance
-    }
-    response = requests.post(generate_images_url, data=data, stream=True)
-    
-    if response.ok:
-        img_idx = 1
-
-        # We will create a new BytesIO for each image
-        buffers = {}
-
-        try:
-            for chunk in response.iter_content(chunk_size=1024):
-                if chunk:
-                    # Use a boundary detection to determine new image start
-                    if img_idx not in buffers:
-                        buffers[img_idx] = io.BytesIO()
-
-                    buffers[img_idx].write(chunk)
-
-                    # Attempt to open the image
-                    try:
-                        buffer = buffers[img_idx]
-                        buffer.seek(0)
-                        image = Image.open(buffer)
-                        img_path = f"generated_image_{img_idx}.png"
-                        image.save(img_path)
-                        print(f"Saved: {img_path}")
-
-                        # Prepare the next image buffer
-                        buffer.close()
-                        img_idx += 1
-
-                    except Exception as e:
-                        # Continue loading data into the current buffer
-                        continue
-
-        except Exception as e:
-            print("Error processing image:", e)
-    else:
-        print("Failed to generate images.")
-
-
-# Example usage
-if __name__ == "__main__":
-    # Call the image understanding API
-    understand_image_and_question(image_path, "What is this image about?")
-    
-    # Call the image generation API
-    generate_images("A beautiful sunset over a mountain range, digital art.")