expand tests

app-old.py

@@ -1,501 +0,0 @@
-import gradio as gr
-import torch
-from transformers import AutoConfig, AutoModelForCausalLM
-from janus.models import MultiModalityCausalLM, VLChatProcessor
-from janus.utils.io import load_pil_images
-from demo.cam import generate_gradcam, AttentionGuidedCAMJanus, AttentionGuidedCAMClip, AttentionGuidedCAMChartGemma, AttentionGuidedCAMLLaVA
-from demo.model_utils import Clip_Utils, Janus_Utils, LLaVA_Utils, ChartGemma_Utils, add_title_to_image
-
-import numpy as np
-import matplotlib.pyplot as plt
-import gc
-import os
-import spaces
-from PIL import Image
-
-def set_seed(model_seed = 42):
-    torch.manual_seed(model_seed)
-    np.random.seed(model_seed)
-    torch.cuda.manual_seed(model_seed) if torch.cuda.is_available() else None
-
-set_seed()
-clip_utils = Clip_Utils()
-clip_utils.init_Clip()
-model_utils, vl_gpt, tokenizer = None, None, None
-model_name = "Clip"
-language_model_max_layer = 24
-language_model_best_layer = 8
-vision_model_best_layer = 24
-
-def clean():
-    global model_utils, vl_gpt, tokenizer, clip_utils
-    # Move models to CPU first (prevents CUDA references)
-    if 'vl_gpt' in globals() and vl_gpt is not None:
-        vl_gpt.to("cpu")
-    if 'clip_utils' in globals() and clip_utils is not None:
-        del clip_utils
-
-    # Delete all references
-    del model_utils, vl_gpt, tokenizer
-    model_utils, vl_gpt, tokenizer, clip_utils = None, None, None, None
-    gc.collect()
-
-    # Empty CUDA cache
-    if torch.cuda.is_available():
-        torch.cuda.empty_cache()
-        torch.cuda.ipc_collect()  # Frees inter-process CUDA memory
-    
-    # Empty MacOS Metal backend (if using Apple Silicon)
-    if torch.backends.mps.is_available():
-        torch.mps.empty_cache()
-
-# Multimodal Understanding function
-@spaces.GPU(duration=120)
-def multimodal_understanding(model_type, 
-                             activation_map_method, 
-                             visual_pooling_method, 
-                             image, question, seed, top_p, temperature, target_token_idx,
-                             visualization_layer_min, visualization_layer_max, focus, response_type, chart_type):
-    # Clear CUDA cache before generating
-    gc.collect()
-    if torch.cuda.is_available():
-        torch.cuda.empty_cache()
-        torch.cuda.ipc_collect()
-
-    # set seed
-    torch.manual_seed(seed)
-    np.random.seed(seed)
-    torch.cuda.manual_seed(seed) if torch.cuda.is_available() else None
-
-    input_text_decoded = ""
-    answer = ""
-    if model_name == "Clip":
-        
-        inputs = clip_utils.prepare_inputs([question], image)
-
-
-        if activation_map_method == "GradCAM":
-            # Generate Grad-CAM
-            all_layers = [layer.layer_norm1 for layer in clip_utils.model.vision_model.encoder.layers]
-
-            if visualization_layer_min != visualization_layer_max:
-                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max-1]
-            else:
-                target_layers = [all_layers[visualization_layer_min-1]]
-            grad_cam = AttentionGuidedCAMClip(clip_utils.model, target_layers)
-            cam, outputs, grid_size = grad_cam.generate_cam(inputs, class_idx=0, visual_pooling_method=visual_pooling_method)
-            cam = cam.to("cpu")
-            cam = [generate_gradcam(cam, image, size=(224, 224))]
-            grad_cam.remove_hooks()
-            target_token_decoded = ""
-            
-    
-
-    else:
-        
-        for param in vl_gpt.parameters():
-            param.requires_grad = True
-
-
-        prepare_inputs = model_utils.prepare_inputs(question, image)
-
-        if response_type == "answer + visualization":
-            if model_name.split('-')[0] == "Janus":
-                inputs_embeds = model_utils.generate_inputs_embeddings(prepare_inputs)
-                outputs = model_utils.generate_outputs(inputs_embeds, prepare_inputs, temperature, top_p)
-            else:
-                outputs = model_utils.generate_outputs(prepare_inputs, temperature, top_p)
-
-            sequences = outputs.sequences.cpu().tolist()
-            answer = tokenizer.decode(sequences[0], skip_special_tokens=True)
-            attention_raw = outputs.attentions
-            print("answer generated")
-
-        input_ids = prepare_inputs.input_ids[0].cpu().tolist()
-        input_ids_decoded = [tokenizer.decode([input_ids[i]]) for i in range(len(input_ids))]
-
-        if activation_map_method == "GradCAM":
-            # target_layers = vl_gpt.vision_model.vision_tower.blocks
-            if focus == "Visual Encoder":
-                if model_name.split('-')[0] == "Janus":
-                    all_layers = [block.norm1 for block in vl_gpt.vision_model.vision_tower.blocks]
-                else:
-                    all_layers = [block.layer_norm1 for block in vl_gpt.vision_tower.vision_model.encoder.layers]
-            else:
-                all_layers = [layer.self_attn for layer in vl_gpt.language_model.model.layers]
-            
-            print("layer values:", visualization_layer_min, visualization_layer_max)
-            if visualization_layer_min != visualization_layer_max:
-                print("multi layers")
-                target_layers = all_layers[visualization_layer_min-1 : visualization_layer_max]
-            else:
-                print("single layer")
-                target_layers = [all_layers[visualization_layer_min-1]]
-            
-
-            if model_name.split('-')[0] == "Janus":
-                gradcam = AttentionGuidedCAMJanus(vl_gpt, target_layers)
-            elif model_name.split('-')[0] == "LLaVA":
-                gradcam = AttentionGuidedCAMLLaVA(vl_gpt, target_layers)
-            elif model_name.split('-')[0] == "ChartGemma":
-                gradcam = AttentionGuidedCAMChartGemma(vl_gpt, target_layers)
-
-            start = 0
-            cam = []
-            if focus == "Visual Encoder":
-                if target_token_idx != -1:
-                    cam_tensors, grid_size, start = gradcam.generate_cam(prepare_inputs, tokenizer, temperature, top_p, target_token_idx, visual_pooling_method, focus)
-                    cam_grid = cam_tensors.reshape(grid_size, grid_size)
-                    cam_i = generate_gradcam(cam_grid, image)
-                    cam_i = add_title_to_image(cam_i, input_ids_decoded[start + target_token_idx])
-                    cam = [cam_i]
-                else:
-                    i = 0
-                    cam = []
-                    while start + i < len(input_ids_decoded):
-                        if model_name.split('-')[0] == "Janus":
-                            gradcam = AttentionGuidedCAMJanus(vl_gpt, target_layers)
-                        elif model_name.split('-')[0] == "LLaVA":
-                            gradcam = AttentionGuidedCAMLLaVA(vl_gpt, target_layers)
-                        elif model_name.split('-')[0] == "ChartGemma":
-                            gradcam = AttentionGuidedCAMChartGemma(vl_gpt, target_layers)
-                        cam_tensors, grid_size, start = gradcam.generate_cam(prepare_inputs, tokenizer, temperature, top_p, i, visual_pooling_method, focus)
-                        cam_grid = cam_tensors.reshape(grid_size, grid_size)
-                        cam_i = generate_gradcam(cam_grid, image)
-                        cam_i = add_title_to_image(cam_i, input_ids_decoded[start + i])
-                        cam.append(cam_i)
-                        gradcam.remove_hooks()
-                        i += 1
-            else:
-                cam_tensors, grid_size, start = gradcam.generate_cam(prepare_inputs, tokenizer, temperature, top_p, target_token_idx, visual_pooling_method, focus)
-                if target_token_idx != -1:
-                    input_text_decoded = input_ids_decoded[start + target_token_idx]
-                    for i, cam_tensor in enumerate(cam_tensors):
-                        if i == target_token_idx:
-                            cam_grid = cam_tensor.reshape(grid_size, grid_size)
-                            cam_i = generate_gradcam(cam_grid, image)
-                            cam = [add_title_to_image(cam_i, input_text_decoded)]
-                            break
-                else:
-                    cam = []
-                    for i, cam_tensor in enumerate(cam_tensors):
-                        cam_grid = cam_tensor.reshape(grid_size, grid_size)
-                        cam_i = generate_gradcam(cam_grid, image)
-                        cam_i = add_title_to_image(cam_i, input_ids_decoded[start + i])
-
-                        cam.append(cam_i)
-                     
-            gradcam.remove_hooks()
-                
-
-    # Collect Results
-    RESULTS_ROOT = "./results"
-    FILES_ROOT = f"{RESULTS_ROOT}/{model_name}/{focus}/{chart_type}/layer{visualization_layer_min}-{visualization_layer_max}"
-    os.makedirs(FILES_ROOT, exist_ok=True)
-    if focus == "Visual Encoder":
-        cam[0].save(f"{FILES_ROOT}/{visual_pooling_method}.png")
-    else:
-        for i, cam_p in enumerate(cam):
-            cam_p.save(f"{FILES_ROOT}/{i}.png")
-            
-    with open(f"{FILES_ROOT}/input_text_decoded.txt", "w") as f:
-        f.write(input_text_decoded)
-        f.close()
-
-    with open(f"{FILES_ROOT}/answer.txt", "w") as f:
-        f.write(answer)
-        f.close()
-            
-
-
-    return answer, cam, input_text_decoded
-
-
-
-
-# Gradio interface
-
-def model_slider_change(model_type):
-    global model_utils, vl_gpt, tokenizer, clip_utils, model_name, language_model_max_layer, language_model_best_layer, vision_model_best_layer
-    model_name = model_type
-    if model_type == "Clip":
-        clean()
-        set_seed()
-        clip_utils = Clip_Utils()
-        clip_utils.init_Clip()
-        res = (
-            gr.Dropdown(choices=["Visualization only"], value="Visualization only", label="response_type"),
-            gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus"),
-            gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
-        )
-        return res
-    elif model_type.split('-')[0] == "Janus":
-        
-        clean()
-        set_seed()
-        model_utils = Janus_Utils()
-        vl_gpt, tokenizer = model_utils.init_Janus(model_type.split('-')[-1])
-        language_model_max_layer = 24
-        language_model_best_layer = 8
-
-        res = (
-            gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="answer + visualization", label="response_type"),
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Visual Encoder", "Language Model"], value="Visual Encoder", label="focus"),
-            gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
-        )
-        return res
-    
-    elif model_type.split('-')[0] == "LLaVA":
-        
-        clean()
-        set_seed()
-        model_utils = LLaVA_Utils()
-        version = model_type.split('-')[1]
-        vl_gpt, tokenizer = model_utils.init_LLaVA(version=version)
-        language_model_max_layer = 32 if version == "1.5" else 28
-        language_model_best_layer = 10
-
-        res = (
-            gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="answer + visualization", label="response_type"),
-            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Language Model"], value="Language Model", label="focus"),
-            gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
-        )
-        return res
-    
-    elif model_type.split('-')[0] == "ChartGemma":
-        clean()
-        set_seed()
-        model_utils = ChartGemma_Utils()
-        vl_gpt, tokenizer = model_utils.init_ChartGemma()
-        language_model_max_layer = 18
-        vision_model_best_layer = 19
-        language_model_best_layer = 15
-
-        res = (
-            gr.Dropdown(choices=["Visualization only", "answer + visualization"], value="answer + visualization", label="response_type"),
-            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers min"),
-            gr.Slider(minimum=1, maximum=language_model_best_layer, value=language_model_best_layer, step=1, label="visualization layers max"),
-            gr.Dropdown(choices=["Visual Encoder", "Language Model"], value="Language Model", label="focus"),
-            gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type")
-        )
-        return res
-
-    
-
-
-def focus_change(focus):
-    global model_name, language_model_max_layer
-    if model_name == "Clip":
-        res = (
-                gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers max")
-            )
-        return res
-
-    if focus == "Language Model":
-        if response_type.value == "answer + visualization":
-            res = (
-                gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max")
-            )
-            return res
-        else:
-            res = (
-                gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=language_model_max_layer, value=language_model_best_layer, step=1, label="visualization layers max")
-            )
-            return res
-
-    else:
-        if model_name.split('-')[0] == "ChartGemma":
-            res = (
-                gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=26, value=vision_model_best_layer, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=26, value=vision_model_best_layer, step=1, label="visualization layers max")
-            )
-            return res
-        
-        else:
-            res = (
-                gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="activation map type"),
-                gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers min"), 
-                gr.Slider(minimum=1, maximum=24, value=24, step=1, label="visualization layers max")
-            )
-            return res
-
-
-
-
-
-with gr.Blocks() as demo:
-    gr.Markdown(value="# Multimodal Understanding")
-    
-    with gr.Row():
-        image_input = gr.Image(height=500, label="Image")
-        activation_map_output = gr.Gallery(label="Visualization", height=500, columns=1, preview=True)
-
-    with gr.Row():
-        chart_type = gr.Textbox(label="Chart Type")
-        understanding_output = gr.Textbox(label="Answer")
-
-    with gr.Row():
-
-        with gr.Column():
-            model_selector = gr.Dropdown(choices=["Clip", "ChartGemma-3B", "Janus-Pro-1B", "Janus-Pro-7B", "LLaVA-1.5-7B"], value="Clip", label="model")
-            question_input = gr.Textbox(label="Input Prompt")
-            und_seed_input = gr.Number(label="Seed", precision=0, value=42)
-            top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
-            temperature = gr.Slider(minimum=0, maximum=1, value=0.1, step=0.05, label="temperature")
-            target_token_idx = gr.Number(label="target_token_idx (-1 means all)", precision=0, value=-1)
-
-        
-        with gr.Column():
-            response_type = gr.Dropdown(choices=["Visualization only"], value="Visualization only", label="response_type")
-            focus = gr.Dropdown(choices=["Visual Encoder"], value="Visual Encoder", label="focus")
-            activation_map_method = gr.Dropdown(choices=["GradCAM"], value="GradCAM", label="visualization type")
-            visual_pooling_method = gr.Dropdown(choices=["CLS", "max", "avg"], value="CLS", label="visual pooling method")
-            
-
-            visualization_layers_min = gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers min")
-            visualization_layers_max = gr.Slider(minimum=1, maximum=12, value=12, step=1, label="visualization layers max")
-
-        
-        
-
-
-        model_selector.change(
-            fn=model_slider_change, 
-            inputs=model_selector, 
-            outputs=[
-                response_type,
-                visualization_layers_min,
-                visualization_layers_max,
-                focus,
-                activation_map_method
-            ]
-        )
-        
-        focus.change(
-            fn = focus_change,
-            inputs = focus,
-            outputs=[
-                activation_map_method,
-                visualization_layers_min,
-                visualization_layers_max,
-            ]
-        )
-
-        # response_type.change(
-        #     fn = response_type_change,
-        #     inputs = response_type,
-        #     outputs = [activation_map_method]
-        # )
-
-        
-
-    understanding_button = gr.Button("Submit")
-    
-    understanding_target_token_decoded_output = gr.Textbox(label="Target Token Decoded")
-
-
-    examples_inpainting = gr.Examples(
-        label="Multimodal Understanding examples",
-        examples=[
-
-            [
-                "LineChart",
-                "What was the price of a barrel of oil in February 2020?", 
-                "images/LineChart.png" 
-            ],
-
-            [
-                "BarChart",
-                "What is the average internet speed in Japan?",
-                "images/BarChart.png"
-            ],
-
-            [
-                "StackedBar",
-                "What is the cost of peanuts in Seoul?", 
-                "images/StackedBar.png"
-            ],
-
-            [
-                "100%StackedBar",
-                "Which country has the lowest proportion of Gold medals?", 
-                "images/Stacked100.png"
-            ],
-            
-            [
-                "PieChart",
-                "What is the approximate global smartphone market share of Samsung?",
-                "images/PieChart.png"
-            ],
-            
-            [
-                "Histogram",
-                "What distance have customers traveled in the taxi the most?", 
-                "images/Histogram.png"
-            ],
-
-            [
-                "Scatterplot",
-                "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
-                "images/Scatterplot.png"
-            ],
-
-            [
-                "AreaChart",
-                "What was the average price of pount of coffee beans in October 2019?",
-                "images/AreaChart.png"
-            ],
-
-            [
-                "StackedArea",
-                "What was the ratio of girls named 'Isla' to girls named 'Amelia' in 2012 in the UK?", 
-                "images/StackedArea.png"
-            ],
-
-            [
-                "BubbleChart",
-                "Which city's metro system has the largest number of stations?", 
-                "images/BubbleChart.png"
-            ],
-
-            [ 
-                "Choropleth",
-                "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
-                "images/Choropleth_New.png"
-            ],
-
-            [
-                "TreeMap",
-                "True/False: eBay is nested in the Software category.", 
-                "images/TreeMap.png"
-            ]
-            
-        ],
-        inputs=[chart_type, question_input, image_input],
-    )
-    
-
-
-        
-    understanding_button.click(
-        multimodal_understanding,
-        inputs=[model_selector, activation_map_method, visual_pooling_method, image_input, question_input, und_seed_input, top_p, temperature, target_token_idx, 
-                visualization_layers_min, visualization_layers_max, focus, response_type, chart_type],
-        outputs=[understanding_output, activation_map_output, understanding_target_token_decoded_output]
-    )
-    
-demo.launch(share=True)
-# demo.queue(concurrency_count=1, max_size=10).launch(server_name="0.0.0.0", server_port=37906, root_path="/path")

app.py

@@ -6,7 +6,9 @@ from janus.utils.io import load_pil_images
 from demo.visualization import generate_gradcam, VisualizationJanus, VisualizationClip, VisualizationChartGemma, VisualizationLLaVA
 from demo.model_utils import Clip_Utils, Janus_Utils, LLaVA_Utils, ChartGemma_Utils, add_title_to_image
 from demo.modified_attn import ModifiedLlamaAttention, ModifiedGemmaAttention
-
+from questions.mini_VLAT import mini_VLAT_questions
+from questions.VLAT_old import VLAT_old_questions
+from questions.VLAT import VLAT_questions
 import numpy as np
 import matplotlib.pyplot as plt
 import gc
@@ -411,12 +413,6 @@ with gr.Blocks() as demo:
             ]
         )
 
-        # response_type.change(
-        #     fn = response_type_change,
-        #     inputs = response_type,
-        #     outputs = [activation_map_method]
-        # )
-
         
 
     understanding_button = gr.Button("Submit")
@@ -426,81 +422,8 @@ with gr.Blocks() as demo:
 
     examples_inpainting = gr.Examples(
         label="Multimodal Understanding examples",
-        examples=[
-
-            [
-                "LineChart",
-                "What was the price of a barrel of oil in February 2020?", 
-                "images/LineChart.png" 
-            ],
-
-            [
-                "BarChart",
-                "What is the average internet speed in Japan?",
-                "images/BarChart.png"
-            ],
-
-            [
-                "StackedBar",
-                "What is the cost of peanuts in Seoul?", 
-                "images/StackedBar.png"
-            ],
-
-            [
-                "100%StackedBar",
-                "Which country has the lowest proportion of Gold medals?", 
-                "images/Stacked100.png"
-            ],
-            
-            [
-                "PieChart",
-                "What is the approximate global smartphone market share of Samsung?",
-                "images/PieChart.png"
-            ],
-            
-            [
-                "Histogram",
-                "What distance have customers traveled in the taxi the most?", 
-                "images/Histogram.png"
-            ],
-
-            [
-                "Scatterplot",
-                "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
-                "images/Scatterplot.png"
-            ],
-
-            [
-                "AreaChart",
-                "What was the average price of pount of coffee beans in October 2019?",
-                "images/AreaChart.png"
-            ],
-
-            [
-                "StackedArea",
-                "What was the ratio of girls named 'Isla' to girls named 'Amelia' in 2012 in the UK?", 
-                "images/StackedArea.png"
-            ],
-
-            [
-                "BubbleChart",
-                "Which city's metro system has the largest number of stations?", 
-                "images/BubbleChart.png"
-            ],
-
-            [ 
-                "Choropleth",
-                "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
-                "images/Choropleth_New.png"
-            ],
-
-            [
-                "TreeMap",
-                "True/False: eBay is nested in the Software category.", 
-                "images/TreeMap.png"
-            ]
-            
-        ],
+        # examples=mini_VLAT_questions,
+        examples=VLAT_questions,
         inputs=[chart_type, question_input, image_input],
     )
     

evaluate/evaluate.py

@@ -1,7 +1,9 @@
 import os
 import torch
+import base64
 import numpy as np
 from PIL import Image
+from openai import OpenAI
 from demo.model_utils import *
 from evaluate.questions import questions
 
@@ -20,6 +22,9 @@ def clean():
     if torch.backends.mps.is_available():
         torch.mps.empty_cache()
 
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
 
 def evaluate(model_type, num_eval = 10):
     for eval_idx in range(num_eval):
@@ -39,6 +44,13 @@ def evaluate(model_type, num_eval = 10):
         elif model_type.split('-')[0] == "ChartGemma":
             model_utils = ChartGemma_Utils()
             vl_gpt, tokenizer = model_utils.init_ChartGemma()
+        
+        elif model_type.split('-')[0] == "GPT":
+            client = OpenAI(api_key=os.environ["OPENAI_HCI_API_KEY"])
+        
+        elif model_type.split('-')[0] == "Gemini":
+            client = OpenAI(api_key=os.environ["GEMINI_HCI_API_KEY"], 
+                            base_url="https://generativelanguage.googleapis.com/v1beta/openai/")
 
         for question in questions:
             chart_type = question[0]
@@ -46,18 +58,63 @@ def evaluate(model_type, num_eval = 10):
             img_path = question[2]
             image = np.array(Image.open(img_path).convert("RGB"))
 
-            prepare_inputs = model_utils.prepare_inputs(q, image)
-            temperature = 0.9
-            top_p = 0.1
+            
+            
+            if model_type.split('-')[0] == "GPT":
+                base64_image = encode_image(img_path)
+                completion = client.chat.completions.create(
+                    model="gpt-4o",
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": [
+                                { "type": "text", "text": f"{q}" },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:image/jpeg;base64,{base64_image}",
+                                    },
+                                },
+                            ],
+                        }
+                    ],
+                )
+                answer = completion.choices[0].message.content
+
+            elif model_type.split('-')[0] == "Gemini":
+                base64_image = encode_image(img_path)
+                completion = client.chat.completions.create(
+                    model="gemini-2.0-flash",
+                    messages=[
+                        {
+                            "role": "user",
+                            "content": [
+                                { "type": "text", "text": f"{q}" },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": f"data:image/jpeg;base64,{base64_image}",
+                                    },
+                                },
+                            ],
+                        }
+                    ],
+                )
+                answer = completion.choices[0].message.content
 
-            if model_type.split('-')[0] == "Janus":
-                inputs_embeds = model_utils.generate_inputs_embeddings(prepare_inputs)
-                outputs = model_utils.generate_outputs(inputs_embeds, prepare_inputs, temperature, top_p)
             else:
-                outputs = model_utils.generate_outputs(prepare_inputs, temperature, top_p)
+                prepare_inputs = model_utils.prepare_inputs(q, image)
+                temperature = 0.9
+                top_p = 0.1
+
+                if model_type.split('-')[0] == "Janus":
+                    inputs_embeds = model_utils.generate_inputs_embeddings(prepare_inputs)
+                    outputs = model_utils.generate_outputs(inputs_embeds, prepare_inputs, temperature, top_p)
+                else:
+                    outputs = model_utils.generate_outputs(prepare_inputs, temperature, top_p)
 
-            sequences = outputs.sequences.cpu().tolist()
-            answer = tokenizer.decode(sequences[0], skip_special_tokens=True)
+                sequences = outputs.sequences.cpu().tolist()
+                answer = tokenizer.decode(sequences[0], skip_special_tokens=True)
 
             RESULTS_ROOT = "./evaluate/results"
             FILES_ROOT = f"{RESULTS_ROOT}/{model_type}/{eval_idx}"
@@ -71,9 +128,9 @@ def evaluate(model_type, num_eval = 10):
 
 if __name__ == '__main__':
     
-    # models = ["ChartGemma", "Janus-Pro-1B", "Janus-Pro-7B", "LLaVA-1.5-7B"]
+    # models = ["ChartGemma", "Janus-Pro-1B", "Janus-Pro-7B", "LLaVA-1.5-7B", "GPT-4o", "Gemini-2.0-flash"]
     # models = ["ChartGemma", "Janus-Pro-1B"]
     # models = ["Janus-Pro-7B", "LLaVA-1.5-7B"]
-    models = ["LLaVA-1.5-7B"]
+    models = ["GPT-4o", "Gemini-2.0-flash"]
     for model_type in models:
         evaluate(model_type=model_type, num_eval=10)

image_modifier.py

@@ -0,0 +1,19 @@
+from PIL import Image
+import os
+
+def augment(img_path):
+    ROOT = "./images/VLAT"
+    NEW_ROOT = "./changed"
+    img = Image.open(f"{ROOT}/{img_path}").convert("RGBA")
+
+    # Create a white background
+    white_bg = Image.new("RGB", img.size, (255, 255, 255))
+    white_bg.paste(img, mask=img.split()[3])  # Paste using alpha channel as mask
+
+    # Save the result
+    white_bg.save(f'{NEW_ROOT}/{img_path}')  # Or save as .png if you prefer
+
+if __name__ == "__main__":
+    ROOT = "./images/VLAT"
+    for img in os.listdir(ROOT):
+        augment(img)

questions/VLAT.py

@@ -0,0 +1,214 @@
+VLAT_questions=[
+    [
+        "StackedBar",
+        "What percentage of the room service cost in Copenhagen, comes from the sandwich?", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "In which city is the cost of soda the highest?", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "True/False: The cost of vodka in Helsinki is higher than that of Toronto.", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "True/False: The ratio of the cost of Soda to the cost of Water is higher in N.Y.C than in Stockholm.", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "What is the cost of peanuts in room service in Seoul?", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedArea",
+        "What was the number of girls named 'Isla' in 2010 in the UK?", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "True/False: Over the course of years between 2009 and 2014, the number of girls named 'Isla' was always more than 'Amelia'.", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "Over the course of years between 2009 and 2014, when was the number of girls named 'Isla' at the maximum?", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "The number of girls named 'Olivia' was raising or falling from 2009 to 2012.", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "True/False: In the UK, the number of girls named 'Isla' in 2014 was more than it was in 2013.", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the weight for the person who is 161 cm tall?", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the height for the tallest person among the 85 individuals?", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the range in weight for the 85 individuals?", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: There is a group of individuals centered at the height of 176 cm and the weight of 55 kg.", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: There is a negative linear relationship between the height and the weight of the 85 individuals", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the height of the most prominent outlier in the data?", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "LineChart",
+        "What was the price of a barrel of oil in February 2020?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "In which month was the price of a barrel of oil the lowest in 2020?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "What was the price range of a barrel of oil in 2020?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "Over the course of the first quarter of 2020, the price of a barrel of oil was rising or falling?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "About how much did the price of a barrel of oil rise from April to August in 2020?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "BubbleChart",
+        "What is the range of metro system lengths in the data shown in this chart?", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True or False: Tokyo has a bigger ridership than Guangzhow.", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True/False: In general, the ridership of the metro system increases as the number of stations increases.", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True/False: The average metro system in this chart has approximately 300 stations and is 200km in length.", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "Which city's metro system has the largest number of stations?", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+
+
+
+
+
+    
+
+    [
+        "BarChart",
+        "What is the average internet speed in Japan?",
+        "images/mini-VLAT/BarChart.png"
+    ],
+
+    [
+        "StackedBar",
+        "What is the cost of peanuts in Seoul?", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "100%StackedBar",
+        "Which country has the lowest proportion of Gold medals?", 
+        "images/mini-VLAT/Stacked100.png"
+    ],
+    
+    [
+        "PieChart",
+        "What is the approximate global smartphone market share of Samsung?",
+        "images/mini-VLAT/PieChart.png"
+    ],
+    
+    [
+        "Histogram",
+        "What distance have customers traveled in the taxi the most?", 
+        "images/mini-VLAT/Histogram.png"
+    ],
+
+
+    [
+        "AreaChart",
+        "What was the average price of pount of coffee beans in October 2019?",
+        "images/mini-VLAT/AreaChart.png"
+    ],
+
+
+    [ 
+        "Choropleth",
+        "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
+        "images/mini-VLAT/Choropleth_New.png"
+    ],
+
+    [
+        "TreeMap",
+        "True/False: eBay is nested in the Software category.", 
+        "images/mini-VLAT/TreeMap.png"
+    ]
+]

questions/VLAT_old.py

@@ -0,0 +1,325 @@
+VLAT_old_questions = [
+    [
+        "LineChart",
+        "What was the price of a barrel of oil in February 2015?", 
+        "images/VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "In which month was the price of a barrel of oil the lowest in 2015?", 
+        "images/VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "What was the price range of a barrel of oil in 2015?", 
+        "images/VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "Over the course of the second half of 2015, the price of a barrel of oil was increasing or decreasing?", 
+        "images/VLAT/LineChart.png" 
+    ],
+
+    [
+        "LineChart",
+        "About how much did the price of a barrel of oil fall from April to September in 2015?", 
+        "images/VLAT/LineChart.png" 
+    ],
+
+    [
+        "BarChart",
+        "What is the average internet speed in Japan?",
+        "images/VLAT/BarChart.png"
+    ],
+
+    [
+        "BarChart",
+        "In which country is the average internet speed the fastest in Asia?",
+        "images/VLAT/BarChart.png"
+    ],
+
+    [
+        "BarChart",
+        "What is the range of the average internet speed in Asia?",
+        "images/VLAT/BarChart.png"
+    ],
+
+    [
+        "BarChart",
+        "How many countries in Asia is the average internet speed slower than Thailand?",
+        "images/VLAT/BarChart.png"
+    ],
+
+    [
+        "StackedBar",
+        "What is the cost of peanuts in Las Vegas?", 
+        "images/VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "About what is the ratio of the cost of a sandwich to the total cost of room service in Seattle?", 
+        "images/VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "In which city is the cost of soda the highest?", 
+        "images/VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "True/False: The cost of vodka in Atlanta is higher than that of Honolulu.", 
+        "images/VLAT/StackedBar.png"
+    ],
+
+    [
+        "StackedBar",
+        "True/False: The ratio of the cost of Soda to the cost of Water in Orlando is higher than that of Washington D.C.", 
+        "images/VLAT/StackedBar.png"
+    ],
+
+    [
+        "100%StackedBar",
+        "What is the approval rating of Republicans among the people who have the education level of Postgraduate Study?", 
+        "images/VLAT/Stacked100.png"
+    ],
+
+    [
+        "100%StackedBar",
+        "What is the education level of people in which the Democrats have the lowest approval rating", 
+        "images/VLAT/Stacked100.png"
+    ],
+
+    [
+        "100%StackedBar",
+        "True/False: The approval rating of Republicans for the people who have the education level of Some College Degree is lower than that for the people who have the education level of Postgraduate Study.", 
+        "images/VLAT/Stacked100.png"
+    ],
+    
+    [
+        "PieChart",
+        "About what is the global smartphone market share of Samsung?",
+        "images/VLAT/PieChart.png"
+    ],
+
+    [
+        "PieChart",
+        "In which company is the global smartphone market share the smallest?",
+        "images/VLAT/PieChart.png"
+    ],
+
+    [
+        "PieChart",
+        "True/False: The global smartphone market share of Apple is larger than that of Huawei.",
+        "images/VLAT/PieChart.png"
+    ],
+    
+    [
+        "Histogram",
+        "How many people have rated the taxi between 4.0 and 4.2?", 
+        "images/VLAT/Histogram.png"
+    ],
+
+    [
+        "Histogram",
+        "What is the rating that the people have rated the taxi the most?", 
+        "images/VLAT/Histogram.png"
+    ],
+
+    [
+        "Histogram",
+        "True/False: The distribution of the taxi passenger rating is generally skewed to the left.", 
+        "images/VLAT/Histogram.png"
+    ],
+
+    [
+        "Histogram",
+        "True/False: More people have rated the taxi between 4.6 and 4.8 than between 4.2 and 4.4.", 
+        "images/VLAT/Histogram.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the weight for the person who is 165.1 cm tall?", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the height for the tallest person among the 85 males?", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the range in weight for the 85 males?", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "What is the height for a person who lies outside the others the most?", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: A group of males are gathered around the height of 176 cm and the weight of 70 kg.", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: The weights for males with the height of 188 cm are all the same.", 
+        "images/VLAT/Scatterplot.png"
+    ],
+
+    [
+        "AreaChart",
+        "What was the average price of a pound of coffee beans in September 2013?",
+        "images/VLAT/AreaChart.png"
+    ],
+
+    [
+        "AreaChart",
+        "When was the average price of a pound of coffee beans at minimum?",
+        "images/VLAT/AreaChart.png"
+    ],
+
+    [
+        "AreaChart",
+        "What was the range of the average price of a pound of coffee beans between January 2013 and December 2014?",
+        "images/VLAT/AreaChart.png"
+    ],
+
+    [
+        "AreaChart",
+        "Over the course of 2013, the average price of a pound of coffee beans was ____________?",
+        "images/VLAT/AreaChart.png"
+    ],
+
+    [
+        "StackedArea",
+        "What was the number of girls named 'Amelia' in 2010 in the UK?", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "About what was the ratio of the number of girls named ‘Olivia’ to those named ‘Isla’ in 2014 in the UK?", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "Over the course of years between 2009 and 2014, when was the number of girls named 'Amelia' at the maximum?", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "The number of girls named 'Isla' was __________ from 2009 to 2012?", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "True/False: In the UK, the number of girls named 'Amelia' in 2014 was more than it was in 2013", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "StackedArea",
+        "True/False: Over the course of years between 2009 and 2014, the number of girls named 'Isla' was always more than 'Olivia'", 
+        "images/VLAT/StackedArea.png"
+    ],
+
+    [
+        "BubbleChart",
+        "What is the total length of the metro system in Beijing?", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "Which city's metro system has the largest number of stations?", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "What is the range of the total length of the metro systems?", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "Which city's metro system does lie outside the relationship between the total system length and the number of stations most?", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True/False: A group of the metro systems of the world has approximately 300 stations and around a 200km system length.", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True/False: In general, the ridership of the metro system increases as the number of stations increases.", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [
+        "BubbleChart",
+        "True/False: The metro system in Shanghai has more ridership than the metro system in Beijing.", 
+        "images/VLAT/BubbleChart.png"
+    ],
+
+    [ 
+        "Choropleth",
+        "What was the unemployment rate for Indiana (IN) in 2015?", 
+        "images/VLAT/Choropleth_New.png"
+    ],
+
+    [ 
+        "Choropleth",
+        "In which state was the unemployment rate the highest in 2015?", 
+        "images/VLAT/Choropleth_New.png"
+    ],
+
+    [ 
+        "Choropleth",
+        "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
+        "images/VLAT/Choropleth_New.png"
+    ],
+
+    [
+        "TreeMap",
+        "For which website was the number of unique visitors the largest in 2010?", 
+        "images/VLAT/TreeMap.png"
+    ],
+
+    [
+        "TreeMap",
+        "True/False: The number of unique visitors for Amazon was more than that of Yahoo in 2010.", 
+        "images/VLAT/TreeMap.png"
+    ],
+
+    [
+        "TreeMap",
+        "True/False: Samsung is nested in the Financial category.", 
+        "images/VLAT/TreeMap.png"
+    ]
+]

questions/mini_VLAT.py

@@ -0,0 +1,73 @@
+mini_VLAT_questions=[
+    [
+        "LineChart",
+        "What was the price of a barrel of oil in February 2020?", 
+        "images/mini-VLAT/LineChart.png" 
+    ],
+
+    [
+        "BarChart",
+        "What is the average internet speed in Japan?",
+        "images/mini-VLAT/BarChart.png"
+    ],
+
+    [
+        "StackedBar",
+        "What is the cost of peanuts in Seoul?", 
+        "images/mini-VLAT/StackedBar.png"
+    ],
+
+    [
+        "100%StackedBar",
+        "Which country has the lowest proportion of Gold medals?", 
+        "images/mini-VLAT/Stacked100.png"
+    ],
+    
+    [
+        "PieChart",
+        "What is the approximate global smartphone market share of Samsung?",
+        "images/mini-VLAT/PieChart.png"
+    ],
+    
+    [
+        "Histogram",
+        "What distance have customers traveled in the taxi the most?", 
+        "images/mini-VLAT/Histogram.png"
+    ],
+
+    [
+        "Scatterplot",
+        "True/False: There is a negative linear relationship between the height and the weight of the 85 males.", 
+        "images/mini-VLAT/Scatterplot.png"
+    ],
+
+    [
+        "AreaChart",
+        "What was the average price of pount of coffee beans in October 2019?",
+        "images/mini-VLAT/AreaChart.png"
+    ],
+
+    [
+        "StackedArea",
+        "What was the ratio of girls named 'Isla' to girls named 'Amelia' in 2012 in the UK?", 
+        "images/mini-VLAT/StackedArea.png"
+    ],
+
+    [
+        "BubbleChart",
+        "Which city's metro system has the largest number of stations?", 
+        "images/mini-VLAT/BubbleChart.png"
+    ],
+
+    [ 
+        "Choropleth",
+        "True/False: In 2020, the unemployment rate for Washington (WA) was higher than that of Wisconsin (WI).", 
+        "images/mini-VLAT/Choropleth_New.png"
+    ],
+
+    [
+        "TreeMap",
+        "True/False: eBay is nested in the Software category.", 
+        "images/mini-VLAT/TreeMap.png"
+    ]
+]