modified saving and outputs

app.py

@@ -58,7 +58,7 @@ def multimodal_understanding(model_type,
                              activation_map_method, 
                              visual_method, 
                              image, question, seed, top_p, temperature, target_token_idx,
-                             visualization_layer_min, visualization_layer_max, focus, response_type, chart_type, accumulate_method):
+                             visualization_layer_min, visualization_layer_max, focus, response_type, chart_type, accumulate_method, test_selector):
     # Clear CUDA cache before generating
     gc.collect()
     if torch.cuda.is_available():
@@ -191,7 +191,7 @@ def multimodal_understanding(model_type,
 
     # Collect Results
     RESULTS_ROOT = "./results"
-    FILES_ROOT = f"{RESULTS_ROOT}/{model_name}/{focus}/{visual_method}/{chart_type}/layer{visualization_layer_min}-{visualization_layer_max}/{'all_tokens' if target_token_idx == -1 else f'--{input_ids_decoded[start + target_token_idx]}--'}"
+    FILES_ROOT = f"{RESULTS_ROOT}/{model_name}/{focus}/{visual_method}/{test_selector}/{chart_type}/layer{visualization_layer_min}-{visualization_layer_max}/{'all_tokens' if target_token_idx == -1 else f'--{input_ids_decoded[start + target_token_idx]}--'}"
     os.makedirs(FILES_ROOT, exist_ok=True)
     
     for i, cam_p in enumerate(cam):
@@ -350,7 +350,19 @@ def focus_change(focus):
             return res
 
 
-
+def test_change(test_selector):
+    if test_selector == "mini-VLAT":
+        return gr.Dataset(
+                samples=mini_VLAT_questions,
+            )
+    elif test_selector == "VLAT":
+        return gr.Dataset(
+                samples=VLAT_questions,
+            )
+    else:
+        return gr.Dataset(
+                samples=VLAT_old_questions,
+            )
 
 
 with gr.Blocks() as demo:
@@ -368,6 +380,7 @@ with gr.Blocks() as demo:
 
         with gr.Column():
             model_selector = gr.Dropdown(choices=["Clip", "ChartGemma-3B", "Janus-Pro-1B", "Janus-Pro-7B", "LLaVA-1.5-7B"], value="Clip", label="model")
+            test_selector = gr.Dropdown(choices=["mini-VLAT", "VLAT", "VLAT-old"], value="mini-VLAT", label="test")
             question_input = gr.Textbox(label="Input Prompt")
             und_seed_input = gr.Number(label="Seed", precision=0, value=42)
             top_p = gr.Slider(minimum=0, maximum=1, value=0.95, step=0.05, label="top_p")
@@ -422,10 +435,14 @@ with gr.Blocks() as demo:
 
     examples_inpainting = gr.Examples(
         label="Multimodal Understanding examples",
-        # examples=mini_VLAT_questions,
-        examples=VLAT_questions,
+        examples=mini_VLAT_questions,
         inputs=[chart_type, question_input, image_input],
     )
+
+    test_selector.change(
+        fn=test_change, 
+        inputs=test_selector,
+        outputs=examples_inpainting.dataset)
     
 
 
@@ -433,7 +450,7 @@ with gr.Blocks() as demo:
     understanding_button.click(
         multimodal_understanding,
         inputs=[model_selector, activation_map_method, visual_method, image_input, question_input, und_seed_input, top_p, temperature, target_token_idx, 
-                visualization_layers_min, visualization_layers_max, focus, response_type, chart_type, accumulate_method],
+                visualization_layers_min, visualization_layers_max, focus, response_type, chart_type, accumulate_method, test_selector],
         outputs=[understanding_output, activation_map_output, understanding_target_token_decoded_output]
     )
     

demo/cam.py

@@ -534,7 +534,7 @@ class AttentionGuidedCAMChartGemma(AttentionGuidedCAM):
 
         elif focus == "Language Model":
             self.model.zero_grad()
-            # print(outputs_raw)
+            print("logits shape:", outputs_raw.logits.shape)
             # loss = outputs_raw.logits.max(dim=-1).values.sum()
             if class_idx == -1:
                 loss = outputs_raw.logits.max(dim=-1).values.sum()

demo/visualization.py

@@ -82,8 +82,8 @@ class Visualization:
             
             grad_weights = grad.mean(dim=-1, keepdim=True)
             
-            print("act shape", act.shape)
-            print("grad_weights shape", grad_weights.shape)
+            # print("act shape", act.shape)
+            # print("grad_weights shape", grad_weights.shape)
             
             # cam = (act * grad_weights).sum(dim=-1)
             cam, _ = (act * grad_weights).max(dim=-1)
@@ -132,8 +132,8 @@ class Visualization:
         
         cams = []
         for act, grad in zip(self.activations, self.gradients):
-            print("act shape", act.shape)
-            print("grad shape", grad.shape)
+            # print("act shape", act.shape)
+            # print("grad shape", grad.shape)
             
             grad = F.relu(grad)
 
@@ -160,7 +160,7 @@ class Visualization:
 
         num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
         grid_size = int(num_patches ** 0.5)
-        print(f"Detected grid size: {grid_size}x{grid_size}")
+        # print(f"Detected grid size: {grid_size}x{grid_size}")
         cam_sum = cam_sum.view(grid_size, grid_size).detach()
 
         # Normalize
@@ -184,10 +184,10 @@ class Visualization:
         for i in range(start, cam_sum_raw.shape[1]):
             cam_sum = cam_sum_raw[:, i, :] # shape: [1: seq_len]
             cam_sum = cam_sum[images_seq_mask].unsqueeze(0) # shape: [1, img_seq_len]
-            print("cam_sum shape: ", cam_sum.shape)
+            # print("cam_sum shape: ", cam_sum.shape)
             num_patches = cam_sum.shape[-1]  # Last dimension of CAM output
             grid_size = int(num_patches ** 0.5)
-            print(f"Detected grid size: {grid_size}x{grid_size}")
+            # print(f"Detected grid size: {grid_size}x{grid_size}")
 
             cam_sum = cam_sum.view(grid_size, grid_size)
             if normalize:
@@ -418,6 +418,7 @@ class VisualizationChartGemma(Visualization):
         
         elif focus == "Language Model":
             self.model.zero_grad()
+            print("logits shape:", outputs_raw.logits.shape)
             if target_token_idx == -1:
                 loss = outputs_raw.logits.max(dim=-1).values.sum()
             else:
@@ -486,15 +487,17 @@ def generate_gradcam(
     Generates a Grad-CAM heatmap overlay on top of the input image.
 
     Parameters:
-      attributions (torch.Tensor): A tensor of shape (C, H, W) representing the
-        intermediate activations or gradients at the target layer.
-      image (PIL.Image): The original image.
-      alpha (float): The blending factor for the heatmap overlay (default 0.5).
-      colormap (int): OpenCV colormap to apply (default cv2.COLORMAP_JET).
-      aggregation (str): How to aggregate across channels; either 'mean' or 'sum'.
+        cam (torch.Tensor): A tensor of shape (C, H, W) representing the
+            intermediate activations or gradients at the target layer.
+        image (PIL.Image): The original image.
+        size (tuple): The desired size of the heatmap overlay (default (384, 384)).
+        alpha (float): The blending factor for the heatmap overlay (default 0.5).
+        colormap (int): OpenCV colormap to apply (default cv2.COLORMAP_JET).
+        aggregation (str): How to aggregate across channels; either 'mean' or 'sum'.
+        normalize (bool): Whether to normalize the heatmap (default False).
 
     Returns:
-      PIL.Image: The image overlaid with the Grad-CAM heatmap.
+        PIL.Image: The image overlaid with the Grad-CAM heatmap.
     """
     # print("Generating Grad-CAM with shape:", cam.shape)