Upload files with `vila-upload`.

modeling_vila.py

@@ -1082,7 +1082,8 @@ class VILAForCasualLM(VILAPretrainedModel):
 
         return outputs
 
-    @torch.inference_mode()
+    # TODO(ligeng): check how qwen implements this function
+    # @torch.inference_mode()
     def generate(
         self,
         input_ids: Optional[torch.FloatTensor] = None,
@@ -1096,14 +1097,14 @@ class VILAForCasualLM(VILAPretrainedModel):
         input_tokens: <image> describe the image
         media:        [Tensor(1, 3, 384, 384), ]
         ----------->
-        input_tokens:     36000       001 002 003 004
+        input_tokens:      36000      001 002 003 004
         input_emds:     <media emd>   001 002 003 004
         """
         # NOTE: hard code to move to GPU
-        input_ids = input_ids.cuda()
-        media = {k: [v.cuda() for v in media[k]] for k in media}
-        if attention_mask is not None:
-            attention_mask = attention_mask.cuda()
+        # input_ids = input_ids.cuda()
+        # media = {k: [v.cuda() if v is not None for v in media[k]] for k in media}
+        # if attention_mask is not None:
+        #     attention_mask = attention_mask.cuda()
 
         inputs_embeds, _, attention_mask = self._embed(input_ids, media, media_config, None, attention_mask)
         output_ids = self.llm.generate(inputs_embeds=inputs_embeds, attention_mask=attention_mask, **generation_kwargs)