huggingface leaderboard and github action to update markdown leaderboard from json

.github/scripts/generate-leaderboard.py

@@ -0,0 +1,63 @@
+import json
+
+
+def generate_markdown(data):
+    markdown = """<div align="center">\n\n"""
+    markdown += "# 🔥🏅️GenCeption Leaderboard 🏅️🔥\n\n"
+    markdown += """\n\n</div>\n\n"""
+    markdown += "#### GC@3 scores for different models and categories:\n"
+    markdown += "| Model | **Mean** | Exist. | Count | Posi. | Col. | Post. | Cel. | Sce. | Lan. | Art. | Comm. | **Vis Mean** | Code | Num. | Tran. | OCR | **Text Mean** |\n"
+    markdown += (
+        "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n"
+    )
+    for model in data["models"]:
+        scores = model["scores"]
+        markdown += f"| [{model['name']}]({model['url']}) "
+        for score_key in [
+            "Mean",
+            "Exist",
+            "Count",
+            "Posi",
+            "Col",
+            "Post",
+            "Cel",
+            "Sce",
+            "Lan",
+            "Art",
+            "Comm",
+            "VisMean",
+            "Code",
+            "Num",
+            "Tran",
+            "OCR",
+            "TextMean",
+        ]:
+            if "Mean" in score_key:
+                markdown += f"| **{scores[score_key]}** "
+            else:
+                markdown += f"| {scores[score_key]} "
+        markdown += "|\n"
+    markdown += """\n\nLegend:
+- Exist.: Existence
+- Count: Count
+- Posi.: Position
+- Col.: Color
+- Post.: Poster
+- Cel.: Celebrity
+- Sce.: Scene
+- Lan.: Landmark
+- Art.: Artwork
+- Com. R.: Commonsense Reasoning
+- Code: Code Reasoning
+- Num.: Numerical Calculation
+- Tran.: Text Translation
+- OCR: OCR"""
+    return markdown
+
+
+if __name__ == "__main__":
+    with open("leaderboard/leaderboard.json", "r") as f:
+        data = json.load(f)
+    markdown_content = generate_markdown(data)
+    with open("leaderboard/Leaderboard.md", "w") as f:
+        f.write(markdown_content)

.github/workflows/update-leaderboard.yaml

@@ -0,0 +1,29 @@
+name: Update Markdown from JSON
+
+on:
+  push:
+    paths:
+      - 'leaderboard/leaderboard.json'
+
+jobs:
+  update-markdown:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pyyaml
+      - name: Generate Markdown
+        run: python .github/scripts/generate_leaderboard.py
+      - name: Commit and push if changed
+        run: |
+          git config --global user.email "action@github.com"
+          git config --global user.name "GitHub Action"
+          git add README.md
+          git commit -m "Update README.md from JSON data" -a || echo "No changes to commit"
+          git push

Leaderboard.md

@@ -1,163 +0,0 @@
-# 🔥🏅️GenCeption Leaderboard 🏅️🔥
-
-Evaluated MLLMs: [ChatGPT-4V](https://cdn.openai.com/papers/GPTV_System_Card.pdf), [mPLUG-Owl2](https://arxiv.org/pdf/2311.04257.pdf), [LLaVA-13B](https://arxiv.org/pdf/2304.08485.pdf), [LLaVA-7B](https://arxiv.org/pdf/2304.08485.pdf)
-
-<table>
-<tr><th>Existence </th><th>Count</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.422 |
-| mPLUG-Owl2|0.323 |
-| LLaVA-7B|0.308 |
-| LLaVA-13B|0.305 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.404 |
-| mPLUG-Owl2|0.299 |
-| LLaVA-13B|0.294 |
-| LLaVA-7B|0.353 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Position </th><th>Color</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.408|
-| mPLUG-Owl2|0.306 |
-| LLaVA-7B|0.285 |
-| LLaVA-13B|0.255 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.403 |
-| LLaVA-13B|0.300 |
-| mPLUG-Owl2|0.290 |
-| LLaVA-7B|0.284 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Poster </th><th>Celebrity</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.324|
-| mPLUG-Owl2|0.243 |
-| LLaVA-13B|0.215 |
-| LLaVA-7B|0.214 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.332 |
-| mPLUG-Owl2|0.232 |
-| LLaVA-13B|0.206 |
-| LLaVA-7B|0.188 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Scene </th><th>Landmark</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.393|
-| mPLUG-Owl2|0.299 |
-| LLaVA-13B|0.277 |
-| LLaVA-7B|0.266 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.353 |
-| mPLUG-Owl2|0.275 |
-| LLaVA-7B|0.252 |
-| LLaVA-13B|0.242 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Artwork </th><th>Commonsense Reasoning</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.421|
-| mPLUG-Owl2|0.252 |
-| LLaVA-13B|0.212 |
-| LLaVA-7B|0.210 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.471 |
-| mPLUG-Owl2|0.353 |
-| LLaVA-13B|0.334 |
-| LLaVA-7B|0.294 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Code Reasoning </th><th>Numerical Calculation</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.193|
-| mPLUG-Owl2|0.176 |
-| LLaVA-13B|0.144 |
-| LLaVA-7B|0.107 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.240 |
-| LLaVA-13B|0.195 |
-| mPLUG-Owl2|0.192 |
-| LLaVA-7B|0.155 |
-
-</td></tr> </table>
-
-
-<table>
-<tr><th>Text Translation </th><th>OCR</th></tr>
-<tr><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.157|
-| LLaVA-13B|0.116 |
-| LLaVA-7B|0.111 |
-| mPLUG-Owl2|0.081 |
-
-</td><td>
-
-| Model | GC@3|
-|--|--|
-| ChatGPT-4V|0.393 |
-| mPLUG-Owl2|0.276 |
-| LLaVA-13B|0.239 |
-| LLaVA-7B|0.222 |
-
-</td></tr> </table>

README.md

@@ -22,7 +22,7 @@ We demostrate a 5-iteration GenCeption procedure below run on a seed images to e
 
 
 ## Contribute
-Please **create PR (Pull-Request)** to contribute your results to the [🔥🏅️**Leaderboard**🏅️🔥](https://github.com/EQTPartners/GenCeption/blob/main/Leaderboard.md). Start by creating your virtual environment:
+Please add your model details and results to `leaderboard/leaderboard.json` and **create a PR (Pull-Request)** to contribute your results to the [🔥🏅️**Leaderboard**🏅️🔥](https://github.com/EQTPartners/GenCeption/blob/main/leaderboard/Leaderboard.md). Start by creating your virtual environment:
 
 ```{bash}
 conda create --name genception python=3.10 -y

app.py

@@ -0,0 +1,107 @@
+from apscheduler.schedulers.background import BackgroundScheduler
+from leaderboard.content import (
+    TITLE,
+    BANNER,
+    INTRO,
+    INTRO2,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+)
+import gradio as gr
+import pandas as pd
+import json
+
+
+df = pd.DataFrame()
+
+
+def update_data():
+    global df
+    with open("leaderboard/leaderboard.json", "r") as f:
+        data = json.load(f)
+    df = create_dataframe(data)
+
+
+def filter_columns(df, show_all):
+    if show_all:
+        return df
+    else:
+        mean_columns = [col for col in df.columns if "Mean" in col or col == "Model"]
+        return df[mean_columns]
+
+
+def create_dataframe(data):
+    rows = []
+    for model in data["models"]:
+        name_with_link = f'<a href="{model["url"]}" target="_blank" style="color: blue; text-decoration: underline;">{model["name"]}</a>'
+        row = {"Model": name_with_link}
+        row.update(model["scores"])
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+
+    for col in df.columns:
+        if "Mean" in col:
+            df[col] = df[col].apply(lambda x: f"<strong>{x}</strong>")
+
+    return df
+
+
+def update_display(show_all, df):
+    filtered_df = filter_columns(df, show_all)
+    legend_visibility = gr.update(visible=show_all)
+    return filtered_df, legend_visibility
+
+update_data()
+demo = gr.Blocks()
+with demo:
+    gr.HTML(TITLE)
+    gr.HTML(BANNER)
+    gr.Markdown(INTRO, elem_classes="markdown-text")
+    gr.Markdown(INTRO2, elem_classes="markdown-text")
+    show_all_columns = gr.Checkbox(label="Show all datasets", value=True)
+    data_display = gr.Dataframe(df, datatype="markdown")
+
+    legend_accordion = gr.Accordion("Legend:", open=False, visible=True)
+    with legend_accordion:
+        gr.Markdown(
+            """
+        - Exist.: Existence
+        - Count: Count
+        - Posi.: Position
+        - Col.: Color
+        - Post.: Poster
+        - Cel.: Celebrity
+        - Sce.: Scene
+        - Lan.: Landmark
+        - Art.: Artwork
+        - Com. R.: Commonsense Reasoning
+        - Code: Code Reasoning
+        - Num.: Numerical Calculation
+        - Tran.: Text Translation
+        - OCR: OCR
+        """
+        )
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                elem_id="citation-button",
+                lines=10,
+                show_copy_button=True,
+            )
+
+    show_all_columns.change(
+        update_display,
+        inputs=[show_all_columns, gr.State(df)],
+        outputs=[data_display, legend_accordion],
+    )
+
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(update_data, "cron", hour=0)  # Update data once a day at midnight
+scheduler.start()
+
+demo.queue(default_concurrency_limit=40).launch(share=True)

leaderboard/Leaderboard.md

@@ -0,0 +1,32 @@
+<div align="center">
+
+# 🔥🏅️GenCeption Leaderboard 🏅️🔥
+
+
+
+</div>
+
+#### GC@3 scores for different models and categories:
+| Model | **Mean** | Exist. | Count | Posi. | Col. | Post. | Cel. | Sce. | Lan. | Art. | Comm. | **Vis Mean** | Code | Num. | Tran. | OCR | **Text Mean** |
+|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
+| [ChatGPT-4V](https://cdn.openai.com/papers/GPTV_System_Card.pdf) | **0.351** | 0.422 | 0.404 | 0.408 | 0.403 | 0.324 | 0.332 | 0.393 | 0.353 | 0.421 | 0.471 | **0.393** | 0.193 | 0.24 | 0.157 | 0.393 | **0.246** |
+| [mPLUG-Owl2](https://arxiv.org/pdf/2311.04257.pdf) | **0.257** | 0.323 | 0.299 | 0.306 | 0.29 | 0.243 | 0.232 | 0.299 | 0.275 | 0.252 | 0.353 | **0.287** | 0.176 | 0.192 | 0.081 | 0.276 | **0.181** |
+| [LLaVA-13B](https://arxiv.org/pdf/2304.08485.pdf) | **0.238** | 0.305 | 0.294 | 0.255 | 0.3 | 0.215 | 0.206 | 0.277 | 0.242 | 0.212 | 0.334 | **0.264** | 0.144 | 0.195 | 0.116 | 0.239 | **0.174** |
+| [LLaVA-7B](https://arxiv.org/pdf/2304.08485.pdf) | **0.225** | 0.308 | 0.253 | 0.285 | 0.284 | 0.214 | 0.188 | 0.266 | 0.252 | 0.21 | 0.294 | **0.255** | 0.107 | 0.155 | 0.111 | 0.222 | **0.149** |
+
+
+Legend:
+- Exist.: Existence
+- Count: Count
+- Posi.: Position
+- Col.: Color
+- Post.: Poster
+- Cel.: Celebrity
+- Sce.: Scene
+- Lan.: Landmark
+- Art.: Artwork
+- Com. R.: Commonsense Reasoning
+- Code: Code Reasoning
+- Num.: Numerical Calculation
+- Tran.: Text Translation
+- OCR: OCR

leaderboard/content.py

@@ -0,0 +1,27 @@
+TITLE = """<h1 align="center" id="space-title">🔥🏅️GenCeption Leaderboard 🏅️🔥</h1>"""
+
+BANNER = """<div>
+<p align="center">
+  <a href="https://github.com/EQTPartners/GenCeption/">GitHub</a>&emsp;•&emsp;
+  <a href="https://github.com/EQTPartners/GenCeption#contribute">Contribute</a>&emsp;•&emsp;
+  <a href="https://arxiv.org/abs/2402.14973">Paper</a>&emsp;•&emsp;
+  <a href="https://github.com/EQTPartners/GenCeption#cite-this-work">Citation</a> 
+</p>
+"""
+
+INTRO = """GenCeption is an annotation-free MLLM (Multimodal Large Language Model) evaluation framework that merely requires unimodal data to assess inter-modality semantic coherence and inversely reflects the models' inclination to hallucinate."""
+INTRO2 = """This leaderboard displays the evaluated models ranked by their performance on the **GC@3** metric, as defined in [GenCeption: Evaluate Multimodal LLMs with Unlabeled Unimodal Data](https://arxiv.org/abs/2402.14973). For contributing a model evaluation, please submit a pull request on [GitHub](https://github.com/EQTPartners/GenCeption)."""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite this benchmark"
+CITATION_BUTTON_TEXT = r"""
+@article{cao2023genception,
+    author = {Lele Cao and
+              Valentin Buchner and
+              Zineb Senane and
+              Fangkai Yang},
+    title = {{GenCeption}: Evaluate Multimodal LLMs with Unlabeled Unimodal Data},
+    year={2023},
+    journal={arXiv preprint arXiv:2402.14973},
+    primaryClass={cs.AI,cs.CL,cs.LG}
+}
+"""

leaderboard/leaderboard.json

@@ -0,0 +1,97 @@
+{
+    "models": [
+      {
+        "name": "ChatGPT-4V",
+        "url": "https://cdn.openai.com/papers/GPTV_System_Card.pdf",
+        "scores": {
+          "Mean": 0.351,
+          "Exist": 0.422,
+          "Count": 0.404,
+          "Posi": 0.408,
+          "Col": 0.403,
+          "Post": 0.324,
+          "Cel": 0.332,
+          "Sce": 0.393,
+          "Lan": 0.353,
+          "Art": 0.421,
+          "Comm": 0.471,
+          "VisMean": 0.393,
+          "Code": 0.193,
+          "Num": 0.240,
+          "Tran": 0.157,
+          "OCR": 0.393,
+          "TextMean": 0.246
+        }
+      },
+      {
+        "name": "mPLUG-Owl2",
+        "url": "https://arxiv.org/pdf/2311.04257.pdf",
+        "scores": {
+          "Mean": 0.257,
+          "Exist": 0.323,
+          "Count": 0.299,
+          "Posi": 0.306,
+          "Col": 0.290,
+          "Post": 0.243,
+          "Cel": 0.232,
+          "Sce": 0.299,
+          "Lan": 0.275,
+          "Art": 0.252,
+          "Comm": 0.353,
+          "VisMean": 0.287,
+          "Code": 0.176,
+          "Num": 0.192,
+          "Tran": 0.081,
+          "OCR": 0.276,
+          "TextMean": 0.181
+        }
+      },
+      {
+        "name": "LLaVA-13B",
+        "url": "https://arxiv.org/pdf/2304.08485.pdf",
+        "scores": {
+          "Mean": 0.238,
+          "Exist": 0.305,
+          "Count": 0.294,
+          "Posi": 0.255,
+          "Col": 0.300,
+          "Post": 0.215,
+          "Cel": 0.206,
+          "Sce": 0.277,
+          "Lan": 0.242,
+          "Art": 0.212,
+          "Comm": 0.334,
+          "VisMean": 0.264,
+          "Code": 0.144,
+          "Num": 0.195,
+          "Tran": 0.116,
+          "OCR": 0.239,
+          "TextMean": 0.174
+        }
+      },
+      {
+        "name": "LLaVA-7B",
+        "url": "https://arxiv.org/pdf/2304.08485.pdf",
+        "scores": {
+          "Mean": 0.225,
+          "Exist": 0.308,
+          "Count": 0.253,
+          "Posi": 0.285,
+          "Col": 0.284,
+          "Post": 0.214,
+          "Cel": 0.188,
+          "Sce": 0.266,
+          "Lan": 0.252,
+          "Art": 0.210,
+          "Comm": 0.294,
+          "VisMean": 0.255,
+          "Code": 0.107,
+          "Num": 0.155,
+          "Tran": 0.111,
+          "OCR": 0.222,
+          "TextMean": 0.149
+        }
+      }
+    ]
+  }
+