Spaces:
Running
Running
| [ | |
| { | |
| "Model": "o3-mini-2025-01-31-high", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "91.70", | |
| "Cell Acc": "95.70", | |
| "No answer": "0.30", | |
| "Easy Puzzle Acc": "99.64", | |
| "Hard Puzzle Acc": "88.61", | |
| "Small Puzzle Acc": "99.69", | |
| "Medium Puzzle Acc": "97.14", | |
| "Large Puzzle Acc": "87.50", | |
| "XL Puzzle Acc": "75.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1983.34", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o3-mini-2025-01-31-medium", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "88.90", | |
| "Cell Acc": "90.41", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "99.64", | |
| "Hard Puzzle Acc": "84.72", | |
| "Small Puzzle Acc": "99.69", | |
| "Medium Puzzle Acc": "97.86", | |
| "Large Puzzle Acc": "88.00", | |
| "XL Puzzle Acc": "60.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "2067.98", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-2024-12-17", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "81.00", | |
| "Cell Acc": "78.74", | |
| "No answer": "0.20", | |
| "Easy Puzzle Acc": "98.21", | |
| "Hard Puzzle Acc": "74.31", | |
| "Small Puzzle Acc": "97.19", | |
| "Medium Puzzle Acc": "92.14", | |
| "Large Puzzle Acc": "78.00", | |
| "XL Puzzle Acc": "42.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1197.51", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-R1", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "78.70", | |
| "Cell Acc": "80.54", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "98.57", | |
| "Hard Puzzle Acc": "70.97", | |
| "Small Puzzle Acc": "98.44", | |
| "Medium Puzzle Acc": "95.71", | |
| "Large Puzzle Acc": "73.50", | |
| "XL Puzzle Acc": "28.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "586.33", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o3-mini-2025-01-31-low", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "74.80", | |
| "Cell Acc": "72.60", | |
| "No answer": "1.60", | |
| "Easy Puzzle Acc": "99.29", | |
| "Hard Puzzle Acc": "65.28", | |
| "Small Puzzle Acc": "99.38", | |
| "Medium Puzzle Acc": "91.07", | |
| "Large Puzzle Acc": "64.50", | |
| "XL Puzzle Acc": "23.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "2080.78", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-preview-2024-09-12", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "71.40", | |
| "Cell Acc": "75.14", | |
| "No answer": "0.30", | |
| "Easy Puzzle Acc": "98.57", | |
| "Hard Puzzle Acc": "60.83", | |
| "Small Puzzle Acc": "98.12", | |
| "Medium Puzzle Acc": "88.21", | |
| "Large Puzzle Acc": "59.50", | |
| "XL Puzzle Acc": "17.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1565.88", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-preview-2024-09-12-v2", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "70.40", | |
| "Cell Acc": "74.18", | |
| "No answer": "0.40", | |
| "Easy Puzzle Acc": "98.21", | |
| "Hard Puzzle Acc": "59.58", | |
| "Small Puzzle Acc": "97.81", | |
| "Medium Puzzle Acc": "88.57", | |
| "Large Puzzle Acc": "55.50", | |
| "XL Puzzle Acc": "16.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1559.71", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-mini-2024-09-12-v3", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "59.70", | |
| "Cell Acc": "70.32", | |
| "No answer": "1.00", | |
| "Easy Puzzle Acc": "86.07", | |
| "Hard Puzzle Acc": "49.44", | |
| "Small Puzzle Acc": "87.50", | |
| "Medium Puzzle Acc": "76.79", | |
| "Large Puzzle Acc": "39.00", | |
| "XL Puzzle Acc": "12.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1166.38", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-mini-2024-09-12-v2", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "56.80", | |
| "Cell Acc": "69.87", | |
| "No answer": "1.30", | |
| "Easy Puzzle Acc": "82.86", | |
| "Hard Puzzle Acc": "46.67", | |
| "Small Puzzle Acc": "83.44", | |
| "Medium Puzzle Acc": "76.43", | |
| "Large Puzzle Acc": "36.00", | |
| "XL Puzzle Acc": "7.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1164.95", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "o1-mini-2024-09-12", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "52.60", | |
| "Cell Acc": "52.29", | |
| "No answer": "0.80", | |
| "Easy Puzzle Acc": "87.14", | |
| "Hard Puzzle Acc": "39.17", | |
| "Small Puzzle Acc": "87.81", | |
| "Medium Puzzle Acc": "67.50", | |
| "Large Puzzle Acc": "24.50", | |
| "XL Puzzle Acc": "3.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "993.28", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-v3", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "42.10", | |
| "Cell Acc": "42.04", | |
| "No answer": "27.90", | |
| "Easy Puzzle Acc": "90.00", | |
| "Hard Puzzle Acc": "23.47", | |
| "Small Puzzle Acc": "85.62", | |
| "Medium Puzzle Acc": "44.64", | |
| "Large Puzzle Acc": "10.00", | |
| "XL Puzzle Acc": "1.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "2158.00", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-5-sonnet-20241022", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "36.20", | |
| "Cell Acc": "54.27", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "91.07", | |
| "Hard Puzzle Acc": "14.86", | |
| "Small Puzzle Acc": "84.69", | |
| "Medium Puzzle Acc": "28.93", | |
| "Large Puzzle Acc": "4.00", | |
| "XL Puzzle Acc": "1.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "861.18", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-5-sonnet-20240620", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "33.40", | |
| "Cell Acc": "54.34", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "87.50", | |
| "Hard Puzzle Acc": "12.36", | |
| "Small Puzzle Acc": "83.44", | |
| "Medium Puzzle Acc": "21.79", | |
| "Large Puzzle Acc": "3.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1141.94", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Llama-3.1-405B-Inst-fp8@together", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "32.60", | |
| "Cell Acc": "45.80", | |
| "No answer": "12.50", | |
| "Easy Puzzle Acc": "87.14", | |
| "Hard Puzzle Acc": "11.39", | |
| "Small Puzzle Acc": "81.25", | |
| "Medium Puzzle Acc": "22.50", | |
| "Large Puzzle Acc": "1.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "314.66", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-4o-2024-08-06", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "31.70", | |
| "Cell Acc": "50.34", | |
| "No answer": "3.60", | |
| "Easy Puzzle Acc": "84.64", | |
| "Hard Puzzle Acc": "11.11", | |
| "Small Puzzle Acc": "80.00", | |
| "Medium Puzzle Acc": "19.64", | |
| "Large Puzzle Acc": "2.50", | |
| "XL Puzzle Acc": "0.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1106.51", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemini-1.5-pro-exp-0827", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "30.50", | |
| "Cell Acc": "50.84", | |
| "No answer": "0.80", | |
| "Easy Puzzle Acc": "79.64", | |
| "Hard Puzzle Acc": "11.39", | |
| "Small Puzzle Acc": "75.31", | |
| "Medium Puzzle Acc": "20.71", | |
| "Large Puzzle Acc": "3.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1594.47", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Llama-3.1-405B-Inst@sambanova", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "30.10", | |
| "Cell Acc": "39.06", | |
| "No answer": "24.70", | |
| "Easy Puzzle Acc": "84.64", | |
| "Hard Puzzle Acc": "8.89", | |
| "Small Puzzle Acc": "79.06", | |
| "Medium Puzzle Acc": "16.43", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.50", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "2001.12", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "chatgpt-4o-latest-24-09-07", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "29.90", | |
| "Cell Acc": "48.83", | |
| "No answer": "4.20", | |
| "Easy Puzzle Acc": "81.43", | |
| "Hard Puzzle Acc": "9.86", | |
| "Small Puzzle Acc": "76.88", | |
| "Medium Puzzle Acc": "17.86", | |
| "Large Puzzle Acc": "1.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1539.99", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Mistral-Large-2", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "29.00", | |
| "Cell Acc": "47.64", | |
| "No answer": "1.70", | |
| "Easy Puzzle Acc": "80.36", | |
| "Hard Puzzle Acc": "9.03", | |
| "Small Puzzle Acc": "75.94", | |
| "Medium Puzzle Acc": "15.00", | |
| "Large Puzzle Acc": "2.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1592.39", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-4-turbo-2024-04-09", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "28.40", | |
| "Cell Acc": "47.90", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "80.71", | |
| "Hard Puzzle Acc": "8.06", | |
| "Small Puzzle Acc": "75.31", | |
| "Medium Puzzle Acc": "15.00", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1148.46", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-4o-2024-05-13", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "28.20", | |
| "Cell Acc": "38.72", | |
| "No answer": "19.30", | |
| "Easy Puzzle Acc": "77.86", | |
| "Hard Puzzle Acc": "8.89", | |
| "Small Puzzle Acc": "73.75", | |
| "Medium Puzzle Acc": "16.43", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1643.51", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "grok-2-1212", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "27.70", | |
| "Cell Acc": "48.16", | |
| "No answer": "3.50", | |
| "Easy Puzzle Acc": "76.43", | |
| "Hard Puzzle Acc": "8.75", | |
| "Small Puzzle Acc": "71.88", | |
| "Medium Puzzle Acc": "13.93", | |
| "Large Puzzle Acc": "4.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "2551.39", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-4-0314", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "27.10", | |
| "Cell Acc": "47.43", | |
| "No answer": "0.20", | |
| "Easy Puzzle Acc": "77.14", | |
| "Hard Puzzle Acc": "7.64", | |
| "Small Puzzle Acc": "71.25", | |
| "Medium Puzzle Acc": "13.57", | |
| "Large Puzzle Acc": "2.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1203.17", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-opus-20240229", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "27.00", | |
| "Cell Acc": "48.91", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "78.21", | |
| "Hard Puzzle Acc": "7.08", | |
| "Small Puzzle Acc": "73.44", | |
| "Medium Puzzle Acc": "12.14", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "855.72", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2.5-72B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "26.60", | |
| "Cell Acc": "40.92", | |
| "No answer": "11.90", | |
| "Easy Puzzle Acc": "76.43", | |
| "Hard Puzzle Acc": "7.22", | |
| "Small Puzzle Acc": "72.50", | |
| "Medium Puzzle Acc": "12.14", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1795.90", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2.5-32B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "26.10", | |
| "Cell Acc": "43.39", | |
| "No answer": "6.30", | |
| "Easy Puzzle Acc": "77.50", | |
| "Hard Puzzle Acc": "6.11", | |
| "Small Puzzle Acc": "72.19", | |
| "Medium Puzzle Acc": "10.36", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1333.07", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemini-1.5-pro-exp-0801", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "25.20", | |
| "Cell Acc": "48.50", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "72.50", | |
| "Hard Puzzle Acc": "6.81", | |
| "Small Puzzle Acc": "66.56", | |
| "Medium Puzzle Acc": "13.93", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1389.75", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Llama-3.1-405B-Inst@hyperbolic", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "25.00", | |
| "Cell Acc": "46.62", | |
| "No answer": "6.25", | |
| "Easy Puzzle Acc": "66.67", | |
| "Hard Puzzle Acc": "15.38", | |
| "Small Puzzle Acc": "50.00", | |
| "Medium Puzzle Acc": "33.33", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 16, | |
| "Reason Lens": "1517.13", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemini-1.5-flash-exp-0827", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "25.00", | |
| "Cell Acc": "43.56", | |
| "No answer": "8.50", | |
| "Easy Puzzle Acc": "70.71", | |
| "Hard Puzzle Acc": "7.22", | |
| "Small Puzzle Acc": "65.00", | |
| "Medium Puzzle Acc": "13.57", | |
| "Large Puzzle Acc": "2.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1705.11", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Meta-Llama-3.1-70B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "24.90", | |
| "Cell Acc": "27.98", | |
| "No answer": "43.00", | |
| "Easy Puzzle Acc": "73.57", | |
| "Hard Puzzle Acc": "5.97", | |
| "Small Puzzle Acc": "67.81", | |
| "Medium Puzzle Acc": "10.36", | |
| "Large Puzzle Acc": "1.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1483.68", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-v2-chat-0628", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "22.70", | |
| "Cell Acc": "42.46", | |
| "No answer": "5.20", | |
| "Easy Puzzle Acc": "68.57", | |
| "Hard Puzzle Acc": "4.86", | |
| "Small Puzzle Acc": "63.44", | |
| "Medium Puzzle Acc": "8.57", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1260.23", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-v2.5-0908", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "22.10", | |
| "Cell Acc": "38.01", | |
| "No answer": "12.70", | |
| "Easy Puzzle Acc": "68.21", | |
| "Hard Puzzle Acc": "4.17", | |
| "Small Puzzle Acc": "62.19", | |
| "Medium Puzzle Acc": "7.86", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1294.46", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2-72B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "21.40", | |
| "Cell Acc": "38.32", | |
| "No answer": "10.20", | |
| "Easy Puzzle Acc": "63.93", | |
| "Hard Puzzle Acc": "4.86", | |
| "Small Puzzle Acc": "60.94", | |
| "Medium Puzzle Acc": "6.79", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1813.82", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-v2-coder-0614", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "21.10", | |
| "Cell Acc": "41.58", | |
| "No answer": "4.90", | |
| "Easy Puzzle Acc": "64.64", | |
| "Hard Puzzle Acc": "4.17", | |
| "Small Puzzle Acc": "59.69", | |
| "Medium Puzzle Acc": "7.14", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1324.55", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "deepseek-v2-coder-0724", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "20.50", | |
| "Cell Acc": "42.35", | |
| "No answer": "3.40", | |
| "Easy Puzzle Acc": "61.79", | |
| "Hard Puzzle Acc": "4.44", | |
| "Small Puzzle Acc": "57.50", | |
| "Medium Puzzle Acc": "7.14", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1230.63", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-4o-mini-2024-07-18", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "20.10", | |
| "Cell Acc": "41.26", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "62.50", | |
| "Hard Puzzle Acc": "3.61", | |
| "Small Puzzle Acc": "58.75", | |
| "Medium Puzzle Acc": "4.64", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "943.52", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemini-1.5-flash", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "19.40", | |
| "Cell Acc": "31.77", | |
| "No answer": "22.70", | |
| "Easy Puzzle Acc": "59.29", | |
| "Hard Puzzle Acc": "3.89", | |
| "Small Puzzle Acc": "55.00", | |
| "Medium Puzzle Acc": "6.43", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1538.18", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemini-1.5-pro", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "19.40", | |
| "Cell Acc": "44.59", | |
| "No answer": "0.80", | |
| "Easy Puzzle Acc": "55.71", | |
| "Hard Puzzle Acc": "5.28", | |
| "Small Puzzle Acc": "52.19", | |
| "Medium Puzzle Acc": "9.64", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1336.17", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "yi-large-preview", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "18.90", | |
| "Cell Acc": "42.61", | |
| "No answer": "1.40", | |
| "Easy Puzzle Acc": "58.93", | |
| "Hard Puzzle Acc": "3.33", | |
| "Small Puzzle Acc": "53.75", | |
| "Medium Puzzle Acc": "6.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "833.36", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "yi-large", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "18.80", | |
| "Cell Acc": "39.83", | |
| "No answer": "1.80", | |
| "Easy Puzzle Acc": "58.21", | |
| "Hard Puzzle Acc": "3.47", | |
| "Small Puzzle Acc": "54.37", | |
| "Medium Puzzle Acc": "5.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "757.01", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-5-haiku-20241022", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "18.70", | |
| "Cell Acc": "43.22", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "57.86", | |
| "Hard Puzzle Acc": "3.47", | |
| "Small Puzzle Acc": "53.12", | |
| "Medium Puzzle Acc": "6.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "660.91", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-sonnet-20240229", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "18.70", | |
| "Cell Acc": "43.66", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "58.93", | |
| "Hard Puzzle Acc": "3.06", | |
| "Small Puzzle Acc": "54.06", | |
| "Medium Puzzle Acc": "4.29", | |
| "Large Puzzle Acc": "1.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1095.37", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Meta-Llama-3-70B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "16.80", | |
| "Cell Acc": "42.31", | |
| "No answer": "0.20", | |
| "Easy Puzzle Acc": "52.86", | |
| "Hard Puzzle Acc": "2.78", | |
| "Small Puzzle Acc": "48.44", | |
| "Medium Puzzle Acc": "4.64", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "809.95", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Athene-70B", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "16.70", | |
| "Cell Acc": "32.98", | |
| "No answer": "21.10", | |
| "Easy Puzzle Acc": "52.50", | |
| "Hard Puzzle Acc": "2.78", | |
| "Small Puzzle Acc": "48.75", | |
| "Medium Puzzle Acc": "3.93", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "391.19", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemma-2-27b-it", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "16.30", | |
| "Cell Acc": "41.18", | |
| "No answer": "1.10", | |
| "Easy Puzzle Acc": "50.71", | |
| "Hard Puzzle Acc": "2.92", | |
| "Small Puzzle Acc": "46.56", | |
| "Medium Puzzle Acc": "5.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1014.56", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "claude-3-haiku-20240307", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "14.30", | |
| "Cell Acc": "37.87", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "47.86", | |
| "Hard Puzzle Acc": "1.25", | |
| "Small Puzzle Acc": "43.75", | |
| "Medium Puzzle Acc": "1.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1015.06", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "command-r-plus", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "13.90", | |
| "Cell Acc": "39.01", | |
| "No answer": "0.20", | |
| "Easy Puzzle Acc": "44.64", | |
| "Hard Puzzle Acc": "1.94", | |
| "Small Puzzle Acc": "40.94", | |
| "Medium Puzzle Acc": "2.86", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "810.53", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "reka-core-20240501", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "13.00", | |
| "Cell Acc": "33.88", | |
| "No answer": "4.00", | |
| "Easy Puzzle Acc": "43.21", | |
| "Hard Puzzle Acc": "1.25", | |
| "Small Puzzle Acc": "39.38", | |
| "Medium Puzzle Acc": "1.43", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1078.29", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemma-2-9b-it", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "12.80", | |
| "Cell Acc": "36.79", | |
| "No answer": "0.00", | |
| "Easy Puzzle Acc": "41.79", | |
| "Hard Puzzle Acc": "1.53", | |
| "Small Puzzle Acc": "37.81", | |
| "Medium Puzzle Acc": "2.50", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "849.84", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Meta-Llama-3.1-8B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "12.80", | |
| "Cell Acc": "13.68", | |
| "No answer": "61.50", | |
| "Easy Puzzle Acc": "43.57", | |
| "Hard Puzzle Acc": "0.83", | |
| "Small Puzzle Acc": "39.38", | |
| "Medium Puzzle Acc": "0.71", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1043.90", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2.5-7B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "12.00", | |
| "Cell Acc": "30.67", | |
| "No answer": "9.50", | |
| "Easy Puzzle Acc": "38.93", | |
| "Hard Puzzle Acc": "1.53", | |
| "Small Puzzle Acc": "36.25", | |
| "Medium Puzzle Acc": "1.43", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "850.93", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Meta-Llama-3-8B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "11.90", | |
| "Cell Acc": "23.70", | |
| "No answer": "29.20", | |
| "Easy Puzzle Acc": "40.71", | |
| "Hard Puzzle Acc": "0.69", | |
| "Small Puzzle Acc": "36.88", | |
| "Medium Puzzle Acc": "0.36", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1216.40", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Mistral-Nemo-Instruct-2407", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "11.80", | |
| "Cell Acc": "34.93", | |
| "No answer": "1.60", | |
| "Easy Puzzle Acc": "38.93", | |
| "Hard Puzzle Acc": "1.25", | |
| "Small Puzzle Acc": "35.31", | |
| "Medium Puzzle Acc": "1.79", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "925.88", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Phi-3-mini-4k-instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "11.60", | |
| "Cell Acc": "13.50", | |
| "No answer": "59.00", | |
| "Easy Puzzle Acc": "38.21", | |
| "Hard Puzzle Acc": "1.25", | |
| "Small Puzzle Acc": "35.94", | |
| "Medium Puzzle Acc": "0.36", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "790.29", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Yi-1.5-34B-Chat", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "11.50", | |
| "Cell Acc": "32.73", | |
| "No answer": "4.40", | |
| "Easy Puzzle Acc": "37.50", | |
| "Hard Puzzle Acc": "1.39", | |
| "Small Puzzle Acc": "35.00", | |
| "Medium Puzzle Acc": "1.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "869.65", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gpt-3.5-turbo-0125", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "10.10", | |
| "Cell Acc": "33.06", | |
| "No answer": "0.10", | |
| "Easy Puzzle Acc": "33.57", | |
| "Hard Puzzle Acc": "0.97", | |
| "Small Puzzle Acc": "30.31", | |
| "Medium Puzzle Acc": "1.07", | |
| "Large Puzzle Acc": "0.50", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "820.66", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "command-r", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "9.90", | |
| "Cell Acc": "32.66", | |
| "No answer": "1.50", | |
| "Easy Puzzle Acc": "32.14", | |
| "Hard Puzzle Acc": "1.25", | |
| "Small Puzzle Acc": "30.31", | |
| "Medium Puzzle Acc": "0.71", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1005.17", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "reka-flash-20240226", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "9.30", | |
| "Cell Acc": "25.67", | |
| "No answer": "18.70", | |
| "Easy Puzzle Acc": "30.71", | |
| "Hard Puzzle Acc": "0.97", | |
| "Small Puzzle Acc": "28.44", | |
| "Medium Puzzle Acc": "0.71", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1074.80", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "mathstral-7B-v0.1", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "9.00", | |
| "Cell Acc": "20.42", | |
| "No answer": "36.00", | |
| "Easy Puzzle Acc": "30.00", | |
| "Hard Puzzle Acc": "0.83", | |
| "Small Puzzle Acc": "27.19", | |
| "Medium Puzzle Acc": "1.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1148.16", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Mixtral-8x7B-Instruct-v0.1", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "8.70", | |
| "Cell Acc": "26.47", | |
| "No answer": "20.30", | |
| "Easy Puzzle Acc": "28.93", | |
| "Hard Puzzle Acc": "0.83", | |
| "Small Puzzle Acc": "26.25", | |
| "Medium Puzzle Acc": "1.07", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1177.21", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2-7B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "8.40", | |
| "Cell Acc": "22.06", | |
| "No answer": "24.40", | |
| "Easy Puzzle Acc": "29.29", | |
| "Hard Puzzle Acc": "0.28", | |
| "Small Puzzle Acc": "26.25", | |
| "Medium Puzzle Acc": "0.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1473.23", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Llama-3.2-3B-Instruct@together", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "7.40", | |
| "Cell Acc": "13.14", | |
| "No answer": "54.50", | |
| "Easy Puzzle Acc": "25.71", | |
| "Hard Puzzle Acc": "0.28", | |
| "Small Puzzle Acc": "23.12", | |
| "Medium Puzzle Acc": "0.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "963.47", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Phi-3.5-mini-instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "6.40", | |
| "Cell Acc": "5.98", | |
| "No answer": "80.60", | |
| "Easy Puzzle Acc": "21.79", | |
| "Hard Puzzle Acc": "0.42", | |
| "Small Puzzle Acc": "19.38", | |
| "Medium Puzzle Acc": "0.71", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "718.43", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Qwen2.5-3B-Instruct", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "4.80", | |
| "Cell Acc": "11.44", | |
| "No answer": "56.70", | |
| "Easy Puzzle Acc": "17.14", | |
| "Hard Puzzle Acc": "0.00", | |
| "Small Puzzle Acc": "15.00", | |
| "Medium Puzzle Acc": "0.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "906.58", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "gemma-2-2b-it", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "4.20", | |
| "Cell Acc": "9.97", | |
| "No answer": "57.20", | |
| "Easy Puzzle Acc": "14.29", | |
| "Hard Puzzle Acc": "0.28", | |
| "Small Puzzle Acc": "13.12", | |
| "Medium Puzzle Acc": "0.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1032.89", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| }, | |
| { | |
| "Model": "Yi-1.5-9B-Chat", | |
| "Mode": "greedy", | |
| "Puzzle Acc": "2.30", | |
| "Cell Acc": "7.53", | |
| "No answer": "11.30", | |
| "Easy Puzzle Acc": "8.21", | |
| "Hard Puzzle Acc": "0.00", | |
| "Small Puzzle Acc": "7.19", | |
| "Medium Puzzle Acc": "0.00", | |
| "Large Puzzle Acc": "0.00", | |
| "XL Puzzle Acc": "0.00", | |
| "Total Puzzles": 1000, | |
| "Reason Lens": "1592.60", | |
| "N_Mode": "single", | |
| "N_Size": 1 | |
| } | |
| ] |