| { | |
| "metadata": { | |
| "timestamp": "2025-06-26T06:21:27.128658", | |
| "vidore_benchmark_version": "5.0.1.dev12+ge0eb903" | |
| }, | |
| "metrics": { | |
| "vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.834, | |
| "ndcg_at_3": 0.87602, | |
| "ndcg_at_5": 0.88351, | |
| "ndcg_at_10": 0.89382, | |
| "ndcg_at_20": 0.89856, | |
| "ndcg_at_50": 0.9021, | |
| "ndcg_at_100": 0.90271, | |
| "map_at_1": 0.834, | |
| "map_at_3": 0.86567, | |
| "map_at_5": 0.86987, | |
| "map_at_10": 0.87409, | |
| "map_at_20": 0.87549, | |
| "map_at_50": 0.87605, | |
| "map_at_100": 0.87609, | |
| "recall_at_1": 0.834, | |
| "recall_at_3": 0.906, | |
| "recall_at_5": 0.924, | |
| "recall_at_10": 0.956, | |
| "recall_at_20": 0.974, | |
| "recall_at_50": 0.992, | |
| "recall_at_100": 0.996, | |
| "precision_at_1": 0.834, | |
| "precision_at_3": 0.302, | |
| "precision_at_5": 0.1848, | |
| "precision_at_10": 0.0956, | |
| "precision_at_20": 0.0487, | |
| "precision_at_50": 0.01984, | |
| "precision_at_100": 0.00996, | |
| "mrr_at_1": 0.834, | |
| "mrr_at_3": 0.8656666666666666, | |
| "mrr_at_5": 0.8698666666666666, | |
| "mrr_at_10": 0.8740904761904763, | |
| "mrr_at_20": 0.8754917376740906, | |
| "mrr_at_50": 0.8760454757022237, | |
| "mrr_at_100": 0.8760882870575352, | |
| "naucs_at_1_max": 0.6703392545655197, | |
| "naucs_at_1_std": 0.4132759009409903, | |
| "naucs_at_1_diff1": 0.9520605650682898, | |
| "naucs_at_3_max": 0.6840495063273536, | |
| "naucs_at_3_std": 0.45158630828217644, | |
| "naucs_at_3_diff1": 0.9315116117368949, | |
| "naucs_at_5_max": 0.7034989434370238, | |
| "naucs_at_5_std": 0.46818025455796014, | |
| "naucs_at_5_diff1": 0.9329205366357055, | |
| "naucs_at_10_max": 0.6890968508615603, | |
| "naucs_at_10_std": 0.474195738901625, | |
| "naucs_at_10_diff1": 0.9457813428401673, | |
| "naucs_at_20_max": 0.7880126409538182, | |
| "naucs_at_20_std": 0.5496301084536358, | |
| "naucs_at_20_diff1": 0.9283559577677175, | |
| "naucs_at_50_max": 0.7480158730158629, | |
| "naucs_at_50_std": 0.8190943043884249, | |
| "naucs_at_50_diff1": 0.8978758169934562, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.58537, | |
| "ndcg_at_3": 0.64531, | |
| "ndcg_at_5": 0.65942, | |
| "ndcg_at_10": 0.67939, | |
| "ndcg_at_20": 0.69023, | |
| "ndcg_at_50": 0.70398, | |
| "ndcg_at_100": 0.71126, | |
| "map_at_1": 0.58537, | |
| "map_at_3": 0.63008, | |
| "map_at_5": 0.63762, | |
| "map_at_10": 0.64581, | |
| "map_at_20": 0.64889, | |
| "map_at_50": 0.65114, | |
| "map_at_100": 0.65181, | |
| "recall_at_1": 0.58537, | |
| "recall_at_3": 0.68958, | |
| "recall_at_5": 0.72506, | |
| "recall_at_10": 0.78714, | |
| "recall_at_20": 0.82927, | |
| "recall_at_50": 0.898, | |
| "recall_at_100": 0.94235, | |
| "precision_at_1": 0.58537, | |
| "precision_at_3": 0.22986, | |
| "precision_at_5": 0.14501, | |
| "precision_at_10": 0.07871, | |
| "precision_at_20": 0.04146, | |
| "precision_at_50": 0.01796, | |
| "precision_at_100": 0.00942, | |
| "mrr_at_1": 0.5853658536585366, | |
| "mrr_at_3": 0.6300813008130081, | |
| "mrr_at_5": 0.6376201034737617, | |
| "mrr_at_10": 0.6458064970260089, | |
| "mrr_at_20": 0.6488879496173225, | |
| "mrr_at_50": 0.6511443628922823, | |
| "mrr_at_100": 0.6518089180219389, | |
| "naucs_at_1_max": 0.2585563149452475, | |
| "naucs_at_1_std": 0.3159595366492725, | |
| "naucs_at_1_diff1": 0.8678053985855875, | |
| "naucs_at_3_max": 0.22128557788133152, | |
| "naucs_at_3_std": 0.258482179935891, | |
| "naucs_at_3_diff1": 0.8393573283409745, | |
| "naucs_at_5_max": 0.21064270306895383, | |
| "naucs_at_5_std": 0.23472397866953829, | |
| "naucs_at_5_diff1": 0.8274386494106453, | |
| "naucs_at_10_max": 0.15097771381217617, | |
| "naucs_at_10_std": 0.2839241224708214, | |
| "naucs_at_10_diff1": 0.7893641996302178, | |
| "naucs_at_20_max": 0.15466073118900037, | |
| "naucs_at_20_std": 0.33553862379172555, | |
| "naucs_at_20_diff1": 0.7724781458006538, | |
| "naucs_at_50_max": 0.10906657289614331, | |
| "naucs_at_50_std": 0.5786081651360842, | |
| "naucs_at_50_diff1": 0.7426293121947367, | |
| "naucs_at_100_max": 0.072673798370254, | |
| "naucs_at_100_std": 0.8933863552254951, | |
| "naucs_at_100_diff1": 0.7602442332060952 | |
| }, | |
| "vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.91498, | |
| "ndcg_at_3": 0.94325, | |
| "ndcg_at_5": 0.94908, | |
| "ndcg_at_10": 0.95095, | |
| "ndcg_at_20": 0.95363, | |
| "ndcg_at_50": 0.95442, | |
| "ndcg_at_100": 0.95476, | |
| "map_at_1": 0.91498, | |
| "map_at_3": 0.93623, | |
| "map_at_5": 0.93947, | |
| "map_at_10": 0.94019, | |
| "map_at_20": 0.94099, | |
| "map_at_50": 0.94111, | |
| "map_at_100": 0.94114, | |
| "recall_at_1": 0.91498, | |
| "recall_at_3": 0.96356, | |
| "recall_at_5": 0.97773, | |
| "recall_at_10": 0.98381, | |
| "recall_at_20": 0.99393, | |
| "recall_at_50": 0.99798, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.91498, | |
| "precision_at_3": 0.32119, | |
| "precision_at_5": 0.19555, | |
| "precision_at_10": 0.09838, | |
| "precision_at_20": 0.0497, | |
| "precision_at_50": 0.01996, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9149797570850202, | |
| "mrr_at_3": 0.936234817813765, | |
| "mrr_at_5": 0.9394736842105261, | |
| "mrr_at_10": 0.9401902191375874, | |
| "mrr_at_20": 0.9409887775689154, | |
| "mrr_at_50": 0.9411103076140448, | |
| "mrr_at_100": 0.9411405209199847, | |
| "naucs_at_1_max": 0.6703820792124877, | |
| "naucs_at_1_std": 0.16833937392893533, | |
| "naucs_at_1_diff1": 0.9498534501270284, | |
| "naucs_at_3_max": 0.5822492726969997, | |
| "naucs_at_3_std": 0.06259669593622658, | |
| "naucs_at_3_diff1": 0.9492105456414766, | |
| "naucs_at_5_max": 0.739556315880861, | |
| "naucs_at_5_std": 0.5078895506993929, | |
| "naucs_at_5_diff1": 0.9643814216187027, | |
| "naucs_at_10_max": 0.6766067765559199, | |
| "naucs_at_10_std": 0.3559984957278538, | |
| "naucs_at_10_diff1": 0.9836748182418953, | |
| "naucs_at_20_max": 0.9564661819784134, | |
| "naucs_at_20_std": 0.8638879360590604, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": 0.8693985459351681, | |
| "naucs_at_50_std": 0.8693985459351681, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.91786, | |
| "ndcg_at_3": 0.95383, | |
| "ndcg_at_5": 0.95935, | |
| "ndcg_at_10": 0.95935, | |
| "ndcg_at_20": 0.96032, | |
| "ndcg_at_50": 0.96109, | |
| "ndcg_at_100": 0.96109, | |
| "map_at_1": 0.91786, | |
| "map_at_3": 0.94524, | |
| "map_at_5": 0.9481, | |
| "map_at_10": 0.9481, | |
| "map_at_20": 0.94839, | |
| "map_at_50": 0.94854, | |
| "map_at_100": 0.94854, | |
| "recall_at_1": 0.91786, | |
| "recall_at_3": 0.97857, | |
| "recall_at_5": 0.99286, | |
| "recall_at_10": 0.99286, | |
| "recall_at_20": 0.99643, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.91786, | |
| "precision_at_3": 0.32619, | |
| "precision_at_5": 0.19857, | |
| "precision_at_10": 0.09929, | |
| "precision_at_20": 0.04982, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9178571428571428, | |
| "mrr_at_3": 0.9452380952380953, | |
| "mrr_at_5": 0.948095238095238, | |
| "mrr_at_10": 0.948095238095238, | |
| "mrr_at_20": 0.9483928571428571, | |
| "mrr_at_50": 0.9485416666666666, | |
| "mrr_at_100": 0.9485416666666666, | |
| "naucs_at_1_max": 0.04487882109365634, | |
| "naucs_at_1_std": 0.15499533146591998, | |
| "naucs_at_1_diff1": 0.928754110339789, | |
| "naucs_at_3_max": 0.8358232181761669, | |
| "naucs_at_3_std": 0.9101307189542569, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 0.9346405228758269, | |
| "naucs_at_5_std": 0.9346405228758269, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 0.9346405228758269, | |
| "naucs_at_10_std": 0.9346405228758269, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": 1.0, | |
| "naucs_at_50_std": 1.0, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.70535, | |
| "ndcg_at_3": 0.7868, | |
| "ndcg_at_5": 0.80621, | |
| "ndcg_at_10": 0.82194, | |
| "ndcg_at_20": 0.82672, | |
| "ndcg_at_50": 0.83053, | |
| "ndcg_at_100": 0.83277, | |
| "map_at_1": 0.70535, | |
| "map_at_3": 0.76742, | |
| "map_at_5": 0.77826, | |
| "map_at_10": 0.78488, | |
| "map_at_20": 0.78628, | |
| "map_at_50": 0.78693, | |
| "map_at_100": 0.78712, | |
| "recall_at_1": 0.70535, | |
| "recall_at_3": 0.84265, | |
| "recall_at_5": 0.88943, | |
| "recall_at_10": 0.93742, | |
| "recall_at_20": 0.95565, | |
| "recall_at_50": 0.97448, | |
| "recall_at_100": 0.98846, | |
| "precision_at_1": 0.70535, | |
| "precision_at_3": 0.28088, | |
| "precision_at_5": 0.17789, | |
| "precision_at_10": 0.09374, | |
| "precision_at_20": 0.04778, | |
| "precision_at_50": 0.01949, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.7035236938031592, | |
| "mrr_at_3": 0.767010935601458, | |
| "mrr_at_5": 0.7775212636695018, | |
| "mrr_at_10": 0.7841125190456905, | |
| "mrr_at_20": 0.7855296898659594, | |
| "mrr_at_50": 0.7861619428321974, | |
| "mrr_at_100": 0.7863629532691659, | |
| "naucs_at_1_max": 0.22208704220171518, | |
| "naucs_at_1_std": 0.15415493193166072, | |
| "naucs_at_1_diff1": 0.85100407554343, | |
| "naucs_at_3_max": 0.21518686518713465, | |
| "naucs_at_3_std": 0.23043576383644288, | |
| "naucs_at_3_diff1": 0.8025394364016292, | |
| "naucs_at_5_max": 0.2641753480647899, | |
| "naucs_at_5_std": 0.2706978631234192, | |
| "naucs_at_5_diff1": 0.7806280461098983, | |
| "naucs_at_10_max": 0.2523834187563826, | |
| "naucs_at_10_std": 0.32315905277258156, | |
| "naucs_at_10_diff1": 0.7532415640618384, | |
| "naucs_at_20_max": 0.26510647248798225, | |
| "naucs_at_20_std": 0.3525755808522635, | |
| "naucs_at_20_diff1": 0.7348455395997588, | |
| "naucs_at_50_max": 0.242804665789723, | |
| "naucs_at_50_std": 0.44822875988285776, | |
| "naucs_at_50_diff1": 0.7286029440909012, | |
| "naucs_at_100_max": 0.05932622574605986, | |
| "naucs_at_100_std": 0.28624024988935604, | |
| "naucs_at_100_diff1": 0.7502131245549767 | |
| }, | |
| "vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.81, | |
| "ndcg_at_3": 0.88678, | |
| "ndcg_at_5": 0.907, | |
| "ndcg_at_10": 0.907, | |
| "ndcg_at_20": 0.907, | |
| "ndcg_at_50": 0.90904, | |
| "ndcg_at_100": 0.90904, | |
| "map_at_1": 0.81, | |
| "map_at_3": 0.86833, | |
| "map_at_5": 0.87933, | |
| "map_at_10": 0.87933, | |
| "map_at_20": 0.87933, | |
| "map_at_50": 0.87968, | |
| "map_at_100": 0.87968, | |
| "recall_at_1": 0.81, | |
| "recall_at_3": 0.94, | |
| "recall_at_5": 0.99, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.81, | |
| "precision_at_3": 0.31333, | |
| "precision_at_5": 0.198, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.81, | |
| "mrr_at_3": 0.8683333333333334, | |
| "mrr_at_5": 0.8793333333333334, | |
| "mrr_at_10": 0.8793333333333334, | |
| "mrr_at_20": 0.8793333333333334, | |
| "mrr_at_50": 0.8796781609195403, | |
| "mrr_at_100": 0.8796781609195403, | |
| "naucs_at_1_max": -0.18730682592068792, | |
| "naucs_at_1_std": -0.7260202210697273, | |
| "naucs_at_1_diff1": 0.8433652889098441, | |
| "naucs_at_3_max": 0.3544195455960126, | |
| "naucs_at_3_std": -0.26914098972922335, | |
| "naucs_at_3_diff1": 0.9319172113289744, | |
| "naucs_at_5_max": 0.12278244631185926, | |
| "naucs_at_5_std": 0.35807656395891135, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 0.12278244631185926, | |
| "naucs_at_10_std": 0.35807656395891135, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 0.12278244631185926, | |
| "naucs_at_20_std": 0.35807656395891135, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.99, | |
| "ndcg_at_3": 0.99631, | |
| "ndcg_at_5": 0.99631, | |
| "ndcg_at_10": 0.99631, | |
| "ndcg_at_20": 0.99631, | |
| "ndcg_at_50": 0.99631, | |
| "ndcg_at_100": 0.99631, | |
| "map_at_1": 0.99, | |
| "map_at_3": 0.995, | |
| "map_at_5": 0.995, | |
| "map_at_10": 0.995, | |
| "map_at_20": 0.995, | |
| "map_at_50": 0.995, | |
| "map_at_100": 0.995, | |
| "recall_at_1": 0.99, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.99, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.99, | |
| "mrr_at_3": 0.995, | |
| "mrr_at_5": 0.995, | |
| "mrr_at_10": 0.995, | |
| "mrr_at_20": 0.995, | |
| "mrr_at_50": 0.995, | |
| "mrr_at_100": 0.995, | |
| "naucs_at_1_max": 0.12278244631185359, | |
| "naucs_at_1_std": 0.12278244631185359, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.96, | |
| "ndcg_at_3": 0.96631, | |
| "ndcg_at_5": 0.96631, | |
| "ndcg_at_10": 0.96946, | |
| "ndcg_at_20": 0.97209, | |
| "ndcg_at_50": 0.97406, | |
| "ndcg_at_100": 0.97406, | |
| "map_at_1": 0.96, | |
| "map_at_3": 0.965, | |
| "map_at_5": 0.965, | |
| "map_at_10": 0.96625, | |
| "map_at_20": 0.96702, | |
| "map_at_50": 0.96732, | |
| "map_at_100": 0.96732, | |
| "recall_at_1": 0.96, | |
| "recall_at_3": 0.97, | |
| "recall_at_5": 0.97, | |
| "recall_at_10": 0.98, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.96, | |
| "precision_at_3": 0.32333, | |
| "precision_at_5": 0.194, | |
| "precision_at_10": 0.098, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.96, | |
| "mrr_at_3": 0.965, | |
| "mrr_at_5": 0.965, | |
| "mrr_at_10": 0.96625, | |
| "mrr_at_20": 0.9670192307692308, | |
| "mrr_at_50": 0.9673222610722612, | |
| "mrr_at_100": 0.9673222610722612, | |
| "naucs_at_1_max": 0.7169701213818873, | |
| "naucs_at_1_std": -0.03863211951446941, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 0.7152194211017727, | |
| "naucs_at_3_std": -0.34126984126984133, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 0.7152194211017693, | |
| "naucs_at_5_std": -0.3412698412698435, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 0.7957516339869297, | |
| "naucs_at_10_std": 0.35807656395892185, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 0.7222222222222276, | |
| "naucs_at_20_std": 0.35807656395891135, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.95, | |
| "ndcg_at_3": 0.97393, | |
| "ndcg_at_5": 0.97823, | |
| "ndcg_at_10": 0.97823, | |
| "ndcg_at_20": 0.97823, | |
| "ndcg_at_50": 0.97823, | |
| "ndcg_at_100": 0.97823, | |
| "map_at_1": 0.95, | |
| "map_at_3": 0.96833, | |
| "map_at_5": 0.97083, | |
| "map_at_10": 0.97083, | |
| "map_at_20": 0.97083, | |
| "map_at_50": 0.97083, | |
| "map_at_100": 0.97083, | |
| "recall_at_1": 0.95, | |
| "recall_at_3": 0.99, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.95, | |
| "precision_at_3": 0.33, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.95, | |
| "mrr_at_3": 0.9683333333333333, | |
| "mrr_at_5": 0.9708333333333333, | |
| "mrr_at_10": 0.9708333333333333, | |
| "mrr_at_20": 0.9708333333333333, | |
| "mrr_at_50": 0.9708333333333333, | |
| "mrr_at_100": 0.9708333333333333, | |
| "naucs_at_1_max": 0.6765639589168986, | |
| "naucs_at_1_std": 0.5556489262371623, | |
| "naucs_at_1_diff1": 0.9738562091503253, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 0.8692810457516356, | |
| "naucs_at_3_diff1": 0.8692810457516356, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.98, | |
| "ndcg_at_3": 0.99262, | |
| "ndcg_at_5": 0.99262, | |
| "ndcg_at_10": 0.99262, | |
| "ndcg_at_20": 0.99262, | |
| "ndcg_at_50": 0.99262, | |
| "ndcg_at_100": 0.99262, | |
| "map_at_1": 0.98, | |
| "map_at_3": 0.99, | |
| "map_at_5": 0.99, | |
| "map_at_10": 0.99, | |
| "map_at_20": 0.99, | |
| "map_at_50": 0.99, | |
| "map_at_100": 0.99, | |
| "recall_at_1": 0.98, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.98, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.98, | |
| "mrr_at_3": 0.99, | |
| "mrr_at_5": 0.99, | |
| "mrr_at_10": 0.99, | |
| "mrr_at_20": 0.99, | |
| "mrr_at_50": 0.99, | |
| "mrr_at_100": 0.99, | |
| "naucs_at_1_max": 0.6381886087768457, | |
| "naucs_at_1_std": -0.14122315592903503, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.49561, | |
| "ndcg_at_3": 0.53551, | |
| "ndcg_at_5": 0.57573, | |
| "ndcg_at_10": 0.62797, | |
| "ndcg_at_20": 0.66435, | |
| "ndcg_at_50": 0.68753, | |
| "ndcg_at_100": 0.69778, | |
| "map_at_1": 0.24991, | |
| "map_at_3": 0.39804, | |
| "map_at_5": 0.45596, | |
| "map_at_10": 0.50329, | |
| "map_at_20": 0.52856, | |
| "map_at_50": 0.54302, | |
| "map_at_100": 0.54853, | |
| "recall_at_1": 0.24991, | |
| "recall_at_3": 0.49812, | |
| "recall_at_5": 0.62081, | |
| "recall_at_10": 0.78505, | |
| "recall_at_20": 0.89032, | |
| "recall_at_50": 0.94781, | |
| "recall_at_100": 0.97423, | |
| "precision_at_1": 0.49561, | |
| "precision_at_3": 0.37865, | |
| "precision_at_5": 0.30965, | |
| "precision_at_10": 0.20921, | |
| "precision_at_20": 0.13246, | |
| "precision_at_50": 0.06912, | |
| "precision_at_100": 0.03846, | |
| "mrr_at_1": 0.4956140350877193, | |
| "mrr_at_3": 0.6271929824561403, | |
| "mrr_at_5": 0.642982456140351, | |
| "mrr_at_10": 0.6540744221665276, | |
| "mrr_at_20": 0.6558978051818611, | |
| "mrr_at_50": 0.6561780443192584, | |
| "mrr_at_100": 0.6561780443192584, | |
| "naucs_at_1_max": -0.07400167664801681, | |
| "naucs_at_1_std": 0.03458415022230023, | |
| "naucs_at_1_diff1": 0.36837819306504144, | |
| "naucs_at_3_max": -0.12444851869211698, | |
| "naucs_at_3_std": -0.03350495496687875, | |
| "naucs_at_3_diff1": 0.14678568781036586, | |
| "naucs_at_5_max": -0.18380991972520577, | |
| "naucs_at_5_std": -0.04485546676356389, | |
| "naucs_at_5_diff1": 0.09319332805741351, | |
| "naucs_at_10_max": -0.2287733254906937, | |
| "naucs_at_10_std": -0.11817355407871401, | |
| "naucs_at_10_diff1": 0.039989126851164826, | |
| "naucs_at_20_max": -0.26978462701811906, | |
| "naucs_at_20_std": -0.17072707397422024, | |
| "naucs_at_20_diff1": -0.043988113501541394, | |
| "naucs_at_50_max": -0.2775075319567234, | |
| "naucs_at_50_std": -0.20957246437121108, | |
| "naucs_at_50_diff1": -0.1133061107255248, | |
| "naucs_at_100_max": -0.27585910810842096, | |
| "naucs_at_100_std": -0.2097908968784823, | |
| "naucs_at_100_diff1": -0.14037054801741544 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.60469, | |
| "ndcg_at_3": 0.61057, | |
| "ndcg_at_5": 0.63196, | |
| "ndcg_at_10": 0.66415, | |
| "ndcg_at_20": 0.68919, | |
| "ndcg_at_50": 0.71209, | |
| "ndcg_at_100": 0.72404, | |
| "map_at_1": 0.3749, | |
| "map_at_3": 0.50426, | |
| "map_at_5": 0.54282, | |
| "map_at_10": 0.57771, | |
| "map_at_20": 0.59195, | |
| "map_at_50": 0.60066, | |
| "map_at_100": 0.60317, | |
| "recall_at_1": 0.3749, | |
| "recall_at_3": 0.56825, | |
| "recall_at_5": 0.65837, | |
| "recall_at_10": 0.75073, | |
| "recall_at_20": 0.81942, | |
| "recall_at_50": 0.8876, | |
| "recall_at_100": 0.93527, | |
| "precision_at_1": 0.60469, | |
| "precision_at_3": 0.37448, | |
| "precision_at_5": 0.28031, | |
| "precision_at_10": 0.17922, | |
| "precision_at_20": 0.10523, | |
| "precision_at_50": 0.05022, | |
| "precision_at_100": 0.02742, | |
| "mrr_at_1": 0.6046875, | |
| "mrr_at_3": 0.693489583333333, | |
| "mrr_at_5": 0.7104427083333327, | |
| "mrr_at_10": 0.7158568948412692, | |
| "mrr_at_20": 0.7190933507966805, | |
| "mrr_at_50": 0.7196874906299467, | |
| "mrr_at_100": 0.7198739903840827, | |
| "naucs_at_1_max": 0.21052397754348515, | |
| "naucs_at_1_std": 0.09375197289505234, | |
| "naucs_at_1_diff1": 0.5111101758127156, | |
| "naucs_at_3_max": 0.06090465086494804, | |
| "naucs_at_3_std": -0.001418024019873419, | |
| "naucs_at_3_diff1": -0.03565597745007234, | |
| "naucs_at_5_max": 0.004465748594415919, | |
| "naucs_at_5_std": -0.0484521722756207, | |
| "naucs_at_5_diff1": -0.1245720478106472, | |
| "naucs_at_10_max": -0.059499017411910264, | |
| "naucs_at_10_std": -0.07086245514678893, | |
| "naucs_at_10_diff1": -0.22255807507622197, | |
| "naucs_at_20_max": -0.08861071305293747, | |
| "naucs_at_20_std": -0.04972647301862899, | |
| "naucs_at_20_diff1": -0.28111304038576185, | |
| "naucs_at_50_max": -0.0772093711850375, | |
| "naucs_at_50_std": -0.03833832084634795, | |
| "naucs_at_50_diff1": -0.3229404564565436, | |
| "naucs_at_100_max": -0.09326229510606512, | |
| "naucs_at_100_std": -0.05851062266000862, | |
| "naucs_at_100_diff1": -0.33725273449156307 | |
| }, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.65086, | |
| "ndcg_at_3": 0.60789, | |
| "ndcg_at_5": 0.57982, | |
| "ndcg_at_10": 0.56497, | |
| "ndcg_at_20": 0.5906, | |
| "ndcg_at_50": 0.66142, | |
| "ndcg_at_100": 0.6981, | |
| "map_at_1": 0.09835, | |
| "map_at_3": 0.19265, | |
| "map_at_5": 0.2438, | |
| "map_at_10": 0.31201, | |
| "map_at_20": 0.37105, | |
| "map_at_50": 0.43282, | |
| "map_at_100": 0.45993, | |
| "recall_at_1": 0.09835, | |
| "recall_at_3": 0.2383, | |
| "recall_at_5": 0.31559, | |
| "recall_at_10": 0.44357, | |
| "recall_at_20": 0.59021, | |
| "recall_at_50": 0.79179, | |
| "recall_at_100": 0.90968, | |
| "precision_at_1": 0.65086, | |
| "precision_at_3": 0.55316, | |
| "precision_at_5": 0.49828, | |
| "precision_at_10": 0.40603, | |
| "precision_at_20": 0.30948, | |
| "precision_at_50": 0.1981, | |
| "precision_at_100": 0.12724, | |
| "mrr_at_1": 0.6508620689655172, | |
| "mrr_at_3": 0.7586206896551727, | |
| "mrr_at_5": 0.7706896551724141, | |
| "mrr_at_10": 0.7741362205801864, | |
| "mrr_at_20": 0.7757055120898937, | |
| "mrr_at_50": 0.7760672193481526, | |
| "mrr_at_100": 0.7760672193481526, | |
| "naucs_at_1_max": -0.1577982313304122, | |
| "naucs_at_1_std": 0.07083760025943213, | |
| "naucs_at_1_diff1": 0.1180440061698451, | |
| "naucs_at_3_max": -0.03551684594794198, | |
| "naucs_at_3_std": 0.18649544217765762, | |
| "naucs_at_3_diff1": 0.006931180468183028, | |
| "naucs_at_5_max": -0.042439023438686566, | |
| "naucs_at_5_std": 0.1463028288463992, | |
| "naucs_at_5_diff1": 0.0052961279206988725, | |
| "naucs_at_10_max": -0.014346231321749392, | |
| "naucs_at_10_std": 0.13820096240926596, | |
| "naucs_at_10_diff1": 0.060959204965535974, | |
| "naucs_at_20_max": -0.04036150486209418, | |
| "naucs_at_20_std": 0.10161400684234778, | |
| "naucs_at_20_diff1": 0.058238772027959955, | |
| "naucs_at_50_max": -0.027805254364547293, | |
| "naucs_at_50_std": 0.06305093612338106, | |
| "naucs_at_50_diff1": 0.014479645829478357, | |
| "naucs_at_100_max": -0.04967371405554246, | |
| "naucs_at_100_std": -0.0014108802561097272, | |
| "naucs_at_100_diff1": 0.004463197803405348 | |
| }, | |
| "vidore/restaurant_esg_reports_beir": { | |
| "ndcg_at_1": 0.66026, | |
| "ndcg_at_3": 0.71844, | |
| "ndcg_at_5": 0.74746, | |
| "ndcg_at_10": 0.78463, | |
| "ndcg_at_20": 0.79647, | |
| "ndcg_at_50": 0.80898, | |
| "ndcg_at_100": 0.81195, | |
| "map_at_1": 0.46731, | |
| "map_at_3": 0.6321, | |
| "map_at_5": 0.67856, | |
| "map_at_10": 0.71154, | |
| "map_at_20": 0.71984, | |
| "map_at_50": 0.72621, | |
| "map_at_100": 0.72685, | |
| "recall_at_1": 0.46731, | |
| "recall_at_3": 0.71218, | |
| "recall_at_5": 0.7989, | |
| "recall_at_10": 0.89575, | |
| "recall_at_20": 0.92767, | |
| "recall_at_50": 0.97191, | |
| "recall_at_100": 0.981, | |
| "precision_at_1": 0.67308, | |
| "precision_at_3": 0.41026, | |
| "precision_at_5": 0.30769, | |
| "precision_at_10": 0.18654, | |
| "precision_at_20": 0.10096, | |
| "precision_at_50": 0.04462, | |
| "precision_at_100": 0.02308, | |
| "mrr_at_1": 0.6730769230769231, | |
| "mrr_at_3": 0.7756410256410257, | |
| "mrr_at_5": 0.7852564102564104, | |
| "mrr_at_10": 0.7940705128205129, | |
| "mrr_at_20": 0.7940705128205129, | |
| "mrr_at_50": 0.7949862637362638, | |
| "mrr_at_100": 0.7949862637362638, | |
| "naucs_at_1_max": 0.2155210634937776, | |
| "naucs_at_1_std": 0.20643949634517547, | |
| "naucs_at_1_diff1": 0.5572918449312647, | |
| "naucs_at_3_max": -0.22870270992303324, | |
| "naucs_at_3_std": -0.16008064610097344, | |
| "naucs_at_3_diff1": -0.1052498381156227, | |
| "naucs_at_5_max": -0.20249887866652966, | |
| "naucs_at_5_std": -0.10577079083523987, | |
| "naucs_at_5_diff1": -0.21560021268353965, | |
| "naucs_at_10_max": -0.13190257359821772, | |
| "naucs_at_10_std": -0.02595056097795122, | |
| "naucs_at_10_diff1": -0.25059078573348686, | |
| "naucs_at_20_max": -0.10804998986280905, | |
| "naucs_at_20_std": 0.009844091874970871, | |
| "naucs_at_20_diff1": -0.3106578658424983, | |
| "naucs_at_50_max": -0.14217724030501422, | |
| "naucs_at_50_std": -0.0010783625765841411, | |
| "naucs_at_50_diff1": -0.2900029814490325, | |
| "naucs_at_100_max": -0.15203872114868228, | |
| "naucs_at_100_std": -0.01643757244869887, | |
| "naucs_at_100_diff1": -0.2814362025783248 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.52632, | |
| "ndcg_at_3": 0.55122, | |
| "ndcg_at_5": 0.58777, | |
| "ndcg_at_10": 0.63895, | |
| "ndcg_at_20": 0.66853, | |
| "ndcg_at_50": 0.69944, | |
| "ndcg_at_100": 0.7087, | |
| "map_at_1": 0.28596, | |
| "map_at_3": 0.42149, | |
| "map_at_5": 0.48239, | |
| "map_at_10": 0.52458, | |
| "map_at_20": 0.54701, | |
| "map_at_50": 0.56361, | |
| "map_at_100": 0.56876, | |
| "recall_at_1": 0.28596, | |
| "recall_at_3": 0.5136, | |
| "recall_at_5": 0.6131, | |
| "recall_at_10": 0.77097, | |
| "recall_at_20": 0.8575, | |
| "recall_at_50": 0.93975, | |
| "recall_at_100": 0.96272, | |
| "precision_at_1": 0.52632, | |
| "precision_at_3": 0.37427, | |
| "precision_at_5": 0.31228, | |
| "precision_at_10": 0.21053, | |
| "precision_at_20": 0.12807, | |
| "precision_at_50": 0.06947, | |
| "precision_at_100": 0.03842, | |
| "mrr_at_1": 0.5263157894736842, | |
| "mrr_at_3": 0.6461988304093568, | |
| "mrr_at_5": 0.6549707602339182, | |
| "mrr_at_10": 0.6659844054580898, | |
| "mrr_at_20": 0.6673339331234068, | |
| "mrr_at_50": 0.6679388938009629, | |
| "mrr_at_100": 0.6679388938009629, | |
| "naucs_at_1_max": -0.018448353550608226, | |
| "naucs_at_1_std": 0.0821493132969922, | |
| "naucs_at_1_diff1": 0.30290509894323814, | |
| "naucs_at_3_max": -0.16441638665467292, | |
| "naucs_at_3_std": 0.05181451792946125, | |
| "naucs_at_3_diff1": 0.17081667434507056, | |
| "naucs_at_5_max": -0.2223104695809391, | |
| "naucs_at_5_std": -0.0034901998225501346, | |
| "naucs_at_5_diff1": 0.11827209266301716, | |
| "naucs_at_10_max": -0.3490810408001137, | |
| "naucs_at_10_std": -0.10137127012539782, | |
| "naucs_at_10_diff1": 0.024290790916341346, | |
| "naucs_at_20_max": -0.41179472120133376, | |
| "naucs_at_20_std": -0.18184189514711724, | |
| "naucs_at_20_diff1": -0.026948094739752244, | |
| "naucs_at_50_max": -0.4169831140986695, | |
| "naucs_at_50_std": -0.2419827968566681, | |
| "naucs_at_50_diff1": -0.14707184113572777, | |
| "naucs_at_100_max": -0.41046730276590754, | |
| "naucs_at_100_std": -0.2436064377498405, | |
| "naucs_at_100_diff1": -0.16240322922206768 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.7931, | |
| "ndcg_at_3": 0.69066, | |
| "ndcg_at_5": 0.65993, | |
| "ndcg_at_10": 0.62291, | |
| "ndcg_at_20": 0.64258, | |
| "ndcg_at_50": 0.70498, | |
| "ndcg_at_100": 0.73984, | |
| "map_at_1": 0.11768, | |
| "map_at_3": 0.22479, | |
| "map_at_5": 0.28825, | |
| "map_at_10": 0.35934, | |
| "map_at_20": 0.42423, | |
| "map_at_50": 0.48412, | |
| "map_at_100": 0.51142, | |
| "recall_at_1": 0.11768, | |
| "recall_at_3": 0.26124, | |
| "recall_at_5": 0.34909, | |
| "recall_at_10": 0.46824, | |
| "recall_at_20": 0.62417, | |
| "recall_at_50": 0.80904, | |
| "recall_at_100": 0.92098, | |
| "precision_at_1": 0.7931, | |
| "precision_at_3": 0.62069, | |
| "precision_at_5": 0.56552, | |
| "precision_at_10": 0.43966, | |
| "precision_at_20": 0.325, | |
| "precision_at_50": 0.20172, | |
| "precision_at_100": 0.12914, | |
| "mrr_at_1": 0.7931034482758621, | |
| "mrr_at_3": 0.8534482758620691, | |
| "mrr_at_5": 0.8568965517241378, | |
| "mrr_at_10": 0.8597701149425288, | |
| "mrr_at_20": 0.8620158408190863, | |
| "mrr_at_50": 0.8620158408190863, | |
| "mrr_at_100": 0.8620158408190863, | |
| "naucs_at_1_max": 0.271935938073248, | |
| "naucs_at_1_std": 0.43674674991288676, | |
| "naucs_at_1_diff1": 0.3617820074090711, | |
| "naucs_at_3_max": 0.045282507585294354, | |
| "naucs_at_3_std": 0.14844039042832088, | |
| "naucs_at_3_diff1": 0.11059599624516278, | |
| "naucs_at_5_max": 0.06086984174322351, | |
| "naucs_at_5_std": 0.15073825039690825, | |
| "naucs_at_5_diff1": 0.07597281303252662, | |
| "naucs_at_10_max": 0.030803253605626704, | |
| "naucs_at_10_std": 0.12023189876899391, | |
| "naucs_at_10_diff1": 0.09409994930103974, | |
| "naucs_at_20_max": 0.06111477495291343, | |
| "naucs_at_20_std": 0.1377686346379398, | |
| "naucs_at_20_diff1": 0.08342034215323584, | |
| "naucs_at_50_max": -0.0214422324455313, | |
| "naucs_at_50_std": 0.0629645228435432, | |
| "naucs_at_50_diff1": 0.03433912496739525, | |
| "naucs_at_100_max": -0.07270216260742496, | |
| "naucs_at_100_std": -0.009713954266926159, | |
| "naucs_at_100_diff1": -0.00289458364944889 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.63125, | |
| "ndcg_at_3": 0.63272, | |
| "ndcg_at_5": 0.65663, | |
| "ndcg_at_10": 0.68997, | |
| "ndcg_at_20": 0.71467, | |
| "ndcg_at_50": 0.73633, | |
| "ndcg_at_100": 0.74718, | |
| "map_at_1": 0.39991, | |
| "map_at_3": 0.5245, | |
| "map_at_5": 0.57059, | |
| "map_at_10": 0.6074, | |
| "map_at_20": 0.62094, | |
| "map_at_50": 0.62966, | |
| "map_at_100": 0.63202, | |
| "recall_at_1": 0.39991, | |
| "recall_at_3": 0.57796, | |
| "recall_at_5": 0.67437, | |
| "recall_at_10": 0.77096, | |
| "recall_at_20": 0.84121, | |
| "recall_at_50": 0.90687, | |
| "recall_at_100": 0.94874, | |
| "precision_at_1": 0.63125, | |
| "precision_at_3": 0.38958, | |
| "precision_at_5": 0.29375, | |
| "precision_at_10": 0.18688, | |
| "precision_at_20": 0.10844, | |
| "precision_at_50": 0.05075, | |
| "precision_at_100": 0.02756, | |
| "mrr_at_1": 0.63125, | |
| "mrr_at_3": 0.715625, | |
| "mrr_at_5": 0.7274999999999998, | |
| "mrr_at_10": 0.7328447420634919, | |
| "mrr_at_20": 0.7370251189782439, | |
| "mrr_at_50": 0.7376058481449105, | |
| "mrr_at_100": 0.737786021875342, | |
| "naucs_at_1_max": 0.40116615786322174, | |
| "naucs_at_1_std": 0.12804177750728857, | |
| "naucs_at_1_diff1": 0.5318484780069298, | |
| "naucs_at_3_max": 0.031070174638639564, | |
| "naucs_at_3_std": -0.03258401349687653, | |
| "naucs_at_3_diff1": -0.08211207879257611, | |
| "naucs_at_5_max": -0.03804589294380602, | |
| "naucs_at_5_std": -0.0638529839983047, | |
| "naucs_at_5_diff1": -0.1625118765251509, | |
| "naucs_at_10_max": -0.08589738686452425, | |
| "naucs_at_10_std": -0.0652662376777225, | |
| "naucs_at_10_diff1": -0.27588514824978205, | |
| "naucs_at_20_max": -0.14280800843691288, | |
| "naucs_at_20_std": -0.050772727555263505, | |
| "naucs_at_20_diff1": -0.33260125020460346, | |
| "naucs_at_50_max": -0.18794085961226306, | |
| "naucs_at_50_std": -0.09315309781822757, | |
| "naucs_at_50_diff1": -0.3809855756340824, | |
| "naucs_at_100_max": -0.20443798069201818, | |
| "naucs_at_100_std": -0.09564469982214785, | |
| "naucs_at_100_diff1": -0.416619080759233 | |
| } | |
| } | |
| } |