antoineedy commited on
Commit
bf183ba
·
1 Parent(s): f7e0323

display ViDoRe V3 tab

Browse files
Files changed (1) hide show
  1. app.py +188 -175
app.py CHANGED
@@ -80,107 +80,21 @@ def main():
80
 
81
  with gr.Blocks(css=css) as block:
82
  with gr.Tabs():
83
- with gr.TabItem("ViDoRe V1"):
84
- gr.Markdown("# ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
85
- gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
86
-
87
- gr.Markdown(
88
- """
89
- Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
90
-
91
- Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
92
- """
93
- )
94
- datasets_columns_1 = list(data_benchmark_1.columns[4:])
95
-
96
- with gr.Row():
97
- metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
98
- research_textbox_1 = gr.Textbox(
99
- placeholder="🔍 Search Models... [press enter]",
100
- label="Filter Models by Name",
101
- )
102
- column_checkboxes_1 = gr.CheckboxGroup(
103
- choices=datasets_columns_1, value=datasets_columns_1, label="Select Columns to Display"
104
- )
105
-
106
- with gr.Row():
107
- datatype_1 = ["number", "markdown"] + ["number"] * (num_datasets_1 + 1)
108
- dataframe_1 = gr.Dataframe(data_benchmark_1, datatype=datatype_1, type="pandas")
109
-
110
- def update_data_1(metric, search_term, selected_columns):
111
- model_handler.get_vidore_data(metric)
112
- data = model_handler.render_df(metric, benchmark_version=1)
113
- data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
114
- data = filter_models(data, search_term)
115
- if selected_columns:
116
- data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
117
- return data
118
-
119
- with gr.Row():
120
- refresh_button_1 = gr.Button("Refresh")
121
- refresh_button_1.click(
122
- get_refresh_function(model_handler, benchmark_version=1),
123
- inputs=[metric_dropdown_1],
124
- outputs=dataframe_1,
125
- concurrency_limit=20,
126
- )
127
-
128
- # Automatically refresh the dataframe when the dropdown value changes
129
- metric_dropdown_1.change(
130
- get_refresh_function(model_handler, benchmark_version=1),
131
- inputs=[metric_dropdown_1],
132
- outputs=dataframe_1,
133
- )
134
- research_textbox_1.submit(
135
- lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
136
- inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
137
- outputs=dataframe_1,
138
- )
139
- column_checkboxes_1.change(
140
- lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
141
- inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
142
- outputs=dataframe_1,
143
- )
144
-
145
- gr.Markdown(
146
- f"""
147
- - **Total Datasets**: {num_datasets_1}
148
- - **Total Scores**: {num_scores_1}
149
- - **Total Models**: {num_models_1}
150
- """
151
- + r"""
152
- Please consider citing:
153
-
154
- ```bibtex
155
- @misc{faysse2024colpaliefficientdocumentretrieval,
156
- title={ColPali: Efficient Document Retrieval with Vision Language Models},
157
- author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
158
- year={2024},
159
- eprint={2407.01449},
160
- archivePrefix={arXiv},
161
- primaryClass={cs.IR},
162
- url={https://arxiv.org/abs/2407.01449},
163
- }
164
- @misc{macé2025vidorebenchmarkv2raising,
165
- title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
166
- author={Quentin Macé and António Loison and Manuel Faysse},
167
- year={2025},
168
- eprint={2505.17166},
169
- archivePrefix={arXiv},
170
- primaryClass={cs.IR},
171
- url={https://arxiv.org/abs/2505.17166},
172
- }
173
-
174
- ```
175
- """
176
- )
177
- with gr.TabItem("ViDoRe V2"):
178
  gr.Markdown("# ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍")
179
  gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
180
 
181
  gr.Markdown(
182
  """
183
- Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
184
 
185
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
186
  """
@@ -223,7 +137,7 @@ def main():
223
  with gr.Row():
224
  gr.Markdown(
225
  """
226
- **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
227
  Those numbers are not numbers obtained from the organisations that released those models.
228
  """
229
  )
@@ -256,150 +170,146 @@ def main():
256
 
257
  ```bibtex
258
  @misc{faysse2024colpaliefficientdocumentretrieval,
259
- title={ColPali: Efficient Document Retrieval with Vision Language Models},
260
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
261
  year={2024},
262
  eprint={2407.01449},
263
  archivePrefix={arXiv},
264
  primaryClass={cs.IR},
265
- url={https://arxiv.org/abs/2407.01449},
266
  }
267
 
268
  @misc{macé2025vidorebenchmarkv2raising,
269
- title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
270
  author={Quentin Macé and António Loison and Manuel Faysse},
271
  year={2025},
272
  eprint={2505.17166},
273
  archivePrefix={arXiv},
274
  primaryClass={cs.IR},
275
- url={https://arxiv.org/abs/2505.17166},
276
  }
277
  ```
278
  """
279
  )
280
- with gr.TabItem("📚 Submit your model"):
281
- gr.Markdown("# How to Submit a New Model to the Leaderboard")
282
- gr.Markdown(
283
- """
284
- To submit a new model to the ViDoRe leaderboard, follow these steps:
285
-
286
- 1. **Evaluate your model**:
287
- - Follow the evaluation procedure provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/) that uses MTEB.
288
-
289
- 2. **Format your submission file**:
290
- - Add the generated files to [MTEB results](https://github.com/embeddings-benchmark/results) project. Check the [Colpali results](https://github.com/embeddings-benchmark/results/tree/main/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0) for an example.
291
-
292
- And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
293
- gets rebooted, it will appear on startup.
294
 
295
- Note: For proper hyperlink redirection, please ensure that your model repository name is in
296
- kebab-case, e.g. `my-model-name`.
297
- """
298
- )
299
- with gr.TabItem("Deprecated ViDoRe V1"):
300
- gr.Markdown(
301
- "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
302
- "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
303
- "which is no longer maintained. Results should be computed using the "
304
- "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
305
- "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
306
- )
307
- gr.Markdown("## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>")
308
- gr.Markdown("# <span style='color:red'>[Deprecated]</span> ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
309
  gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
310
 
311
  gr.Markdown(
312
  """
313
- Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
314
 
315
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
316
  """
317
  )
318
- deprecated_datasets_columns_1 = list(deprecated_data_benchmark_1.columns[3:])
319
 
320
  with gr.Row():
321
- deprecated_metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
322
- deprecated_research_textbox_1 = gr.Textbox(
323
  placeholder="🔍 Search Models... [press enter]",
324
  label="Filter Models by Name",
325
  )
326
- deprecated_column_checkboxes_1 = gr.CheckboxGroup(
327
- choices=deprecated_datasets_columns_1, value=deprecated_datasets_columns_1, label="Select Columns to Display"
328
  )
329
 
330
  with gr.Row():
331
- deprecated_datatype_1 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_1 + 1)
332
- deprecated_dataframe_1 = gr.Dataframe(deprecated_data_benchmark_1, datatype=deprecated_datatype_1, type="pandas")
333
 
334
- def deprecated_update_data_1(metric, search_term, selected_columns):
335
- deprecated_model_handler.get_vidore_data(metric)
336
- data = deprecated_model_handler.render_df(metric, benchmark_version=1)
337
  data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
338
  data = filter_models(data, search_term)
339
- # data = remove_duplicates(data) # Add this line
340
  if selected_columns:
341
- data = data[["Rank", "Model", "Average"] + selected_columns]
342
  return data
343
 
344
  with gr.Row():
345
- deprecated_refresh_button_1 = gr.Button("Refresh")
346
- deprecated_refresh_button_1.click(
347
- deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
348
- inputs=[deprecated_metric_dropdown_1],
349
- outputs=deprecated_dataframe_1,
350
  concurrency_limit=20,
351
  )
352
 
353
  # Automatically refresh the dataframe when the dropdown value changes
354
- deprecated_metric_dropdown_1.change(
355
- deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
356
- inputs=[deprecated_metric_dropdown_1],
357
- outputs=deprecated_dataframe_1,
358
  )
359
- deprecated_research_textbox_1.submit(
360
- lambda metric, search_term, selected_columns: deprecated_update_data_1(metric, search_term, selected_columns),
361
- inputs=[deprecated_metric_dropdown_1, deprecated_research_textbox_1, deprecated_column_checkboxes_1],
362
- outputs=deprecated_dataframe_1,
363
  )
364
- deprecated_column_checkboxes_1.change(
365
- lambda metric, search_term, selected_columns: deprecated_update_data_1(metric, search_term, selected_columns),
366
- inputs=[deprecated_metric_dropdown_1, deprecated_research_textbox_1, deprecated_column_checkboxes_1],
367
- outputs=deprecated_dataframe_1,
368
  )
369
 
370
  gr.Markdown(
371
  f"""
372
- - **Total Datasets**: {deprecated_num_datasets_1}
373
- - **Total Scores**: {deprecated_num_scores_1}
374
- - **Total Models**: {deprecated_num_models_1}
375
  """
376
  + r"""
377
  Please consider citing:
378
 
379
  ```bibtex
380
  @misc{faysse2024colpaliefficientdocumentretrieval,
381
- title={ColPali: Efficient Document Retrieval with Vision Language Models},
382
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
383
  year={2024},
384
  eprint={2407.01449},
385
  archivePrefix={arXiv},
386
  primaryClass={cs.IR},
387
- url={https://arxiv.org/abs/2407.01449},
388
  }
389
-
390
  @misc{macé2025vidorebenchmarkv2raising,
391
- title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
392
  author={Quentin Macé and António Loison and Manuel Faysse},
393
  year={2025},
394
  eprint={2505.17166},
395
  archivePrefix={arXiv},
396
  primaryClass={cs.IR},
397
- url={https://arxiv.org/abs/2505.17166},
398
  }
 
399
  ```
400
  """
401
  )
402
- with gr.TabItem("Deprecated ViDoRe V2"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
403
  gr.Markdown(
404
  "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
405
  "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
@@ -413,7 +323,7 @@ def main():
413
 
414
  gr.Markdown(
415
  """
416
- Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
417
 
418
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
419
  """
@@ -456,7 +366,7 @@ def main():
456
  with gr.Row():
457
  gr.Markdown(
458
  """
459
- **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
460
  Those numbers are not numbers obtained from the organisations that released those models.
461
  """
462
  )
@@ -489,30 +399,133 @@ def main():
489
 
490
  ```bibtex
491
  @misc{faysse2024colpaliefficientdocumentretrieval,
492
- title={ColPali: Efficient Document Retrieval with Vision Language Models},
493
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
494
  year={2024},
495
  eprint={2407.01449},
496
  archivePrefix={arXiv},
497
  primaryClass={cs.IR},
498
- url={https://arxiv.org/abs/2407.01449},
499
  }
500
 
501
  @misc{macé2025vidorebenchmarkv2raising,
502
- title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
503
  author={Quentin Macé and António Loison and Manuel Faysse},
504
  year={2025},
505
  eprint={2505.17166},
506
  archivePrefix={arXiv},
507
  primaryClass={cs.IR},
508
- url={https://arxiv.org/abs/2505.17166},
509
  }
510
  ```
511
  """
512
  )
513
 
514
- block.queue(max_size=10).launch(debug=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
515
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
516
 
517
  if __name__ == "__main__":
518
  main()
 
80
 
81
  with gr.Blocks(css=css) as block:
82
  with gr.Tabs():
83
+ with gr.TabItem("3️⃣ ViDoRe V3"):
84
+ gr.Markdown("# ViDoRe V3: a comprehensive evaluation of retrieval for enterprise use-cases 📚🔍")
85
+ with gr.Row(variant="panel"):
86
+ gr.Markdown("""
87
+ ### ⚠️ To access the ViDoRe V3 results, please refer directly to the [MTEB Leaderboard](https://huggingface.co/spaces/mteb/leaderboard).
88
+ **ViDoRe V3 is fully integrated into MTEB, which provides a unified platform for evaluating embedding models across various tasks, including document retrieval.**
89
+ **We decided to display ViDoRe V3 results directly on MTEB to leverage its extensive features and community.**
90
+ """)
91
+ with gr.TabItem("2️⃣ ViDoRe V2"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  gr.Markdown("# ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍")
93
  gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
94
 
95
  gr.Markdown(
96
  """
97
+ Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
98
 
99
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
100
  """
 
137
  with gr.Row():
138
  gr.Markdown(
139
  """
140
+ **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
141
  Those numbers are not numbers obtained from the organisations that released those models.
142
  """
143
  )
 
170
 
171
  ```bibtex
172
  @misc{faysse2024colpaliefficientdocumentretrieval,
173
+ title={ColPali: Efficient Document Retrieval with Vision Language Models},
174
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
175
  year={2024},
176
  eprint={2407.01449},
177
  archivePrefix={arXiv},
178
  primaryClass={cs.IR},
179
+ url={https://arxiv.org/abs/2407.01449},
180
  }
181
 
182
  @misc{macé2025vidorebenchmarkv2raising,
183
+ title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
184
  author={Quentin Macé and António Loison and Manuel Faysse},
185
  year={2025},
186
  eprint={2505.17166},
187
  archivePrefix={arXiv},
188
  primaryClass={cs.IR},
189
+ url={https://arxiv.org/abs/2505.17166},
190
  }
191
  ```
192
  """
193
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
194
 
195
+ with gr.TabItem("1️⃣ ViDoRe V1"):
196
+ gr.Markdown("# ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
 
 
 
 
 
 
 
 
 
 
 
 
197
  gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
198
 
199
  gr.Markdown(
200
  """
201
+ Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
202
 
203
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
204
  """
205
  )
206
+ datasets_columns_1 = list(data_benchmark_1.columns[4:])
207
 
208
  with gr.Row():
209
+ metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
210
+ research_textbox_1 = gr.Textbox(
211
  placeholder="🔍 Search Models... [press enter]",
212
  label="Filter Models by Name",
213
  )
214
+ column_checkboxes_1 = gr.CheckboxGroup(
215
+ choices=datasets_columns_1, value=datasets_columns_1, label="Select Columns to Display"
216
  )
217
 
218
  with gr.Row():
219
+ datatype_1 = ["number", "markdown"] + ["number"] * (num_datasets_1 + 1)
220
+ dataframe_1 = gr.Dataframe(data_benchmark_1, datatype=datatype_1, type="pandas")
221
 
222
+ def update_data_1(metric, search_term, selected_columns):
223
+ model_handler.get_vidore_data(metric)
224
+ data = model_handler.render_df(metric, benchmark_version=1)
225
  data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
226
  data = filter_models(data, search_term)
 
227
  if selected_columns:
228
+ data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
229
  return data
230
 
231
  with gr.Row():
232
+ refresh_button_1 = gr.Button("Refresh")
233
+ refresh_button_1.click(
234
+ get_refresh_function(model_handler, benchmark_version=1),
235
+ inputs=[metric_dropdown_1],
236
+ outputs=dataframe_1,
237
  concurrency_limit=20,
238
  )
239
 
240
  # Automatically refresh the dataframe when the dropdown value changes
241
+ metric_dropdown_1.change(
242
+ get_refresh_function(model_handler, benchmark_version=1),
243
+ inputs=[metric_dropdown_1],
244
+ outputs=dataframe_1,
245
  )
246
+ research_textbox_1.submit(
247
+ lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
248
+ inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
249
+ outputs=dataframe_1,
250
  )
251
+ column_checkboxes_1.change(
252
+ lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
253
+ inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
254
+ outputs=dataframe_1,
255
  )
256
 
257
  gr.Markdown(
258
  f"""
259
+ - **Total Datasets**: {num_datasets_1}
260
+ - **Total Scores**: {num_scores_1}
261
+ - **Total Models**: {num_models_1}
262
  """
263
  + r"""
264
  Please consider citing:
265
 
266
  ```bibtex
267
  @misc{faysse2024colpaliefficientdocumentretrieval,
268
+ title={ColPali: Efficient Document Retrieval with Vision Language Models},
269
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
270
  year={2024},
271
  eprint={2407.01449},
272
  archivePrefix={arXiv},
273
  primaryClass={cs.IR},
274
+ url={https://arxiv.org/abs/2407.01449},
275
  }
 
276
  @misc{macé2025vidorebenchmarkv2raising,
277
+ title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
278
  author={Quentin Macé and António Loison and Manuel Faysse},
279
  year={2025},
280
  eprint={2505.17166},
281
  archivePrefix={arXiv},
282
  primaryClass={cs.IR},
283
+ url={https://arxiv.org/abs/2505.17166},
284
  }
285
+
286
  ```
287
  """
288
  )
289
+
290
+ with gr.TabItem("📚 Submit your model"):
291
+ gr.Markdown("# How to Submit a New Model to the Leaderboard")
292
+ gr.Markdown(
293
+ """
294
+ To submit a new model to the ViDoRe leaderboard, follow these steps:
295
+
296
+ 1. **Evaluate your model**:
297
+ - Follow the evaluation procedure provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/) that uses MTEB.
298
+
299
+ 2. **Format your submission file**:
300
+ - Add the generated files to [MTEB results](https://github.com/embeddings-benchmark/results) project. Check the [Colpali results](https://github.com/embeddings-benchmark/results/tree/main/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0) for an example.
301
+
302
+ And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
303
+ gets rebooted, it will appear on startup.
304
+
305
+ Note: For proper hyperlink redirection, please ensure that your model repository name is in
306
+ kebab-case, e.g. `my-model-name`.
307
+ """
308
+ )
309
+
310
+ ### Deprecated Tabs ###
311
+
312
+ with gr.TabItem("⚠️ Deprecated ViDoRe V2"):
313
  gr.Markdown(
314
  "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
315
  "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
 
323
 
324
  gr.Markdown(
325
  """
326
+ Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.
327
 
328
  Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
329
  """
 
366
  with gr.Row():
367
  gr.Markdown(
368
  """
369
+ **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side.
370
  Those numbers are not numbers obtained from the organisations that released those models.
371
  """
372
  )
 
399
 
400
  ```bibtex
401
  @misc{faysse2024colpaliefficientdocumentretrieval,
402
+ title={ColPali: Efficient Document Retrieval with Vision Language Models},
403
  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
404
  year={2024},
405
  eprint={2407.01449},
406
  archivePrefix={arXiv},
407
  primaryClass={cs.IR},
408
+ url={https://arxiv.org/abs/2407.01449},
409
  }
410
 
411
  @misc{macé2025vidorebenchmarkv2raising,
412
+ title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
413
  author={Quentin Macé and António Loison and Manuel Faysse},
414
  year={2025},
415
  eprint={2505.17166},
416
  archivePrefix={arXiv},
417
  primaryClass={cs.IR},
418
+ url={https://arxiv.org/abs/2505.17166},
419
  }
420
  ```
421
  """
422
  )
423
 
424
+ with gr.TabItem("⚠️ Deprecated ViDoRe V1"):
425
+ gr.Markdown(
426
+ "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
427
+ "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
428
+ "which is no longer maintained. Results should be computed using the "
429
+ "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
430
+ "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
431
+ )
432
+ gr.Markdown("## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>")
433
+ gr.Markdown("# <span style='color:red'>[Deprecated]</span> ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
434
+ gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
435
+
436
+ gr.Markdown(
437
+ """
438
+ Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.
439
+
440
+ Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
441
+ """
442
+ )
443
+ deprecated_datasets_columns_1 = list(deprecated_data_benchmark_1.columns[3:])
444
+
445
+ with gr.Row():
446
+ deprecated_metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
447
+ deprecated_research_textbox_1 = gr.Textbox(
448
+ placeholder="🔍 Search Models... [press enter]",
449
+ label="Filter Models by Name",
450
+ )
451
+ deprecated_column_checkboxes_1 = gr.CheckboxGroup(
452
+ choices=deprecated_datasets_columns_1, value=deprecated_datasets_columns_1, label="Select Columns to Display"
453
+ )
454
+
455
+ with gr.Row():
456
+ deprecated_datatype_1 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_1 + 1)
457
+ deprecated_dataframe_1 = gr.Dataframe(deprecated_data_benchmark_1, datatype=deprecated_datatype_1, type="pandas")
458
+
459
+ def deprecated_update_data_1(metric, search_term, selected_columns):
460
+ deprecated_model_handler.get_vidore_data(metric)
461
+ data = deprecated_model_handler.render_df(metric, benchmark_version=1)
462
+ data = add_rank_and_format(data, benchmark_version=1, selected_columns=selected_columns)
463
+ data = filter_models(data, search_term)
464
+ # data = remove_duplicates(data) # Add this line
465
+ if selected_columns:
466
+ data = data[["Rank", "Model", "Average"] + selected_columns]
467
+ return data
468
 
469
+ with gr.Row():
470
+ deprecated_refresh_button_1 = gr.Button("Refresh")
471
+ deprecated_refresh_button_1.click(
472
+ deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
473
+ inputs=[deprecated_metric_dropdown_1],
474
+ outputs=deprecated_dataframe_1,
475
+ concurrency_limit=20,
476
+ )
477
+
478
+ # Automatically refresh the dataframe when the dropdown value changes
479
+ deprecated_metric_dropdown_1.change(
480
+ deprecated_get_refresh_function(deprecated_model_handler, benchmark_version=1),
481
+ inputs=[deprecated_metric_dropdown_1],
482
+ outputs=deprecated_dataframe_1,
483
+ )
484
+ deprecated_research_textbox_1.submit(
485
+ lambda metric, search_term, selected_columns: deprecated_update_data_1(metric, search_term, selected_columns),
486
+ inputs=[deprecated_metric_dropdown_1, deprecated_research_textbox_1, deprecated_column_checkboxes_1],
487
+ outputs=deprecated_dataframe_1,
488
+ )
489
+ deprecated_column_checkboxes_1.change(
490
+ lambda metric, search_term, selected_columns: deprecated_update_data_1(metric, search_term, selected_columns),
491
+ inputs=[deprecated_metric_dropdown_1, deprecated_research_textbox_1, deprecated_column_checkboxes_1],
492
+ outputs=deprecated_dataframe_1,
493
+ )
494
+
495
+ gr.Markdown(
496
+ f"""
497
+ - **Total Datasets**: {deprecated_num_datasets_1}
498
+ - **Total Scores**: {deprecated_num_scores_1}
499
+ - **Total Models**: {deprecated_num_models_1}
500
+ """
501
+ + r"""
502
+ Please consider citing:
503
+
504
+ ```bibtex
505
+ @misc{faysse2024colpaliefficientdocumentretrieval,
506
+ title={ColPali: Efficient Document Retrieval with Vision Language Models},
507
+ author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
508
+ year={2024},
509
+ eprint={2407.01449},
510
+ archivePrefix={arXiv},
511
+ primaryClass={cs.IR},
512
+ url={https://arxiv.org/abs/2407.01449},
513
+ }
514
+
515
+ @misc{macé2025vidorebenchmarkv2raising,
516
+ title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval},
517
+ author={Quentin Macé and António Loison and Manuel Faysse},
518
+ year={2025},
519
+ eprint={2505.17166},
520
+ archivePrefix={arXiv},
521
+ primaryClass={cs.IR},
522
+ url={https://arxiv.org/abs/2505.17166},
523
+ }
524
+ ```
525
+ """
526
+ )
527
+
528
+ block.queue(max_size=10).launch(debug=True)
529
 
530
  if __name__ == "__main__":
531
  main()