pip install vidore-benchmark

vidore-benchmark pipeline list-datasets

vidore-benchmark pipeline evaluate \
    --dataset-name vidore/vidore_v3_hr \
    --module-path path/to/my_pipeline.py \
    --class-name MyCustomPipeline \
    --language english \
    --pipeline-args '{"model_name": "my-model"}'

from vidore_benchmark.pipeline_evaluation import BasePipeline

class MyCustomPipeline(BasePipeline):
    def __init__(self, model_name):
        self.model_name = model_name
        # Initialize your model here

    def index(self, corpus_ids, corpus_images, corpus_texts):
        # Indexing function to process corpus, should store anything
        # relevant as class attributes
        self.corpus_ids = corpus_ids
        ...

    def search(self, query_ids, queries):
        # Your search logic, returns scores dict (see BasePipeline file for description)
        return {query_id: {corpus_id: score}}

vidore-benchmark pipeline evaluate \
    --dataset-name vidore/vidore_v3_hr \
    --pipeline-type random \
    --language english

vidore-benchmark pipeline evaluate-all \
    --pipeline-type random \
    --pipeline-args '{"seed": 42}' \
    --output-dir results/

vidore-benchmark pipeline evaluate-all \
    --module-path my_pipeline.py \
    --class-name MyCustomPipeline \
    --output-dir results/

from path_to_pipeline import MyCustomPipeline
from vidore_benchmark.pipeline_evaluation import (
    load_vidore_dataset,
    evaluate_retrieval,
    aggregate_results,
)

# Load dataset
query_ids, queries, corpus_ids, corpus_images, corpus_texts, qrels = load_vidore_dataset(
    dataset_name="vidore/vidore_v3_hr",
    split="test"
)

# Initialize your pipeline
pipeline = MyCustomPipeline(retriever=my_retriever, reranker=my_reranker)

# Run evaluation
results = evaluate_retrieval(
    pipeline=pipeline,
    query_ids=query_ids,
    queries=queries,
    corpus_ids=corpus_ids,
    corpus_images=corpus_images,
    corpus_texts=corpus_texts,
    qrels=qrels,
    metrics=["ndcg_cut_10", "recall_10"]
)

# Get aggregate scores
aggregated = aggregate_results(results)
print(f"NDCG@10: {aggregated['ndcg_cut_10']:.4f}")

from typing import Dict, List, Any, Optional, Tuple

class PipelineWithMetrics(BasePipeline):
    def index(
        self,
        corpus_ids: List[str],
        corpus_images: List[Any],
        corpus_texts: List[str],
    ) -> None:
        # Indexing logic
        ...

    def search(
        self,
        query_ids: List[str],
        queries: List[str],
    ) -> Tuple[Dict[str, Dict[str, float]], Optional[Dict[str, Any]]]:
        """
        Return both retrieval results and optional tracking metrics.
        
        Returns:
            Tuple of (results, infos) where infos can contain:
            - Cost tracking (e.g., API costs, GPU hours)
            - Granular timing information
            - Resource usage (num_gpus, memory, etc.)
            - Model-specific metadata
        """
        # Your retrieval logic here
        results = {...}
        
        # Optional: track additional metrics
        infos = {
            "estimated_cost_usd": 0.05,
            "num_gpus": 1,
            "total_time_ms": 1234.5,
            "model_name": "my-model-v1",
        }
        
        return results, infos

class SimplePipeline(BasePipeline):
    def search(...) -> Dict[str, Dict[str, float]]:
        # Just return results, no tracking needed
        return results

from vidore_benchmark.pipeline_evaluation import (
    load_vidore_dataset,
    print_dataset_info,
    get_available_datasets,
)

# List available datasets
datasets = get_available_datasets()
print(datasets)

# Load and inspect a dataset
query_ids, queries, corpus_ids, corpus, qrels = load_vidore_dataset(
    "vidore/vidore_v3_industrial"
)

print_dataset_info(
    dataset_name="vidore/vidore_v3_industrial",
    query_ids=query_ids,
    queries=queries,
    corpus_ids=corpus_ids,
    corpus=corpus,
    qrels=qrels,
)

results = evaluate_retrieval(
    pipeline=pipeline,
    query_ids=query_ids,
    queries=queries,
    corpus_ids=corpus_ids,
    corpus=corpus,
    qrels=qrels,
    metrics=[
        "ndcg_cut_5",
        "ndcg_cut_10",
        "recall_5",
        "recall_10",
        "map",
    ]
)

vidore_benchmark/
├── pipeline_evaluation/
│   ├── base_pipeline.py          # BasePipeline abstract class
│   ├── dataset_loader.py          # ViDoRe v3 dataset loading
│   ├── evaluator.py               # Evaluation orchestration
│   ├── utils.py                   # Helper utilities
└── cli/
    └── pipeline_evaluation.py     # CLI for pipeline evaluation

@misc{faysse2024colpaliefficientdocumentretrieval,
      title={ColPali: Efficient Document Retrieval with Vision Language Models}, 
      author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
      year={2024},
      eprint={2407.01449},
      archivePrefix={arXiv},
      primaryClass={cs.IR},
      url={https://arxiv.org/abs/2407.01449}, 
}

@misc{macé2025vidorebenchmarkv2raising,
      title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval}, 
      author={Quentin Macé and António Loison and Manuel Faysse},
      year={2025},
      eprint={2505.17166},
      archivePrefix={arXiv},
      primaryClass={cs.IR},
      url={https://arxiv.org/abs/2505.17166}, 
}

@misc{loison2026vidore,
      title={ViDoRe V3: A Comprehensive Evaluation of Retrieval Augmented Generation in Complex Real-World Scenarios},
      author={Loison, Ant{\'o}nio and Mac{\'e}, Quentin and Edy, Antoine and Xing, Victor and Balough, Tom and Moreira, Gabriel and Liu, Bo and Faysse, Manuel and Hudelot, C{\'e}line and Viaud, Gautier},
      journal={arXiv preprint arXiv:2601.08620},
      year={2026}
}

pip install vidore-benchmark