@inproceedings{kuru2025evaluating, month = nov, booktitle = {48th International ACM SIGIR Conference on Research and Development in Information Retrieval, Workshop on Information Retrieval's Role in RAG Systems}, doi = {10.13140/RG.2.2.24324.18561}, author = {Kuru, Emre and Keskin, Mehmet Onur}, year = {2025}, pages = {}, title = {Evaluating Modern RAG: Textual, Multimodal, Dense, and Late Interaction Pipelines}, abstract = {Retrieval-augmented generation (RAG) systems traditionally rely on text-based pipelines to extract and retrieve information from documents. While efficient, these systems often struggle with documents where meaning is conveyed through layout, tables, and visual elements. Recent advancements in multimodal pipelines, utilizing advanced vision-language models (VLMs), have significantly improved retrieval quality by jointly encoding visual and textual signals. However, these improvements come with increased memory requirements and higher indexing and retrieval latency. To address these trade-offs, a quantitative, data-driven methodology is proposed for selecting the most suitable RAG pipeline based on empirical performance and resource constraints. This involves evaluating leading contemporary textual and multimodal pipelines, including dense and late-interaction architectures, to analyze their strengths, limitations, and the types of documents they are best equipped to handle. The evaluation focuses on key design dimensions in modern RAG systems: input modality (textual vs. visual) and retrieval architecture (dense vs. late interaction). The study highlights that text-based pipelines typically follow a multi-stage preprocessing procedure, and while efficient, they can lose fine-grained alignment crucial for structured or layout-dependent documents. Multimodal approaches, on the other hand, preserve token-level or patch-level granularity during indexing. The goal is to guide practitioners in identifying the most efficient and accurate pipeline for specific tasks.} }