diff --git a/llama-index-core/llama_index/core/evaluation/batch_runner.py b/llama-index-core/llama_index/core/evaluation/batch_runner.py
index 33b335c8bf049fcb4279af6eaa46f8f832317500..e7ce75b5c45c687785b66b427a00f0f79961092d 100644
--- a/llama-index-core/llama_index/core/evaluation/batch_runner.py
+++ b/llama-index-core/llama_index/core/evaluation/batch_runner.py
@@ -207,7 +207,9 @@ class BatchEvalRunner:
         eval_kwargs_lists = self._validate_nested_eval_kwargs_types(eval_kwargs_lists)
 
         # boolean to check if using multi kwarg evaluator
-        multi_kwargs = isinstance(next(iter(eval_kwargs_lists.values())), dict)
+        multi_kwargs = len(eval_kwargs_lists) > 0 and isinstance(
+            next(iter(eval_kwargs_lists.values())), dict
+        )
 
         # run evaluations
         eval_jobs = []
@@ -265,7 +267,9 @@ class BatchEvalRunner:
         eval_kwargs_lists = self._validate_nested_eval_kwargs_types(eval_kwargs_lists)
 
         # boolean to check if using multi kwarg evaluator
-        multi_kwargs = isinstance(next(iter(eval_kwargs_lists.values())), dict)
+        multi_kwargs = len(eval_kwargs_lists) > 0 and isinstance(
+            next(iter(eval_kwargs_lists.values())), dict
+        )
 
         # run evaluations
         eval_jobs = []