[Benchmark] Support Video-MME-v2 (#1508)

* support videomme-v2

* fix lint

Author: dongyh20

vlmeval/dataset/__init__.py

@@ -139,6 +139,7 @@
 from .video_holmes import Video_Holmes
 from .video_mmlu import Video_MMLU_CAP, Video_MMLU_QA
 from .videomme import VideoMME
+from .videommev2 import VideoMMEv2
 from .videommmu import VideoMMMU
 from .videott import VideoTT
 from .viewspatialbench import ViewSpatialBench
@@ -308,7 +309,8 @@ def evaluate(self, eval_file, **judge_kwargs):
     QBench_Video, QBench_Video_MCQ, QBench_Video_VQA,
     Video_MMLU_CAP, Video_MMLU_QA,
     Video_Holmes, VCRBench, CGAVCounting,
-    EgoExoBench_MCQ, DREAM, VideoTT, VideoMMMU, MVUEval, OMTGBench, V2PBench, AVSpeakerBench
+    EgoExoBench_MCQ, DREAM, VideoTT, VideoMMMU, MVUEval, OMTGBench, V2PBench, AVSpeakerBench,
+    VideoMMEv2
 ]
 
 # add by EASI team

vlmeval/dataset/utils/videomme.py

@@ -150,3 +150,30 @@ def extract_characters_regex(s):
     if matches is None:
         return ''
     return matches[0]
+
+
+def extract_characters_regex_v2(s):
+    """Extract answer letter from A-H for Video-MME-v2 (supports up to 8 options)."""
+    s = s.strip()
+    answer_prefixes = [
+        'Final Answer:',
+        'The best answer is',
+        'The correct answer is',
+        'The answer is',
+        'The answer',
+        'The best option is',
+        'The correct option is',
+        'Best answer:',
+        'Best option:',
+        'Answer:',
+        'Option:',
+    ]
+    for answer_prefix in answer_prefixes:
+        s = s.replace(answer_prefix, '')
+
+    if len(s.split()) > 10 and not re.search('[A-H]', s):
+        return ''
+    matches = re.search(r'[A-H]', s)
+    if matches is None:
+        return ''
+    return matches[0]

vlmeval/dataset/video_dataset_config.py

@@ -52,6 +52,54 @@
     'Video-MME_0.5fps_subs': partial(VideoMME, dataset='Video-MME', fps=0.5, use_subtitle=True),
 }
 
+videommev2_dataset = {
+    # ── No subtitle ──
+    'Video-MME-v2_64frame': partial(VideoMMEv2, dataset='Video-MME-v2', nframe=64),
+    'Video-MME-v2_1fps': partial(VideoMMEv2, dataset='Video-MME-v2', fps=1.0),
+    # ── Subtitle (non-interleave, concatenated as text block) ──
+    'Video-MME-v2_64frame_subs': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64, with_subtitle=True),
+    'Video-MME-v2_1fps_subs': partial(
+        VideoMMEv2, dataset='Video-MME-v2', fps=1.0, with_subtitle=True),
+    # ── Subtitle (interleave, timestamp-aligned between frames) ──
+    'Video-MME-v2_64frame_subs_interleave': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        with_subtitle=True, subtitle_interleave=True),
+    'Video-MME-v2_1fps_subs_interleave': partial(
+        VideoMMEv2, dataset='Video-MME-v2', fps=1.0,
+        with_subtitle=True, subtitle_interleave=True),
+    # ── Reasoning (no subtitle) ──
+    'Video-MME-v2_64frame_reasoning': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64, reasoning=True),
+    # ── Reasoning + subtitle (non-interleave) ──
+    'Video-MME-v2_64frame_reasoning_subs': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        reasoning=True, with_subtitle=True),
+    # ── Reasoning + subtitle (interleave) ──
+    'Video-MME-v2_64frame_reasoning_subs_interleave': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        reasoning=True, with_subtitle=True, subtitle_interleave=True),
+    # ── Resize (no subtitle) ──
+    'Video-MME-v2_64frame_resize': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        resize_target_area=448 * 448),
+    'Video-MME-v2_1fps_resize': partial(
+        VideoMMEv2, dataset='Video-MME-v2', fps=1.0,
+        resize_target_area=448 * 448),
+    # ── Resize + subtitle ──
+    'Video-MME-v2_64frame_resize_subs': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        resize_target_area=448 * 448, with_subtitle=True),
+    # ── Resize + subtitle interleave ──
+    'Video-MME-v2_64frame_resize_subs_interleave': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        resize_target_area=448 * 448, with_subtitle=True, subtitle_interleave=True),
+    # ── Resize + reasoning ──
+    'Video-MME-v2_64frame_resize_reasoning': partial(
+        VideoMMEv2, dataset='Video-MME-v2', nframe=64,
+        resize_target_area=448 * 448, reasoning=True),
+}
+
 videommmu_dataset = {
     'VideoMMMU_8frame': partial(VideoMMMU, dataset='VideoMMMU', nframe=8),
     'VideoMMMU_64frame': partial(VideoMMMU, dataset='VideoMMMU', nframe=64),
@@ -329,8 +377,8 @@ def _build_video_variants(subsets, cls, variants=VSI_FRAME_VARIANTS):
 supported_video_datasets = {}
 
 dataset_groups = [
-    mmbench_video_dataset, mvbench_dataset, videomme_dataset, videommmu_dataset, longvideobench_dataset,
-    mlvu_dataset, tempcompass_dataset, cgbench_dataset, worldsense_dataset, tamperbench_dataset,
+    mmbench_video_dataset, mvbench_dataset, videomme_dataset, videommev2_dataset, videommmu_dataset,
+    longvideobench_dataset, mlvu_dataset, tempcompass_dataset, cgbench_dataset, worldsense_dataset, tamperbench_dataset,
     megabench_dataset, qbench_video_dataset, moviechat1k_dataset, vdc_dataset, video_holmes_dataset, vcrbench_dataset,
     cg_av_counting_dataset, video_mmlu_dataset, egoexobench_dataset, dream_1k_dataset, video_tt_dataset,
     video_vsi_dataset, mvu_eval_dataset, omtg_dataset, v2pbench_dataset, av_speakerbench_dataset