{
  "updated_at": "2026-09-27",
  "links": {
    "paper": "https://arxiv.org/abs/2603.16859v3",
    "code": "https://github.com/MAC-AutoML/SocialOmni",
    "dataset": "https://huggingface.co/datasets/alexisty/SocialOmni"
  },
  "records": [
    {
      "id": "paper-gpt4o",
      "model": "GPT-4o",
      "variant": "Cascade",
      "status": "published",
      "metrics": {
        "who": 35.05,
        "when": 50.5,
        "qgold": 77.1484375,
        "qens": 76.4957264957265,
        "cov_plus": 30.46875,
        "qens_joint": 23.307291666666668
      },
      "classification": {
        "who_macro_f1": 31.184205577225555,
        "when_macro_f1": 49.83658888804439
      },
      "judge_panel": "paper-v3",
      "judges": [
        "gpt-4o",
        "gemini-2.5-pro",
        "qwen3-omni"
      ],
      "source": {
        "label": {
          "en": "Paper · Table 2",
          "zh": "论文 · 表 2"
        },
        "url": "https://arxiv.org/abs/2603.16859v3"
      },
      "evaluation": {
        "paper": "https://arxiv.org/abs/2603.16859v3",
        "table": "2",
        "source_csv": "https://github.com/MAC-AutoML/SocialOmni/blob/main/reproducibility/arxiv-v3/results/extension/main_extension_summary.csv",
        "quality_scores": "Original paper three-judge panel"
      }
    },
    {
      "id": "modern-gemini_2_5_pro",
      "model": "Gemini 2.5 Pro",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 5.800000000000001,
        "when": 78,
        "qgold": 80.40364583333333,
        "qens": 82.30158730158729,
        "cov_plus": 82.03125,
        "qens_joint": 67.51302083333333
      },
      "classification": {
        "who_macro_f1": 10.988937631256162,
        "when_macro_f1": 76.27008952648042
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-native-av-20260924/gemini25pro"
      },
      "evaluation": {
        "model": "gemini-2.5-pro",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "modern-gemini_2_5_flash",
      "model": "Gemini 2.5 Flash",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 1.7000000000000002,
        "when": 68,
        "qgold": 67.57812499999999,
        "qens": 72.11934156378601,
        "cov_plus": 63.28125,
        "qens_joint": 45.638020833333336
      },
      "classification": {
        "who_macro_f1": 2.999295187419116,
        "when_macro_f1": 67.26342710997443
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-native-av-20260924/gemini25flash"
      },
      "evaluation": {
        "model": "gemini-2.5-flash",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 360
      }
    },
    {
      "id": "modern-gemini_3_flash_preview",
      "model": "Gemini 3 Flash",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 0.75,
        "when": 78.5,
        "qgold": 86.5885416666667,
        "qens": 87.78877887788778,
        "cov_plus": 78.90625,
        "qens_joint": 69.27083333333333
      },
      "classification": {
        "who_macro_f1": 1.5252367438870582,
        "when_macro_f1": 80.70402298850574
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-native-av-20260924/gemini3flash"
      },
      "evaluation": {
        "model": "gemini-3-flash-preview",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "paper-gemini3-pro-preview",
      "model": "Gemini 3 Pro",
      "variant": "Visual-only",
      "status": "published",
      "metrics": {
        "who": 45.4,
        "when": 52,
        "qgold": 21.549479166666668,
        "qens": 25,
        "cov_plus": 32.03125,
        "qens_joint": 8.0078125
      },
      "classification": {
        "who_macro_f1": 45.20445213463031,
        "when_macro_f1": 51.412086243546916
      },
      "judge_panel": "paper-v3",
      "judges": [
        "gpt-4o",
        "gemini-2.5-pro",
        "qwen3-omni"
      ],
      "source": {
        "label": {
          "en": "Paper · Table 2",
          "zh": "论文 · 表 2"
        },
        "url": "https://arxiv.org/abs/2603.16859v3"
      },
      "evaluation": {
        "paper": "https://arxiv.org/abs/2603.16859v3",
        "table": "2",
        "source_csv": "https://github.com/MAC-AutoML/SocialOmni/blob/main/reproducibility/arxiv-v3/results/extension/main_extension_summary.csv",
        "quality_scores": "Original paper three-judge panel"
      }
    },
    {
      "id": "modern-qwen3_omni",
      "model": "Qwen3-Omni",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 70.85000000000001,
        "when": 64,
        "qgold": 49.218749999999986,
        "qens": 45.49019607843137,
        "cov_plus": 66.40625,
        "qens_joint": 30.208333333333332
      },
      "classification": {
        "who_macro_f1": 70.37233594388586,
        "when_macro_f1": 62.339156815566476
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/qwen3_omni"
      },
      "evaluation": {
        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
        "revision": "ea64d566a2a0731823dc44965e7f89bad78f95cb",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "modern-qwen3_omni_thinking",
      "model": "Qwen3-Omni-Thinking",
      "variant": "Native AV · Thinking",
      "status": "published",
      "metrics": {
        "who": 75.65,
        "when": 50.5,
        "qgold": 63.1510416667,
        "qens": 78.8461538462,
        "cov_plus": 30.46875,
        "qens_joint": 24.0234375
      },
      "classification": {
        "who_macro_f1": 75.31305535041041,
        "when_macro_f1": 50.17486948147397
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/qwen3_omni_thinking"
      },
      "evaluation": {
        "model": "Qwen/Qwen3-Omni-30B-A3B-Thinking",
        "revision": "601ad8c82e091cf431a6f36143afb98908b1cf13",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 363
      }
    },
    {
      "id": "modern-qwen2_5_omni",
      "model": "Qwen2.5-Omni",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 4.15,
        "when": 61.5,
        "qgold": 49.34895833333333,
        "qens": 49.89837398373984,
        "cov_plus": 64.0625,
        "qens_joint": 31.966145833333332
      },
      "classification": {
        "who_macro_f1": 6.472834613772034,
        "when_macro_f1": 59.81105978757275
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/qwen2_5_omni"
      },
      "evaluation": {
        "model": "Qwen/Qwen2.5-Omni-7B",
        "revision": "966424e71a6ef4669fbba56d958f0ed23a5d17ab",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "paper-omnivinci",
      "model": "OmniVinci",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 29.75,
        "when": 64.5,
        "qgold": 40.8203125,
        "qens": 38.51851851851852,
        "cov_plus": 70.3125,
        "qens_joint": 27.083333333333332
      },
      "classification": {
        "who_macro_f1": 29.99703543216497,
        "when_macro_f1": 62.03107034947459
      },
      "judge_panel": "paper-v3",
      "judges": [
        "gpt-4o",
        "gemini-2.5-pro",
        "qwen3-omni"
      ],
      "source": {
        "label": {
          "en": "Paper · Table 2",
          "zh": "论文 · 表 2"
        },
        "url": "https://arxiv.org/abs/2603.16859v3"
      },
      "evaluation": {
        "paper": "https://arxiv.org/abs/2603.16859v3",
        "table": "2",
        "source_csv": "https://github.com/MAC-AutoML/SocialOmni/blob/main/reproducibility/arxiv-v3/results/extension/main_extension_summary.csv",
        "quality_scores": "Original paper three-judge panel"
      }
    },
    {
      "id": "modern-vita_1_5",
      "model": "VITA-1.5",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 34.65,
        "when": 56.0,
        "qgold": 50.1953125,
        "qens": 49.43502824858757,
        "cov_plus": 46.09375,
        "qens_joint": 22.786458333333332
      },
      "classification": {
        "who_macro_f1": 41.48866134735696,
        "when_macro_f1": 55.96036432789511
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/vita_1_5"
      }
    },
    {
      "id": "modern-baichuan_omni_1_5",
      "model": "Baichuan-Omni-1.5",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 8.4,
        "when": 16,
        "qgold": 44.270833333333336,
        "qens": 40.625,
        "cov_plus": 6.25,
        "qens_joint": 2.5390625
      },
      "classification": {
        "who_macro_f1": 13.311024605884175,
        "when_macro_f1": 22.740824509098385
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/baichuan_omni_1_5"
      },
      "evaluation": {
        "model": "baichuan-inc/Baichuan-Omni-1d5",
        "revision": "0ab0aff5ceb02d9dbec8349151442ab839b17e61",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "modern-qwen38-omni-flash-nonthinking",
      "model": "Qwen3.8-Omni-Flash",
      "variant": "Native AV · Non-thinking",
      "status": "published",
      "metrics": {
        "who": 89.45,
        "when": 44,
        "qgold": 77.01822916666666,
        "qens": 69.92753623188408,
        "cov_plus": 17.96875,
        "qens_joint": 12.56510416666667
      },
      "classification": {
        "who_macro_f1": 89.43111188946638,
        "when_macro_f1": 41.41646615754786
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/modern-panel-20260924/qwen38"
      }
    },
    {
      "id": "modern-gemini38-flash",
      "model": "Gemini 3.8 Flash",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 25.4,
        "when": 82.5,
        "qgold": 86.5234375,
        "qens": 86.47798742138366,
        "cov_plus": 82.8125,
        "qens_joint": 71.61458333333334
      },
      "classification": {
        "who_macro_f1": 40.29710707255999,
        "when_macro_f1": 83.05625017174576
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/modern-panel-20260924/gemini38"
      }
    },
    {
      "id": "modern-qwen35-omni-plus",
      "model": "Qwen3.5-Omni-Plus",
      "variant": "Native AV · Non-thinking",
      "status": "published",
      "metrics": {
        "who": 91.05,
        "when": 58.5,
        "qgold": 80.2734375,
        "qens": 77.8225806451613,
        "cov_plus": 48.4375,
        "qens_joint": 37.6953125
      },
      "classification": {
        "who_macro_f1": 91.09415774680379,
        "when_macro_f1": 58.44910014768091
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/modern-panel-20260924/qwen35plus"
      }
    },
    {
      "id": "modern-qwen35-omni-flash-20260315",
      "model": "Qwen3.5-Omni-Flash (2026-03-15)",
      "variant": "Native AV · Non-thinking",
      "status": "published",
      "metrics": {
        "who": 86.55,
        "when": 71,
        "qgold": 33.138020833333336,
        "qens": 34.722222222222236,
        "cov_plus": 70.3125,
        "qens_joint": 24.414062500000007
      },
      "classification": {
        "who_macro_f1": 86.72272675439051,
        "when_macro_f1": 69.91389148251893
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/modern-panel-20260924/qwen35flash"
      }
    },
    {
      "id": "modern-gemini-3.1-pro-preview",
      "model": "Gemini 3.1 Pro Preview",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 45.05,
        "when": 84,
        "qgold": 80.01302083333333,
        "qens": 81.9182389937107,
        "cov_plus": 82.8125,
        "qens_joint": 67.83854166666667
      },
      "classification": {
        "who_macro_f1": 59.38685587776218,
        "when_macro_f1": 83.1862126944094
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-native-av-20260924/gemini31pro"
      }
    },
    {
      "id": "modern-gemini-3.5-flash",
      "model": "Gemini 3.5 Flash",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 6,
        "when": 60.5,
        "qgold": 79.55729166666667,
        "qens": 82.94871794871796,
        "cov_plus": 50.78125,
        "qens_joint": 42.122395833333336
      },
      "classification": {
        "who_macro_f1": 11.267280757667388,
        "when_macro_f1": 71.68949771689496
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-native-av-20260924/gemini35flash"
      }
    },
    {
      "id": "modern-gemini-3.5-flash-lite",
      "model": "Gemini 3.5 Flash-Lite",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 78.45,
        "when": 38.5,
        "qgold": 74.609375,
        "qens": 65.27777777777779,
        "cov_plus": 4.6875,
        "qens_joint": 3.0598958333333335
      },
      "classification": {
        "who_macro_f1": 81.2155284129654,
        "when_macro_f1": 31.236897274633122
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-catalog-av-20260924/gemini35flashlite"
      }
    },
    {
      "id": "modern-gemini-3.6-flash",
      "model": "Gemini 3.6 Flash",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 13.2,
        "when": 88.0,
        "qgold": 79.23177083333333,
        "qens": 79.49852507374632,
        "cov_plus": 88.28125,
        "qens_joint": 70.18229166666667
      },
      "classification": {
        "who_macro_f1": 23.19509897377386,
        "when_macro_f1": 87.38152610441767
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-catalog-av-20260924/gemini36flash"
      }
    },
    {
      "id": "modern-gemini-3.7-flash",
      "model": "Gemini 3.7 Flash",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 16.7,
        "when": 73.0,
        "qgold": 84.89583333333333,
        "qens": 84.640522875817,
        "cov_plus": 79.6875,
        "qens_joint": 67.44791666666667
      },
      "classification": {
        "who_macro_f1": 28.46574000141087,
        "when_macro_f1": 76.169805298436
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-catalog-av-20260924/gemini37flash"
      }
    },
    {
      "id": "modern-gemini-3.1-flash-lite",
      "model": "Gemini 3.1 Flash-Lite",
      "variant": "Native AV · Default thinking",
      "status": "published",
      "metrics": {
        "who": 73.9,
        "when": 47.0,
        "qgold": 83.984375,
        "qens": 83.80952380952381,
        "cov_plus": 27.34375,
        "qens_joint": 22.916666666666668
      },
      "classification": {
        "who_macro_f1": 80.60309864094296,
        "when_macro_f1": 46.22564935064935
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/gemini-catalog-av-20260924/gemini31flashlite"
      },
      "evaluation": {
        "model": "gemini-3.1-flash-lite",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "modern-minicpmo-o-4-5",
      "model": "MiniCPM-o 4.5",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 72.8,
        "when": 38.0,
        "qgold": 57.8125,
        "qens": 47.916666666666664,
        "cov_plus": 6.25,
        "qens_joint": 2.9947916666666665
      },
      "classification": {
        "who_macro_f1": 72.52443805382262,
        "when_macro_f1": 31.868131868131872
      },
      "judge_panel": "modern-20260924",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260924/minicpmo_o_4_5"
      },
      "evaluation": {
        "model": "openbmb/MiniCPM-o-4_5",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    },
    {
      "id": "modern-ming-omni-2-0",
      "model": "Ming-Omni 2.0",
      "variant": "Native AV",
      "status": "published",
      "metrics": {
        "who": 52.3,
        "when": 49.5,
        "qgold": 75.78125,
        "qens": 77.34375000000001,
        "cov_plus": 25.0,
        "qens_joint": 19.3359375
      },
      "classification": {
        "who_macro_f1": 50.472745614230405,
        "when_macro_f1": 47.90457769181173
      },
      "judge_panel": "standard",
      "judges": [
        "gemini-3.8-flash",
        "qwen3.8-omni",
        "gpt-5.6-sol"
      ],
      "source": {
        "label": {
          "en": "Answers and scores",
          "zh": "回答与评分"
        },
        "url": "https://github.com/MAC-AutoML/SocialOmni/tree/main/evaluation/results/local-native-av-20260926/ming_flash_omni_2_0"
      },
      "evaluation": {
        "model": "ming-flash-omni-2.0",
        "who_items": 2000,
        "when_items": 200,
        "how_items": 128,
        "judge_scores": 384
      }
    }
  ],
  "judges": [
    "gemini-3.8-flash",
    "dashscope/qwen3.8-omni-flash",
    "gpt-5.6-sol"
  ]
}