🏆 FM4SE Leaderboard

Community-Driven Evaluation of Top Foundation Models (FMs) in Software Engineering (SE) Tasks

The SWE-Model-Arena is an open-source platform designed to evaluate foundation models through human preference, fostering transparency and collaboration. This platform aims to empower the SE community to assess and compare the performance of leading FMs in related tasks. For technical details, check out our paper.
{
  • "headers": [
    • "Rank",
    • "Model",
    • "Website",
    • "Elo Score",
    • "Conversation Efficiency Index",
    • "Model Consistency Score",
    • "Average Win Rate",
    • "Bradley-Terry Coefficient",
    • "Eigenvector Centrality Value",
    • "Newman Modularity Score",
    • "PageRank Score"
    ],
  • "data": [
    • [
      • 1,
      • "claude-3-5-sonnet-20241022",
      • "https://www.anthropic.com/news/claude-3-5-sonnet",
      • 1005.97,
      • 1,
      • null,
      • 1,
      • 0,
      • 1,
      • null,
      • 0.07
      ],
    • [
      • 2,
      • "grok-3-fast-beta",
      • "https://x.ai/news/grok-3",
      • 1003.99,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.06
      ],
    • [
      • 3,
      • "llama-3.1-405b",
      • "",
      • 1003.95,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.06
      ],
    • [
      • 4,
      • "o3",
      • "https://openai.com/index/introducing-o3-and-o4-mini",
      • 1002,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.05
      ],
    • [
      • 4,
      • "grok-3-mini-beta",
      • "https://x.ai/news/grok-3",
      • 1002,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 4,
      • "gemini-2.0-flash-lite-preview",
      • "",
      • 1002,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 4,
      • "deepseek-v3.1",
      • "https://www.deepseek.com",
      • 1002,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 8,
      • "o3-mini",
      • "https://openai.com/index/introducing-o3-and-o4-mini",
      • 1001.99,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 9,
      • "mistral-large-latest",
      • "https://mistral.ai/news/mistral-large",
      • 1001.98,
      • 0.72,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 10,
      • "deepseek-chat",
      • "https://www.deepseek.com",
      • 1001.94,
      • 1,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.04
      ],
    • [
      • 11,
      • "claude-3-opus-20240229",
      • "https://www.anthropic.com/news/claude-3-family",
      • 999.99,
      • 0.53,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.03
      ],
    • [
      • 12,
      • "gpt-4-turbo",
      • "https://openai.com/index/gpt-4",
      • 999.98,
      • -0.33,
      • null,
      • 1,
      • 0,
      • 0,
      • null,
      • 0.03
      ],
    • [
      • 12,
      • "gpt-oss-120b",
      • "https://openai.com/index/introducing-gpt-oss",
      • 999.98,
      • 1,
      • 1,
      • 0.5,
      • 0,
      • 0,
      • 0,
      • 0.03
      ],
    • [
      • 14,
      • "gemma-3-27b-it",
      • "https://ai.google.dev/gemma",
      • 998.02,
      • 0.3,
      • null,
      • 0.33,
      • 0,
      • 0,
      • 0,
      • 0.03
      ],
    • [
      • 14,
      • "gpt-4o-mini",
      • "https://openai.com/index/gpt-4o-mini-advancing-cost-efficient-intelligence",
      • 998.02,
      • -1,
      • null,
      • 0.33,
      • 0,
      • 0,
      • 0,
      • 0.03
      ],
    • [
      • 16,
      • "gemini-1.5-flash",
      • "",
      • 998.01,
      • -0.6,
      • null,
      • 0.25,
      • 0,
      • 0,
      • 0,
      • 0.03
      ],
    • [
      • 16,
      • "claude-3-haiku-20240307",
      • "https://www.anthropic.com/news/claude-3-haiku",
      • 998.01,
      • -1,
      • null,
      • 0.12,
      • 0,
      • 0,
      • 0,
      • 0.03
      ],
    • [
      • 16,
      • "o4-mini",
      • "https://openai.com/index/introducing-o3-and-o4-mini",
      • 998.01,
      • 0.3,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 16,
      • "gpt-4o",
      • "https://openai.com/index/hello-gpt-4o",
      • 998.01,
      • -0.2,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "o1-mini",
      • "https://openai.com/o1",
      • 998,
      • 0.3,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "deepseek-r1",
      • "https://www.deepseek.com",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "gemini-2.0-pro-exp",
      • "",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "gemini-2.5-pro-preview-06-05",
      • "https://deepmind.google/models/gemini/pro",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "qwen3-30b-a3b",
      • "",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "o1",
      • "https://openai.com/o1",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 20,
      • "gpt-5-mini",
      • "https://openai.com/gpt-5",
      • 998,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 27,
      • "grok-3-beta",
      • "https://x.ai/news/grok-3",
      • 997.99,
      • 0.3,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 28,
      • "gpt-4.1-mini",
      • "https://openai.com/index/gpt-4-1",
      • 996.01,
      • -1,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 28,
      • "claude-3-7-sonnet-latest",
      • "https://www.anthropic.com/claude/sonnet",
      • 996.01,
      • -0.13,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 30,
      • "claude-3-7-sonnet-20250219#thinking",
      • "https://www.anthropic.com/claude/sonnet",
      • 996,
      • -0.43,
      • null,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 30,
      • "gemini-1.5-pro",
      • "",
      • 996,
      • null,
      • 1,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ],
    • [
      • 32,
      • "gpt-3.5-turbo",
      • "https://openai.com",
      • 988.14,
      • -0.63,
      • 0.33,
      • 0,
      • 0,
      • 0,
      • 0,
      • 0.02
      ]
    ],
  • "metadata": null
}

Made with ❤️ for SWE-Model-Arena. If this work is useful to you, please consider citing our vision paper:

@inproceedings{zhao2025se,
title={SE Arena: An Interactive Platform for Evaluating Foundation Models in Software Engineering},
author={Zhao, Zhimin},
booktitle={2025 IEEE/ACM Second International Conference on AI Foundation Models and Software Engineering (Forge)},
pages={78--81},
year={2025},
organization={IEEE}
}