update v1.1

.gitattributes

@@ -34,4 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-OpenS2V-Nexus_logo.png filter=lfs diff=lfs merge=lfs -text
+OpenS2V-Nexus_logo.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -25,7 +25,7 @@ thumbnail: >-
 **OpenS2V-Eval** introduces 180 prompts from seven major categories of S2V, which incorporate both real and synthetic test data. Furthermore, 
 to accurately align human preferences with S2V benchmarks, we propose three automatic metrics: **NexusScore**, **NaturalScore**, **GmeScore**
 to separately quantify subject consistency, naturalness, and text relevance in generated videos. Building on this, we conduct a comprehensive 
-evaluation of 14 representative S2V models, highlighting their strengths and weaknesses across different content.
+evaluation of 18 representative S2V models, highlighting their strengths and weaknesses across different content.
 
 ## 📣 Evaluate Your Own Models
 For how to evaluate your customized model like OpenS2V-Eval in the [OpenS2V-Nexus paper](https://huggingface.co/papers/2505.20292), please refer to [here](https://github.com/PKU-YuanGroup/OpenS2V-Nexus/tree/main/eval).

app.py

@@ -41,7 +41,8 @@ def compute_scores(input_data):
         [
             input_data["total_score"],
             input_data["aes_score"],
-            input_data["motion_score"],
+            input_data["motion_amplitude"],
+            input_data["motion_smoothness"],
             input_data["facesim_cur"],
             input_data["gme_score"],
             input_data["nexus_score"],
@@ -56,7 +57,8 @@ def compute_scores_human_domain(input_data):
         [
             input_data["total_score"],
             input_data["aes_score"],
-            input_data["motion_score"],
+            input_data["motion_amplitude"],
+            input_data["motion_smoothness"],
             input_data["facesim_cur"],
             input_data["gme_score"],
             input_data["natural_score"],
@@ -125,6 +127,7 @@ def add_opendomain_eval(
             f"{input_data[4] * 100:.2f}%",
             f"{input_data[5] * 100:.2f}%",
             f"{input_data[6] * 100:.2f}%",
+            f"{input_data[7] * 100:.2f}%",
         ]
         csv_data.loc[col] = new_data
         csv_data.to_csv(CSV_DIR_OPEN_DOMAIN_RESULTS, index=False)
@@ -191,6 +194,7 @@ def add_humandomain_eval(
             f"{input_data[3] * 100:.2f}%",
             f"{input_data[4] * 100:.2f}%",
             f"{input_data[5] * 100:.2f}%",
+            f"{input_data[6] * 100:.2f}%",
         ]
         csv_data.loc[col] = new_data
         csv_data.to_csv(CSV_DIR_HUMAN_DOMAIN_RESULTS, index=False)
@@ -258,6 +262,7 @@ def add_singledomain_eval(
             f"{input_data[4] * 100:.2f}%",
             f"{input_data[5] * 100:.2f}%",
             f"{input_data[6] * 100:.2f}%",
+            f"{input_data[7] * 100:.2f}%",
         ]
         csv_data.loc[col] = new_data
         csv_data.to_csv(CSV_DIR_SINGLE_DOMAIN_RESULTS, index=False)

constants.py

@@ -3,7 +3,8 @@ MODEL_INFO = ["Model", "Venue", "Evaluated by"]
 ALL_RESULTS = [
     "TotalScore↑",
     "Aesthetics↑",
-    "Motion↑",
+    "MotionSmoothness↑",
+    "MotionAmplitude↑",
     "FaceSim↑",
     "GmeScore↑",
     "NexusScore↑",
@@ -13,7 +14,8 @@ ALL_RESULTS = [
 OPEN_DOMAIN_RESULTS = [
     "TotalScore↑",
     "Aesthetics↑",
-    "Motion↑",
+    "MotionSmoothness↑",
+    "MotionAmplitude↑",
     "FaceSim↑",
     "GmeScore↑",
     "NexusScore↑",
@@ -22,7 +24,8 @@ OPEN_DOMAIN_RESULTS = [
 HUMAN_DOMAIN_RESULTS = [
     "TotalScore↑",
     "Aesthetics↑",
-    "Motion↑",
+    "MotionSmoothness↑",
+    "MotionAmplitude↑",
     "FaceSim↑",
     "GmeScore↑",
     "NaturalScore↑",
@@ -30,7 +33,8 @@ HUMAN_DOMAIN_RESULTS = [
 SINGLE_DOMAIN_RESULTS = [
     "TotalScore↑",
     "Aesthetics↑",
-    "Motion↑",
+    "MotionSmoothness↑",
+    "MotionAmplitude↑",
     "FaceSim↑",
     "GmeScore↑",
     "NexusScore↑",
@@ -49,23 +53,23 @@ NEW_DATA_TITLE_TYPE = [
     "number",
 ]
 
-CSV_DIR_OPEN_DOMAIN_RESULTS = "./file/results_Open-Domain.csv"
-CSV_DIR_HUMAN_DOMAIN_RESULTS = "./file/results_Human-Domain.csv"
-CSV_DIR_SINGLE_DOMAIN_RESULTS = "./file/results_Single-Domain.csv"
+CSV_DIR_OPEN_DOMAIN_RESULTS = "./file_v1.1/results_Open-Domain.csv"
+CSV_DIR_HUMAN_DOMAIN_RESULTS = "./file_v1.1/results_Human-Domain.csv"
+CSV_DIR_SINGLE_DOMAIN_RESULTS = "./file_v1.1/results_Single-Domain.csv"
 
 COLUMN_NAMES = MODEL_INFO + ALL_RESULTS
 COLUMN_NAMES_HUMAN = MODEL_INFO + HUMAN_DOMAIN_RESULTS
 
 LEADERBORAD_INTRODUCTION = """
-    # OpenS2V-Eval Leaderboard
+    # OpenS2V-Eval-1.1 Leaderboard
     
-    Welcome to the leaderboard of the OpenS2V-Eval!
+    Welcome to the leaderboard of the OpenS2V-Eval-1.1! The v1.1 version adds motion smoothness on top of v1.0 to provide a more accurate measurement of motion quality.
      
-    🏆 OpenS2V-Eval is a core component of **OpenS2V-Nexus**, designed to establish a foundational infrastructure for *Subject-to-Video* (S2V) generation. It presents 180 prompts spanning seven major categories of S2V, incorporating both real and synthetic test data. To better align evaluation with human preferences, it introduce three new automatic metrics—NexusScore, NaturalScore, and GmeScore—that independently assess subject consistency, naturalness, and textual relevance in generated videos.
-    
+    🏆 OpenS2V-Eval is a core component of **OpenS2V-Nexus**, designed to establish a foundational infrastructure for *Subject-to-Video* (S2V) generation. It presents 180 prompts spanning seven major categories of S2V, incorporating both real and synthetic test data. To better align evaluation with human preferences, it introduce three new automatic metrics—NexusScore, NaturalScore, and GmeScore—that independently assess subject consistency, naturalness, and textual relevance in generated videos. 
+
     If you like our project, please give us a star ⭐ on GitHub for the latest update.
 
-    [GitHub](https://github.com/PKU-YuanGroup/OpenS2V-Nexus) | [Arxiv](https://arxiv.org/) | [Home Page](https://pku-yuangroup.github.io/OpenS2V-Nexus/) | [OpenS2V-Eval](https://huggingface.co/datasets/BestWishYsh/OpenS2V-Eval) | [OpenS2V-5M](https://huggingface.co/datasets/BestWishYsh/OpenS2V-5M)
+    [GitHub](https://github.com/PKU-YuanGroup/OpenS2V-Nexus) | [Arxiv](https://arxiv.org/) | [Home Page](https://pku-yuangroup.github.io/OpenS2V-Nexus/) | [OpenS2V-Eval](https://huggingface.co/datasets/BestWishYsh/OpenS2V-Eval) | [OpenS2V-5M](https://huggingface.co/datasets/BestWishYsh/OpenS2V-5M) | [OpenS2V-Eval-v1.0-LeaderBoard](https://huggingface.co/spaces/BestWishYsh/OpenS2V-Eval/tree/main/file_v1.0)
 """
 
 SUBMIT_INTRODUCTION = """# Submission Guidelines
@@ -78,7 +82,7 @@ SUBMIT_INTRODUCTION = """# Submission Guidelines
 
 TABLE_INTRODUCTION = """In the table below, we use six dimensions as the primary evaluation metrics for each task.
         1. Visual Quality: Aesthetics.
-        2. Motion Amplitude: Motion.
+        2. Motion Quality: Motion Smoothness and Motion Amplitude.
         3. Text Relevance: GmeScore.
         4. Subject Consistency: FaceSim and NexusScore.
         5. Subject Naturalness: NaturalScore.
@@ -86,7 +90,7 @@ TABLE_INTRODUCTION = """In the table below, we use six dimensions as the primary
 
 TABLE_INTRODUCTION_HUMAN = """In the table below, we use six dimensions as the primary evaluation metrics for each task.
         1. Visual Quality: Aesthetics.
-        2. Motion Amplitude: Motion.
+        2. Motion Quality: Motion Smoothness and Motion Amplitude.
         3. Text Relevance: GmeScore.
         4. Subject Consistency: FaceSim.
         5. Naturalness: NaturalScore.

file_v1.0/app.py

@@ -0,0 +1,588 @@
+import gradio as gr
+import pandas as pd
+import json
+import io
+
+from constants import (
+    SUBMIT_INTRODUCTION,
+    COLUMN_NAMES,
+    MODEL_INFO,
+    ALL_RESULTS,
+    NEW_DATA_TITLE_TYPE,
+    SINGLE_DOMAIN_RESULTS,
+    TABLE_INTRODUCTION,
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    COLUMN_NAMES_HUMAN,
+    CSV_DIR_HUMAN_DOMAIN_RESULTS,
+    CSV_DIR_OPEN_DOMAIN_RESULTS,
+    HUMAN_DOMAIN_RESULTS,
+    CSV_DIR_SINGLE_DOMAIN_RESULTS,
+    TABLE_INTRODUCTION_HUMAN,
+    LEADERBORAD_INTRODUCTION,
+    OPEN_DOMAIN_RESULTS,
+)
+
+global \
+    filter_component, \
+    data_component_opendomain, \
+    data_component_humandomain, \
+    data_component_singledomain
+
+
+def upload_file(files):
+    file_paths = [file.name for file in files]
+    return file_paths
+
+
+def compute_scores(input_data):
+    return [
+        None,
+        [
+            input_data["total_score"],
+            input_data["aes_score"],
+            input_data["motion_score"],
+            input_data["facesim_cur"],
+            input_data["gme_score"],
+            input_data["nexus_score"],
+            input_data["natural_score"],
+        ],
+    ]
+
+
+def compute_scores_human_domain(input_data):
+    return [
+        None,
+        [
+            input_data["total_score"],
+            input_data["aes_score"],
+            input_data["motion_score"],
+            input_data["facesim_cur"],
+            input_data["gme_score"],
+            input_data["natural_score"],
+        ],
+    ]
+
+
+def add_opendomain_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    venue_type_dropdown: str,
+    team_name_textbox: str,
+    model_link: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        selected_model_data = json.load(io.BytesIO(input_file))
+
+        scores = compute_scores(selected_model_data)
+        input_data = scores[1]
+        input_data = [float(i) for i in input_data]
+
+        csv_data = pd.read_csv(CSV_DIR_OPEN_DOMAIN_RESULTS)
+
+        if revision_name_textbox == "":
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in csv_data["Model"]
+            ]
+            assert model_name not in name_list
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data["Model"]
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in model_name_list
+            ]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)
+
+        if model_link == "":
+            model_name = model_name  # no url
+        else:
+            model_name = "[" + model_name + "](" + model_link + ")"
+
+        venue = venue_type_dropdown
+        if team_name_textbox == "":
+            team = "User Upload"
+        else:
+            team = team_name_textbox
+
+        new_data = [
+            model_name,
+            venue,
+            team,
+            f"{input_data[0] * 100:.2f}%",
+            f"{input_data[1] * 100:.2f}%",
+            f"{input_data[2] * 100:.2f}%",
+            f"{input_data[3] * 100:.2f}%",
+            f"{input_data[4] * 100:.2f}%",
+            f"{input_data[5] * 100:.2f}%",
+            f"{input_data[6] * 100:.2f}%",
+        ]
+        csv_data.loc[col] = new_data
+        csv_data.to_csv(CSV_DIR_OPEN_DOMAIN_RESULTS, index=False)
+    return "Evaluation successfully submitted!"
+
+
+def add_humandomain_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    venue_type_dropdown: str,
+    team_name_textbox: str,
+    model_link: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        selected_model_data = json.load(io.BytesIO(input_file))
+
+        scores = compute_scores_human_domain(selected_model_data)
+        input_data = scores[1]
+        input_data = [float(i) for i in input_data]
+
+        csv_data = pd.read_csv(CSV_DIR_HUMAN_DOMAIN_RESULTS)
+
+        if revision_name_textbox == "":
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in csv_data["Model"]
+            ]
+            assert model_name not in name_list
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data["Model"]
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in model_name_list
+            ]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)
+
+        if model_link == "":
+            model_name = model_name  # no url
+        else:
+            model_name = "[" + model_name + "](" + model_link + ")"
+
+        venue = venue_type_dropdown
+        if team_name_textbox == "":
+            team = "User Upload"
+        else:
+            team = team_name_textbox
+
+        new_data = [
+            model_name,
+            venue,
+            team,
+            f"{input_data[0] * 100:.2f}%",
+            f"{input_data[1] * 100:.2f}%",
+            f"{input_data[2] * 100:.2f}%",
+            f"{input_data[3] * 100:.2f}%",
+            f"{input_data[4] * 100:.2f}%",
+            f"{input_data[5] * 100:.2f}%",
+        ]
+        csv_data.loc[col] = new_data
+        csv_data.to_csv(CSV_DIR_HUMAN_DOMAIN_RESULTS, index=False)
+    return "Evaluation successfully submitted!"
+
+
+def add_singledomain_eval(
+    input_file,
+    model_name_textbox: str,
+    revision_name_textbox: str,
+    venue_type_dropdown: str,
+    team_name_textbox: str,
+    model_link: str,
+):
+    if input_file is None:
+        return "Error! Empty file!"
+    else:
+        selected_model_data = json.load(io.BytesIO(input_file))
+
+        scores = compute_scores(selected_model_data)
+        input_data = scores[1]
+        input_data = [float(i) for i in input_data]
+
+        csv_data = pd.read_csv(CSV_DIR_SINGLE_DOMAIN_RESULTS)
+
+        if revision_name_textbox == "":
+            col = csv_data.shape[0]
+            model_name = model_name_textbox
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in csv_data["Model"]
+            ]
+            assert model_name not in name_list
+        else:
+            model_name = revision_name_textbox
+            model_name_list = csv_data["Model"]
+            name_list = [
+                name.split("]")[0][1:] if name.endswith(")") else name
+                for name in model_name_list
+            ]
+            if revision_name_textbox not in name_list:
+                col = csv_data.shape[0]
+            else:
+                col = name_list.index(revision_name_textbox)
+
+        if model_link == "":
+            model_name = model_name  # no url
+        else:
+            model_name = "[" + model_name + "](" + model_link + ")"
+
+        venue = venue_type_dropdown
+        if team_name_textbox == "":
+            team = "User Upload"
+        else:
+            team = team_name_textbox
+
+        new_data = [
+            model_name,
+            venue,
+            team,
+            f"{input_data[0] * 100:.2f}%",
+            f"{input_data[1] * 100:.2f}%",
+            f"{input_data[2] * 100:.2f}%",
+            f"{input_data[3] * 100:.2f}%",
+            f"{input_data[4] * 100:.2f}%",
+            f"{input_data[5] * 100:.2f}%",
+            f"{input_data[6] * 100:.2f}%",
+        ]
+        csv_data.loc[col] = new_data
+        csv_data.to_csv(CSV_DIR_SINGLE_DOMAIN_RESULTS, index=False)
+    return "Evaluation successfully submitted!"
+
+
+def get_all_df_opendomain():
+    df = pd.read_csv(CSV_DIR_OPEN_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    return df
+
+
+def get_baseline_df_opendomain():
+    df = pd.read_csv(CSV_DIR_OPEN_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group_opendomain.value
+    df = df[present_columns]
+    return df
+
+
+def get_all_df_humandomain():
+    df = pd.read_csv(CSV_DIR_HUMAN_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    return df
+
+
+def get_baseline_df_humandomain():
+    df = pd.read_csv(CSV_DIR_HUMAN_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group_humandomain.value
+    df = df[present_columns]
+    return df
+
+
+def get_all_df_singledomain():
+    df = pd.read_csv(CSV_DIR_SINGLE_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    return df
+
+
+def get_baseline_df_singledomain():
+    df = pd.read_csv(CSV_DIR_SINGLE_DOMAIN_RESULTS)
+    df = df.sort_values(by="TotalScore↑", ascending=False)
+    present_columns = MODEL_INFO + checkbox_group_singledomain.value
+    df = df[present_columns]
+    return df
+
+
+block = gr.Blocks()
+
+
+with block:
+    gr.HTML("""
+        <div style='display: flex; align-items: center; justify-content: center; text-align: center;'>
+            <img src="https://www.pnglog.com/6xm07l.png" style='width: 400px; height: auto; margin-right: 10px;' />
+        </div>
+    """)
+    gr.Markdown(LEADERBORAD_INTRODUCTION)
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # table Opendomain
+        with gr.TabItem("🏅 Open-Domain", elem_id="OpenS2V-Nexus-tab-table", id=0):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        show_copy_button=True,
+                    )
+
+            gr.Markdown(TABLE_INTRODUCTION)
+
+            checkbox_group_opendomain = gr.CheckboxGroup(
+                choices=ALL_RESULTS,
+                value=OPEN_DOMAIN_RESULTS,
+                label="Select options",
+                interactive=True,
+            )
+
+            data_component_opendomain = gr.components.Dataframe(
+                value=get_baseline_df_opendomain,
+                headers=COLUMN_NAMES,
+                type="pandas",
+                datatype=NEW_DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
+            )
+
+            def on_checkbox_group_change_opendomain(selected_columns):
+                selected_columns = [
+                    item for item in ALL_RESULTS if item in selected_columns
+                ]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_baseline_df_opendomain()[present_columns]
+                updated_data = updated_data.sort_values(
+                    by=present_columns[1], ascending=False
+                )
+                updated_headers = present_columns
+                update_datatype = [
+                    NEW_DATA_TITLE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers
+                ]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data,
+                    headers=updated_headers,
+                    type="pandas",
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                )
+
+                return filter_component
+
+            checkbox_group_opendomain.change(
+                fn=on_checkbox_group_change_opendomain,
+                inputs=checkbox_group_opendomain,
+                outputs=data_component_opendomain,
+            )
+
+        # table HumanDomain
+        with gr.TabItem("🏅 Human-Domain", elem_id="OpenS2V-Nexus-tab-table", id=1):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        show_copy_button=True,
+                    )
+
+            gr.Markdown(TABLE_INTRODUCTION_HUMAN)
+
+            checkbox_group_humandomain = gr.CheckboxGroup(
+                choices=HUMAN_DOMAIN_RESULTS,
+                value=HUMAN_DOMAIN_RESULTS,
+                label="Select options",
+                interactive=True,
+            )
+
+            data_component_humandomain = gr.components.Dataframe(
+                value=get_baseline_df_humandomain,
+                headers=COLUMN_NAMES_HUMAN,
+                type="pandas",
+                datatype=NEW_DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
+            )
+
+            def on_checkbox_group_change_humandomain(selected_columns):
+                selected_columns = [
+                    item for item in ALL_RESULTS if item in selected_columns
+                ]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_baseline_df_humandomain()[present_columns]
+                updated_data = updated_data.sort_values(
+                    by=present_columns[1], ascending=False
+                )
+                updated_headers = present_columns
+                update_datatype = [
+                    NEW_DATA_TITLE_TYPE[COLUMN_NAMES_HUMAN.index(x)]
+                    for x in updated_headers
+                ]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data,
+                    headers=updated_headers,
+                    type="pandas",
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                )
+
+                return filter_component
+
+            checkbox_group_humandomain.change(
+                fn=on_checkbox_group_change_humandomain,
+                inputs=checkbox_group_humandomain,
+                outputs=data_component_humandomain,
+            )
+
+        # table SingleDomain
+        with gr.TabItem("🏅 Single-Domain", elem_id="OpenS2V-Nexus-tab-table", id=2):
+            with gr.Row():
+                with gr.Accordion("Citation", open=False):
+                    citation_button = gr.Textbox(
+                        value=CITATION_BUTTON_TEXT,
+                        label=CITATION_BUTTON_LABEL,
+                        elem_id="citation-button",
+                        show_copy_button=True,
+                    )
+
+            gr.Markdown(TABLE_INTRODUCTION)
+
+            checkbox_group_singledomain = gr.CheckboxGroup(
+                choices=ALL_RESULTS,
+                value=SINGLE_DOMAIN_RESULTS,
+                label="Select options",
+                interactive=True,
+            )
+
+            data_component_singledomain = gr.components.Dataframe(
+                value=get_baseline_df_singledomain,
+                headers=COLUMN_NAMES,
+                type="pandas",
+                datatype=NEW_DATA_TITLE_TYPE,
+                interactive=False,
+                visible=True,
+            )
+
+            def on_checkbox_group_change_singledomain(selected_columns):
+                selected_columns = [
+                    item for item in ALL_RESULTS if item in selected_columns
+                ]
+                present_columns = MODEL_INFO + selected_columns
+                updated_data = get_baseline_df_singledomain()[present_columns]
+                updated_data = updated_data.sort_values(
+                    by=present_columns[1], ascending=False
+                )
+                updated_headers = present_columns
+                update_datatype = [
+                    NEW_DATA_TITLE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers
+                ]
+
+                filter_component = gr.components.Dataframe(
+                    value=updated_data,
+                    headers=updated_headers,
+                    type="pandas",
+                    datatype=update_datatype,
+                    interactive=False,
+                    visible=True,
+                )
+
+                return filter_component
+
+            checkbox_group_singledomain.change(
+                fn=on_checkbox_group_change_singledomain,
+                inputs=checkbox_group_singledomain,
+                outputs=data_component_singledomain,
+            )
+
+        # table Submission
+        with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=4):
+            with gr.Row():
+                gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
+
+            with gr.Row():
+                gr.Markdown(
+                    "# ✉️✨ Submit your result here!", elem_classes="markdown-text"
+                )
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(
+                        label="Model name", placeholder="ConsisID"
+                    )
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision Model Name (Optinal)", placeholder="ConsisID"
+                    )
+                    venue_type_dropdown = gr.Dropdown(
+                        label="Venue Type",
+                        choices=["Open-Source", "Close-Source"],
+                        value="Open-Source",
+                    )
+                    team_name_textbox = gr.Textbox(
+                        label="Your Team Name (If left blank, it will be user upload))", placeholder="User Upload"
+                    )
+                    model_link = gr.Textbox(
+                        label="Model Link",
+                        placeholder="https://github.com/PKU-YuanGroup/ConsisID",
+                    )
+
+            with gr.Column():
+                input_file = gr.File(label="Click to Upload a json File", type="binary")
+
+                submit_button_opendomain = gr.Button("Submit Result (Open-Domain)")
+                submit_button_humandomain = gr.Button("Submit Result (Human-Domain)")
+                submit_button_singledomain = gr.Button("Submit Result (Single-Domain)")
+
+                submission_result = gr.Markdown()
+
+                submit_button_opendomain.click(
+                    add_opendomain_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        venue_type_dropdown,
+                        team_name_textbox,
+                        model_link,
+                    ],
+                    outputs=submission_result,
+                )
+
+                submit_button_humandomain.click(
+                    add_humandomain_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        venue_type_dropdown,
+                        team_name_textbox,
+                        model_link,
+                    ],
+                    outputs=submission_result,
+                )
+
+                submit_button_singledomain.click(
+                    add_singledomain_eval,
+                    inputs=[
+                        input_file,
+                        model_name_textbox,
+                        revision_name_textbox,
+                        venue_type_dropdown,
+                        team_name_textbox,
+                        model_link,
+                    ],
+                    outputs=submission_result,
+                )
+
+    with gr.Row():
+        data_run = gr.Button("Refresh")
+        data_run.click(get_baseline_df_opendomain, outputs=data_component_opendomain)
+        data_run.click(get_baseline_df_humandomain, outputs=data_component_humandomain)
+        data_run.click(
+            get_baseline_df_singledomain, outputs=data_component_singledomain
+        )
+
+block.launch()

file_v1.0/constants.py

@@ -0,0 +1,101 @@
+MODEL_INFO = ["Model", "Venue", "Evaluated by"]
+
+ALL_RESULTS = [
+    "TotalScore↑",
+    "Aesthetics↑",
+    "Motion↑",
+    "FaceSim↑",
+    "GmeScore↑",
+    "NexusScore↑",
+    "NaturalScore↑",
+]
+
+OPEN_DOMAIN_RESULTS = [
+    "TotalScore↑",
+    "Aesthetics↑",
+    "Motion↑",
+    "FaceSim↑",
+    "GmeScore↑",
+    "NexusScore↑",
+    "NaturalScore↑",
+]
+HUMAN_DOMAIN_RESULTS = [
+    "TotalScore↑",
+    "Aesthetics↑",
+    "Motion↑",
+    "FaceSim↑",
+    "GmeScore↑",
+    "NaturalScore↑",
+]
+SINGLE_DOMAIN_RESULTS = [
+    "TotalScore↑",
+    "Aesthetics↑",
+    "Motion↑",
+    "FaceSim↑",
+    "GmeScore↑",
+    "NexusScore↑",
+    "NaturalScore↑",
+]
+
+NEW_DATA_TITLE_TYPE = [
+    "markdown",
+    "markdown",
+    "number",
+    "number",
+    "number",
+    "number",
+    "number",
+    "number",
+    "number",
+]
+
+CSV_DIR_OPEN_DOMAIN_RESULTS = "./results_Open-Domain.csv"
+CSV_DIR_HUMAN_DOMAIN_RESULTS = "./results_Human-Domain.csv"
+CSV_DIR_SINGLE_DOMAIN_RESULTS = "./results_Single-Domain.csv"
+
+COLUMN_NAMES = MODEL_INFO + ALL_RESULTS
+COLUMN_NAMES_HUMAN = MODEL_INFO + HUMAN_DOMAIN_RESULTS
+
+LEADERBORAD_INTRODUCTION = """
+    # OpenS2V-Eval Leaderboard
+    
+    Welcome to the leaderboard of the OpenS2V-Eval!
+     
+    🏆 OpenS2V-Eval is a core component of **OpenS2V-Nexus**, designed to establish a foundational infrastructure for *Subject-to-Video* (S2V) generation. It presents 180 prompts spanning seven major categories of S2V, incorporating both real and synthetic test data. To better align evaluation with human preferences, it introduce three new automatic metrics—NexusScore, NaturalScore, and GmeScore—that independently assess subject consistency, naturalness, and textual relevance in generated videos.
+    
+    If you like our project, please give us a star ⭐ on GitHub for the latest update.
+
+    [GitHub](https://github.com/PKU-YuanGroup/OpenS2V-Nexus) | [Arxiv](https://arxiv.org/) | [Home Page](https://pku-yuangroup.github.io/OpenS2V-Nexus/) | [OpenS2V-Eval](https://huggingface.co/datasets/BestWishYsh/OpenS2V-Eval) | [OpenS2V-5M](https://huggingface.co/datasets/BestWishYsh/OpenS2V-5M)
+"""
+
+SUBMIT_INTRODUCTION = """# Submission Guidelines
+    1. Fill in *'Model Name'* if it is your first time to submit your result **or** Fill in *'Revision Model Name'* if you want to update your result.
+    2. Fill in your home page to *'Model Link'* and your team name to *'Your Team Name'*.
+    3. After evaluation, follow the guidance in the [github repository](https://github.com/PKU-YuanGroup/OpenS2V-Nexus) to obtain `model_name.json` and upload it here.
+    4. Click the *'Submit Eval'* button.
+    5. Click *'Refresh'* to obtain the updated leaderboard.
+"""
+
+TABLE_INTRODUCTION = """In the table below, we use six dimensions as the primary evaluation metrics for each task.
+        1. Visual Quality: Aesthetics.
+        2. Motion Amplitude: Motion.
+        3. Text Relevance: GmeScore.
+        4. Subject Consistency: FaceSim and NexusScore.
+        5. Subject Naturalness: NaturalScore.
+    """
+
+TABLE_INTRODUCTION_HUMAN = """In the table below, we use six dimensions as the primary evaluation metrics for each task.
+        1. Visual Quality: Aesthetics.
+        2. Motion Amplitude: Motion.
+        3. Text Relevance: GmeScore.
+        4. Subject Consistency: FaceSim.
+        5. Naturalness: NaturalScore.
+    """
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{yuan2025opens2v,
+  title={OpenS2V-Nexus: A Detailed Benchmark and Million-Scale Dataset for Subject-to-Video Generation},
+  author={Yuan, Shenghai and He, Xianyi and Deng, Yufan and Ye, Yang and Huang, Jinfa and Lin, Bin and Luo, Jiebo and Yuan, Li},
+  journal={arXiv preprint arXiv:2505.20292},
+  year={2025}
+}"""

{file → file_v1.0}/results_Human-Domain.csv

@@ -6,7 +6,7 @@ Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,Motion↑,FaceSim↑,GmeSco
 [VACE-1.3B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,49.02%,53.18%,16.87%,22.29%,73.61%,73.00%
 [VACE-14B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,58.57%,52.78%,11.76%,64.65%,69.53%,74.33%
 [Phantom-1.3B](https://github.com/Phantom-video/Phantom),Open-Source,OpenS2V Team,53.64%,50.80%,14.14%,46.30%,72.17%,71.67%
-[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-V2),Open-Source,OpenS2V Team,54.27%,39.88%,31.98%,55.02%,63.63%,67.33%
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Open-Source,OpenS2V Team,54.27%,39.88%,31.98%,55.02%,63.63%,67.33%
 [HunyuanCustom](https://github.com/Tencent-Hunyuan/HunyuanCustom),Open-Source,OpenS2V Team,55.85%,49.67%,15.13%,62.25%,69.78%,67.00%
 [Hailuo(20250503)](https://hailuoai.video/),Closed-Source,OpenS2V Team,60.20%,52.75%,31.83%,57.79%,71.42%,74.52%
 [ConsisID](https://github.com/PKU-YuanGroup/ConsisID),Open-Source,OpenS2V Team,52.97%,41.76%,38.12%,43.14%,72.03%,64.67%

{file → file_v1.0}/results_Open-Domain.csv

@@ -6,6 +6,6 @@ Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,Motion↑,FaceSim↑,GmeSco
 [VACE-1.3B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,45.53%,48.24%,18.83%,20.58%,71.26%,37.95%,71.78%
 [VACE-14B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,52.87%,47.21%,15.02%,55.09%,67.27%,44.20%,72.78%
 [Phantom-1.3B](https://github.com/Phantom-video/Phantom),Open-Source,OpenS2V Team,50.71%,46.67%,14.29%,48.55%,69.43%,42.44%,70.26%
-[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-V2),Open-Source,OpenS2V Team,49.61%,39.40%,25.60%,45.95%,64.54%,43.77%,67.22%
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Open-Source,OpenS2V Team,49.61%,39.40%,25.60%,45.95%,64.54%,43.77%,67.22%
 [Phantom-14B](https://github.com/Phantom-video/Phantom),Open-Source,OpenS2V Team,52.32%,46.39%,33.42%,51.48%,70.65%,37.43%,68.66%
 [MAGREF-480P](https://github.com/MAGREF-Video/MAGREF),Open-Source,OpenS2V Team,47.93%,45.02%,21.81%,30.83%,70.47%,43.04%,69.49%

{file → file_v1.0}/results_Single-Domain.csv

@@ -6,7 +6,7 @@ Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,Motion↑,FaceSim↑,GmeSco
 [VACE-1.3B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,47.33%,41.81%,33.78%,22.38%,65.35%,38.52%,76.00%
 [VACE-14B](https://github.com/ali-vilab/VACE),Open-Source,OpenS2V Team,58.00%,41.30%,35.54%,64.65%,58.55%,51.33%,77.33%
 [Phantom-1.3B](https://github.com/Phantom-video/Phantom),Open-Source,OpenS2V Team,49.95%,42.98%,19.30%,44.03%,65.61%,37.78%,76.00%
-[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-V2),Open-Source,OpenS2V Team,51.64%,33.83%,21.60%,54.42%,61.93%,48.63%,70.60%
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Open-Source,OpenS2V Team,51.64%,33.83%,21.60%,54.42%,61.93%,48.63%,70.60%
 [HunyuanCustom](https://github.com/Tencent-Hunyuan/HunyuanCustom),Open-Source,OpenS2V Team,51.64%,34.08%,26.83%,55.93%,54.31%,50.75%,68.66%
 [Phantom-14B](https://github.com/Phantom-video/Phantom),Open-Source,OpenS2V Team,53.17%,47.46%,41.55%,51.82%,70.07%,35.35%,69.35%
 [MAGREF-480P](https://github.com/MAGREF-Video/MAGREF),Open-Source,OpenS2V Team,49.13%,46.31%,27.43%,33.77%,69.02%,42.45%,69.81%

file_v1.1/model_name_Open-Domain.json

@@ -0,0 +1,10 @@
+{
+    "total_score": 0.562269546381486,
+    "aes_score": 0.44594517904800374,
+    "motion_smoothness": 0.8693211435903602,
+    "motion_amplitude": 0.4160163568750958,
+    "facesim_cur": 0.4010164068564032,
+    "gme_score": 0.6620000359470859,
+    "nexus_score": 0.4588851569398767,
+    "natural_score": 0.745910020449898
+}

file_v1.1/results_Human-Domain.csv

@@ -0,0 +1,20 @@
+Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,MotionSmoothness↑,MotionAmplitude↑,FaceSim↑,GmeScore↑,NaturalScore↑
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Closed-Source,OpenS2V Team,56.43%,39.89%,80.19%,31.49%,55.01%,63.63%,59.31%
+[Vidu2.0(20250503)](https://www.vidu.cn/),Closed-Source,OpenS2V Team,57.7%,47.33%,91.31%,14.54%,38.5%,70.43%,67.78%
+[EchoVideo](https://github.com/bytedance/EchoVideo),Closed-Source,OpenS2V Team,56.36%,39.93%,77.96%,35.58%,48.65%,68.4%,62.22%
+[Hailuo(20250503)](https://hailuoai.video/),Closed-Source,OpenS2V Team,65.26%,52.75%,99.1%,31.8%,57.69%,71.42%,69.2%
+[MAGREF-480P](https://github.com/MAGREF-Video/MAGREF),Closed-Source,OpenS2V Team,57.72%,51.2%,90.26%,14.76%,32.87%,70.88%,70.28%
+[Concat-ID-CogVideoX](https://github.com/ML-GSAI/Concat-ID),Closed-Source,OpenS2V Team,55.89%,44.13%,81.9%,31.07%,43.87%,73.67%,58.75%
+[HunyuanCustom](https://github.com/Tencent-Hunyuan/HunyuanCustom),Closed-Source,OpenS2V Team,61.22%,49.67%,84.73%,15.13%,62.25%,69.78%,60.56%
+[Concat-ID-Wan-AdaLN](https://github.com/ML-GSAI/Concat-ID),Closed-Source,OpenS2V Team,59.85%,43.13%,85.86%,17.19%,50.05%,71.9%,68.47%
+[FantasyID](https://github.com/Fantasy-AMAP/fantasy-id),Closed-Source,OpenS2V Team,54.33%,45.6%,85.44%,23.41%,32.48%,72.68%,62.36%
+[ConsisID](https://github.com/PKU-YuanGroup/ConsisID),Closed-Source,OpenS2V Team,54.19%,41.77%,79.83%,37.99%,43.19%,72.03%,55.83%
+[Kling1.6(20250503)](https://app.klingai.com/cn/),Closed-Source,OpenS2V Team,60.19%,50.94%,84.75%,50.02%,41.02%,67.79%,71.55%
+[VideoMaker](https://github.com/WuTao-CS/VideoMaker),Closed-Source,OpenS2V Team,54.23%,31.76%,77.5%,50.09%,76.45%,45.28%,47.08%
+[VACE-P1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,53.97%,51.91%,95.8%,8.78%,19.98%,73.27%,65.83%
+[VACE-1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,54.9%,53.18%,95.84%,16.87%,22.29%,73.61%,65.28%
+[Pika2.1(20250503)](https://pika.art/),Closed-Source,OpenS2V Team,56.84%,52.39%,85.29%,28.77%,29.42%,75.03%,67.53%
+[ID-Animator](https://github.com/ID-Animator/ID-Animator),Closed-Source,OpenS2V Team,49.75%,42.03%,94.69%,33.54%,31.56%,52.91%,56.11%
+[Phantom-1.3B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,60.0%,50.8%,92.02%,14.09%,46.29%,72.17%,65.83%
+[VACE-14B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,65.78%,52.78%,94.96%,11.76%,64.65%,69.53%,69.31%
+[Phantom-14B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,64.22%,49.14%,94.81%,41.24%,55.04%,72.55%,69.86%

file_v1.1/results_Open-Domain.csv

@@ -0,0 +1,11 @@
+Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,MotionSmoothness↑,MotionAmplitude↑,FaceSim↑,GmeScore↑,NexusScore↑,NaturalScore↑
+[MAGREF-480P](https://github.com/MAGREF-Video/MAGREF),Closed-Source,OpenS2V Team,52.51%,45.02%,93.17%,21.81%,30.83%,70.47%,43.04%,66.9%
+[VACE-P1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,48.98%,47.34%,96.8%,12.03%,16.59%,71.38%,40.19%,64.31%
+[Phantom-14B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,56.77%,46.39%,96.31%,33.42%,51.46%,70.65%,37.43%,69.35%
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Closed-Source,OpenS2V Team,52.25%,39.41%,87.93%,25.6%,45.95%,64.54%,43.75%,60.32%
+[Pika2.1(20250503)](https://pika.art/),Closed-Source,OpenS2V Team,51.88%,46.88%,87.06%,24.71%,30.38%,69.19%,45.4%,63.32%
+[Kling1.6(20250503)](https://app.klingai.com/cn/),Closed-Source,OpenS2V Team,56.23%,44.59%,86.93%,41.6%,40.1%,66.2%,45.89%,74.59%
+[Vidu2.0(20250503)](https://www.vidu.cn/),Closed-Source,OpenS2V Team,51.95%,41.48%,90.45%,13.52%,35.11%,67.57%,43.37%,65.88%
+[Phantom-1.3B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,54.89%,46.67%,93.3%,14.29%,48.56%,69.43%,42.48%,62.5%
+[VACE-1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,49.89%,48.24%,97.2%,18.83%,20.57%,71.26%,37.91%,65.46%
+[VACE-14B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,57.55%,47.21%,94.97%,15.02%,55.09%,67.27%,44.08%,67.04%

file_v1.1/results_Single-Domain.csv

@@ -0,0 +1,12 @@
+Model,Venue,Evaluated by,TotalScore↑,Aesthetics↑,MotionSmoothness↑,MotionAmplitude↑,FaceSim↑,GmeScore↑,NexusScore↑,NaturalScore↑
+[VACE-P1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,49.2%,48.93%,95.68%,11.91%,18.04%,70.78%,36.24%,66.85%
+[VACE-1.3B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,51.13%,49.41%,95.42%,22.51%,22.37%,70.87%,38.34%,68.33%
+[Kling1.6(20250503)](https://app.klingai.com/cn/),Closed-Source,OpenS2V Team,56.67%,45.97%,85.76%,47.17%,39.27%,65.36%,49.3%,73.63%
+[Pika2.1(20250503)](https://pika.art/),Closed-Source,OpenS2V Team,53.12%,47.43%,86.07%,26.32%,32.33%,69.84%,47.35%,64.68%
+[SkyReels-A2-P14B](https://github.com/SkyworkAI/SkyReels-A2),Closed-Source,OpenS2V Team,55.06%,40.85%,85.54%,26.41%,54.42%,61.81%,48.6%,61.85%
+[MAGREF-480P](https://github.com/MAGREF-Video/MAGREF),Closed-Source,OpenS2V Team,53.44%,46.31%,92.63%,27.43%,33.77%,69.02%,42.45%,68.33%
+[Phantom-14B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,57.02%,47.46%,94.86%,41.55%,51.82%,70.07%,35.3%,71.11%
+[Phantom-1.3B](https://github.com/Phantom-video/Phantom),Closed-Source,OpenS2V Team,54.5%,49.0%,93.7%,16.38%,44.03%,69.54%,37.72%,66.76%
+[VACE-14B](https://github.com/ali-vilab/VACE),Closed-Source,OpenS2V Team,61.75%,48.94%,93.16%,19.69%,64.65%,65.86%,50.82%,70.56%
+[Vidu2.0(20250503)](https://www.vidu.cn/),Closed-Source,OpenS2V Team,52.9%,43.32%,91.88%,17.52%,36.19%,66.96%,44.84%,66.11%
+[HunyuanCustom](https://github.com/Tencent-Hunyuan/HunyuanCustom),Closed-Source,OpenS2V Team,56.89%,44.84%,86.49%,17.94%,55.93%,62.71%,56.49%,58.98%