[Add, Ref] pairwise sim, data loading, simple number example demo

app.py

@@ -1,103 +1,70 @@
 import gradio as gr
-import plotly.graph_objects as go
-import numpy as np
 from src.dataloading import get_leaderboard_models_cached, get_leaderboard_datasets
+from src.similarity import compute_similarity
 
-# Optionally, force a renderer (may or may not help)
-import plotly.io as pio
-pio.renderers.default = "iframe"
+def validate_inputs(selected_model_a, selected_model_b, selected_dataset):
+    if not selected_model_a:
+        raise gr.Error("Please select Model A!")
+    if not selected_model_b:
+        raise gr.Error("Please select Model B!")
+    if not selected_dataset:
+        raise gr.Error("Please select a dataset!")
 
-def create_heatmap(selected_models, selected_dataset):
-    if not selected_models or not selected_dataset:
-        return ""  # Return empty HTML if no input
-    size = len(selected_models)
-    similarities = np.random.rand(size, size)
-    similarities = (similarities + similarities.T) / 2
-    similarities = np.round(similarities, 2)
-    
-    fig = go.Figure(data=go.Heatmap(
-        z=similarities,
-        x=selected_models,
-        y=selected_models,
-        colorscale="Viridis",
-        zmin=0, zmax=1,
-        text=similarities,
-        hoverinfo="text"
-    ))
+def display_similarity(model_a, model_b, dataset):
+    # Assuming compute_similarity returns a float or a string
+    similarity_score = compute_similarity(model_a, model_b, dataset)
+    return f"The similarity between {model_a} and {model_b} on {dataset} is: {similarity_score}"
+
+with gr.Blocks(title="LLM Similarity Analyzer") as demo:
+    gr.Markdown("## Model Similarity Comparison Tool")
     
-    fig.update_layout(
-        title=f"Similarity Matrix for {selected_dataset}",
-        xaxis_title="Models",
-        yaxis_title="Models",
-        width=800,
-        height=800,
-        margin=dict(l=100, r=100, t=100, b=100)
+    dataset_dropdown = gr.Dropdown(
+        choices=get_leaderboard_datasets(),
+        label="Select Dataset",
+        filterable=True,
+        interactive=True,
+        info="Leaderboard benchmark datasets"
     )
     
-    # Force categorical ordering with explicit tick settings.
-    fig.update_xaxes(
-        type="category",
-        categoryorder="array",
-        categoryarray=selected_models,
-        tickangle=45,
-        automargin=True
+    model_a_dropdown = gr.Dropdown(
+        choices=get_leaderboard_models_cached(),
+        label="Select Model A",
+        filterable=True,
+        allow_custom_value=False,
+        info="Search and select models"
     )
-    fig.update_yaxes(
-        type="category",
-        categoryorder="array",
-        categoryarray=selected_models,
-        automargin=True
-    )
-    
-    # Convert the figure to an HTML string that includes Plotly.js via CDN.
-    return fig.to_html(full_html=False, include_plotlyjs="cdn")
 
-def validate_inputs(selected_models, selected_dataset):
-    if not selected_models:
-        raise gr.Error("Please select at least one model!")
-    if not selected_dataset:
-        raise gr.Error("Please select a dataset!")
+    model_b_dropdown = gr.Dropdown(
+        choices=get_leaderboard_models_cached(),
+        label="Select Model B",
+        filterable=True,
+        allow_custom_value=False,
+        info="Search and select models"
+    )
 
-with gr.Blocks(title="LLM Similarity Analyzer") as demo:
-    gr.Markdown("## Model Similarity Comparison Tool")
-    
-    with gr.Row():
-        dataset_dropdown = gr.Dropdown(
-            choices=get_leaderboard_datasets(),
-            label="Select Dataset",
-            filterable=True,
-            interactive=True,
-            info="Leaderboard benchmark datasets"
-        )
-        model_dropdown = gr.Dropdown(
-            choices=get_leaderboard_models_cached(),
-            label="Select Models",
-            multiselect=True,
-            filterable=True,
-            allow_custom_value=False,
-            info="Search and select multiple models"
-        )
+    generate_btn = gr.Button("Compute Similarity", variant="primary")
     
-    generate_btn = gr.Button("Generate Heatmap", variant="primary")
-    # Use an HTML component instead of gr.Plot.
-    heatmap = gr.HTML(label="Similarity Heatmap", visible=True)
+    # Textbox to display the similarity result
+    similarity_output = gr.Textbox(
+        label="Similarity Result",
+        interactive=False
+    )
     
     generate_btn.click(
         fn=validate_inputs,
-        inputs=[model_dropdown, dataset_dropdown],
+        inputs=[model_a_dropdown, model_b_dropdown, dataset_dropdown],
         queue=False
     ).then(
-        fn=create_heatmap,
-        inputs=[model_dropdown, dataset_dropdown],
-        outputs=heatmap
+        fn=display_similarity,
+        inputs=[model_a_dropdown, model_b_dropdown, dataset_dropdown],
+        outputs=similarity_output
     )
     
     clear_btn = gr.Button("Clear Selection")
     clear_btn.click(
-        lambda: [None, None, ""],
-        outputs=[model_dropdown, dataset_dropdown, heatmap]
+        lambda: [None, None, None, ""],
+        outputs=[model_a_dropdown, model_b_dropdown, dataset_dropdown, similarity_output]
     )
 
 if __name__ == "__main__":
-    # On Spaces, disable server-side rendering.
-    demo.launch(ssr_mode=False)
+    demo.launch()

app_heatmap.py

@@ -0,0 +1,103 @@
+import gradio as gr
+import plotly.graph_objects as go
+import numpy as np
+from src.dataloading import get_leaderboard_models_cached, get_leaderboard_datasets
+
+# Optionally, force a renderer (may or may not help)
+import plotly.io as pio
+pio.renderers.default = "iframe"
+
+def create_heatmap(selected_models, selected_dataset):
+    if not selected_models or not selected_dataset:
+        return ""  # Return empty HTML if no input
+    size = len(selected_models)
+    similarities = np.random.rand(size, size)
+    similarities = (similarities + similarities.T) / 2
+    similarities = np.round(similarities, 2)
+    
+    fig = go.Figure(data=go.Heatmap(
+        z=similarities,
+        x=selected_models,
+        y=selected_models,
+        colorscale="Viridis",
+        zmin=0, zmax=1,
+        text=similarities,
+        hoverinfo="text"
+    ))
+    
+    fig.update_layout(
+        title=f"Similarity Matrix for {selected_dataset}",
+        xaxis_title="Models",
+        yaxis_title="Models",
+        width=800,
+        height=800,
+        margin=dict(l=100, r=100, t=100, b=100)
+    )
+    
+    # Force categorical ordering with explicit tick settings.
+    fig.update_xaxes(
+        type="category",
+        categoryorder="array",
+        categoryarray=selected_models,
+        tickangle=45,
+        automargin=True
+    )
+    fig.update_yaxes(
+        type="category",
+        categoryorder="array",
+        categoryarray=selected_models,
+        automargin=True
+    )
+    
+    # Convert the figure to an HTML string that includes Plotly.js via CDN.
+    return fig.to_html(full_html=False, include_plotlyjs="cdn")
+
+def validate_inputs(selected_models, selected_dataset):
+    if not selected_models:
+        raise gr.Error("Please select at least one model!")
+    if not selected_dataset:
+        raise gr.Error("Please select a dataset!")
+
+with gr.Blocks(title="LLM Similarity Analyzer") as demo:
+    gr.Markdown("## Model Similarity Comparison Tool")
+    
+    with gr.Row():
+        dataset_dropdown = gr.Dropdown(
+            choices=get_leaderboard_datasets(),
+            label="Select Dataset",
+            filterable=True,
+            interactive=True,
+            info="Leaderboard benchmark datasets"
+        )
+        model_dropdown = gr.Dropdown(
+            choices=get_leaderboard_models_cached(),
+            label="Select Models",
+            multiselect=True,
+            filterable=True,
+            allow_custom_value=False,
+            info="Search and select multiple models"
+        )
+    
+    generate_btn = gr.Button("Generate Heatmap", variant="primary")
+    # Use an HTML component instead of gr.Plot.
+    heatmap = gr.HTML(label="Similarity Heatmap", visible=True)
+    
+    generate_btn.click(
+        fn=validate_inputs,
+        inputs=[model_dropdown, dataset_dropdown],
+        queue=False
+    ).then(
+        fn=create_heatmap,
+        inputs=[model_dropdown, dataset_dropdown],
+        outputs=heatmap
+    )
+    
+    clear_btn = gr.Button("Clear Selection")
+    clear_btn.click(
+        lambda: [None, None, ""],
+        outputs=[model_dropdown, dataset_dropdown, heatmap]
+    )
+
+if __name__ == "__main__":
+    # On Spaces, disable server-side rendering.
+    demo.launch(ssr_mode=False)

requirements.txt

@@ -5,4 +5,5 @@ matplotlib
 seaborn
 plotly
 pandas
-scienceplots
+scienceplots
+lm-sim

src/dataloading.py

@@ -1,9 +1,12 @@
+import datasets
+import numpy as np
+
 from huggingface_hub import HfApi
 from functools import lru_cache
 
 
 def get_leaderboard_models():
-    api = HfApi()
+    #api = HfApi()
     
     # List all datasets in the open-llm-leaderboard organization
     #datasets = api.list_datasets(author="open-llm-leaderboard")
@@ -38,8 +41,48 @@ def get_leaderboard_datasets():
     return [
         "ai2_arc",
         "hellaswag",
-        "mmlu",
+        "mmlu_pro",
         "truthful_qa",
         "winogrande",
         "gsm8k"
-    ]
+    ]
+
+def filter_labels(doc):
+    labels = []
+    if "answer_index" in doc[0].keys():
+        for d in doc:
+            labels.append(int(d["answer_index"]))
+    else:
+        for d in doc:
+            if d["answer"] == "False":
+                labels.append(0)
+            elif d["answer"] == "True":
+                labels.append(1)
+            else:
+                raise ValueError("Invalid label")
+
+def load_run_data(model_name, dataset_name):
+    try:
+        model_name = model_name.replace("/", "__")
+
+        data = datasets.load_dataset("open-llm-leaderboard/" + model_name + "-details",
+                                    name=model_name + "__leaderboard_" + dataset_name,
+                                    split="latest")
+        data = data.sort("doc_id")
+        data = data.to_dict()
+
+        # Get log probabilities for each response
+        log_probs = []
+        for resp in data["filtered_resps"]:
+            log_prob = np.array([float(option[0]) for option in resp])
+            log_probs.append(log_prob)
+
+        # Get ground truth labels
+        labels = filter_labels(data["doc"])
+        
+    except Exception as e:
+        print(e)
+        log_probs = None
+        labels = None
+
+    return log_probs, labels

src/similarity.py

@@ -0,0 +1,25 @@
+from dataloading import load_run_data
+from lmsim.metrics import Kappa_p
+
+
+def compute_similarity(selected_model_a, selected_model_b, selected_dataset):
+    probs_a, gt_a = load_run_data(selected_model_a, selected_dataset)
+    probs_b, gt_b = load_run_data(selected_model_b, selected_dataset)
+
+    assert len(probs_a) == len(probs_b), "Models must have the same number of responses"
+    
+    # Only keep responses where the ground truth is the same
+    output_a = []
+    output_b = []
+    gt = []
+    for i in range(len(probs_a)):
+        if gt_a == gt_b:
+            output_a.append(probs_a[i])
+            output_b.append(probs_b[i])
+            gt.append(gt_a[i])
+
+    # Placeholder similarity value
+    kappa_p = Kappa_p()
+    similarity = kappa_p.compute_k(output_a, output_b, gt)
+    
+    return similarity