Parse judgments with structured output prompting, one response model, one judge model at a time.

app.py

@@ -17,7 +17,7 @@ from constants import (
 )
 from prompts import *
 from judging_dataclasses import (
-    DirectAssessmentJudgingResponse,
+    # DirectAssessmentJudgingResponse,
     DirectAssessmentCriterionScore,
     DirectAssessmentCriteriaScores,
 )
@@ -191,24 +191,24 @@ def get_llm_response_stream(model_identifier, prompt):
 
 
 def create_dataframe_for_direct_assessment_judging_response(
-    response: DirectAssessmentJudgingResponse,
-):
+    response: DirectAssessmentCriteriaScores, judging_model: str
+) -> pd.DataFrame:
     # Initialize empty list to collect data
     data = []
 
     # Loop through models
-    for judging_model in response.judging_models:
-        model_name = judging_model.model
-        # Loop through criteria_scores
-        for criteria_score in judging_model.criteria_scores:
-            data.append(
-                {
-                    "judging_model": model_name,
-                    "criteria": criteria_score.criterion,
-                    "score": criteria_score.score,
-                    "explanation": criteria_score.explanation,
-                }
-            )
+    # for judging_model in response.judging_models:
+    #     model_name = judging_model.model
+    # Loop through criteria_scores
+    for criteria_score in response.criteria_scores:
+        data.append(
+            {
+                "judging_model": judging_model,  # Gets passed in.
+                "criteria": criteria_score.criterion,
+                "score": criteria_score.score,
+                "explanation": criteria_score.explanation,
+            }
+        )
 
     # Create DataFrame
     return pd.DataFrame(data)
@@ -295,26 +295,29 @@ def get_default_aggregator_prompt(user_prompt, llms):
 
 
 def get_parse_judging_response_for_direct_assessment_prompt(
-    judging_responses: dict[str, str],
+    judging_response: str,
     criteria_list,
     options,
-):
-    formatted_judging_responses = "\n\n".join(
-        [
-            f"{get_ui_friendly_name(model)} START\n{judging_responses[model]}\n\n{get_ui_friendly_name(model)} END\n\n\n"
-            for model in judging_responses.keys()
-        ]
+) -> str:
+    # formatted_judging_responses = "\n\n\n".join(
+    #     [
+    #         f"----- {get_ui_friendly_name(model)} START -----\n\n\n{judging_responses[model]}\n\n\n-----{get_ui_friendly_name(model)} END-----\n\n\n"
+    #         for model in judging_responses.keys()
+    #     ]
+    # )
+    formatted_judging_response = (
+        f"----- START -----\n\n\n{judging_response}\n\n\n----- END -----\n\n\n"
     )
     return PARSE_JUDGING_RESPONSE_FOR_DIRECT_ASSESSMENT_PROMPT.format(
-        judging_responses=formatted_judging_responses,
+        judging_response=formatted_judging_response,
         criteria_list=format_criteria_list(criteria_list),
         options=format_likert_comparison_options(options),
     )
 
 
-def parse_judging_responses(
-    prompt: str, judging_responses: dict[str, str]
-) -> DirectAssessmentJudgingResponse:
+def get_parsed_judging_response_obj_using_llm(
+    prompt: str,
+) -> DirectAssessmentCriteriaScores:
     # if os.getenv("DEBUG_MODE") == "True":
     #     return DirectAssessmentJudgingResponse(
     #         judging_models=[
@@ -358,7 +361,7 @@ def parse_judging_responses(
             },
             {"role": "user", "content": prompt},
         ],
-        response_format=DirectAssessmentJudgingResponse,
+        response_format=DirectAssessmentCriteriaScores,
     )
     # Track token usage.
     st.session_state["input_token_usage"][
@@ -443,7 +446,7 @@ def plot_overall_scores(overall_scores_df):
         y="mean_score",
         hue="ui_friendly_name",
         data=summary,
-        palette="prism",
+        palette="rainbow",
         capsize=0.1,
         legend=False,
     )
@@ -663,29 +666,76 @@ def st_direct_assessment_results(user_prompt, direct_assessment_prompt, criteria
                                 judging_stream
                             )
 
-            # Extract actual scores from open-ended responses using structured outputs.
-            # Since we're extracting structured data for the first time, we can save the dataframe
-            # to the session state so that it's cached.
+                        # Parse the judging response. If parsing results are already cached, then
+                        # skip.
+                        # Use Structured Output to parse the judging response.
+                        parse_judging_response_prompt = get_parse_judging_response_for_direct_assessment_prompt(
+                            judging_response=st.session_state.direct_assessment_judging_responses[
+                                response_model
+                            ][
+                                judging_model
+                            ],
+                            criteria_list=criteria_list,
+                            options=SEVEN_POINT_DIRECT_ASSESSMENT_OPTIONS,
+                        )
+
+                        st.write("Parse judging response prompt:")
+                        st.write(parse_judging_response_prompt)
+
+                        if (
+                            response_model
+                            not in st.session_state.direct_assessment_judging_by_response_and_judging_model_df
+                            or judging_model
+                            not in st.session_state.direct_assessment_judging_by_response_and_judging_model_df[
+                                response_model
+                            ]
+                        ):
+                            parsed_judging_response_obj = (
+                                get_parsed_judging_response_obj_using_llm(
+                                    parse_judging_response_prompt
+                                )
+                            )
+                            st.session_state.direct_assessment_judging_by_response_and_judging_model_df[
+                                response_model
+                            ][
+                                judging_model
+                            ] = create_dataframe_for_direct_assessment_judging_response(
+                                parsed_judging_response_obj, judging_model
+                            )
+
+                        # with st.expander("Structured output parsing response"):
+                        st.write("Structured output parsing response:")
+                        st.write(
+                            st.session_state.direct_assessment_judging_by_response_and_judging_model_df[
+                                response_model
+                            ][
+                                judging_model
+                            ]
+                        )
+
+            # Combined the dataframes for each judging model into a single dataframe for each
+            # response model.
             if response_model not in st.session_state.direct_assessment_judging_df:
-                judging_responses = (
-                    st.session_state.direct_assessment_judging_responses[response_model]
-                )
-                parse_judging_response_prompt = (
-                    get_parse_judging_response_for_direct_assessment_prompt(
-                        judging_responses,
-                        criteria_list,
-                        SEVEN_POINT_DIRECT_ASSESSMENT_OPTIONS,
+                # Combine the dataframes for each judging model into a single dataframe.
+                combined_judging_df = pd.DataFrame()
+                for judging_model in st.session_state.selected_models:
+                    combined_judging_df = pd.concat(
+                        [
+                            combined_judging_df,
+                            st.session_state.direct_assessment_judging_by_response_and_judging_model_df[
+                                response_model
+                            ][
+                                judging_model
+                            ],
+                        ]
                     )
-                )
-                parsed_judging_responses = parse_judging_responses(
-                    parse_judging_response_prompt, judging_responses
-                )
                 st.session_state.direct_assessment_judging_df[response_model] = (
-                    create_dataframe_for_direct_assessment_judging_response(
-                        parsed_judging_responses
-                    )
+                    combined_judging_df
                 )
 
+            with st.expander("Judging results from all judges"):
+                st.write(st.session_state.direct_assessment_judging_df[response_model])
+
             # Uses the session state to plot the criteria scores and graphs for a given response
             # model.
             plot_criteria_scores(
@@ -706,13 +756,11 @@ def st_direct_assessment_results(user_prompt, direct_assessment_prompt, criteria
 
             # Save the overall scores to the session state if it's not already there.
             for record in grouped.to_dict(orient="records"):
-                if (
-                    response_model
-                    not in st.session_state.direct_assessment_overall_scores
-                ):
-                    st.session_state.direct_assessment_overall_scores[response_model][
-                        record["judging_model"]
-                    ] = record["overall_score"]
+                st.session_state.direct_assessment_overall_scores[
+                    get_ui_friendly_name(response_model)
+                ][get_ui_friendly_name(record["judging_model"])] = record[
+                    "overall_score"
+                ]
 
             overall_score = grouped["overall_score"].mean()
             controversy = grouped["overall_score"].std()
@@ -796,7 +844,14 @@ def main():
         if "direct_assessment_overall_score" not in st.session_state:
             st.session_state.direct_assessment_overall_score = {}
         if "direct_assessment_judging_df" not in st.session_state:
-            st.session_state.direct_assessment_judging_df = defaultdict(dict)
+            st.session_state.direct_assessment_judging_df = {}
+        if (
+            "direct_assessment_judging_by_response_and_judging_model_df"
+            not in st.session_state
+        ):
+            st.session_state.direct_assessment_judging_by_response_and_judging_model_df = defaultdict(
+                dict
+            )
         if "direct_assessment_judging_responses" not in st.session_state:
             st.session_state.direct_assessment_judging_responses = defaultdict(dict)
         if "direct_assessment_overall_scores" not in st.session_state:
@@ -940,19 +995,22 @@ def main():
                     overall_scores_df["response_model"] = overall_scores_df[
                         "response_model"
                     ].apply(get_ui_friendly_name)
-                    overall_scores_df["judging_model"] = overall_scores_df[
-                        "judging_model"
-                    ].apply(get_ui_friendly_name)
+                    # overall_scores_df["judging_model"] = overall_scores_df[
+                    #     "judging_model"
+                    # ].apply(get_ui_friendly_name)
 
                     with st.expander("Overall scores from all judges"):
+                        st.write(st.session_state.direct_assessment_overall_scores)
+                        st.dataframe(overall_scores_df_raw)
                         st.dataframe(overall_scores_df)
 
                 # All criteria scores.
                 with right_column:
                     all_scores_df = pd.DataFrame()
-                    for response_model, score_df in st.session_state[
-                        "direct_assessment_judging_df"
-                    ].items():
+                    for (
+                        response_model,
+                        score_df,
+                    ) in st.session_state.direct_assessment_judging_df.items():
                         score_df["response_model"] = response_model
                         all_scores_df = pd.concat([all_scores_df, score_df])
                     all_scores_df = all_scores_df.reset_index()
@@ -968,12 +1026,12 @@ def main():
                             "explanation",
                         ]
                     ]
-                    all_scores_df["response_model"] = all_scores_df[
-                        "response_model"
-                    ].apply(get_ui_friendly_name)
-                    all_scores_df["judging_model"] = all_scores_df[
-                        "judging_model"
-                    ].apply(get_ui_friendly_name)
+                    # all_scores_df["response_model"] = all_scores_df[
+                    #     "response_model"
+                    # ].apply(get_ui_friendly_name)
+                    # all_scores_df["judging_model"] = all_scores_df[
+                    #     "judging_model"
+                    # ].apply(get_ui_friendly_name)
 
                     with st.expander(
                         "Criteria-specific scores and explanations from all judges"

judging_dataclasses.py

@@ -35,9 +35,9 @@ class DirectAssessmentCriterionScore(BaseModel):
 
 
 class DirectAssessmentCriteriaScores(BaseModel):
-    model: str
+    # model: str
     criteria_scores: List[DirectAssessmentCriterionScore]
 
 
-class DirectAssessmentJudgingResponse(BaseModel):
-    judging_models: List[DirectAssessmentCriteriaScores]
+# class DirectAssessmentJudgingResponse(BaseModel):
+#     judging_models: List[DirectAssessmentCriteriaScores]

prompts.py

@@ -1,18 +1,21 @@
 from judging_dataclasses import Criteria
 
 
-PARSE_JUDGING_RESPONSE_FOR_DIRECT_ASSESSMENT_PROMPT = """We are trying to parse the responses from the judges for a direct assessment.
+PARSE_JUDGING_RESPONSE_FOR_DIRECT_ASSESSMENT_PROMPT = """We are trying to parse the response from a judge for a direct assessment.
+
+The judge was asked to give a rating for each of the following criteria, along with an explanation:
 
-Each judge was asked to give a rating for each of the following criteria, along with an explanation:
 {criteria_list}
 
 The possible options for each criterion are as follows:
+
 {options}
 
-The responses from the judges are as follows:
-{judging_responses}
+Here is the response from the judge:
+
+{judging_response}
 
-Please provide a JSON object with the following structure that includes the model name and the scores for each of the criteria, along with the explanation.
+Please provide a JSON object the scores for each of the criteria, along with any explanation the judge provided.
 """