[Fix] alpha choices

src/dataloading.py

@@ -4,7 +4,7 @@ from huggingface_hub import HfApi
 
 from functools import lru_cache
 
-from src.utils import opt_in_pars_to_index, get_test_target
+from src.utils import opt_to_index, get_test_target
 
 def get_leaderboard_models_reload():
     api = HfApi()
@@ -104,8 +104,8 @@ def filter_labels(dataset_name, doc):
     test_target, target_key = get_test_target(doc[0])
     if "answer_index" in doc[0].keys():
         labels = [d["answer_index"] for d in doc]
-    elif test_target.startswith("("):
-        labels = [opt_in_pars_to_index(d[target_key]) for d in doc]
+    elif test_target.startswith("(") or test_target.isalpha():
+        labels = [opt_to_index(d[target_key]) for d in doc]
     elif dataset_name in ["bbh_boolean_expressions"]:
         for d in doc:
             if d[target_key] == "True":

src/utils.py

@@ -10,12 +10,17 @@ def one_hot(probs: np.array) -> np.array:
     one_hot[np.argmax(probs)] = 1
     return one_hot
 
-def opt_in_pars_to_index(s):
+def opt_to_index(s):
     if s.startswith("(") and s.endswith(")"):
         letter = s[1]  # Extract the letter inside the parentheses
         return ord(letter) - ord("A")  # Convert to zero-based index
+    elif is_single_letter(s):
+        return ord(s.upper()) - ord("A")
     else:
         raise ValueError("Invalid format")
+
+def is_single_letter(s):
+    return len(s) == 1 and s.isalpha()
     
 def get_test_target(doc):
     if "target" in doc: