Catch unordered list case and cast preds and ref to str for better reliability.

test_of_time_accuracy.py

@@ -113,15 +113,12 @@ class TestOfTimeAccuracy(evaluate.Metric):
 
     @staticmethod
     def _parse_label(s):
-        """Parses a string that could be a JSON object or a Python dict."""
+        """Parses a string that could be a Python dict."""
         try:
-            return json.loads(s)
-        except json.JSONDecodeError:
-            try:
-                # Safe: only parses literals, does not execute code
-                return ast.literal_eval(s)
-            except (ValueError, SyntaxError):
-                return None
+            # Safe: only parses literals, does not execute code
+            return ast.literal_eval(s)
+        except (ValueError, SyntaxError):
+            return None
 
     def _compute(
         self,
@@ -139,8 +136,14 @@ class TestOfTimeAccuracy(evaluate.Metric):
             references = [self._parse_label(r) for r in references]
         else:
             raise ValueError(f"Invalid subset: {subset}")
-        accuracy = [i == j for i, j in zip(predictions, references)]
+        accuracy = []
+        for i, j in zip(predictions, references):
+            if subset == "arithmetic" and "unordered_list" in j:
+                i = sorted(i["unordered_list"])
+                j = sorted(j["unordered_list"])
+            accuracy.append(
+                str(i) == str(j)
+            )  # Semantic subset answer JSON somestimes has int as value. Label is string.
         if return_average:
             return {"accuracy": sum(accuracy) / len(accuracy)}
         return {"accuracy": accuracy}
-

tests.py

@@ -5,12 +5,12 @@ from test_of_time_accuracy import TestOfTimeAccuracy
 arithmetic_test_cases = {
     "predictions": [
         'JSON = {"explanation": "The war began in 360 BC. Since BC years count backwards, adding 8 years to 360 BC means subtracting 8 from 360, resulting in 352 BC.", "answer": "352 BC"}',
-        '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["London"]\n}\n```',
+        '```json\n{\n  "explanation": "The dates provided are March 2012, September 2011, June 2017, September 2019, and June 2015. These correspond to visits to Miami, Sydney, Tokyo, London, and Nairobi respectively. The latest date among these is September 2019, which is associated with London. Therefore, London is the last city visited.",\n  "unordered_list": ["Berlin","London"]\n}\n```',
         ' "To find the date of the second most important game, we need to subtract 7 days from the date of the most important game. We can do this by counting back 7 days from April 14, 2005. April 14 - 7 days = April 7, 2005", "answer": "2005-04-07"}',
     ],
     "references": [
         '{"answer": "352 BC"}',
-        '{"unordered_list": ["London"]}',
+        '{"unordered_list": ["London", "Berlin"]}',
         "{'answer': '2005-04-07'}",
     ],
     "result": {"accuracy": 2 / 3},
@@ -19,9 +19,9 @@ arithmetic_test_cases = {
 
 semantic_test_cases = {
     "predictions": [
-        ' "First, we need to find the third occurrence of E33 being the R53 of E22. We can see that it happened from 1959 to 1962, then from 1967 to 1968, and then from 1982 to 1984. The third occurrence happened from 1982 to 1984. We can then compute the duration by subtracting the start time from the end time.", "answer": 2}',
+        '{"explanation": First, we need to find the third occurrence of E33 being the R53 of E22. We can see that it happened from 1959 to 1962, then from 1967 to 1968, and then from 1982 to 1984. The third occurrence happened from 1982 to 1984. We can then compute the duration by subtracting the start time from the end time.", "answer": 2}',
         ' "To find the duration, we need to find the start and end time when E97 was the R71 of E67. From the given facts, we can see that E97 was the R71 of E67 from 1961 to 1961, and also from 1964 to 1964. We need to find the first occurrence, which is from 1961 to 1961.", "answer": 1}',
-        '{"explanation": "To find when E92 stopped being the R88 of E11, we need to look at the temporal facts where E92 was the R88 of E11 and find the end time. We see that E92 was the R88 of E11 from 1982 to 1985, and there is no other fact that indicates E92 stopped being the R88 of E11 before 1985. However, we also see that E92 was the R17 of E42 from 1986 to 1992, and E92 was the R88 of E42 from 1977 to 1979, but this is irrelevant to the question. Therefore, E92 stopped being the R88 of E11 in 1985.", "answer": "1985"}',
+        '{"explanation": "To find when E92 stopped being the R88 of E11, we need to look at the temporal facts where E92 was the R88 of E11 and find the end time. We see that E92 was the R88 of E11 from 1982 to 1985, and there is no other fact that indicates E92 stopped being the R88 of E11 before 1985. However, we also see that E92 was the R17 of E42 from 1986 to 1992, and E92 was the R88 of E42 from 1977 to 1979, but this is irrelevant to the question. Therefore, E92 stopped being the R88 of E11 in 1985.", "answer": 1985}',
     ],
     "references": ["2", "0", "1985"],
     "result": {"accuracy": 1 / 3},