Commit
Β·
11f068c
1
Parent(s):
f4b0324
adding labels
Browse files
app.py
CHANGED
|
@@ -389,7 +389,7 @@ def show_leaderboard(task):
|
|
| 389 |
st.dataframe(real_tmp, column_config=column_config)
|
| 390 |
|
| 391 |
|
| 392 |
-
def make_roc(results):
|
| 393 |
results["FA"] = 1.0 - results["real_accuracy"]
|
| 394 |
|
| 395 |
chart = (
|
|
@@ -405,7 +405,26 @@ def make_roc(results):
|
|
| 405 |
detail=["submission_id","auc","balanced_accuracy"]
|
| 406 |
)
|
| 407 |
.properties(width=400, height=400, title="Detection vs False Alarm vs Inference Time")
|
|
|
|
| 408 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
|
| 410 |
diag_line = (
|
| 411 |
alt.Chart(pd.DataFrame(dict(tpr=[0, 1], fpr=[0, 1])))
|
|
@@ -413,10 +432,10 @@ def make_roc(results):
|
|
| 413 |
.encode(x="fpr", y="tpr")
|
| 414 |
)
|
| 415 |
|
| 416 |
-
return chart + diag_line
|
| 417 |
|
| 418 |
|
| 419 |
-
def make_acc(results):
|
| 420 |
results = results.loc[results["total_time"] >= 0]
|
| 421 |
|
| 422 |
chart = (
|
|
@@ -433,6 +452,32 @@ def make_acc(results):
|
|
| 433 |
)
|
| 434 |
.properties(width=400, height=400, title="Inference Time vs Balanced Accuracy")
|
| 435 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 436 |
diag_line = (
|
| 437 |
alt.Chart(pd.DataFrame(dict(t=[100, 100000], y=[0.5, 0.5])))
|
| 438 |
.mark_line(color="lightgray", strokeDash=[8, 4])
|
|
@@ -460,6 +505,8 @@ def show_charts(task):
|
|
| 460 |
results = load_results(task, best_only=True)
|
| 461 |
temp = results[f"{split}_score"].reset_index()
|
| 462 |
teams = get_unique_teams(temp["team"])
|
|
|
|
|
|
|
| 463 |
|
| 464 |
if split == "private":
|
| 465 |
|
|
@@ -486,8 +533,8 @@ def show_charts(task):
|
|
| 486 |
|
| 487 |
# with st.spinner("making plots...", show_time=True):
|
| 488 |
|
| 489 |
-
roc_scatter = make_roc(temp)
|
| 490 |
-
acc_vs_time = make_acc(temp)
|
| 491 |
|
| 492 |
if split == "private" and hf_token is not None:
|
| 493 |
if full_curves:
|
|
@@ -506,8 +553,6 @@ def make_plots_for_task(task):
|
|
| 506 |
|
| 507 |
with t2:
|
| 508 |
show_charts(task)
|
| 509 |
-
|
| 510 |
-
|
| 511 |
|
| 512 |
|
| 513 |
updated = get_updated_time()
|
|
|
|
| 389 |
st.dataframe(real_tmp, column_config=column_config)
|
| 390 |
|
| 391 |
|
| 392 |
+
def make_roc(results,show_text = False):
|
| 393 |
results["FA"] = 1.0 - results["real_accuracy"]
|
| 394 |
|
| 395 |
chart = (
|
|
|
|
| 405 |
detail=["submission_id","auc","balanced_accuracy"]
|
| 406 |
)
|
| 407 |
.properties(width=400, height=400, title="Detection vs False Alarm vs Inference Time")
|
| 408 |
+
|
| 409 |
)
|
| 410 |
+
if show_text:
|
| 411 |
+
text = (
|
| 412 |
+
alt.Chart(results)
|
| 413 |
+
.mark_text(
|
| 414 |
+
align="right",
|
| 415 |
+
fontSize=14,
|
| 416 |
+
dx=-5, # shift text to right of point
|
| 417 |
+
dy=-5, # shift text slightly up
|
| 418 |
+
)
|
| 419 |
+
.encode(
|
| 420 |
+
x=alt.X("FA:Q", title="π§βπ€ False Positive Rate", scale=alt.Scale(domain=[0, 1])),
|
| 421 |
+
y=alt.Y("generated_accuracy:Q", title="π€ True Positive Rate", scale=alt.Scale(domain=[0, 1])),
|
| 422 |
+
color="team:N", # Color by categorical field
|
| 423 |
+
text = "team",
|
| 424 |
+
)
|
| 425 |
+
)
|
| 426 |
+
|
| 427 |
+
chart = chart + text
|
| 428 |
|
| 429 |
diag_line = (
|
| 430 |
alt.Chart(pd.DataFrame(dict(tpr=[0, 1], fpr=[0, 1])))
|
|
|
|
| 432 |
.encode(x="fpr", y="tpr")
|
| 433 |
)
|
| 434 |
|
| 435 |
+
return (chart + diag_line)
|
| 436 |
|
| 437 |
|
| 438 |
+
def make_acc(results,show_text = False):
|
| 439 |
results = results.loc[results["total_time"] >= 0]
|
| 440 |
|
| 441 |
chart = (
|
|
|
|
| 452 |
)
|
| 453 |
.properties(width=400, height=400, title="Inference Time vs Balanced Accuracy")
|
| 454 |
)
|
| 455 |
+
|
| 456 |
+
if show_text:
|
| 457 |
+
text = (
|
| 458 |
+
alt.Chart(results)
|
| 459 |
+
.mark_text(
|
| 460 |
+
align="right",
|
| 461 |
+
dx=-5, # shift text to right of point
|
| 462 |
+
dy=-5, # shift text slightly up
|
| 463 |
+
fontSize=14,
|
| 464 |
+
)
|
| 465 |
+
.encode(
|
| 466 |
+
x=alt.X("total_time:Q", title="π Inference Time (sec)", scale=alt.Scale(type = "log",domain = [100,100000])),
|
| 467 |
+
y=alt.Y(
|
| 468 |
+
"balanced_accuracy:Q",
|
| 469 |
+
title="Balanced Accuracy",
|
| 470 |
+
scale=alt.Scale(domain=[0.4, 1]),
|
| 471 |
+
),
|
| 472 |
+
color="team:N", # Color by categorical field # Size by quantitative field
|
| 473 |
+
text = "team",
|
| 474 |
+
)
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
chart = chart + text
|
| 478 |
+
|
| 479 |
+
|
| 480 |
+
|
| 481 |
diag_line = (
|
| 482 |
alt.Chart(pd.DataFrame(dict(t=[100, 100000], y=[0.5, 0.5])))
|
| 483 |
.mark_line(color="lightgray", strokeDash=[8, 4])
|
|
|
|
| 505 |
results = load_results(task, best_only=True)
|
| 506 |
temp = results[f"{split}_score"].reset_index()
|
| 507 |
teams = get_unique_teams(temp["team"])
|
| 508 |
+
|
| 509 |
+
best_only = True
|
| 510 |
|
| 511 |
if split == "private":
|
| 512 |
|
|
|
|
| 533 |
|
| 534 |
# with st.spinner("making plots...", show_time=True):
|
| 535 |
|
| 536 |
+
roc_scatter = make_roc(temp, show_text=best_only)
|
| 537 |
+
acc_vs_time = make_acc(temp, show_text=best_only)
|
| 538 |
|
| 539 |
if split == "private" and hf_token is not None:
|
| 540 |
if full_curves:
|
|
|
|
| 553 |
|
| 554 |
with t2:
|
| 555 |
show_charts(task)
|
|
|
|
|
|
|
| 556 |
|
| 557 |
|
| 558 |
updated = get_updated_time()
|
utils.py
CHANGED
|
@@ -190,11 +190,13 @@ if __name__ == "__main__":
|
|
| 190 |
"safe-challenge/video-challenge-task-1-config",
|
| 191 |
"safe-challenge/video-challenge-task-2-config"]
|
| 192 |
|
| 193 |
-
|
| 194 |
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
|
|
|
|
|
|
| 198 |
|
| 199 |
|
| 200 |
## Loop
|
|
|
|
| 190 |
"safe-challenge/video-challenge-task-1-config",
|
| 191 |
"safe-challenge/video-challenge-task-2-config"]
|
| 192 |
|
| 193 |
+
download_competition_data(competition_names=spaces)
|
| 194 |
|
| 195 |
+
try:
|
| 196 |
+
add_custom_submission(path_to_cache="competition_cache/safe-challenge/video-challenge-task-1-config",
|
| 197 |
+
path_to_subfile="competition_cache/custom/Scores-DSRI-brian.txt")
|
| 198 |
+
except Exception as e:
|
| 199 |
+
print("no custom subs found.. skipping")
|
| 200 |
|
| 201 |
|
| 202 |
## Loop
|