By @rpinho
Refereed correctness bugs in open-source research software, with patches and links to upstream PRs
before:
{
"normal/deccaf_cp_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"normal/ova_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"normal/rand_deferral_func": {
"assignments": {
"10": "model#0",
"20": "human"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"normal/full_auto_func": {
"assignments": {
"10": "auto-reject",
"20": "auto-reject"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"normal/full_model_func": {
"assignments": {
"10": "model#0",
"20": "model#0"
},
"predictions": {
"10": 1,
"20": 0
},
"rows": 2
},
"reordered/deccaf_cp_deferral_func": {
"returned": null
},
"reordered/ova_deferral_func": {
"assignments": {
"10": "human"
},
"predictions": {
"10": 0
},
"rows": 1
},
"reordered/rand_deferral_func": {
"error": "IndexError: Cannot choose from an empty sequence"
},
"reordered/full_auto_func": {
"assignments": {
"10": "auto-reject",
"20": "auto-reject"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"reordered/full_model_func": {
"assignments": {
"10": "model#0",
"20": "model#0"
},
"predictions": {
"10": 1,
"20": 0
},
"rows": 2
}
}
after:
Exact minimum assignment cost: 0.2
{
"normal/deccaf_cp_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"normal/ova_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"normal/rand_deferral_func": {
"assignments": {
"10": "model#0",
"20": "human"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"normal/full_auto_func": {
"assignments": {
"10": "auto-reject",
"20": "auto-reject"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"normal/full_model_func": {
"assignments": {
"10": "model#0",
"20": "model#0"
},
"predictions": {
"10": 1,
"20": 0
},
"rows": 2
},
"reordered/deccaf_cp_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"reordered/ova_deferral_func": {
"assignments": {
"10": "human",
"20": "model#0"
},
"predictions": {
"10": 0,
"20": 0
},
"rows": 2
},
"reordered/rand_deferral_func": {
"assignments": {
"10": "model#0",
"20": "human"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"reordered/full_auto_func": {
"assignments": {
"10": "auto-reject",
"20": "auto-reject"
},
"predictions": {
"10": 1,
"20": 1
},
"rows": 2
},
"reordered/full_model_func": {
"assignments": {
"10": "model#0",
"20": "model#0"
},
"predictions": {
"10": 1,
"20": 0
},
"rows": 2
}
}