before: { "normal/deccaf_cp_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "normal/ova_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "normal/rand_deferral_func": { "assignments": { "10": "model#0", "20": "human" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "normal/full_auto_func": { "assignments": { "10": "auto-reject", "20": "auto-reject" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "normal/full_model_func": { "assignments": { "10": "model#0", "20": "model#0" }, "predictions": { "10": 1, "20": 0 }, "rows": 2 }, "reordered/deccaf_cp_deferral_func": { "returned": null }, "reordered/ova_deferral_func": { "assignments": { "10": "human" }, "predictions": { "10": 0 }, "rows": 1 }, "reordered/rand_deferral_func": { "error": "IndexError: Cannot choose from an empty sequence" }, "reordered/full_auto_func": { "assignments": { "10": "auto-reject", "20": "auto-reject" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "reordered/full_model_func": { "assignments": { "10": "model#0", "20": "model#0" }, "predictions": { "10": 1, "20": 0 }, "rows": 2 } } after: Exact minimum assignment cost: 0.2 { "normal/deccaf_cp_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "normal/ova_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "normal/rand_deferral_func": { "assignments": { "10": "model#0", "20": "human" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "normal/full_auto_func": { "assignments": { "10": "auto-reject", "20": "auto-reject" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "normal/full_model_func": { "assignments": { "10": "model#0", "20": "model#0" }, "predictions": { "10": 1, "20": 0 }, "rows": 2 }, "reordered/deccaf_cp_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "reordered/ova_deferral_func": { "assignments": { "10": "human", "20": "model#0" }, "predictions": { "10": 0, "20": 0 }, "rows": 2 }, "reordered/rand_deferral_func": { "assignments": { "10": "model#0", "20": "human" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "reordered/full_auto_func": { "assignments": { "10": "auto-reject", "20": "auto-reject" }, "predictions": { "10": 1, "20": 1 }, "rows": 2 }, "reordered/full_model_func": { "assignments": { "10": "model#0", "20": "model#0" }, "predictions": { "10": 1, "20": 0 }, "rows": 2 } }