By @rpinho

Refereed correctness bugs in open-source research software, with patches and links to upstream PRs

c4bb77c / feedzai-deccaf/repro/DECCAF-001/RESULTS.txt

Raw
before:
{
  "normal/deccaf_cp_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "normal/ova_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "normal/rand_deferral_func": {
    "assignments": {
      "10": "model#0",
      "20": "human"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "normal/full_auto_func": {
    "assignments": {
      "10": "auto-reject",
      "20": "auto-reject"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "normal/full_model_func": {
    "assignments": {
      "10": "model#0",
      "20": "model#0"
    },
    "predictions": {
      "10": 1,
      "20": 0
    },
    "rows": 2
  },
  "reordered/deccaf_cp_deferral_func": {
    "returned": null
  },
  "reordered/ova_deferral_func": {
    "assignments": {
      "10": "human"
    },
    "predictions": {
      "10": 0
    },
    "rows": 1
  },
  "reordered/rand_deferral_func": {
    "error": "IndexError: Cannot choose from an empty sequence"
  },
  "reordered/full_auto_func": {
    "assignments": {
      "10": "auto-reject",
      "20": "auto-reject"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "reordered/full_model_func": {
    "assignments": {
      "10": "model#0",
      "20": "model#0"
    },
    "predictions": {
      "10": 1,
      "20": 0
    },
    "rows": 2
  }
}
after:
Exact minimum assignment cost: 0.2
{
  "normal/deccaf_cp_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "normal/ova_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "normal/rand_deferral_func": {
    "assignments": {
      "10": "model#0",
      "20": "human"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "normal/full_auto_func": {
    "assignments": {
      "10": "auto-reject",
      "20": "auto-reject"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "normal/full_model_func": {
    "assignments": {
      "10": "model#0",
      "20": "model#0"
    },
    "predictions": {
      "10": 1,
      "20": 0
    },
    "rows": 2
  },
  "reordered/deccaf_cp_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "reordered/ova_deferral_func": {
    "assignments": {
      "10": "human",
      "20": "model#0"
    },
    "predictions": {
      "10": 0,
      "20": 0
    },
    "rows": 2
  },
  "reordered/rand_deferral_func": {
    "assignments": {
      "10": "model#0",
      "20": "human"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "reordered/full_auto_func": {
    "assignments": {
      "10": "auto-reject",
      "20": "auto-reject"
    },
    "predictions": {
      "10": 1,
      "20": 1
    },
    "rows": 2
  },
  "reordered/full_model_func": {
    "assignments": {
      "10": "model#0",
      "20": "model#0"
    },
    "predictions": {
      "10": 1,
      "20": 0
    },
    "rows": 2
  }
}