osverify
    auroc chk 0.8018465909090909 harn 0.8018465909090909 OK
  tau 0.05
    in t* chk 16.0 harn 16.0 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    in cp_fnr chk 0.10333500111522342 harn 0.10333500111522373 OK
    cf ri chk 20 harn 20 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.625 harn 0.625 OK
    cf fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.8] harn [0.0, 0.8] OK
    cf tnr ci chk [0.135, 0.8] harn [0.135, 0.8] OK
    cf fnr ci chk [0.0, 0.146] harn [0.0, 0.146] OK
  tau 0.1
    in t* chk 15.0 harn 15.0 OK
    in rej_inv chk 21 harn 21 OK
    in rej_val chk 4 harn 4 OK
    in tnr chk 0.65625 harn 0.65625 OK
    in fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    in cp_fnr chk 0.1960784968125613 harn 0.19607849681256095 OK
    cf ri chk 21 harn 21 OK
    cf rv chk 4 harn 4 OK
    cf tnr chk 0.65625 harn 0.65625 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 2 harn 2 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.438, 0.813] harn [0.438, 0.813] OK
    cf tnr ci chk [0.424, 0.818] harn [0.424, 0.818] OK
    cf fnr ci chk [0.043, 0.196] harn [0.043, 0.196] OK
referee
    auroc chk 0.7752130681818182 harn 0.7752130681818182 OK
  tau 0.05
    in t* chk 0.6 harn 0.6 OK
    in rej_inv chk 15 harn 15 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.46875 harn 0.46875 OK
    in fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    in cp_fnr chk 0.10333500111522342 harn 0.10333500111522373 OK
    cf ri chk 15 harn 15 OK
    cf rv chk 2 harn 2 OK
    cf tnr chk 0.46875 harn 0.46875 OK
    cf fnr chk 0.045454545454545456 harn 0.045454545454545456 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.273, 0.703] harn [0.273, 0.703] OK
    cf tnr ci chk [0.273, 0.686] harn [0.273, 0.686] OK
    cf fnr ci chk [0.0, 0.125] harn [0.0, 0.125] OK
  tau 0.1
    in t* chk 0.4 harn 0.4 OK
    in rej_inv chk 18 harn 18 OK
    in rej_val chk 3 harn 3 OK
    in tnr chk 0.5625 harn 0.5625 OK
    in fnr chk 0.06818181818181818 harn 0.06818181818181818 OK
    in cp_fnr chk 0.16689884168360836 harn 0.1668988416836079 OK
    cf ri chk 18 harn 18 OK
    cf rv chk 3 harn 3 OK
    cf tnr chk 0.5625 harn 0.5625 OK
    cf fnr chk 0.06818181818181818 harn 0.06818181818181818 OK
    cf violations chk 2 harn 2 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.333, 0.731] harn [0.333, 0.731] OK
    cf tnr ci chk [0.308, 0.725] harn [0.308, 0.725] OK
    cf fnr ci chk [0.02, 0.167] harn [0.02, 0.167] OK
voting
    auroc chk 0.8196022727272727 harn 0.8196022727272727 OK
  tau 0.05
    in t* chk 0.625 harn 0.625 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    in cp_fnr chk 0.10333500111522342 harn 0.10333500111522373 OK
    cf ri chk 20 harn 20 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.625 harn 0.625 OK
    cf fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.412, 0.8] harn [0.412, 0.8] OK
    cf tnr ci chk [0.4, 0.8] harn [0.4, 0.8] OK
    cf fnr ci chk [0.0, 0.105] harn [0.0, 0.105] OK
  tau 0.1
    in t* chk 0.625 harn 0.625 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    in cp_fnr chk 0.10333500111522342 harn 0.10333500111522373 OK
    cf ri chk 21 harn 21 OK
    cf rv chk 2 harn 2 OK
    cf tnr chk 0.65625 harn 0.65625 OK
    cf fnr chk 0.045454545454545456 harn 0.045454545454545456 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.433, 0.833] harn [0.433, 0.833] OK
    cf tnr ci chk [0.467, 0.833] harn [0.467, 0.833] OK
    cf fnr ci chk [0.0, 0.163] harn [0.0, 0.163] OK
length
    auroc chk 0.5809659090909091 harn 0.5809659090909091 OK
  tau 0.05
    in t* chk -223.0 harn -223.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    in cp_fnr chk 0.10333500111522342 harn 0.10333500111522373 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.022727272727272728 harn 0.022727272727272728 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.125] harn [0.0, 0.125] OK
    cf tnr ci chk [0.0, 0.125] harn [0.0, 0.125] OK
    cf fnr ci chk [0.02, 0.114] harn [0.02, 0.114] OK
  tau 0.1
    in t* chk -232.0 harn -232.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 4 harn 4 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    in cp_fnr chk 0.1960784968125613 harn 0.19607849681256095 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 4 harn 4 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 4 harn 4 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.235] harn [0.0, 0.235] OK
    cf tnr ci chk [0.0, 0.235] harn [0.0, 0.235] OK
    cf fnr ci chk [0.056, 0.175] harn [0.056, 0.175] OK
cue
    auroc chk 0.515625 harn 0.515625 OK
  tau 0.05
    in t* chk 1.0 harn 1.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.06581877114493428 harn 0.06581877114493394 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.103] harn [0.0, 0.103] OK
    cf tnr ci chk [0.0, 0.103] harn [0.0, 0.103] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
  tau 0.1
    in t* chk 1.0 harn 1.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.06581877114493428 harn 0.06581877114493394 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.103] harn [0.0, 0.103] OK
    cf tnr ci chk [0.0, 0.103] harn [0.0, 0.103] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
osverify-referee
    tau 0.05 in diff chk 0.15625 harn 0.15625 OK
    tau 0.05 in diff ci chk [-0.526, 0.318] harn [-0.526, 0.318] OK
    tau 0.05 cf diff chk 0.15625 harn 0.15625 OK
    tau 0.05 cf diff ci chk [-0.375, 0.314] harn [-0.375, 0.314] OK
    tau 0.1 in diff chk 0.09375 harn 0.09375 OK
    tau 0.1 in diff ci chk [-0.077, 0.281] harn [-0.077, 0.281] OK
    tau 0.1 cf diff chk 0.09375 harn 0.09375 OK
    tau 0.1 cf diff ci chk [-0.094, 0.313] harn [-0.094, 0.313] OK
osverify-voting
    tau 0.05 in diff chk 0.0 harn 0.0 OK
    tau 0.05 in diff ci chk [-0.575, 0.121] harn [-0.575, 0.121] OK
    tau 0.05 cf diff chk 0.0 harn 0.0 OK
    tau 0.05 cf diff ci chk [-0.459, 0.12] harn [-0.459, 0.12] OK
    tau 0.1 in diff chk 0.03125 harn 0.03125 OK
    tau 0.1 in diff ci chk [-0.18, 0.143] harn [-0.18, 0.143] OK
    tau 0.1 cf diff chk 0.0 harn 0.0 OK
    tau 0.1 cf diff ci chk [-0.19, 0.133] harn [-0.19, 0.133] OK
osverify-length
    tau 0.05 in diff chk 0.59375 harn 0.59375 OK
    tau 0.05 in diff ci chk [-0.044, 0.788] harn [-0.044, 0.788] OK
    tau 0.05 cf diff chk 0.59375 harn 0.59375 OK
    tau 0.05 cf diff ci chk [0.091, 0.788] harn [0.091, 0.788] OK
    tau 0.1 in diff chk 0.625 harn 0.625 OK
    tau 0.1 in diff ci chk [0.281, 0.794] harn [0.281, 0.794] OK
    tau 0.1 cf diff chk 0.625 harn 0.625 OK
    tau 0.1 cf diff ci chk [0.273, 0.8] harn [0.273, 0.8] OK
osverify-cue
    tau 0.05 in diff chk 0.59375 harn 0.59375 OK
    tau 0.05 in diff ci chk [-0.036, 0.769] harn [-0.036, 0.769] OK
    tau 0.05 cf diff chk 0.59375 harn 0.59375 OK
    tau 0.05 cf diff ci chk [0.097, 0.769] harn [0.097, 0.769] OK
    tau 0.1 in diff chk 0.625 harn 0.625 OK
    tau 0.1 in diff ci chk [0.407, 0.793] harn [0.407, 0.793] OK
    tau 0.1 cf diff chk 0.625 harn 0.625 OK
    tau 0.1 cf diff ci chk [0.387, 0.8] harn [0.387, 0.8] OK
referee-voting checker-only (not a baseline pair)
referee-length checker-only (not a baseline pair)
referee-cue checker-only (not a baseline pair)
voting-length checker-only (not a baseline pair)
voting-cue checker-only (not a baseline pair)
length-cue checker-only (not a baseline pair)
decisions (checker, section-5 clauses a+b only): {'osverify-referee': 'UNCLEAR', 'osverify-voting': "DON'T"}
MISMATCHES 0
