osverify
    auroc chk 0.7909722222222222 harn 0.7909722222222222 OK
  tau 0.05
    in t* chk 16.0 harn 16.0 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 2 harn 2 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.044444444444444446 harn 0.044444444444444446 OK
    in cp_fnr chk 0.13337564609192887 harn 0.13337564609192853 OK
    cf ri chk 10 harn 10 OK
    cf rv chk 2 harn 2 OK
    cf tnr chk 0.3125 harn 0.3125 OK
    cf fnr chk 0.044444444444444446 harn 0.044444444444444446 OK
    cf violations chk 2 harn 2 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.786] harn [0.0, 0.786] OK
    cf tnr ci chk [0.0, 0.76] harn [0.0, 0.76] OK
    cf fnr ci chk [0.0, 0.132] harn [0.0, 0.132] OK
  tau 0.1
    in t* chk 16.0 harn 16.0 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 2 harn 2 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.044444444444444446 harn 0.044444444444444446 OK
    in cp_fnr chk 0.13337564609192887 harn 0.13337564609192853 OK
    cf ri chk 20 harn 20 OK
    cf rv chk 5 harn 5 OK
    cf tnr chk 0.625 harn 0.625 OK
    cf fnr chk 0.1111111111111111 harn 0.1111111111111111 OK
    cf violations chk 3 harn 3 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.812] harn [0.0, 0.812] OK
    cf tnr ci chk [0.241, 0.808] harn [0.241, 0.808] OK
    cf fnr ci chk [0.043, 0.2] harn [0.043, 0.2] OK
referee
    auroc chk 0.7715277777777778 harn 0.7715277777777778 OK
  tau 0.05
    in t* chk 0.6 harn 0.6 OK
    in rej_inv chk 15 harn 15 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.46875 harn 0.46875 OK
    in fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    in cp_fnr chk 0.10113378359391367 harn 0.10113378359391356 OK
    cf ri chk 15 harn 15 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.46875 harn 0.46875 OK
    cf fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.268, 0.704] harn [0.268, 0.704] OK
    cf tnr ci chk [0.281, 0.694] harn [0.281, 0.694] OK
    cf fnr ci chk [0.0, 0.091] harn [0.0, 0.091] OK
  tau 0.1
    in t* chk 0.4 harn 0.4 OK
    in rej_inv chk 18 harn 18 OK
    in rej_val chk 3 harn 3 OK
    in tnr chk 0.5625 harn 0.5625 OK
    in fnr chk 0.06666666666666667 harn 0.06666666666666667 OK
    in cp_fnr chk 0.16338838109952247 harn 0.16338838109952253 OK
    cf ri chk 18 harn 18 OK
    cf rv chk 3 harn 3 OK
    cf tnr chk 0.5625 harn 0.5625 OK
    cf fnr chk 0.06666666666666667 harn 0.06666666666666667 OK
    cf violations chk 3 harn 3 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.333, 0.735] harn [0.333, 0.735] OK
    cf tnr ci chk [0.324, 0.733] harn [0.324, 0.733] OK
    cf fnr ci chk [0.0, 0.128] harn [0.0, 0.128] OK
voting
    auroc chk 0.8177083333333334 harn 0.8177083333333334 OK
  tau 0.05
    in t* chk 0.625 harn 0.625 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    in cp_fnr chk 0.10113378359391367 harn 0.10113378359391356 OK
    cf ri chk 20 harn 20 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.625 harn 0.625 OK
    cf fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.423, 0.808] harn [0.423, 0.808] OK
    cf tnr ci chk [0.414, 0.8] harn [0.414, 0.8] OK
    cf fnr ci chk [0.0, 0.119] harn [0.0, 0.119] OK
  tau 0.1
    in t* chk 0.625 harn 0.625 OK
    in rej_inv chk 20 harn 20 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.625 harn 0.625 OK
    in fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    in cp_fnr chk 0.10113378359391367 harn 0.10113378359391356 OK
    cf ri chk 21 harn 21 OK
    cf rv chk 3 harn 3 OK
    cf tnr chk 0.65625 harn 0.65625 OK
    cf fnr chk 0.06666666666666667 harn 0.06666666666666667 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.444, 0.844] harn [0.444, 0.844] OK
    cf tnr ci chk [0.462, 0.839] harn [0.462, 0.839] OK
    cf fnr ci chk [0.0, 0.167] harn [0.0, 0.167] OK
length
    auroc chk 0.5819444444444445 harn 0.5819444444444445 OK
  tau 0.05
    in t* chk -223.0 harn -223.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    in cp_fnr chk 0.10113378359391367 harn 0.10113378359391356 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.022222222222222223 harn 0.022222222222222223 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.125] harn [0.0, 0.125] OK
    cf tnr ci chk [0.0, 0.135] harn [0.0, 0.135] OK
    cf fnr ci chk [0.02, 0.13] harn [0.02, 0.13] OK
  tau 0.1
    in t* chk -232.0 harn -232.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 4 harn 4 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.08888888888888889 harn 0.08888888888888889 OK
    in cp_fnr chk 0.19197631701164264 harn 0.19197631701164264 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 5 harn 5 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.1111111111111111 harn 0.1111111111111111 OK
    cf violations chk 3 harn 3 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.237] harn [0.0, 0.237] OK
    cf tnr ci chk [0.0, 0.293] harn [0.0, 0.293] OK
    cf fnr ci chk [0.065, 0.196] harn [0.065, 0.196] OK
cue
    auroc chk 0.515625 harn 0.515625 OK
  tau 0.05
    in t* chk 1.0 harn 1.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.06440428880778315 harn 0.06440428880778348 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.107] harn [0.0, 0.107] OK
    cf tnr ci chk [0.0, 0.107] harn [0.0, 0.107] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
  tau 0.1
    in t* chk 1.0 harn 1.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.03125 harn 0.03125 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.06440428880778315 harn 0.06440428880778348 OK
    cf ri chk 1 harn 1 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.03125 harn 0.03125 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.107] harn [0.0, 0.107] OK
    cf tnr ci chk [0.0, 0.107] harn [0.0, 0.107] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
osverify-referee
    tau 0.05 in diff chk 0.15625 harn 0.15625 OK
    tau 0.05 in diff ci chk [-0.64, 0.286] harn [-0.64, 0.286] OK
    tau 0.05 cf diff chk -0.15625 harn -0.15625 OK
    tau 0.05 cf diff ci chk [-0.533, 0.273] harn [-0.533, 0.273] OK
    tau 0.1 in diff chk 0.0625 harn 0.0625 OK
    tau 0.1 in diff ci chk [-0.514, 0.278] harn [-0.514, 0.278] OK
    tau 0.1 cf diff chk 0.0625 harn 0.0625 OK
    tau 0.1 cf diff ci chk [-0.294, 0.278] harn [-0.294, 0.278] OK
osverify-voting
    tau 0.05 in diff chk 0.0 harn 0.0 OK
    tau 0.05 in diff ci chk [-0.743, 0.103] harn [-0.743, 0.103] OK
    tau 0.05 cf diff chk -0.3125 harn -0.3125 OK
    tau 0.05 cf diff ci chk [-0.632, 0.08] harn [-0.632, 0.08] OK
    tau 0.1 in diff chk 0.0 harn 0.0 OK
    tau 0.1 in diff ci chk [-0.594, 0.129] harn [-0.594, 0.129] OK
    tau 0.1 cf diff chk -0.03125 harn -0.03125 OK
    tau 0.1 cf diff ci chk [-0.421, 0.103] harn [-0.421, 0.103] OK
osverify-length
    tau 0.05 in diff chk 0.59375 harn 0.59375 OK
    tau 0.05 in diff ci chk [-0.091, 0.774] harn [-0.091, 0.774] OK
    tau 0.05 cf diff chk 0.28125 harn 0.28125 OK
    tau 0.05 cf diff ci chk [-0.039, 0.75] harn [-0.039, 0.75] OK
    tau 0.1 in diff chk 0.59375 harn 0.59375 OK
    tau 0.1 in diff ci chk [-0.031, 0.795] harn [-0.031, 0.795] OK
    tau 0.1 cf diff chk 0.59375 harn 0.59375 OK
    tau 0.1 cf diff ci chk [0.105, 0.786] harn [0.105, 0.786] OK
osverify-cue
    tau 0.05 in diff chk 0.59375 harn 0.59375 OK
    tau 0.05 in diff ci chk [-0.075, 0.759] harn [-0.075, 0.759] OK
    tau 0.05 cf diff chk 0.28125 harn 0.28125 OK
    tau 0.05 cf diff ci chk [-0.03, 0.733] harn [-0.03, 0.733] OK
    tau 0.1 in diff chk 0.59375 harn 0.59375 OK
    tau 0.1 in diff ci chk [-0.028, 0.778] harn [-0.028, 0.778] OK
    tau 0.1 cf diff chk 0.59375 harn 0.59375 OK
    tau 0.1 cf diff ci chk [0.194, 0.774] harn [0.194, 0.774] OK
referee-voting checker-only (not a baseline pair)
referee-length checker-only (not a baseline pair)
referee-cue checker-only (not a baseline pair)
voting-length checker-only (not a baseline pair)
voting-cue checker-only (not a baseline pair)
length-cue checker-only (not a baseline pair)
decisions (checker, section-5 clauses a+b only): {'osverify-referee': 'UNCLEAR', 'osverify-voting': "DON'T"}
MISMATCHES 0
