osverify
    auroc chk 0.7272727272727273 harn 0.7272727272727273 OK
  tau 0.05
    in t* chk 10.0 harn 10.0 OK
    in rej_inv chk 5 harn 5 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.45454545454545453 harn 0.45454545454545453 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 5 harn 5 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.45454545454545453 harn 0.45454545454545453 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    cf tnr ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
  tau 0.1
    in t* chk 10.0 harn 10.0 OK
    in rej_inv chk 5 harn 5 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.45454545454545453 harn 0.45454545454545453 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 5 harn 5 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.45454545454545453 harn 0.45454545454545453 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    cf tnr ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
referee
    auroc chk 0.6157024793388429 harn 0.6157024793388429 OK
  tau 0.05
    in t* chk 0.4 harn 0.4 OK
    in rej_inv chk 3 harn 3 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.2727272727272727 harn 0.2727272727272727 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 3 harn 3 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.2727272727272727 harn 0.2727272727272727 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.571] harn [0.0, 0.571] OK
    cf tnr ci chk [0.0, 0.6] harn [0.0, 0.6] OK
    cf fnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
  tau 0.1
    in t* chk 0.4 harn 0.4 OK
    in rej_inv chk 3 harn 3 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.2727272727272727 harn 0.2727272727272727 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 3 harn 3 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.2727272727272727 harn 0.2727272727272727 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.583] harn [0.0, 0.583] OK
    cf tnr ci chk [0.057, 0.615] harn [0.057, 0.615] OK
    cf fnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
voting
    auroc chk 0.6983471074380165 harn 0.6983471074380165 OK
  tau 0.05
    in t* chk 0.5 harn 0.5 OK
    in rej_inv chk 4 harn 4 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.36363636363636365 harn 0.36363636363636365 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 4 harn 4 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.36363636363636365 harn 0.36363636363636365 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.1, 0.7] harn [0.1, 0.7] OK
    cf tnr ci chk [0.1, 0.7] harn [0.1, 0.7] OK
    cf fnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
  tau 0.1
    in t* chk 0.125 harn 0.125 OK
    in rej_inv chk 5 harn 5 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.45454545454545453 harn 0.45454545454545453 OK
    in fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    in cp_fnr chk 0.3643594892472426 harn 0.3643594892472426 OK
    cf ri chk 4 harn 4 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.36363636363636365 harn 0.36363636363636365 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.125, 0.75] harn [0.125, 0.75] OK
    cf tnr ci chk [0.111, 0.7] harn [0.111, 0.7] OK
    cf fnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
length
    auroc chk 0.2396694214876033 harn 0.2396694214876033 OK
  tau 0.05
    in t* chk +inf harn +inf OK
    in rej_inv chk 0 harn 0 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.0 harn 0.0 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 0 harn 0 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.0 harn 0.0 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.25] harn [0.0, 0.25] OK
    cf tnr ci chk [0.0, 0.273] harn [0.0, 0.273] OK
    cf fnr ci chk [0.0, 0.5] harn [0.0, 0.5] OK
  tau 0.1
    in t* chk -223.0 harn -223.0 OK
    in rej_inv chk 1 harn 1 OK
    in rej_val chk 1 harn 1 OK
    in tnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    in fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    in cp_fnr chk 0.3643594892472426 harn 0.3643594892472426 OK
    cf ri chk 0 harn 0 OK
    cf rv chk 1 harn 1 OK
    cf tnr chk 0.0 harn 0.0 OK
    cf fnr chk 0.09090909090909091 harn 0.09090909090909091 OK
    cf violations chk 1 harn 1 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
    cf tnr ci chk [0.0, 0.3] harn [0.0, 0.3] OK
    cf fnr ci chk [0.0, 0.5] harn [0.0, 0.5] OK
cue
    auroc chk 0.5 harn 0.5 OK
  tau 0.05
    in t* chk +inf harn +inf OK
    in rej_inv chk 0 harn 0 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.0 harn 0.0 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 0 harn 0 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.0 harn 0.0 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
    cf tnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
  tau 0.1
    in t* chk +inf harn +inf OK
    in rej_inv chk 0 harn 0 OK
    in rej_val chk 0 harn 0 OK
    in tnr chk 0.0 harn 0.0 OK
    in fnr chk 0.0 harn 0.0 OK
    in cp_fnr chk 0.23840419038085292 harn 0.2384041903808526 OK
    cf ri chk 0 harn 0 OK
    cf rv chk 0 harn 0 OK
    cf tnr chk 0.0 harn 0.0 OK
    cf fnr chk 0.0 harn 0.0 OK
    cf violations chk 0 harn 0 OK
    cf folds_without_valid chk 0 harn 0 OK
    in tnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
    cf tnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
    cf fnr ci chk [0.0, 0.0] harn [0.0, 0.0] OK
osverify-referee
    tau 0.05 in diff chk 0.18181818181818182 harn 0.18181818181818182 OK
    tau 0.05 in diff ci chk [0.0, 0.444] harn [0.0, 0.444] OK
    tau 0.05 cf diff chk 0.18181818181818182 harn 0.18181818181818182 OK
    tau 0.05 cf diff ci chk [-0.125, 0.429] harn [-0.125, 0.429] OK
    tau 0.1 in diff chk 0.18181818181818182 harn 0.18181818181818182 OK
    tau 0.1 in diff ci chk [-0.111, 0.444] harn [-0.111, 0.444] OK
    tau 0.1 cf diff chk 0.18181818181818182 harn 0.18181818181818182 OK
    tau 0.1 cf diff ci chk [-0.167, 0.429] harn [-0.167, 0.429] OK
osverify-voting
    tau 0.05 in diff chk 0.09090909090909088 harn 0.09090909090909088 OK
    tau 0.05 in diff ci chk [-0.231, 0.375] harn [-0.231, 0.375] OK
    tau 0.05 cf diff chk 0.09090909090909088 harn 0.09090909090909088 OK
    tau 0.05 cf diff ci chk [-0.231, 0.375] harn [-0.231, 0.375] OK
    tau 0.1 in diff chk 0.0 harn 0.0 OK
    tau 0.1 in diff ci chk [-0.25, 0.333] harn [-0.25, 0.333] OK
    tau 0.1 cf diff chk 0.09090909090909088 harn 0.09090909090909088 OK
    tau 0.1 cf diff ci chk [-0.25, 0.357] harn [-0.25, 0.357] OK
osverify-length
    tau 0.05 in diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.05 in diff ci chk [0.061, 0.75] harn [0.061, 0.75] OK
    tau 0.05 cf diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.05 cf diff ci chk [0.0, 0.75] harn [0.0, 0.75] OK
    tau 0.1 in diff chk 0.36363636363636365 harn 0.36363636363636365 OK
    tau 0.1 in diff ci chk [0.0, 0.727] harn [0.0, 0.727] OK
    tau 0.1 cf diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.1 cf diff ci chk [0.0, 0.727] harn [0.0, 0.727] OK
osverify-cue
    tau 0.05 in diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.05 in diff ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    tau 0.05 cf diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.05 cf diff ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    tau 0.1 in diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.1 in diff ci chk [0.154, 0.75] harn [0.154, 0.75] OK
    tau 0.1 cf diff chk 0.45454545454545453 harn 0.45454545454545453 OK
    tau 0.1 cf diff ci chk [0.154, 0.75] harn [0.154, 0.75] OK
referee-voting checker-only (not a baseline pair)
referee-length checker-only (not a baseline pair)
referee-cue checker-only (not a baseline pair)
voting-length checker-only (not a baseline pair)
voting-cue checker-only (not a baseline pair)
length-cue checker-only (not a baseline pair)
decisions (checker, section-5 clauses a+b only): {'osverify-referee': 'UNCLEAR', 'osverify-voting': 'UNCLEAR'}
MISMATCHES 0
