{
 "n_tasks": 60,
 "models": {
  "@cf/meta/llama-3.1-8b-instruct-fast": {
   "n": 60,
   "pending": 0,
   "pass1": 2,
   "best_of_4": 4,
   "repair_4": 4,
   "repair_minus_sample": {
    "pts": 0.0,
    "ci95": [
     -6.7,
     6.7
    ],
    "repair_only": 2,
    "sample_only": 2
   },
   "avg_calls": {
    "sample": 4,
    "repair": 3.83
   },
   "by_set": {
    "textbook": {
     "n": 16,
     "pass1": 1,
     "best_of_4": 1,
     "repair_4": 3
    },
    "substituted": {
     "n": 16,
     "pass1": 0,
     "best_of_4": 0,
     "repair_4": 0
    },
    "gen_small": {
     "n": 10,
     "pass1": 1,
     "best_of_4": 3,
     "repair_4": 1
    },
    "gen_medium": {
     "n": 10,
     "pass1": 0,
     "best_of_4": 0,
     "repair_4": 0
    },
    "gen_large": {
     "n": 8,
     "pass1": 0,
     "best_of_4": 0,
     "repair_4": 0
    }
   },
   "textbook_vs_substituted": {
    "pairs": 16,
    "pass1": {
     "textbook": 1,
     "substituted": 0,
     "diff_pts": 6.2,
     "ci95": [
      0.0,
      18.8
     ]
    },
    "repair_4": {
     "textbook": 3,
     "substituted": 0,
     "diff_pts": 18.8,
     "ci95": [
      0.0,
      37.5
     ]
    }
   },
   "first_attempt_outcome": {
    "subproof/scope": 29,
    "rule misapplied": 22,
    "accepted": 2,
    "numbering/references": 6,
    "other": 1
   },
   "all_sample_outcomes": {
    "subproof/scope": 120,
    "rule misapplied": 89,
    "numbering/references": 23,
    "accepted": 7,
    "other": 1
   },
   "repair_fixed_after": {
    "subproof/scope": 2
   },
   "per_task": {
    "tb_identity": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 2: line 1 is inside a closed subproof and cannot be used here"
    },
    "tb_weakening": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "tb_contraposition": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: reit does not justify 'p -> q' \u2014 reit must repeat the referenced formula exactly"
    },
    "tb_hyp_syllogism": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "tb_and_comm": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_or_comm": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "tb_exportation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify '(p & q -> r) -> p -> q' \u2014 reit must repeat the referenced formula exactly"
    },
    "tb_importation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 6: line 2 is inside a closed subproof and cannot be used here"
    },
    "tb_dn_intro": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: expected a subproof range like 3-5, got 1"
    },
    "tb_demorgan_or": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "tb_non_contradiction": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 5: notI does not justify '~~(p & ~p)' \u2014 subproof assuming A and ending in F gives ~A"
    },
    "tb_excluded_middle": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: reit does not justify '~p' \u2014 reit must repeat the referenced formula exactly"
    },
    "tb_peirce": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify '(p -> q) -> p' \u2014 reit must repeat the referenced formula exactly"
    },
    "tb_distrib": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: orE takes 3 reference(s), got 2"
    },
    "tb_demorgan_and_cl": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: subproof 2-1 is still open; this line must be one level shallower"
    },
    "tb_modus_tollens": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_identity": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: notI does not justify 'F' \u2014 subproof assuming A and ending in F gives ~A"
    },
    "sub_weakening": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "sub_contraposition": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: notE does not justify 'u' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_hyp_syllogism": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: andE1 does not justify '~(s & u) -> s & t' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "sub_and_comm": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: line 1 is inside a closed subproof and cannot be used here"
    },
    "sub_or_comm": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 6: orE does not justify '~u' \u2014 needs A | B, a subproof A..C and a subproof B..C, concluding C"
    },
    "sub_exportation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: expected a subproof range like 3-5, got 1"
    },
    "sub_importation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: trailing input in '(u -> v) -> t & ~v assume'"
    },
    "sub_dn_intro": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: expected a subproof range like 3-5, got 3"
    },
    "sub_demorgan_or": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "sub_non_contradiction": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: notE takes 2 reference(s), got 1"
    },
    "sub_excluded_middle": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_peirce": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify '(t & ~v -> ~u) -> t & ~v' \u2014 reit must repeat the referenced formula exactly"
    },
    "sub_distrib": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: andE2 does not justify '~v' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "sub_demorgan_and_cl": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: 2-3 is not a subproof (it must start at an 'assume' line)"
    },
    "sub_modus_tollens": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "gen_small_00": {
     "p1": false,
     "bo4": true,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify 'b & a' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_small_01": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: depth 2 but only 1 subproof(s) open \u2014 open a subproof with 'assume'"
    },
    "gen_small_02": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: subproof starting at 1 ends at line 4, not 3"
    },
    "gen_small_03": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_small_04": {
     "p1": false,
     "bo4": true,
     "r4": false,
     "rc": 4,
     "err0": "line 5: impE does not justify 'c -> b' \u2014 needs A -> B (first ref) and A (second ref)"
    },
    "gen_small_05": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_small_06": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_small_07": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: impE takes 2 reference(s), got 1"
    },
    "gen_small_08": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: orE takes 3 reference(s), got 2"
    },
    "gen_small_09": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_medium_00": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: impE takes 2 reference(s), got 1"
    },
    "gen_medium_01": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_02": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: andE1 does not justify 'a' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "gen_medium_03": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_04": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: andE2 does not justify 'c & a' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "gen_medium_05": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_06": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: impE takes 2 reference(s), got 1"
    },
    "gen_medium_07": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: reit does not justify 'a | (c -> c)' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_medium_08": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: andE1 does not justify '~a -> a -> c' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "gen_medium_09": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_large_00": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: line 5 is inside a closed subproof and cannot be used here"
    },
    "gen_large_01": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_large_02": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify '~b | (a -> b) | c' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_large_03": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: orI2 does not justify 'a | c' \u2014 orI1 turns A into A | B, orI2 turns B into A | B"
    },
    "gen_large_04": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: notE does not justify '~c & ~c' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "gen_large_05": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: reit does not justify 'b' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_large_06": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_large_07": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: reit does not justify '~a -> (a -> b) & a' \u2014 reit must repeat the referenced formula exactly"
    }
   },
   "lenient_order": {
    "pass1": 2,
    "best_of_4": 4,
    "repair_4": 4,
    "repair_minus_sample_pts": 0.0,
    "ci95": [
     -6.7,
     6.7
    ]
   }
  },
  "@cf/meta/llama-3.3-70b-instruct-fp8-fast": {
   "n": 53,
   "pending": 7,
   "pass1": 14,
   "best_of_4": 16,
   "repair_4": 23,
   "repair_minus_sample": {
    "pts": 13.2,
    "ci95": [
     3.8,
     24.5
    ],
    "repair_only": 8,
    "sample_only": 1
   },
   "avg_calls": {
    "sample": 4,
    "repair": 2.94
   },
   "by_set": {
    "textbook": {
     "n": 16,
     "pass1": 5,
     "best_of_4": 6,
     "repair_4": 9
    },
    "substituted": {
     "n": 16,
     "pass1": 2,
     "best_of_4": 2,
     "repair_4": 4
    },
    "gen_small": {
     "n": 10,
     "pass1": 7,
     "best_of_4": 7,
     "repair_4": 9
    },
    "gen_medium": {
     "n": 10,
     "pass1": 0,
     "best_of_4": 1,
     "repair_4": 1
    },
    "gen_large": {
     "n": 1,
     "pass1": 0,
     "best_of_4": 0,
     "repair_4": 0
    }
   },
   "textbook_vs_substituted": {
    "pairs": 16,
    "pass1": {
     "textbook": 5,
     "substituted": 2,
     "diff_pts": 18.8,
     "ci95": [
      0.0,
      37.5
     ]
    },
    "repair_4": {
     "textbook": 9,
     "substituted": 4,
     "diff_pts": 31.2,
     "ci95": [
      12.5,
      56.2
     ]
    }
   },
   "first_attempt_outcome": {
    "accepted": 14,
    "subproof/scope": 18,
    "rule misapplied": 20,
    "numbering/references": 1
   },
   "all_sample_outcomes": {
    "accepted": 55,
    "subproof/scope": 77,
    "rule misapplied": 77,
    "numbering/references": 3
   },
   "repair_fixed_after": {
    "subproof/scope": 4,
    "rule misapplied": 5
   },
   "per_task": {
    "tb_identity": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_weakening": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 5: subproof 1-4 is still open; this line must be one level shallower"
    },
    "tb_contraposition": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_hyp_syllogism": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: line 1 is inside a closed subproof and cannot be used here"
    },
    "tb_and_comm": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_or_comm": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 3,
     "err0": "line 5: orE does not justify 'q | p' \u2014 needs A | B, a subproof A..C and a subproof B..C, concluding C"
    },
    "tb_exportation": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_importation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: depth 3 but only 2 subproof(s) open \u2014 open a subproof with 'assume'"
    },
    "tb_dn_intro": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "tb_demorgan_or": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "tb_non_contradiction": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "tb_excluded_middle": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: notE does not justify '~p & ~~p' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "tb_peirce": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "tb_distrib": {
     "p1": false,
     "bo4": true,
     "r4": true,
     "rc": 2,
     "err0": "line 8: andI does not justify 'p & r' \u2014 expected the conjunction of the two referenced lines, in order"
    },
    "tb_demorgan_and_cl": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "tb_modus_tollens": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 6: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_identity": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 3: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_weakening": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 6: line 2 is inside a closed subproof and cannot be used here"
    },
    "sub_contraposition": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: subproof 3-3 is still open; this line must be one level shallower"
    },
    "sub_hyp_syllogism": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: andI does not justify 's & u' \u2014 expected the conjunction of the two referenced lines, in order"
    },
    "sub_and_comm": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "sub_or_comm": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: orI2 does not justify 'u -> v' \u2014 orI1 turns A into A | B, orI2 turns B into A | B"
    },
    "sub_exportation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: line 5 is inside a closed subproof and cannot be used here"
    },
    "sub_importation": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: line 1 is inside a closed subproof and cannot be used here"
    },
    "sub_dn_intro": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 5: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_demorgan_or": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 2: notE takes 2 reference(s), got 1"
    },
    "sub_non_contradiction": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "sub_excluded_middle": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: reit does not justify '~(s & u)' \u2014 reit must repeat the referenced formula exactly"
    },
    "sub_peirce": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: orI2 does not justify '~t | ~v' \u2014 orI1 turns A into A | B, orI2 turns B into A | B"
    },
    "sub_distrib": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 8: andE1 does not justify 's | t' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "sub_demorgan_and_cl": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 7: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "sub_modus_tollens": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 6: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "gen_small_00": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_01": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 2,
     "err0": "line 4: botE does not justify 'b | c' \u2014 referenced line must be F"
    },
    "gen_small_02": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_03": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_04": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_05": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 4,
     "err0": "line 4: notE does not justify 'F' \u2014 needs A (first ref) and ~A (second ref), concludes F"
    },
    "gen_small_06": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "gen_small_07": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_08": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_small_09": {
     "p1": true,
     "bo4": true,
     "r4": true,
     "rc": 1,
     "err0": "QED"
    },
    "gen_medium_00": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 6: line 4 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_01": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: impI does not justify 'c | a' \u2014 subproof assuming A and ending in B gives A -> B"
    },
    "gen_medium_02": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: line 2 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_03": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 4: reit does not justify 'a' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_medium_04": {
     "p1": false,
     "bo4": true,
     "r4": false,
     "rc": 4,
     "err0": "line 3: andE2 does not justify 'c & a' \u2014 referenced line must be A & B; andE1 gives A, andE2 gives B"
    },
    "gen_medium_05": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_06": {
     "p1": false,
     "bo4": false,
     "r4": true,
     "rc": 3,
     "err0": "line 3: andI does not justify 'c & a | a & a' \u2014 expected the conjunction of the two referenced lines, in order"
    },
    "gen_medium_07": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: reit does not justify 'a' \u2014 reit must repeat the referenced formula exactly"
    },
    "gen_medium_08": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    },
    "gen_medium_09": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: impE does not justify 'c' \u2014 needs A -> B (first ref) and A (second ref)"
    },
    "gen_large_00": {
     "p1": false,
     "bo4": false,
     "r4": false,
     "rc": 4,
     "err0": "line 3: line 1 is inside a closed subproof and cannot be used here"
    }
   },
   "lenient_order": {
    "pass1": 17,
    "best_of_4": 20,
    "repair_4": 24,
    "repair_minus_sample_pts": 7.5,
    "ci95": [
     0.0,
     17.0
    ]
   }
  }
 },
 "kernel_audit": {
  "audited": 472,
  "good": 4,
  "bad": 3,
  "mutants": 8000,
  "mutants_rejected": 6578,
  "mutants_accepted": 1422
 }
}