{"n_reviews": 74380, "n_reviews_ge3": 72828, "opening": [{"key": "problem_framing", "first_share": 0.0552, "last_share": 0.0327, "overall_share": 0.0296, "open_lift": 1.865, "close_lift": 1.105}, {"key": "novelty", "first_share": 0.0437, "last_share": 0.0249, "overall_share": 0.0279, "open_lift": 1.57, "close_lift": 0.893}, {"key": "method_design", "first_share": 0.2001, "last_share": 0.1261, "overall_share": 0.1464, "open_lift": 1.366, "close_lift": 0.861}, {"key": "theory", "first_share": 0.1793, "last_share": 0.1079, "overall_share": 0.1392, "open_lift": 1.288, "close_lift": 0.775}, {"key": "empirical_scope", "first_share": 0.2376, "last_share": 0.1875, "overall_share": 0.2179, "open_lift": 1.091, "close_lift": 0.861}, {"key": "compute_cost", "first_share": 0.0613, "last_share": 0.0817, "overall_share": 0.0817, "open_lift": 0.75, "close_lift": 1.0}, {"key": "related_work", "first_share": 0.0188, "last_share": 0.0417, "overall_share": 0.0251, "open_lift": 0.747, "close_lift": 1.662}, {"key": "clarity", "first_share": 0.0557, "last_share": 0.1454, "overall_share": 0.0804, "open_lift": 0.694, "close_lift": 1.809}, {"key": "stats_metrics", "first_share": 0.0627, "last_share": 0.1007, "overall_share": 0.1001, "open_lift": 0.626, "close_lift": 1.005}, {"key": "baselines_ablations", "first_share": 0.0462, "last_share": 0.0688, "overall_share": 0.0797, "open_lift": 0.58, "close_lift": 0.862}, {"key": "reproducibility", "first_share": 0.0115, "last_share": 0.0296, "overall_share": 0.0209, "open_lift": 0.552, "close_lift": 1.413}, {"key": "robustness_sensitivity", "first_share": 0.0278, "last_share": 0.0531, "overall_share": 0.0511, "open_lift": 0.544, "close_lift": 1.04}], "position": {"bins": 5, "neg_rate": [0.7033, 0.7695, 0.8184, 0.7611, 0.6044], "pos_rate": [0.2646, 0.1402, 0.0617, 0.054, 0.1288], "cat_share": [{"clarity": 0.0562, "theory": 0.1791, "problem_framing": 0.0484, "method_design": 0.1906, "empirical_scope": 0.2364, "baselines_ablations": 0.0527, "robustness_sensitivity": 0.0305, "novelty": 0.04, "reproducibility": 0.0131, "stats_metrics": 0.0711, "compute_cost": 0.0634, "related_work": 0.0185}, {"empirical_scope": 0.2332, "baselines_ablations": 0.0984, "method_design": 0.1453, "theory": 0.1461, "robustness_sensitivity": 0.0471, "stats_metrics": 0.1136, "clarity": 0.0541, "compute_cost": 0.0804, "reproducibility": 0.017, "novelty": 0.0267, "problem_framing": 0.0203, "related_work": 0.0177}, {"theory": 0.1328, "method_design": 0.1332, "empirical_scope": 0.2223, "robustness_sensitivity": 0.059, "clarity": 0.0627, "baselines_ablations": 0.0998, "compute_cost": 0.0908, "related_work": 0.0204, "stats_metrics": 0.1148, "problem_framing": 0.0197, "novelty": 0.0241, "reproducibility": 0.0204}, {"empirical_scope": 0.2126, "clarity": 0.078, "theory": 0.1255, "novelty": 0.0227, "robustness_sensitivity": 0.0646, "baselines_ablations": 0.0906, "compute_cost": 0.0936, "related_work": 0.0253, "stats_metrics": 0.1103, "method_design": 0.1315, "reproducibility": 0.0232, "problem_framing": 0.0221}, {"method_design": 0.1262, "theory": 0.1133, "clarity": 0.1313, "empirical_scope": 0.1921, "baselines_ablations": 0.0711, "related_work": 0.0384, "reproducibility": 0.0294, "compute_cost": 0.0851, "robustness_sensitivity": 0.058, "problem_framing": 0.0306, "stats_metrics": 0.1007, "novelty": 0.0238}]}, "transitions": [{"a": "clarity", "b": "empirical_scope", "n": 3933, "lift": 0.83}, {"a": "empirical_scope", "b": "empirical_scope", "n": 19469, "lift": 1.207}, {"a": "empirical_scope", "b": "method_design", "n": 10153, "lift": 0.999}, {"a": "clarity", "b": "baselines_ablations", "n": 1717, "lift": 0.888}, {"a": "baselines_ablations", "b": "clarity", "n": 2511, "lift": 1.059}, {"a": "clarity", "b": "theory", "n": 2892, "lift": 1.0}, {"a": "theory", "b": "method_design", "n": 5294, "lift": 0.801}, {"a": "method_design", "b": "theory", "n": 5426, "lift": 0.82}, {"a": "theory", "b": "theory", "n": 12321, "lift": 1.924}, {"a": "problem_framing", "b": "method_design", "n": 1516, "lift": 1.159}, {"a": "method_design", "b": "baselines_ablations", "n": 4475, "lift": 1.012}, {"a": "baselines_ablations", "b": "method_design", "n": 3291, "lift": 0.885}, {"a": "method_design", "b": "method_design", "n": 9649, "lift": 1.413}, {"a": "method_design", "b": "novelty", "n": 1157, "lift": 0.936}, {"a": "novelty", "b": "clarity", "n": 845, "lift": 1.027}, {"a": "clarity", "b": "method_design", "n": 2492, "lift": 0.835}, {"a": "empirical_scope", "b": "baselines_ablations", "n": 6805, "lift": 1.034}, {"a": "baselines_ablations", "b": "theory", "n": 2905, "lift": 0.807}, {"a": "theory", "b": "empirical_scope", "n": 8871, "lift": 0.846}, {"a": "empirical_scope", "b": "robustness_sensitivity", "n": 4390, "lift": 1.034}, {"a": "robustness_sensitivity", "b": "empirical_scope", "n": 3673, "lift": 1.011}, {"a": "robustness_sensitivity", "b": "novelty", "n": 332, "lift": 0.801}, {"a": "novelty", "b": "robustness_sensitivity", "n": 431, "lift": 0.799}, {"a": "baselines_ablations", "b": "related_work", "n": 737, "lift": 1.007}, {"a": "related_work", "b": "reproducibility", "n": 213, "lift": 1.285}, {"a": "robustness_sensitivity", "b": "robustness_sensitivity", "n": 1206, "lift": 1.26}, {"a": "robustness_sensitivity", "b": "clarity", "n": 1240, "lift": 0.849}, {"a": "clarity", "b": "compute_cost", "n": 1346, "lift": 0.704}, {"a": "compute_cost", "b": "robustness_sensitivity", "n": 1987, "lift": 1.287}, {"a": "novelty", "b": "stats_metrics", "n": 1187, "lift": 1.142}, {"a": "stats_metrics", "b": "compute_cost", "n": 2797, "lift": 0.965}, {"a": "compute_cost", "b": "theory", "n": 3268, "lift": 0.913}, {"a": "problem_framing", "b": "clarity", "n": 895, "lift": 1.073}, {"a": "clarity", "b": "clarity", "n": 3392, "lift": 1.782}, {"a": "reproducibility", "b": "baselines_ablations", "n": 524, "lift": 0.94}, {"a": "baselines_ablations", "b": "baselines_ablations", "n": 2490, "lift": 1.034}, {"a": "baselines_ablations", "b": "empirical_scope", "n": 5802, "lift": 0.984}, {"a": "empirical_scope", "b": "compute_cost", "n": 6656, "lift": 1.023}, {"a": "method_design", "b": "empirical_scope", "n": 10857, "lift": 1.002}, {"a": "empirical_scope", "b": "related_work", "n": 1687, "lift": 0.843}, {"a": "related_work", "b": "compute_cost", "n": 453, "lift": 0.73}, {"a": "empirical_scope", "b": "stats_metrics", "n": 8014, "lift": 0.979}, {"a": "stats_metrics", "b": "empirical_scope", "n": 6666, "lift": 0.928}, {"a": "baselines_ablations", "b": "compute_cost", "n": 2919, "lift": 1.226}, {"a": "compute_cost", "b": "empirical_scope", "n": 6036, "lift": 1.03}, {"a": "method_design", "b": "compute_cost", "n": 4217, "lift": 0.964}, {"a": "compute_cost", "b": "compute_cost", "n": 2822, "lift": 1.193}, {"a": "empirical_scope", "b": "theory", "n": 7693, "lift": 0.782}, {"a": "method_design", "b": "reproducibility", "n": 962, "lift": 0.824}, {"a": "reproducibility", "b": "compute_cost", "n": 516, "lift": 0.936}, {"a": "compute_cost", "b": "method_design", "n": 3708, "lift": 1.004}, {"a": "baselines_ablations", "b": "problem_framing", "n": 597, "lift": 0.903}, {"a": "problem_framing", "b": "empirical_scope", "n": 1998, "lift": 0.963}, {"a": "empirical_scope", "b": "clarity", "n": 5494, "lift": 0.848}, {"a": "compute_cost", "b": "clarity", "n": 1989, "lift": 0.844}, {"a": "clarity", "b": "stats_metrics", "n": 2394, "lift": 0.995}, {"a": "theory", "b": "robustness_sensitivity", "n": 2792, "lift": 1.011}, {"a": "robustness_sensitivity", "b": "stats_metrics", "n": 1757, "lift": 0.952}, {"a": "stats_metrics", "b": "theory", "n": 3289, "lift": 0.75}, {"a": "related_work", "b": "clarity", "n": 1106, "lift": 1.789}, {"a": "clarity", "b": "novelty", "n": 654, "lift": 1.211}, {"a": "novelty", "b": "theory", "n": 1150, "lift": 0.92}, {"a": "reproducibility", "b": "stats_metrics", "n": 841, "lift": 1.213}, {"a": "stats_metrics", "b": "stats_metrics", "n": 4628, "lift": 1.269}, {"a": "compute_cost", "b": "related_work", "n": 616, "lift": 0.847}, {"a": "robustness_sensitivity", "b": "related_work", "n": 389, "lift": 0.862}, {"a": "method_design", "b": "robustness_sensitivity", "n": 2622, "lift": 0.919}, {"a": "compute_cost", "b": "baselines_ablations", "n": 2588, "lift": 1.082}, {"a": "method_design", "b": "clarity", "n": 3518, "lift": 0.808}, {"a": "method_design", "b": "stats_metrics", "n": 5560, "lift": 1.011}, {"a": "stats_metrics", "b": "baselines_ablations", "n": 3091, "lift": 1.055}, {"a": "stats_metrics", "b": "clarity", "n": 3185, "lift": 1.104}, {"a": "empirical_scope", "b": "novelty", "n": 1725, "lift": 0.938}, {"a": "novelty", "b": "method_design", "n": 1246, "lift": 0.965}, {"a": "stats_metrics", "b": "method_design", "n": 4206, "lift": 0.929}, {"a": "stats_metrics", "b": "robustness_sensitivity", "n": 1885, "lift": 0.997}, {"a": "robustness_sensitivity", "b": "baselines_ablations", "n": 1421, "lift": 0.958}, {"a": "clarity", "b": "problem_framing", "n": 769, "lift": 1.448}, {"a": "robustness_sensitivity", "b": "theory", "n": 2330, "lift": 1.05}, {"a": "theory", "b": "baselines_ablations", "n": 3839, "lift": 0.897}, {"a": "empirical_scope", "b": "problem_framing", "n": 1687, "lift": 0.933}, {"a": "clarity", "b": "related_work", "n": 1124, "lift": 1.912}, {"a": "clarity", "b": "robustness_sensitivity", "n": 901, "lift": 0.722}, {"a": "related_work", "b": "theory", "n": 857, "lift": 0.913}, {"a": "theory", "b": "stats_metrics", "n": 4347, "lift": 0.816}, {"a": "novelty", "b": "compute_cost", "n": 794, "lift": 0.961}, {"a": "problem_framing", "b": "baselines_ablations", "n": 781, "lift": 0.922}, {"a": "theory", "b": "clarity", "n": 3942, "lift": 0.935}, {"a": "related_work", "b": "robustness_sensitivity", "n": 285, "lift": 0.704}, {"a": "stats_metrics", "b": "reproducibility", "n": 966, "lift": 1.248}, {"a": "reproducibility", "b": "method_design", "n": 744, "lift": 0.864}, {"a": "method_design", "b": "problem_framing", "n": 1210, "lift": 0.996}, {"a": "theory", "b": "problem_framing", "n": 887, "lift": 0.754}, {"a": "problem_framing", "b": "theory", "n": 1140, "lift": 0.9}, {"a": "theory", "b": "compute_cost", "n": 3922, "lift": 0.926}, {"a": "compute_cost", "b": "reproducibility", "n": 665, "lift": 1.053}, {"a": "robustness_sensitivity", "b": "reproducibility", "n": 387, "lift": 0.988}, {"a": "reproducibility", "b": "empirical_scope", "n": 1271, "lift": 0.931}, {"a": "theory", "b": "related_work", "n": 1088, "lift": 0.835}, {"a": "robustness_sensitivity", "b": "method_design", "n": 2115, "lift": 0.923}, {"a": "empirical_scope", "b": "reproducibility", "n": 1740, "lift": 1.001}, {"a": "related_work", "b": "novelty", "n": 211, "lift": 1.203}, {"a": "stats_metrics", "b": "related_work", "n": 1007, "lift": 1.13}, {"a": "stats_metrics", "b": "problem_framing", "n": 968, "lift": 1.203}, {"a": "problem_framing", "b": "compute_cost", "n": 692, "lift": 0.826}, {"a": "baselines_ablations", "b": "robustness_sensitivity", "n": 1659, "lift": 1.068}, {"a": "baselines_ablations", "b": "stats_metrics", "n": 3214, "lift": 1.073}, {"a": "related_work", "b": "baselines_ablations", "n": 565, "lift": 0.9}, {"a": "robustness_sensitivity", "b": "compute_cost", "n": 1835, "lift": 1.251}, {"a": "problem_framing", "b": "stats_metrics", "n": 1108, "lift": 1.051}, {"a": "clarity", "b": "reproducibility", "n": 569, "lift": 1.115}, {"a": "novelty", "b": "baselines_ablations", "n": 1003, "lift": 1.2}, {"a": "related_work", "b": "stats_metrics", "n": 852, "lift": 1.091}, {"a": "baselines_ablations", "b": "novelty", "n": 716, "lift": 1.065}, {"a": "reproducibility", "b": "related_work", "n": 241, "lift": 1.422}, {"a": "problem_framing", "b": "novelty", "n": 351, "lift": 1.483}, {"a": "novelty", "b": "related_work", "n": 277, "lift": 1.09}, {"a": "method_design", "b": "related_work", "n": 1091, "lift": 0.811}, {"a": "novelty", "b": "problem_framing", "n": 324, "lift": 1.411}, {"a": "related_work", "b": "method_design", "n": 833, "lift": 0.859}, {"a": "compute_cost", "b": "stats_metrics", "n": 2548, "lift": 0.856}, {"a": "reproducibility", "b": "theory", "n": 547, "lift": 0.656}, {"a": "related_work", "b": "problem_framing", "n": 240, "lift": 1.392}, {"a": "theory", "b": "reproducibility", "n": 747, "lift": 0.661}, {"a": "novelty", "b": "empirical_scope", "n": 1940, "lift": 0.947}, {"a": "stats_metrics", "b": "novelty", "n": 940, "lift": 1.148}, {"a": "related_work", "b": "empirical_scope", "n": 1271, "lift": 0.826}, {"a": "theory", "b": "novelty", "n": 1083, "lift": 0.905}, {"a": "baselines_ablations", "b": "reproducibility", "n": 781, "lift": 1.229}, {"a": "related_work", "b": "related_work", "n": 317, "lift": 1.66}, {"a": "compute_cost", "b": "problem_framing", "n": 541, "lift": 0.823}, {"a": "compute_cost", "b": "novelty", "n": 685, "lift": 1.025}, {"a": "reproducibility", "b": "reproducibility", "n": 262, "lift": 1.78}, {"a": "problem_framing", "b": "problem_framing", "n": 310, "lift": 1.332}, {"a": "problem_framing", "b": "robustness_sensitivity", "n": 373, "lift": 0.682}, {"a": "novelty", "b": "reproducibility", "n": 225, "lift": 1.02}, {"a": "robustness_sensitivity", "b": "problem_framing", "n": 335, "lift": 0.822}, {"a": "problem_framing", "b": "related_work", "n": 337, "lift": 1.308}, {"a": "problem_framing", "b": "reproducibility", "n": 220, "lift": 0.983}, {"a": "reproducibility", "b": "clarity", "n": 733, "lift": 1.336}, {"a": "reproducibility", "b": "robustness_sensitivity", "n": 373, "lift": 1.037}], "arcs": [{"first": "negative", "last": "negative", "n": 31486, "share": 0.4323}, {"first": "negative", "last": "hedged", "n": 11851, "share": 0.1627}, {"first": "positive", "last": "negative", "n": 10538, "share": 0.1447}, {"first": "negative", "last": "positive", "n": 9078, "share": 0.1246}, {"first": "positive", "last": "hedged", "n": 7430, "share": 0.102}, {"first": "positive", "last": "positive", "n": 1071, "share": 0.0147}, {"first": "hedged", "last": "negative", "n": 789, "share": 0.0108}, {"first": "hedged", "last": "hedged", "n": 408, "share": 0.0056}, {"first": "hedged", "last": "positive", "n": 177, "share": 0.0024}], "lengths": {"1": 31, "2": 1521, "3": 7930, "4": 14276, "5": 17312, "6": 14188, "7": 8646, "8": 5387, "9": 2440, "10": 1480, "11": 581, "12": 317, "13": 137, "14": 62, "15": 72}}