Patient Selection

  • Patients meeting inclusion criteria: 16242
  • Reasons for exclusion:
    • Pregnant: 203
    • Prisoner: 77
    • ICU stay < 12 hours: 1077
    • Diagnosis coding contained both ICD-9-CM and ICD-10-CM: 548
    • Missing data needed to calculate risk scores: 9165
    • Re-encounters (only included each patient once) : 1782
  • Total number of eligible patients remaining to sample from: 3390

Primary endpoint

APACHE II

  • Mean and median score for each comorbidity set
scores <- ls(pattern = "score_apache2")
all_scores <- map(scores, ~mutate(get(.x), score = .x)) %>%
    map(~select(.x, pie.id, score, apache2)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "score_apache2_", replacement = "") 
man <- filter(all_scores, score == "manual")$apache2
result <- all_scores %>%
    select(-pie.id) %>%
    group_by(score) %>%
    summarize_all(funs(mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(result)
score mean sd median IQR_low IQR_high
ahrq 20.80000 7.161100 20 15 24
ahrq_drg 20.43333 7.291431 20 15 24
ek 20.73333 7.182527 20 15 24
ek_drg 20.40000 7.279411 20 15 24
elixhauser 20.76667 7.150600 20 15 24
elixhauser_drg 20.40000 7.279411 20 15 24
manual 20.36667 7.097306 20 15 24
quan 20.76667 7.150600 20 15 24
quan_drg 20.43333 7.249469 20 15 24
  • Comparison of scores
    • Results run using both the t-test and Wilcoxon signed rank test (assumes non-parametric data, is more difficult to achieve significance)
t <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(t.test(.$apache2, man, paired = TRUE)))
knitr::kable(t)
score estimate statistic p.value parameter conf.low conf.high method alternative
ahrq 0.4333333 1.5669418 0.1224765 59 -0.1200363 0.9867030 Paired t-test two.sided
ahrq_drg 0.0666667 0.2380106 0.8126975 59 -0.4938112 0.6271445 Paired t-test two.sided
ek 0.3666667 1.3181069 0.1925626 59 -0.1899637 0.9232970 Paired t-test two.sided
ek_drg 0.0333333 0.1181156 0.9063773 59 -0.5313665 0.5980332 Paired t-test two.sided
elixhauser 0.4000000 1.4315041 0.1575620 59 -0.1591309 0.9591309 Paired t-test two.sided
elixhauser_drg 0.0333333 0.1181156 0.9063773 59 -0.5313665 0.5980332 Paired t-test two.sided
quan 0.4000000 1.4315041 0.1575620 59 -0.1591309 0.9591309 Paired t-test two.sided
quan_drg 0.0666667 0.2346551 0.8152889 59 -0.5018261 0.6351594 Paired t-test two.sided 
mw <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(wilcox.test(.$apache2, man, paired = TRUE, conf.int = TRUE, exact = FALSE)))
knitr::kable(mw)
score estimate statistic p.value conf.low conf.high method alternative
ahrq 2.0000007 117.0 0.1639934 -1.499960 3.500034 Wilcoxon signed rank test with continuity correction two.sided
ahrq_drg 0.0000427 78.0 0.9610189 -1.500066 3.499924 Wilcoxon signed rank test with continuity correction two.sided
ek 1.9999973 112.5 0.2341505 -1.499999 3.500062 Wilcoxon signed rank test with continuity correction two.sided
ek_drg -0.0000623 85.5 1.0000000 -1.500016 2.000013 Wilcoxon signed rank test with continuity correction two.sided
elixhauser 1.9999404 127.0 0.1916011 -1.499995 3.499992 Wilcoxon signed rank test with continuity correction two.sided
elixhauser_drg -0.0000623 85.5 1.0000000 -1.500016 2.000013 Wilcoxon signed rank test with continuity correction two.sided
quan 1.9999404 127.0 0.1916011 -1.499995 3.499992 Wilcoxon signed rank test with continuity correction two.sided
quan_drg 0.0000360 98.0 0.9174555 -1.500032 2.000026 Wilcoxon signed rank test with continuity correction two.sided
  • Compare differences in score from comorbidity sets vs. manual
    • Column num_diff indicates the number of patients with a score that differed from manual
    • Mean and median indicate the difference from the manual score
      • These were only calculated for those patients whose score was different from manual
      • If all patients were included, the number is skewed towards 0 by all the patients with the same scores
diff <- all_scores %>%
    spread(score, apache2) %>%
    mutate_at(vars(-pie.id, -manual), funs(abs(. - manual))) %>%
    gather(score, apache2, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    filter(apache2 > 0) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = length, mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(diff, caption = "Differences in APACHE II Scores")
score num_diff mean sd median IQR_low IQR_high
ahrq 18 3.666667 1.533930 5 2 5
ahrq_drg 17 3.764706 1.521899 5 2 5
ek 18 3.666667 1.533930 5 2 5
ek_drg 18 3.666667 1.533930 5 2 5
elixhauser 19 3.578947 1.538968 5 2 5
elixhauser_drg 18 3.666667 1.533930 5 2 5
quan 19 3.578947 1.538968 5 2 5
quan_drg 19 3.578947 1.538968 5 2 5

APACHE III

  • There were 10 patients with different comorbidities identified by ICD code vs. manual
    • Of these, only 6 had a different APACHE III score; the other 4 were all “elective” admissions where comorbidity score is ignored
    • 1 person was identified as being chronic HD by ICD code but not by manual review, accounting for the 7 patients where the APACHE III score by ICD varied from manual
scores <- ls(pattern = "score_apache3")
all_scores <- map(scores, ~mutate(get(.x), score = .x)) %>%
    map(~select(.x, pie.id, score, apache3)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "score_apache3_", replacement = "") 
man <- filter(all_scores, score == "manual")$apache3
result <- all_scores %>%
    select(-pie.id) %>%
    group_by(score) %>%
    summarize_all(funs(mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(result)
score mean sd median IQR_low IQR_high
ahrq 85.26667 29.43414 77.5 68.00 103.00
ahrq_drg 85.26667 29.43414 77.5 68.00 103.00
ek 85.26667 29.43414 77.5 68.00 103.00
ek_drg 85.26667 29.43414 77.5 68.00 103.00
elixhauser 85.26667 29.43414 77.5 68.00 103.00
elixhauser_drg 85.26667 29.43414 77.5 68.00 103.00
manual 84.08333 29.02383 77.5 66.75 98.25
quan 85.26667 29.43414 77.5 68.00 103.00
quan_drg 85.26667 29.43414 77.5 68.00 103.00
  • Comparison of scores
t <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(t.test(.$apache3, man, paired = TRUE)))
knitr::kable(t)
score estimate statistic p.value parameter conf.low conf.high method alternative
ahrq 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
ahrq_drg 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
ek 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
ek_drg 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
elixhauser 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
elixhauser_drg 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
quan 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided
quan_drg 1.183333 2.487694 0.0157028 59 0.2315103 2.135156 Paired t-test two.sided 
mw <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(wilcox.test(.$apache3, man, paired = TRUE, conf.int = TRUE, exact = FALSE)))
requested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievable
knitr::kable(mw)
score estimate statistic p.value conf.low conf.high method alternative
ahrq 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
ahrq_drg 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
ek 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
ek_drg 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
elixhauser 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
elixhauser_drg 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
quan 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
quan_drg 10.99996 28 0.0220149 6.999939 13.99999 Wilcoxon signed rank test with continuity correction two.sided
  • Differences in score
diff <- all_scores %>%
    spread(score, apache3) %>%
    mutate_at(vars(-pie.id, -manual), funs(abs(. - manual))) %>%
    gather(score, apache3, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    filter(apache3 > 0) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = length, mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(diff, caption = "Difference in APACHE III Scores")
score num_diff mean sd median IQR_low IQR_high
ahrq 7 10.14286 5.241774 11 7 13.5
ahrq_drg 7 10.14286 5.241774 11 7 13.5
ek 7 10.14286 5.241774 11 7 13.5
ek_drg 7 10.14286 5.241774 11 7 13.5
elixhauser 7 10.14286 5.241774 11 7 13.5
elixhauser_drg 7 10.14286 5.241774 11 7 13.5
quan 7 10.14286 5.241774 11 7 13.5
quan_drg 7 10.14286 5.241774 11 7 13.5

SAPS II

scores <- ls(pattern = "score_saps2")
all_scores <- map(scores, ~mutate(get(.x), score = .x)) %>%
    map(~select(.x, pie.id, score, saps2)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "score_saps2_", replacement = "") 
man <- filter(all_scores, score == "manual")$saps2
result <- all_scores %>%
    select(-pie.id) %>%
    group_by(score) %>%
    summarize_all(funs(mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(result)
score mean sd median IQR_low IQR_high
ahrq 37.53333 13.60018 36.5 27.75 45
ahrq_drg 37.53333 13.60018 36.5 27.75 45
ek 37.53333 13.60018 36.5 27.75 45
ek_drg 37.53333 13.60018 36.5 27.75 45
elixhauser 37.53333 13.60018 36.5 27.75 45
elixhauser_drg 37.53333 13.60018 36.5 27.75 45
manual 37.06667 13.76666 34.0 27.00 45
quan 37.53333 13.60018 36.5 27.75 45
quan_drg 37.53333 13.60018 36.5 27.75 45
  • Comparison of scores
t <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(t.test(.$saps2, man, paired = TRUE)))
knitr::kable(t)
score estimate statistic p.value parameter conf.low conf.high method alternative
ahrq 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
ahrq_drg 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
ek 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
ek_drg 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
elixhauser 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
elixhauser_drg 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
quan 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
quan_drg 0.4666667 1.759815 0.0836229 59 -0.0639562 0.9972896 Paired t-test two.sided
  • Confidence interval was unable to be calculated for Wilcoxon analysis here
mw <- all_scores[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(wilcox.test(.$saps2, man, paired = TRUE, exact = FALSE)))
knitr::kable(mw)
score statistic p.value method alternative
ahrq 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
ahrq_drg 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
ek 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
ek_drg 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
elixhauser 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
elixhauser_drg 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
quan 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
quan_drg 6 0.1735682 Wilcoxon signed rank test with continuity correction two.sided
  • Differences in score
diff <- all_scores %>%
    spread(score, saps2) %>%
    mutate_at(vars(-pie.id, -manual), funs(abs(. - manual))) %>%
    gather(score, saps2, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    filter(saps2 > 0) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = length, mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(diff, caption = "Difference in SAPS II Scores")
score num_diff mean sd median IQR_low IQR_high
ahrq 3 9.333333 0.5773503 9 9 9.5
ahrq_drg 3 9.333333 0.5773503 9 9 9.5
ek 3 9.333333 0.5773503 9 9 9.5
ek_drg 3 9.333333 0.5773503 9 9 9.5
elixhauser 3 9.333333 0.5773503 9 9 9.5
elixhauser_drg 3 9.333333 0.5773503 9 9 9.5
quan 3 9.333333 0.5773503 9 9 9.5
quan_drg 3 9.333333 0.5773503 9 9 9.5

Secondary endpoints

Predicted mortality

SAPS II

  • Mean and median risk of mortality (as a percent) for each comorbidity set
comorbid <- ls(pattern = "score_saps2_")
all_df <- map(comorbid, ~mutate(get(.x), score = .x)) %>%
    map(~select(.x, pie.id, score, saps2)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "score_saps2_", replacement = "") %>%
    distinct(pie.id, score, .keep_all = TRUE) %>%
    mutate(logit = -7.763 + 0.0737 * saps2 + 0.9971 * log(saps2 + 1),
           mortality = exp(logit) / (1 + exp(logit))) %>%
    select(pie.id, score, mortality) 
man <- filter(all_df, score == "manual")$mortality
result <- all_df %>%
    select(-pie.id) %>%
    group_by(score) %>%
    summarize_all(funs(mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(result)
score mean sd median IQR_low IQR_high
ahrq 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
ahrq_drg 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
ek 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
ek_drg 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
elixhauser 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
elixhauser_drg 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
manual 0.2441350 0.2260207 0.1545467 0.0793977 0.347746
quan 0.2498587 0.2238646 0.1887083 0.0856351 0.347746
quan_drg 0.2498587 0.2238646 0.1887083 0.0856351 0.347746 
t <- all_df[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(t.test(.$mortality, man, paired = TRUE)))
knitr::kable(t)
score estimate statistic p.value parameter conf.low conf.high method alternative
ahrq 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
ahrq_drg 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
ek 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
ek_drg 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
elixhauser 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
elixhauser_drg 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
quan 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided
quan_drg 0.0057237 1.678483 0.0985426 59 -0.0010998 0.0125472 Paired t-test two.sided 
mw <- all_df[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(wilcox.test(.$mortality, man, paired = TRUE, conf.int = TRUE, exact = FALSE)))
requested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievablerequested conf.level not achievable
knitr::kable(mw)
score estimate statistic p.value conf.low conf.high method alternative
ahrq 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
ahrq_drg 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
ek 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
ek_drg 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
elixhauser 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
elixhauser_drg 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
quan 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
quan_drg 0.1153318 6 0.1814492 0.0701968 0.1573562 Wilcoxon signed rank test with continuity correction two.sided
  • Difference in risk of mortality
    • Mean and median calculated using only those patients whose predicted mortality score was different from manual
diff <- all_df %>%
    spread(score, mortality) %>%
    mutate_at(vars(-pie.id, -manual), funs(abs(. - manual))) %>%
    gather(score, saps2, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    filter(saps2 > 0) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = length, mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(diff, caption = "Difference in Predicted Mortality using SAPS II")
score num_diff mean sd median IQR_low IQR_high
ahrq 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
ahrq_drg 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
ek 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
ek_drg 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
elixhauser 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
elixhauser_drg 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
quan 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591
quan_drg 3 0.1144744 0.0436883 0.1158702 0.0929876 0.1366591

Number of comorbidities

APACHE II

  • Mean represents the average number of comorbidities per patient
comorbid <- ls(pattern = "data_apache2_")
all_df <- map(comorbid, ~mutate(get(.x), score = .x)) %>%
    map(~distinct(.x, pie.id, score, liver, cardiovasc, respiratory, renal, immunocomp, comorbidity)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "data_apache2_", replacement = "") %>%
    mutate_if(is.logical, as.numeric) %>%
    by_row(function(x) sum(x[, 3:7], na.rm = TRUE), .collate = "rows", .to = "num_comorbid") %>%
    select(pie.id, score, num_comorbid) 
man <- filter(all_df, score == "manual")$num_comorbid
result <- all_df %>%
    select(-pie.id) %>%
    group_by(score) %>%
    summarize_all(funs(mean, sd, median, IQR_low = quantile(., 0.25), IQR_high = quantile(., 0.75)))
knitr::kable(result)
score mean sd median IQR_low IQR_high
ahrq 0.9166667 0.7656097 1 0 1
ahrq_drg 0.7666667 0.7448505 1 0 1
ek 0.8500000 0.7552079 1 0 1
ek_drg 0.7166667 0.7152496 1 0 1
elixhauser 0.8833333 0.7611692 1 0 1
elixhauser_drg 0.7333333 0.7333847 1 0 1
manual 0.6833333 0.7476895 1 0 1
quan 0.9000000 0.7745967 1 0 1
quan_drg 0.7666667 0.7448505 1 0 1 
t <- all_df[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(t.test(.$num_comorbid, man, paired = TRUE)))
knitr::kable(t)
score estimate statistic p.value parameter conf.low conf.high method alternative
ahrq 0.2333333 2.2905947 0.0255771 59 0.0295003 0.4371664 Paired t-test two.sided
ahrq_drg 0.0833333 0.7982162 0.4279463 59 -0.1255695 0.2922362 Paired t-test two.sided
ek 0.1666667 1.6923586 0.0958528 59 -0.0303951 0.3637285 Paired t-test two.sided
ek_drg 0.0333333 0.3308504 0.7419297 59 -0.1682679 0.2349346 Paired t-test two.sided
elixhauser 0.2000000 1.9406075 0.0570888 59 -0.0062236 0.4062236 Paired t-test two.sided
elixhauser_drg 0.0500000 0.4772832 0.6349238 59 -0.1596235 0.2596235 Paired t-test two.sided
quan 0.2166667 2.0862042 0.0412923 59 0.0088495 0.4244838 Paired t-test two.sided
quan_drg 0.0833333 0.7782991 0.4395029 59 -0.1309154 0.2975821 Paired t-test two.sided 
mw <- all_df[, -1] %>%
    filter(score != "manual") %>%
    group_by(score) %>%
    do(tidy(wilcox.test(.$num_comorbid, man, paired = TRUE, conf.int = TRUE, exact = FALSE)))
knitr::kable(mw)
score estimate statistic p.value conf.low conf.high method alternative
ahrq 0.5000436 293.5 0.0262454 3.80e-06 1.0000061 Wilcoxon signed rank test with continuity correction two.sided
ahrq_drg 0.0000655 220.5 0.4200794 -5.51e-05 0.9999098 Wilcoxon signed rank test with continuity correction two.sided
ek 0.0000028 253.0 0.0947684 -1.50e-05 0.9999502 Wilcoxon signed rank test with continuity correction two.sided
ek_drg 0.0000365 202.0 0.7422385 -2.53e-05 0.5000059 Wilcoxon signed rank test with continuity correction two.sided
elixhauser 0.4999231 281.0 0.0556365 -3.25e-05 1.0000382 Wilcoxon signed rank test with continuity correction two.sided
elixhauser_drg 0.0000114 208.5 0.6211835 -8.23e-05 0.5000579 Wilcoxon signed rank test with continuity correction two.sided
quan 0.4999387 305.0 0.0410004 -3.75e-05 1.0000033 Wilcoxon signed rank test with continuity correction two.sided
quan_drg 0.0000694 251.5 0.4313550 -2.36e-05 0.9999478 Wilcoxon signed rank test with continuity correction two.sided

Proportion with differing comorbidity

APACHE II

  • Proportion of patients where a comorbidity was identified by ICD or by manual, but not both
comorbid <- ls(pattern = "data_apache2_")
all_df <- map(comorbid, ~mutate(get(.x), score = .x)) %>%
    map(~distinct(.x, pie.id, score, comorbidity)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "data_apache2_", replacement = "") %>%
    spread(score, comorbidity) %>%
    mutate_at(vars(-pie.id, -manual), funs(. != manual)) %>%
    mutate_if(is.logical, as.numeric) %>%
    ungroup() %>%
    gather(score, comorbidity, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = sum(. > 0), proportion = mean))
knitr::kable(all_df)
score num_diff proportion
ahrq 18 0.3000000
ahrq_drg 17 0.2833333
ek 18 0.3000000
ek_drg 18 0.3000000
elixhauser 19 0.3166667
elixhauser_drg 18 0.3000000
quan 19 0.3166667
quan_drg 19 0.3166667

APACHE III

  • Proportion of patients where the primary comorbidity identified by ICD varied from primary comorbidity identified by manual
comorbid <- ls(pattern = "data_apache3_")
all_df <- map(comorbid, ~mutate(get(.x), score = .x)) %>%
    map(~distinct(.x, pie.id, score, comorbidity)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "data_apache3_", replacement = "") %>%
    spread(score, comorbidity, fill = "") %>%
    dmap(str_replace_all, pattern = "none", replacement = "") %>%
    mutate_at(vars(-pie.id, -manual), funs(. != manual)) %>%
    mutate_if(is.logical, as.numeric) %>%
    gather(score, comorbidity, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = sum(. > 0), proportion = mean))
knitr::kable(all_df)
score num_diff proportion
ahrq 10 0.1666667
ahrq_drg 10 0.1666667
ek 10 0.1666667
ek_drg 10 0.1666667
elixhauser 10 0.1666667
elixhauser_drg 10 0.1666667
quan 10 0.1666667
quan_drg 10 0.1666667

SAPS II

  • Proportion of patients where the primary comorbidity identified by ICD varied from primary comorbidity identified by manual
comorbid <- ls(pattern = "data_saps2_")
all_df <- map(comorbid, ~mutate(get(.x), score = .x)) %>%
    map(~distinct(.x, pie.id, score, comorbidity)) %>%
    map_df(~semi_join(.x, manual_patients, by = "pie.id")) %>%
    arrange(pie.id, score) %>%
    dmap_at("score", str_replace_all, pattern = "data_saps2_", replacement = "") %>%
    spread(score, comorbidity, fill = "") %>%
    dmap(str_replace_all, pattern = "none", replacement = "") %>%
    mutate_at(vars(-pie.id, -manual), funs(. != manual)) %>%
    mutate_if(is.logical, as.numeric) %>%
    gather(score, comorbidity, -pie.id, -manual) %>%
    select(-pie.id, -manual) %>%
    group_by(score) %>%
    summarize_all(funs(num_diff = sum(. > 0), proportion = mean))
knitr::kable(all_df)
score num_diff proportion
ahrq 3 0.05
ahrq_drg 3 0.05
ek 3 0.05
ek_drg 3 0.05
elixhauser 3 0.05
elixhauser_drg 3 0.05
quan 3 0.05
quan_drg 3 0.05
LS0tDQp0aXRsZTogVmFsaWRhdGlvbiBvZiBJQ0QtOS1DTS9JQ0QtMTAtQ00gQ29kZXMgZm9yIEF1dG9tYXRlZCBFbGVjdHJvbmljIFNjb3Jpbmcgb2YNCiAgQVBBQ0hFIElJLCBBUEFDSEUgSUlJLCBhbmQgU0FQUyBJSQ0Kc3VidGl0bGU6IFByZWxpbWluYXJ5IEFuYWx5c2lzIGZvciBBU0hQIEFic3RyYWN0IFN1Ym1pc3Npb24NCmRhdGU6ICdVcGRhdGVkOiBgciBmb3JtYXQoU3lzLkRhdGUoKSwgIiVCICVkLCAlWSIpYCcNCm91dHB1dDoNCiAgaHRtbF9ub3RlYm9vazoNCiAgICBjb2RlX2ZvbGRpbmc6IGhpZGUNCiAgICB0b2M6IHllcw0KICAgIHRvY19mbG9hdDogeWVzDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9DQpsaWJyYXJ5KHRpZHl2ZXJzZSkNCmxpYnJhcnkoc3RyaW5ncikNCiMgbGlicmFyeSh0YWJsZW9uZSkNCmxpYnJhcnkoYnJvb20pDQoNCmRpcnI6OmdldF9yZHMoIi4uL2RhdGEvZmluYWwiKQ0KYGBgDQoNCiMjIFBhdGllbnQgU2VsZWN0aW9uDQoNCiogUGF0aWVudHMgbWVldGluZyBpbmNsdXNpb24gY3JpdGVyaWE6IGByIGV4Y2x1ZGUkc2NyZWVuYA0KKiBSZWFzb25zIGZvciBleGNsdXNpb246DQogICAgLSBQcmVnbmFudDogYHIgZXhjbHVkZSRwcmVnbmFudGANCiAgICAtIFByaXNvbmVyOiBgciBleGNsdWRlJHByaXNvbmVyc2ANCiAgICAtIElDVSBzdGF5IDwgMTIgaG91cnM6IGByIGV4Y2x1ZGUkaWN1X3Nob3J0YA0KICAgIC0gRGlhZ25vc2lzIGNvZGluZyBjb250YWluZWQgYm90aCBJQ0QtOS1DTSBhbmQgSUNELTEwLUNNOiBgciBleGNsdWRlJG11bHRfaWNkX3R5cGVzYA0KICAgIC0gTWlzc2luZyBkYXRhIG5lZWRlZCB0byBjYWxjdWxhdGUgcmlzayBzY29yZXM6IGByIGV4Y2x1ZGUkbGFic19taXNzaW5nYA0KICAgIC0gUmUtZW5jb3VudGVycyAob25seSBpbmNsdWRlZCBlYWNoIHBhdGllbnQgb25jZSkgOiBgciBleGNsdWRlJHJlZW5jb3VudGVyYA0KKiBUb3RhbCBudW1iZXIgb2YgZWxpZ2libGUgcGF0aWVudHMgcmVtYWluaW5nIHRvIHNhbXBsZSBmcm9tOiBgciBmbGF0dGVuX2ludChleGNsdWRlKVsxXSAtIHN1bShmbGF0dGVuX2ludChleGNsdWRlKVstMV0pYA0KDQojIyBQcmltYXJ5IGVuZHBvaW50DQoNCiMjIyBBUEFDSEUgSUkNCg0KKiBNZWFuIGFuZCBtZWRpYW4gc2NvcmUgZm9yIGVhY2ggY29tb3JiaWRpdHkgc2V0DQoNCmBgYHtyfQ0Kc2NvcmVzIDwtIGxzKHBhdHRlcm4gPSAic2NvcmVfYXBhY2hlMiIpDQoNCmFsbF9zY29yZXMgPC0gbWFwKHNjb3Jlcywgfm11dGF0ZShnZXQoLngpLCBzY29yZSA9IC54KSkgJT4lDQogICAgbWFwKH5zZWxlY3QoLngsIHBpZS5pZCwgc2NvcmUsIGFwYWNoZTIpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJzY29yZV9hcGFjaGUyXyIsIHJlcGxhY2VtZW50ID0gIiIpIA0KDQptYW4gPC0gZmlsdGVyKGFsbF9zY29yZXMsIHNjb3JlID09ICJtYW51YWwiKSRhcGFjaGUyDQoNCnJlc3VsdCA8LSBhbGxfc2NvcmVzICU+JQ0KICAgIHNlbGVjdCgtcGllLmlkKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgc3VtbWFyaXplX2FsbChmdW5zKG1lYW4sIHNkLCBtZWRpYW4sIElRUl9sb3cgPSBxdWFudGlsZSguLCAwLjI1KSwgSVFSX2hpZ2ggPSBxdWFudGlsZSguLCAwLjc1KSkpDQoNCmtuaXRyOjprYWJsZShyZXN1bHQpDQpgYGANCg0KKiBDb21wYXJpc29uIG9mIHNjb3Jlcw0KICAgIC0gUmVzdWx0cyBydW4gdXNpbmcgYm90aCB0aGUgdC10ZXN0IGFuZCBXaWxjb3hvbiBzaWduZWQgcmFuayB0ZXN0IChhc3N1bWVzIG5vbi1wYXJhbWV0cmljIGRhdGEsIGlzIG1vcmUgZGlmZmljdWx0IHRvIGFjaGlldmUgc2lnbmlmaWNhbmNlKQ0KDQpgYGB7cn0NCnQgPC0gYWxsX3Njb3Jlc1ssIC0xXSAlPiUNCiAgICBmaWx0ZXIoc2NvcmUgIT0gIm1hbnVhbCIpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBkbyh0aWR5KHQudGVzdCguJGFwYWNoZTIsIG1hbiwgcGFpcmVkID0gVFJVRSkpKQ0KDQprbml0cjo6a2FibGUodCkNCmBgYA0KDQpgYGB7cn0NCm13IDwtIGFsbF9zY29yZXNbLCAtMV0gJT4lDQogICAgZmlsdGVyKHNjb3JlICE9ICJtYW51YWwiKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgZG8odGlkeSh3aWxjb3gudGVzdCguJGFwYWNoZTIsIG1hbiwgcGFpcmVkID0gVFJVRSwgY29uZi5pbnQgPSBUUlVFLCBleGFjdCA9IEZBTFNFKSkpDQoNCmtuaXRyOjprYWJsZShtdykNCmBgYA0KDQoqIENvbXBhcmUgZGlmZmVyZW5jZXMgaW4gc2NvcmUgZnJvbSBjb21vcmJpZGl0eSBzZXRzIHZzLiBtYW51YWwNCiAgICAtIENvbHVtbiAqbnVtX2RpZmYqIGluZGljYXRlcyB0aGUgbnVtYmVyIG9mIHBhdGllbnRzIHdpdGggYSBzY29yZSB0aGF0IGRpZmZlcmVkIGZyb20gbWFudWFsDQogICAgLSBNZWFuIGFuZCBtZWRpYW4gaW5kaWNhdGUgdGhlIGRpZmZlcmVuY2UgZnJvbSB0aGUgbWFudWFsIHNjb3JlDQogICAgICAgICsgVGhlc2Ugd2VyZSBvbmx5IGNhbGN1bGF0ZWQgZm9yIHRob3NlIHBhdGllbnRzIHdob3NlIHNjb3JlIHdhcyBkaWZmZXJlbnQgZnJvbSBtYW51YWwNCiAgICAgICAgKyBJZiBhbGwgcGF0aWVudHMgd2VyZSBpbmNsdWRlZCwgdGhlIG51bWJlciBpcyBza2V3ZWQgdG93YXJkcyAwIGJ5IGFsbCB0aGUgcGF0aWVudHMgd2l0aCB0aGUgc2FtZSBzY29yZXMNCg0KYGBge3J9DQpkaWZmIDwtIGFsbF9zY29yZXMgJT4lDQogICAgc3ByZWFkKHNjb3JlLCBhcGFjaGUyKSAlPiUNCiAgICBtdXRhdGVfYXQodmFycygtcGllLmlkLCAtbWFudWFsKSwgZnVucyhhYnMoLiAtIG1hbnVhbCkpKSAlPiUNCiAgICBnYXRoZXIoc2NvcmUsIGFwYWNoZTIsIC1waWUuaWQsIC1tYW51YWwpICU+JQ0KICAgIHNlbGVjdCgtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBmaWx0ZXIoYXBhY2hlMiA+IDApICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBzdW1tYXJpemVfYWxsKGZ1bnMobnVtX2RpZmYgPSBsZW5ndGgsIG1lYW4sIHNkLCBtZWRpYW4sIElRUl9sb3cgPSBxdWFudGlsZSguLCAwLjI1KSwgSVFSX2hpZ2ggPSBxdWFudGlsZSguLCAwLjc1KSkpDQoNCmtuaXRyOjprYWJsZShkaWZmLCBjYXB0aW9uID0gIkRpZmZlcmVuY2VzIGluIEFQQUNIRSBJSSBTY29yZXMiKQ0KYGBgDQoNCiMjIyBBUEFDSEUgSUlJDQoNCiogVGhlcmUgd2VyZSAxMCBwYXRpZW50cyB3aXRoIGRpZmZlcmVudCBjb21vcmJpZGl0aWVzIGlkZW50aWZpZWQgYnkgSUNEIGNvZGUgdnMuIG1hbnVhbA0KICAgIC0gT2YgdGhlc2UsIG9ubHkgNiBoYWQgYSBkaWZmZXJlbnQgQVBBQ0hFIElJSSBzY29yZTsgdGhlIG90aGVyIDQgd2VyZSBhbGwgImVsZWN0aXZlIiBhZG1pc3Npb25zIHdoZXJlIGNvbW9yYmlkaXR5IHNjb3JlIGlzIGlnbm9yZWQNCiAgICAtIDEgcGVyc29uIHdhcyBpZGVudGlmaWVkIGFzIGJlaW5nIGNocm9uaWMgSEQgYnkgSUNEIGNvZGUgYnV0IG5vdCBieSBtYW51YWwgcmV2aWV3LCBhY2NvdW50aW5nIGZvciB0aGUgNyBwYXRpZW50cyB3aGVyZSB0aGUgQVBBQ0hFIElJSSBzY29yZSBieSBJQ0QgdmFyaWVkIGZyb20gbWFudWFsDQoNCmBgYHtyfQ0Kc2NvcmVzIDwtIGxzKHBhdHRlcm4gPSAic2NvcmVfYXBhY2hlMyIpDQoNCmFsbF9zY29yZXMgPC0gbWFwKHNjb3Jlcywgfm11dGF0ZShnZXQoLngpLCBzY29yZSA9IC54KSkgJT4lDQogICAgbWFwKH5zZWxlY3QoLngsIHBpZS5pZCwgc2NvcmUsIGFwYWNoZTMpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJzY29yZV9hcGFjaGUzXyIsIHJlcGxhY2VtZW50ID0gIiIpIA0KDQptYW4gPC0gZmlsdGVyKGFsbF9zY29yZXMsIHNjb3JlID09ICJtYW51YWwiKSRhcGFjaGUzDQoNCnJlc3VsdCA8LSBhbGxfc2NvcmVzICU+JQ0KICAgIHNlbGVjdCgtcGllLmlkKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgc3VtbWFyaXplX2FsbChmdW5zKG1lYW4sIHNkLCBtZWRpYW4sIElRUl9sb3cgPSBxdWFudGlsZSguLCAwLjI1KSwgSVFSX2hpZ2ggPSBxdWFudGlsZSguLCAwLjc1KSkpDQoNCmtuaXRyOjprYWJsZShyZXN1bHQpDQpgYGANCg0KKiBDb21wYXJpc29uIG9mIHNjb3Jlcw0KDQpgYGB7cn0NCnQgPC0gYWxsX3Njb3Jlc1ssIC0xXSAlPiUNCiAgICBmaWx0ZXIoc2NvcmUgIT0gIm1hbnVhbCIpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBkbyh0aWR5KHQudGVzdCguJGFwYWNoZTMsIG1hbiwgcGFpcmVkID0gVFJVRSkpKQ0KDQprbml0cjo6a2FibGUodCkNCmBgYA0KDQpgYGB7cn0NCm13IDwtIGFsbF9zY29yZXNbLCAtMV0gJT4lDQogICAgZmlsdGVyKHNjb3JlICE9ICJtYW51YWwiKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgZG8odGlkeSh3aWxjb3gudGVzdCguJGFwYWNoZTMsIG1hbiwgcGFpcmVkID0gVFJVRSwgY29uZi5pbnQgPSBUUlVFLCBleGFjdCA9IEZBTFNFKSkpDQoNCmtuaXRyOjprYWJsZShtdykNCmBgYA0KDQoqIERpZmZlcmVuY2VzIGluIHNjb3JlDQoNCmBgYHtyfQ0KZGlmZiA8LSBhbGxfc2NvcmVzICU+JQ0KICAgIHNwcmVhZChzY29yZSwgYXBhY2hlMykgJT4lDQogICAgbXV0YXRlX2F0KHZhcnMoLXBpZS5pZCwgLW1hbnVhbCksIGZ1bnMoYWJzKC4gLSBtYW51YWwpKSkgJT4lDQogICAgZ2F0aGVyKHNjb3JlLCBhcGFjaGUzLCAtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBzZWxlY3QoLXBpZS5pZCwgLW1hbnVhbCkgJT4lDQogICAgZmlsdGVyKGFwYWNoZTMgPiAwKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgc3VtbWFyaXplX2FsbChmdW5zKG51bV9kaWZmID0gbGVuZ3RoLCBtZWFuLCBzZCwgbWVkaWFuLCBJUVJfbG93ID0gcXVhbnRpbGUoLiwgMC4yNSksIElRUl9oaWdoID0gcXVhbnRpbGUoLiwgMC43NSkpKQ0KDQprbml0cjo6a2FibGUoZGlmZiwgY2FwdGlvbiA9ICJEaWZmZXJlbmNlIGluIEFQQUNIRSBJSUkgU2NvcmVzIikNCmBgYA0KDQojIyMgU0FQUyBJSQ0KDQpgYGB7cn0NCnNjb3JlcyA8LSBscyhwYXR0ZXJuID0gInNjb3JlX3NhcHMyIikNCg0KYWxsX3Njb3JlcyA8LSBtYXAoc2NvcmVzLCB+bXV0YXRlKGdldCgueCksIHNjb3JlID0gLngpKSAlPiUNCiAgICBtYXAofnNlbGVjdCgueCwgcGllLmlkLCBzY29yZSwgc2FwczIpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJzY29yZV9zYXBzMl8iLCByZXBsYWNlbWVudCA9ICIiKSANCg0KbWFuIDwtIGZpbHRlcihhbGxfc2NvcmVzLCBzY29yZSA9PSAibWFudWFsIikkc2FwczINCg0KcmVzdWx0IDwtIGFsbF9zY29yZXMgJT4lDQogICAgc2VsZWN0KC1waWUuaWQpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBzdW1tYXJpemVfYWxsKGZ1bnMobWVhbiwgc2QsIG1lZGlhbiwgSVFSX2xvdyA9IHF1YW50aWxlKC4sIDAuMjUpLCBJUVJfaGlnaCA9IHF1YW50aWxlKC4sIDAuNzUpKSkNCg0Ka25pdHI6OmthYmxlKHJlc3VsdCkNCmBgYA0KDQoqIENvbXBhcmlzb24gb2Ygc2NvcmVzDQoNCmBgYHtyfQ0KdCA8LSBhbGxfc2NvcmVzWywgLTFdICU+JQ0KICAgIGZpbHRlcihzY29yZSAhPSAibWFudWFsIikgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIGRvKHRpZHkodC50ZXN0KC4kc2FwczIsIG1hbiwgcGFpcmVkID0gVFJVRSkpKQ0KDQprbml0cjo6a2FibGUodCkNCmBgYA0KDQoqIENvbmZpZGVuY2UgaW50ZXJ2YWwgd2FzIHVuYWJsZSB0byBiZSBjYWxjdWxhdGVkIGZvciBXaWxjb3hvbiBhbmFseXNpcyBoZXJlDQoNCmBgYHtyfQ0KbXcgPC0gYWxsX3Njb3Jlc1ssIC0xXSAlPiUNCiAgICBmaWx0ZXIoc2NvcmUgIT0gIm1hbnVhbCIpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBkbyh0aWR5KHdpbGNveC50ZXN0KC4kc2FwczIsIG1hbiwgcGFpcmVkID0gVFJVRSwgZXhhY3QgPSBGQUxTRSkpKQ0KDQprbml0cjo6a2FibGUobXcpDQpgYGANCg0KKiBEaWZmZXJlbmNlcyBpbiBzY29yZQ0KDQpgYGB7cn0NCmRpZmYgPC0gYWxsX3Njb3JlcyAlPiUNCiAgICBzcHJlYWQoc2NvcmUsIHNhcHMyKSAlPiUNCiAgICBtdXRhdGVfYXQodmFycygtcGllLmlkLCAtbWFudWFsKSwgZnVucyhhYnMoLiAtIG1hbnVhbCkpKSAlPiUNCiAgICBnYXRoZXIoc2NvcmUsIHNhcHMyLCAtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBzZWxlY3QoLXBpZS5pZCwgLW1hbnVhbCkgJT4lDQogICAgZmlsdGVyKHNhcHMyID4gMCkgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIHN1bW1hcml6ZV9hbGwoZnVucyhudW1fZGlmZiA9IGxlbmd0aCwgbWVhbiwgc2QsIG1lZGlhbiwgSVFSX2xvdyA9IHF1YW50aWxlKC4sIDAuMjUpLCBJUVJfaGlnaCA9IHF1YW50aWxlKC4sIDAuNzUpKSkNCg0Ka25pdHI6OmthYmxlKGRpZmYsIGNhcHRpb24gPSAiRGlmZmVyZW5jZSBpbiBTQVBTIElJIFNjb3JlcyIpDQpgYGANCg0KIyMgU2Vjb25kYXJ5IGVuZHBvaW50cw0KDQojIyMgUHJlZGljdGVkIG1vcnRhbGl0eQ0KDQojIyMjIFNBUFMgSUkNCg0KKiBNZWFuIGFuZCBtZWRpYW4gcmlzayBvZiBtb3J0YWxpdHkgKGFzIGEgcGVyY2VudCkgZm9yIGVhY2ggY29tb3JiaWRpdHkgc2V0DQoNCmBgYHtyfQ0KY29tb3JiaWQgPC0gbHMocGF0dGVybiA9ICJzY29yZV9zYXBzMl8iKQ0KDQphbGxfZGYgPC0gbWFwKGNvbW9yYmlkLCB+bXV0YXRlKGdldCgueCksIHNjb3JlID0gLngpKSAlPiUNCiAgICBtYXAofnNlbGVjdCgueCwgcGllLmlkLCBzY29yZSwgc2FwczIpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJzY29yZV9zYXBzMl8iLCByZXBsYWNlbWVudCA9ICIiKSAlPiUNCiAgICBkaXN0aW5jdChwaWUuaWQsIHNjb3JlLCAua2VlcF9hbGwgPSBUUlVFKSAlPiUNCiAgICBtdXRhdGUobG9naXQgPSAtNy43NjMgKyAwLjA3MzcgKiBzYXBzMiArIDAuOTk3MSAqIGxvZyhzYXBzMiArIDEpLA0KICAgICAgICAgICBtb3J0YWxpdHkgPSBleHAobG9naXQpIC8gKDEgKyBleHAobG9naXQpKSkgJT4lDQogICAgc2VsZWN0KHBpZS5pZCwgc2NvcmUsIG1vcnRhbGl0eSkgDQoNCm1hbiA8LSBmaWx0ZXIoYWxsX2RmLCBzY29yZSA9PSAibWFudWFsIikkbW9ydGFsaXR5DQoNCnJlc3VsdCA8LSBhbGxfZGYgJT4lDQogICAgc2VsZWN0KC1waWUuaWQpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBzdW1tYXJpemVfYWxsKGZ1bnMobWVhbiwgc2QsIG1lZGlhbiwgSVFSX2xvdyA9IHF1YW50aWxlKC4sIDAuMjUpLCBJUVJfaGlnaCA9IHF1YW50aWxlKC4sIDAuNzUpKSkNCg0Ka25pdHI6OmthYmxlKHJlc3VsdCkNCmBgYA0KDQpgYGB7cn0NCnQgPC0gYWxsX2RmWywgLTFdICU+JQ0KICAgIGZpbHRlcihzY29yZSAhPSAibWFudWFsIikgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIGRvKHRpZHkodC50ZXN0KC4kbW9ydGFsaXR5LCBtYW4sIHBhaXJlZCA9IFRSVUUpKSkNCg0Ka25pdHI6OmthYmxlKHQpDQpgYGANCg0KYGBge3J9DQptdyA8LSBhbGxfZGZbLCAtMV0gJT4lDQogICAgZmlsdGVyKHNjb3JlICE9ICJtYW51YWwiKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgZG8odGlkeSh3aWxjb3gudGVzdCguJG1vcnRhbGl0eSwgbWFuLCBwYWlyZWQgPSBUUlVFLCBjb25mLmludCA9IFRSVUUsIGV4YWN0ID0gRkFMU0UpKSkNCg0Ka25pdHI6OmthYmxlKG13KQ0KYGBgDQoNCiogRGlmZmVyZW5jZSBpbiByaXNrIG9mIG1vcnRhbGl0eQ0KICAgIC0gTWVhbiBhbmQgbWVkaWFuIGNhbGN1bGF0ZWQgdXNpbmcgb25seSB0aG9zZSBwYXRpZW50cyB3aG9zZSBwcmVkaWN0ZWQgbW9ydGFsaXR5IHNjb3JlIHdhcyBkaWZmZXJlbnQgZnJvbSBtYW51YWwNCiAgICANCmBgYHtyfQ0KZGlmZiA8LSBhbGxfZGYgJT4lDQogICAgc3ByZWFkKHNjb3JlLCBtb3J0YWxpdHkpICU+JQ0KICAgIG11dGF0ZV9hdCh2YXJzKC1waWUuaWQsIC1tYW51YWwpLCBmdW5zKGFicyguIC0gbWFudWFsKSkpICU+JQ0KICAgIGdhdGhlcihzY29yZSwgc2FwczIsIC1waWUuaWQsIC1tYW51YWwpICU+JQ0KICAgIHNlbGVjdCgtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBmaWx0ZXIoc2FwczIgPiAwKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgc3VtbWFyaXplX2FsbChmdW5zKG51bV9kaWZmID0gbGVuZ3RoLCBtZWFuLCBzZCwgbWVkaWFuLCBJUVJfbG93ID0gcXVhbnRpbGUoLiwgMC4yNSksIElRUl9oaWdoID0gcXVhbnRpbGUoLiwgMC43NSkpKQ0KDQprbml0cjo6a2FibGUoZGlmZiwgY2FwdGlvbiA9ICJEaWZmZXJlbmNlIGluIFByZWRpY3RlZCBNb3J0YWxpdHkgdXNpbmcgU0FQUyBJSSIpDQpgYGANCg0KIyMjIE51bWJlciBvZiBjb21vcmJpZGl0aWVzDQoNCiMjIyMgQVBBQ0hFIElJDQoNCiogTWVhbiByZXByZXNlbnRzIHRoZSBhdmVyYWdlIG51bWJlciBvZiBjb21vcmJpZGl0aWVzIHBlciBwYXRpZW50DQoNCmBgYHtyfQ0KY29tb3JiaWQgPC0gbHMocGF0dGVybiA9ICJkYXRhX2FwYWNoZTJfIikNCg0KYWxsX2RmIDwtIG1hcChjb21vcmJpZCwgfm11dGF0ZShnZXQoLngpLCBzY29yZSA9IC54KSkgJT4lDQogICAgbWFwKH5kaXN0aW5jdCgueCwgcGllLmlkLCBzY29yZSwgbGl2ZXIsIGNhcmRpb3Zhc2MsIHJlc3BpcmF0b3J5LCByZW5hbCwgaW1tdW5vY29tcCwgY29tb3JiaWRpdHkpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJkYXRhX2FwYWNoZTJfIiwgcmVwbGFjZW1lbnQgPSAiIikgJT4lDQogICAgbXV0YXRlX2lmKGlzLmxvZ2ljYWwsIGFzLm51bWVyaWMpICU+JQ0KICAgIGJ5X3JvdyhmdW5jdGlvbih4KSBzdW0oeFssIDM6N10sIG5hLnJtID0gVFJVRSksIC5jb2xsYXRlID0gInJvd3MiLCAudG8gPSAibnVtX2NvbW9yYmlkIikgJT4lDQogICAgc2VsZWN0KHBpZS5pZCwgc2NvcmUsIG51bV9jb21vcmJpZCkgDQoNCm1hbiA8LSBmaWx0ZXIoYWxsX2RmLCBzY29yZSA9PSAibWFudWFsIikkbnVtX2NvbW9yYmlkDQoNCnJlc3VsdCA8LSBhbGxfZGYgJT4lDQogICAgc2VsZWN0KC1waWUuaWQpICU+JQ0KICAgIGdyb3VwX2J5KHNjb3JlKSAlPiUNCiAgICBzdW1tYXJpemVfYWxsKGZ1bnMobWVhbiwgc2QsIG1lZGlhbiwgSVFSX2xvdyA9IHF1YW50aWxlKC4sIDAuMjUpLCBJUVJfaGlnaCA9IHF1YW50aWxlKC4sIDAuNzUpKSkNCg0Ka25pdHI6OmthYmxlKHJlc3VsdCkNCmBgYA0KDQpgYGB7cn0NCnQgPC0gYWxsX2RmWywgLTFdICU+JQ0KICAgIGZpbHRlcihzY29yZSAhPSAibWFudWFsIikgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIGRvKHRpZHkodC50ZXN0KC4kbnVtX2NvbW9yYmlkLCBtYW4sIHBhaXJlZCA9IFRSVUUpKSkNCg0Ka25pdHI6OmthYmxlKHQpDQpgYGANCg0KYGBge3J9DQptdyA8LSBhbGxfZGZbLCAtMV0gJT4lDQogICAgZmlsdGVyKHNjb3JlICE9ICJtYW51YWwiKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgZG8odGlkeSh3aWxjb3gudGVzdCguJG51bV9jb21vcmJpZCwgbWFuLCBwYWlyZWQgPSBUUlVFLCBjb25mLmludCA9IFRSVUUsIGV4YWN0ID0gRkFMU0UpKSkNCg0Ka25pdHI6OmthYmxlKG13KQ0KYGBgDQoNCiMjIyBQcm9wb3J0aW9uIHdpdGggZGlmZmVyaW5nIGNvbW9yYmlkaXR5DQoNCiMjIyMgQVBBQ0hFIElJDQoNCiogUHJvcG9ydGlvbiBvZiBwYXRpZW50cyB3aGVyZSBhIGNvbW9yYmlkaXR5IHdhcyBpZGVudGlmaWVkIGJ5IElDRCBvciBieSBtYW51YWwsIGJ1dCBub3QgYm90aA0KDQpgYGB7cn0NCmNvbW9yYmlkIDwtIGxzKHBhdHRlcm4gPSAiZGF0YV9hcGFjaGUyXyIpDQoNCmFsbF9kZiA8LSBtYXAoY29tb3JiaWQsIH5tdXRhdGUoZ2V0KC54KSwgc2NvcmUgPSAueCkpICU+JQ0KICAgIG1hcCh+ZGlzdGluY3QoLngsIHBpZS5pZCwgc2NvcmUsIGNvbW9yYmlkaXR5KSkgJT4lDQogICAgbWFwX2RmKH5zZW1pX2pvaW4oLngsIG1hbnVhbF9wYXRpZW50cywgYnkgPSAicGllLmlkIikpICU+JQ0KICAgIGFycmFuZ2UocGllLmlkLCBzY29yZSkgJT4lDQogICAgZG1hcF9hdCgic2NvcmUiLCBzdHJfcmVwbGFjZV9hbGwsIHBhdHRlcm4gPSAiZGF0YV9hcGFjaGUyXyIsIHJlcGxhY2VtZW50ID0gIiIpICU+JQ0KICAgIHNwcmVhZChzY29yZSwgY29tb3JiaWRpdHkpICU+JQ0KICAgIG11dGF0ZV9hdCh2YXJzKC1waWUuaWQsIC1tYW51YWwpLCBmdW5zKC4gIT0gbWFudWFsKSkgJT4lDQogICAgbXV0YXRlX2lmKGlzLmxvZ2ljYWwsIGFzLm51bWVyaWMpICU+JQ0KICAgIHVuZ3JvdXAoKSAlPiUNCiAgICBnYXRoZXIoc2NvcmUsIGNvbW9yYmlkaXR5LCAtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBzZWxlY3QoLXBpZS5pZCwgLW1hbnVhbCkgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIHN1bW1hcml6ZV9hbGwoZnVucyhudW1fZGlmZiA9IHN1bSguID4gMCksIHByb3BvcnRpb24gPSBtZWFuKSkNCg0Ka25pdHI6OmthYmxlKGFsbF9kZikNCmBgYA0KDQojIyMjIEFQQUNIRSBJSUkNCg0KKiBQcm9wb3J0aW9uIG9mIHBhdGllbnRzIHdoZXJlIHRoZSBwcmltYXJ5IGNvbW9yYmlkaXR5IGlkZW50aWZpZWQgYnkgSUNEIHZhcmllZCBmcm9tIHByaW1hcnkgY29tb3JiaWRpdHkgaWRlbnRpZmllZCBieSBtYW51YWwNCg0KYGBge3J9DQpjb21vcmJpZCA8LSBscyhwYXR0ZXJuID0gImRhdGFfYXBhY2hlM18iKQ0KDQphbGxfZGYgPC0gbWFwKGNvbW9yYmlkLCB+bXV0YXRlKGdldCgueCksIHNjb3JlID0gLngpKSAlPiUNCiAgICBtYXAofmRpc3RpbmN0KC54LCBwaWUuaWQsIHNjb3JlLCBjb21vcmJpZGl0eSkpICU+JQ0KICAgIG1hcF9kZih+c2VtaV9qb2luKC54LCBtYW51YWxfcGF0aWVudHMsIGJ5ID0gInBpZS5pZCIpKSAlPiUNCiAgICBhcnJhbmdlKHBpZS5pZCwgc2NvcmUpICU+JQ0KICAgIGRtYXBfYXQoInNjb3JlIiwgc3RyX3JlcGxhY2VfYWxsLCBwYXR0ZXJuID0gImRhdGFfYXBhY2hlM18iLCByZXBsYWNlbWVudCA9ICIiKSAlPiUNCiAgICBzcHJlYWQoc2NvcmUsIGNvbW9yYmlkaXR5LCBmaWxsID0gIiIpICU+JQ0KICAgIGRtYXAoc3RyX3JlcGxhY2VfYWxsLCBwYXR0ZXJuID0gIm5vbmUiLCByZXBsYWNlbWVudCA9ICIiKSAlPiUNCiAgICBtdXRhdGVfYXQodmFycygtcGllLmlkLCAtbWFudWFsKSwgZnVucyguICE9IG1hbnVhbCkpICU+JQ0KICAgIG11dGF0ZV9pZihpcy5sb2dpY2FsLCBhcy5udW1lcmljKSAlPiUNCiAgICBnYXRoZXIoc2NvcmUsIGNvbW9yYmlkaXR5LCAtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBzZWxlY3QoLXBpZS5pZCwgLW1hbnVhbCkgJT4lDQogICAgZ3JvdXBfYnkoc2NvcmUpICU+JQ0KICAgIHN1bW1hcml6ZV9hbGwoZnVucyhudW1fZGlmZiA9IHN1bSguID4gMCksIHByb3BvcnRpb24gPSBtZWFuKSkNCg0Ka25pdHI6OmthYmxlKGFsbF9kZikNCmBgYA0KDQojIyMjIFNBUFMgSUkNCg0KKiBQcm9wb3J0aW9uIG9mIHBhdGllbnRzIHdoZXJlIHRoZSBwcmltYXJ5IGNvbW9yYmlkaXR5IGlkZW50aWZpZWQgYnkgSUNEIHZhcmllZCBmcm9tIHByaW1hcnkgY29tb3JiaWRpdHkgaWRlbnRpZmllZCBieSBtYW51YWwNCg0KYGBge3J9DQpjb21vcmJpZCA8LSBscyhwYXR0ZXJuID0gImRhdGFfc2FwczJfIikNCg0KYWxsX2RmIDwtIG1hcChjb21vcmJpZCwgfm11dGF0ZShnZXQoLngpLCBzY29yZSA9IC54KSkgJT4lDQogICAgbWFwKH5kaXN0aW5jdCgueCwgcGllLmlkLCBzY29yZSwgY29tb3JiaWRpdHkpKSAlPiUNCiAgICBtYXBfZGYofnNlbWlfam9pbigueCwgbWFudWFsX3BhdGllbnRzLCBieSA9ICJwaWUuaWQiKSkgJT4lDQogICAgYXJyYW5nZShwaWUuaWQsIHNjb3JlKSAlPiUNCiAgICBkbWFwX2F0KCJzY29yZSIsIHN0cl9yZXBsYWNlX2FsbCwgcGF0dGVybiA9ICJkYXRhX3NhcHMyXyIsIHJlcGxhY2VtZW50ID0gIiIpICU+JQ0KICAgIHNwcmVhZChzY29yZSwgY29tb3JiaWRpdHksIGZpbGwgPSAiIikgJT4lDQogICAgZG1hcChzdHJfcmVwbGFjZV9hbGwsIHBhdHRlcm4gPSAibm9uZSIsIHJlcGxhY2VtZW50ID0gIiIpICU+JQ0KICAgIG11dGF0ZV9hdCh2YXJzKC1waWUuaWQsIC1tYW51YWwpLCBmdW5zKC4gIT0gbWFudWFsKSkgJT4lDQogICAgbXV0YXRlX2lmKGlzLmxvZ2ljYWwsIGFzLm51bWVyaWMpICU+JQ0KICAgIGdhdGhlcihzY29yZSwgY29tb3JiaWRpdHksIC1waWUuaWQsIC1tYW51YWwpICU+JQ0KICAgIHNlbGVjdCgtcGllLmlkLCAtbWFudWFsKSAlPiUNCiAgICBncm91cF9ieShzY29yZSkgJT4lDQogICAgc3VtbWFyaXplX2FsbChmdW5zKG51bV9kaWZmID0gc3VtKC4gPiAwKSwgcHJvcG9ydGlvbiA9IG1lYW4pKQ0KDQprbml0cjo6a2FibGUoYWxsX2RmKQ0KYGBgDQo=