## ----include = FALSE----------------------------------------------------------
knitr::opts_chunk$set(collapse = TRUE, comment = "#>")
set.seed(3)

## ----setup--------------------------------------------------------------------
library(evaluatellm)

## -----------------------------------------------------------------------------
passage_difficulty <- rnorm(50, 0, 1.3)
item_difficulty <- rep(passage_difficulty, each = 8) + rnorm(400, 0, 0.4)

skill <- c(new = 0.95, old = 0.72)
results <- do.call(rbind, lapply(names(skill), function(m) {
  data.frame(
    q       = 1:400,
    passage = rep(1:50, each = 8),
    model   = m,
    correct = rbinom(400, 1, plogis(skill[[m]] - item_difficulty))
  )
}))

e <- as_eval(results, score = correct, item = q, model = model,
             cluster = passage)
e

## -----------------------------------------------------------------------------
ev_score(e, "new")

## -----------------------------------------------------------------------------
ev_score(e, "new", cluster = FALSE)

## -----------------------------------------------------------------------------
ev_icc(e, "new")

## -----------------------------------------------------------------------------
ev_paired(e, model_a = "new", model_b = "old")

## -----------------------------------------------------------------------------
ev_mde(n_items = 400, p_a = 0.72, p_b = 0.70, correlation = 0.7,
       icc = 0.25, cluster_size = 8)

## -----------------------------------------------------------------------------
ev_power(delta = 0.02, p_a = 0.72, p_b = 0.70, correlation = 0.7,
         icc = 0.25, cluster_size = 8)

## -----------------------------------------------------------------------------
p_i <- plogis(rnorm(200, 0.7, 1.1))
rep_d <- data.frame(
  q    = rep(1:200, each = 6),
  draw = rep(1:6, times = 200),
  s    = rbinom(1200, 1, rep(p_i, each = 6))
)
ev_resample(as_eval(rep_d, score = s, item = q, sample = draw))

## -----------------------------------------------------------------------------
truth <- rbinom(500, 1, 0.55)
judge_pilot <- truth
judge_pilot[truth == 0] <- rbinom(sum(truth == 0), 1, 0.35)
ev_judge_agreement(judge_pilot, truth)

## -----------------------------------------------------------------------------
n_total <- 5000
truth_all <- rbinom(n_total, 1, 0.62)
judge <- ifelse(runif(n_total) < 0.87, truth_all, 1 - truth_all)
judge[truth_all == 0 & runif(n_total) < 0.15] <- 1

gold <- rep(NA_real_, n_total)
labelled <- sample(n_total, 250)
gold[labelled] <- truth_all[labelled]

ev_judge_debias(judge, gold)

## -----------------------------------------------------------------------------
ev_judge_power(n_total = 20000, correlation = 0.8, target_se = 0.01)

## -----------------------------------------------------------------------------
suite <- lapply(1:6, function(i) {
  a <- rbinom(250, 1, 0.72)
  b <- rbinom(250, 1, 0.68)
  d <- data.frame(q = rep(1:250, 2), m = rep(c("new", "old"), each = 250),
                  s = c(a, b))
  ev_paired(as_eval(d, score = s, item = q, model = m), "new", "old")
})
names(suite) <- paste0("task_", 1:6)
ev_multi(suite)

## ----fig.width = 7, fig.height = 3.5------------------------------------------
ev_plot(suite, reference = 0, xlab = "accuracy gain")