1
#
2
#   This file contains the mcmcRocPrc() S3 generic, which constructs objects
3
#   of class "mcmcRocPrc". For methods for this class, see mcmcRocPrc-methods.R
4
#   S3 methods for the mcmcRocPrc() generic handle different types of input
5
#   e.g. "rjags" input produced by R2jags.
6
#
7

8

9

10
#' ROC and Precision-Recall Curves using Bayesian MCMC estimates
11
#' 
12
#' Generate ROC and Precision-Recall curves after fitting a Bayesian logit or 
13
#' probit regression using [rstan::stan()], [rstanarm::stan_glm()], 
14
#' [R2jags::jags()], [R2WinBUGS::bugs()], [MCMCpack::MCMClogit()], or other 
15
#' functions that provide samples from a posterior density. 
16
#' 
17
#' @param object A fitted binary choice model, e.g. "rjags" object 
18
#'   (see [R2jags::jags()]), or a `[N, iter]` matrix of predicted probabilites.
19
#' @param curves logical indicator of whether or not to return values to plot 
20
#'   the ROC or Precision-Recall curves. If set to `FALSE` (default), 
21
#'   results are returned as a list without the extra values. 
22
#' @param fullsims logical indicator of whether full object (based on all MCMC
23
#'   draws rather than their average) will be returned. Default is `FALSE`. 
24
#'   Note: If `TRUE` is chosen, the function takes notably longer to execute.
25
#' @param yvec A `numeric(N)` vector of observed outcomes. 
26
#' @param yname (`character(1)`)\cr
27
#'   The name of the dependent variable, should match the variable name in the 
28
#'   JAGS data object.
29
#' @param xnames ([base::character()])\cr
30
#'   A character vector of the independent variable names, should match the 
31
#'   corresponding names in the JAGS data object.
32
#' @param posterior_samples a "mcmc" object with the posterior samples
33
#' @param ... Used by methods
34
#' @param x a `mcmcRocPrc()` object
35
#' 
36
#' @details If only the average AUC-ROC and PR are of interest, setting 
37
#'   `curves = FALSE` and `fullsims = FALSE` can greatly speed up calculation 
38
#'   time. The curve data (`curves = TRUE`) is needed for plotting. The plot
39
#'   method will always plot both the ROC and PR curves, but the underlying
40
#'   data can easily be extracted from the output for your own plotting; 
41
#'   see the documentation of the value returned below. 
42
#'   
43
#'   The default method works with a matrix of predicted probabilities and the 
44
#'   vector of observed incomes as input. Other methods accommodate some of the 
45
#'   common Bayesian modeling packages like rstan (which returns class "stanfit"),
46
#'   rstanarm ("stanreg"), R2jags ("jags"), R2WinBUGS ("bugs"), and 
47
#'   MCMCpack ("mcmc"). Even if a package-specific method is not implemented, 
48
#'   the default method can always be used as a fallback by manually calculating
49
#'   the matrix of predicted probabilities for each posterior sample. 
50
#'   
51
#'   Note that MCMCpack returns generic "mcmc" output that is annotated with 
52
#'   some additional information as attributes, including the original function
53
#'   call. There is no inherent way to distinguish any other kind of "mcmc" 
54
#'   object from one generated by a proper MCMCpack modeling function, but as a
55
#'   basic precaution, `mcmcRocPrc()` will check the saved call and return an 
56
#'   error if the function called was not `MCMClogit()` or `MCMCprobit()`. 
57
#'   This behavior can be suppressed by setting `force = TRUE`. 
58
#' 
59
#' @references Beger, Andreas. 2016. “Precision-Recall Curves.” Available at 
60
#'   SSRN: [http://dx.doi.org/10.2139/ssrn.2765419](http://dx.doi.org/10.2139/ssrn.2765419)
61
#' 
62
#' @return Returns a list with length 2 or 4, depending on the on the "curves" 
63
#'   and "fullsims" argument values:
64
#'   
65
#'   - "area_under_roc": `numeric()`; either length 1 if `fullsims = FALSE`, or 
66
#'     one value for each posterior sample otherwise
67
#'   - "area_under_prc": `numeric()`; either length 1 if `fullsims = FALSE`, or 
68
#'     one value for each posterior sample otherwise
69
#'   - "prc_dat": only if `curves = TRUE`; a list with length 1 if 
70
#'     `fullsims = FALSE`, longer otherwise
71
#'   - "roc_dat": only if `curves = TRUE`; a list with length 1 if 
72
#'     `fullsims = FALSE`, longer otherwise
73
#'
74
#' @examples
75
#' # load simulated data and fitted model (see ?sim_data and ?jags_logit)
76
#' data("jags_logit")
77
#' 
78
#' # using mcmcRocPrc
79
#' fit_sum <- mcmcRocPrc(jags_logit,
80
#'                       yname = "Y",
81
#'                       xnames = c("X1", "X2"),
82
#'                       curves = TRUE,
83
#'                       fullsims = FALSE)
84
#' fit_sum                     
85
#' plot(fit_sum)
86
#' 
87
#' # Equivalently, we can calculate the matrix of predicted probabilities 
88
#' # ourselves; using the example from ?jags_logit:
89
#' library(R2jags)
90
#' 
91
#' data("sim_data")
92
#' yvec <- sim_data$Y
93
#' xmat <- sim_data[, c("X1", "X2")]
94
#' 
95
#' # add intercept to the X data
96
#' xmat <- as.matrix(cbind(Intercept = 1L, xmat))
97
#' 
98
#' beta <- as.matrix(as.mcmc(jags_logit))[, c("b[1]", "b[2]", "b[3]")]
99
#' pred_mat <- plogis(xmat %*% t(beta)) 
100
#' 
101
#' # the matrix of predictions has rows matching the number of rows in the data;
102
#' # the column are the predictions for each of the 2,000 posterior samples
103
#' nrow(sim_data)
104
#' dim(pred_mat)
105
#' 
106
#' # now we can call mcmcRocPrc; the default method works with the matrix
107
#' # of predictions and vector of outcomes as input
108
#' mcmcRocPrc(object = pred_mat, curves = TRUE, fullsims = FALSE, yvec = yvec)
109
#' 
110
#' @export
111
#' @md
112
mcmcRocPrc <- function(object, curves = FALSE, fullsims = FALSE, ...) {
113 1
  UseMethod("mcmcRocPrc", object)
114
}
115

116
#' Constructor for mcmcRocPrc objects
117
#' 
118
#' This function actually does the heavy lifting once we have a matrix of 
119
#' predicted probabilities from a model, plus the vector of observed outcomes.
120
#' The reason to have it here in a single function is that we don't replicate 
121
#' it in each function that accomodates a JAGS, BUGS, RStan, etc. object.
122
#' 
123
#' @param pred_prob a `\[N, iter\]` matrix of predicted probabilities 
124
#' @param yvec a `numeric(N)` vector of observed outcomes
125
#' @param curves include curve data in output?
126
#' @param fullsims collapse posterior samples into single summary?
127
#' 
128
#' @md
129
#' @keywords internal
130
new_mcmcRocPrc <- function(pred_prob, yvec, curves, fullsims) {
131
  
132 1
  stopifnot(
133 1
    "number of predictions and observed outcomes do not match" = nrow(pred_prob)==length(yvec),
134 1
    "yvec must be 0 or 1"                                      = all(yvec %in% c(0L, 1L)),
135 1
    "pred_prob must be in the interval [0, 1]"                 = all(pred_prob >= 0 & pred_prob <= 1)
136
  )
137
  
138
  # pred_prob is a [N, iter] matrix, i.e. each column are preds from one 
139
  # set of posterior samples
140
  # if not using fullsims, summarize across columns
141 1
  if (isFALSE(fullsims)) {
142
    
143 1
    pred_prob <- as.matrix(apply(pred_prob, MARGIN = 1, median))
144
    
145
  }
146
  
147 1
  pred_prob  <- as.data.frame(pred_prob)
148 1
  curve_data <- lapply(pred_prob, yy = yvec, FUN = function(x, yy) {
149 1
    prc_data <- compute_pr(yvec = yy, pvec = x)
150 1
    roc_data <- compute_roc(yvec = yy, pvec = x)
151 1
    list(
152 1
      prc_dat = prc_data,
153 1
      roc_dat = roc_data
154
    )
155
  })
156 1
  prc_dat <- lapply(curve_data, `[[`, "prc_dat")
157 1
  roc_dat <- lapply(curve_data, `[[`, "roc_dat")
158
  
159
  # Compute AUC-ROC values
160 1
  v_auc_roc <- sapply(roc_dat, function(xy) {
161 1
    caTools::trapz(xy$x, xy$y)
162
  })
163 1
  v_auc_pr  <- sapply(prc_dat, function(xy) {
164 1
    xy <- subset(xy, !is.nan(xy$y))
165 1
    caTools::trapz(xy$x, xy$y)
166
  })
167
  
168
  # Recreate original output formats
169 1
  if (curves & fullsims) {
170 1
    out <- list(
171 1
      area_under_roc = v_auc_roc,
172 1
      area_under_prc = v_auc_pr,
173 1
      prc_dat = prc_dat,
174 1
      roc_dat = roc_dat
175
    )
176
  }
177 1
  if (curves & !fullsims) {
178 1
    out <- list(
179 1
      area_under_roc = v_auc_roc,
180 1
      area_under_prc = v_auc_pr,
181 1
      prc_dat = prc_dat[1],
182 1
      roc_dat = roc_dat[1]
183
    )
184
  }
185 1
  if (!curves & !fullsims) {
186 1
    out <- list(
187 1
      area_under_roc = v_auc_roc[[1]],
188 1
      area_under_prc = v_auc_pr[[1]]
189
    )
190
  }
191 1
  if (!curves & fullsims) {
192 1
    out <- data.frame(
193 1
      area_under_roc = v_auc_roc,
194 1
      area_under_prc = v_auc_pr
195
    )
196
  }
197 1
  structure(
198 1
    out,
199 1
    y_pos_rate = mean(yvec),
200 1
    class = "mcmcRocPrc"
201
  )
202
}
203

204
#' @rdname mcmcRocPrc
205
#' 
206
#' @md
207
#' @export
208
mcmcRocPrc.default <- function(object, curves, fullsims, yvec, ...) {
209 1
  pred_prob <- object
210
  
211 1
  stopifnot(
212 1
    "mcmcRocPrc.default requires 'matrix' like input" = inherits(pred_prob, "matrix")
213
  )
214
  
215 1
  new_mcmcRocPrc(pred_prob, yvec, curves, fullsims)
216
}
217

218
# Under the hood ROC/PRC calculations -------------------------------------
219

220
#' Compute ROC and PR curve points
221
#' 
222
#' Faster replacements for calculating ROC and PR curve data than with 
223
#' [ROCR::prediction()] and [ROCR::performance()]
224
#' 
225
#' @details Replacements to use instead of a combination of [ROCR::prediction()] 
226
#' and [ROCR::performance()] to calculate ROC and PR curves. These functions are
227
#' about 10 to 20 times faster when using [mcmcRocPrc()] with `curves = TRUE` 
228
#' and/or `fullsims = TRUE`. 
229
#' 
230
#' See this [issue on GH (ShanaScogin/BayesPostEst#25)](https://github.com/ShanaScogin/BayesPostEst/issues/25) for more general details.
231
#' 
232
#' And [here is a note](https://github.com/andybega/BayesPostEst/blob/f1da23b9db86461d4f9c671d9393265dd10578c5/tests/profile-mcmcRocPrc.md) with specific performance benchmarks, compared to the 
233
#' old approach relying on ROCR.
234
#' 
235
#' @keywords internal
236
#' @md
237
compute_roc <- function(yvec, pvec) {
238 1
  porder <- order(pvec, decreasing = TRUE)
239 1
  yvecs  <- yvec[porder]
240 1
  pvecs  <- pvec[porder]
241 1
  p      <- sum(yvecs)
242 1
  n      <- length(yvecs) - p
243 1
  tp     <- cumsum(yvecs)
244 1
  tpr    <- tp/p
245 1
  fp     <- 1:length(yvecs) - tp
246 1
  fpr    <- fp/n
247
  
248 1
  dup_pred  <- rev(duplicated(pvecs))
249 1
  dup_stats <- duplicated(tpr) & duplicated(fpr)
250 1
  dups <- dup_pred | dup_stats
251
  
252 1
  fpr <- c(0, fpr[!dups])
253 1
  tpr <- c(0, tpr[!dups])
254
  
255 1
  roc_data <- data.frame(x = fpr,
256 1
                         y = tpr)
257 1
  roc_data
258
}
259

260
#' @rdname compute_roc
261
#' @aliases compute_pr
262
compute_pr <- function(yvec, pvec) {
263 1
  porder <- order(pvec, decreasing = TRUE)
264 1
  yvecs  <- yvec[porder]
265 1
  pvecs  <- pvec[porder]
266 1
  p      <- sum(yvecs)
267 1
  n      <- length(yvecs) - p
268 1
  tp     <- cumsum(yvecs)
269 1
  tpr    <- tp/p
270 1
  pp     <- 1:length(yvecs) 
271 1
  prec   <- tp/pp
272
  
273 1
  dup_pred  <- rev(duplicated(pvecs))
274 1
  dup_stats <- duplicated(tpr) & duplicated(prec)
275 1
  dups <- dup_pred | dup_stats
276
  
277 1
  prec <- c(NaN, prec[!dups])
278 1
  tpr <- c(0, tpr[!dups])
279
  
280 1
  prc_data <- data.frame(x = tpr,
281 1
                         y = prec)
282 1
  prc_data
283
}
284

285

286
# auc_roc and auc_pr are not really used, but keep around just in case
287
auc_roc <- function(obs, pred) {
288 1
  values <- compute_roc(obs, pred)
289 1
  caTools::trapz(values$x, values$y)
290
}
291

292
auc_pr <- function(obs, pred) {
293 1
  values <- compute_pr(obs, pred)
294 1
  caTools::trapz(values$x, values$y)
295
}
296

297

298

299
# JAGS-like input (rjags, R2jags, runjags) --------------------------------
300

301
#' @rdname mcmcRocPrc
302
#' 
303
#' @export
304
mcmcRocPrc.jags <- function(object, curves = FALSE, fullsims = FALSE, yname, 
305
                            xnames, posterior_samples, ...) {
306
  
307 1
  stopifnot(
308 1
    inherits(posterior_samples, c("mcmc", "mcmc.list"))
309
  )
310
  
311 1
  link_logit  <- any(grepl("logit", object$model()))
312 1
  link_probit <- any(grepl("probit", object$model()))
313
  
314 1
  if (isFALSE(link_logit | link_probit)) {
315 1
    stop("Could not identify model link function")
316
  }
317
  
318 1
  mdl_data <- object$data()
319 1
  stopifnot(all(xnames %in% names(mdl_data)))
320 1
  stopifnot(all(yname %in% names(mdl_data)))
321
  
322
  # add intercept by default, maybe revisit this
323 1
  xdata <- as.matrix(cbind(X0 = 1L, as.data.frame(mdl_data[xnames])))
324 1
  yvec  <- mdl_data[[yname]]
325
  
326 1
  pardraws <- as.matrix(posterior_samples)
327
  # this is not very robust, assumes pars are 'b[x]'
328
  # for both this and the intercept addition above, maybe a more robust solution
329
  # down the road would be to dig into the object$model$model() string
330 1
  betadraws <- pardraws[, c(sprintf("b[%s]", 1:ncol(xdata - 1)))]
331
  
332 1
  if(isTRUE(link_logit)) {
333 1
    pred_prob <- plogis(xdata %*% t(betadraws))  
334 1
  } else if (isTRUE(link_probit)) {
335 1
    pred_prob <- pnorm(xdata %*% t(betadraws))
336
  } 
337
  
338 1
  new_mcmcRocPrc(pred_prob = pred_prob, yvec = yvec, curves = curves, 
339 1
                 fullsims = fullsims)
340
}
341

342
#' @rdname mcmcRocPrc
343
#' 
344
#' @export
345
mcmcRocPrc.rjags <- function(object, curves = FALSE, fullsims = FALSE, yname, 
346
                             xnames, ...) {
347
  
348 1
  if (!requireNamespace("R2jags", quietly = TRUE)) {
349
    stop("Package \"R2jags\" is needed for this function to work. Please install it.", call. = FALSE)  # nocov
350
  }
351
  
352 1
  jags_object <- object$model
353 1
  pardraws    <- coda::as.mcmc(object)
354
  
355
  # pass it on to the "jags" method
356 1
  mcmcRocPrc(object = jags_object, curves = curves, fullsims = fullsims, 
357 1
             yname = yname, xnames = xnames, posterior_samples = pardraws, ...)
358
}
359

360
#' @rdname mcmcRocPrc
361
#' 
362
#' @export
363
mcmcRocPrc.runjags <- function(object, curves = FALSE, fullsims = FALSE, yname, 
364
                               xnames, ...) {
365 1
  jags_object <- runjags::as.jags(object, quiet = TRUE)
366
  # as.mcmc.runjags will issue a warning when converting multiple chains
367
  # because it combines them
368 1
  pardraws    <- suppressWarnings(coda::as.mcmc(object))
369
  
370
  # pass it on to the "jags" method
371 1
  mcmcRocPrc(object = jags_object, curves = curves, fullsims = fullsims, 
372 1
             yname = yname, xnames = xnames, posterior_samples = pardraws, ...)
373
}
374

375

376
# STAN-like input (rstan, rstanarm, brms) ---------------------------------
377

378

379

380
#' @rdname mcmcRocPrc
381
#' 
382
#' @param data the data that was used in the `stan(data = ?, ...)` call
383
#' 
384
#' @export
385
mcmcRocPrc.stanfit <- function(object, curves = FALSE, fullsims = FALSE, data, 
386
                               xnames, yname, ...) {
387 1
  if (!requireNamespace("rstan", quietly = TRUE)) {
388
    stop("Package \"rstan\" is needed for this function to work. Please install it.", call. = FALSE)  # nocov
389
  }
390
  
391 1
  if (!is_binary_model(object)) {
392 0
    stop("the input model does not seem to be a binary choice model; if this is a mistake please file an issue at https://github.com/ShanaScogin/BayesPostEst/issues/")
393
  }
394 1
  link_type <- identify_link_function(object)
395 1
  if (is.na(link_type)) {
396 0
    stop("could not identify model link function; please file an issue at https://github.com/ShanaScogin/BayesPostEst/issues/")
397
  }
398
  
399 1
  mdl_data <- data
400 1
  stopifnot(all(xnames %in% names(mdl_data)))
401 1
  stopifnot(all(yname %in% names(mdl_data)))
402
   
403
  # add intercept by default, maybe revisit this
404 1
  xdata <- as.matrix(cbind(X0 = 1L, as.data.frame(mdl_data[xnames])))
405 1
  yvec  <- mdl_data[[yname]]
406
  
407 1
  pardraws <- as.matrix(object)
408
  # this is not very robust, assumes pars are 'b[x]'
409 1
  betadraws <- pardraws[, c(sprintf("b[%s]", 1:ncol(xdata - 1)))]
410
  
411 1
  if(link_type=="logit") {
412 1
    pred_prob <- plogis(xdata %*% t(betadraws))  
413 0
  } else if (link_type=="probit") {
414 0
    pred_prob <- pnorm(xdata %*% t(betadraws))
415
  } 
416
  
417 1
  new_mcmcRocPrc(pred_prob = pred_prob, yvec = yvec, curves = curves, 
418 1
                 fullsims = fullsims)
419
  
420
  
421
}
422

423
#' Try to identify if a stanfit model is a binary choice model
424
#' 
425
#' @param obj stanfit object
426
#' 
427
#' @keywords internal
428
is_binary_model <- function(obj) {
429 1
  stopifnot(inherits(obj, "stanfit"))
430 1
  model_string <- rstan::get_stancode(obj)
431 1
  grepl("bernoulli", model_string)
432
}
433

434
#' Try to identify link function 
435
#' 
436
#' @param obj stanfit object
437
#' 
438
#' @return Either "logit" or "probit"; if neither can be identified the function
439
#' will return `NA_character_`. 
440
#' 
441
#' @keywords internal
442
identify_link_function <- function(obj) {
443 1
  stopifnot(inherits(obj, "stanfit"))
444 1
  model_string <- rstan::get_stancode(obj)
445 1
  if (grepl("logit", model_string)) return("logit")
446 0
  if (grepl("Phi", model_string)) return("probit")
447
  NA_character_
448
}
449

450
#' @rdname mcmcRocPrc
451
#' 
452
#' @export
453
mcmcRocPrc.stanreg <- function(object, curves = FALSE, fullsims = FALSE, ...) {
454 1
  if (!requireNamespace("rstanarm", quietly = TRUE)) {
455
    stop("Package \"rstanarm\" is needed for this function to work. Please install it.", call. = FALSE)  # nocov
456
  }
457 1
  if (!stats::family(object)$family=="binomial") {
458 1
    stop("the input model does not seem to be a binary choice model; should be like 'obj <- stan_glm(family = binomial(), ...)'") 
459
  }
460 1
  pred_prob <- rstanarm::posterior_linpred(object, transform = TRUE)
461
  # posterior_linepred returns a matrix in which data cases are columns, and 
462
  # MCMC samples are row; we need to transpose this so that columns are samples
463 1
  pred_prob <- t(pred_prob)
464 1
  yvec <- unname(object$y)
465
  
466 1
  new_mcmcRocPrc(pred_prob = pred_prob, yvec = yvec, curves = curves, 
467 1
                 fullsims = fullsims)
468
}
469

470
#' @rdname mcmcRocPrc
471
#' 
472
#' @export
473
mcmcRocPrc.brmsfit <- function(object, curves = FALSE, fullsims = FALSE, ...) {
474 1
  if (!requireNamespace("brms", quietly = TRUE)) {
475
    stop("Package \"brms\" is needed for this function to work. Please install it.", call. = FALSE)  # nocov
476
  }
477 1
  if (!stats::family(object)$family=="bernoulli") {
478 1
    stop("the input model does not seem to be a binary choice model; should be like 'obj <- brm(family = bernoulli(), ...)'") 
479
  }
480
  
481 1
  pred_prob <- brms::posterior_epred(object)
482
  # posterior_epred returns a matrix in which data cases are columns, and 
483
  # MCMC samples are row; we need to transpose this so that columns are samples
484 1
  pred_prob <- t(pred_prob)
485 1
  yvec <- stats::model.response(stats::model.frame(object))
486
  
487 1
  new_mcmcRocPrc(pred_prob = pred_prob, yvec = yvec, curves = curves, 
488 1
                 fullsims = fullsims)
489
}
490

491

492
# Other input types (MCMCpack, ...) ---------------------------------------
493

494

495
#' #' @rdname mcmcRocPrc
496
#' #'
497
#' #' @export
498
#' mcmcRocPrc.bugs <- function(object, curves = FALSE, fullsims = FALSE, ...) {
499
#'   stop("not implemented yet")
500
#' }
501

502

503
#' @rdname mcmcRocPrc
504
#' 
505
#' @param type "logit" or "probit"
506
#' @param force for MCMCpack models, suppress warning if the model does not 
507
#'   appear to be a binary choice model?
508
#'
509
#' @export
510
mcmcRocPrc.mcmc <- function(object, curves = FALSE, fullsims = FALSE, data, 
511
                            xnames, yname, type = c("logit", "probit"), 
512
                            force = FALSE, ...) {
513
  
514 1
  if (!force) {
515 1
    if (is.null(attr(object, "call"))) {
516 1
      stop("object does not have a 'call' attribute; was it generated with a MCMCpack function?")
517
    } else {
518 1
      func <- as.character(attr(object, "call"))[1]
519 1
      if (!func %in% c("MCMClogit", "MCMCprobit")) {
520 1
        stop("object does not appear to have been fitted using MCMCpack::MCMClogit() or MCMCprobit(); mcmcRocPrc only properly works for those function. To be safe, consider manually calculating the matrix of predicted probabilities.")
521
      }
522
    }
523
  }
524
  
525 1
  link_type <- match.arg(type)
526 1
  mdl_data <- data
527 1
  stopifnot(
528 1
    all(xnames %in% names(mdl_data)),
529 1
    all(yname %in% names(mdl_data))
530
  )
531
  
532
  # add intercept by default, maybe revisit this
533 1
  xdata <- as.matrix(cbind(X0 = 1L, as.data.frame(mdl_data[xnames])))
534 1
  yvec  <- mdl_data[[yname]]
535
  
536 1
  betadraws <- as.matrix(object)
537

538 1
  if(link_type=="logit") {
539 1
    pred_prob <- plogis(xdata %*% t(betadraws))  
540 0
  } else if (link_type=="probit") {
541 0
    pred_prob <- pnorm(xdata %*% t(betadraws))
542
  } 
543
  
544 1
  new_mcmcRocPrc(pred_prob = pred_prob, yvec = yvec, curves = curves, 
545 1
                 fullsims = fullsims)
546
}
547

548

549

Read our documentation on viewing source code .

Loading