\name{cvrisk}
\alias{cvrisk}
\alias{cvrisk.mboost}
\alias{print.cvrisk}
\alias{plot.cvrisk}
\alias{cv}

\title{ Cross-Validation }
\description{
  Cross-validated estimation of the empirical risk for hyper-parameter selection.
}
\usage{
\method{cvrisk}{mboost}(object, folds = cv(model.weights(object)),
       grid = 0:mstop(object),
       papply = mclapply,
       fun = NULL, mc.preschedule = FALSE, ...)
cv(weights, type = c("bootstrap", "kfold", "subsampling"),
   B = ifelse(type == "kfold", 10, 25), prob = 0.5, strata = NULL)

## Plot cross-valiation results   
\method{plot}{cvrisk}(x, 
     xlab = "Number of boosting iterations", ylab = attr(x, "risk"),
     ylim = range(x), main = attr(x, "type"), ...)
}
\arguments{
  \item{object}{ an object of class \code{mboost}.}
  \item{folds}{ a weight matrix with number of rows equal to the number
                of observations. The number of columns corresponds to
                the number of cross-validation runs. Can be computed
                using function \code{cv} and defaults to 25 bootstrap samples.}
  \item{grid}{ a vector of stopping parameters the empirical risk
                is to be evaluated for. }
  \item{papply}{
    (parallel) apply function, defaults to  \code{\link[parallel]{mclapply}}.
    Alternatively, \code{\link[parallel]{parLapply}} can be used. In the
    latter case, usually more setup is needed (see example for some
    details). To run \code{cvrisk} sequentially (i.e. not in parallel),
    one can use \code{\link{lapply}}.
  }
  \item{fun}{ if \code{fun} is NULL, the out-of-sample risk is returned. \code{fun},
              as a function of \code{object}, may extract any other characteristic
              of the cross-validated models. These are returned as is.}
  \item{mc.preschedule}{
    preschedule tasks if are parallelized using \code{\link{mclapply}}
    (default: \code{FALSE})? For details see \code{\link{mclapply}}.
  }
  \item{weights}{ a numeric vector of weights for the model to be cross-validated.}
  \item{type}{ character argument for specifying the cross-validation
               method. Currently (stratified) bootstrap, k-fold cross-validation
               and subsampling are implemented.}
  \item{B}{ number of folds, per default 25 for \code{bootstrap} and
            \code{subsampling} and 10 for \code{kfold}.}
  \item{prob}{ percentage of observations to be included in the learning samples
               for subsampling.}
  \item{strata}{ a factor of the same length as \code{weights} for stratification.}
  \item{x}{ an object of class \code{cvrisk}.}
  \item{xlab, ylab}{axis labels.}
  \item{ylim}{limits of y-axis.}
  \item{main}{main title of graphic.}
  \item{...}{additional arguments passed to \code{\link[parallel]{mclapply}} or 
     \code{\link{plot}}.}
}
\details{

  The number of boosting iterations is a hyper-parameter of the
  boosting algorithms implemented in this package \bibcitep{mboost::Mayr_2012}. Honest,
  i.e., cross-validated, estimates of the empirical risk
  for different stopping parameters \code{mstop} are computed by
  this function which can be utilized to choose an appropriate
  number of boosting iterations to be applied.

  Different forms of cross-validation can be applied, for example
  10-fold cross-validation or bootstrapping. The weights (zero weights
  correspond to test cases) are defined via the \code{folds} matrix.

  \code{cvrisk} runs in parallel on OSes where forking is possible
  (i.e., not on Windows) and multiple cores/processors are available.
  The scheduling
  can be changed by the corresponding arguments of
  \code{\link[parallel]{mclapply}} (via the dot arguments).

  The function \code{cv} can be used to build an appropriate
  weight matrix to be used with \code{cvrisk}. If \code{strata} is defined
  sampling is performed in each stratum separately thus preserving
  the distribution of the \code{strata} variable in each fold.
  
  There exist various functions to display and work with 
  cross-validation results. One can \code{print} and \code{plot} (see above)
  results and extract the optimal iteration via \code{\link{mstop}}.
}
\value{
  An object of class \code{cvrisk} (when \code{fun} wasn't specified), basically a matrix
  containing estimates of the empirical risk for a varying number
  of bootstrap iterations. \code{plot} and \code{print} methods
  are available as well as a \code{mstop} method.
}
\references{\bibshow{*}}
\seealso{\code{\link{AIC.mboost}} for
  \code{AIC} based selection of the stopping iteration. Use \code{mstop}
  to extract the optimal stopping iteration from \code{cvrisk}
  object.}
\examples{

  data("bodyfat", package = "TH.data")

  ### fit linear model to data
  model <- glmboost(DEXfat ~ ., data = bodyfat, center = TRUE)

  ### AIC-based selection of number of boosting iterations
  maic <- AIC(model)
  maic

  ### inspect coefficient path and AIC-based stopping criterion
  par(mai = par("mai") * c(1, 1, 1, 1.8))
  plot(model)
  abline(v = mstop(maic), col = "lightgray")

  ### 10-fold cross-validation
  cv10f <- cv(model.weights(model), type = "kfold")
  cvm <- cvrisk(model, folds = cv10f, papply = lapply)
  print(cvm)
  mstop(cvm)
  plot(cvm)

  ### 25 bootstrap iterations (manually)
  set.seed(290875)
  n <- nrow(bodyfat)
  bs25 <- rmultinom(25, n, rep(1, n)/n)
  cvm <- cvrisk(model, folds = bs25, papply = lapply)
  print(cvm)
  mstop(cvm)
  plot(cvm)

  ### same by default
  set.seed(290875)
  cvrisk(model, papply = lapply)

  ### 25 bootstrap iterations (using cv)
  set.seed(290875)
  bs25_2 <- cv(model.weights(model), type="bootstrap")
  all(bs25 == bs25_2)

\dontrun{
############################################################
## Do not run this example automatically as it takes
## some time (~ 5 seconds depending on the system)

  ### trees
  blackbox <- blackboost(DEXfat ~ ., data = bodyfat)
  cvtree <- cvrisk(blackbox, papply = lapply)
  plot(cvtree)
  
## End(Not run this automatically)  
}


### cvrisk in parallel modes:

\dontrun{
## at least not automatically

## parallel::mclapply() which is used here for parallelization only runs 
## on unix systems (here we use 2 cores)

    cvrisk(model, mc.cores = 2)

## infrastructure needs to be set up in advance

    cl <- makeCluster(25) # e.g. to run cvrisk on 25 nodes via PVM
    myApply <- function(X, FUN, ...) {
      myFun <- function(...) {
          library("mboost") # load mboost on nodes
          FUN(...)
      }
      ## further set up steps as required
      parLapply(cl = cl, X, myFun, ...)
    }
    cvrisk(model, papply = myApply)
    stopCluster(cl)
}

}
\keyword{models}
\keyword{regression}
