% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/score_model_out.R
\name{score_model_out}
\alias{score_model_out}
\title{Score model output predictions}
\usage{
score_model_out(
  model_out_tbl,
  oracle_output,
  metrics = NULL,
  relative_metrics = NULL,
  baseline = NULL,
  summarize = TRUE,
  by = "model_id",
  output_type_id_order = NULL,
  compound_taskid_set = NULL,
  transform = NULL,
  transform_append = FALSE,
  transform_label = NULL,
  ...
)
}
\arguments{
\item{model_out_tbl}{Model output tibble with predictions}

\item{oracle_output}{Predictions that would have been generated by an oracle
model that knew the observed target data values in advance}

\item{metrics}{Character vector of scoring metrics to compute. If \code{NULL}
(the default), appropriate metrics are chosen automatically. See details
for more.}

\item{relative_metrics}{Character vector of scoring metrics for which to
compute relative skill scores. The \code{relative_metrics} should be a subset of
\code{metrics} and should only include proper scores. Interval coverage metrics
(e.g., \code{"interval_coverage_90"}) and \code{"bias"} are not supported as relative
metrics: interval coverage targets a nominal level rather than a "lower is
better" direction, and bias is a signed quantity for which a geometric-mean
pairwise ratio has no clean interpretation. If \code{NULL} (the default), no
relative metrics will be computed. Relative metrics are only computed if
\code{summarize = TRUE}, and require that \code{"model_id"} is included in \code{by}.
Relative skill is computed within each comparison group (the \code{by} columns
other than \code{"model_id"}). If a comparison group contains only one model,
that group's relative-skill columns are filled with \code{1} (a model has trivial
skill \code{1} relative to itself) rather than erroring.}

\item{baseline}{String with the name of a model to use as a baseline for
relative skill scores. If a baseline is given, then a scaled relative skill
with respect to the baseline will be returned. By default (\code{NULL}), relative
skill will not be scaled with respect to a baseline model. If the baseline is
absent from a comparison group, that group's relative and scaled relative
skill are reported as \code{NA} (with a warning) while its absolute scores are
still returned. This \code{NA} case only arises when scoring is disaggregated:
with \code{by = "model_id"} alone there is a single comparison group containing
every model, so relative skill is always defined and is never \code{NA}. A
baseline absent from the data entirely is an error.}

\item{summarize}{Boolean indicator of whether summaries of forecast scores
should be computed. Defaults to \code{TRUE}.}

\item{by}{Character vector naming columns to summarize by. For example,
specifying \code{by = "model_id"} (the default) will compute average scores for
each model.}

\item{output_type_id_order}{For ordinal variables in pmf format, this is a
vector of levels for pmf forecasts, in increasing order of the levels. The
order of the values for the output_type_id can be found by referencing the
hub's tasks.json configuration file. For all output types other than pmf,
this is ignored.}

\item{compound_taskid_set}{When \code{NULL} (the default), sample forecasts are
scored marginally (each modeling task scored independently). When a character
vector of task ID column names is provided, it sets the compound grouping:
the specified columns stay constant within each sample draw, while the
remaining task ID dimensions vary within a draw and are scored jointly
(using the energy score). Only applicable when
\code{output_type == "sample"}. The value of \code{compound_taskid_set} can be found
by referencing the hub's \code{tasks.json} configuration file.}

\item{transform}{A function to apply as a scale transformation to both
predictions and observations before scoring. Common choices include
\code{\link[scoringutils]{log_shift}} (recommended for log transformation
as it handles zeros via an offset parameter), \code{sqrt}, or
\code{log1p}. Avoid using \code{log} directly if data may contain zeros.
If \code{NULL} (the default), no transformation is applied. Only supported for
quantile, mean, and median output types.}

\item{transform_append}{Logical. If \code{FALSE} (the default), scores are
computed only on the transformed scale. If \code{TRUE}, scores are computed on
both original and transformed scales, with a \code{scale} column distinguishing
them. Ignored if \code{transform = NULL}.}

\item{transform_label}{A character string label for the transformation
(e.g., "log"). If \code{NULL} (the default), the label is auto-generated from
the function name (e.g., "log_shift" for \code{scoringutils::log_shift}).
Required when using an anonymous transform function. Ignored if
\code{transform = NULL}. Note: the label only appears in output when
\code{transform_append = TRUE}, where it distinguishes transformed rows
(labeled with this value) from original rows (labeled "natural") in the
\code{scale} column.}

\item{...}{Additional arguments passed to the \code{transform} function. For
example, allows use of the \code{offset} and \code{base} arguments of
\code{\link[scoringutils:log_shift]{scoringutils::log_shift()}}. Ignored if \code{transform = NULL}.}
}
\value{
A tibble of scores, inheriting from scoringutils' \code{scores} class so
that downstream scoringutils helpers (e.g. \code{\link[scoringutils:get_metrics]{scoringutils::get_metrics()}})
continue to work. The tibble has a \code{metrics} attribute holding the names of
the scoring rules that were applied.
}
\description{
Scores model outputs with a single \code{output_type} against observed data.
}
\details{
See the hubverse documentation for the expected format of the
\href{https://docs.hubverse.io/en/latest/user-guide/target-data.html#oracle-output}{oracle output data}.

Default metrics are provided by the \code{scoringutils} package. You can select
metrics by passing in a character vector of metric names to the \code{metrics}
argument.

The following metrics can be selected (all are used by default) for the
different \code{output_type}s:

\strong{Quantile forecasts:} (\code{output_type == "quantile"})
\itemize{
\item wis
\item overprediction
\item underprediction
\item dispersion
\item bias
\item ae_median
\item "interval_coverage_XX": interval coverage at the "XX" level. For example,
"interval_coverage_95" is the 95\% interval coverage rate, which would be calculated
based on quantiles at the probability levels 0.025 and 0.975.
}

See \link[scoringutils:get_metrics.forecast_quantile]{scoringutils::get_metrics.forecast_quantile} for details.

\strong{Nominal forecasts:} (\code{output_type == "pmf"} and \code{output_type_id_order} is \code{NULL})
\itemize{
\item log_score
}

See \link[scoringutils:get_metrics.forecast_nominal]{scoringutils::get_metrics.forecast_nominal} for details.

\strong{Ordinal forecasts:} (\code{output_type == "pmf"} and \code{output_type_id_order} is a vector)
\itemize{
\item log_score
\item rps
}

See \link[scoringutils:get_metrics.forecast_ordinal]{scoringutils::get_metrics.forecast_ordinal} for details.

\strong{Median forecasts:} (\code{output_type == "median"})
\itemize{
\item ae_point: absolute error of the point forecast (recommended for the median, see Gneiting (2011))
}

See \link[scoringutils:get_metrics.forecast_point]{scoringutils::get_metrics.forecast_point} for details.

\strong{Mean forecasts:} (\code{output_type == "mean"})
\itemize{
\item \code{se_point}: squared error of the point forecast (recommended for the mean, see Gneiting (2011))
}

\strong{Sample forecasts (marginal):} (\code{output_type == "sample"}, \code{compound_taskid_set = NULL})
\itemize{
\item bias
\item dss
\item crps
\item overprediction
\item underprediction
\item dispersion
\item log_score
\item mad
\item ae_median
\item se_mean
}

Note: \code{log_score} uses kernel density estimation, which may not be
appropriate for integer-valued forecasts. \code{scoringutils} will warn when
this is detected.

See \link[scoringutils:get_metrics.forecast_sample]{scoringutils::get_metrics.forecast_sample} for details.

\strong{Sample forecasts (compound):} (\code{output_type == "sample"}, \code{compound_taskid_set} provided)
\itemize{
\item energy_score
\item variogram_score
}

See \link[scoringutils:get_metrics.forecast_multivariate_sample]{scoringutils::get_metrics.forecast_multivariate_sample} for details.
The output includes a \code{.mv_group_id} column assigned by \code{scoringutils} to
identify the multivariate groups used for scoring (equivalent to the
\code{compound_idx} concept in the
\href{https://docs.hubverse.io/en/latest/user-guide/sample-output-type.html}{hubverse sample output type documentation }).
Correct scoring depends on providing the right \code{compound_taskid_set} from
the hub's \code{tasks.json} configuration. If the specified grouping does not
match the actual dependence structure of the submitted samples (e.g.,
because some models submitted coarser samples than configured),
\code{.mv_group_id} may not correspond to the original sample draws as
indicated by their \code{output_type_id} values.

See \link[scoringutils:add_relative_skill]{scoringutils::add_relative_skill} for details on relative skill scores.
}
\examples{
\dontshow{if (requireNamespace("hubExamples", quietly = TRUE)) withAutoprint(\{ # examplesIf}
# compute WIS and interval coverage rates at 80\% and 90\% levels based on
# quantile forecasts, summarized by the mean score for each model
quantile_scores <- score_model_out(
  model_out_tbl = hubExamples::forecast_outputs |>
    dplyr::filter(.data[["output_type"]] == "quantile"),
  oracle_output = hubExamples::forecast_oracle_output,
  metrics = c("wis", "interval_coverage_80", "interval_coverage_90"),
  relative_metrics = "wis",
  by = "model_id"
)
quantile_scores

# compute log scores based on pmf predictions for categorical targets,
# summarized by the mean score for each combination of model and location.
# Note: if the model_out_tbl had forecasts for multiple targets using a
# pmf output_type with different bins, it would be necessary to score the
# predictions for those targets separately.
pmf_scores <- score_model_out(
  model_out_tbl = hubExamples::forecast_outputs |>
    dplyr::filter(.data[["output_type"]] == "pmf"),
  oracle_output = hubExamples::forecast_oracle_output,
  metrics = c("log_score", "rps"),
  by = c("model_id", "location", "horizon"),
  output_type_id_order = c("low", "moderate", "high", "very high")
)
head(pmf_scores)

# Score sample forecasts marginally (each modeling task scored independently).
# Note: this data has compound structure (samples span horizons), but marginal
# scoring is still valid -- it evaluates each horizon independently.
sample_scores <- score_model_out(
  model_out_tbl = hubExamples::forecast_outputs |>
    dplyr::filter(.data[["output_type"]] == "sample"),
  oracle_output = hubExamples::forecast_oracle_output,
  metrics = "crps",
  by = "model_id"
)
sample_scores

# Score compound sample forecasts jointly using the energy score.
# compound_taskid_set specifies which task IDs stay constant within
# a sample group -- here, each sample draw spans all horizons for a
# given reference_date and location (i.e., a trajectory over time).
compound_scores <- score_model_out(
  model_out_tbl = hubExamples::forecast_outputs |>
    dplyr::filter(.data[["output_type"]] == "sample"),
  oracle_output = hubExamples::forecast_oracle_output,
  compound_taskid_set = c("reference_date", "location"),
  by = "model_id"
)
compound_scores
\dontshow{\}) # examplesIf}
}
\references{
Gneiting, Tilmann. 2011. "Making and Evaluating Point Forecasts." Journal of the
American Statistical Association 106 (494): 746–62. \doi{10.1198/jasa.2011.r10138}.
}
