% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/chunk.R, R/offload.R
\name{collect_chunked}
\alias{collect_chunked}
\alias{collect_chunked.default}
\alias{collect_chunked.vectra_node}
\alias{collect_chunked.vectra_partition}
\title{Fold a function over a query, one batch at a time}
\usage{
collect_chunked(x, f, .init = NULL, combine = NULL, commutative = FALSE)

\method{collect_chunked}{default}(x, f, .init = NULL, combine = NULL, commutative = FALSE)

\method{collect_chunked}{vectra_node}(x, f, .init = NULL, combine = NULL, commutative = FALSE)

\method{collect_chunked}{vectra_partition}(x, f, .init = NULL, combine = NULL, commutative = FALSE)
}
\arguments{
\item{x}{A \code{vectra_node} (from \code{\link[=tbl]{tbl()}}, \code{\link[=tbl_csv]{tbl_csv()}}, \code{\link[=tbl_tiff]{tbl_tiff()}}, ... and
any chain of verbs). It is consumed: after \code{collect_chunked()} returns, the
stream is drained and \code{x} cannot be collected again.}

\item{f}{A function of two arguments \verb{function(acc, chunk)} returning the
updated accumulator. \code{chunk} is a data.frame holding the next batch of
rows.}

\item{.init}{Initial accumulator value. Passed to \code{f} with the first batch
and returned unchanged if the query yields no rows. When \code{combine} is
supplied this is also the monoid identity (the value \code{combine} leaves
unchanged).}

\item{combine}{Optional function \verb{function(acc, acc)} that merges two
accumulators. Supplying it declares the reduction a monoid with \code{.init} as
identity, which is what lets the fold run over the independent shards of a
partition (\code{offload(x, by = ...)}) and have the partial results merged
correctly. For a plain node the stream is a single sequence, so \code{combine}
is not needed and is ignored.}

\item{commutative}{Logical; declare that \code{combine} does not depend on the
order of its arguments. Lets a partitioned fold merge shards in any order
(no stable sort required). Default \code{FALSE}.}
}
\value{
The final accumulator. For a node: \code{f} applied left-to-right across
every batch, seeded with \code{.init}. For a partition: each shard folded with
\code{f}/\code{.init}, then those per-shard accumulators merged with \code{combine}.
}
\description{
Streams a lazy query through R in bounded pieces and reduces them with \code{f},
instead of materializing the whole result the way \code{\link[=collect]{collect()}} does. The
engine pulls one batch (a data.frame of up to a few hundred thousand rows)
at a time; \code{f} is called as \code{f(acc, chunk)} and its return value becomes the
accumulator for the next batch. Peak memory is one batch plus whatever the
accumulator holds, so a result far larger than RAM can be reduced to a small
summary in a single pass.
}
\details{
This is the streaming counterpart to a fold (\code{Reduce()}): use it when the
query returns more rows than fit in memory but the \emph{reduction} is small. A
running count, per-group sufficient statistics, the cross-products \verb{X'X} and
\verb{X'y} behind a linear fit, an online mean or histogram - all accumulate in
bounded space across the stream. When you instead need the model-fitting
consumer to drive the iteration (and to re-read the data on each pass, as an
iteratively reweighted GLM does), use \code{\link[=chunk_feeder]{chunk_feeder()}}.
}
\examples{
f <- tempfile(fileext = ".vtr")
write_vtr(mtcars, f)

# Row count without materializing the result.
collect_chunked(tbl(f), function(acc, chunk) acc + nrow(chunk), .init = 0L)

# Accumulate the normal-equation pieces X'X and X'y for an exact OLS fit
# of mpg ~ wt + hp, in one streaming pass.
acc <- collect_chunked(
  tbl(f) |> select(mpg, wt, hp),
  function(acc, chunk) {
    X <- cbind(1, chunk$wt, chunk$hp)
    y <- chunk$mpg
    list(XtX = acc$XtX + crossprod(X), Xty = acc$Xty + crossprod(X, y))
  },
  .init = list(XtX = matrix(0, 3, 3), Xty = matrix(0, 3, 1))
)
solve(acc$XtX, acc$Xty)            # same as coef(lm(mpg ~ wt + hp, mtcars))
unlink(f)

}
\seealso{
\code{\link[=chunk_feeder]{chunk_feeder()}} for pull-based consumers such as \code{biglm::bigglm()},
\code{\link[=offload]{offload()}} for the replay cache and the partitioned monoidal reduce,
\code{\link[=group_map]{group_map()}} and \code{\link[=group_modify]{group_modify()}} for per-shard application, and
\code{\link[=collect]{collect()}} to materialize the full result.
}
