% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/shard_map.R
\name{shard_map}
\alias{shard_map}
\title{Parallel Execution with shard_map}
\usage{
shard_map(
  shards,
  fun = NULL,
  borrow = list(),
  out = list(),
  kernel = NULL,
  scheduler_policy = NULL,
  autotune = NULL,
  dispatch_mode = c("rpc_chunked", "shm_queue"),
  dispatch_opts = NULL,
  workers = NULL,
  chunk_size = 1L,
  profile = c("default", "memory", "speed"),
  mem_cap = "2GB",
  recycle = TRUE,
  cow = c("deny", "audit", "allow"),
  seed = NULL,
  diagnostics = TRUE,
  packages = NULL,
  init_expr = NULL,
  timeout = 3600,
  max_retries = 3L,
  health_check_interval = 10L
)
}
\arguments{
\item{shards}{A \code{shard_descriptor} from \code{\link[=shards]{shards()}}, or an integer N to
auto-generate shards.}

\item{fun}{Function to execute per shard. Receives the shard descriptor
as first argument, followed by borrowed inputs and outputs. You can also
select a registered kernel via \verb{kernel=} instead of providing \verb{fun=}.}

\item{borrow}{Named list of shared inputs. These are exported to workers
once and reused across shards. Treated as read-only by default.}

\item{out}{Named list of output buffers (from \code{buffer()}). Workers write
results directly to these buffers.}

\item{kernel}{Optional. Name of a registered kernel (see \code{\link[=list_kernels]{list_kernels()}}).
If provided, \code{fun} must be NULL.}

\item{scheduler_policy}{Optional list of scheduling hints (advanced). Currently:
\itemize{
\item \code{max_huge_concurrency}: cap concurrent chunks whose kernel footprint is
classified as \code{"huge"} (see \code{\link[=register_kernel]{register_kernel()}}).
}}

\item{autotune}{Optional. Online autotuning for scalar-N sharding (advanced).
When \code{shards} is an integer \code{N}, shard_map can adjust shard block sizes over
time based on observed wall time and worker RSS.

Accepted values:
\itemize{
\item \code{NULL} (default): enable online autotuning for \code{shard_map(N, ...)}, off for
precomputed shard descriptors.
\item \code{TRUE} / \code{"online"}: force online autotuning (only applies when \code{shards} is
an integer \code{N}).
\item \code{FALSE} / \code{"none"}: disable autotuning.
\item a list: \code{list(mode="online", max_rounds=..., probe_shards_per_worker=..., min_shard_time=...)}
}}

\item{dispatch_mode}{Dispatch mode (advanced). \code{"rpc_chunked"} is the default
supervised socket-based dispatcher. \code{"shm_queue"} is an opt-in fast mode
that uses a shared-memory task queue to reduce per-task overhead for tiny
tasks. In v1, \code{"shm_queue"} is only supported for \code{shard_map(N, ...)} with
\code{chunk_size=1} and is intended for out-buffer/sink workflows (results are
not gathered).}

\item{dispatch_opts}{Optional list of dispatch-mode specific knobs (advanced).
Currently:
\itemize{
\item For \code{dispatch_mode="rpc_chunked"}:
\itemize{
\item \code{auto_table}: logical. If TRUE, shard_map treats data.frame/tibble return
values as row-group outputs and writes them to a table sink
automatically (one partition per shard id). This avoids building a large
list of tibbles and calling bind_rows() on the master. Requires \verb{out=}
to be empty (use explicit \code{out=list(sink=table_sink(...))} otherwise).
\item \code{auto_table_materialize}: \code{"never"}, \code{"auto"}, or \code{"always"} (default \code{"auto"}).
\item \code{auto_table_max_bytes}: numeric/integer. For \code{"auto"}, materialize only
if estimated output size <= this threshold (default 256MB).
\item \code{auto_table_mode}: \code{"row_groups"} (default) or \code{"partitioned"}.
\item \code{auto_table_path}: optional output directory (default tempdir()).
\item \code{auto_table_format}: \code{"auto"}, \code{"rds"} (default), or \code{"native"}.
\item \code{auto_table_schema}: optional \code{shard_schema} for validation/native encoding.
}
\item For \code{dispatch_mode="shm_queue"}:
\itemize{
\item \code{block_size}: integer. If provided, overrides the default heuristic for
contiguous shard block sizing.
\item \code{queue_backing}: one of \code{"mmap"} or \code{"shm"} (default \code{"mmap"}).
\item \code{error_log}: logical. If TRUE, workers write a bounded per-worker error
log to disk to aid debugging failed tasks (default FALSE).
\item \code{error_log_max_lines}: integer. Maximum lines per worker in the error
log (default 100).
}
}}

\item{workers}{Integer. Number of worker processes. If NULL, uses existing
pool or creates one with \code{detectCores() - 1}.}

\item{chunk_size}{Integer. Shards to batch per worker dispatch (default 1).
Higher values reduce RPC overhead but may hurt load balancing.}

\item{profile}{Execution profile: \code{"default"}, \code{"memory"} (aggressive recycling),
or \code{"speed"} (minimal overhead). With \code{profile="speed"}, shard_map will
automatically enable \code{dispatch_mode="shm_queue"} when possible for
\code{shard_map(N, ...)} out-buffer workflows (scalar \code{N}, \code{chunk_size=1}),
unless \code{dispatch_mode} is explicitly specified.}

\item{mem_cap}{Memory cap per worker (e.g., "2GB"). Workers exceeding this
are recycled.}

\item{recycle}{Logical or numeric. If TRUE, recycle workers on RSS drift.
If numeric, specifies drift threshold (default 0.5 = 50\% growth).}

\item{cow}{Copy-on-write policy for borrowed inputs: \code{"deny"} (error on mutation),
\code{"audit"} (detect and flag), or \code{"allow"} (permit with tracking).}

\item{seed}{Integer. RNG seed for reproducibility. If NULL, no seed is set.}

\item{diagnostics}{Logical. Collect detailed diagnostics (default TRUE).}

\item{packages}{Character vector. Additional packages to load in workers.}

\item{init_expr}{Expression to evaluate in each worker on startup.}

\item{timeout}{Numeric. Seconds to wait for each shard (default 3600).}

\item{max_retries}{Integer. Maximum retries per shard on failure (default 3).}

\item{health_check_interval}{Integer. Check worker health every N shards (default 10).}
}
\value{
A \code{shard_result} object containing:
\itemize{
\item \code{results}: List of results from each shard (if fun returns values)
\item \code{failures}: Any permanently failed shards
\item \code{diagnostics}: Timing, memory, and worker statistics
\item \code{pool_stats}: Pool-level statistics
}
}
\description{
Core parallel execution engine with supervision, shared inputs, and output buffers.

Executes a function over shards in parallel with worker supervision,
shared inputs, and explicit output buffers. This is the primary entry
point for shard's parallel execution model.
}
\examples{
\donttest{
blocks <- shards(1000, workers = 2)
result <- shard_map(blocks, function(shard) {
  sum(shard$idx^2)
}, workers = 2)
pool_stop()
}
}
