% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/add_content.R
\name{add_content}
\alias{add_content}
\title{Add Content from PDF Documents}
\usage{
add_content(
  pub_id,
  year,
  type = c("text", "image"),
  ...,
  srcdir = "archive",
  destdir = tempdir(),
  ignore = NULL,
  pubs = inlpubs::pubs,
  overwrite = FALSE
)
}
\arguments{
\item{pub_id}{'character' vector.
Unique identifier(s) of the publication(s) to process.
Publications may instead, or additionally, be selected with \code{year}.}

\item{year}{'integer' vector.
Year(s) of publication. All publications from the given year(s) are selected,
in addition to any specified by \code{pub_id}.}

\item{type}{'character' string.
Type of content to extract from each PDF, either \code{"text"} (the default) or
\code{"image"} (a cover image).}

\item{...}{Additional arguments passed to the extraction function:
\code{\link{get_pdf_text}} when \code{type = "text"}, or \code{\link{get_pdf_image}} when \code{type = "image"}.}

\item{srcdir}{'character' string.
Source directory containing the publication PDFs, organized into
year subdirectories (see Details). Defaults to \code{"archive"} in the working directory.}

\item{destdir}{'character' string.
Destination directory for the extracted output files. Created if it does not
already exist. Defaults to the session's temporary directory.}

\item{ignore}{'character' vector.
Publication identifier(s) to skip. Useful for documents that are known to fail
automatic extraction.}

\item{pubs}{'pub' table.
Publications of the INLPO; see the \code{\link{pubs}} dataset for the expected data format.}

\item{overwrite}{'logical' flag.
Whether to overwrite an existing output file. Defaults to \code{FALSE}, in which case
publications with an existing output are skipped.}
}
\value{
Invisibly returns a 'character' vector of paths to the saved output files,
carrying a \code{"failed"} attribute with the identifiers of any publications whose
extraction failed. Returns \code{NULL} (invisibly) when no publications remain to
process.
}
\description{
Extract the text or cover image from publication PDF documents and
save the result to disk, for incorporation into the \pkg{inlpubs} package corpus.
Publications are selected by identifier or by year, and each must already be
present in the \code{pubs} dataset before this function is run.
}
\details{
For each selected publication, the function locates its primary PDF (the
first file listed in the publication's \code{files} entry) within the source directory,
then delegates extraction to \code{\link{get_pdf_text}} (for \code{type = "text"}) or
\code{\link{get_pdf_image}} (for \code{type = "image"}). Extra arguments passed through \code{...}
are forwarded to whichever of those functions is used.

Source documents are expected to follow the archive layout
\verb{srcdir/<year>/<pub_id>/<file>}, for example
\code{archive/2005/KnobelOthers2005/ofr20051223.pdf}. Output files are written to
\code{destdir} and named \verb{pub-<pub_id>}, with the appropriate extension (\code{.txt} or
\code{.jpg}) supplied by the extraction function.

Publications whose \code{files} entry is empty, and those listed in \code{ignore}, are
skipped with a message. By default, existing output files are left in place;
set \code{overwrite = TRUE} to regenerate them.

Extraction failures (including renderer crashes or timeouts on problematic
PDFs) are caught per publication: the offending document is reported and
skipped, and processing continues with the next one. The identifiers of any
failed documents are returned in the \code{"failed"} attribute of the result.

Processing the full archive can take a considerable amount of time. For routine
use, run this function interactively and one publication year at a time so that
progress can be monitored.
}
\examples{
\dontrun{
  # Extract text for a single publication into the corpus folder
  add_content("KnobelOthers2005", destdir = "data-raw/corpus")

  # Extract text for every publication from a given year
  add_content(year = 2005, destdir = "data-raw/corpus")

  # Extract cover images into the vignettes folder
  add_content(year = 2005, type = "image", destdir = "vignettes")

  # Process several years, skipping a document that fails extraction
  add_content(year = 2003:2005, ignore = "Mazurek2004", destdir = "data-raw/corpus")
}
}
\seealso{
\code{\link{get_pdf_text}} and \code{\link{get_pdf_image}} for the underlying extraction
functions, and the \code{\link{pubs}} dataset for publication metadata.
}
\author{
J.C. Fisher, U.S. Geological Survey, Idaho Water Science Center
}
