% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/awkreader_v2.R
\name{filtered.fread}
\alias{filtered.fread}
\title{Fast, Filtered Reading of Multiple Files via AWK}
\usage{
filtered.fread(
  the.files,
  path.to.awk = NULL,
  file.pattern = NULL,
  recursive = FALSE,
  header = TRUE,
  delim = ",",
  the.filter = NULL,
  the.variables = ".",
  include.filename = TRUE,
  skip = 0,
  file.header = "file",
  num.files.per.batch = 1000,
  return.as = "result",
  envir = parent.frame(),
  and.symbol = "&",
  or.symbol = "|",
  in.symbol = "\%in\%",
  nin.symbol = "\%nin\%",
  show.warnings = FALSE,
  return.data.table = TRUE,
  nrows = Inf,
  drop = NULL,
  ...
)
}
\arguments{
\item{the.files}{A character vector of file paths to process. Non-existent files are automatically filtered out.}

\item{path.to.awk}{A character string specifying the path to the AWK binary. If \code{NULL} (default),
the function attempts to invoke a global system call to \code{"awk"}.}

\item{file.pattern}{Optional character string to filter file names when \code{the.files} contains directory paths. Accepts simple extensions (e.g., \code{"csv"} or \code{".csv"}),
wildcards (e.g., \code{"*.csv"}), or regular expressions (e.g., \code{"\\.csv$"}). Default is \code{NULL} (includes all files).}

\item{recursive}{Logical. Should directory searches recurse into subdirectories when \code{the.files} contains directory paths? Default is \code{FALSE}.}

\item{header}{A logical value indicating whether the target files contain a header row. Default is \code{TRUE}.
If \code{FALSE}, columns are auto-assigned as \code{V1}, \code{V2}, etc.}

\item{delim}{A character string specifying the column separator within the files. Default is \code{","}.}

\item{the.filter}{A character string or unquoted expression outlining the filtering logic to pass to AWK.
Supports implicit translation of basic math and symbolic calls. Default is \code{NULL} (no filtering).}

\item{the.variables}{A character vector specifying which columns to retain. Use \code{"."} (default)
to retain all columns.}

\item{include.filename}{A logical value indicating whether to include a source file tracking column
in the returned dataset. Default is \code{TRUE}.}

\item{skip}{A numeric offset, a character regex pattern, or a structured list indicating lines to bypass.
If a list is used, it \strong{must follow dot notation}:
\itemize{
\item \code{skip.metadata.rows}: An integer count or a character regex pattern used to identify where
the metadata block ends before hitting the core table.
\item \code{skip.data.rows}: An integer specifying the number of data rows to explicitly skip after the header.
}
Default is \code{0}.}

\item{file.header}{A character string defining the column name for the tracked file origin.
Only utilized if \code{include.filename = TRUE}. Default is \code{"file"}.}

\item{num.files.per.batch}{An integer specifying how many files to aggregate per AWK system pipeline call.
Default is \code{1000}.}

\item{return.as}{A character string specifying the desired return object. Options are:
\itemize{
\item \code{"result"} (default): Returns the compiled dataset.
\item \code{"code"}: Bypasses compilation and returns a character vector of the raw generated AWK scripts.
\item \code{"all"}: Returns a structured list containing both the compiled dataset and the underlying AWK statements.
}}

\item{envir}{The environment context in which evaluation characters or external metadata strings are parsed.
Default is \code{parent.frame()}.}

\item{and.symbol}{A character replacement flag for logical AND statements. Default is \code{"&"}.}

\item{or.symbol}{A character replacement flag for logical OR statements. Default is \code{"|"}.}

\item{in.symbol}{A character replacement flag for inclusion tests. Default is \code{"\%in\%"}.}

\item{nin.symbol}{A character replacement flag for exclusion tests. Default is \code{"\%nin\%"}.}

\item{show.warnings}{A logical value determining whether underlying \code{data.table::fread()} shell messages
should be displayed or suppressed. Default is \code{FALSE}.}

\item{return.data.table}{A logical value indicating whether to return a \code{data.table} object
or a standard \code{data.frame}. Default is \code{TRUE}.}

\item{nrows}{An integer specifying the maximum total rows to parse out from the batch pipeline. Default is \code{Inf}.}

\item{drop}{A character or numeric index vector specifying columns that should be explicitly excluded from the final output.}

\item{...}{Extra parameters forwarded to underlying internal setup routines.}
}
\value{
A \code{data.table} (or \code{data.frame}), or a character vector containing the raw shell commands,
depending on the value passed to \code{return.as}.
}
\description{
Translates R-style filtering statements into highly efficient AWK commands
to process, filter, and select specific columns from multiple flat files simultaneously.
Batched outputs are fast-loaded and bound together into a single dataset.
}
\examples{
\donttest{
# Create a sample CSV file using base R mtcars dataset
tmp_file <- tempfile(fileext = ".csv")
write.csv(mtcars, tmp_file, row.names = FALSE)

# Standard usage with a numeric filter and dot-notation row skipping
my_data <- filtered.fread(
  the.files = tmp_file,
  the.filter = "hp > 100",
  skip = list(skip.metadata.rows = 0, skip.data.rows = 2)
)
print(my_data)

# Clean up temporary file
unlink(tmp_file)
}
}
