% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/connection_spark.R, R/spark_submit.R
\name{spark-connections}
\alias{spark-connections}
\alias{spark_connect}
\alias{spark_connection_is_open}
\alias{spark_disconnect}
\alias{spark_disconnect_all}
\alias{spark_submit}
\title{Manage Spark Connections}
\usage{
spark_connect(
  master,
  spark_home = Sys.getenv("SPARK_HOME"),
  method = c("shell", "livy", "databricks", "test", "qubole", "synapse"),
  app_name = "sparklyr",
  version = NULL,
  config = spark_config(),
  extensions = sparklyr::registered_extensions(),
  packages = NULL,
  scala_version = NULL,
  ...
)

spark_connection_is_open(sc)

spark_disconnect(sc, ...)

spark_disconnect_all(...)

spark_submit(
  master,
  file,
  spark_home = Sys.getenv("SPARK_HOME"),
  app_name = "sparklyr",
  version = NULL,
  config = spark_config(),
  extensions = sparklyr::registered_extensions(),
  scala_version = NULL,
  ...
)
}
\arguments{
\item{master}{Spark cluster url to connect to. Use \code{"local"} to
connect to a local instance of Spark installed via
\code{\link[=spark_install]{spark_install}}.}

\item{spark_home}{The path to a Spark installation. Defaults to the path
provided by the \code{SPARK_HOME} environment variable. If
\code{SPARK_HOME} is defined, it will always be used unless the
\code{version} parameter is specified to force the use of a locally
installed version.}

\item{method}{The method used to connect to Spark. Default connection method
is \code{"shell"} to connect using spark-submit, use \code{"livy"} to
perform remote connections using HTTP, or \code{"databricks"} when using a
Databricks clusters.}

\item{app_name}{The application name to be used while running in the Spark
cluster.}

\item{version}{The version of Spark to use. Required for \code{"local"} Spark
connections, optional otherwise.}

\item{config}{Custom configuration for the generated Spark connection. See
\code{\link{spark_config}} for details.}

\item{extensions}{Extension R packages to enable for this connection. By
default, all packages enabled through the use of
\code{\link[=register_extension]{sparklyr::register_extension}} will be passed here.}

\item{packages}{A list of Spark packages to load. For example, \code{"delta"} or
\code{"kafka"} to enable Delta Lake or Kafka. Also supports full versions like
\code{"io.delta:delta-core_2.11:0.4.0"}. This is similar to adding packages into the
\code{sparklyr.shell.packages} configuration option. Notice that the \code{version}
parameter is used to choose the correct package, otherwise assumes the latest version
is being used.}

\item{scala_version}{Load the sparklyr jar file that is built with the version of
Scala specified (this currently only makes sense for Spark 2.4, where sparklyr will
by default assume Spark 2.4 on current host is built with Scala 2.11, and therefore
`scala_version = '2.12'` is needed if sparklyr is connecting to Spark 2.4 built with
Scala 2.12)}

\item{...}{Additional params to be passed to each `spark_disconnect()` call
(e.g., `terminate = TRUE`)}

\item{sc}{A \code{spark_connection}.}

\item{file}{Path to R source file to submit for batch execution.}
}
\description{
These routines allow you to manage your connections to Spark.

Call `spark_disconnect()` on each open Spark connection
}
\details{
By default, when using \code{method = "livy"}, jars are downloaded from GitHub. But
an alternative path (local to Livy server or on HDFS or HTTP(s)) to \code{sparklyr}
JAR can also be specified through the \code{sparklyr.livy.jar} setting.
}
\examples{
conf <- spark_config()
conf$`sparklyr.shell.conf` <- c(
  "spark.executor.extraJavaOptions=-Duser.timezone='UTC'",
  "spark.driver.extraJavaOptions=-Duser.timezone='UTC'",
  "spark.sql.session.timeZone='UTC'"
)

sc <- spark_connect(
  master = "spark://HOST:PORT", config = conf
)
connection_is_open(sc)

spark_disconnect(sc)
}
