#' Submit programs for execution in parallel. Each program in a list will be submitted
#' at the same time.
#'
#' @param prog_list A list of R program paths to execute
#' @param submit_func function that will be used to submit jobs
#' @param wait_func function that checks job status and returns when job finishes executing
#' @param ... arguments that will be passed to submit_func and wait_func functions
#'
#' @return list job IDs associated with executed programs
#' @noRd
#'
#' @examples \dontrun{
#' job_ids <- batch_submit_parallel(
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_dm.sas",
#'     "/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_ae.sas"))
#'  }
batch_submit_parallel <- function(prog_list,
                                  submit_func=abba_rslauncher_submit_job_local,
                                  wait_func=abba_rslauncher_watch_job_local,
                                  ...) {
  # Submit jobs for the programs and wait until everything has been executed
  job_ids <- sapply(prog_list, function(x) submit_func(x, ...))

  # wait until all programs finish executing
  results <- wait_func(job_ids, ...)

  # return job ids generated by submitting programs for execution in workbench
  return(job_ids)
}

#' Submit programs for execution in order defined by structure of input list.
#' Programs inside sublists will be executed in parallel, and sublists themselves
#' would be submitted sequentially.
#'
#' @param prog_list A list of R program paths to execute
#' @param sequential when sequential=TRUE, prog_list is flattened and everything is executed sequentially.
#' @param submit_func function that will be used to submit jobs
#' @param wait_func function that checks job status and returns when job finishes executing
#' @param succeed_func function that returns TRUE if job finished running without errors and FALSE otherwise
#' @param col_name Name of the column that contains run group numbers when prog_list is a data frame
#' @param halt_on_error If TRUE: if prog_list contains program inputs/outputs - programs that depend on failed
#' program will not be executed; if prog_list contains only program paths - when program fails,
#' entire batch will stop executing. TRUE by default
#' @param rerun_unchanged_programs If FALSE: will not re-run programs whose code
#' and inputs have not been modified since last batch run with update_cache=TRUE.
#' @param update_cache if TRUE, file hash for programs and their inputs will be
#' calculated and written to `cache_folder` after the batch run. If FALSE, the
#' cache will not be created or updated. FALSE by default.
#' @param cache_folder Path to the folder where hash-sums of programs and
#' their inputs will be stored. Required when `update_cache=TRUE` or
#' `rerun_unchanged_programs=FALSE`. The default value can be set via the
#' `abba.default_cache_folder` option; abba never falls back to writing caches
#' under the user's filespace.
#' @param ... arguments that will be passed to submit_func and wait_func functions
#'
#' @return list job IDs associated with executed programs
#' @export
#'
#' @examples \dontrun{
#' job_ids <- abba_submit_batch(list(
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/sdtm/dm.sas"),
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/sdtm/ae.sas"),
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_dm.sas",
#'     "/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_ae.sas")))
#'  }
abba_submit_batch <- function(prog_list,
                              sequential=FALSE,
                              submit_func=abba_rslauncher_submit_job_local,
                              wait_func=abba_rslauncher_watch_job_local,
                              succeed_func=abba_rslauncher_get_job_succeeded_local,
                              col_name='run_group',
                              halt_on_error=TRUE,
                              rerun_unchanged_programs=TRUE,
                              update_cache=FALSE,
                              cache_folder=getOption("abba.default_cache_folder"),
                              ...) {

  if ((update_cache || !rerun_unchanged_programs) && is.null(cache_folder)) {
    stop("cache_folder must be supplied when update_cache=TRUE or rerun_unchanged_programs=FALSE. Set it via the cache_folder argument or the abba.default_cache_folder option.")
  }

  # if sequential=TRUE is specified - flatten the list and this will execute everything sequentially
  # REGARDLESS of whether prog_list is a list, a character vector or a data frame
  if (sequential==TRUE){
    prog_list_converted <- unlist(dataframe_to_batch_list(prog_list))
  }
  else {prog_list_converted <- prog_list}

  # execute each item in the prog_list_converted sequentially. Programs inside
  # each list element, which can also be a vector of program paths,
  # will be executed in parallel, unless 'sequential=TRUE' is specified
  job_ids <- list()
  previous_run_ok <- TRUE
  previous_run_programs <- c()

  # check if run groups are present for data frame
  unique_run_groups <- get_run_groups(prog_list_converted)

  for (rg in unique_run_groups){

    # check if programs/their inputs had any changes and remove programs whose code
    # and inputs(if specified) have not changed since last hash collection(typicaly from previous batch run)
    if (!rerun_unchanged_programs){
      prog_list_converted <- remove_unchanged_programs(prog_list_converted,
                                                       run_group=rg,
                                                       col_name=col_name,
                                                       cache_folder=cache_folder)
    }
    # check if programs from previous run group were executed completely
    batch_check_results <-
      batch_run_control(prog_list_converted,
                        current_group=rg,
                        previous_run_programs=previous_run_programs,
                        previous_run_ok=previous_run_ok,
                        col_name='run_group',
                        halt_on_error=halt_on_error,
                        ...)

    prog_list_converted <- batch_check_results$prog_list

    # if batch input is a list, do not run the next group
    if (batch_check_results$stop == TRUE && halt_on_error){break}

    # select programs for running in parallel
    parallel_run <- select_parallel_run(prog_list_converted, rg, col_name=col_name)
    # message that would show batch progress. Only show it if there is at least one program to run
    if (length(parallel_run) > 1){
      message(sprintf("Submitting programs for parallel run:\n\t%s", paste(parallel_run, collapse='\n\t')))
    }
    else if (length(parallel_run) == 1){
      message(sprintf("Submitting program:\n\t%s", paste(parallel_run, collapse='\n\t')))
    }
    # submit the run
    new_run <- batch_submit_parallel(parallel_run,
                                     submit_func=submit_func,
                                     wait_func=wait_func,
                                     ...)
    # add executed jobs to the list
    job_ids <- c(job_ids, new_run)
    # collect information about this run group exit codes to decide if we should
    # continue with the batch run on the next loop
    previous_run_ok <- succeed_func(new_run)
    previous_run_programs <- parallel_run
  }
  # update program and inputs hash after batch has been run
  # only update hashes if re-run check has been explicitly enabled
  if (update_cache){
    update_program_hashes(prog_list, cache_folder=cache_folder)
    }

  return(unlist(job_ids))
}


#' Submit programs for execution in order defined by structure of input list.
#'
#' @param prog_list A list of R program paths to execute
#' @param sequential when sequential=TRUE, prog_list is flattened and everything is executed sequentially.
#' @param submit_func function that will be used to submit jobs
#' @param wait_func function that checks job status and returns when job finishes executing
#' @param succeed_func function that returns TRUE if job finished running without errors and FALSE otherwise
#' @param status_func function that returns descriptive job statuses
#' @param col_name Name of the column that contains run group numbers when prog_list is a data frame
#' @param halt_on_error If TRUE: if prog_list contains program inputs/outputs - programs that depend on failed
#' program will not be executed; if prog_list contains only program paths - when program fails,
#' entire batch will stop executing. TRUE by default
#' @param rerun_unchanged_programs If FALSE: will not re-run programs whose code
#' and inputs have not been modified since last batch run with update_cache=TRUE.
#' @param update_cache if TRUE, file hash for programs and their inputs will be
#' calculated and written to `cache_folder` after the batch run. If FALSE, the
#' cache will not be created or updated. FALSE by default.
#' @param cache_folder Path to the folder where hash-sums of programs and
#' their inputs will be stored. Required when `update_cache=TRUE` or
#' `rerun_unchanged_programs=FALSE`. The default value can be set via the
#' `abba.default_cache_folder` option; abba never falls back to writing caches
#' under the user's filespace.
#' @param ... arguments that will be passed to submit_func and wait_func functions
#'
#' @return Data frame containing program names, job ids, execution statuses
#' @export
#'
#' @examples \dontrun{
#' job_ids <- abba_submit_batch_and_get_results(list(
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/sdtm/dm.sas"),
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/sdtm/ae.sas"),
#'   c("/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_dm.sas",
#'     "/mnt/work_drive/proj/comp/prot/task/development/prod/program/tfl/t1_ae.sas")),
#'   sequential=TRUE)
#'  }
abba_submit_batch_and_get_results <- function(prog_list,
                                              sequential=FALSE,
                                              submit_func=abba_rslauncher_submit_job_local,
                                              wait_func=abba_rslauncher_watch_job_local,
                                              succeed_func=abba_rslauncher_get_job_succeeded_local,
                                              status_func=rslauncher_get_job_display_status,
                                              col_name='run_group',
                                              halt_on_error=TRUE,
                                              rerun_unchanged_programs=TRUE,
                                              update_cache=FALSE,
                                              cache_folder=getOption("abba.default_cache_folder"),
                                              ...) {
  # use universal batch runner to submit programs and wait for completion
  job_ids <- abba_submit_batch(prog_list,
                               sequential=sequential,
                               submit_func=submit_func,
                               wait_func=wait_func,
                               succeed_func=succeed_func,
                               col_name=col_name,
                               halt_on_error=halt_on_error,
                               rerun_unchanged_programs=rerun_unchanged_programs,
                               update_cache=update_cache,
                               cache_folder=cache_folder,
                               ...)
  # collect batch results in a data frame
  results <- compose_batch_results(
    job_ids=job_ids,
    prog_names=unlist(dataframe_to_batch_list(prog_list)),
    status_func=status_func)

  return(results)
}
