#! /usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

#
# Environment variables that can be set to influence the behavior
# of this script
#
# ACCUMULO_LOCALHOST_ADDRESSES - set to a space delimited string of localhost names
#                                and addresses to override the default lookups
#

function print_usage() {
  cat <<EOF
$(cyan Usage): $(green accumulo-cluster) <$(yellow command)> [<$(yellow option)> ...]

$(cyan Options):
  $(cyan General options):
    $(green --dry-run)                Prints information and commands, but does not execute them
    $(green --local)                  Operates on matching local services only (no SSH)

  $(cyan 'Service type selection options (if none are used, all service types are selected)'):
    $(green --manager)                Selects the manager service (oversees cluster operations)
    $(green --gc)                     Selects the gc service (cleans up unused files)
    $(green --monitor)                Selects the monitor web service (shows cluster information)
    $(green --compaction-coordinator) Selects the compaction-coordinator service (external compactions)
    $(green --tservers)               Selects the tablet server services (read/write operations on tablets)
    $(green --sservers)[$(yellow '=group')]       Selects the scan server services (read-only eventually consistent scans)
    $(green --compactors)[$(yellow '=group')]     Selects the compactor services (external compactions)
    $(green --no-tservers)            $(yellow Deprecated). Selects service types except tservers (for backwards compatibility)
    $(yellow NOTE): some server types support an optional $(yellow group) name. If it is not provided or is empty, then
          all groups are considered. Declare multiple groups as a single space-separated parameter. Later
          options overwrite earlier ones, so $(yellow '--sservers="g1 g2" --sserver') operates on all sserver groups.

$(cyan Commands):
  $(green create-config)              Creates cluster config (ignores service selection options)
  $(green start)                      Starts Accumulo cluster services
  $(green stop)                       Stops Accumulo cluster services
  $(green restart)                    Restarts Accumulo cluster services
  $(green kill)                       Kills Accumulo cluster services
  $(green prune)                      Reomves zookeeper locks of extra processes

  $(cyan Deprecated commands):
    $(green start-non-tservers)       $(yellow Deprecated). Alias for "start --no-tservers"
    $(green start-servers)            $(yellow Deprecated). Alias for "start"
    $(green stop-servers)             $(yellow Deprecated). Alias for "stop"
    $(green start-tservers)           $(yellow Deprecated). Alias for "start --tservers"
    $(green stop-tservers)            $(yellow Deprecated). Alias for "stop --tservers"
    $(green start-here)               $(yellow Deprecated). Alias for "start --local"
    $(green stop-here)                $(yellow Deprecated). Alias for "stop --local"

$(cyan Examples):
  $(purple 'accumulo-cluster start')                              $(blue '# start all servers')
  $(purple 'accumulo-cluster start --dry-run')                    $(blue '# print debug information and commands to be executed')
  $(purple 'accumulo-cluster start --local')                      $(blue '# start all local services')
  $(purple 'accumulo-cluster start --local --manager')            $(blue '# start local manager services')
  $(purple 'accumulo-cluster start --tservers')                   $(blue '# start all tservers')
  $(purple 'accumulo-cluster start --sservers=group1')            $(blue '# start all group1 sservers')
  $(purple 'accumulo-cluster start --sservers="group1 group2"')   $(blue '# start all group1 and group2 sservers')
  $(purple 'accumulo-cluster start --local --manager --tservers') $(blue '# Start the local manager and local tservers')
  $(purple 'accumulo-cluster prune --compactors')                 $(blue '# prune all extra compactors across all groups')
  $(purple 'accumulo-cluster prune --compactors="group1"')        $(blue '# prune extra compactors running in group1')

EOF
}

function checkTerminalSupportsColor() {
  local c
  # get the number of colors supported
  c=$(tput colors 2>/dev/null) || c=-1
  # if STDOUT is a terminal and the number of colors is at least 8
  [[ -t 1 && $c -ge 8 ]]
}

function decolorize() {
  # this only decolorizes escape sequences that we've added
  # namely, the color codes 0;31m through 0;37m and the 0m reset
  # it also can't deal with arrays, so can only work on a single param
  if [[ $# -ne 1 ]]; then
    echo "$(red ERROR): Couldn't decolorize multiple items: $*"
  fi
  local myvar=$1
  myvar="${myvar//[[:cntrl:]]\[0;3[1-7]m/}" # remove the color codes
  myvar="${myvar//[[:cntrl:]]\[0m/}"        # remove the reset code
  echo "$myvar"
}
function colorize() {
  local c=$1
  shift
  [[ $COLOR_ENABLED == 1 ]] && echo -e "\\e[0;${c}m${*}\\e[0m" || echo "$@"
}
function red() { colorize 31 "$@"; }
function green() { colorize 32 "$@"; }
function yellow() { colorize 33 "$@"; }
function blue() { colorize 34 "$@"; }
function purple() { colorize 35 "$@"; }
function cyan() { colorize 36 "$@"; }
function white() { colorize 37 "$@"; }

function parse_args() {
  local originalArgs=("$@")

  DEBUG=0
  ARG_LOCAL=0
  ARG_ALL=1
  ARG_MANAGER=0
  ARG_GC=0
  ARG_MONITOR=0
  ARG_COORDINATOR=0
  ARG_TSERVER=0
  ARG_TSERVER_GROUP=""
  ARG_SSERVER=0
  ARG_SSERVER_GROUP=""
  ARG_COMPACTOR=0
  ARG_COMPACTOR_GROUP=""

  # the correct version of getopt will always exit with exit code 4 when provided the `-T` flag
  getopt -T &>/dev/null
  if (($? != 4)); then
    echo "$(red ERROR): Missing $(yellow util-linux) (or equivalent) version of $(green getopt). Unable to continue."
    exit 2
  fi

  if ! PARSE_OUTPUT=$(getopt -o "" --long "dry-run,local,manager,gc,monitor,compaction-coordinator,no-tservers,tservers,sservers::,compactors::" -n 'accumulo-cluster' -- "$@"); then
    print_usage
    exit 1
  fi

  eval set -- "$PARSE_OUTPUT"

  while true; do
    case "$1" in
      --dry-run)
        DEBUG=1
        debug "original args: $(quote "${originalArgs[@]}")"
        debug "parsed args: $PARSE_OUTPUT"
        shift 1
        ;;
      --local)
        ARG_LOCAL=1
        shift 1
        ;;
      --manager)
        ARG_ALL=0
        ARG_MANAGER=1
        shift 1
        ;;
      --gc)
        ARG_ALL=0
        ARG_GC=1
        shift 1
        ;;
      --monitor)
        ARG_ALL=0
        ARG_MONITOR=1
        shift 1
        ;;
      --compaction-coordinator)
        ARG_ALL=0
        ARG_COORDINATOR=1
        shift 1
        ;;
      --tservers)
        ARG_ALL=0
        ARG_TSERVER=1
        shift 1
        ;;
      --sservers)
        ARG_ALL=0
        ARG_SSERVER=1
        ARG_SSERVER_GROUP=$2
        shift 2
        ;;
      --compactors)
        ARG_ALL=0
        ARG_COMPACTOR=1
        ARG_COMPACTOR_GROUP=$2
        shift 2
        ;;
      --no-tservers)
        echo "$(yellow WARN): The $(yellow "--no-tservers") option is deprecated. Please specify the servers you wish to manage instead"
        ARG_ALL=0
        ARG_MANAGER=1
        ARG_GC=1
        ARG_MONITOR=1
        ARG_COORDINATOR=1
        ARG_SSERVER=1
        ARG_COMPACTOR=1
        shift 1
        ;;
      --)
        shift
        break
        ;;
      *)
        echo "$(red ERROR): Unhandled option: $(yellow "$1")"
        print_usage
        exit 1
        ;;
    esac
  done

  if [[ $# -eq 0 ]]; then
    invalid_args "<command> cannot be empty"
  elif [[ $# -ne 1 ]]; then
    # use getopt to display the remaining non-option parameters quoted for readability
    local remaining
    remaining="$(getopt -o "" -- "$@")"
    invalid_args "Only one <$(yellow command)> is allowed, but found:$(yellow "${remaining#*--}")"
  fi
  ARG_CMD=$1
  debug "ARG_CMD=$ARG_CMD"
  debug "ARG_ALL=$ARG_ALL"
  debug "ARG_LOCAL=$ARG_LOCAL"
  debug "ARG_MANAGER=$ARG_MANAGER"
  debug "ARG_GC=$ARG_GC"
  debug "ARG_MONITOR=$ARG_MONITOR"
  debug "ARG_COORDINATOR=$ARG_COORDINATOR"
  debug "ARG_TSERVER=$ARG_TSERVER"
  debug "ARG_TSERVER_GROUP=$ARG_TSERVER_GROUP"
  debug "ARG_SSERVER=$ARG_SSERVER"
  debug "ARG_SSERVER_GROUP=$ARG_SSERVER_GROUP"
  debug "ARG_COMPACTOR=$ARG_COMPACTOR"
  debug "ARG_COMPACTOR_GROUP=$ARG_COMPACTOR_GROUP"

}

function invalid_args() {
  echo "$(red ERROR): $(yellow invalid arguments): $*"
  echo
  print_usage 1>&2
  exit 1
}

function parse_fail() {
  echo "$(red ERROR): Failed to parse $(yellow "$conf/cluster.yaml")"
  exit 1
}

function isDebug() {
  [[ $DEBUG == 1 ]]
}

# if debug is on, print and return true; otherwise, return false
function debug() {
  isDebug && echo "$(blue DEBUG): $(cyan "$*")"
}

function quote() {
  # use getopt to quote, because it uses quotes, rather than escaping spaces, and that's easier to
  # read; POSIXLY_CORRECT makes it ignore unrecognized options, so it still quotes things that start
  # with '-', but it only does this after the first non-option argument, so we provide an empty
  # string to trick it, and then we remove it in the regex that removes " -- '' " from the front of
  # the parsed output, which is now quoted
  local quoted
  quoted=$(POSIXLY_CORRECT=1 getopt -o "" -- '' "$@") &&
    [[ $quoted =~ ^[[:space:]]*--[[:space:]]*\'\'[[:space:]]*(.*)$ ]] && echo "${BASH_REMATCH[1]}" &&
    return
  echo "$(red ERROR): $(yellow internal script error): unable to quote: $(yellow "$*")"
  exit 1
}

# call debug to print the command only, or execute asynchronously if debug is off
function debugOrRunAsync() {
  debug "$(quote "$@")" || ("$@") &
}

function canRunOnHost() {

  # always true when non-local
  if [[ $ARG_LOCAL == 0 ]]; then
    return 0
  fi

  if [[ -z $1 ]]; then
    echo "$(red ERROR): Host argument expected but missing"
    exit 1
  fi

  local found=0
  local addr
  for addr in "${LOCAL_HOST_ADDRESSES[@]}"; do
    if [[ $1 == "$addr" ]]; then
      found=1
      break
    fi
  done
  [[ $found == 1 ]]
}

function parse_config() {

  for file in slaves tservers monitor tracers gc managers masters; do
    if [[ -f $conf/$file ]]; then
      echo "$(red ERROR): A $(yellow "$file") file was found in $(yellow "$conf/")"
      echo "$(red ERROR): Accumulo now uses cluster host configuration information from $(yellow cluster.yaml)"
      echo "$(red ERROR): and requires that the $(yellow "$file") file not be present to reduce confusion."
      exit 1
    fi
  done

  local manager1
  local tservers_found
  local group
  local G

  if [[ ! -f $conf/cluster.yaml ]]; then
    echo "$(red ERROR): File not found $(yellow "$conf/cluster.yaml")"
    echo "$(red ERROR): Please make sure it exists and is configured with the host information."
    echo "$(red ERROR): Run $(yellow accumulo-cluster create-config) to create an example configuration."
    exit 1
  fi

  AC_TMP_DIR=$(mktemp -t -d "accumulo-cluster-XXXXXXXX") || exit 1
  if isDebug; then
    echo "$(blue DEBUG): Temporary files for this run are in $AC_TMP_DIR"
  else
    trap 'rm -rf -- "$AC_TMP_DIR"' EXIT
  fi

  CONFIG_FILE="$AC_TMP_DIR/ClusterConfigParser.out"
  "$accumulo_cmd" org.apache.accumulo.core.conf.cluster.ClusterConfigParser "$conf/cluster.yaml" "$CONFIG_FILE" || parse_fail
  #shellcheck source=/dev/null
  . "$CONFIG_FILE"
  debug "Parsed config from $(white "$conf/cluster.yaml")"
  local line
  if isDebug; then
    while read -r line; do
      debug "$(white "$line")"
    done <"$CONFIG_FILE"
  fi
  rm -f "$CONFIG_FILE"

  # this might not be possible, since the ClusterConfigParser would probably fail instead
  if [[ -z $MANAGER_HOSTS ]]; then
    echo "$(red ERROR): $(yellow managers) not found in $(yellow "$conf/cluster.yaml")"
    exit 1
  fi

  # This version doesn't support configurable tserver groups, so
  # use a default group to allow code reuse with newer branches
  # Shellcheck thinks these aren't used, but they are referenced
  # indirectly by group name.
  TSERVER_GROUPS="default"
  #shellcheck disable=SC2034
  TSERVERS_PER_HOST_default=${NUM_TSERVERS:-1}
  #shellcheck disable=SC2034
  TSERVER_HOSTS_default=$TSERVER_HOSTS

  # Rename variables from this version's config parser to ones that
  # this script expects, to support code reuse with newer branches
  COMPACTOR_GROUPS=$COMPACTION_QUEUES
  local var_name
  for group in $SSERVER_GROUPS; do
    var_name="NUM_SSERVERS_$group"
    if [[ -n ${!var_name} ]]; then
      declare -g "SSERVERS_PER_HOST_$group"="${!var_name}"
    else
      declare -g "SSERVERS_PER_HOST_$group"="${NUM_SSERVERS:-1}"
    fi
  done
  for group in $COMPACTOR_GROUPS; do
    var_name="NUM_COMPACTORS_$group"
    if [[ -n ${!var_name} ]]; then
      declare -g "COMPACTORS_PER_HOST_$group"="${!var_name}"
    else
      declare -g "COMPACTORS_PER_HOST_$group"="${NUM_COMPACTORS:-1}"
    fi
  done

  # This version requires a compaction coordinator for compactors
  if [[ -n $COMPACTOR_GROUPS && -z $COORDINATOR_HOSTS ]]; then
    echo "$(yellow WARN): External compactor group(s) configured, but no coordinator configured"
  fi

  if [[ -z $COMPACTOR_GROUPS ]]; then
    # no compactor groups are required for this version
    debug "No compactor groups configured"
  else
    for group in $COMPACTOR_GROUPS; do
      G="COMPACTOR_HOSTS_$group"
      if [[ -z ${!G} ]]; then
        echo "$(yellow WARN): External compactor group $(yellow "$group") configured, but no compactors configured for it"
      fi
    done
  fi

  tservers_found="false"
  if [[ -z $TSERVER_GROUPS ]]; then
    echo "$(yellow WARN): No tablet server groups configured"
  else
    for group in $TSERVER_GROUPS; do
      G="TSERVER_HOSTS_$group"
      if [[ -z ${!G} ]]; then
        echo "$(yellow WARN): tablet server group $(yellow "$group") configured, but no hosts configured for it"
      else
        tservers_found="true"
      fi
    done
  fi

  if [[ $tservers_found != "true" ]]; then
    echo "$(red ERROR): There are no tablet servers configured, Accumulo requires at least $(yellow 1) tablets server to host system tables"
    exit 1
  fi

  if [[ -n $SSERVER_GROUPS ]]; then
    for group in $SSERVER_GROUPS; do
      G="SSERVER_HOSTS_$group"
      if [[ -z ${!G} ]]; then
        echo "$(yellow WARN): scan server group $(yellow "$group") configured, but no hosts configured for it"
      fi
    done
  fi

  manager1=$(echo "$MANAGER_HOSTS" | cut -d" " -f1)

  if [[ -z $MONITOR_HOSTS ]]; then
    echo "$(yellow WARN): monitors not found in $(yellow "$conf/cluster.yaml"), using first manager host $(green "$manager1")"
    MONITOR_HOSTS=$manager1
  fi

  if [[ -z $GC_HOSTS ]]; then
    echo "$(yellow WARN): gc not found in $(yellow "$conf/cluster.yaml"), using first manager host $(green "$manager1")"
    GC_HOSTS=$manager1
  fi

}

function execute_command() {
  control_cmd=$1
  host=$2
  service=$3
  group=$4
  shift 4

  local S
  local servers_per_host

  S="${service^^}S_PER_HOST_$group"
  S="${S//-/_}" # replace dashes in service/group name with underscores
  servers_per_host="${!S:-1}"

  if [[ $ARG_LOCAL == 1 ]]; then
    debugOrRunAsync bash -c "ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -a $host $*"
  else
    debugOrRunAsync "${SSH[@]}" "$host" "bash -c 'ACCUMULO_CLUSTER_ARG=$servers_per_host \"$bin/accumulo-service\" $service $control_cmd -a $host $*'"
  fi
}

function get_localhost_addresses() {
  local localaddresses
  local localinterfaces
  local x
  if [[ -n $ACCUMULO_LOCALHOST_ADDRESSES ]]; then
    read -r -a localaddresses <<<"$ACCUMULO_LOCALHOST_ADDRESSES"
  else
    read -r -a localinterfaces <<<"$(hostname -I)"
    read -r -a localaddresses <<<"$(getent hosts 127.0.0.1 ::1 "${localinterfaces[@]}" | paste -sd' ')"
  fi
  for x in "${localaddresses[@]}"; do echo "$x"; done | sort -u
}

function control_services() {
  unset DISPLAY
  local operation=$1

  if [[ $operation != "start" && $operation != "stop" && $operation != "kill" ]]; then
    echo "$(red ERROR): Invalid operation: $(yellow "$operation")"
    exit 1
  fi

  local tserver_groups
  local addr
  local group
  local tserver
  local G
  local sserver
  local gc
  if [[ $ARG_ALL == 1 && $ARG_LOCAL == 0 && $operation == "stop" ]]; then
    echo "Stopping Accumulo cluster..."
    if ! isDebug; then
      # Stop all of the the Scan Server processes
      for group in $SSERVER_GROUPS; do
        echo "Executing $(green "$ARG_CMD") on $(purple scan servers) for group $(yellow "$group")"
        hosts="SSERVER_HOSTS_$group"
        for sserver in ${!hosts}; do
          if canRunOnHost "$sserver"; then
            execute_command "$operation" "$sserver" sserver "$group" "-g" "$group"
          fi
        done
      done
      # Stop the GC processes, they scan and write to the metadata table
      for gc in $GC_HOSTS; do
        if canRunOnHost "$gc"; then
          echo "Executing $(green "$ARG_CMD") on $(purple garbage collectors)"
          execute_command "$operation" "$gc" gc "default"
        fi
      done
      # Try to cleanly stop the TabletServers and Manager
      if ! "$accumulo_cmd" admin stopAll; then
        echo "Invalid password or unable to connect to the manager"
        echo "Initiating forced shutdown in 15 seconds (Ctrl-C to abort)"
        sleep 10
        echo "Initiating forced shutdown in  5 seconds (Ctrl-C to abort)"
      else
        echo "Accumulo shut down cleanly"
        echo "Utilities and unresponsive servers will shut down in 5 seconds (Ctrl-C to abort)"
      fi
      sleep 5
    fi
  elif [[ $ARG_LOCAL == 1 && $ARG_TSERVER == 1 && $operation == "stop" ]]; then
    tserver_groups=$TSERVER_GROUPS
    if [[ -n $ARG_TSERVER_GROUP ]]; then
      tserver_groups=$ARG_TSERVER_GROUP
    fi
    for addr in "${LOCAL_HOST_ADDRESSES[@]}"; do
      for group in $tserver_groups; do
        G="TSERVER_HOSTS_$group"
        for tserver in ${!G}; do
          if echo "$tserver" | grep -q "$addr"; then
            if ! isDebug; then
              "$accumulo_cmd" admin stop "$addr"
            else
              debug "Stopping tservers on $addr via admin command"
            fi
          fi
        done
      done
    done
  elif [[ $ARG_ALL == 1 && $operation == "kill" ]]; then
    echo "Killing Accumulo cluster..."
  fi

  local count
  local hosts
  if [[ $ARG_ALL == 1 || $ARG_TSERVER == 1 ]]; then
    tserver_groups=$TSERVER_GROUPS
    if [[ -n $ARG_TSERVER_GROUP ]]; then
      tserver_groups=$ARG_TSERVER_GROUP
    fi
    for group in $tserver_groups; do
      local msg
      local msgNoColor
      msg="Executing $(green "$ARG_CMD") on $(purple tablet servers) for group $(yellow "$group") ..."
      msgNoColor=$(decolorize "$msg")
      count=${#msgNoColor}
      ((count > 71)) && count=69 # only print up to 3 more dots if the line is too long
      echo -n "$msg"
      hosts="TSERVER_HOSTS_$group"
      for tserver in ${!hosts}; do
        if canRunOnHost "$tserver"; then
          echo -n "."
          execute_command "$operation" "$tserver" tserver "$group"
          if ((++count % 72 == 0)); then
            echo
            wait
          fi
        fi
      done
      echo "done"
    done
  fi

  local manager
  if [[ $ARG_ALL == 1 || $ARG_MANAGER == 1 ]]; then
    for manager in $MANAGER_HOSTS; do
      if canRunOnHost "$manager"; then
        echo "Executing $(green "$ARG_CMD") on $(purple managers)"
        execute_command "$operation" "$manager" manager "default"
      fi
    done
  fi

  if [[ $ARG_ALL == 1 || $ARG_GC == 1 ]]; then
    for gc in $GC_HOSTS; do
      if canRunOnHost "$gc"; then
        echo "Executing $(green "$ARG_CMD") on $(purple garbage collectors)"
        execute_command "$operation" "$gc" gc "default"
      fi
    done
  fi

  local monitor
  if [[ $ARG_ALL == 1 || $ARG_MONITOR == 1 ]]; then
    for monitor in $MONITOR_HOSTS; do
      if canRunOnHost "$monitor"; then
        echo "Executing $(green "$ARG_CMD") on $(purple monitors)"
        execute_command "$operation" "$monitor" monitor "default"
      fi
    done
  fi

  local coordinator
  if [[ $ARG_ALL == 1 || $ARG_COORDINATOR == 1 ]]; then
    for coordinator in $COORDINATOR_HOSTS; do
      if canRunOnHost "$coordinator"; then
        echo "Executing $(green "$ARG_CMD") on $(purple compaction coordinators)"
        execute_command "$operation" "$coordinator" compaction-coordinator "default"
      fi
    done
  fi

  local sserver_groups
  if [[ $ARG_ALL == 1 || $ARG_SSERVER == 1 ]]; then
    sserver_groups=$SSERVER_GROUPS
    if [[ -n $ARG_SSERVER_GROUP ]]; then
      sserver_groups=$ARG_SSERVER_GROUP
    fi
    for group in $sserver_groups; do
      echo "Executing $(green "$ARG_CMD") on $(purple scan servers) for group $(yellow "$group")"
      hosts="SSERVER_HOSTS_$group"
      for sserver in ${!hosts}; do
        if canRunOnHost "$sserver"; then
          execute_command "$operation" "$sserver" sserver "$group" "-g" "$group"
        fi
      done
    done
  fi

  local compactor_groups
  local compactor
  if [[ $ARG_ALL == 1 || $ARG_COMPACTOR == 1 ]]; then
    compactor_groups=$COMPACTOR_GROUPS
    if [[ -n $ARG_COMPACTOR_GROUP ]]; then
      compactor_groups=$ARG_COMPACTOR_GROUP
    fi
    for group in $compactor_groups; do
      echo "Executing $(green "$ARG_CMD") on $(purple compactors) for group $(yellow "$group")"
      hosts="COMPACTOR_HOSTS_$group"
      for compactor in ${!hosts}; do
        if canRunOnHost "$compactor"; then
          execute_command "$operation" "$compactor" compactor "$group" "-q" "$group"
        fi
      done
    done
  fi

  if [[ $ARG_LOCAL == 0 && $ARG_ALL == 1 && ($operation == "stop" || $operation == "kill") ]]; then
    if ! isDebug; then
      echo "Cleaning all server entries in ZooKeeper"
      "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap -manager -tservers -compaction-coordinators -compactors -sservers --gc --monitor
    fi
  fi

  wait

}

function prune_group() {
  local service_type=$1
  local group=$2
  local expectedCount=$3
  declare -a hosts
  read -r -a hosts <<<"$4"

  if isDebug; then
    echo "$(blue DEBUG) starting prune for service:$service_type group:$group expected:$expectedCount"
  fi

  if [ -z ${AC_TMP_DIR+x} ]; then
    echo "$(red ERROR): AC_TMP_DIR is not set"
    exit 1
  fi
  local exclude_file="$AC_TMP_DIR/accumulo-zoozap-exclude-$service_type-$group.txt"
  touch "$exclude_file"

  # Determine the host:ports known by the accumulo cluster script, these should be kept
  for host in "${hosts[@]}"; do
    "${SSH[@]}" "$host" bash -c "'$bin/accumulo-service $service_type list'" | grep -E "^[a-zA-Z0-9]+_${group}_[0-9]+" | head -n "$expectedCount" | awk '{print $3}' | tr ',' '\n' | awk '{print "'"$host"':" $1}' >>"$exclude_file"
  done

  local lockTypeOpt
  case $service_type in
    manager)
      lockTypeOpt="-manager"
      ;;
    compaction-coordinator)
      lockTypeOpt="-compaction-coordinators"
      ;;
    compactor)
      lockTypeOpt="-compactors"
      ;;
    tserver)
      lockTypeOpt="-tservers"
      ;;
    sserver)
      lockTypeOpt="-sservers"
      ;;
    gc)
      lockTypeOpt="--gc"
      ;;
    monitor)
      lockTypeOpt="--monitor"
      ;;
    *)
      echo "Prune does not support $service_type"
      exit 1
      ;;
  esac

  if isDebug; then
    "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap "$lockTypeOpt" -verbose --include-groups "$group" --exclude-host-ports "$exclude_file" --dry-run
  else
    "$accumulo_cmd" org.apache.accumulo.server.util.ZooZap "$lockTypeOpt" -verbose --include-groups "$group" --exclude-host-ports "$exclude_file"
  fi
}

# Kills extra server processes that are not needed according to the
# cluster.yaml file.  Conceptually this code is trying to reconcile the
# following three sets of servers.
#
#  1. The notional goal set of servers specified by cluster.yaml
#  2. The set of servers processes seen in zookeeper
#  3. The set of server processes known to the accumulo-cluster script.  This
#     is derived from pid files on hosts in set 1.
#
# This function attempts to find extra servers in set 2 that are not specified
# by set 1.  When it does find extra servers it removes their zookeeper locks
# avoiding removing locks of servers in set 3. The following are different
# situations the code will see and handle.
#
#  * When a host is not cluster.yaml but has some processes listed in
#    zookeeper.  For this case all of the process with that host can be killed.
#  * When a resource group is not in cluster.yaml but has some processes listed
#    in zookeeper.  For this case all of the processes with that resource group
#    can be killed.
#  * When a host is in cluster.yaml with a target of 3 processes but has 6
#    processes listed in zookeeper.  For this case want to kill 3 processes that
#    do not have pid files on the host.
#
function prune() {
  if [[ $ARG_LOCAL == 1 ]]; then
    # Currently the code is structured to remove all extra servers in a single resource group.  Finer granularity is not supported.
    echo "$(red ERROR): Prune does not support running locally"
    exit 1
  fi

  if ! jq -h >&/dev/null; then
    echo "$(red ERROR:) Missing $(green jq). Unable to continue."
    exit 1
  fi

  if [[ -z ${AC_TMP_DIR+x} ]]; then
    echo "AC_TMP_DIR is not set"
    exit 1
  fi
  local service_json="$AC_TMP_DIR/accumulo-service.json"
  "$accumulo_cmd" admin serviceStatus --json >"$service_json" 2>/dev/null || exit 1

  local var_name
  local hosts
  declare -a groups

  local manager
  if [[ $ARG_ALL == 1 || $ARG_MANAGER == 1 ]]; then
    prune_group "manager" "default" "1" "$MANAGER_HOSTS"
  fi

  if [[ $ARG_ALL == 1 || $ARG_GC == 1 ]]; then
    prune_group "gc" "default" "1" "$GC_HOSTS"
  fi

  if [[ $ARG_ALL == 1 || $ARG_MONITOR == 1 ]]; then
    prune_group "monitor" "default" "1" "$MONITOR_HOSTS"
  fi

  if [[ $ARG_ALL == 1 || $ARG_COORDINATOR == 1 ]]; then
    prune_group "compaction-coordinator" "default" "1" "$COORDINATOR_HOSTS"
  fi

  if [[ $ARG_ALL == 1 || $ARG_TSERVER == 1 ]]; then
    #TODO in main need to adapt to having RGs for tservers
    prune_group "tserver" "default" "$TSERVERS_PER_HOST_default" "$TSERVER_HOSTS_default"
  fi

  if [[ $ARG_ALL == 1 || $ARG_SSERVER == 1 ]]; then
    groups=()
    if [[ -n $ARG_SSERVER_GROUP ]]; then
      read -r -a groups <<<"$ARG_SSERVER_GROUP"
    else
      # find all groups known in zookeeper, this will allow pruning entire groups that do not even exist in cluster.yaml
      readarray -t groups < <(jq -r ".summaries.S_SERVER.resourceGroups | keys | .[]?" "$service_json")
    fi

    for group in "${groups[@]}"; do
      var_name="SSERVERS_PER_HOST_$group"
      local expected=${!var_name:-0}

      hosts="SSERVER_HOSTS_$group"
      prune_group "sserver" "$group" "$expected" "${!hosts}"
    done

  fi

  if [[ $ARG_ALL == 1 || $ARG_COMPACTOR == 1 ]]; then
    groups=()
    if [[ -n $ARG_COMPACTOR_GROUP ]]; then
      read -r -a groups <<<"$ARG_COMPACTOR_GROUP"
    else
      # find all groups known in zookeeper, this will allow pruning entire groups that do not even exist in cluster.yaml
      readarray -t groups < <(jq -r ".summaries.COMPACTOR.resourceGroups | keys | .[]?" "$service_json")
    fi

    for group in "${groups[@]}"; do
      var_name="COMPACTORS_PER_HOST_$group"
      local expected=${!var_name:-0}

      hosts="COMPACTOR_HOSTS_$group"
      prune_group "compactor" "$group" "$expected" "${!hosts}"
    done
  fi
}

function main() {

  checkTerminalSupportsColor && COLOR_ENABLED=1 || COLOR_ENABLED=0
  parse_args "$@"

  # Resolve base directory
  local SOURCE
  SOURCE="${BASH_SOURCE[0]}"
  while [[ -L $SOURCE ]]; do
    bin="$(cd -P "$(dirname "$SOURCE")" && pwd)"
    SOURCE="$(readlink "$SOURCE")"
    [[ $SOURCE != /* ]] && SOURCE="$bin/$SOURCE"
  done
  bin="$(cd -P "$(dirname "$SOURCE")" && pwd)"
  basedir=$(cd -P "$bin/.." && pwd)
  conf="${ACCUMULO_CONF_DIR:-$basedir/conf}"

  accumulo_cmd="$bin/accumulo"
  SSH=('ssh' '-qn' '-o' 'ConnectTimeout=2' '-o' 'BatchMode=yes')

  mapfile -t LOCAL_HOST_ADDRESSES < <(get_localhost_addresses)
  debug "LOCAL_HOST_ADDRESSES=${LOCAL_HOST_ADDRESSES[*]}"

  case "$ARG_CMD" in
    create-config)
      if [[ -f "$conf"/cluster.yaml ]]; then
        echo "ERROR : $conf/cluster.yaml already exists, not overwriting"
        exit 1
      fi
      cat <<EOF >"$conf"/cluster.yaml
manager:
  - localhost

monitor:
  - localhost

gc:
  - localhost

tserver:
  - localhost

#sserver:
#  - default:
#    - localhost
#
#compaction:
#  coordinator:
#    - localhost
#  compactor:
#    - q1:
#        - localhost
#    - q2:
#        - localhost
#

#
# The following are used by the accumulo-cluster script to determine how many servers
# to start on each host. If the following variables are not set, then they default to 1.
# If the environment variable NUM_TSERVERS is set when running accumulo_cluster
# then its value will override what is set in this file for tservers_per_host. Likewise if
# NUM_SSERVERS or NUM_COMPACTORS are set then it will override sservers_per_host and
# compactors_per_host.
#
tservers_per_host: 1
#sservers_per_host:
# - default: 1
#compactors_per_host:
# - q1: 1
# - q2: 1

EOF
      ;;
    restart)
      parse_config
      control_services stop
      control_services kill
      # Make sure the JVM has a chance to fully exit
      sleep 1
      control_services start
      ;;
    start)
      parse_config
      control_services start
      ;;
    stop)
      parse_config
      control_services stop
      ;;
    kill)
      parse_config
      control_services kill
      ;;
    start-here)
      parse_config
      ARG_ALL=1
      ARG_LOCAL=1
      control_services start
      ;;
    stop-here)
      parse_config
      ARG_ALL=1
      ARG_LOCAL=1
      control_services stop
      control_services kill
      ;;
    prune)
      parse_config
      prune
      ;;
    start-non-tservers)
      echo "'$ARG_CMD' is deprecated. Please specify the servers you wish to start instead"
      parse_config
      ARG_MANAGER=1
      ARG_GC=1
      ARG_MONITOR=1
      ARG_COORDINATOR=1
      ARG_SSERVER=1
      ARG_COMPACTOR=1
      control_services start
      ;;
    start-servers)
      echo "'$ARG_CMD' is deprecated. Please use 'start' instead"
      parse_config
      control_services start
      ;;
    stop-servers)
      echo "'$ARG_CMD' is deprecated. Please use 'stop' instead"
      parse_config
      control_services stop
      ;;
    start-tservers)
      echo "'$ARG_CMD' is deprecated. Please use 'start --tservers' instead"
      ARG_TSERVER=1
      control_services start
      ;;
    stop-tservers)
      echo "'$ARG_CMD' is deprecated. Please use 'stop --tservers' instead"
      ARG_TSERVER=1
      control_services stop
      ;;
    *)
      invalid_args "'$ARG_CMD' is an invalid <command>"
      ;;
  esac
}

main "$@"
