#' @title Negative log likelihood of distances
#' 
#' @description Return the negative log likelihood of 
#' observed detection distances given a likelihood
#'   and the estimated parameters.
#'
#' @param a A vector of likelihood parameter values. Length and 
#' meaning depend on \code{ml$series} and \code{ml$expansions}. If no expansion 
#' terms were called for (i.e., \code{ml$expansions = 0}), the distance 
#' likelihood contain one or two canonical parameters (see Details). 
#' If one or more expansions are called for, coefficients for the 
#' expansion terms follow coefficients for the canonical parameters.  
#' i.e., length of this vector is 
#'   \code{(num Covars incl. intercept) + expansions + 1*(like \%in\% c("hazrate"))}.
#'   
#' @inheritParams startLimits
#' 
#' @param verbosity The level of output produced during estimation. 
#' \code{verbosity} = 0 produces no output. Increasing values 
#' of \code{verbosity} >= 1 produce increasing levels of intermediate details.
#' This is mostly used for de-bugging and checking convergence.
#'  
#' @details
#' \bold{Expansion Terms}: If \code{ml$expansions} = k (k > 0), 
#' the expansion function specified by \code{ml$series} is 
#' called (see for example \code{\link{cosine.expansion}}). 
#' Assuming \eqn{h_{ij}(x)}{h_ij(x)} is the 
#' \eqn{j^{th}}{j-th} expansion term for the 
#' \eqn{i^{th}}{i-th} distance and that 
#' \eqn{c_1, c_2, \dots, c_k}{c(1), c(2), ..., c(k)} 
#' are (estimated) coefficients for the expansion terms, 
#' the likelihood contribution for the \eqn{i^{th}}{i-th} 
#' distance is, \deqn{f(x|a,b,c_1,c_2,\dots,c_k) = 
#' f(x|a,b)(1 + \sum_{j=1}^{k} c_j h_{ij}(x)).}{f(x|a,b,c_1,c_2,...,c_k) 
#' = f(x|a,b)(1 + c(1) h_i1(x) + c(2) h_i2(x) + ... + c(k) h_ik(x)). }
#'   
#' @return A scalar, the negative of the log likelihood evaluated at 
#' parameters \code{a}.
#' 
#' @seealso See \code{\link{halfnorm.like}} and links there; 
#'  \code{\link{dfuncEstim}}
#' 
#' @examples
#' 
#' # A halfnorm distance function 
#' fit <- dfuncEstim(sparrowDf, dist~1, likelihood = "halfnorm")
#' nLL(fit$par, fit, 3)
#' fit$loglik
#' ESW(fit)[1]
#' 
#' # Another way, b/c we have pnorm()
#' d <- distances(fit)
#' ones <- matrix(1, nrow = length(d), ncol = 1)
#' l <- halfnorm.like(fit$par, d, ones)
#' esw <-(pnorm(units::drop_units(fit$w.hi)
#'   , units::drop_units(fit$w.lo)
#'   , sd = exp(l$params)) - 0.5) * sqrt(2*pi) * exp(l$params)
#' -sum(log(l$L.unscaled/esw))
#' 
#' # A third way, b/c we have pnorm() and dnorm(). 
#' l2 <- dnorm(units::drop_units(d), mean = 0, sd = exp(fit$par))
#' scaler <- pnorm(units::drop_units(fit$w.hi), mean = 0, sd = exp(fit$par)) - 0.5 
#' -sum(log(l2/scaler))
#'  
#' @export

nLL <- function(a
                , ml
                , verbosity = 0
                ){
  
  
  # Pull data from input list ----
  # rule is: parameters in 'a' never have units,  
  # even though they could (e.g., sigma of halfnorm)

  # Because of na.pass when building model frame, and 
  # other code in parseModel(), there could be missing distances 
  # ONLY because they are inherently missing (crew did not get a distance
  # on an observation). Observations outside the strip are not present 
  # in the model frame (ml$mf). 

  # Evaluate the "key" function ----
  # I call Buckland's "key" function just "likelihood".
  # Returns one value per observation. Expansions, if any, 
  # are applied below, after this.
  fuzz <- getOption("Rdistance_fuzz")
  f.like <- utils::getFromNamespace(paste0( ml$likelihood, ".like"), "Rdistance")    
  d <- distances(ml) - ml$w.lo # observed distances, n of them
  X <- stats::model.matrix(ml)
  L <- f.like( a = a
             , dist = d
             , covars = X
             , w.hi = ml$w.hi - ml$w.lo # needed for some but not all Likes
             )
  key <- L$L.unscaled  # (n X k matrix)
  parms <- L$params

  if( verbosity >= 3 ){
    likExpan <- paste0(ml$likelihood, "_", ml$expansions, "_", transectType(ml))
    cat(colorize(paste(paste(rep("-", 40), collapse = ""))))
    cat(paste(" "
              , colorize(likExpan)
              , "\n"))
  }
  
  
  if( all(is.na(key)) ){
    # Parameters are out of bounds.  Happens when using optim.
    # Save a bunch of computation by doing this...
    key <- matrix(fuzz, nrow = nrow(key), ncol = ncol(key))
    if( verbosity >= 4 ){
      # for later
      exp.terms <- rep(NA, 5)
      W <- NA
    }
    
  } else {
    # Evaluate and apply the expansions ----
    if( ml$expansions > 0 ){
      W <- expandW(ml = ml
                 , params = parms
                 , k = ncol(key)  # I think this is always 1
                 )
      # The following call to expansionTerms returns matrix size (n X k); here,
      #  either nx1 or nxn.
      exp.terms <- Rdistance::expansionTerms(a = a
                                             , d = d
                                             , series = ml$series
                                             , nexp = ml$expansions
                                             , w = W)
      
      if( ncol(exp.terms) > 1 ){
        # W is len k; so exp.terms is kxk
        exp.terms <- diag(exp.terms)
      }
      
      key <- key * exp.terms
  
      # b/c no monotonicity restraints yet, function can go negative, 
      key[ !is.na(key) & (key <= 0) ] <- getOption("Rdistance_zero")
    }
  
    # Apply d if point survey ----
    if(is.points(ml)){
      key <- d * key  # element-wise
    }

    
    # Integrate under the distance function so can scale to density ----
    # This yields area under g(x), which is ESW, and is used to get f(x) = g(x)/ESW
    parms[,1] <- exp(parms[,1]) # variable param
    if( ml$expansions > 0 ){
      # append expansion coefs to parms; needed during integration
      coefLocs <- length(a) - (ml$expansions:1) + 1 
      parms <- cbind(parms
                      , matrix( a[coefLocs]
                               , nrow = nrow(parms)
                               , ncol = ml$expansions
                               , byrow = TRUE
                      ))  # n X ([#canonical] + nexp)
    }
    
    outArea <- integrateDfuncs( parms, ml )
    
    if( verbosity >= 3 ){
      cat(paste(" "
                , "integral ="
                , colorize(unique(outArea))
                , "\n"))
    }
    
    # Scale g(x) into f(x) ----
    key <- key / outArea

    # Note: Key or d*Key should integrate to 1.0 every iteration
    # Note 2: if POINTS, key has units, remove them b/c nlminb can't handle em.
    key <- dropUnits(key)
  
    # Check for really bad values ----
    #   RULE: Any impossible observations (i.e., key exactly 0) get 
    #   replaced by fuzz so that log(L) does not return -Inf. "exactly 0" is 
    #   less than fuzz. Worst fitting objects now contribute -log(fuzz) ~ 31.5 to LL
    
  
    ind <- is.na(key) | (key < fuzz)
    if( any(ind) ){
      key[ ind ] <- fuzz # replace them
    }

  } # endif - if(all(is.na(key)))  
  
  # Compute neg log likelihood ----
  nLL <- -sum(log(key), na.rm=TRUE)  # Note that distances > w in L are set to NA

  # Rules: 
  #   RULE 1 FOR LIKELIHOODS: No matter how bad the guess at a, you cannot return Inf, -Inf, NA, or NaN
  #   This means f.like can return NA, but not NaN (div by 0), Inf or -Inf for any row of data
  #   Must program the likelihoods to trap these values and return the appropriate .Machine constants
  #
  #   RULE 2 FOR NLL: It is possible for all rows of returned L to be non-Inf and non-NaN, 
  #   but there be enough of them that the sum overflows to Inf. This is unlikely because 
  #   we replace any LL < fuzz with fuzz (above) and log(fuzz) ~ -32. But, nonetheless
  #   trap infinite likelihoods here.
  #
  if( is.infinite(nLL) ){
    nLL <- getOption("Rdistance_posInf") # positive b/c already flipped over by -1: Should not happen
  }
  
  # Diagnostics ----
  if( verbosity >= 1 ){
    cat(paste0("  Coeffs: ", colorize(paste(a, collapse = ", "))))
    cat(paste(" -log(likelihood) =", colorize(nLL), "\n"))
  }
  if( length(unique(outArea)) <= 1 ){
    # check of integral only works w/o covars
    if( verbosity >= 3 ){
      integrateKey(ml, key, f0 = 1 / unique(outArea), plot=TRUE )
    } else if( verbosity >= 2) {
      integrateKey(ml, key, f0 = 1 / unique(outArea), plot=FALSE )
    }
  }
  if( verbosity >= 4 ){
    cat(crayon::bgRed( paste("data[1:5]      =", paste(d[1:5], collapse = ", "), "\n")))
    cat(crayon::bgRed( paste("key[1:5]       =", paste(key[1:5], collapse = ", "), "\n")))
    cat(crayon::bgRed( paste("exp.terms[1:5] =", paste(exp.terms[1:5], collapse = ", "), "\n")))
    cat(crayon::bgRed( paste("W              =", paste(W, collapse = ", "), "\n")))
  }
  
  
  if( verbosity >= 5 ){
    yn <- readline("Abort [y/N]?")
    if( toupper(yn) == "Y" ){
      stop("Aborted by user.")
    }
  }

  
  nLL
}
