tokenize <- function(x){

  part <- split_sumerian(x)

  if(str_detect(x,"<")){
    df <- data.frame(
      start = which(str_ends(  part$separators, "<")),
      end   = which(str_starts(part$separators, ">"))-1,
      stringsAsFactors = FALSE)

    tok_sign   <- part$signs
    tok_length <- rep(1, length(tok_sign))
    tok_sep    <- part$separators

    for(i in nrow(df):1){
      N <- length(tok_sign)
      start <- df$start[i]
      end   <- df$end[i]
      v <- tok_sign[start:end]
      w <- c(tok_sep[(start+1):end], "")
      this_token <- paste0(v, w, collapse="")

      first_ones <- integer(0)
      last_ones  <- integer(0)

      if(start>1)first_ones <- 1:(start-1)
      if(end<N) last_ones <- (end+1):N

      tok_sign   <- c(tok_sign[first_ones],           this_token,  tok_sign[last_ones])
      tok_sep    <- c(tok_sep[c(first_ones, start)],               tok_sep[c(last_ones, N+1)])
      tok_length <- c(tok_length[first_ones],         end-start+1, tok_length[last_ones])
    }
  }else{
    tok_sign     <- part$signs
    tok_length <- rep(1,length(part$signs))
    tok_sep      <- part$separators
  }

  token <- list(
    sign   = tok_sign,
    length = tok_length,
    sep    = tok_sep
  )
  return(token)
}



mark_skeleton_entries <- function(x){
  token <- tokenize(x)
  
  token$sep <- str_replace_all(token$sep, fixed("<"), "(")
  token$sep <- str_replace_all(token$sep, fixed(">"), ")")

  token$sep <- str_replace_all(token$sep, fixed("("), "{(")
  token$sep <- str_replace_all(token$sep, fixed(")"), ")}")

  N <- length(token$sign)
  for(i in 1:N){
    has_brackets <- str_ends(token$sep[i],fixed("(")) && str_starts(token$sep[i+1],fixed(")"))
    if(!has_brackets){
      token$sep[i] <- paste0(token$sep[i], "(")
      token$sep[i+1] <- paste0(")", token$sep[i+1])
    }
  }
  token$sep <- str_replace_all(token$sep, "  ", " ")

  result <- paste0(c("", token$sign), token$sep, collapse = "")

  return(result)
}





mark_skeleton_entries_alt <- function(x){
  token <- tokenize(x)
  token$sep <- str_replace_all(token$sep, "[<>]", " ")

  N <- length(token$sign)
  for(i in 1:N){
    has_brackets <- str_ends(token$sep[i],fixed("(")) && str_starts(token$sep[i+1],fixed(")"))
    if(!has_brackets){
      token$sep[i] <- paste0(token$sep[i], "(")
      token$sep[i+1] <- paste0(")", token$sep[i+1])
    }
  }
  token$sep <- str_replace_all(token$sep, "  ", " ")

  result <- paste0(c("", token$sign), token$sep, collapse = "")

  return(result)
}
