#' Get all Wikidata Q identifiers of all Wikipedia pages (or files, or
#' subcategories) that are members of the given category,
#'
#' @param url Full URL to a Wikipedia category page. If given, title and
#'   language can be left empty.
#' @param category Title of a Wikipedia category page or final parts of its url.
#'   Must include "Category:", or equivalent in other languages. If given, url
#'   can be left empty, but language must be provided.
#' @param type Defaults to "page", defines which kind of members of a category
#'   are returned. Valid values include "page", "file", and "subcat" (for
#'   sub-category). Corresponds to `cmtype`. For details, see
#'   \href{https://www.mediawiki.org/wiki/API:Categorymembers}{the relevant page
#'   of the official documentation}.
#' @param attempts Defaults to 10. Number of times it re-attempts to reach the
#'   API before failing.
#' @inheritParams tw_get
#'
#' @return A data frame (a tibble) with eight columns: `source_title_url`,
#'   `source_wikipedia_title`, `source_qid`, `wikipedia_title`, `wikipedia_id`,
#'   `qid`, `description`, and `language`.
#' @export
#'
#' @examples
#' if (interactive()) {
#'   sub_categories <- tw_get_wikipedia_category_members(
#'     category = "Category:American women anthropologists",
#'     type = "subcat"
#'   )
#'
#'   sub_categories
#'
#'   tw_get_wikipedia_category_members(
#'     category = sub_categories$wikipedia_title,
#'     type = "page"
#'   )
#' }
tw_get_wikipedia_category_members <- function(
  url = NULL,
  category = NULL,
  type = "page",
  language = tidywikidatar::tw_get_language(),
  cache = NULL,
  overwrite_cache = FALSE,
  cache_connection = NULL,
  disconnect_db = TRUE,
  wait = 1,
  attempts = 10
) {
  if (is.null(category) & !is.null(url)) {
    language <- stringr::str_extract(
      string = url,
      pattern = "(?<=https://)[[a-z]][[a-z]](?=.wikipedia.org/)"
    )
  }

  db <- tw_connect_to_cache(
    connection = cache_connection,
    language = language,
    cache = cache
  )

  wikipedia_page_qid_df <- tw_get_wikipedia_page_qid(
    title = category,
    language = language,
    url = url,
    cache = cache,
    overwrite_cache = overwrite_cache,
    cache_connection = db,
    disconnect_db = FALSE,
    wait = wait,
    attempts = attempts
  )

  source_df <- wikipedia_page_qid_df %>%
    dplyr::transmute(
      source_title_url = .data$title_url,
      source_wikipedia_category = .data$wikipedia_title,
      source_qid = .data$qid,
      language = .data$language
    ) %>%
    dplyr::distinct(.data$source_title_url, .keep_all = TRUE)

  wikipedia_category_members_df <- purrr::map_dfr(
    .x = seq_along(source_df$source_wikipedia_category),
    .f = function(i) {
      current_slice_df <- source_df %>%
        dplyr::slice(i)

      linked_df <- tw_get_wikipedia_category_members_single(
        category = current_slice_df$source_title_url,
        language = current_slice_df$language,
        url = NULL,
        type = type,
        cache = cache,
        overwrite_cache = overwrite_cache,
        cache_connection = db,
        disconnect_db = FALSE,
        wait = wait,
        attempts = attempts
      )

      linked_df
    }
  )

  tw_disconnect_from_cache(
    cache = cache,
    cache_connection = db,
    disconnect_db = disconnect_db,
    language = language
  )

  wikipedia_category_members_df
}


#' Get all Wikidata Q identifiers of all Wikipedia pages that appear in a given page
#'
#' @inheritParams tw_get_wikipedia_category_members
#'
#' @return A data frame (a tibble) with four columns: `wikipedia_title`, `wikipedia_id`, `wikidata_id`, `wikidata_description`.
#'
#' @examples
#' if (interactive()) {
#'   tidywikidatar:::tw_get_wikipedia_category_members_single(
#'     category = "Category:American women anthropologists",
#'     type = "subcat"
#'   )
#'
#'   tidywikidatar:::tw_get_wikipedia_category_members_single(
#'     category = "Category:Puerto Rican women anthropologists",
#'     type = "page"
#'   )
#' }
tw_get_wikipedia_category_members_single <- function(
  url = NULL,
  category = NULL,
  type = "page",
  language = tidywikidatar::tw_get_language(),
  cache = NULL,
  overwrite_cache = FALSE,
  cache_connection = NULL,
  disconnect_db = TRUE,
  wait = 1,
  attempts = 10
) {
  db <- tw_connect_to_cache(
    connection = cache_connection,
    language = language,
    cache = cache
  )

  if (is.null(category)) {
    category <- stringr::str_extract(
      string = url,
      pattern = "wiki/.*$"
    ) %>%
      stringr::str_remove(pattern = stringr::fixed("wiki/"))
  }

  if (tw_check_cache(cache) & !overwrite_cache) {
    db_result <- tw_get_cached_wikipedia_category_members(
      category = category,
      type = type,
      language = language,
      cache = cache,
      cache_connection = db,
      disconnect_db = FALSE
    )
    if (is.data.frame(db_result) & nrow(db_result) > 0) {
      category_df <- db_result %>%
        dplyr::collect() %>%
        dplyr::distinct()
    } else {
      category_df <- tidywikidatar::tw_empty_wikipedia_category_members
    }
  } else {
    category_df <- tidywikidatar::tw_empty_wikipedia_category_members
  }

  if (nrow(category_df) == 0) {
    json_url <- stringr::str_c(
      tw_get_wikipedia_base_api_url(
        url = url,
        title = category,
        language = language,
        type = "category"
      ),
      "&cmlimit=max&cmtype=",
      type
    )

    api_result <- FALSE

    attempt_n <- 1

    while (isFALSE(api_result) & attempt_n <= attempts) {
      attempt_n <- sum(attempt_n, 1)
      api_result <- tryCatch(
        jsonlite::read_json(path = json_url),
        error = function(e) {
          logical(1L)
        }
      )
      Sys.sleep(time = wait)
    }

    if (isFALSE(api_result)) {
      cli::cli_abort(c(
        "Could not reach the API with {attempts} attempts.",
        i = "Consider increasing the waiting time between calls with the {.arg wait} parameter or check your internet connection."
      ))
    } else if (length(api_result) == 1) {
      cli::cli_abort(
        "Page not found. Make sure that language parameter is consistent with the language of the input title or url."
      )
    } else {
      base_json <- api_result
    }

    continue_check <- base_json %>%
      purrr::pluck(
        "continue",
        "cmcontinue"
      )

    all_jsons <- list()

    page_number <- 1

    all_jsons[[page_number]] <- base_json

    while (is.null(continue_check) == FALSE & page_number < 1000) {
      page_number <- page_number + 1

      json_url <- stringr::str_c(
        json_url,
        "&cmcontinue=",
        continue_check
      )

      api_result <- FALSE

      attempt_n <- 1
      while (isFALSE(api_result) & attempt_n <= attempts) {
        attempt_n <- sum(attempt_n, 1)
        api_result <- tryCatch(
          jsonlite::read_json(path = json_url),
          error = function(e) {
            logical(1L)
          }
        )
        Sys.sleep(time = wait)
      }

      if (isFALSE(api_result)) {
        cli::cli_abort(c(
          "Could not reach the API with {attempts} attempts.",
          i = "Consider increasing the waiting time between calls with the {.arg wait} parameter or check your internet connection."
        ))
      } else {
        base_json <- api_result
      }

      all_jsons[[page_number]] <- base_json

      continue_check <- base_json %>%
        purrr::pluck(
          "continue",
          "cmcontinue"
        )
    }

    all_pages <- purrr::map(
      .x = all_jsons,
      .f = purrr::pluck,
      "query",
      "categorymembers"
    ) %>%
      purrr::flatten()

    page_title_df <- purrr::map_dfr(
      .x = all_pages,
      .f = function(current_page) {
        tibble::tibble(
          wikipedia_title = current_page %>%
            purrr::pluck(
              "title"
            ),
          wikipedia_id = current_page %>%
            purrr::pluck(
              "pageid"
            )
        )
      }
    )

    category_df <- page_title_df %>%
      dplyr::transmute(
        category = category,
        .data$wikipedia_title,
        wikipedia_id = as.numeric(.data$wikipedia_id)
      )

    if (tw_check_cache(cache)) {
      tw_write_wikipedia_category_members_to_cache(
        df = category_df,
        type = type,
        cache_connection = db,
        language = language,
        overwrite_cache = overwrite_cache,
        disconnect_db = disconnect_db
      )
    }
  }

  wikipedia_page_qid_df <- tw_get_wikipedia_page_qid(
    title = category_df$wikipedia_title,
    language = language,
    url = NULL,
    cache = cache,
    overwrite_cache = overwrite_cache,
    cache_connection = cache_connection,
    disconnect_db = FALSE,
    wait = wait,
    attempts = attempts
  )

  tw_disconnect_from_cache(
    cache = cache,
    cache_connection = db,
    disconnect_db = disconnect_db,
    language = language
  )

  wikipedia_page_qid_df
}


#' Gets members of Wikipedia categories from local cache
#'
#' Mostly used internally.
#'
#' @inheritParams tw_get_wikipedia_category_members
#'
#' @return If data present in cache, returns a data frame with cached data.
#' @export
#'
#' @examples
#' if (interactive()) {
#'   tw_set_cache_folder(path = tempdir())
#'   tw_enable_cache()
#'   tw_create_cache_folder(ask = FALSE)
#'
#'   df_from_api <- tw_get_wikipedia_page_qid(category = "Margaret Mead", language = "en")
#'
#'   df_from_cache <- tw_get_cached_wikipedia_category_members(
#'     category = "Margaret Mead",
#'     language = "en"
#'   )
#'
#'   df_from_cache
#' }
tw_get_cached_wikipedia_category_members <- function(
  category,
  type = "page",
  language = tidywikidatar::tw_get_language(),
  cache = NULL,
  cache_connection = NULL,
  disconnect_db = TRUE
) {
  if (isFALSE(tw_check_cache(cache = cache))) {
    return(invisible(NULL))
  }

  db <- tw_connect_to_cache(
    connection = cache_connection,
    language = language,
    cache = cache
  )

  table_name <- tw_get_cache_table_name(
    type = stringr::str_c("wikipedia_category_members_", type),
    language = language
  )

  if (pool::dbExistsTable(conn = db, name = table_name) == FALSE) {
    if (disconnect_db) {
      tw_disconnect_from_cache(
        cache = cache,
        cache_connection = db,
        disconnect_db = disconnect_db,
        language = language
      )
    }
    return(tidywikidatar::tw_empty_wikipedia_category_members)
  }

  db_result <- tryCatch(
    dplyr::tbl(src = db, table_name) %>%
      dplyr::filter(
        .data$category %in% !!stringr::str_c(category)
      ),
    error = function(e) {
      logical(1L)
    }
  )

  if (isFALSE(db_result)) {
    if (disconnect_db) {
      tw_disconnect_from_cache(
        cache = cache,
        cache_connection = db,
        disconnect_db = disconnect_db,
        language = language
      )
    }
    return(tidywikidatar::tw_empty_wikipedia_category_members)
  }

  cached_df <- db_result %>%
    dplyr::collect()

  tw_disconnect_from_cache(
    cache = cache,
    cache_connection = db,
    disconnect_db = disconnect_db,
    language = language
  )

  cached_df
}


#' Write Wikipedia page links to cache
#'
#' Mostly used internally by `tidywikidatar`, use with caution to keep caching consistent.
#'
#' @param df A data frame typically generated with `tw_get_wikipedia_category_members()`.
#' @param language Defaults to language set with `tw_set_language()`; if not set, "en". Use "all_available" to keep all languages. For available language values, see https://www.wikidata.org/wiki/Help:Wikimedia_language_codes/lists/all
#' @param type Defaults to "page", defines which kind of members of a category are returned. Valid values include "page", "file", and "subcat" (for sub-category). Corresponds to `cmtype`. For details, see \url{https://www.mediawiki.org/wiki/API:Categorymembers}
#' @param overwrite_cache Logical, defaults to FALSE. If TRUE, it overwrites the table in the local sqlite database. Useful if the original Wikidata object has been updated.
#' @param cache Defaults to NULL. If given, it should be given either TRUE or FALSE. Typically set with `tw_enable_cache()` or `tw_disable_cache()`.
#' @param cache_connection Defaults to NULL. If NULL, and caching is enabled, `tidywikidatar` will use a local sqlite database. A custom connection to other databases can be given (see vignette `caching` for details).
#' @param disconnect_db Defaults to TRUE. If FALSE, leaves the connection to cache open.
#'
#' @return Silently returns the same data frame provided as input. Mostly used internally for its side effects.
#'
#' @export
#'
#' @examples
#' if (interactive()) {
#'   df <- tw_get_wikipedia_category_members(
#'     category = "American women anthropologists",
#'     language = "en",
#'     cache = FALSE
#'   )
#'
#'   tw_write_wikipedia_category_members_to_cache(
#'     df = df,
#'     language = "en"
#'   )
#' }
tw_write_wikipedia_category_members_to_cache <- function(
  df,
  language = tidywikidatar::tw_get_language(),
  type = "page",
  cache = NULL,
  overwrite_cache = FALSE,
  cache_connection = NULL,
  disconnect_db = TRUE
) {
  if (isFALSE(tw_check_cache(cache = cache))) {
    return(invisible(NULL))
  }

  db <- tw_connect_to_cache(
    connection = cache_connection,
    language = language,
    cache = cache
  )

  table_name <- tw_get_cache_table_name(
    type = stringr::str_c("wikipedia_category_members_", type),
    language = language
  )

  if (pool::dbExistsTable(conn = db, name = table_name) == FALSE) {
    # do nothing: if table does not exist, previous data cannot be there
  } else {
    if (overwrite_cache) {
      statement <- glue::glue_sql(
        "DELETE FROM {`table_name`} WHERE category = {category*}",
        source_title_url = unique(df$source_title_url),
        table_name = table_name,
        .con = db
      )
      result <- pool::dbExecute(
        conn = db,
        statement = statement
      )
    }
  }

  pool::dbWriteTable(db, name = table_name, value = df, append = TRUE)

  tw_disconnect_from_cache(
    cache = cache,
    cache_connection = db,
    disconnect_db = disconnect_db,
    language = language
  )

  invisible(df)
}

#' Reset Wikipedia category members cache
#'
#' Removes from cache the table where data typically gathered with
#' [tw_get_wikipedia_category_members()] are stored.
#'
#' @inheritParams tw_reset_wikipedia_page_links_cache
#' @inheritParams tw_get_wikipedia_category_members
#' @inheritParams tw_reset_item_cache
#'
#' @return Nothing, used for its side effects.
#' @export
#'
#' @examples
#' if (interactive()) {
#'   tw_reset_wikipedia_category_members_cache()
#' }
tw_reset_wikipedia_category_members_cache <- function(
  language = tidywikidatar::tw_get_language(),
  type = "page",
  cache = NULL,
  cache_connection = NULL,
  disconnect_db = TRUE,
  ask = TRUE
) {
  db <- tw_connect_to_cache(
    connection = cache_connection,
    language = language,
    cache = cache
  )

  table_name <- tw_get_cache_table_name(
    type = stringr::str_c("wikipedia_category_members_", type),
    language = language
  )

  if (pool::dbExistsTable(conn = db, name = table_name) == FALSE) {
    # do nothing: if table does not exist, nothing to delete
  } else if (isFALSE(ask)) {
    pool::dbRemoveTable(conn = db, name = table_name)
    cli::cli_inform(
      "Wikipedia category members cache reset for language {.val {language}} completed."
    )
  } else if (
    utils::menu(
      c("Yes", "No"),
      title = paste0(
        "Are you sure you want to remove from cache the Wikipedia category members cache for language: ",
        sQuote(language),
        "?"
      )
    ) ==
      1
  ) {
    pool::dbRemoveTable(conn = db, name = table_name)
    cli::cli_inform(
      "Wikipedia category members cache reset for language {.val {language}} completed."
    )
  }

  tw_disconnect_from_cache(
    cache = cache,
    cache_connection = db,
    disconnect_db = disconnect_db,
    language = language
  )
}
