nc-campaign-finance/process/R/prepare_candidates.R

prepare_candidates_for_matching <- function(
  path_data_prep_officers,
  path_out_report_list,
  path_addresses = "data-out/addresses"
) {
  lg_info_target(lg_get_logger())

  report_list <- out_open_dataset_db(path_out_report_list)
  addresses <- out_open_dataset_db(path_addresses)

  candidates <-
    out_open_dataset_db(path_data_prep_officers) |>
    semi_join(report_list, by = "report_id") |>
    filter(type == "Candidate") |>
    mutate(
      name_clean = toupper(name),
      name_clean = REGEXP_REPLACE(name_clean, " FOR .+$", "", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "\\s+", " ", "g")
    ) |>
    distinct(sboe_id, name_clean, address) |>
    rename(address_raw = address) |>
    left_join(
      addresses |> select(1:2),
      by = join_by(address_raw == address_lookup)
    ) |>
    rename(address = address_resolved)


  # Add new rows with aliases for people like `"ROLLANDE \"ROLIE\" SAMPSON"`
  candidates_aliases <-
    candidates |>
    filter(grepl('"[A-Z]+"', name_clean)) |>
    mutate(
      name_clean = REGEXP_REPLACE(name_clean, '.+? "([A-Z]+)"(.+)$', "\\1 \\2", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "[^A-Z ]", "", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "\\s+", " ", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "^\\s+|\\s+$", "", "g")
    ) |>
    collect()

  candidates <- candidates |>
    mutate(
      name_clean = REGEXP_REPLACE(name_clean, "[^A-Z ]", "", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "\\s+", " ", "g"),
      name_clean = REGEXP_REPLACE(name_clean, "^\\s+|\\s+$", "", "g")
    ) |>
    collect()

  candidates |>
    bind_rows(candidates_aliases) |>
    filter(!is.na(name_clean)) |>
    tidyr::extract(
      address,
      c("street", "city", "state", "postal_code"),
      "(.+), (.+), ([A-Z]{2}), (\\d{5})"
    ) |>
    mutate(
      street = if_else(!grepl("\\d", street), NA_character_, street),
    )
}

prepare_candidate_listing_for_matching <- function(candidate_listing) {
  lg_info_target(lg_get_logger())

  cl <-
    candidate_listing |>
    mutate(
      name_clean = toupper(name_on_ballot),
      name_clean = gsub("[^A-Z ]", "", name_clean)
    ) |>
    distinct(
      name_on_ballot, name_clean,
      street = street_address,
      city, state,
      postal_code = zip_code
    )
}

fastlink_candidates <- function(candidates_for_matching, candidate_listing_for_matching) {
  lg_info_target(lg_get_logger())

  fastLink::fastLink(
    candidates_for_matching,
    candidate_listing_for_matching,
    varnames = c("name_clean", "street", "city"),
    stringdist.match = c("name_clean", "street"),
    partial.match = c("name_clean", "street"),
    stringdist.method = "dl"
  )
}

fastlink_match_candidates <- function(
  candidates_for_matching,
  candidate_listing_for_matching,
  candidates_linked
) {
  lg_info_target(lg_get_logger())

  matches <- fastLink::getMatches(
    candidates_for_matching,
    candidate_listing_for_matching,
    candidates_linked,
    threshold.match = 0.8
  )

  names(matches)[ncol(candidates_for_matching) + 1] <- "name_on_ballot"

  candidates_for_matching |>
    left_join(
      matches |> select(sboe_id, name_on_ballot),
      by = join_by(sboe_id == sboe_id)
    )
}

candidates_match <- function(
  candidates_for_matching,
  candidate_listing_for_matching,
  candidates_linked
) {
  # First, direct matches
  candidates_matched_1 <-
    candidates_for_matching |>
    inner_join(
      candidate_listing_for_matching |>
        select(name_on_ballot, name_clean, street, city),
      by = join_by(
        name_clean == name_clean,
        street == street,
        city == city
      ),
      relationship = "many-to-many"
    ) |>
    distinct(sboe_id, name_on_ballot)

  # Then unambiguous matches on street + city
  matches_street_city <-
    candidates_for_matching |>
    anti_join(candidates_matched_1, by = "sboe_id") |>
    inner_join(
      candidate_listing_for_matching |>
        select(name_on_ballot, street, city),
      by = join_by(
        street == street,
        city == city
      ),
      relationship = "many-to-many"
    ) |>
    distinct(sboe_id, name_on_ballot, street, city) |>
    group_by(street, city) |>
    mutate(n_names = n_distinct(name_on_ballot)) |>
    ungroup() |>
    filter(n_names == 1) |>
    select(sboe_id, name_on_ballot)

  candidates_matched_2 <- union(candidates_matched_1, matches_street_city)

  candidates_for_matching_left <-
    candidates_for_matching |>
    anti_join(candidates_matched_2, by = "sboe_id")

  # Now fuzzyjoin...
  candidates_fuzzy_name <-
    zoomerjoin::jaccard_inner_join(
      candidates_for_matching_left,
      candidate_listing_for_matching |>
        select(name_on_ballot, name_clean, street, city),
      by = "name_clean",
      threshold = 0.85
    )

  candidates_fuzzy_name |>
    distinct(sboe_id, name_on_ballot) |>
    group_by(sboe_id) |>
    mutate(n_names = n_distinct(name_on_ballot)) |>
    ungroup() |>
    filter(n_names != 1) |>
    arrange(desc(n_names), sboe_id)


}