nc-campaign-finance/process/R/db_connect.R

.globals <- new.env(parent = emptyenv())

duckdb_global_con <- function() {
  if (is.null(.globals$duckdb_con)) {
    .globals$duckdb_con <- con <- DBI::dbConnect(duckdb::duckdb())
  }

  .globals$duckdb_con
}

cf_prep_db_create <- function(data_dir = here::here("data-prep")) {
  tables <- dir_ls(data_dir)
  names(tables) <- path_file(tables)

  idx_report_list <- which(names(tables) == "report_list")

  cli::cli_progress_bar(name = "Opening parquet files", total = length(tables))
  env <- rlang::current_env()

  tbls_arrow <- tables[-idx_report_list] |>
    imap(function(path, name) {
      cli::cli_progress_update(inc = 1, status = name, .envir = env)
      arrow::open_dataset(path, partitioning = "sboe_id")
    })

  cli::cli_progress_update(inc = 1, status = "report_list")
  tbls_arrow$report_list <- arrow::open_dataset(tables["report_list"])
  cli::cli_progress_done()

  con <- duckdb_global_con()

  cli::cli_progress_step("Creating db")

  imap(tbls_arrow, function(df, name) {
    duckdb::duckdb_register_arrow(con, name, df)
    dplyr::tbl(con, name)
  })
}