nc-campaign-finance/process/R/db_connect.R

cf_db_create <- function(data_dir = here::here("data")) {
  tables <- dir_ls(data_dir)
  names(tables) <- path_file(tables)

  idx_report_list <- which(names(tables) == "report_list")

  cli::cli_progress_bar(name = "Opening parquet files", total = length(tables))
  env <- rlang::current_env()

  tbls_arrow <- tables[-idx_report_list] |>
    imap(function(path, name) {
      cli::cli_progress_update(inc = 1, status = name, .envir = env)
      arrow::open_dataset(path, partitioning = "sboe_id")
    })

  cli::cli_progress_update(inc = 1, status = "report_list")
  tbls_arrow$report_list <- arrow::open_dataset(tables["report_list"])
  cli::cli_progress_done()

  con <- DBI::dbConnect(duckdb::duckdb())

  cli::cli_progress_step("Creating db")

  imap(tbls_arrow, function(df, name) {
    duckdb::duckdb_register_arrow(con, name, df)
    dplyr::tbl(con, name)
  })
}