`cover` table should have distinct rows, fill in missing covers

process/R/read_report_file.R

@@ -93,6 +93,8 @@ pre_process_table_body <- function(table, header, body) {
 post_process_steps_for_table <- function(data, table) {
   switch(
     table,
+    cover = ,
+    accounts = distinct(data),
     forgiven_loans = ,
     loan_proceeds = filter(data, !is.na(amount)),
     data
@@ -103,10 +105,29 @@ process_report_export <- function(dir_sboe_id, report_list = tar_read(report_lis
   all_exports <- dir_ls(dir_sboe_id, glob = "*.txt")
   info <- report_info_in_report_list(all_exports, report_list)
 
-  info$path |>
+  all <-
+    info$path |>
     map(read_report_file) |>
     list_transpose_bind() |>
     map(report_data_set_column_type)
+
+  if (!"cover" %in% names(all)) {
+    all$cover <- semi_join(report_list, info, by = "report_id") |> create_missing_cover()
+    return(all)
+  }
+
+  missing_cover <-
+    report_list |>
+    semi_join(info, by = "report_id") |>   # in this export group
+    anti_join(all$cover, by = "report_id") # but doesn't have cover
+
+  if (nrow(missing_cover) == 0) {
+    return(all)
+  }
+
+  all$cover <- bind_rows(all$cover, create_missing_cover(missing_cover))
+
+  all
 }
 
 report_data_set_column_type <- function(data) {
@@ -145,3 +166,24 @@ write_processed_report_export <- function(dir_sboe_id, report_list = tar_read(re
 
   return_path
 }
+
+create_missing_cover <- function(missing_cover) {
+  missing_cover |>
+    mutate(report_type = paste(year, doc_name)) |>
+    select(
+      report_id,
+      sboe_id,
+      committee_name,
+      report_type,
+      date_from = start_date,
+      date_to = end_date,
+      date_filed = received
+    ) |>
+    mutate(
+      street_1 = NA_character_, street_2 = NA_character_, city = NA_character_,
+      state = NA_character_, zip_code = NA_character_, country = NA_character_,
+      postal_code = NA_character_, committee_type = NA_character_,
+      .before = report_type
+    ) |>
+    mutate(fund_type = NA_character_, fund_name = NA_character_, .after = report_type)
+}