rework pipeline to locally store complete report as raw text

DESCRIPTION

@@ -12,13 +12,15 @@ Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.2.3
 Depends:
     dplyr,
+    fs,
     glue,
     httr2,
     readr,
-    rlang
+    rlang,
+    targets
 Imports:
     cli,
     crew,
     desc,
-    targets,
+    here,
     visNetwork

R/get.R

@@ -172,3 +172,27 @@ get_report_section <- function(
 
   res
 }
+
+get_raw_report_all <- function(report_id) {
+  delay()
+
+  res <-
+    req_report_detail(report_id, "all") |>
+    req_perform()
+
+  if (identical(res$body, raw(0))) return("")
+
+  resp_body_string(res)
+}
+
+save_raw_report_all <- function(report_id, sboe_id) {
+  res <- get_raw_report_all(report_id)
+
+  dir <- here::here("data-raw", "reports", sboe_id)
+  dir_create(dir)
+
+  path <- path(dir, report_id, ext = "txt")
+  brio::write_lines(res, path)
+
+  invisible(path)
+}

R/utils-delay.R

@@ -1,5 +1,5 @@
 delay <- function() {
   if (identical(Sys.getenv("IN_TARGETS"), "true")) {
-    Sys.sleep(runif(1, 0, 2))
+    Sys.sleep(runif(1, 0, 1))
   }
 }

_targets.R

@@ -43,14 +43,15 @@ list(
     report_list,
     doc_list |>
       filter(!is.na(report_id)) |>
-      filter(committee_name == "JOHN BELL COMM" | report_id == 188139)
+      mutate(received = coalesce(received_data, received_image)) |>
+      slice_max(received, by = report_id)
   ),
   tar_target(report_list_report_id, report_list$report_id),
   tar_target(report_list_sboe_id, report_list$sboe_id),
   tar_target(
-    receipts,
-    get_report_section(report_list_report_id, "receipts", report_list_sboe_id),
+    receipts_raw_path,
+    save_raw_report_all(report_list_report_id, report_list_sboe_id),
     pattern = map(report_list_report_id, report_list_sboe_id),
-    format = "parquet"
+    format = "file_fast"
   )
 )