reorganize targets an ensure up to date

R/get.R

@@ -176,6 +176,10 @@ get_report_section <- function(
 get_raw_report_all <- function(report_id) {
   delay()
 
+  if (!identical(Sys.getenv("ALLOW_DOWNLOADS"), "true")) {
+    stop("Shouldn't be downloading reports now...")
+  }
+
   res <-
     req_report_detail(report_id, "all") |>
     req_perform()
@@ -185,13 +189,20 @@ get_raw_report_all <- function(report_id) {
   resp_body_string(res)
 }
 
-save_raw_report_all <- function(report_id, sboe_id) {
+save_raw_report_all <- function(report_id, sboe_id, received) {
+  dir <- here::here("data-raw", "reports", sboe_id)
+  file_name <- paste0(report_id, "_", received)
+  path <- path(dir, file_name, ext = "txt")
+
+  if (file_exists(path)) {
+    # We don't need to re-download any reports
+    return(path)
+  }
+
   res <- get_raw_report_all(report_id)
 
-  dir <- here::here("data-raw", "reports", sboe_id)
   dir_create(dir)
 
-  path <- path(dir, report_id, ext = "txt")
   brio::write_lines(res, path)
 
   invisible(path)

_targets.R

@@ -48,10 +48,19 @@ list(
   ),
   tar_target(report_list_report_id, report_list$report_id),
   tar_target(report_list_sboe_id, report_list$sboe_id),
+  tar_target(report_list_received, report_list$received),
   tar_target(
-    receipts_raw_path,
-    save_raw_report_all(report_list_report_id, report_list_sboe_id),
-    pattern = map(report_list_report_id, report_list_sboe_id),
+    reports_raw_path,
+    save_raw_report_all(
+      report_list_report_id,
+      report_list_sboe_id,
+      report_list_received
+    ),
+    pattern = map(
+      report_list_report_id,
+      report_list_sboe_id,
+      report_list_received
+    ),
     format = "file_fast"
   )
 )

run.R

@@ -6,7 +6,8 @@
 # to learn about your options.
 
 Sys.setenv("IN_TARGETS" = "true")
+Sys.setenv("ALLOW_DOWNLOADS" = "false")
 
+targets::tar_make_future(workers = 8)
 targets::tar_make()
 # targets::tar_make_clustermq(workers = 2) # nolint
-# targets::tar_make_future(workers = 2) # nolint