|
- #+ setup
- library(tidyverse)
- library(fs)
- load_all(here::here("process"))
-
- #+ candidates
- john_bell <- "STA-8S285O-C-001"
- dan_forest <- "STA-M4HR0Y-C-001"
- mitch_setzer <- "STA-C1877N-C-001"
-
- #+ load-data
- cf <- cf_db_create()
-
- cf_dm <- dm::dm(!!!cf)
- cf_dm <- dm::dm_add_pk(cf_dm, "report_list", c("sboe_id", "report_id"))
-
- ## Report List vs Cover ----
-
- #+ basic-validation
- cf$report_list |> count()
- cf$cover |> count()
-
- ## When de-duplicated there should be no more reports in "cover" than in "report_list"
- cf$report_list |> distinct(report_id, sboe_id) |> count()
- cf$cover |> distinct(report_id, sboe_id) |> count()
-
- # duplicated sboe/report ids
- duplicated <-
- cf$cover |>
- semi_join(
- cf$cover |> count(sboe_id, report_id) |> filter(n > 1),
- by = c("sboe_id", "report_id")
- )
-
- # are all of the duplicated rows full duplicates? yes, both are the same.
- # => I'm going to go back and call `distinct()` when adding the file
- duplicated |> distinct(sboe_id, report_id) |> count()
- duplicated |> distinct() |> count()
-
- # The filed date is missing from some listed reports
- # => I'm going to leave this alone
- cf$report_list |> filter(is.na(received))
-
- # How many reports are missing from cover?
- cf$report_list |>
- anti_join(cf$cover, by = c("report_id", "sboe_id")) |>
- collect() |>
- arrange(sboe_id, year, report_id) |>
- select(1:5, received) |>
- mutate(path = glue::glue("../../data-raw/reports/{sboe_id}/all/{report_id}_{received}.txt"))
-
- #> * `../../data-raw/reports/STA-C3372N-C-001/all/174237_2020-04-01.txt`: missing cover
- #> * `../../data-raw/reports/STA-C4270N-C-001/all/169030_2019-08-15.txt`: missing cover
- #> * `../../data-raw/reports/STA-MW53OC-C-001/all/194017_2022-01-28.txt`: missing cover
- #> * `../../data-raw/reports/FED-JUU72L-C-001/all/201413_2021-07-27.txt`: no records
- #> * `../../data-raw/reports/STA-H3F77S-C-002/all/201481_2021-12-29.txt`: no records
- #> * `../../data-raw/reports/STA-O079OC-C-001/all/200919_2021-07-17.txt`: no records
- #> * `../../data-raw/reports/STA-XD82JF-C-001/all/210255_2022-10-24.txt`: no records
-
- # TODO: Identify reports that are missing a cover entry...
|
