#+ setup
library(tidyverse)
library(fs)
pkgload::load_all(here::here("process"))

#+ candidates
john_bell <- "STA-8S285O-C-001"
dan_forest <- "STA-M4HR0Y-C-001"
mitch_setzer <- "STA-C1877N-C-001"

#+ load-data
cf <- cf_db_create()

cf_dm <- dm::dm(!!!cf)
cf_dm <- dm::dm_add_pk(cf_dm, "report_list", c("sboe_id", "report_id"))

## Report List vs Cover ----

#+ basic-validation
cf$report_list |> count()
cf$report_list |> distinct(report_id, sboe_id) |> count()

## Cover should have the same number of reports as report_list
cf$cover |> count()
cf$cover |> distinct(report_id, sboe_id) |> count()

# duplicated sboe/report ids
duplicated <-
  cf$cover |>
  semi_join(
    cf$cover |> count(sboe_id, report_id) |> filter(n > 1),
    by = c("sboe_id", "report_id")
  )

# FIXED: are all of the duplicated rows full duplicates? yes, both are the same.
# => I'm going to go back and call `distinct()` when adding the file
duplicated |> distinct(sboe_id, report_id) |> count()
duplicated |> distinct() |> count()

# The filed date is missing from some listed reports
# => I'm going to leave this alone
cf$report_list |> filter(is.na(received))

# FIXED: How many reports are missing from cover?
cf$report_list |>
  anti_join(cf$cover, by = c("report_id", "sboe_id")) |>
  collect() |>
  arrange(sboe_id, year, report_id) |>
  select(1:5, received) |>
  mutate(path = glue::glue("../../data-raw/reports/{sboe_id}/all/{report_id}_{received}.txt"))

#> * `../../data-raw/reports/STA-C3372N-C-001/all/174237_2020-04-01.txt`: missing cover
#> * `../../data-raw/reports/STA-C4270N-C-001/all/169030_2019-08-15.txt`: missing cover
#> * `../../data-raw/reports/STA-MW53OC-C-001/all/194017_2022-01-28.txt`: missing cover
#> * `../../data-raw/reports/FED-JUU72L-C-001/all/201413_2021-07-27.txt`: no records
#> * `../../data-raw/reports/STA-H3F77S-C-002/all/201481_2021-12-29.txt`: no records
#> * `../../data-raw/reports/STA-O079OC-C-001/all/200919_2021-07-17.txt`: no records
#> * `../../data-raw/reports/STA-XD82JF-C-001/all/210255_2022-10-24.txt`: no records

# FIXED: Identify reports that are missing a cover entry...
no_records <-
  cf$report_list |>
  anti_join(distinct(cf$receipts, sboe_id, report_id)) |>
  anti_join(distinct(cf$expenditures, sboe_id, report_id)) |>
  anti_join(distinct(cf$summary, sboe_id, report_id))

missing_cover <-
  cf$report_list |>
  anti_join(cf$cover, by = c("sboe_id", "report_id")) |>
  anti_join(no_records, by = c("sboe_id", "report_id"))

cf$report_list |> semi_join(missing_cover)
cf$summary |>semi_join(missing_cover)


# Some committees have "No Id" as their SBOE ID
# => I'll make these "NOID-{report_id}"
unique_committees_report <- cf$report_list |> distinct(sboe_id, committee_name)

unique_committees_report |>
  semi_join(
    unique_committees_report |> count(sboe_id) |> filter(n > 1),
    by = "sboe_id"
  )

# Bad report dates
cover <-
  cf$cover |>
  select(1:3, report_type, contains("date")) |>
  collect()

cover |>
  filter(
    date_filed > today() | date_to > today() | date_from > today()
  )

cover |> filter(date_filed < date_from)

cover |> filter(sboe_id == "FED-361L24-C-001")

cover |>
  filter(
    !(date_filed > today() | date_to > today() | date_from > today())
  ) |>
  ggplot() +
  aes(x = date_to, y = date_filed) +
  geom_point()

# Committee names change over time?
unique_committees <- cf$cover |> distinct(sboe_id, committee_name)

cf$cover |>
  semi_join(
    unique_committees |> count(sboe_id) |> filter(n > 1),
    by = "sboe_id"
  ) |>
  arrange(sboe_id, date_filed) |>
  select(1, date_filed, 2:zip_code)

# Committee registration ----
cf$cover |>
  slice_max(date_filed, by = sboe_id) |>
  select(sboe_id:committee_type, last_date_filed = date_filed) |>
  left_join(
    cf$cover |> slice_min(date_filed, by = sboe_id) |> select(sboe_id, first_date_filed = date_filed),
    by = "sboe_id"
  ) |>
  arrange(desc(last_date_filed), first_date_filed)


# simple committee registration (best effort, not perfect) FIXME
committees <-
  report_list |>
  slice_max(end_date, by = sboe_id, n = 1) |>
  slice_max(report_id, by = sboe_id, n = 1) |>
  distinct(sboe_id, committee_name)


# Dealing with appended reports -------------------------------------------

## Which reports have been appended?

reports_summary <-
  report_list |>
  summarize(
    n_total = n(),
    n_amended = sum(amended),
    .by = c(sboe_id, year, doc_name)
  ) |>
  arrange(desc(n_total)) |>
  left_join(committees, by = "sboe_id")

# Dan Forest, 2016-Q2 Report, 6 reports, 5 ammendments
# [x] Can be resolved by looking at last received date!
report_list |>
  filter(
    sboe_id == "STA-M4HR0Y-C-001",
    year == 2016,
    doc_name == "Second Quarter"
  ) |>
  slice_max(received)


report_list |>
  group_by(sboe_id, year, doc_name) |>
  slice_max(received) |>
  count(sort = TRUE) |>
  filter(n > 1)

# report_id is basically sequential
report_list |>
  filter(!is.na(received_data)) |>
  filter(received < today()) |>
  filter(received > ymd("2016-01-01")) |>
  ggplot() +
  aes(received, report_id) +
  geom_point()

report_list |>
  filter(report_id == min(report_id))

report_list |>
  filter(
    between(received_image, ymd("2016-01-01"), today()),
    between(received_data, ymd("2016-01-01"), today())
  ) |>
  ggplot() +
  aes(received_image, received_data) +
  geom_point()

report_list |>
  filter(
    between(received_image, ymd("2016-01-01"), today()),
    between(received_data, ymd("2016-01-01"), today())
  ) |>
  pivot_longer(contains("received_"), names_to = "received_item", values_to = "received_when") |>
  mutate(received_item = str_remove(received_item, "received_")) |>
  ggplot() +
  aes(end_date, received_when) +
  geom_point() +
  facet_wrap(~ received_item)


report_list |> filter(end_date > today())

levels_doc_name <- str_to_title(c(
  paste(c("first", "second", "third", "fourth"), "quarter"),
  paste(c("mid year", "year end"), "Semi-Annual")
))

# wait, what are the correct reporting dates?
report_list |>
  mutate(doc_name = factor(doc_name, levels_doc_name)) |>
  group_by(year, doc_name) |>
  count(start_date, end_date, sort = TRUE) |>
  slice_max(n) |>
  slice_max(end_date) |>
  print(n = 48)


# https://www.ncleg.gov/EnactedLegislation/Statutes/HTML/BySection/Chapter_163/GS_163-278.9.html
report_list |>
  mutate(doc_name = factor(doc_name, levels_doc_name)) |>
  filter(
    between(start_date, make_date(year), make_date(year + 1)),
    between(end_date, make_date(year), make_date(year + 1))
  ) |>
  group_by(year, doc_name) |>
  count(start_date, end_date, sort = TRUE) |>
  ungroup() |>
  arrange(year, doc_name) |>
  mutate(quarterly = ifelse(grepl("Quarter", doc_name), "Quarterly", "Semi-Annual")) |>
  ggplot() +
  aes(y = year, color = doc_name, group = doc_name) +
  geom_linerange(
    aes(xmin = start_date, xmax = end_date, alpha = n),
    linewidth = 20,
  ) +
  facet_wrap(~quarterly) +
  theme_minimal()

report_list |>
  left_join(report_dates, by = c("year", "doc_name"), relationship = "many-to-one") |>
  filter(
    !(between(start_date, sboe_start_date, sboe_end_date) | between(end_date, sboe_start_date, sboe_end_date))
  ) |>
  View()
