diff --git a/tests/testthat/helper-walkthrough-raw.R b/tests/testthat/helper-walkthrough-raw.R new file mode 100644 index 0000000..9f76592 --- /dev/null +++ b/tests/testthat/helper-walkthrough-raw.R @@ -0,0 +1,44 @@ +# Helper for the Madison-walkthrough finding tests (uscogdata #11-#16). +# +# Those tests all assert something about what a `cog_*` verb includes or +# excludes. The expected amounts must therefore come from the RAW corpus, never +# from the verb under test: verifying an absence through the filter that creates +# it proves nothing. `wt_raw_*()` opens its own DuckDB connection straight onto +# the corpus's `long` parquet partitions, bypassing uscogdata's SQL views (and +# therefore its `flow_prefixes` filtering) entirely. + +wt_corpus_glob <- function() { + url <- Sys.getenv("USCOGDATA_URL") + if (!nzchar(url)) testthat::skip("USCOGDATA_URL is not set") + paste0(sub("/$", "", url), "/data/long/**/*.parquet") +} + +wt_raw_query <- function(sql) { + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + DBI::dbGetQuery(con, sql) +} + +# Sum of `amt` (in $1,000s, as the corpus stores it) for one government-year, +# restricted either to an explicit set of item codes or to a set of first-letter +# prefixes. Aggregate rows are excluded, matching every published verb. +wt_raw_amt <- function(govid, year, codes = NULL, prefixes = NULL) { + stopifnot(xor(is.null(codes), is.null(prefixes))) + filter_sql <- if (!is.null(codes)) { + paste0("item_code IN (", paste0("'", codes, "'", collapse = ", "), ")") + } else { + paste0("LEFT(item_code, 1) IN (", paste0("'", prefixes, "'", collapse = ", "), ")") + } + out <- wt_raw_query(paste0( + "SELECT COALESCE(SUM(amt), 0) AS amt FROM read_parquet('", wt_corpus_glob(), "') ", + "WHERE canonical_govid = '", govid, "' AND year = ", year, + " AND NOT is_aggregate AND ", filter_sql + )) + out$amt[[1]] +} + +# The item codes a verb reports having summed, flattened out of the +# comma-separated `codes_included` column. +wt_codes_included <- function(df) { + sort(unique(trimws(unlist(strsplit(stats::na.omit(df$codes_included), ","))))) +} diff --git a/tests/testthat/test-amount-units-documented.R b/tests/testthat/test-amount-units-documented.R new file mode 100644 index 0000000..7f0c8d2 --- /dev/null +++ b/tests/testthat/test-amount-units-documented.R @@ -0,0 +1,46 @@ +# Madison walkthrough audit -- finding F-004. Tracked as uscogdata#15. +# See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# The raw Census files report thousands of dollars; this package multiplies by +# 1000 and returns full US dollars. That is the friendlier choice and is not +# wrong -- but cog_explorer's CLAUDE.md states "All raw `amt` values are in +# $1,000s", so a reader who applies that rule to amt_nominal overstates every +# figure by 1000x, and gets a plausible-looking number rather than an obvious +# error. The audit rates this the highest-consequence definitional gap it found. +# +# Deliberately NOT asserted here: man/cog_spending.Rd and man/cog_revenue.Rd, +# which ALREADY carry the statement in their @return sections (verified +# 2026-07-29), as does cog-api's data-dictionary.md (since 2b71b41). The gap is +# in the surfaces a reader meets first and in cog_explorer's own conventions +# doc -- see uscogdata#15 for the full surface-by-surface table and for the two +# secondary tasks (cog_explorer/CLAUDE.md, which has no git remote, and +# cog-api's llms.txt, which is silent on units). + +test_that("returned amounts are documented as full US dollars where readers meet the package", { + testthat::skip("Blocked on uscogdata#15 (finding F-004)") + + says_units <- function(path) { + txt <- paste(readLines(path, warn = FALSE), collapse = " ") + grepl("full US dollars|full U\\.S\\. dollars", txt, ignore.case = TRUE) && + grepl("\\$1,000s|thousands of dollars", txt, ignore.case = TRUE) + } + + expect_true(says_units(testthat::test_path("..", "..", "README.md"))) + expect_true(says_units(testthat::test_path("..", "..", "vignettes", "total-spending.Rmd"))) + expect_true(says_units(testthat::test_path("..", "..", "vignettes", + "population-denominators.Rmd"))) + + # Pin the documented claim to the actual behaviour, so the two cannot drift. + # The expected raw amount is read straight from the corpus's parquet + # partitions -- never through cog_spending(), which is the thing being + # described. Madison FY2020: E/F/G = 623,347 ($1,000s) -> $623,347,000. + raw_thousands <- wt_raw_amt("552025209777", 2020L, prefixes = c("E", "F", "G")) + expect_equal(raw_thousands, 623347) + + returned <- cog_spending(govid = "552025209777", years = 2020L) + expect_equal(sum(returned$amt_nominal), raw_thousands * 1000) + + units <- attr(returned, "provenance")$transformations$units_conversion + expect_true(units$applied) + expect_equal(units$multiplier, 1000) +}) diff --git a/tests/testthat/test-coverage-disclosure.R b/tests/testthat/test-coverage-disclosure.R new file mode 100644 index 0000000..62e97cb --- /dev/null +++ b/tests/testthat/test-coverage-disclosure.R @@ -0,0 +1,92 @@ +# Madison walkthrough audit -- findings F-020 and F-023. Tracked as uscogdata#13. +# See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# The owner's settled design (2026-07-28): a `coverage` argument on +# cog_geographic_rollup(), cog_find_peers()/cog_peer_compare() and their +# cog-api equivalents -- +# "all" every unit that reported that year (today's behaviour, DEFAULT) +# "census" census years only (years ending 2 or 7) +# "consistent" only units reporting in every requested year (balanced panel) +# -- PLUS always-on coverage metadata on every result regardless of mode: +# n_units_reporting, n_units_expected, is_census_year. +# +# Motivating principle: using these verbs correctly must not require the user to +# know that the Census of Governments is a complete census only in years ending +# in 2 and 7. +# +# The helper below accepts that metadata either as columns on the returned +# tibble or as a per-year table in provenance$coverage -- the design fixes the +# three field names and that they reach the caller, not the container. + +wt_coverage <- function(x) { + prov <- attr(x, "provenance") + cov <- prov$coverage + if (is.null(cov)) { + needed <- c("year", "n_units_reporting", "n_units_expected", "is_census_year") + expect_true(all(needed %in% names(x))) + cov <- unique(x[, needed]) + } + cov[order(cov$year), ] +} + +test_that("multi-government aggregates disclose reporting coverage on every result", { + testthat::skip("Blocked on uscogdata#13 (findings F-020, F-023)") + + # -- F-020: geographic rollups ------------------------------------------- + # Wisconsin's city/village universe is 608 governments. On the bundled + # fixture, FY2012 (a census year) has 597 of them reporting while FY2019 and + # FY2020 (sample years) have 112 and 114 -- an 18%-98% swing that today's + # return value says nothing about. Counts cross-checked against the raw + # corpus, not through cog_geographic_rollup(), which is under test. + wi <- cog_gov_search(name = NULL, state = "WI", type = "city") + expect_equal(nrow(wi), 608L) + + roll <- cog_geographic_rollup(govids = list(city = wi$canonical_govid), + category = NULL, years = c(2011L, 2012L, 2019L, 2020L)) + cov <- wt_coverage(roll) + + expect_equal(cov$n_units_expected, rep(608L, 4L)) + expect_equal(cov$n_units_reporting, c(152L, 597L, 112L, 114L)) + expect_equal(cov$is_census_year, c(FALSE, TRUE, FALSE, FALSE)) + + raw_2012 <- wt_raw_query(paste0( + "SELECT COUNT(DISTINCT canonical_govid) n FROM read_parquet('", wt_corpus_glob(), "') ", + "WHERE type = 2 AND fips_state = 55 AND year = 2012 ", + "AND LEFT(item_code, 1) IN ('E','F','G') AND NOT is_aggregate")) + expect_equal(cov$n_units_reporting[cov$year == 2012], as.integer(raw_2012$n[[1]])) + + # -- F-023: peer cohorts -------------------------------------------------- + # CHILTON CITY, WI (ACS population 4,017): a 15-peer cohort fixed at FY2012 + # reports 15 of 15 in FY2012 and only 3 of 15 in FY2019 and FY2020. Nothing + # in cog_peer_compare()'s return distinguishes those years today. + chilton <- "552015177095" + peers <- cog_find_peers(chilton, year = 2012L, max_peers = 15L) + expect_equal(nrow(peers), 15L) + + cmp <- cog_peer_compare(target_govid = chilton, peers = peers, category = NULL, + years = c(2012L, 2019L, 2020L), per_capita = TRUE) + cov_peers <- wt_coverage(cmp) + expect_equal(cov_peers$n_units_expected, rep(15L, 3L)) + expect_equal(cov_peers$n_units_reporting, c(15L, 3L, 3L)) + expect_equal(cov_peers$is_census_year, c(TRUE, FALSE, FALSE)) + + # -- the three coverage modes -------------------------------------------- + expect_equal(attr(cog_peer_compare(target_govid = chilton, peers = peers, + category = NULL, years = c(2012L, 2019L, 2020L), + per_capita = TRUE), + "provenance")$coverage_mode, "all") # unchanged default + + consistent <- cog_peer_compare(target_govid = chilton, peers = peers, + category = NULL, years = c(2012L, 2019L, 2020L), + per_capita = TRUE, coverage = "consistent") + n_by_year <- tapply(consistent$canonical_govid[consistent$role == "peer"], + consistent$year[consistent$role == "peer"], + function(g) length(unique(g))) + expect_equal(unname(as.integer(n_by_year)), c(3L, 3L, 3L)) # balanced panel + + census_only <- cog_geographic_rollup(govids = list(city = wi$canonical_govid), + category = NULL, + years = c(2011L, 2012L, 2019L, 2020L), + coverage = "census") + expect_equal(sort(unique(census_only$year)), 2012) +}) diff --git a/tests/testthat/test-expenditure-concepts.R b/tests/testthat/test-expenditure-concepts.R new file mode 100644 index 0000000..1349e2c --- /dev/null +++ b/tests/testthat/test-expenditure-concepts.R @@ -0,0 +1,75 @@ +# Madison walkthrough audit -- findings F-012, F-017, F-018. +# Tracked as uscogdata#11. See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# The owner's settled three-concept model (2026-07-28): +# total = primary + interest + intergovernmental transfers +# direct = primary + interest (Census's published Direct Expenditure) +# primary = direct minus debt service (the NEW DEFAULT) +# implemented by reclassifying on the crosswalk's `spend_type` column, NOT on +# item-code first letters -- F-018 shows prefix `Y` carries both revenue +# (Y01/Y02) and expenditure (Y05/Y06) codes, so no first-letter allowlist can +# route them correctly. +# +# Fixture reproducibility: the finding's headline reconciliation is Madison +# FY2022, where the corpus carries I89 = 46,609 (thousands) and Census's +# published Direct Expenditure is $654,893,000 against cog_spending()'s +# $608,284,000 (-7.1%). FY2022 is outside the bundled fixture's year window +# (2011/2012/2019/2020), so the same invariant is asserted on FY2020, where the +# fixture carries I89 = 27,704. Anyone running against the full corpus should +# also check the FY2022 numbers above. + +test_that("expenditure concepts classify on spend_type, not item-code prefix", { + testthat::skip("Blocked on uscogdata#11 (findings F-012, F-017, F-018)") + + mad <- "552025209777" # MADISON CITY, WI + wi_state <- "550000227544" # WISCONSIN (state government) + + # -- F-012: `primary` is the new default, and equals today's E/F/G figure --- + primary <- cog_spending(govid = mad, years = 2020L) + expect_equal(attr(primary, "provenance")$expenditure_concept, "primary") + expect_equal(sum(primary$amt_nominal), 623347000) + + # -- F-012: `direct` adds interest on long-term debt ------------------------ + # Expected interest read from the RAW corpus, never through cog_spending(), + # which is the filter under test. + interest <- wt_raw_amt(mad, 2020L, prefixes = "I") + expect_equal(interest, 27704) # I89, in $1,000s + + direct <- cog_spending(govid = mad, years = 2020L, expenditure_concept = "direct") + expect_equal(sum(direct$amt_nominal), 651051000) # 623,347 + 27,704 thousands + expect_equal(sum(direct$amt_nominal) - sum(primary$amt_nominal), interest * 1000) + expect_true("I89" %in% wt_codes_included(direct)) + + # -- F-017: `total` carries Q12/Q18, state IG transfers to school districts -- + # Wisconsin FY2019: Q12 = 6,431,530 and Q18 = 533,391 (thousands). Today + # neither verb's flow_prefixes contains "Q", so both are dropped from the one + # concept that is supposed to include intergovernmental transfers. + ig_expected <- wt_raw_amt(wi_state, 2019L, prefixes = c("M", "L", "Q")) + expect_equal(ig_expected, 11609814) # M 4,644,893 + Q 6,964,921 + + wi_direct <- cog_spending(govid = wi_state, years = 2019L, + expenditure_concept = "direct") + wi_total <- cog_spending(govid = wi_state, years = 2019L, + expenditure_concept = "total") + + # total - direct is exactly the intergovernmental component. Asserted as a + # delta rather than a grand total so this stays correct however the J and Y + # families land inside `primary`. + expect_equal(sum(wi_total$amt_nominal) - sum(wi_direct$amt_nominal), + ig_expected * 1000) + expect_true(all(c("Q12", "Q18") %in% wt_codes_included(wi_total))) + + # -- F-018: prefix Y splits revenue from expenditure, by spend_type --------- + # Y01/Y02 are Insurance Trust revenue; Y05/Y06 are Insurance Trust benefit + # payments. All four share the first letter `Y` and the spend_type + # "Insurance Trust", so this pair of assertions is the concrete proof that + # classification is no longer keyed on the first letter. + wi_revenue <- cog_revenue(govid = wi_state, years = 2019L) + spend_codes <- wt_codes_included(wi_total) + rev_codes <- wt_codes_included(wi_revenue) + + expect_true("Y05" %in% spend_codes) + expect_false("Y05" %in% rev_codes) + expect_true("Y01" %in% rev_codes) + expect_false("Y01" %in% spend_codes) +}) diff --git a/tests/testthat/test-gov-search-literal-match.R b/tests/testthat/test-gov-search-literal-match.R new file mode 100644 index 0000000..3ea1bdd --- /dev/null +++ b/tests/testthat/test-gov-search-literal-match.R @@ -0,0 +1,58 @@ +# Madison walkthrough audit -- finding F-025. Tracked as uscogdata#16. +# See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# cog_gov_search()'s UTILITY mode interpolates `name` into +# regexp_matches(gov_name, , 'i') +# unescaped (R/search.R:102), while BASKET mode in the same file already routes +# it through .escape_regex() (R/search.R:307) with the comment "so `name` is +# treated as a literal substring". Two failure modes result: +# correctness -- a real government cannot be found by its own exact name, and +# a single "." matches everything (HTTP 200 both ways via the API); +# robustness -- malformed regex reaches the engine and errors, which cog-api +# surfaces as a 500, reachable by typing a real name one +# character at a time. +# +# NOT asserted here: the finding's `q=St. Louis` example. Under correct literal +# matching that search still returns 0 rows, because the stored name is +# "ST LOUIS CITY" with no period -- it demonstrates today's over-matching +# semantics, not a row the fix makes findable. + +test_that("cog_gov_search() matches name literally, not as an unescaped regex", { + testthat::skip("Blocked on uscogdata#16 (finding F-025)") + + # -- correctness (1): a government must be findable by its own exact name --- + # FREDONIA (BRISCOE) CITY is real; today the parentheses are read as regex + # grouping, so its own complete name matches nothing. + fredonia <- cog_gov_search(name = "FREDONIA (BRISCOE) CITY") + expect_equal(nrow(fredonia), 1L) + expect_equal(fredonia$canonical_govid, "052117184386") + expect_equal(cog_gov_search(name = "FREDONIA (BRISCOE)")$canonical_govid, + "052117184386") + + # -- correctness (2): a metacharacter must not become a wildcard ------------ + # No Wisconsin city or village name contains a literal period -- established + # against the raw registry below, NOT through the verb under test. A literal + # search for "." must therefore return nothing; today it returns all 608. + con <- DBI::dbConnect(duckdb::duckdb()) + on.exit(DBI::dbDisconnect(con, shutdown = TRUE), add = TRUE) + xwalk <- paste0(sub("/$", "", Sys.getenv("USCOGDATA_URL")), + "/data/canonical_fips_xwalk.parquet") + with_dot <- DBI::dbGetQuery(con, paste0( + "SELECT COUNT(*) n FROM read_parquet('", xwalk, "') ", + "WHERE fips_state = '55' AND govs_type = 2 AND gov_name LIKE '%.%'")) + expect_equal(as.integer(with_dot$n[[1]]), 0L) + + expect_equal(nrow(cog_gov_search(name = ".", state = "WI", type = "city")), 0L) + expect_equal(nrow(cog_gov_search(name = "M.dison", state = "WI", type = "city")), 0L) + expect_equal(nrow(cog_gov_search(name = "Mad(i|o)son", state = "WI", type = "city")), 0L) + + # A metacharacter-free name still resolves exactly as before. + expect_equal(nrow(cog_gov_search(name = "Madison", state = "WI", type = "city")), 1L) + + # -- robustness: malformed pattern text returns no rows, and does not error -- + # "[" alone, and "Athens-Clarke County (bal" -- an in-progress substring of + # ATHENS-CLARKE COUNTY (BALANCE), a real government -- both currently raise + # (DuckDB: "Invalid Input Error: missing ]"). + expect_equal(nrow(cog_gov_search(name = "[")), 0L) + expect_equal(nrow(cog_gov_search(name = "Athens-Clarke County (bal")), 0L) +}) diff --git a/tests/testthat/test-peer-summary-scope.R b/tests/testthat/test-peer-summary-scope.R new file mode 100644 index 0000000..c08b702 --- /dev/null +++ b/tests/testthat/test-peer-summary-scope.R @@ -0,0 +1,45 @@ +# Madison walkthrough audit -- finding F-021. Tracked as uscogdata#14. +# See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# .peer_summary_rows() computes stats::quantile() separately INSIDE each +# (year, spend_subtype, category) cell. A summary_p50 row is therefore "the +# median peer's value in that one category", not "the value of the median +# peer's total". Summing those rows across categories -- the obvious move for a +# caller who wants one peer-median total line and reads only the column names -- +# misstated a total-spending band by -32.7% to +251.0% across the 24 years the +# audit tested, with a sign flip at FY2012. +# +# The verb is not wrong and its documented use (faceting by role AND category) +# is unaffected, so the fix is documentation: one sentence in @return. + +test_that("cog_peer_compare() documents that summary_* rows are per-category quantiles", { + testthat::skip("Blocked on uscogdata#14 (finding F-021)") + + rd <- paste(readLines(testthat::test_path("..", "..", "man", "cog_peer_compare.Rd"), + warn = FALSE), collapse = " ") + + # The @return section must say the quantile is computed within each cell... + expect_match(rd, "within each|per-category|per category", ignore.case = TRUE) + # ...and must warn that the rows are not additive across category. + expect_match(rd, "not additive|do(es)? not sum|cannot be summed", ignore.case = TRUE) + # ...naming the grouping explicitly. + expect_match(rd, "spend_subtype", fixed = TRUE) + + # Pin the mechanism numerically so a future refactor that quietly changes the + # quantile grouping fails here rather than silently invalidating the sentence + # above. Fixture: Madison, 10 peers found at FY2020, category = NULL. + peers <- cog_find_peers("552025209777", year = 2020L, max_peers = 10L) + cmp <- cog_peer_compare(target_govid = "552025209777", peers = peers, + category = NULL, years = 2020L, per_capita = TRUE) + + naive <- sum(cmp$amt_per_capita_nominal[cmp$role == "summary_p50"], na.rm = TRUE) + + peer_rows <- cmp[cmp$role == "peer", ] + per_gov <- tapply(peer_rows$amt_per_capita_nominal, peer_rows$canonical_govid, + sum, na.rm = TRUE) + correct <- unname(stats::quantile(per_gov, 0.5, na.rm = TRUE)) + + expect_equal(round(naive), 6180) # summing the built-in summary rows + expect_equal(round(correct), 2043) # quantile of each peer's OWN total + expect_gt(naive / correct, 2) # a +200% misstatement on this cohort +}) diff --git a/tests/testthat/test-revenue-concept-insurance-trust.R b/tests/testthat/test-revenue-concept-insurance-trust.R new file mode 100644 index 0000000..e53d9d2 --- /dev/null +++ b/tests/testthat/test-revenue-concept-insurance-trust.R @@ -0,0 +1,55 @@ +# Madison walkthrough audit -- finding F-014. Tracked as uscogdata#12. +# See docs/walkthroughs/FINDINGS.md in cog_explorer. +# +# cog_revenue()'s flow_prefixes = c("T","A","U","B","C","D") never returns +# item-code prefix X (Employee Retirement) or Y (other Insurance Trust). Per +# Census's standard identity, Total Revenue = General + Utility + Liquor Store + +# Insurance Trust Revenue, and Employee Retirement System contributions and +# earnings ARE the Insurance Trust Revenue component -- so prefix X sits inside +# a published Census revenue concept exactly the way I89 sits inside Census's +# Direct Expenditure concept (finding F-012). +# +# CAVEAT FOR WHOEVER PICKS THIS UP: the argument name below (`revenue_concept = +# "total"`) is this test's *proposal*, not a settled decision. The owner's +# 2026-07-28 resolution covers expenditure concepts only; no revenue-side +# naming has been ruled on. If the eventual argument is named differently, +# change the two calls here -- the asserted dollar invariants are what matter +# and are independent of the naming. +# +# Fixture reproducibility: Madison's own X-prefix revenue (FY1970-FY1986, +# $15,098,000 nominal, $0 thereafter) is outside the bundled fixture's year +# window (2011/2012/2019/2020), so the same invariant is asserted on Wisconsin +# state government FY2012, where the fixture carries nonzero X01/X05/X08. + +test_that("cog_revenue() can return Census Total Revenue including Insurance Trust (prefix X)", { + testthat::skip("Blocked on uscogdata#12 (finding F-014)") + + wi_state <- "550000227544" # WISCONSIN (state government) + + # Revenue-shaped Employee Retirement codes, read from the RAW corpus rather + # than through cog_revenue(), which is the filter under test: + # X01 local employee contribution, X04/X05 contributions and transfers from + # other governments, X08 earnings on investments. + x_revenue <- wt_raw_amt(wi_state, 2012L, codes = c("X01", "X04", "X05", "X08")) + expect_equal(x_revenue, 2038800) # 615,835 + 0 + 560,382 + 862,583 ($1,000s) + + general <- cog_revenue(govid = wi_state, years = 2012L) + expect_equal(sum(general$amt_nominal), 31338293000) + + total <- cog_revenue(govid = wi_state, years = 2012L, revenue_concept = "total") + expect_equal(sum(total$amt_nominal) - sum(general$amt_nominal), x_revenue * 1000) + expect_equal(sum(total$amt_nominal), 33377093000) + expect_true(all(c("X01", "X05", "X08") %in% wt_codes_included(total))) + + # Sibling codes under the SAME first letter must stay out: X11/X12 are + # benefit payments (an expenditure) and X21/X30/X47 are cash and securities + # holdings (a balance-sheet stock). This is the F-018 point restated on the + # revenue side -- the split has to come from the crosswalk's spend_type, not + # from the letter X. + expect_false(any(c("X11", "X12", "X21", "X30", "X47") %in% wt_codes_included(total))) + + # Every returned row still resolves to a category. summary_categories has + # zero rows for prefix X today, so relaxing the prefix filter alone would + # produce category = NA rows -- see census_of_governments_finance_pipeline#60. + expect_false(any(is.na(total$category))) +})