From db35fa90583d94d22117f676e6762d21c6399779 Mon Sep 17 00:00:00 2001 From: Jared Knowles Date: Tue, 4 Aug 2026 21:38:22 -0400 Subject: [PATCH] feat: measure structurally-suppressed recipe component dollars (#9) --- R/spending.R | 12 +++++ R/suggestions.R | 86 +++++++++++++++++++++++++++++++++++ tests/testthat/test-recipes.R | 68 +++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) diff --git a/R/spending.R b/R/spending.R index 9642d42..3ae8600 100644 --- a/R/spending.R +++ b/R/spending.R @@ -512,6 +512,18 @@ cog_spending <- function(govid, years, category = NULL, if (identical(basis, "harmonized")) paste0(view_base, "_harmonized") else view_base } +#' The `*_long`/`*_long_harmonized` view behind an annotated view base -- +#' `"spending_annotated"` -> `"spending_long_harmonized"`. `.build_suggestions()` +#' anti-joins the LONG view rather than the annotated one: they have identical +#' row membership (the annotated views are the long views plus LEFT JOINs, see +#' inst/sql/42-spending_annotated_harmonized.sql), but the long view is the +#' one that actually owns the `NOT is_aggregate` + crosswalk-membership rule +#' the suppression test is asking about. +#' @noRd +.select_long_view <- function(view_base, basis) { + .select_view(sub("_annotated$", "_long", view_base), basis) +} + #' @noRd .select_ig_view <- function(basis) { if (identical(basis, "harmonized")) "ig_annotated_harmonized" else "ig_annotated" diff --git a/R/suggestions.R b/R/suggestions.R index 6e64c72..a89d346 100644 --- a/R/suggestions.R +++ b/R/suggestions.R @@ -131,6 +131,92 @@ .attach_ig_counterparts(con, suggestions, flow_prefixes) } +#' Measure, per (recipe, year), the component dollars this government holds +#' that the calling verb's own long view structurally excludes. +#' +#' This is the second qualifying path for a suggestion (uscogdata#9). The +#' first -- row absence -- only fires when a category returns NOTHING in a +#' requested year, which is how Corrections behaves in the wide era. Public +#' Welfare is the failure mode it misses: E74/E75/E77/E79 still return rows, +#' so there is no absence to detect, while E67/E68 (aggregate-flagged 1967- +#' 2011, and absent from `summary_categories` entirely) are dropped. The +#' caller gets a plausible number a third too low, silently. +#' +#' "Structurally excluded" is decided by anti-joining the verb's REAL long +#' view rather than restating its WHERE clause, so this stays correct if +#' `spending_long_harmonized` / `revenue_long_harmonized` ever change. That +#' anti-join is keyed on `item_code`, which is sound only because +#' harmonization never renames a recipe component -- asserted by the "no +#' recipe component is ever renamed by harmonization" test in +#' tests/testthat/test-recipes.R. +#' +#' Note what this deliberately does NOT count as suppressed: a component +#' excluded from the RESULT for scoping reasons -- because it belongs to a +#' different `category`, or because `expenditure_concept` narrowed the +#' subtypes -- is still present in the view, so it never fires. Suggesting a +#' recipe is a coverage fix, not a category redefinition. Measured on the +#' bundled fixture, this keeps `higher_ed_e18_wide` and `general_gov_e89_wide` +#' silent (E18/E89 are leaf-and-classified even in the wide era) and confines +#' every fire to 2011. +#' +#' @param con Active DuckDB connection. +#' @param candidates Character vector of recipe ids to measure. +#' @param govid Character vector of canonical_govid values. +#' @param years Integer vector of requested years. +#' @param long_view Name of the verb's long view, from `.select_long_view()`. +#' @return Tibble of `recipe_id`, `year`, `suppressed_amount` (full US +#' dollars), `suppressed_codes` (comma-joined, sorted). Zero rows when +#' nothing is suppressed. +#' @noRd +.suppressed_components <- function(con, candidates, govid, years, long_view) { + empty <- tibble::tibble( + recipe_id = character(0), year = numeric(0), + suppressed_amount = numeric(0), suppressed_codes = character(0) + ) + if (length(candidates) == 0L) return(empty) + + # long_view is interpolated as a SQL IDENTIFIER, not a literal, so it can + # never be quoted safely. It is always internally derived from a fixed + # view_base, so an off-allowlist value is a programming error, not input. + if (!long_view %in% c("spending_long", "spending_long_harmonized", + "revenue_long", "revenue_long_harmonized")) { + cli::cli_abort( + "Internal error: unexpected `long_view` {.val {long_view}}.", + class = "uscogdata_internal_error" + ) + } + + sql <- sprintf( + "SELECT r.recipe_id, + l.year, + SUM(l.amt) * 1000.0 AS suppressed_amount, + string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code) + AS suppressed_codes + FROM long l + JOIN harmonization_recipes r + ON l.item_code = r.component_code + AND l.year BETWEEN r.year_min AND r.year_max + AND (r.gov_type_scope = 'all' + OR (r.gov_type_scope = 'state' AND l.type = 0) + OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) + WHERE r.recipe_id IN (%1$s) + AND l.canonical_govid IN (%2$s) + AND l.year IN (%3$s) + AND l.amt <> 0 + AND NOT EXISTS ( + SELECT 1 FROM %4$s v + WHERE v.canonical_govid = l.canonical_govid + AND v.year = l.year + AND v.item_code = l.item_code + ) + GROUP BY 1, 2 + ORDER BY 1, 2", + .sql_lit_chr(candidates), .sql_lit_chr(govid), + paste(as.integer(years), collapse = ","), long_view + ) + tibble::as_tibble(DBI::dbGetQuery(con, sql)) +} + #' Attach `ig_recipe_id` to each suggestion: the intergovernmental-expenditure #' recipe (an M-to-local or L-to-state recipe) whose component codes cover #' exactly the same set of function suffixes as the firing recipe's own diff --git a/tests/testthat/test-recipes.R b/tests/testthat/test-recipes.R index 0bf9518..e65dcf5 100644 --- a/tests/testthat/test-recipes.R +++ b/tests/testthat/test-recipes.R @@ -214,3 +214,71 @@ test_that("no signposting under basis = 'raw'", { prov <- attr(r, "provenance") expect_length(prov$suggestions, 0L) }) + +# --- uscogdata#9: partial-coverage signposting ------------------------------ + +test_that("no recipe component is ever renamed by harmonization", { + # The suppression trigger anti-joins the verb's long view on item_code. + # That is only sound because harmonization never rewrites a recipe + # component's code -- every component whose harmonized_code differs has + # harmonized_code IS NULL (and is aggregate-flagged). If this ever fails, + # .suppressed_components() would report reachable dollars as suppressed. + skip_if_no_corpus() + con <- uscogdata:::.ensure_session() + n <- DBI::dbGetQuery(con, + "SELECT COUNT(*) AS renamed FROM long + WHERE item_code IN (SELECT DISTINCT component_code FROM harmonization_recipes) + AND harmonized_code IS NOT NULL + AND harmonized_code <> item_code")$renamed + expect_equal(as.integer(n), 0L) +}) + +test_that(".select_long_view maps annotated view bases to their long views", { + expect_equal( + uscogdata:::.select_long_view("spending_annotated", "harmonized"), + "spending_long_harmonized") + expect_equal( + uscogdata:::.select_long_view("revenue_annotated", "harmonized"), + "revenue_long_harmonized") + expect_equal( + uscogdata:::.select_long_view("spending_annotated", "raw"), + "spending_long") +}) + +test_that(".suppressed_components measures the E67/E68 dollars Public Welfare drops", { + skip_if_no_corpus() + con <- uscogdata:::.ensure_session() + s <- uscogdata:::.suppressed_components( + con, + candidates = c("welfare_cash_e67_wide", "welfare_cash_e68_wide"), + govid = "061037123085", years = 2011L, + long_view = "spending_long_harmonized") + + expect_s3_class(s, "tbl_df") + expect_equal(nrow(s), 2L) + s <- s[order(s$recipe_id), ] + expect_equal(s$recipe_id, c("welfare_cash_e67_wide", "welfare_cash_e68_wide")) + expect_equal(s$suppressed_amount, c(1803872000, 271589000)) + expect_equal(s$suppressed_codes, c("E67", "E68")) +}) + +test_that(".suppressed_components finds nothing in a modern year", { + skip_if_no_corpus() + con <- uscogdata:::.ensure_session() + s <- uscogdata:::.suppressed_components( + con, + candidates = c("welfare_cash_e67_wide", "welfare_cash_e68_wide"), + govid = "061037123085", years = 2019L, + long_view = "spending_long_harmonized") + expect_equal(nrow(s), 0L) +}) + +test_that(".suppressed_components rejects a long_view outside the allowlist", { + skip_if_no_corpus() + con <- uscogdata:::.ensure_session() + expect_error( + uscogdata:::.suppressed_components( + con, candidates = "welfare_cash_e67_wide", govid = "061037123085", + years = 2011L, long_view = "long; DROP TABLE x"), + class = "uscogdata_internal_error") +})