diff --git a/R/spending.R b/R/spending.R index 3ae8600..29393c8 100644 --- a/R/spending.R +++ b/R/spending.R @@ -395,7 +395,8 @@ cog_spending <- function(govid, years, category = NULL, } suggestions <- .build_suggestions(con, govid, years, category, direct_leg_result, - resolved$basis, flow_prefixes) + resolved$basis, flow_prefixes, + .select_long_view(view_base, resolved$basis)) } # C1(b): when expenditure_concept = "total", flag any row where the IG diff --git a/R/suggestions.R b/R/suggestions.R index a89d346..1163f06 100644 --- a/R/suggestions.R +++ b/R/suggestions.R @@ -1,8 +1,16 @@ # R/suggestions.R -# Recipe-component-driven signposting: when a basis = "harmonized" query for -# a category comes back with a coverage gap in some requested years (the -# result has no rows at all in that year) that a harmonization recipe would -# actually fill for this government, surface that recipe as a suggestion. +# Recipe-component-driven signposting. When a basis = "harmonized" query for +# a category comes back incomplete in some requested year -- and a +# harmonization recipe would actually fill it for this government -- surface +# that recipe as a suggestion. "Incomplete" has two forms, and a recipe +# qualifies on either: +# 1. empty_year -- the result has no rows at all in that year. +# 2. suppressed_component -- the result HAS rows, but a component code +# carries dollars the verb's own long view structurally excludes +# (aggregate-published, or absent from summary_categories). This is +# uscogdata#9: Public Welfare kept returning E74/E79 rows while dropping +# aggregate-only E67/E68, so form 1 never fired and the caller got a +# number a third too low with no signpost at all. # # This is deliberately keyed off the recipe catalog's component codes, not # off harmonization_map rows: no live map row carries a non-blank @@ -48,11 +56,15 @@ #' "D")` for `cog_revenue()` -- see `.verb_spendrev()`). Passed through to #' `.attach_ig_counterparts()` to keep the intergovernmental-counterpart #' lookup scoped to the calling verb's own flow family. +#' @param long_view Name of the verb's own long view (from +#' `.select_long_view()`), passed through to `.suppressed_components()` to +#' measure the second qualifying path (uscogdata#9). #' @return List of `list(recipe_id, label, available_years, hint, -#' ig_recipe_id)`, possibly empty. +#' ig_recipe_id, trigger, suppressed_amount, suppressed_years, +#' suppressed_codes)`, possibly empty. #' @noRd .build_suggestions <- function(con, govid, years, category, result, basis, - flow_prefixes) { + flow_prefixes, long_view) { if (!identical(basis, "harmonized") || is.null(category)) return(list()) # Exclude any recipe that is ITSELF an intergovernmental (M/L) recipe -- @@ -86,7 +98,14 @@ unique(as.integer(result$year)) } gap_years <- setdiff(as.integer(years), result_years) - if (length(gap_years) == 0L) return(list()) + + # Path 2 (uscogdata#9): component dollars this government holds that the + # verb's own view structurally excludes. Measured across ALL requested + # years, not just gap years -- the whole point is that a year with rows can + # still be missing dollars. + supp <- .suppressed_components(con, candidates, govid, years, long_view) + + if (length(gap_years) == 0L && nrow(supp) == 0L) return(list()) meta <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf( "SELECT recipe_id, any_value(label) AS label, @@ -97,35 +116,55 @@ .sql_lit_chr(candidates) ))) - # Which (recipe_id, year) pairs the recipe's own generic join actually - # covers for this government, restricted to the gap years -- the same - # join .run_recipe() uses (component year_min/year_max + gov_type_scope, - # no is_aggregate filter), just checking existence instead of summing. - covered <- DBI::dbGetQuery(con, sprintf( - "SELECT DISTINCT r.recipe_id, l.year - FROM long l - JOIN harmonization_recipes r - ON l.item_code = r.component_code - AND l.year BETWEEN r.year_min AND r.year_max - AND (r.gov_type_scope = 'all' - OR (r.gov_type_scope = 'state' AND l.type = 0) - OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) - WHERE r.recipe_id IN (%s) - AND l.canonical_govid IN (%s) - AND l.year IN (%s)", - .sql_lit_chr(candidates), .sql_lit_chr(govid), - paste(gap_years, collapse = ",") - )) + # Path 1 (unchanged): (recipe, year) pairs the recipe's own generic join + # covers for this government, restricted to the gap years. + covered <- if (length(gap_years) == 0L) { + data.frame(recipe_id = character(0), year = integer(0)) + } else { + DBI::dbGetQuery(con, sprintf( + "SELECT DISTINCT r.recipe_id, l.year + FROM long l + JOIN harmonization_recipes r + ON l.item_code = r.component_code + AND l.year BETWEEN r.year_min AND r.year_max + AND (r.gov_type_scope = 'all' + OR (r.gov_type_scope = 'state' AND l.type = 0) + OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) + WHERE r.recipe_id IN (%s) + AND l.canonical_govid IN (%s) + AND l.year IN (%s)", + .sql_lit_chr(candidates), .sql_lit_chr(govid), + paste(gap_years, collapse = ",") + )) + } suggestions <- list() for (rid in candidates) { - if (!rid %in% covered$recipe_id) next + empty_hit <- rid %in% covered$recipe_id + s_rows <- supp[supp$recipe_id == rid, , drop = FALSE] + supp_hit <- nrow(s_rows) > 0L + if (!empty_hit && !supp_hit) next m <- meta[meta$recipe_id == rid, ] suggestions[[length(suggestions) + 1L]] <- list( recipe_id = rid, label = m$label[[1]], available_years = c(as.integer(m$year_min), as.integer(m$year_max)), - hint = sprintf("re-run with recipe = '%s'", rid) + hint = sprintf("re-run with recipe = '%s'", rid), + # An empty year is the stronger claim -- the category returned nothing + # at all -- so it wins when both paths qualify. The suppressed_* fields + # are still populated, so an empty_year fire also reports its dollars. + trigger = if (empty_hit) "empty_year" else "suppressed_component", + suppressed_amount = if (supp_hit) sum(s_rows$suppressed_amount) else 0, + suppressed_years = if (supp_hit) { + sort(unique(as.integer(s_rows$year))) + } else { + integer(0) + }, + suppressed_codes = if (supp_hit) { + sort(unique(unlist(strsplit(s_rows$suppressed_codes, ",", fixed = TRUE)))) + } else { + character(0) + } ) } .attach_ig_counterparts(con, suggestions, flow_prefixes) diff --git a/tests/testthat/test-recipes.R b/tests/testthat/test-recipes.R index e65dcf5..612402b 100644 --- a/tests/testthat/test-recipes.R +++ b/tests/testthat/test-recipes.R @@ -282,3 +282,99 @@ test_that(".suppressed_components rejects a long_view outside the allowlist", { years = 2011L, long_view = "long; DROP TABLE x"), class = "uscogdata_internal_error") }) + +test_that("uscogdata#9: Public Welfare signposts its suppressed E67/E68 dollars", { + # The bug: E74/E79 return rows for FY2011, so there is no row-absence gap, + # so nothing fired -- while E67 ($1,803,872,000) and E68 ($271,589,000) were + # dropped for being aggregate-published. LA County reports $3,185,943,000 + # and omits $2,075,461,000, a 39% understatement, silently. + skip_if_no_corpus() + r <- suppressMessages( + cog_spending("061037123085", years = 2011L, category = "Public Welfare")) + sugg <- attr(r, "provenance")$suggestions + + expect_length(sugg, 2L) + ids <- vapply(sugg, function(s) s$recipe_id, character(1)) + expect_setequal(ids, c("welfare_cash_e67_wide", "welfare_cash_e68_wide")) + + e67 <- sugg[[which(ids == "welfare_cash_e67_wide")]] + expect_equal(e67$trigger, "suppressed_component") + expect_equal(e67$suppressed_amount, 1803872000) + expect_equal(e67$suppressed_years, 2011L) + expect_equal(e67$suppressed_codes, "E67") + expect_equal(e67$hint, "re-run with recipe = 'welfare_cash_e67_wide'") + + e68 <- sugg[[which(ids == "welfare_cash_e68_wide")]] + expect_equal(e68$trigger, "suppressed_component") + expect_equal(e68$suppressed_amount, 271589000) + expect_equal(e68$suppressed_codes, "E68") +}) + +test_that("uscogdata#9: an empty_year fire keeps its trigger and gains the dollars", { + # Corrections is the case that already worked: zero rows in FY2011, so the + # row-absence path fires. It must keep firing, keep trigger = "empty_year", + # keep its IG counterpart -- and now also report what was suppressed. + skip_if_no_corpus() + r <- suppressMessages( + cog_spending("061037123085", years = 2011L, category = "Corrections")) + sugg <- attr(r, "provenance")$suggestions + + expect_length(sugg, 3L) + ids <- vapply(sugg, function(s) s$recipe_id, character(1)) + expect_setequal(ids, c("corrections_combined", "corrections_capital_combined", + "corrections_other_capital_combined")) + expect_true(all(vapply(sugg, function(s) s$trigger, character(1)) == "empty_year")) + + cc <- sugg[[which(ids == "corrections_combined")]] + expect_equal(cc$suppressed_amount, 1371460000) + expect_equal(cc$suppressed_codes, "E05") + expect_equal(cc$ig_recipe_id, "corrections_ig_local_combined") +}) + +test_that("uscogdata#9: the revenue verb inherits the same trigger", { + # Alaska state FY2011 Miscellaneous Revenue reports $943,842,000 from + # U11/U20/U30 while dropping $1,899,995,000 of aggregate-published `U4-` + # rents and royalties -- the omission is LARGER than the reported figure. + skip_if_no_corpus() + r <- suppressMessages( + cog_revenue("020000227749", years = 2011L, + category = "Miscellaneous Revenue")) + sugg <- attr(r, "provenance")$suggestions + + expect_length(sugg, 1L) + expect_equal(sugg[[1]]$recipe_id, "rents_royalties_u4_wide") + expect_equal(sugg[[1]]$trigger, "suppressed_component") + expect_equal(sugg[[1]]$suppressed_amount, 1899995000) + expect_equal(sugg[[1]]$suppressed_codes, "U4-") + # A revenue recipe must never be handed an M/L expenditure counterpart. + expect_null(sugg[[1]]$ig_recipe_id) +}) + +test_that("uscogdata#9: no partial-coverage fire in a modern year", { + skip_if_no_corpus() + r <- cog_spending("061037123085", years = 2019L, category = "Public Welfare") + expect_length(attr(r, "provenance")$suggestions, 0L) +}) + +test_that("uscogdata#9: leaf-and-classified wide-era families never fire", { + # higher_ed_e18_wide and general_gov_e89_wide are the control group: their + # components (E16/E18, E85/E89) are ordinary classified leaves even in the + # wide era, so widening the trigger must leave them silent. This is the + # measurement that refutes "it would fire on every category in every legacy + # year" -- corpus-wide on the fixture, these two produce zero suppressed rows. + skip_if_no_corpus() + con <- uscogdata:::.ensure_session() + n <- DBI::dbGetQuery(con, + "SELECT COUNT(*) AS n + FROM long l + JOIN harmonization_recipes r + ON l.item_code = r.component_code + AND l.year BETWEEN r.year_min AND r.year_max + WHERE r.recipe_id IN ('higher_ed_e18_wide', 'general_gov_e89_wide') + AND l.amt <> 0 + AND NOT EXISTS ( + SELECT 1 FROM spending_long_harmonized v + WHERE v.canonical_govid = l.canonical_govid + AND v.year = l.year AND v.item_code = l.item_code)")$n + expect_equal(as.integer(n), 0L) +})