feat: measure structurally-suppressed recipe component dollars (#9)

R/spending.R

@@ -512,6 +512,18 @@ cog_spending <- function(govid, years, category = NULL,
   if (identical(basis, "harmonized")) paste0(view_base, "_harmonized") else view_base
 }
 
+#' The `*_long`/`*_long_harmonized` view behind an annotated view base --
+#' `"spending_annotated"` -> `"spending_long_harmonized"`. `.build_suggestions()`
+#' anti-joins the LONG view rather than the annotated one: they have identical
+#' row membership (the annotated views are the long views plus LEFT JOINs, see
+#' inst/sql/42-spending_annotated_harmonized.sql), but the long view is the
+#' one that actually owns the `NOT is_aggregate` + crosswalk-membership rule
+#' the suppression test is asking about.
+#' @noRd
+.select_long_view <- function(view_base, basis) {
+  .select_view(sub("_annotated$", "_long", view_base), basis)
+}
+
 #' @noRd
 .select_ig_view <- function(basis) {
   if (identical(basis, "harmonized")) "ig_annotated_harmonized" else "ig_annotated"

R/suggestions.R

@@ -131,6 +131,92 @@
   .attach_ig_counterparts(con, suggestions, flow_prefixes)
 }
 
+#' Measure, per (recipe, year), the component dollars this government holds
+#' that the calling verb's own long view structurally excludes.
+#'
+#' This is the second qualifying path for a suggestion (uscogdata#9). The
+#' first -- row absence -- only fires when a category returns NOTHING in a
+#' requested year, which is how Corrections behaves in the wide era. Public
+#' Welfare is the failure mode it misses: E74/E75/E77/E79 still return rows,
+#' so there is no absence to detect, while E67/E68 (aggregate-flagged 1967-
+#' 2011, and absent from `summary_categories` entirely) are dropped. The
+#' caller gets a plausible number a third too low, silently.
+#'
+#' "Structurally excluded" is decided by anti-joining the verb's REAL long
+#' view rather than restating its WHERE clause, so this stays correct if
+#' `spending_long_harmonized` / `revenue_long_harmonized` ever change. That
+#' anti-join is keyed on `item_code`, which is sound only because
+#' harmonization never renames a recipe component -- asserted by the "no
+#' recipe component is ever renamed by harmonization" test in
+#' tests/testthat/test-recipes.R.
+#'
+#' Note what this deliberately does NOT count as suppressed: a component
+#' excluded from the RESULT for scoping reasons -- because it belongs to a
+#' different `category`, or because `expenditure_concept` narrowed the
+#' subtypes -- is still present in the view, so it never fires. Suggesting a
+#' recipe is a coverage fix, not a category redefinition. Measured on the
+#' bundled fixture, this keeps `higher_ed_e18_wide` and `general_gov_e89_wide`
+#' silent (E18/E89 are leaf-and-classified even in the wide era) and confines
+#' every fire to 2011.
+#'
+#' @param con Active DuckDB connection.
+#' @param candidates Character vector of recipe ids to measure.
+#' @param govid Character vector of canonical_govid values.
+#' @param years Integer vector of requested years.
+#' @param long_view Name of the verb's long view, from `.select_long_view()`.
+#' @return Tibble of `recipe_id`, `year`, `suppressed_amount` (full US
+#'   dollars), `suppressed_codes` (comma-joined, sorted). Zero rows when
+#'   nothing is suppressed.
+#' @noRd
+.suppressed_components <- function(con, candidates, govid, years, long_view) {
+  empty <- tibble::tibble(
+    recipe_id = character(0), year = numeric(0),
+    suppressed_amount = numeric(0), suppressed_codes = character(0)
+  )
+  if (length(candidates) == 0L) return(empty)
+
+  # long_view is interpolated as a SQL IDENTIFIER, not a literal, so it can
+  # never be quoted safely. It is always internally derived from a fixed
+  # view_base, so an off-allowlist value is a programming error, not input.
+  if (!long_view %in% c("spending_long", "spending_long_harmonized",
+                        "revenue_long", "revenue_long_harmonized")) {
+    cli::cli_abort(
+      "Internal error: unexpected `long_view` {.val {long_view}}.",
+      class = "uscogdata_internal_error"
+    )
+  }
+
+  sql <- sprintf(
+    "SELECT r.recipe_id,
+            l.year,
+            SUM(l.amt) * 1000.0 AS suppressed_amount,
+            string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code)
+              AS suppressed_codes
+     FROM long l
+     JOIN harmonization_recipes r
+       ON l.item_code = r.component_code
+      AND l.year BETWEEN r.year_min AND r.year_max
+      AND (r.gov_type_scope = 'all'
+           OR (r.gov_type_scope = 'state' AND l.type = 0)
+           OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3))
+     WHERE r.recipe_id IN (%1$s)
+       AND l.canonical_govid IN (%2$s)
+       AND l.year IN (%3$s)
+       AND l.amt <> 0
+       AND NOT EXISTS (
+         SELECT 1 FROM %4$s v
+         WHERE v.canonical_govid = l.canonical_govid
+           AND v.year = l.year
+           AND v.item_code = l.item_code
+       )
+     GROUP BY 1, 2
+     ORDER BY 1, 2",
+    .sql_lit_chr(candidates), .sql_lit_chr(govid),
+    paste(as.integer(years), collapse = ","), long_view
+  )
+  tibble::as_tibble(DBI::dbGetQuery(con, sql))
+}
+
 #' Attach `ig_recipe_id` to each suggestion: the intergovernmental-expenditure
 #' recipe (an M-to-local or L-to-state recipe) whose component codes cover
 #' exactly the same set of function suffixes as the firing recipe's own

tests/testthat/test-recipes.R

@@ -214,3 +214,71 @@ test_that("no signposting under basis = 'raw'", {
   prov <- attr(r, "provenance")
   expect_length(prov$suggestions, 0L)
 })
+
+# --- uscogdata#9: partial-coverage signposting ------------------------------
+
+test_that("no recipe component is ever renamed by harmonization", {
+  # The suppression trigger anti-joins the verb's long view on item_code.
+  # That is only sound because harmonization never rewrites a recipe
+  # component's code -- every component whose harmonized_code differs has
+  # harmonized_code IS NULL (and is aggregate-flagged). If this ever fails,
+  # .suppressed_components() would report reachable dollars as suppressed.
+  skip_if_no_corpus()
+  con <- uscogdata:::.ensure_session()
+  n <- DBI::dbGetQuery(con,
+    "SELECT COUNT(*) AS renamed FROM long
+     WHERE item_code IN (SELECT DISTINCT component_code FROM harmonization_recipes)
+       AND harmonized_code IS NOT NULL
+       AND harmonized_code <> item_code")$renamed
+  expect_equal(as.integer(n), 0L)
+})
+
+test_that(".select_long_view maps annotated view bases to their long views", {
+  expect_equal(
+    uscogdata:::.select_long_view("spending_annotated", "harmonized"),
+    "spending_long_harmonized")
+  expect_equal(
+    uscogdata:::.select_long_view("revenue_annotated", "harmonized"),
+    "revenue_long_harmonized")
+  expect_equal(
+    uscogdata:::.select_long_view("spending_annotated", "raw"),
+    "spending_long")
+})
+
+test_that(".suppressed_components measures the E67/E68 dollars Public Welfare drops", {
+  skip_if_no_corpus()
+  con <- uscogdata:::.ensure_session()
+  s <- uscogdata:::.suppressed_components(
+    con,
+    candidates = c("welfare_cash_e67_wide", "welfare_cash_e68_wide"),
+    govid = "061037123085", years = 2011L,
+    long_view = "spending_long_harmonized")
+
+  expect_s3_class(s, "tbl_df")
+  expect_equal(nrow(s), 2L)
+  s <- s[order(s$recipe_id), ]
+  expect_equal(s$recipe_id, c("welfare_cash_e67_wide", "welfare_cash_e68_wide"))
+  expect_equal(s$suppressed_amount, c(1803872000, 271589000))
+  expect_equal(s$suppressed_codes, c("E67", "E68"))
+})
+
+test_that(".suppressed_components finds nothing in a modern year", {
+  skip_if_no_corpus()
+  con <- uscogdata:::.ensure_session()
+  s <- uscogdata:::.suppressed_components(
+    con,
+    candidates = c("welfare_cash_e67_wide", "welfare_cash_e68_wide"),
+    govid = "061037123085", years = 2019L,
+    long_view = "spending_long_harmonized")
+  expect_equal(nrow(s), 0L)
+})
+
+test_that(".suppressed_components rejects a long_view outside the allowlist", {
+  skip_if_no_corpus()
+  con <- uscogdata:::.ensure_session()
+  expect_error(
+    uscogdata:::.suppressed_components(
+      con, candidates = "welfare_cash_e67_wide", govid = "061037123085",
+      years = 2011L, long_view = "long; DROP TABLE x"),
+    class = "uscogdata_internal_error")
+})