From 8bf9c4ccc1471616ecc0830dea842a9235753528 Mon Sep 17 00:00:00 2001 From: Jared Knowles Date: Wed, 5 Aug 2026 08:11:15 -0400 Subject: [PATCH] refactor: split .suppressed_components() into R/suppression.R (#9) R/suggestions.R crossed the project's 400-line limit (424 lines). Pure move of .suppressed_components() and its roxygen block per the plan's Task 5 Step 2 remedy; no logic, SQL, or wording changed. --- R/suggestions.R | 107 -------------------------------------------- R/suppression.R | 115 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 107 deletions(-) create mode 100644 R/suppression.R diff --git a/R/suggestions.R b/R/suggestions.R index 0a148c0..d0d45af 100644 --- a/R/suggestions.R +++ b/R/suggestions.R @@ -184,113 +184,6 @@ .attach_ig_counterparts(con, suggestions, flow_prefixes) } -#' Measure, per (recipe, year), the component dollars this government holds -#' that the calling verb's own long view structurally excludes. -#' -#' This is the second qualifying path for a suggestion (uscogdata#9). The -#' first -- row absence -- only fires when a category returns NOTHING in a -#' requested year, which is how Corrections behaves in the wide era. Public -#' Welfare is the failure mode it misses: E74/E75/E77/E79 still return rows, -#' so there is no absence to detect, while E67/E68 (aggregate-flagged 1967- -#' 2011, and absent from `summary_categories` entirely) are dropped. The -#' caller gets a plausible number a third too low, silently. -#' -#' "Structurally excluded" is decided by anti-joining the verb's REAL long -#' view rather than restating its WHERE clause, so this stays correct if -#' `spending_long_harmonized` / `revenue_long_harmonized` ever change. That -#' anti-join is keyed on `item_code`, which is sound only because -#' harmonization never renames a recipe component -- asserted by the "no -#' recipe component is ever renamed by harmonization" test in -#' tests/testthat/test-recipes.R. -#' -#' Note what this deliberately does NOT count as suppressed: a component -#' excluded from the RESULT for scoping reasons -- because it belongs to a -#' different `category`, or because `expenditure_concept` narrowed the -#' subtypes -- is still present in the view, so it never fires. Suggesting a -#' recipe is a coverage fix, not a category redefinition. -#' -#' `flow_prefixes` (uscogdata#9 review, finding I1) restricts the measured -#' components to the CALLING VERB's own flow family (`c("E","F","G")` for -#' spending, `c("T","A","U","B","C","D")` for revenue). Without this, a -#' candidate recipe belonging to the OTHER flow family is always absent from -#' this verb's view (by construction -- `cog_revenue()`'s view never carries -#' an E-coded row) and so was always reported as "suppressed", fabricating a -#' dollar claim across flow families (`cog_revenue(category = "Corrections")` -#' claimed $3.63B excluded that `cog_spending()` reports and fully accounts -#' for). Filtering on `LEFT(r.component_code, 1)` also drops M/L-prefixed -#' components from measurement under `cog_spending()` (`flow_prefixes` never -#' includes "M"/"L") -- harmless today, because a recipe's own M/L components -#' (e.g. `corrections_ig_local_combined`'s M04/M05) are present in the view -#' in every year they exist and so never fired as suppressed anyway, but -#' worth recording since this filter is now the thing relied on to prevent -#' it. -#' -#' @param con Active DuckDB connection. -#' @param candidates Character vector of recipe ids to measure. -#' @param govid Character vector of canonical_govid values. -#' @param years Integer vector of requested years. -#' @param long_view Name of the verb's long view, from `.select_long_view()`. -#' @param flow_prefixes The calling verb's own flow-type prefixes (see -#' `.build_suggestions()`). Only recipe components whose first character is -#' in this set are measured. -#' @return Tibble of `recipe_id`, `year`, `suppressed_amount` (full US -#' dollars), `suppressed_codes` (comma-joined, sorted). Zero rows when -#' nothing is suppressed. -#' @noRd -.suppressed_components <- function(con, candidates, govid, years, long_view, - flow_prefixes) { - empty <- tibble::tibble( - recipe_id = character(0), year = numeric(0), - suppressed_amount = numeric(0), suppressed_codes = character(0) - ) - if (length(candidates) == 0L) return(empty) - - # long_view is interpolated as a SQL IDENTIFIER, not a literal, so it can - # never be quoted safely. It is always internally derived from a fixed - # view_base, so an off-allowlist value is a programming error, not input. - if (!long_view %in% c("spending_long", "spending_long_harmonized", - "revenue_long", "revenue_long_harmonized")) { - cli::cli_abort( - "Internal error: unexpected `long_view` {.val {long_view}}.", - class = "uscogdata_internal_error" - ) - } - - sql <- sprintf( - "SELECT r.recipe_id, - l.year, - SUM(l.amt) * 1000.0 AS suppressed_amount, - string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code) - AS suppressed_codes - FROM long l - JOIN harmonization_recipes r - ON l.item_code = r.component_code - AND l.year BETWEEN r.year_min AND r.year_max - AND (r.gov_type_scope = 'all' - OR (r.gov_type_scope = 'state' AND l.type = 0) - OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) - WHERE r.recipe_id IN (%1$s) - AND l.canonical_govid IN (%2$s) - AND l.year IN (%3$s) - AND l.amt <> 0 - AND LEFT(r.component_code, 1) IN (%5$s) - AND NOT EXISTS ( - SELECT 1 FROM %4$s v - WHERE v.canonical_govid = l.canonical_govid - AND v.year = l.year - AND v.item_code = l.item_code - AND v.year IN (%3$s) -- restated: enables partition pruning (I3a) - AND v.canonical_govid IN (%2$s) -- restated: pushes the govid filter (I3a) - ) - GROUP BY 1, 2 - ORDER BY 1, 2", - .sql_lit_chr(candidates), .sql_lit_chr(govid), - paste(as.integer(years), collapse = ","), long_view, - .sql_lit_chr(flow_prefixes) - ) - tibble::as_tibble(DBI::dbGetQuery(con, sql)) -} - #' Attach `ig_recipe_id` to each suggestion: the intergovernmental-expenditure #' recipe (an M-to-local or L-to-state recipe) whose component codes cover #' exactly the same set of function suffixes as the firing recipe's own diff --git a/R/suppression.R b/R/suppression.R new file mode 100644 index 0000000..e9a01b4 --- /dev/null +++ b/R/suppression.R @@ -0,0 +1,115 @@ +# R/suppression.R +# Split out of R/suggestions.R (2026-08-05) to keep files under the project's +# 400-line limit. Owns the second qualifying path for coverage signposting +# (uscogdata#9): measuring, per government, the component dollars the +# calling verb's own long view structurally excludes (aggregate-published, +# or absent from summary_categories). See R/suggestions.R for the +# orchestrator (`.build_suggestions()`) that calls this and the full +# uscogdata#9 background. + +#' Measure, per (recipe, year), the component dollars this government holds +#' that the calling verb's own long view structurally excludes. +#' +#' This is the second qualifying path for a suggestion (uscogdata#9). The +#' first -- row absence -- only fires when a category returns NOTHING in a +#' requested year, which is how Corrections behaves in the wide era. Public +#' Welfare is the failure mode it misses: E74/E75/E77/E79 still return rows, +#' so there is no absence to detect, while E67/E68 (aggregate-flagged 1967- +#' 2011, and absent from `summary_categories` entirely) are dropped. The +#' caller gets a plausible number a third too low, silently. +#' +#' "Structurally excluded" is decided by anti-joining the verb's REAL long +#' view rather than restating its WHERE clause, so this stays correct if +#' `spending_long_harmonized` / `revenue_long_harmonized` ever change. That +#' anti-join is keyed on `item_code`, which is sound only because +#' harmonization never renames a recipe component -- asserted by the "no +#' recipe component is ever renamed by harmonization" test in +#' tests/testthat/test-recipes.R. +#' +#' Note what this deliberately does NOT count as suppressed: a component +#' excluded from the RESULT for scoping reasons -- because it belongs to a +#' different `category`, or because `expenditure_concept` narrowed the +#' subtypes -- is still present in the view, so it never fires. Suggesting a +#' recipe is a coverage fix, not a category redefinition. +#' +#' `flow_prefixes` (uscogdata#9 review, finding I1) restricts the measured +#' components to the CALLING VERB's own flow family (`c("E","F","G")` for +#' spending, `c("T","A","U","B","C","D")` for revenue). Without this, a +#' candidate recipe belonging to the OTHER flow family is always absent from +#' this verb's view (by construction -- `cog_revenue()`'s view never carries +#' an E-coded row) and so was always reported as "suppressed", fabricating a +#' dollar claim across flow families (`cog_revenue(category = "Corrections")` +#' claimed $3.63B excluded that `cog_spending()` reports and fully accounts +#' for). Filtering on `LEFT(r.component_code, 1)` also drops M/L-prefixed +#' components from measurement under `cog_spending()` (`flow_prefixes` never +#' includes "M"/"L") -- harmless today, because a recipe's own M/L components +#' (e.g. `corrections_ig_local_combined`'s M04/M05) are present in the view +#' in every year they exist and so never fired as suppressed anyway, but +#' worth recording since this filter is now the thing relied on to prevent +#' it. +#' +#' @param con Active DuckDB connection. +#' @param candidates Character vector of recipe ids to measure. +#' @param govid Character vector of canonical_govid values. +#' @param years Integer vector of requested years. +#' @param long_view Name of the verb's long view, from `.select_long_view()`. +#' @param flow_prefixes The calling verb's own flow-type prefixes (see +#' `.build_suggestions()`). Only recipe components whose first character is +#' in this set are measured. +#' @return Tibble of `recipe_id`, `year`, `suppressed_amount` (full US +#' dollars), `suppressed_codes` (comma-joined, sorted). Zero rows when +#' nothing is suppressed. +#' @noRd +.suppressed_components <- function(con, candidates, govid, years, long_view, + flow_prefixes) { + empty <- tibble::tibble( + recipe_id = character(0), year = numeric(0), + suppressed_amount = numeric(0), suppressed_codes = character(0) + ) + if (length(candidates) == 0L) return(empty) + + # long_view is interpolated as a SQL IDENTIFIER, not a literal, so it can + # never be quoted safely. It is always internally derived from a fixed + # view_base, so an off-allowlist value is a programming error, not input. + if (!long_view %in% c("spending_long", "spending_long_harmonized", + "revenue_long", "revenue_long_harmonized")) { + cli::cli_abort( + "Internal error: unexpected `long_view` {.val {long_view}}.", + class = "uscogdata_internal_error" + ) + } + + sql <- sprintf( + "SELECT r.recipe_id, + l.year, + SUM(l.amt) * 1000.0 AS suppressed_amount, + string_agg(DISTINCT l.item_code, ',' ORDER BY l.item_code) + AS suppressed_codes + FROM long l + JOIN harmonization_recipes r + ON l.item_code = r.component_code + AND l.year BETWEEN r.year_min AND r.year_max + AND (r.gov_type_scope = 'all' + OR (r.gov_type_scope = 'state' AND l.type = 0) + OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) + WHERE r.recipe_id IN (%1$s) + AND l.canonical_govid IN (%2$s) + AND l.year IN (%3$s) + AND l.amt <> 0 + AND LEFT(r.component_code, 1) IN (%5$s) + AND NOT EXISTS ( + SELECT 1 FROM %4$s v + WHERE v.canonical_govid = l.canonical_govid + AND v.year = l.year + AND v.item_code = l.item_code + AND v.year IN (%3$s) -- restated: enables partition pruning (I3a) + AND v.canonical_govid IN (%2$s) -- restated: pushes the govid filter (I3a) + ) + GROUP BY 1, 2 + ORDER BY 1, 2", + .sql_lit_chr(candidates), .sql_lit_chr(govid), + paste(as.integer(years), collapse = ","), long_view, + .sql_lit_chr(flow_prefixes) + ) + tibble::as_tibble(DBI::dbGetQuery(con, sql)) +}