From 72d8922ee81064c10406d4092ebecc66790fb9ca Mon Sep 17 00:00:00 2001 From: Jared Knowles Date: Sun, 19 Jul 2026 19:20:41 -0400 Subject: [PATCH] feat: narrow harmonization signposting to per-code gap detection .build_suggestions() previously flagged a recipe only when the WHOLE category result had zero rows in a requested year, so a multi-code category where one recipe component was genuinely gapped never fired if any sibling code (same recipe or not) had data that year. Each recipe's own in-category component is now checked individually -- a component fires when it has no rows in a requested (in-scope) year the recipe's own generic join otherwise covers, even when the overall category result looks complete. Decomposes .build_suggestions() into .category_recipe_components/ .recipe_meta/.component_presence/.recipe_coverage/.recipe_component_gapped helpers, drops the now-unused `result` param, and rewrites the header comment to describe the new, deliberately wider scope plus the per-government `covered` guard that still filters recipes with no data at all (ordinary reporting variance vs. a real format-boundary gap). Tests pin the multi-code case the coarse check missed (Cleburne County FY2012: G05 gapped, G04 covers, masked because E04/E05 have data) next to the still-guarded no-recipe-coverage case (F04/F05 both absent), and update the Broward 2019-2020 case to its new, correct expectation (fires for corrections_combined/corrections_other_capital_combined, still silent for corrections_capital_combined) plus a fresh true-full-coverage negative case (Maricopa County). --- R/spending.R | 2 +- R/suggestions.R | 188 ++++++++++++++++++++++++---------- tests/testthat/test-recipes.R | 79 +++++++++++++- 3 files changed, 214 insertions(+), 55 deletions(-) diff --git a/R/spending.R b/R/spending.R index bb4f641..d465bc1 100644 --- a/R/spending.R +++ b/R/spending.R @@ -141,7 +141,7 @@ cog_spending <- function(govid, years, category = NULL, harmonization <- .build_harmonization_block( con, govid, years, resolved, flow_prefixes ) - suggestions <- .build_suggestions(con, govid, years, category, result, resolved$basis) + suggestions <- .build_suggestions(con, govid, years, category, resolved$basis) } prov <- .build_provenance( diff --git a/R/suggestions.R b/R/suggestions.R index fc59d21..f075f82 100644 --- a/R/suggestions.R +++ b/R/suggestions.R @@ -1,8 +1,9 @@ # R/suggestions.R # Recipe-component-driven signposting: when a basis = "harmonized" query for -# a category comes back with a coverage gap in some requested years (the -# result has no rows at all in that year) that a harmonization recipe would -# actually fill for this government, surface that recipe as a suggestion. +# a category asks for a code that is itself a harmonization recipe +# component, and that specific code has no rows in some requested years +# while the recipe's own generic join would still fill those years for this +# government, surface that recipe as a suggestion. # # This is deliberately keyed off the recipe catalog's component codes, not # off harmonization_map rows: no live map row carries a non-blank @@ -12,53 +13,52 @@ # suggestion off of, just a leaf-code absence a recipe happens to fill). # See docs/phase_r_harmonization_review.md § 0.3. # -# Scope is deliberately narrow: signposting only runs when the caller +# Scope is deliberately narrow in one respect and, as of Phase R3 Task 19c, +# deliberately WIDE in another: signposting only runs when the caller # supplied a `category` (an un-scoped, all-categories query has no single -# coverage question to answer) and only flags a recipe when the ACTUAL -# result has zero rows in a requested year AND the candidate recipe's own -# generic join (same join .run_recipe() uses, including its wide-era -# aggregate rows) produces at least one row for this government in that -# year. Checking presence per-government (not corpus-wide) avoids false -# positives from ordinary reporting variance -- most governments don't use -# every sibling code in a multi-code category every year, and that is not -# a format-boundary gap worth signposting. +# coverage question to answer), but within that category it now checks +# EACH recipe component that is itself a category member individually, +# rather than asking whether the whole category *result* has zero rows +# that year. A recipe fires when one of its own components has zero rows +# for this government in a requested year the recipe's own generic join +# (same join .run_recipe() uses, aggregate rows included) otherwise covers +# -- even if OTHER, unrelated codes in the same category have full data +# that year and the overall result looks complete. That is a deliberate +# narrowing of the R2-era false-positive guard: most governments don't use +# every sibling code in a multi-code category every year, and per-code +# detection WILL flag some of that as a "gap" even though it's really just +# a government not having that particular sub-type of spending, not a +# format-boundary artifact. The remaining guard against ordinary reporting +# variance is the per-government `covered` check below (a component is +# only flagged when the recipe's OWN join -- not some unrelated code -- +# actually has something to offer in that year); it no longer tries to +# avoid noise from sibling *codes*, only from a recipe with genuinely +# nothing to contribute. The acceptable noise level this trade produces is +# a product decision, measured (not tuned here) by +# data-raw/measure_signposting_rate.R and ruled on at Checkpoint R3. -#' Build the `prov$suggestions` list for a (non-recipe) basis = "harmonized" -#' verb call: recipes whose generic join would fill a real gap in `result`. -#' -#' @param con Active DuckDB connection. -#' @param govid Character vector of canonical_govid values (the verb's raw -#' `govid`). -#' @param years Integer vector of requested years. -#' @param category `category` argument as passed to the verb (character -#' vector or `NULL`; suggestions are only computed when non-NULL). -#' @param result The verb's already-computed result tibble (post basis -#' query, pre per_capita/adjust_to_year). -#' @param basis The *resolved* basis (`"harmonized"` or `"raw"`). -#' @return List of `list(recipe_id, label, available_years, hint)`, possibly -#' empty. +#' Recipe components that are classified under the requested category -- +#' the codes a category-scoped query actually "requests". A recipe can +#' have components outside the category (e.g. general_gov_e89_wide's E85 +#' leg has no category assignment); those never trigger on their own, they +#' just were never part of what this query asked for. #' @noRd -.build_suggestions <- function(con, govid, years, category, result, basis) { - if (!identical(basis, "harmonized") || is.null(category)) return(list()) - - candidates <- DBI::dbGetQuery(con, sprintf( - "SELECT DISTINCT recipe_id FROM harmonization_recipes - WHERE component_code IN ( - SELECT DISTINCT item_code FROM summary_categories WHERE category IN (%s) - )", +.category_recipe_components <- function(con, category) { + DBI::dbGetQuery(con, sprintf( + "SELECT DISTINCT r.recipe_id, r.component_code, r.year_min, r.year_max, + r.gov_type_scope + FROM harmonization_recipes r + JOIN summary_categories sc + ON sc.item_code = r.component_code AND sc.category IN (%s)", .sql_lit_chr(category) - ))$recipe_id - if (length(candidates) == 0L) return(list()) + )) +} - result_years <- if (is.null(result) || nrow(result) == 0L) { - integer(0) - } else { - unique(as.integer(result$year)) - } - gap_years <- setdiff(as.integer(years), result_years) - if (length(gap_years) == 0L) return(list()) - - meta <- tibble::as_tibble(DBI::dbGetQuery(con, sprintf( +#' Label + overall year coverage for a set of recipe ids (the suggestion's +#' `label`/`available_years`). +#' @noRd +.recipe_meta <- function(con, candidates) { + tibble::as_tibble(DBI::dbGetQuery(con, sprintf( "SELECT recipe_id, any_value(label) AS label, MIN(year_min) AS year_min, MAX(year_max) AS year_max FROM harmonization_recipes @@ -66,12 +66,45 @@ GROUP BY recipe_id", .sql_lit_chr(candidates) ))) +} - # Which (recipe_id, year) pairs the recipe's own generic join actually - # covers for this government, restricted to the gap years -- the same - # join .run_recipe() uses (component year_min/year_max + gov_type_scope, - # no is_aggregate filter), just checking existence instead of summing. - covered <- DBI::dbGetQuery(con, sprintf( +#' Which (recipe_id, component_code, year) triples have at least one +#' NOT-aggregate row for these governments -- i.e. that specific requested +#' code itself has data, scoped exactly like .run_recipe()'s join +#' (component year_min/year_max + gov_type_scope). NOT-aggregate mirrors +#' what basis = "harmonized" itself excludes: an aggregate-only year is a +#' gap for that code exactly as it would be in a plain category query. +#' @noRd +.component_presence <- function(con, candidates, govid, years_lit) { + DBI::dbGetQuery(con, sprintf( + "SELECT DISTINCT r.recipe_id, r.component_code, l.year + FROM long l + JOIN harmonization_recipes r + ON l.item_code = r.component_code + AND l.year BETWEEN r.year_min AND r.year_max + AND (r.gov_type_scope = 'all' + OR (r.gov_type_scope = 'state' AND l.type = 0) + OR (r.gov_type_scope = 'local' AND l.type BETWEEN 1 AND 3)) + WHERE NOT l.is_aggregate + AND r.recipe_id IN (%s) + AND l.canonical_govid IN (%s) + AND l.year IN (%s)", + .sql_lit_chr(candidates), .sql_lit_chr(govid), years_lit + )) +} + +#' Which (recipe_id, year) pairs the recipe's own generic join actually +#' covers for these governments -- the same join .run_recipe() uses +#' (component year_min/year_max + gov_type_scope, no is_aggregate filter), +#' just checking existence instead of summing. This is the per-government, +#' whole-recipe guard against ordinary reporting variance: unlike +#' .component_presence(), it is aggregate-inclusive and unioned across ALL +#' of a recipe's components, not just the one requested code being tested, +#' so a recipe with genuinely nothing to offer (no component, aggregate or +#' leaf, has ever reported) never fires. +#' @noRd +.recipe_coverage <- function(con, candidates, govid, years_lit) { + DBI::dbGetQuery(con, sprintf( "SELECT DISTINCT r.recipe_id, l.year FROM long l JOIN harmonization_recipes r @@ -83,13 +116,62 @@ WHERE r.recipe_id IN (%s) AND l.canonical_govid IN (%s) AND l.year IN (%s)", - .sql_lit_chr(candidates), .sql_lit_chr(govid), - paste(gap_years, collapse = ",") + .sql_lit_chr(candidates), .sql_lit_chr(govid), years_lit )) +} + +#' TRUE if recipe `rid` has at least one requested component with an +#' in-scope requested year that has no data (`present`), in a year the +#' recipe's own generic join is otherwise fillable (`covered`) -- the +#' per-code gap the R2 whole-result check couldn't see. +#' @noRd +.recipe_component_gapped <- function(rid, requested, present, covered, years) { + covered_years <- covered$year[covered$recipe_id == rid] + if (length(covered_years) == 0L) return(FALSE) + + comps <- requested[requested$recipe_id == rid, , drop = FALSE] + for (i in seq_len(nrow(comps))) { + in_scope <- years[years >= comps$year_min[i] & years <= comps$year_max[i]] + if (length(in_scope) == 0L) next + has_data <- present$year[ + present$recipe_id == rid & present$component_code == comps$component_code[i] + ] + gap_years <- setdiff(in_scope, has_data) + if (any(gap_years %in% covered_years)) return(TRUE) + } + FALSE +} + +#' Build the `prov$suggestions` list for a (non-recipe) basis = "harmonized" +#' verb call: recipes whose generic join would fill a real per-code gap for +#' the requested category. +#' +#' @param con Active DuckDB connection. +#' @param govid Character vector of canonical_govid values (the verb's raw +#' `govid`). +#' @param years Integer vector of requested years. +#' @param category `category` argument as passed to the verb (character +#' vector or `NULL`; suggestions are only computed when non-NULL). +#' @param basis The *resolved* basis (`"harmonized"` or `"raw"`). +#' @return List of `list(recipe_id, label, available_years, hint)`, possibly +#' empty. +#' @noRd +.build_suggestions <- function(con, govid, years, category, basis) { + if (!identical(basis, "harmonized") || is.null(category)) return(list()) + + requested <- .category_recipe_components(con, category) + if (nrow(requested) == 0L) return(list()) + candidates <- unique(requested$recipe_id) + years_int <- as.integer(years) + years_lit <- paste(years_int, collapse = ",") + + meta <- .recipe_meta(con, candidates) + present <- .component_presence(con, candidates, govid, years_lit) + covered <- .recipe_coverage(con, candidates, govid, years_lit) suggestions <- list() for (rid in candidates) { - if (!rid %in% covered$recipe_id) next + if (!.recipe_component_gapped(rid, requested, present, covered, years_int)) next m <- meta[meta$recipe_id == rid, ] suggestions[[length(suggestions) + 1L]] <- list( recipe_id = rid, diff --git a/tests/testthat/test-recipes.R b/tests/testthat/test-recipes.R index 0bf9518..87b2318 100644 --- a/tests/testthat/test-recipes.R +++ b/tests/testthat/test-recipes.R @@ -176,6 +176,16 @@ test_that("recipe = requires schema_version >= 5", { }) # --- signposting ------------------------------------------------------- +# +# Phase R3 / Task 19c: .build_suggestions() was narrowed from a whole-result +# gap check (R2: does the ENTIRE category result have zero rows in a +# requested year) to per-code gap detection (does a specific recipe +# component -- itself a member of the requested category -- have zero rows +# in a year the recipe's own generic join otherwise covers). See +# R/suggestions.R's header comment and docs/phase_r_harmonization_review.md +# § 0.3. The R2 test below ("...across the 2011->2012 gap") is unaffected +# by the refinement (it already passed under both the coarse and per-code +# rule). The next few pin cases the coarse rule specifically could NOT see. test_that("signposting suggests corrections_combined across the 2011->2012 gap", { skip_if_no_corpus() @@ -193,10 +203,77 @@ test_that("signposting suggests corrections_combined across the 2011->2012 gap", expect_equal(hit$available_years, c(1967L, 2023L)) }) -test_that("no signposting when the result already has full year coverage", { +test_that("per-code gap fires even when a sibling code masks the whole-result check (Cleburne County, FY2012)", { skip_if_no_corpus() + # Cleburne County, AL (canonical_govid 011029122489), FY2012: E04 ($854) + # and E05 ($1) both report ("operations" subtype), and G04 ($14,000, + # corrections_other_capital_combined's modern-only leg) also reports + # ("capital" subtype) -- so the WHOLE category result is non-empty for + # 2012 (2 rows) and the R2 whole-result check would never look further. + # But G05 -- G04's OWN recipe sibling, the 1967-2023 wide leg -- has + # ZERO rows at all that year: a genuine, per-code gap the recipe exists + # to bridge, invisible at the category-result grain because it's masked + # by G04's own data, let alone the unrelated E04/E05 pair. + r <- cog_spending("011029122489", years = 2012L, category = "Corrections") + expect_equal(nrow(r), 2L) # operations + capital rows: a non-empty result + + prov <- attr(r, "provenance") + ids <- vapply(prov$suggestions, function(s) s$recipe_id, character(1)) + expect_true("corrections_other_capital_combined" %in% ids) + hit <- prov$suggestions[[which(ids == "corrections_other_capital_combined")]] + expect_equal(hit$hint, "re-run with recipe = 'corrections_other_capital_combined'") + expect_equal(hit$available_years, c(1967L, 2023L)) + + # corrections_combined must NOT fire: E04 AND E05 both have real 2012 + # data for this government, so neither of ITS OWN components is gapped. + expect_false("corrections_combined" %in% ids) +}) + +test_that("per-code gap does not fire when no recipe component has any data at all (ordinary reporting variance, not a format-boundary gap)", { + skip_if_no_corpus() + # Same government/year as above: F04 and F05 (corrections_capital_combined) + # are BOTH completely absent -- Cleburne simply never reported capital + # corrections spending under that code family in 2012, wide-era or + # modern. The recipe's own generic join (aggregate-inclusive, either + # component) has nothing to offer either, so this must stay silent -- + # the per-government `covered` guard the header comment describes is + # unchanged and still does this filtering. + r <- cog_spending("011029122489", years = 2012L, category = "Corrections") + prov <- attr(r, "provenance") + ids <- vapply(prov$suggestions, function(s) s$recipe_id, character(1)) + expect_false("corrections_capital_combined" %in% ids) +}) + +test_that("per-code gap fires for Broward 2019-2020 even though the category result looks complete", { + skip_if_no_corpus() + # Broward reports E04 + G04 (modern leaf codes) in BOTH 2019 and 2020 but + # never reports E05 or G05 (their own recipe siblings) in either year -- + # a real per-code gap in two of the three Corrections recipes, invisible + # under the R2 coarse check because the category *result* is non-empty + # both years (this replaces the old R2-era "full year coverage" test, + # whose premise -- that a non-empty result implies nothing to signpost -- + # is exactly what this refinement narrows; see data-raw/ + # measure_signposting_rate.R for the measured rate change this causes). + # corrections_capital_combined correctly stays silent: Broward reports + # neither F04 nor F05 in 2019 or 2020, so that recipe's own join has + # nothing to offer either (ordinary non-reporting, not a format-boundary + # gap) -- the per-government `covered` guard still does its job here too. r <- cog_spending("121011212191", years = 2019:2020, category = "Corrections") prov <- attr(r, "provenance") + ids <- vapply(prov$suggestions, function(s) s$recipe_id, character(1)) + expect_true("corrections_combined" %in% ids) + expect_true("corrections_other_capital_combined" %in% ids) + expect_false("corrections_capital_combined" %in% ids) +}) + +test_that("no signposting when every recipe component genuinely has data (true full per-code coverage)", { + skip_if_no_corpus() + # Maricopa County, AZ (canonical_govid 041013160815): all six Corrections + # codes (E04, E05, F04, F05, G04, G05) report real, nonzero, non-aggregate + # amounts in BOTH 2019 and 2020 -- genuinely nothing for any recipe to + # fill, even at the finer per-code grain this refinement now checks. + r <- cog_spending("041013160815", years = 2019:2020, category = "Corrections") + prov <- attr(r, "provenance") expect_length(prov$suggestions, 0L) })